forked from Blendman974/faceclaw
Work on voice input
This commit is contained in:
1 parent
7b8fc8f37f
commit
097d40efb2
12 files changed
+599
-234
No files matched your search
@@ -0,0 +1,175 @@
|
||||
declare const com: any;
|
||||
declare const android: any;
|
||||
|
||||
/**
|
||||
* ElevenLabs realtime speech-to-text over WebSocket.
|
||||
* https://elevenlabs.io/docs/api-reference/speech-to-text/v-1-speech-to-text-realtime
|
||||
*
|
||||
* We push 16 kHz signed-16-bit-LE PCM (base64) as it arrives from the G2 mic
|
||||
* and, because push-to-talk gives us the utterance boundary, use manual commit:
|
||||
* commit=true is sent when the button is released, which finalizes the
|
||||
* transcript. Partial transcripts stream in the meantime.
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
|
||||
// The only model the realtime endpoint accepts (verified against the live API;
|
||||
// it rejects "scribe_v1" with an explicit message naming this one).
|
||||
const MODEL_ID = "scribe_v2_realtime";
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
export type ElevenLabsTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type ElevenLabsSttOptions = {
|
||||
apiKey: string;
|
||||
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private open = false;
|
||||
private closed = false;
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private latestText = "";
|
||||
|
||||
constructor(private readonly options: ElevenLabsSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
// The API key rides an xi-api-key header, added by FaceclawWebSocket.
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", this.options.apiKey);
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Feed PCM (16 kHz signed-16-bit LE). */
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.closed || pcm.length === 0) return;
|
||||
const base64 = encodeBase64(pcm);
|
||||
if (this.open) {
|
||||
this.sendChunk(base64);
|
||||
} else {
|
||||
this.pendingChunks.push(base64);
|
||||
}
|
||||
}
|
||||
|
||||
/** End of utterance: flush and commit for a final transcript. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
const commitMessage = JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: "",
|
||||
commit: true,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
});
|
||||
if (this.open) {
|
||||
this.trySend(commitMessage);
|
||||
} else {
|
||||
this.pendingChunks.push("__commit__");
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.finish();
|
||||
return;
|
||||
}
|
||||
this.trySend(
|
||||
JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: base64OrCommit,
|
||||
commit: false,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
} catch (error) {
|
||||
console.warn("elevenlabs send failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
private handleMessage(text: string): void {
|
||||
let message: any;
|
||||
try {
|
||||
message = JSON.parse(text);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
switch (message?.message_type) {
|
||||
case "session_started":
|
||||
this.options.onStatus("Listening (ElevenLabs)...");
|
||||
return;
|
||||
case "partial_transcript":
|
||||
this.latestText = String(message.text ?? "");
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
return;
|
||||
case "committed_transcript":
|
||||
case "committed_transcript_with_timestamps":
|
||||
this.latestText = String(message.text ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
return;
|
||||
case "error":
|
||||
case "auth_error":
|
||||
case "quota_exceeded":
|
||||
case "rate_limited":
|
||||
case "input_error":
|
||||
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function encodeBase64(bytes: Uint8Array): string {
|
||||
if (!global.isAndroid) return "";
|
||||
const javaBytes = Array.create("byte", bytes.length);
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const value = bytes[i]!;
|
||||
javaBytes[i] = value > 127 ? value - 256 : value;
|
||||
}
|
||||
return String(android.util.Base64.encodeToString(javaBytes, android.util.Base64.NO_WRAP));
|
||||
}
|
||||
+69
-22
@@ -1,17 +1,31 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type VoiceControlState = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type VoiceInputMode = "off" | "wakeword" | "full";
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
/**
|
||||
* Complete best transcript of the current utterance. REPLACE semantics —
|
||||
* render as-is, replacing any previous partial. Not a delta.
|
||||
*/
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type PushToTalkOptions = {
|
||||
communicator: any;
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
};
|
||||
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
|
||||
@@ -20,7 +34,8 @@ export class FaceclawVoiceControlBridge {
|
||||
private listenerProxy: any | null = null;
|
||||
private status = "Voice control stopped.";
|
||||
private started = false;
|
||||
private mode: VoiceInputMode = "off";
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: ElevenLabsSttClient | null = null;
|
||||
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||
this.statusListeners.add(listener);
|
||||
@@ -38,30 +53,55 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.transcriptListeners.delete(listener);
|
||||
}
|
||||
|
||||
start(communicator?: any, mode: VoiceInputMode = "wakeword"): void {
|
||||
if (mode === "off") {
|
||||
this.stop();
|
||||
/** Begin push-to-talk capture with the selected provider. */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
if (!global.isAndroid) return;
|
||||
if (this.started) this.stop();
|
||||
this.ensureController();
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
|
||||
if (options.provider === "elevenlabs" && options.elevenLabsApiKey.trim()) {
|
||||
this.cloudClient = new ElevenLabsSttClient({
|
||||
apiKey: options.elevenLabsApiKey.trim(),
|
||||
onTranscript: (event) => this.emitTranscript(event.text, event.isFinal),
|
||||
onStatus: (status) => this.setStatus(status),
|
||||
onError: (message) => this.setStatus(message),
|
||||
});
|
||||
this.cloudClient.start();
|
||||
this.started = true;
|
||||
this.controller?.start("cloud");
|
||||
return;
|
||||
}
|
||||
if (!global.isAndroid) return;
|
||||
this.ensureController();
|
||||
if (communicator) {
|
||||
this.controller?.setCommunicator(communicator);
|
||||
}
|
||||
if (this.started && this.mode !== mode) {
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
|
||||
if (options.provider === "elevenlabs") {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
}
|
||||
this.cloudClient = null;
|
||||
this.started = true;
|
||||
this.mode = mode;
|
||||
this.controller?.start(mode);
|
||||
this.controller?.start("onboard");
|
||||
}
|
||||
|
||||
/** End push-to-talk: stop the mic and, for cloud, commit for a final result. */
|
||||
stopPushToTalk(): void {
|
||||
if (!this.started) {
|
||||
this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
// Order matters for cloud: stopping the Java controller flushes any final
|
||||
// decode/PCM; then commit so ElevenLabs finalizes the transcript. The
|
||||
// socket is closed once the committed transcript arrives, or on stop().
|
||||
this.controller?.stop();
|
||||
this.started = false;
|
||||
this.cloudClient?.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.started || !global.isAndroid) return;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
this.started = false;
|
||||
this.mode = "off";
|
||||
this.controller?.stop();
|
||||
this.cloudClient?.stop();
|
||||
this.cloudClient = null;
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
|
||||
@@ -82,15 +122,22 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
},
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
const event = { text: String(text), isFinal: Boolean(isFinal) };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
onPcm: (pcm: any) => {
|
||||
this.cloudClient?.acceptPcm(toUint8Array(pcm));
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private emitTranscript(text: string, isFinal: boolean): void {
|
||||
const event = { text, isFinal };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
|
||||
private setStatus(status: string): void {
|
||||
this.status = status;
|
||||
for (const listener of this.statusListeners) {
|
||||
|
||||
Reference in new issue
Block a user