import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt"; import { openSocket, type SocketConnection } from "./socket"; /** * ElevenLabs realtime speech-to-text over WebSocket. * https://elevenlabs.io/docs/api-reference/speech-to-text/v-1-speech-to-text-realtime * * We push 16 kHz signed-16-bit-LE PCM (base64) as it arrives from the G2 mic * and, because push-to-talk gives us the utterance boundary, use manual commit: * commit=true is sent when the button is released, which finalizes the * transcript. Partial transcripts stream in the meantime. */ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime"; // The only model the realtime endpoint accepts (verified against the live API; // it rejects "scribe_v1" with an explicit message naming this one). const MODEL_ID = "scribe_v2_realtime"; const SAMPLE_RATE = 16000; export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent; export type ElevenLabsSttOptions = CloudSttOptions; export class ElevenLabsSttClient { private ws: SocketConnection | null = null; private open = false; private closed = false; // PCM that arrived before the socket finished opening; flushed on open. private readonly pendingChunks: string[] = []; private latestText = ""; private readonly commits: boolean[] = []; constructor(private readonly options: ElevenLabsSttOptions) {} start(): void { if (this.closed || this.ws) return; // Without language_code the model detects the language, which can go // badly wrong on short or noisy segments (Japanese heard as Russian). const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : ""; const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`; const listener = { onOpen: () => { if (this.closed) return; this.open = true; for (const chunk of this.pendingChunks.splice(0)) { this.sendChunk(chunk); } if (!this.closed) this.options.onReady?.(); }, onTextMessage: (message: string) => { if (!this.closed) this.handleMessage(String(message)); }, onClosed: () => { this.open = false; if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed."); }, onFailure: (message: string) => { if (this.closed) return; this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`); }, }; try { // The API key rides an xi-api-key header on the handshake. const key = this.options.apiKey; console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`); this.ws = openSocket(url, listener, { name: "xi-api-key", value: key }); this.options.onStatus("Connecting to ElevenLabs..."); } catch (error) { this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`); } } /** Feed PCM (16 kHz signed-16-bit LE). */ acceptPcm(pcm: Uint8Array): void { if (this.closed || pcm.length === 0) return; const base64 = encodeBase64(pcm); if (this.open) { this.sendChunk(base64); } else { this.pendingChunks.push(base64); } } commitSegment(): void { this.commit(true); } /** End of utterance: flush and commit for a final transcript. */ finish(): void { this.commit(false); } private commit(paragraphBreakAfter: boolean): void { if (this.closed) return; this.commits.push(paragraphBreakAfter); if (this.open) this.sendChunk("__commit__"); else this.pendingChunks.push("__commit__"); } stop(): void { this.closed = true; this.open = false; if (this.ws) { try { this.ws.close(1000, "bye"); } catch { // ignore } this.ws = null; } this.pendingChunks.length = 0; } private sendChunk(base64OrCommit: string): void { const commit = base64OrCommit === "__commit__"; this.trySend(JSON.stringify({ message_type: "input_audio_chunk", audio_base_64: commit ? "" : base64OrCommit, commit, sample_rate: SAMPLE_RATE, })); } private trySend(message: string): void { try { if (this.ws?.sendText(message) === false) { this.options.onDisconnected?.("ElevenLabs send failed."); } } catch (error) { this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`); } } private handleMessage(text: string): void { let message: any; try { message = JSON.parse(text); } catch { return; } switch (message?.message_type) { case "session_started": this.options.onStatus("Listening (ElevenLabs)..."); return; case "partial_transcript": this.latestText = String(message.text ?? ""); this.options.onTranscript({ text: this.latestText, isFinal: false }); return; case "committed_transcript": this.latestText = String(message.text ?? this.latestText); this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false }); this.latestText = ""; return; case "committed_transcript_with_timestamps": // The timestamp event follows the plain commit; do not append twice. return; case "rate_limited": this.options.onDisconnected?.("ElevenLabs rate limited."); return; case "error": case "auth_error": case "quota_exceeded": case "input_error": this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`); return; default: return; } } }