Improve the Transcribe app

This commit is contained in:
James Babcock committed 2026-09-05 19:16:56 -07:00
1 parent c9a35b0922
commit 9b70880a5b
12 files changed
+540 -140

No files matched your search

+7
View File
@@ -9,6 +9,9 @@
export type CloudSttTranscriptEvent = {
text: string;
isFinal: boolean;
/** Optional presentation for Transcribe only; dictation uses plain text. */
transcribeText?: string;
paragraphBreakAfter?: boolean;
};
export type CloudSttOptions = {
@@ -16,6 +19,8 @@ export type CloudSttOptions = {
onTranscript: (event: CloudSttTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
onReady?: () => void;
onDisconnected?: (message: string) => void;
};
export interface CloudSttClient {
@@ -25,6 +30,8 @@ export interface CloudSttClient {
acceptPcm(pcm: Uint8Array): void;
/** End of utterance: produce a final transcript. */
finish(): void;
/** Commit a pause boundary while keeping the audio session open. */
commitSegment?(): void;
/** Abandon the session without finalizing. */
stop(): void;
}
+45 -44
View File
@@ -1,4 +1,4 @@
import { encodeBase64 } from "./cloud-stt";
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
declare const com: any;
@@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
const MODEL_ID = "scribe_v2_realtime";
const SAMPLE_RATE = 16000;
export type ElevenLabsTranscriptEvent = {
text: string;
isFinal: boolean;
};
export type ElevenLabsSttOptions = {
apiKey: string;
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
};
export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
export type ElevenLabsSttOptions = CloudSttOptions;
export class ElevenLabsSttClient {
private ws: any = null;
@@ -38,26 +29,32 @@ export class ElevenLabsSttClient {
// PCM that arrived before the socket finished opening; flushed on open.
private readonly pendingChunks: string[] = [];
private latestText = "";
private readonly commits: boolean[] = [];
constructor(private readonly options: ElevenLabsSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
this.open = true;
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
if (!this.closed) this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
},
});
try {
@@ -67,7 +64,7 @@ export class ElevenLabsSttClient {
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
this.options.onStatus("Connecting to ElevenLabs...");
} catch (error) {
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -82,24 +79,25 @@ export class ElevenLabsSttClient {
}
}
commitSegment(): void {
this.commit(true);
}
/** End of utterance: flush and commit for a final transcript. */
finish(): void {
this.commit(false);
}
private commit(paragraphBreakAfter: boolean): void {
if (this.closed) return;
const commitMessage = JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: "",
commit: true,
sample_rate: SAMPLE_RATE,
});
if (this.open) {
this.trySend(commitMessage);
} else {
this.pendingChunks.push("__commit__");
}
this.commits.push(paragraphBreakAfter);
if (this.open) this.sendChunk("__commit__");
else this.pendingChunks.push("__commit__");
}
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -109,28 +107,26 @@ export class ElevenLabsSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
private sendChunk(base64OrCommit: string): void {
if (base64OrCommit === "__commit__") {
this.finish();
return;
}
this.trySend(
JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: base64OrCommit,
commit: false,
sample_rate: SAMPLE_RATE,
}),
);
const commit = base64OrCommit === "__commit__";
this.trySend(JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: commit ? "" : base64OrCommit,
commit,
sample_rate: SAMPLE_RATE,
}));
}
private trySend(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("ElevenLabs send failed.");
}
} catch (error) {
console.warn("elevenlabs send failed", error);
this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`);
}
}
@@ -150,14 +146,19 @@ export class ElevenLabsSttClient {
this.options.onTranscript({ text: this.latestText, isFinal: false });
return;
case "committed_transcript":
case "committed_transcript_with_timestamps":
this.latestText = String(message.text ?? this.latestText);
this.options.onTranscript({ text: this.latestText, isFinal: true });
this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false });
this.latestText = "";
return;
case "committed_transcript_with_timestamps":
// The timestamp event follows the plain commit; do not append twice.
return;
case "rate_limited":
this.options.onDisconnected?.("ElevenLabs rate limited.");
return;
case "error":
case "auth_error":
case "quota_exceeded":
case "rate_limited":
case "input_error":
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
return;
+74 -16
View File
@@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
// Base64 audio (or the commit sentinel) queued until the socket opens.
private readonly pendingChunks: string[] = [];
private readonly upsampler = new PcmUpsampler();
private latestText = "";
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
private readonly commits: boolean[] = [];
private committed = false;
constructor(private readonly options: OpenAiSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig()));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
if (!this.closed) this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`OpenAI connection failed: ${String(message)}`);
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
},
});
try {
@@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
);
this.options.onStatus("Connecting to OpenAI...");
} catch (error) {
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
}
commitSegment(): void {
this.commit(true);
}
/** End of utterance: commit the buffer for a final transcript. */
finish(): void {
this.committed = true;
this.commit(false);
}
private commit(paragraphBreakAfter: boolean): void {
if (this.closed) return;
this.commits.push(paragraphBreakAfter);
if (this.open) {
this.sendChunk("__commit__");
} else {
@@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
private sendChunk(base64OrCommit: string): void {
if (base64OrCommit === "__commit__") {
this.committed = true;
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
return;
}
@@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
private trySend(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("OpenAI send failed.");
}
} catch (error) {
console.warn("openai send failed", error);
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
}
}
@@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
case "session.updated":
this.options.onStatus("Listening (OpenAI)...");
return;
case "conversation.item.input_audio_transcription.delta":
this.latestText += String(message.delta ?? "");
this.options.onTranscript({ text: this.latestText, isFinal: false });
case "input_audio_buffer.committed": {
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
return;
case "conversation.item.input_audio_transcription.completed":
this.latestText = String(message.transcript ?? this.latestText);
this.options.onTranscript({ text: this.latestText, isFinal: true });
// The utterance is done; nothing further is coming after the commit.
if (this.committed) this.stop();
}
case "conversation.item.input_audio_transcription.delta": {
this.item(String(message.item_id)).text += String(message.delta ?? "");
this.emitItems();
return;
}
case "conversation.item.input_audio_transcription.completed": {
const item = this.item(String(message.item_id));
item.text = String(message.transcript ?? item.text);
item.final = true;
this.emitItems();
if (this.committed && !this.items.size && !this.commits.length) this.stop();
return;
}
case "error":
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
return;
}
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
return;
default:
return;
}
}
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
let item = this.items.get(id);
if (!item) {
item = { text: "", final: false, paragraphBreakAfter: false };
this.items.set(id, item);
}
return item;
}
private emitItems(): void {
// Committed notifications precede transcription events, so Map insertion
// order is audio order even when completion events arrive out of order.
for (const [id, item] of this.items) {
if (!item.final) break;
this.items.delete(id);
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
}
if (this.items.size) {
const items = [...this.items.values()];
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
const transcribeText = items.map((item, index) =>
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
).join("");
this.options.onTranscript({ text, transcribeText, isFinal: false });
}
}
}
function sessionConfig(): object {
+141
View File
@@ -0,0 +1,141 @@
import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
import { SpeechPauseDetector } from "./speech-pause";
// Ten seconds of unsent PCM. Never replay audio already sent: that would
// duplicate words already shown. An extended outage drops the oldest audio.
const MAX_PENDING_BYTES = 16000 * 2 * 10;
/** Owns retries and audio buffering independently of the provider protocol. */
export class ReconnectingSttClient implements CloudSttClient {
private client: CloudSttClient | null = null;
private generation = 0;
private stopped = false;
private finishing = false;
private ready = false;
private retry: ReturnType<typeof setTimeout> | null = null;
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
private attempt = 0;
private pending: Uint8Array[] = [];
private pendingBytes = 0;
private latest: CloudSttTranscriptEvent | null = null;
private readonly pause = new SpeechPauseDetector();
constructor(
private readonly create: (options: CloudSttOptions) => CloudSttClient,
private readonly options: CloudSttOptions,
private readonly continuous: () => boolean,
) {}
start(): void {
if (this.stopped || this.client || this.retry !== null) return;
const generation = ++this.generation;
const current = () => !this.stopped && generation === this.generation;
this.client = this.create({
...this.options,
onReady: () => {
if (!current()) return;
this.ready = true;
const chunks = this.pending;
this.pending = [];
this.pendingBytes = 0;
for (const chunk of chunks) this.acceptPcm(chunk);
if (this.finishing) this.client?.finish();
},
onTranscript: (event) => {
if (!current()) return;
// A handshake alone can immediately be followed by another failure.
this.attempt = 0;
this.latest = event.isFinal ? null : event;
this.options.onTranscript(event);
},
onStatus: (status) => { if (current()) this.options.onStatus(status); },
onError: (message) => {
if (!current()) return;
this.stop();
this.options.onError(message);
},
onDisconnected: (message) => {
if (current()) this.disconnected(message);
},
});
try {
this.client.start();
} catch (error) {
this.disconnected(`Transcription connection failed: ${String(error)}`);
}
}
acceptPcm(pcm: Uint8Array): void {
if (this.stopped || !pcm.length) return;
if (!this.ready) {
const copy = pcm.slice(-MAX_PENDING_BYTES);
this.pending.push(copy);
this.pendingBytes += copy.length;
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
return;
}
this.client?.acceptPcm(pcm);
// A rejected send synchronously reports disconnection. Keep that packet
// with the unsent audio instead of losing it at the outage boundary.
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
}
finish(): void {
if (this.stopped || this.finishing) return;
this.finishing = true;
// Releasing the last mic holder must not leave a reconnect loop alive.
if (this.retry !== null) {
this.preservePartial();
this.stop();
return;
}
this.finishTimeout = setTimeout(() => this.stop(), 5000);
if (this.ready) this.client?.finish();
}
stop(): void {
this.stopped = true;
++this.generation;
if (this.retry !== null) clearTimeout(this.retry);
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
this.retry = this.finishTimeout = null;
this.client?.stop();
this.client = null;
this.ready = false;
this.pending = [];
this.pendingBytes = 0;
this.latest = null;
}
private preservePartial(): void {
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
this.latest = null;
}
private disconnected(message: string): void {
if (this.stopped) return;
++this.generation; // Ignore late finals / closes from the abandoned socket.
this.ready = false;
this.client?.stop();
this.client = null;
this.pause.reset();
this.preservePartial();
if (this.stopped) return; // A transcript listener can release capture.
if (/\b(401|402|403)\b/.test(message)) {
this.stop();
this.options.onError(message);
return;
}
if (this.finishing) {
this.stop();
return;
}
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
this.retry = setTimeout(() => {
this.retry = null;
this.start();
}, delay);
}
}
+48 -13
View File
@@ -1,4 +1,5 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
import { TimedTranscript } from "./transcript-format";
declare const com: any;
@@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient {
// PCM queued until the socket opens and the config message is sent.
private readonly pendingPcm: Uint8Array[] = [];
private pendingFinish = false;
private finishing = false;
// Concatenation of all final tokens so far.
private finalText = "";
private formatted = new TimedTranscript();
constructor(private readonly options: SonioxSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
// The API key rides in the config message; there is no auth header.
this.trySendText(
JSON.stringify({
@@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient {
audio_format: "pcm_s16le",
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
enable_speaker_diarization: true,
}),
);
if (this.closed) return;
this.open = true;
for (const chunk of this.pendingPcm.splice(0)) {
this.sendPcm(chunk);
@@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient {
this.finish();
}
this.options.onStatus("Listening (Soniox)...");
this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("Soniox connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`Soniox connection failed: ${String(message)}`);
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
},
});
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
this.options.onStatus("Connecting to Soniox...");
} catch (error) {
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient {
/** End of utterance: an empty text frame tells Soniox the audio is done. */
finish(): void {
if (this.closed) return;
this.finishing = true;
if (this.open) {
this.trySendText("");
} else {
@@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient {
}
}
commitSegment(): void {
if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" }));
}
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingPcm.length = 0;
}
private sendPcm(pcm: Uint8Array): void {
try {
this.ws?.sendBinary(toJavaBytes(pcm));
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
this.options.onDisconnected?.("Soniox audio send failed.");
}
} catch (error) {
console.warn("soniox send failed", error);
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
}
}
private trySendText(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("Soniox send failed.");
}
} catch (error) {
console.warn("soniox send failed", error);
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
}
}
@@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient {
return;
}
if (message?.error_code != null) {
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
const error = `Soniox: ${String(message.error_message ?? message.error_code)}`;
if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error);
else this.options.onError(error);
return;
}
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
let nonFinal = "";
let preview: TimedTranscript | null = null;
for (const token of tokens) {
const tokenText = String(token?.text ?? "");
// Markers emitted by endpoint detection / manual finalize; not speech.
if (tokenText === "<end>" || tokenText === "<fin>") continue;
if (tokenText === "<end>" || tokenText === "<fin>") {
if (token?.is_final && this.finalText) {
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true });
this.finalText = "";
this.formatted = new TimedTranscript();
}
continue;
}
if (token?.is_final) {
this.finalText += tokenText;
this.formatted.append({ ...token, text: tokenText });
} else {
nonFinal += tokenText;
preview ??= this.formatted.copy();
preview.append({ ...token, text: tokenText });
}
}
if (message?.finished) {
this.options.onTranscript({ text: this.finalText, isFinal: true });
this.stop();
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true });
if (this.finishing) this.stop();
else this.options.onDisconnected?.("Soniox session ended; restarting transcription.");
return;
}
if (tokens.length > 0) {
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false });
}
}
}
+44
View File
@@ -0,0 +1,44 @@
/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */
export const TRANSCRIPT_PAUSE_MS = 1500;
export class SpeechPauseDetector {
private noiseFloor = 220;
private speaking = false;
private quietSamples = 0;
reset(): void {
this.noiseFloor = 220;
this.speaking = false;
this.quietSamples = 0;
}
/** PCM16LE, 16 kHz. Returns true once per pause after speech. */
accept(pcm: Uint8Array): boolean {
const count = pcm.length >> 1;
if (!count) return false;
let squares = 0;
for (let i = 0; i < count; i++) {
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
squares += sample * sample;
}
const rms = Math.sqrt(squares / count);
const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3);
if (rms >= threshold) {
this.speaking = true;
this.quietSamples = 0;
} else {
// Slowly follow background levels without learning speech as noise.
this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02);
if (this.speaking) {
this.quietSamples += count;
if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) {
this.speaking = false;
this.quietSamples = 0;
return true;
}
}
}
return false;
}
}
+32
View File
@@ -0,0 +1,32 @@
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string };
/** A separate presentation buffer keeps paragraph breaks out of dictation. */
export class TimedTranscript {
text = "";
private endMs: number | undefined;
private speaker: string | undefined;
copy(): TimedTranscript {
const copy = new TimedTranscript();
copy.text = this.text;
copy.endMs = this.endMs;
copy.speaker = this.speaker;
return copy;
}
append(token: TranscriptToken): void {
const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker;
const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS;
if (token.text.trim() && this.text && (changedSpeaker || paused)) {
this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`;
} else {
this.text += token.text;
}
if (token.text.trim()) {
if (token.end_ms != null) this.endMs = token.end_ms;
if (token.speaker != null) this.speaker = token.speaker;
}
}
}
+28 -9
View File
@@ -1,6 +1,8 @@
import { Utils } from "@nativescript/core";
import { CloudSttClient } from "./cloud-stt";
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SpeechPauseDetector } from "./speech-pause";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { SonioxSttClient } from "./soniox-stt";
@@ -14,7 +16,7 @@ export type VoiceControlState = {
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
export type VoiceTranscriptEvent = {
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
* Complete best transcript of the current utterance. REPLACE semantics —
* render as-is, replacing any previous partial. Not a delta.
@@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void;
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
private readonly speechPauseListeners = new Set<() => void>();
private readonly speechPause = new SpeechPauseDetector();
private readonly speechEndListeners = new Set<() => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
@@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge {
return () => this.speechEndListeners.delete(listener);
}
/** Pauses in on-device recognition, for Transcribe's presentation only. */
onSpeechPause(listener: () => void): () => void {
this.speechPauseListeners.add(listener);
return () => this.speechPauseListeners.delete(listener);
}
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
startPushToTalk(options: PushToTalkOptions): void {
this.acquireCapture("ptt", options);
@@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge {
this.controller?.clearSpeakerVerification();
}
this.speechPause.reset();
// A previous push-to-talk commit may still be awaiting its final result.
this.cloudClient?.stop();
const cloudClient = this.createCloudClient(options);
if (cloudClient) {
this.cloudClient = cloudClient;
@@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge {
if (options.provider === "onboard") return null;
const sttOptions = {
apiKey: "",
onTranscript: (event: { text: string; isFinal: boolean }) =>
this.emitTranscript(event.text, event.isFinal),
onTranscript: (event: CloudSttTranscriptEvent) =>
this.emitTranscript(event.text, event.isFinal, event),
onStatus: (status: string) => this.setStatus(status),
onError: (message: string) => this.setStatus(message),
};
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
if (options.provider === "elevenlabs") {
const apiKey = options.elevenLabsApiKey.trim();
if (!apiKey) {
this.setStatus("No ElevenLabs key set; using on-device voice.");
return null;
}
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
}
if (options.provider === "soniox") {
const apiKey = options.sonioxApiKey.trim();
@@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge {
this.setStatus("No Soniox key set; using on-device voice.");
return null;
}
return new SonioxSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new SonioxSttClient(config), apiKey);
}
const apiKey = options.openAiApiKey.trim();
if (!apiKey) {
this.setStatus("No OpenAI key set; using on-device voice.");
return null;
}
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
}
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
@@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge {
onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener();
}
if (this.rawPcmListeners.size > 0) {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
@@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge {
this.controller.setListener(this.listenerProxy);
}
private emitTranscript(text: string, isFinal: boolean): void {
private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void {
// "My voice only": a rejected session's final transcript is emptied so
// nothing downstream (assistant, dictation) acts on another speaker's
// words. The verification verdict is posted before the final, and cloud
// finals arrive over the network later still, so the flag is settled.
if (isFinal && this.verificationActive && this.verificationRejected) {
text = "";
metadata = undefined;
}
const event = { text, isFinal };
const event = { ...metadata, text, isFinal };
for (const listener of this.transcriptListeners) {
listener(event);
}