forked from Blendman974/faceclaw
Improve the Transcribe app
This commit is contained in:
1 parent
c9a35b0922
commit
9b70880a5b
12 files changed
+540
-140
No files matched your search
@@ -9,6 +9,9 @@
|
||||
export type CloudSttTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
/** Optional presentation for Transcribe only; dictation uses plain text. */
|
||||
transcribeText?: string;
|
||||
paragraphBreakAfter?: boolean;
|
||||
};
|
||||
|
||||
export type CloudSttOptions = {
|
||||
@@ -16,6 +19,8 @@ export type CloudSttOptions = {
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
onReady?: () => void;
|
||||
onDisconnected?: (message: string) => void;
|
||||
};
|
||||
|
||||
export interface CloudSttClient {
|
||||
@@ -25,6 +30,8 @@ export interface CloudSttClient {
|
||||
acceptPcm(pcm: Uint8Array): void;
|
||||
/** End of utterance: produce a final transcript. */
|
||||
finish(): void;
|
||||
/** Commit a pause boundary while keeping the audio session open. */
|
||||
commitSegment?(): void;
|
||||
/** Abandon the session without finalizing. */
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { encodeBase64 } from "./cloud-stt";
|
||||
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
|
||||
const MODEL_ID = "scribe_v2_realtime";
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
export type ElevenLabsTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type ElevenLabsSttOptions = {
|
||||
apiKey: string;
|
||||
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
|
||||
export type ElevenLabsSttOptions = CloudSttOptions;
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
@@ -38,26 +29,32 @@ export class ElevenLabsSttClient {
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private latestText = "";
|
||||
private readonly commits: boolean[] = [];
|
||||
|
||||
constructor(private readonly options: ElevenLabsSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
if (!this.closed) this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
@@ -67,7 +64,7 @@ export class ElevenLabsSttClient {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,24 +79,25 @@ export class ElevenLabsSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
this.commit(true);
|
||||
}
|
||||
|
||||
/** End of utterance: flush and commit for a final transcript. */
|
||||
finish(): void {
|
||||
this.commit(false);
|
||||
}
|
||||
|
||||
private commit(paragraphBreakAfter: boolean): void {
|
||||
if (this.closed) return;
|
||||
const commitMessage = JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: "",
|
||||
commit: true,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
});
|
||||
if (this.open) {
|
||||
this.trySend(commitMessage);
|
||||
} else {
|
||||
this.pendingChunks.push("__commit__");
|
||||
}
|
||||
this.commits.push(paragraphBreakAfter);
|
||||
if (this.open) this.sendChunk("__commit__");
|
||||
else this.pendingChunks.push("__commit__");
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -109,28 +107,26 @@ export class ElevenLabsSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.finish();
|
||||
return;
|
||||
}
|
||||
this.trySend(
|
||||
JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: base64OrCommit,
|
||||
commit: false,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}),
|
||||
);
|
||||
const commit = base64OrCommit === "__commit__";
|
||||
this.trySend(JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: commit ? "" : base64OrCommit,
|
||||
commit,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}));
|
||||
}
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("ElevenLabs send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("elevenlabs send failed", error);
|
||||
this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,14 +146,19 @@ export class ElevenLabsSttClient {
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
return;
|
||||
case "committed_transcript":
|
||||
case "committed_transcript_with_timestamps":
|
||||
this.latestText = String(message.text ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false });
|
||||
this.latestText = "";
|
||||
return;
|
||||
case "committed_transcript_with_timestamps":
|
||||
// The timestamp event follows the plain commit; do not append twice.
|
||||
return;
|
||||
case "rate_limited":
|
||||
this.options.onDisconnected?.("ElevenLabs rate limited.");
|
||||
return;
|
||||
case "error":
|
||||
case "auth_error":
|
||||
case "quota_exceeded":
|
||||
case "rate_limited":
|
||||
case "input_error":
|
||||
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
|
||||
return;
|
||||
|
||||
+74
-16
@@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private readonly upsampler = new PcmUpsampler();
|
||||
private latestText = "";
|
||||
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
|
||||
private readonly commits: boolean[] = [];
|
||||
private committed = false;
|
||||
|
||||
constructor(private readonly options: OpenAiSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
if (!this.closed) this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`OpenAI connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
@@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
);
|
||||
this.options.onStatus("Connecting to OpenAI...");
|
||||
} catch (error) {
|
||||
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
this.commit(true);
|
||||
}
|
||||
|
||||
/** End of utterance: commit the buffer for a final transcript. */
|
||||
finish(): void {
|
||||
this.committed = true;
|
||||
this.commit(false);
|
||||
}
|
||||
|
||||
private commit(paragraphBreakAfter: boolean): void {
|
||||
if (this.closed) return;
|
||||
this.commits.push(paragraphBreakAfter);
|
||||
if (this.open) {
|
||||
this.sendChunk("__commit__");
|
||||
} else {
|
||||
@@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.committed = true;
|
||||
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
||||
return;
|
||||
}
|
||||
@@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("OpenAI send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("openai send failed", error);
|
||||
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
case "session.updated":
|
||||
this.options.onStatus("Listening (OpenAI)...");
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
this.latestText += String(message.delta ?? "");
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
case "input_audio_buffer.committed": {
|
||||
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
this.latestText = String(message.transcript ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
// The utterance is done; nothing further is coming after the commit.
|
||||
if (this.committed) this.stop();
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.delta": {
|
||||
this.item(String(message.item_id)).text += String(message.delta ?? "");
|
||||
this.emitItems();
|
||||
return;
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.completed": {
|
||||
const item = this.item(String(message.item_id));
|
||||
item.text = String(message.transcript ?? item.text);
|
||||
item.final = true;
|
||||
this.emitItems();
|
||||
if (this.committed && !this.items.size && !this.commits.length) this.stop();
|
||||
return;
|
||||
}
|
||||
case "error":
|
||||
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
|
||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||
return;
|
||||
}
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
|
||||
let item = this.items.get(id);
|
||||
if (!item) {
|
||||
item = { text: "", final: false, paragraphBreakAfter: false };
|
||||
this.items.set(id, item);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
private emitItems(): void {
|
||||
// Committed notifications precede transcription events, so Map insertion
|
||||
// order is audio order even when completion events arrive out of order.
|
||||
for (const [id, item] of this.items) {
|
||||
if (!item.final) break;
|
||||
this.items.delete(id);
|
||||
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
|
||||
}
|
||||
if (this.items.size) {
|
||||
const items = [...this.items.values()];
|
||||
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
|
||||
const transcribeText = items.map((item, index) =>
|
||||
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
|
||||
).join("");
|
||||
this.options.onTranscript({ text, transcribeText, isFinal: false });
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
|
||||
// Ten seconds of unsent PCM. Never replay audio already sent: that would
|
||||
// duplicate words already shown. An extended outage drops the oldest audio.
|
||||
const MAX_PENDING_BYTES = 16000 * 2 * 10;
|
||||
|
||||
/** Owns retries and audio buffering independently of the provider protocol. */
|
||||
export class ReconnectingSttClient implements CloudSttClient {
|
||||
private client: CloudSttClient | null = null;
|
||||
private generation = 0;
|
||||
private stopped = false;
|
||||
private finishing = false;
|
||||
private ready = false;
|
||||
private retry: ReturnType<typeof setTimeout> | null = null;
|
||||
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
private attempt = 0;
|
||||
private pending: Uint8Array[] = [];
|
||||
private pendingBytes = 0;
|
||||
private latest: CloudSttTranscriptEvent | null = null;
|
||||
private readonly pause = new SpeechPauseDetector();
|
||||
|
||||
constructor(
|
||||
private readonly create: (options: CloudSttOptions) => CloudSttClient,
|
||||
private readonly options: CloudSttOptions,
|
||||
private readonly continuous: () => boolean,
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
if (this.stopped || this.client || this.retry !== null) return;
|
||||
const generation = ++this.generation;
|
||||
const current = () => !this.stopped && generation === this.generation;
|
||||
this.client = this.create({
|
||||
...this.options,
|
||||
onReady: () => {
|
||||
if (!current()) return;
|
||||
this.ready = true;
|
||||
const chunks = this.pending;
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
for (const chunk of chunks) this.acceptPcm(chunk);
|
||||
if (this.finishing) this.client?.finish();
|
||||
},
|
||||
onTranscript: (event) => {
|
||||
if (!current()) return;
|
||||
// A handshake alone can immediately be followed by another failure.
|
||||
this.attempt = 0;
|
||||
this.latest = event.isFinal ? null : event;
|
||||
this.options.onTranscript(event);
|
||||
},
|
||||
onStatus: (status) => { if (current()) this.options.onStatus(status); },
|
||||
onError: (message) => {
|
||||
if (!current()) return;
|
||||
this.stop();
|
||||
this.options.onError(message);
|
||||
},
|
||||
onDisconnected: (message) => {
|
||||
if (current()) this.disconnected(message);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.client.start();
|
||||
} catch (error) {
|
||||
this.disconnected(`Transcription connection failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.stopped || !pcm.length) return;
|
||||
if (!this.ready) {
|
||||
const copy = pcm.slice(-MAX_PENDING_BYTES);
|
||||
this.pending.push(copy);
|
||||
this.pendingBytes += copy.length;
|
||||
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
|
||||
return;
|
||||
}
|
||||
this.client?.acceptPcm(pcm);
|
||||
// A rejected send synchronously reports disconnection. Keep that packet
|
||||
// with the unsent audio instead of losing it at the outage boundary.
|
||||
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
|
||||
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (this.stopped || this.finishing) return;
|
||||
this.finishing = true;
|
||||
// Releasing the last mic holder must not leave a reconnect loop alive.
|
||||
if (this.retry !== null) {
|
||||
this.preservePartial();
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
this.finishTimeout = setTimeout(() => this.stop(), 5000);
|
||||
if (this.ready) this.client?.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
++this.generation;
|
||||
if (this.retry !== null) clearTimeout(this.retry);
|
||||
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
|
||||
this.retry = this.finishTimeout = null;
|
||||
this.client?.stop();
|
||||
this.client = null;
|
||||
this.ready = false;
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
this.latest = null;
|
||||
}
|
||||
|
||||
private preservePartial(): void {
|
||||
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
|
||||
this.latest = null;
|
||||
}
|
||||
|
||||
private disconnected(message: string): void {
|
||||
if (this.stopped) return;
|
||||
++this.generation; // Ignore late finals / closes from the abandoned socket.
|
||||
this.ready = false;
|
||||
this.client?.stop();
|
||||
this.client = null;
|
||||
this.pause.reset();
|
||||
this.preservePartial();
|
||||
if (this.stopped) return; // A transcript listener can release capture.
|
||||
if (/\b(401|402|403)\b/.test(message)) {
|
||||
this.stop();
|
||||
this.options.onError(message);
|
||||
return;
|
||||
}
|
||||
if (this.finishing) {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
|
||||
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
|
||||
this.retry = setTimeout(() => {
|
||||
this.retry = null;
|
||||
this.start();
|
||||
}, delay);
|
||||
}
|
||||
}
|
||||
+48
-13
@@ -1,4 +1,5 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
|
||||
import { TimedTranscript } from "./transcript-format";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
// PCM queued until the socket opens and the config message is sent.
|
||||
private readonly pendingPcm: Uint8Array[] = [];
|
||||
private pendingFinish = false;
|
||||
private finishing = false;
|
||||
// Concatenation of all final tokens so far.
|
||||
private finalText = "";
|
||||
private formatted = new TimedTranscript();
|
||||
|
||||
constructor(private readonly options: SonioxSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
// The API key rides in the config message; there is no auth header.
|
||||
this.trySendText(
|
||||
JSON.stringify({
|
||||
@@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
audio_format: "pcm_s16le",
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
enable_speaker_diarization: true,
|
||||
}),
|
||||
);
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingPcm.splice(0)) {
|
||||
this.sendPcm(chunk);
|
||||
@@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
this.finish();
|
||||
}
|
||||
this.options.onStatus("Listening (Soniox)...");
|
||||
this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("Soniox connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`Soniox connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
|
||||
this.options.onStatus("Connecting to Soniox...");
|
||||
} catch (error) {
|
||||
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
/** End of utterance: an empty text frame tells Soniox the audio is done. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
this.finishing = true;
|
||||
if (this.open) {
|
||||
this.trySendText("");
|
||||
} else {
|
||||
@@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" }));
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingPcm.length = 0;
|
||||
}
|
||||
|
||||
private sendPcm(pcm: Uint8Array): void {
|
||||
try {
|
||||
this.ws?.sendBinary(toJavaBytes(pcm));
|
||||
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
|
||||
this.options.onDisconnected?.("Soniox audio send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private trySendText(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("Soniox send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
return;
|
||||
}
|
||||
if (message?.error_code != null) {
|
||||
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
|
||||
const error = `Soniox: ${String(message.error_message ?? message.error_code)}`;
|
||||
if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error);
|
||||
else this.options.onError(error);
|
||||
return;
|
||||
}
|
||||
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
|
||||
let nonFinal = "";
|
||||
let preview: TimedTranscript | null = null;
|
||||
for (const token of tokens) {
|
||||
const tokenText = String(token?.text ?? "");
|
||||
// Markers emitted by endpoint detection / manual finalize; not speech.
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") continue;
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") {
|
||||
if (token?.is_final && this.finalText) {
|
||||
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true });
|
||||
this.finalText = "";
|
||||
this.formatted = new TimedTranscript();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (token?.is_final) {
|
||||
this.finalText += tokenText;
|
||||
this.formatted.append({ ...token, text: tokenText });
|
||||
} else {
|
||||
nonFinal += tokenText;
|
||||
preview ??= this.formatted.copy();
|
||||
preview.append({ ...token, text: tokenText });
|
||||
}
|
||||
}
|
||||
if (message?.finished) {
|
||||
this.options.onTranscript({ text: this.finalText, isFinal: true });
|
||||
this.stop();
|
||||
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true });
|
||||
if (this.finishing) this.stop();
|
||||
else this.options.onDisconnected?.("Soniox session ended; restarting transcription.");
|
||||
return;
|
||||
}
|
||||
if (tokens.length > 0) {
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */
|
||||
export const TRANSCRIPT_PAUSE_MS = 1500;
|
||||
|
||||
export class SpeechPauseDetector {
|
||||
private noiseFloor = 220;
|
||||
private speaking = false;
|
||||
private quietSamples = 0;
|
||||
|
||||
reset(): void {
|
||||
this.noiseFloor = 220;
|
||||
this.speaking = false;
|
||||
this.quietSamples = 0;
|
||||
}
|
||||
|
||||
/** PCM16LE, 16 kHz. Returns true once per pause after speech. */
|
||||
accept(pcm: Uint8Array): boolean {
|
||||
const count = pcm.length >> 1;
|
||||
if (!count) return false;
|
||||
let squares = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
|
||||
squares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(squares / count);
|
||||
const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3);
|
||||
if (rms >= threshold) {
|
||||
this.speaking = true;
|
||||
this.quietSamples = 0;
|
||||
} else {
|
||||
// Slowly follow background levels without learning speech as noise.
|
||||
this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02);
|
||||
if (this.speaking) {
|
||||
this.quietSamples += count;
|
||||
if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) {
|
||||
this.speaking = false;
|
||||
this.quietSamples = 0;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string };
|
||||
|
||||
/** A separate presentation buffer keeps paragraph breaks out of dictation. */
|
||||
export class TimedTranscript {
|
||||
text = "";
|
||||
private endMs: number | undefined;
|
||||
private speaker: string | undefined;
|
||||
|
||||
copy(): TimedTranscript {
|
||||
const copy = new TimedTranscript();
|
||||
copy.text = this.text;
|
||||
copy.endMs = this.endMs;
|
||||
copy.speaker = this.speaker;
|
||||
return copy;
|
||||
}
|
||||
|
||||
append(token: TranscriptToken): void {
|
||||
const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker;
|
||||
const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS;
|
||||
if (token.text.trim() && this.text && (changedSpeaker || paused)) {
|
||||
this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`;
|
||||
} else {
|
||||
this.text += token.text;
|
||||
}
|
||||
if (token.text.trim()) {
|
||||
if (token.end_ms != null) this.endMs = token.end_ms;
|
||||
if (token.speaker != null) this.speaker = token.speaker;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
@@ -14,7 +16,7 @@ export type VoiceControlState = {
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
* Complete best transcript of the current utterance. REPLACE semantics —
|
||||
* render as-is, replacing any previous partial. Not a delta.
|
||||
@@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void;
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
private readonly speechPauseListeners = new Set<() => void>();
|
||||
private readonly speechPause = new SpeechPauseDetector();
|
||||
private readonly speechEndListeners = new Set<() => void>();
|
||||
private controller: any | null = null;
|
||||
private listenerProxy: any | null = null;
|
||||
@@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.speechEndListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Pauses in on-device recognition, for Transcribe's presentation only. */
|
||||
onSpeechPause(listener: () => void): () => void {
|
||||
this.speechPauseListeners.add(listener);
|
||||
return () => this.speechPauseListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
this.acquireCapture("ptt", options);
|
||||
@@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller?.clearSpeakerVerification();
|
||||
}
|
||||
|
||||
this.speechPause.reset();
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
if (cloudClient) {
|
||||
this.cloudClient = cloudClient;
|
||||
@@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge {
|
||||
if (options.provider === "onboard") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: { text: string; isFinal: boolean }) =>
|
||||
this.emitTranscript(event.text, event.isFinal),
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
this.emitTranscript(event.text, event.isFinal, event),
|
||||
onStatus: (status: string) => this.setStatus(status),
|
||||
onError: (message: string) => this.setStatus(message),
|
||||
};
|
||||
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
|
||||
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
|
||||
if (options.provider === "elevenlabs") {
|
||||
const apiKey = options.elevenLabsApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
|
||||
}
|
||||
if (options.provider === "soniox") {
|
||||
const apiKey = options.sonioxApiKey.trim();
|
||||
@@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge {
|
||||
this.setStatus("No Soniox key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new SonioxSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new SonioxSttClient(config), apiKey);
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
@@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge {
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
@@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private emitTranscript(text: string, isFinal: boolean): void {
|
||||
private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void {
|
||||
// "My voice only": a rejected session's final transcript is emptied so
|
||||
// nothing downstream (assistant, dictation) acts on another speaker's
|
||||
// words. The verification verdict is posted before the final, and cloud
|
||||
// finals arrive over the network later still, so the flag is settled.
|
||||
if (isFinal && this.verificationActive && this.verificationRejected) {
|
||||
text = "";
|
||||
metadata = undefined;
|
||||
}
|
||||
const event = { text, isFinal };
|
||||
const event = { ...metadata, text, isFinal };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user