Improve the Transcribe app
This commit is contained in:
1 parent
c9a35b0922
commit
9b70880a5b
12 files changed
+540
-140
No files matched your search
@@ -2,6 +2,7 @@
|
||||
|
||||
## 0.6.6 - Future
|
||||
* Navigate and terminal app: App-specific screen size/position setting
|
||||
* Transcribe app: Scrolling; automatic reconnection; insert newlines
|
||||
|
||||
## 0.6.5 - 2026-09-04
|
||||
* Changed context menu gestures: — for system, ●— for app context menu.
|
||||
|
||||
@@ -57,10 +57,18 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
closeable: true,
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
}],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
options.stopContinuousVoiceCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
options.onClosed();
|
||||
|
||||
@@ -1,33 +1,35 @@
|
||||
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { TranscriptModel, TranscriptScroll, wrapTranscribeText } from "./transcript-model";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads on click.
|
||||
* it, and saves it to Downloads from the app menu.
|
||||
*/
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Listening...";
|
||||
// Finalized utterances, plus the live (replace-semantics) partial appended
|
||||
// when painting.
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private readonly transcript = new TranscriptModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.onTranscript(event);
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
});
|
||||
}
|
||||
@@ -36,29 +38,34 @@ export class TranscribeLayer implements Layer {
|
||||
const font = getDefaultSmallFont();
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const text = this.displayText() || "Listening...";
|
||||
const wrapped = wrapTranscribeText(font, text, width - 64);
|
||||
const text = this.transcript.text || "Listening...";
|
||||
const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64);
|
||||
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const footerY = height - font.lineHeight - 6;
|
||||
const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / step));
|
||||
const firstLine = Math.max(0, wrapped.length - bodyLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
|
||||
const y = bodyTop + (index - firstLine) * step;
|
||||
image.drawText(font, 32, y, wrapped[index]!, 230);
|
||||
}
|
||||
|
||||
image.drawText(font, 24, footerY, `${GESTURE_CLICK} save ${GESTURE_DOUBLE_CLICK} back`, 110);
|
||||
if (wrapped.length > bodyLines) {
|
||||
const trackHeight = height - bodyTop - 6;
|
||||
const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / wrapped.length));
|
||||
const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (wrapped.length - bodyLines));
|
||||
image.fillRect(width - 14, bodyTop, 2, trackHeight, 45);
|
||||
image.fillRect(width - 14, thumbY, 2, thumbHeight, 140);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "click") {
|
||||
this.saveTranscript();
|
||||
if (event.type === "scroll-up" || event.type === "scroll-down") {
|
||||
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
@@ -68,14 +75,16 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
this.unsubscribeStatus = null;
|
||||
}
|
||||
|
||||
private saveTranscript(): void {
|
||||
const text = this.displayText().trim();
|
||||
saveTranscript(): void {
|
||||
const text = this.transcript.text.trim();
|
||||
if (!text) {
|
||||
this.saveNotice = "Nothing to save yet.";
|
||||
return;
|
||||
@@ -84,25 +93,6 @@ export class TranscribeLayer implements Layer {
|
||||
const path = writeTextToDownloads(filename, `${text}\n`);
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
private displayText(): string {
|
||||
if (!this.finalizedText) return this.liveText;
|
||||
if (!this.liveText) return this.finalizedText;
|
||||
return `${this.finalizedText} ${this.liveText}`;
|
||||
}
|
||||
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
} else {
|
||||
// Replace semantics: the live partial is the whole current best transcript.
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptTimestamp(): string {
|
||||
@@ -113,22 +103,3 @@ function transcriptTimestamp(): string {
|
||||
`-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`
|
||||
);
|
||||
}
|
||||
|
||||
function wrapTranscribeText(font: UiFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) {
|
||||
lines.push(line);
|
||||
}
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
type TranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
transcribeText?: string;
|
||||
paragraphBreakAfter?: boolean;
|
||||
};
|
||||
|
||||
/** Text accumulation belongs to this app, not the shared dictation stream. */
|
||||
export class TranscriptModel {
|
||||
private finalized = "";
|
||||
private live = "";
|
||||
private separator = " ";
|
||||
private breakAfterLive = false;
|
||||
|
||||
get text(): string {
|
||||
return this.finalized && this.live ? this.finalized + this.separator + this.live : this.finalized || this.live;
|
||||
}
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
const text = (event.transcribeText ?? event.text).trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
if (finalText) this.finalized = this.finalized ? this.finalized + this.separator + finalText : finalText;
|
||||
this.live = "";
|
||||
if (finalText) this.separator = event.paragraphBreakAfter || this.breakAfterLive ? "\n" : " ";
|
||||
this.breakAfterLive = false;
|
||||
}
|
||||
|
||||
/** The onboard recognizer may finalize just before or just after the pause. */
|
||||
pause(): void {
|
||||
if (this.live) this.breakAfterLive = true;
|
||||
else if (this.finalized) this.separator = "\n";
|
||||
}
|
||||
}
|
||||
|
||||
/** An absolute history position, with an explicit follow-tail mode. */
|
||||
export class TranscriptScroll {
|
||||
firstLine = 0;
|
||||
private maximum = 0;
|
||||
private following = true;
|
||||
|
||||
layout(totalLines: number, visibleLines: number): number {
|
||||
this.maximum = Math.max(0, totalLines - visibleLines);
|
||||
this.firstLine = this.following ? this.maximum : Math.min(this.firstLine, this.maximum);
|
||||
return this.firstLine;
|
||||
}
|
||||
|
||||
scroll(delta: number): void {
|
||||
this.firstLine = Math.max(0, Math.min(this.maximum, this.firstLine + delta));
|
||||
this.following = this.firstLine === this.maximum;
|
||||
}
|
||||
}
|
||||
|
||||
export function wrapTranscribeText(measure: (text: string) => number, text: string, maxWidth: number): string[] {
|
||||
const lines: string[] = [];
|
||||
for (const paragraph of text.split("\n")) {
|
||||
let line = "";
|
||||
for (const word of paragraph.split(/\s+/).filter(Boolean)) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && measure(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = "";
|
||||
}
|
||||
if (measure(word) <= maxWidth) {
|
||||
line = line ? `${line} ${word}` : word;
|
||||
continue;
|
||||
}
|
||||
// Long URLs and unspaced languages must not run off the display.
|
||||
for (const char of (line ? " " : "") + word) {
|
||||
if (line && measure(line + char) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = "";
|
||||
}
|
||||
line += char;
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
@@ -9,6 +9,9 @@
|
||||
export type CloudSttTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
/** Optional presentation for Transcribe only; dictation uses plain text. */
|
||||
transcribeText?: string;
|
||||
paragraphBreakAfter?: boolean;
|
||||
};
|
||||
|
||||
export type CloudSttOptions = {
|
||||
@@ -16,6 +19,8 @@ export type CloudSttOptions = {
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
onReady?: () => void;
|
||||
onDisconnected?: (message: string) => void;
|
||||
};
|
||||
|
||||
export interface CloudSttClient {
|
||||
@@ -25,6 +30,8 @@ export interface CloudSttClient {
|
||||
acceptPcm(pcm: Uint8Array): void;
|
||||
/** End of utterance: produce a final transcript. */
|
||||
finish(): void;
|
||||
/** Commit a pause boundary while keeping the audio session open. */
|
||||
commitSegment?(): void;
|
||||
/** Abandon the session without finalizing. */
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { encodeBase64 } from "./cloud-stt";
|
||||
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
|
||||
const MODEL_ID = "scribe_v2_realtime";
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
export type ElevenLabsTranscriptEvent = {
|
||||
text: string;
|
||||
isFinal: boolean;
|
||||
};
|
||||
|
||||
export type ElevenLabsSttOptions = {
|
||||
apiKey: string;
|
||||
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
|
||||
export type ElevenLabsSttOptions = CloudSttOptions;
|
||||
|
||||
export class ElevenLabsSttClient {
|
||||
private ws: any = null;
|
||||
@@ -38,26 +29,32 @@ export class ElevenLabsSttClient {
|
||||
// PCM that arrived before the socket finished opening; flushed on open.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private latestText = "";
|
||||
private readonly commits: boolean[] = [];
|
||||
|
||||
constructor(private readonly options: ElevenLabsSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
if (!this.closed) this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
@@ -67,7 +64,7 @@ export class ElevenLabsSttClient {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
|
||||
this.options.onStatus("Connecting to ElevenLabs...");
|
||||
} catch (error) {
|
||||
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,24 +79,25 @@ export class ElevenLabsSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
this.commit(true);
|
||||
}
|
||||
|
||||
/** End of utterance: flush and commit for a final transcript. */
|
||||
finish(): void {
|
||||
this.commit(false);
|
||||
}
|
||||
|
||||
private commit(paragraphBreakAfter: boolean): void {
|
||||
if (this.closed) return;
|
||||
const commitMessage = JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: "",
|
||||
commit: true,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
});
|
||||
if (this.open) {
|
||||
this.trySend(commitMessage);
|
||||
} else {
|
||||
this.pendingChunks.push("__commit__");
|
||||
}
|
||||
this.commits.push(paragraphBreakAfter);
|
||||
if (this.open) this.sendChunk("__commit__");
|
||||
else this.pendingChunks.push("__commit__");
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -109,28 +107,26 @@ export class ElevenLabsSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.finish();
|
||||
return;
|
||||
}
|
||||
this.trySend(
|
||||
JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: base64OrCommit,
|
||||
commit: false,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}),
|
||||
);
|
||||
const commit = base64OrCommit === "__commit__";
|
||||
this.trySend(JSON.stringify({
|
||||
message_type: "input_audio_chunk",
|
||||
audio_base_64: commit ? "" : base64OrCommit,
|
||||
commit,
|
||||
sample_rate: SAMPLE_RATE,
|
||||
}));
|
||||
}
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("ElevenLabs send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("elevenlabs send failed", error);
|
||||
this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,14 +146,19 @@ export class ElevenLabsSttClient {
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
return;
|
||||
case "committed_transcript":
|
||||
case "committed_transcript_with_timestamps":
|
||||
this.latestText = String(message.text ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false });
|
||||
this.latestText = "";
|
||||
return;
|
||||
case "committed_transcript_with_timestamps":
|
||||
// The timestamp event follows the plain commit; do not append twice.
|
||||
return;
|
||||
case "rate_limited":
|
||||
this.options.onDisconnected?.("ElevenLabs rate limited.");
|
||||
return;
|
||||
case "error":
|
||||
case "auth_error":
|
||||
case "quota_exceeded":
|
||||
case "rate_limited":
|
||||
case "input_error":
|
||||
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
|
||||
return;
|
||||
|
||||
+74
-16
@@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
||||
private readonly pendingChunks: string[] = [];
|
||||
private readonly upsampler = new PcmUpsampler();
|
||||
private latestText = "";
|
||||
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
|
||||
private readonly commits: boolean[] = [];
|
||||
private committed = false;
|
||||
|
||||
constructor(private readonly options: OpenAiSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig()));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
if (!this.closed) this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`OpenAI connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
@@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
);
|
||||
this.options.onStatus("Connecting to OpenAI...");
|
||||
} catch (error) {
|
||||
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
this.commit(true);
|
||||
}
|
||||
|
||||
/** End of utterance: commit the buffer for a final transcript. */
|
||||
finish(): void {
|
||||
this.committed = true;
|
||||
this.commit(false);
|
||||
}
|
||||
|
||||
private commit(paragraphBreakAfter: boolean): void {
|
||||
if (this.closed) return;
|
||||
this.commits.push(paragraphBreakAfter);
|
||||
if (this.open) {
|
||||
this.sendChunk("__commit__");
|
||||
} else {
|
||||
@@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingChunks.length = 0;
|
||||
}
|
||||
|
||||
private sendChunk(base64OrCommit: string): void {
|
||||
if (base64OrCommit === "__commit__") {
|
||||
this.committed = true;
|
||||
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
||||
return;
|
||||
}
|
||||
@@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
private trySend(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("OpenAI send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("openai send failed", error);
|
||||
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
case "session.updated":
|
||||
this.options.onStatus("Listening (OpenAI)...");
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.delta":
|
||||
this.latestText += String(message.delta ?? "");
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: false });
|
||||
case "input_audio_buffer.committed": {
|
||||
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
|
||||
return;
|
||||
case "conversation.item.input_audio_transcription.completed":
|
||||
this.latestText = String(message.transcript ?? this.latestText);
|
||||
this.options.onTranscript({ text: this.latestText, isFinal: true });
|
||||
// The utterance is done; nothing further is coming after the commit.
|
||||
if (this.committed) this.stop();
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.delta": {
|
||||
this.item(String(message.item_id)).text += String(message.delta ?? "");
|
||||
this.emitItems();
|
||||
return;
|
||||
}
|
||||
case "conversation.item.input_audio_transcription.completed": {
|
||||
const item = this.item(String(message.item_id));
|
||||
item.text = String(message.transcript ?? item.text);
|
||||
item.final = true;
|
||||
this.emitItems();
|
||||
if (this.committed && !this.items.size && !this.commits.length) this.stop();
|
||||
return;
|
||||
}
|
||||
case "error":
|
||||
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
|
||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||
return;
|
||||
}
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
|
||||
let item = this.items.get(id);
|
||||
if (!item) {
|
||||
item = { text: "", final: false, paragraphBreakAfter: false };
|
||||
this.items.set(id, item);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
private emitItems(): void {
|
||||
// Committed notifications precede transcription events, so Map insertion
|
||||
// order is audio order even when completion events arrive out of order.
|
||||
for (const [id, item] of this.items) {
|
||||
if (!item.final) break;
|
||||
this.items.delete(id);
|
||||
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
|
||||
}
|
||||
if (this.items.size) {
|
||||
const items = [...this.items.values()];
|
||||
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
|
||||
const transcribeText = items.map((item, index) =>
|
||||
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
|
||||
).join("");
|
||||
this.options.onTranscript({ text, transcribeText, isFinal: false });
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(): object {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
|
||||
// Ten seconds of unsent PCM. Never replay audio already sent: that would
|
||||
// duplicate words already shown. An extended outage drops the oldest audio.
|
||||
const MAX_PENDING_BYTES = 16000 * 2 * 10;
|
||||
|
||||
/** Owns retries and audio buffering independently of the provider protocol. */
|
||||
export class ReconnectingSttClient implements CloudSttClient {
|
||||
private client: CloudSttClient | null = null;
|
||||
private generation = 0;
|
||||
private stopped = false;
|
||||
private finishing = false;
|
||||
private ready = false;
|
||||
private retry: ReturnType<typeof setTimeout> | null = null;
|
||||
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
private attempt = 0;
|
||||
private pending: Uint8Array[] = [];
|
||||
private pendingBytes = 0;
|
||||
private latest: CloudSttTranscriptEvent | null = null;
|
||||
private readonly pause = new SpeechPauseDetector();
|
||||
|
||||
constructor(
|
||||
private readonly create: (options: CloudSttOptions) => CloudSttClient,
|
||||
private readonly options: CloudSttOptions,
|
||||
private readonly continuous: () => boolean,
|
||||
) {}
|
||||
|
||||
start(): void {
|
||||
if (this.stopped || this.client || this.retry !== null) return;
|
||||
const generation = ++this.generation;
|
||||
const current = () => !this.stopped && generation === this.generation;
|
||||
this.client = this.create({
|
||||
...this.options,
|
||||
onReady: () => {
|
||||
if (!current()) return;
|
||||
this.ready = true;
|
||||
const chunks = this.pending;
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
for (const chunk of chunks) this.acceptPcm(chunk);
|
||||
if (this.finishing) this.client?.finish();
|
||||
},
|
||||
onTranscript: (event) => {
|
||||
if (!current()) return;
|
||||
// A handshake alone can immediately be followed by another failure.
|
||||
this.attempt = 0;
|
||||
this.latest = event.isFinal ? null : event;
|
||||
this.options.onTranscript(event);
|
||||
},
|
||||
onStatus: (status) => { if (current()) this.options.onStatus(status); },
|
||||
onError: (message) => {
|
||||
if (!current()) return;
|
||||
this.stop();
|
||||
this.options.onError(message);
|
||||
},
|
||||
onDisconnected: (message) => {
|
||||
if (current()) this.disconnected(message);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.client.start();
|
||||
} catch (error) {
|
||||
this.disconnected(`Transcription connection failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (this.stopped || !pcm.length) return;
|
||||
if (!this.ready) {
|
||||
const copy = pcm.slice(-MAX_PENDING_BYTES);
|
||||
this.pending.push(copy);
|
||||
this.pendingBytes += copy.length;
|
||||
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
|
||||
return;
|
||||
}
|
||||
this.client?.acceptPcm(pcm);
|
||||
// A rejected send synchronously reports disconnection. Keep that packet
|
||||
// with the unsent audio instead of losing it at the outage boundary.
|
||||
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
|
||||
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (this.stopped || this.finishing) return;
|
||||
this.finishing = true;
|
||||
// Releasing the last mic holder must not leave a reconnect loop alive.
|
||||
if (this.retry !== null) {
|
||||
this.preservePartial();
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
this.finishTimeout = setTimeout(() => this.stop(), 5000);
|
||||
if (this.ready) this.client?.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
++this.generation;
|
||||
if (this.retry !== null) clearTimeout(this.retry);
|
||||
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
|
||||
this.retry = this.finishTimeout = null;
|
||||
this.client?.stop();
|
||||
this.client = null;
|
||||
this.ready = false;
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
this.latest = null;
|
||||
}
|
||||
|
||||
private preservePartial(): void {
|
||||
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
|
||||
this.latest = null;
|
||||
}
|
||||
|
||||
private disconnected(message: string): void {
|
||||
if (this.stopped) return;
|
||||
++this.generation; // Ignore late finals / closes from the abandoned socket.
|
||||
this.ready = false;
|
||||
this.client?.stop();
|
||||
this.client = null;
|
||||
this.pause.reset();
|
||||
this.preservePartial();
|
||||
if (this.stopped) return; // A transcript listener can release capture.
|
||||
if (/\b(401|402|403)\b/.test(message)) {
|
||||
this.stop();
|
||||
this.options.onError(message);
|
||||
return;
|
||||
}
|
||||
if (this.finishing) {
|
||||
this.stop();
|
||||
return;
|
||||
}
|
||||
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
|
||||
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
|
||||
this.retry = setTimeout(() => {
|
||||
this.retry = null;
|
||||
this.start();
|
||||
}, delay);
|
||||
}
|
||||
}
|
||||
+48
-13
@@ -1,4 +1,5 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
|
||||
import { TimedTranscript } from "./transcript-format";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
// PCM queued until the socket opens and the config message is sent.
|
||||
private readonly pendingPcm: Uint8Array[] = [];
|
||||
private pendingFinish = false;
|
||||
private finishing = false;
|
||||
// Concatenation of all final tokens so far.
|
||||
private finalText = "";
|
||||
private formatted = new TimedTranscript();
|
||||
|
||||
constructor(private readonly options: SonioxSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.ws) return;
|
||||
if (this.closed || this.ws) return;
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
// The API key rides in the config message; there is no auth header.
|
||||
this.trySendText(
|
||||
JSON.stringify({
|
||||
@@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
audio_format: "pcm_s16le",
|
||||
sample_rate: CLOUD_STT_SAMPLE_RATE,
|
||||
num_channels: 1,
|
||||
enable_speaker_diarization: true,
|
||||
}),
|
||||
);
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
for (const chunk of this.pendingPcm.splice(0)) {
|
||||
this.sendPcm(chunk);
|
||||
@@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
this.finish();
|
||||
}
|
||||
this.options.onStatus("Listening (Soniox)...");
|
||||
this.options.onReady?.();
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (!this.closed) this.handleMessage(String(message));
|
||||
},
|
||||
onTextMessage: (message: string) => this.handleMessage(String(message)),
|
||||
onClosed: () => {
|
||||
this.open = false;
|
||||
if (!this.closed) this.options.onDisconnected?.("Soniox connection closed.");
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.closed) return;
|
||||
this.options.onError(`Soniox connection failed: ${String(message)}`);
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
|
||||
this.options.onStatus("Connecting to Soniox...");
|
||||
} catch (error) {
|
||||
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
/** End of utterance: an empty text frame tells Soniox the audio is done. */
|
||||
finish(): void {
|
||||
if (this.closed) return;
|
||||
this.finishing = true;
|
||||
if (this.open) {
|
||||
this.trySendText("");
|
||||
} else {
|
||||
@@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
}
|
||||
}
|
||||
|
||||
commitSegment(): void {
|
||||
if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" }));
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.closed = true;
|
||||
this.open = false;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
@@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.pendingPcm.length = 0;
|
||||
}
|
||||
|
||||
private sendPcm(pcm: Uint8Array): void {
|
||||
try {
|
||||
this.ws?.sendBinary(toJavaBytes(pcm));
|
||||
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
|
||||
this.options.onDisconnected?.("Soniox audio send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private trySendText(message: string): void {
|
||||
try {
|
||||
this.ws?.sendText(message);
|
||||
if (this.ws?.sendText(message) === false) {
|
||||
this.options.onDisconnected?.("Soniox send failed.");
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("soniox send failed", error);
|
||||
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient {
|
||||
return;
|
||||
}
|
||||
if (message?.error_code != null) {
|
||||
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
|
||||
const error = `Soniox: ${String(message.error_message ?? message.error_code)}`;
|
||||
if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error);
|
||||
else this.options.onError(error);
|
||||
return;
|
||||
}
|
||||
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
|
||||
let nonFinal = "";
|
||||
let preview: TimedTranscript | null = null;
|
||||
for (const token of tokens) {
|
||||
const tokenText = String(token?.text ?? "");
|
||||
// Markers emitted by endpoint detection / manual finalize; not speech.
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") continue;
|
||||
if (tokenText === "<end>" || tokenText === "<fin>") {
|
||||
if (token?.is_final && this.finalText) {
|
||||
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true });
|
||||
this.finalText = "";
|
||||
this.formatted = new TimedTranscript();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (token?.is_final) {
|
||||
this.finalText += tokenText;
|
||||
this.formatted.append({ ...token, text: tokenText });
|
||||
} else {
|
||||
nonFinal += tokenText;
|
||||
preview ??= this.formatted.copy();
|
||||
preview.append({ ...token, text: tokenText });
|
||||
}
|
||||
}
|
||||
if (message?.finished) {
|
||||
this.options.onTranscript({ text: this.finalText, isFinal: true });
|
||||
this.stop();
|
||||
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true });
|
||||
if (this.finishing) this.stop();
|
||||
else this.options.onDisconnected?.("Soniox session ended; restarting transcription.");
|
||||
return;
|
||||
}
|
||||
if (tokens.length > 0) {
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
|
||||
this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */
|
||||
export const TRANSCRIPT_PAUSE_MS = 1500;
|
||||
|
||||
export class SpeechPauseDetector {
|
||||
private noiseFloor = 220;
|
||||
private speaking = false;
|
||||
private quietSamples = 0;
|
||||
|
||||
reset(): void {
|
||||
this.noiseFloor = 220;
|
||||
this.speaking = false;
|
||||
this.quietSamples = 0;
|
||||
}
|
||||
|
||||
/** PCM16LE, 16 kHz. Returns true once per pause after speech. */
|
||||
accept(pcm: Uint8Array): boolean {
|
||||
const count = pcm.length >> 1;
|
||||
if (!count) return false;
|
||||
let squares = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
|
||||
squares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(squares / count);
|
||||
const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3);
|
||||
if (rms >= threshold) {
|
||||
this.speaking = true;
|
||||
this.quietSamples = 0;
|
||||
} else {
|
||||
// Slowly follow background levels without learning speech as noise.
|
||||
this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02);
|
||||
if (this.speaking) {
|
||||
this.quietSamples += count;
|
||||
if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) {
|
||||
this.speaking = false;
|
||||
this.quietSamples = 0;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string };
|
||||
|
||||
/** A separate presentation buffer keeps paragraph breaks out of dictation. */
|
||||
export class TimedTranscript {
|
||||
text = "";
|
||||
private endMs: number | undefined;
|
||||
private speaker: string | undefined;
|
||||
|
||||
copy(): TimedTranscript {
|
||||
const copy = new TimedTranscript();
|
||||
copy.text = this.text;
|
||||
copy.endMs = this.endMs;
|
||||
copy.speaker = this.speaker;
|
||||
return copy;
|
||||
}
|
||||
|
||||
append(token: TranscriptToken): void {
|
||||
const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker;
|
||||
const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS;
|
||||
if (token.text.trim() && this.text && (changedSpeaker || paused)) {
|
||||
this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`;
|
||||
} else {
|
||||
this.text += token.text;
|
||||
}
|
||||
if (token.text.trim()) {
|
||||
if (token.end_ms != null) this.endMs = token.end_ms;
|
||||
if (token.speaker != null) this.speaker = token.speaker;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
@@ -14,7 +16,7 @@ export type VoiceControlState = {
|
||||
|
||||
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
export type VoiceTranscriptEvent = {
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
* Complete best transcript of the current utterance. REPLACE semantics —
|
||||
* render as-is, replacing any previous partial. Not a delta.
|
||||
@@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void;
|
||||
export class FaceclawVoiceControlBridge {
|
||||
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
|
||||
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
|
||||
private readonly speechPauseListeners = new Set<() => void>();
|
||||
private readonly speechPause = new SpeechPauseDetector();
|
||||
private readonly speechEndListeners = new Set<() => void>();
|
||||
private controller: any | null = null;
|
||||
private listenerProxy: any | null = null;
|
||||
@@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge {
|
||||
return () => this.speechEndListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Pauses in on-device recognition, for Transcribe's presentation only. */
|
||||
onSpeechPause(listener: () => void): () => void {
|
||||
this.speechPauseListeners.add(listener);
|
||||
return () => this.speechPauseListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
|
||||
startPushToTalk(options: PushToTalkOptions): void {
|
||||
this.acquireCapture("ptt", options);
|
||||
@@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller?.clearSpeakerVerification();
|
||||
}
|
||||
|
||||
this.speechPause.reset();
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
if (cloudClient) {
|
||||
this.cloudClient = cloudClient;
|
||||
@@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge {
|
||||
if (options.provider === "onboard") return null;
|
||||
const sttOptions = {
|
||||
apiKey: "",
|
||||
onTranscript: (event: { text: string; isFinal: boolean }) =>
|
||||
this.emitTranscript(event.text, event.isFinal),
|
||||
onTranscript: (event: CloudSttTranscriptEvent) =>
|
||||
this.emitTranscript(event.text, event.isFinal, event),
|
||||
onStatus: (status: string) => this.setStatus(status),
|
||||
onError: (message: string) => this.setStatus(message),
|
||||
};
|
||||
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
|
||||
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
|
||||
if (options.provider === "elevenlabs") {
|
||||
const apiKey = options.elevenLabsApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No ElevenLabs key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
|
||||
}
|
||||
if (options.provider === "soniox") {
|
||||
const apiKey = options.sonioxApiKey.trim();
|
||||
@@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge {
|
||||
this.setStatus("No Soniox key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new SonioxSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new SonioxSttClient(config), apiKey);
|
||||
}
|
||||
const apiKey = options.openAiApiKey.trim();
|
||||
if (!apiKey) {
|
||||
this.setStatus("No OpenAI key set; using on-device voice.");
|
||||
return null;
|
||||
}
|
||||
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
|
||||
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
|
||||
}
|
||||
|
||||
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
|
||||
@@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge {
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
if (this.rawPcmListeners.size > 0) {
|
||||
this.rawPcmListeners.forEach((listener) => listener(bytes));
|
||||
}
|
||||
@@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge {
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
|
||||
private emitTranscript(text: string, isFinal: boolean): void {
|
||||
private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void {
|
||||
// "My voice only": a rejected session's final transcript is emptied so
|
||||
// nothing downstream (assistant, dictation) acts on another speaker's
|
||||
// words. The verification verdict is posted before the final, and cloud
|
||||
// finals arrive over the network later still, so the flag is settled.
|
||||
if (isFinal && this.verificationActive && this.verificationRejected) {
|
||||
text = "";
|
||||
metadata = undefined;
|
||||
}
|
||||
const event = { text, isFinal };
|
||||
const event = { ...metadata, text, isFinal };
|
||||
for (const listener of this.transcriptListeners) {
|
||||
listener(event);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user