Improve the Transcribe app

This commit is contained in:
James Babcock committed 2026-09-05 19:16:56 -07:00
1 parent c9a35b0922
commit 9b70880a5b
12 files changed
+540 -140

No files matched your search

+1
View File
@@ -2,6 +2,7 @@
## 0.6.6 - Future
* Navigate and terminal app: App-specific screen size/position setting
* Transcribe app: Scrolling; automatic reconnection; insert newlines
## 0.6.5 - 2026-09-04
* Changed context menu gestures: — for system, ●— for app context menu.
+8
View File
@@ -57,10 +57,18 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(layer),
menuItems: () => [{
label: "Save",
onSelect: (ctx) => {
ctx.stack.pop();
layer.saveTranscript();
},
}],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
layer.onRemoved();
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
+29 -58
View File
@@ -1,33 +1,35 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
import { type InputEvent } from "../../ui/gestures";
import { TranscriptModel, TranscriptScroll, wrapTranscribeText } from "./transcript-model";
/**
* Live transcription view. Continuous mic capture is owned by the app wrapper
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
* it, and saves it to Downloads on click.
* it, and saves it to Downloads from the app menu.
*/
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
export class TranscribeLayer implements Layer {
private status = "Listening...";
// Finalized utterances, plus the live (replace-semantics) partial appended
// when painting.
private finalizedText = "";
private liveText = "";
private readonly transcript = new TranscriptModel();
private readonly scroll = new TranscriptScroll();
private saveNotice = "";
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribePause: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
start(requestRender: () => void): void {
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.onTranscript(event);
this.transcript.accept(event);
requestRender();
});
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
this.saveNotice = "";
requestRender();
});
}
@@ -36,29 +38,34 @@ export class TranscribeLayer implements Layer {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const text = this.displayText() || "Listening...";
const wrapped = wrapTranscribeText(font, text, width - 64);
const text = this.transcript.text || "Listening...";
const wrapped = wrapTranscribeText((text) => font.measureText(text), text, width - 64);
image.drawText(font, 24, 20, "Transcribe", 200);
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
const step = lineStep(font) + 2;
const bodyTop = 62;
const footerY = height - font.lineHeight - 6;
const bodyLines = Math.max(1, Math.floor((footerY - bodyTop) / step));
const firstLine = Math.max(0, wrapped.length - bodyLines);
for (let index = firstLine; index < wrapped.length; index++) {
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
const y = bodyTop + (index - firstLine) * step;
image.drawText(font, 32, y, wrapped[index]!, 230);
}
image.drawText(font, 24, footerY, `${GESTURE_CLICK} save ${GESTURE_DOUBLE_CLICK} back`, 110);
if (wrapped.length > bodyLines) {
const trackHeight = height - bodyTop - 6;
const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / wrapped.length));
const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (wrapped.length - bodyLines));
image.fillRect(width - 14, bodyTop, 2, trackHeight, 45);
image.fillRect(width - 14, thumbY, 2, thumbHeight, 140);
}
return image;
}
handleInput(event: InputEvent, ctx: LayerContext): void {
if (event.type === "click") {
this.saveTranscript();
if (event.type === "scroll-up" || event.type === "scroll-down") {
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
ctx.actions.requestRender();
return;
}
@@ -68,14 +75,16 @@ export class TranscribeLayer implements Layer {
}
onRemoved(): void {
this.unsubscribePause?.();
this.unsubscribePause = null;
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
}
private saveTranscript(): void {
const text = this.displayText().trim();
saveTranscript(): void {
const text = this.transcript.text.trim();
if (!text) {
this.saveNotice = "Nothing to save yet.";
return;
@@ -84,25 +93,6 @@ export class TranscribeLayer implements Layer {
const path = writeTextToDownloads(filename, `${text}\n`);
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
}
private displayText(): string {
if (!this.finalizedText) return this.liveText;
if (!this.liveText) return this.finalizedText;
return `${this.finalizedText} ${this.liveText}`;
}
private onTranscript(event: VoiceTranscriptEvent): void {
if (event.isFinal) {
const finalText = event.text.trim() || this.liveText.trim();
if (finalText) {
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
}
this.liveText = "";
} else {
// Replace semantics: the live partial is the whole current best transcript.
this.liveText = event.text.trim();
}
}
}
function transcriptTimestamp(): string {
@@ -113,22 +103,3 @@ function transcriptTimestamp(): string {
`-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`
);
}
function wrapTranscribeText(font: UiFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) {
lines.push(line);
}
return lines.length ? lines : [""];
}
+83
View File
@@ -0,0 +1,83 @@
type TranscriptEvent = {
text: string;
isFinal: boolean;
transcribeText?: string;
paragraphBreakAfter?: boolean;
};
/** Text accumulation belongs to this app, not the shared dictation stream. */
export class TranscriptModel {
private finalized = "";
private live = "";
private separator = " ";
private breakAfterLive = false;
get text(): string {
return this.finalized && this.live ? this.finalized + this.separator + this.live : this.finalized || this.live;
}
accept(event: TranscriptEvent): void {
const text = (event.transcribeText ?? event.text).trim();
if (!event.isFinal) {
this.live = text;
return;
}
const finalText = text || this.live;
if (finalText) this.finalized = this.finalized ? this.finalized + this.separator + finalText : finalText;
this.live = "";
if (finalText) this.separator = event.paragraphBreakAfter || this.breakAfterLive ? "\n" : " ";
this.breakAfterLive = false;
}
/** The onboard recognizer may finalize just before or just after the pause. */
pause(): void {
if (this.live) this.breakAfterLive = true;
else if (this.finalized) this.separator = "\n";
}
}
/** An absolute history position, with an explicit follow-tail mode. */
export class TranscriptScroll {
firstLine = 0;
private maximum = 0;
private following = true;
layout(totalLines: number, visibleLines: number): number {
this.maximum = Math.max(0, totalLines - visibleLines);
this.firstLine = this.following ? this.maximum : Math.min(this.firstLine, this.maximum);
return this.firstLine;
}
scroll(delta: number): void {
this.firstLine = Math.max(0, Math.min(this.maximum, this.firstLine + delta));
this.following = this.firstLine === this.maximum;
}
}
export function wrapTranscribeText(measure: (text: string) => number, text: string, maxWidth: number): string[] {
const lines: string[] = [];
for (const paragraph of text.split("\n")) {
let line = "";
for (const word of paragraph.split(/\s+/).filter(Boolean)) {
const candidate = line ? `${line} ${word}` : word;
if (line && measure(candidate) > maxWidth) {
lines.push(line);
line = "";
}
if (measure(word) <= maxWidth) {
line = line ? `${line} ${word}` : word;
continue;
}
// Long URLs and unspaced languages must not run off the display.
for (const char of (line ? " " : "") + word) {
if (line && measure(line + char) > maxWidth) {
lines.push(line);
line = "";
}
line += char;
}
}
lines.push(line);
}
return lines;
}
+7
View File
@@ -9,6 +9,9 @@
export type CloudSttTranscriptEvent = {
text: string;
isFinal: boolean;
/** Optional presentation for Transcribe only; dictation uses plain text. */
transcribeText?: string;
paragraphBreakAfter?: boolean;
};
export type CloudSttOptions = {
@@ -16,6 +19,8 @@ export type CloudSttOptions = {
onTranscript: (event: CloudSttTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
onReady?: () => void;
onDisconnected?: (message: string) => void;
};
export interface CloudSttClient {
@@ -25,6 +30,8 @@ export interface CloudSttClient {
acceptPcm(pcm: Uint8Array): void;
/** End of utterance: produce a final transcript. */
finish(): void;
/** Commit a pause boundary while keeping the audio session open. */
commitSegment?(): void;
/** Abandon the session without finalizing. */
stop(): void;
}
+45 -44
View File
@@ -1,4 +1,4 @@
import { encodeBase64 } from "./cloud-stt";
import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt";
declare const com: any;
@@ -18,17 +18,8 @@ const WS_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime";
const MODEL_ID = "scribe_v2_realtime";
const SAMPLE_RATE = 16000;
export type ElevenLabsTranscriptEvent = {
text: string;
isFinal: boolean;
};
export type ElevenLabsSttOptions = {
apiKey: string;
onTranscript: (event: ElevenLabsTranscriptEvent) => void;
onStatus: (status: string) => void;
onError: (message: string) => void;
};
export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent;
export type ElevenLabsSttOptions = CloudSttOptions;
export class ElevenLabsSttClient {
private ws: any = null;
@@ -38,26 +29,32 @@ export class ElevenLabsSttClient {
// PCM that arrived before the socket finished opening; flushed on open.
private readonly pendingChunks: string[] = [];
private latestText = "";
private readonly commits: boolean[] = [];
constructor(private readonly options: ElevenLabsSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
this.open = true;
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
if (!this.closed) this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("ElevenLabs connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`ElevenLabs connection failed: ${String(message)}`);
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`);
},
});
try {
@@ -67,7 +64,7 @@ export class ElevenLabsSttClient {
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key);
this.options.onStatus("Connecting to ElevenLabs...");
} catch (error) {
this.options.onError(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -82,24 +79,25 @@ export class ElevenLabsSttClient {
}
}
commitSegment(): void {
this.commit(true);
}
/** End of utterance: flush and commit for a final transcript. */
finish(): void {
this.commit(false);
}
private commit(paragraphBreakAfter: boolean): void {
if (this.closed) return;
const commitMessage = JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: "",
commit: true,
sample_rate: SAMPLE_RATE,
});
if (this.open) {
this.trySend(commitMessage);
} else {
this.pendingChunks.push("__commit__");
}
this.commits.push(paragraphBreakAfter);
if (this.open) this.sendChunk("__commit__");
else this.pendingChunks.push("__commit__");
}
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -109,28 +107,26 @@ export class ElevenLabsSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
private sendChunk(base64OrCommit: string): void {
if (base64OrCommit === "__commit__") {
this.finish();
return;
}
this.trySend(
JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: base64OrCommit,
commit: false,
sample_rate: SAMPLE_RATE,
}),
);
const commit = base64OrCommit === "__commit__";
this.trySend(JSON.stringify({
message_type: "input_audio_chunk",
audio_base_64: commit ? "" : base64OrCommit,
commit,
sample_rate: SAMPLE_RATE,
}));
}
private trySend(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("ElevenLabs send failed.");
}
} catch (error) {
console.warn("elevenlabs send failed", error);
this.options.onDisconnected?.(`ElevenLabs send failed: ${String(error)}`);
}
}
@@ -150,14 +146,19 @@ export class ElevenLabsSttClient {
this.options.onTranscript({ text: this.latestText, isFinal: false });
return;
case "committed_transcript":
case "committed_transcript_with_timestamps":
this.latestText = String(message.text ?? this.latestText);
this.options.onTranscript({ text: this.latestText, isFinal: true });
this.options.onTranscript({ text: this.latestText, isFinal: true, paragraphBreakAfter: this.commits.shift() ?? false });
this.latestText = "";
return;
case "committed_transcript_with_timestamps":
// The timestamp event follows the plain commit; do not append twice.
return;
case "rate_limited":
this.options.onDisconnected?.("ElevenLabs rate limited.");
return;
case "error":
case "auth_error":
case "quota_exceeded":
case "rate_limited":
case "input_error":
this.options.onError(`ElevenLabs: ${String(message.error ?? message.message_type)}`);
return;
+74 -16
View File
@@ -30,28 +30,34 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
// Base64 audio (or the commit sentinel) queued until the socket opens.
private readonly pendingChunks: string[] = [];
private readonly upsampler = new PcmUpsampler();
private latestText = "";
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
private readonly commits: boolean[] = [];
private committed = false;
constructor(private readonly options: OpenAiSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
this.open = true;
this.trySend(JSON.stringify(sessionConfig()));
for (const chunk of this.pendingChunks.splice(0)) {
this.sendChunk(chunk);
}
if (!this.closed) this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`OpenAI connection failed: ${String(message)}`);
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
},
});
try {
@@ -63,7 +69,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
);
this.options.onStatus("Connecting to OpenAI...");
} catch (error) {
this.options.onError(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -80,9 +86,19 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
}
}
commitSegment(): void {
this.commit(true);
}
/** End of utterance: commit the buffer for a final transcript. */
finish(): void {
this.committed = true;
this.commit(false);
}
private commit(paragraphBreakAfter: boolean): void {
if (this.closed) return;
this.commits.push(paragraphBreakAfter);
if (this.open) {
this.sendChunk("__commit__");
} else {
@@ -92,6 +108,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -101,11 +118,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingChunks.length = 0;
}
private sendChunk(base64OrCommit: string): void {
if (base64OrCommit === "__commit__") {
this.committed = true;
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
return;
}
@@ -114,9 +131,11 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
private trySend(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("OpenAI send failed.");
}
} catch (error) {
console.warn("openai send failed", error);
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
}
}
@@ -132,23 +151,62 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
case "session.updated":
this.options.onStatus("Listening (OpenAI)...");
return;
case "conversation.item.input_audio_transcription.delta":
this.latestText += String(message.delta ?? "");
this.options.onTranscript({ text: this.latestText, isFinal: false });
case "input_audio_buffer.committed": {
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
return;
case "conversation.item.input_audio_transcription.completed":
this.latestText = String(message.transcript ?? this.latestText);
this.options.onTranscript({ text: this.latestText, isFinal: true });
// The utterance is done; nothing further is coming after the commit.
if (this.committed) this.stop();
}
case "conversation.item.input_audio_transcription.delta": {
this.item(String(message.item_id)).text += String(message.delta ?? "");
this.emitItems();
return;
}
case "conversation.item.input_audio_transcription.completed": {
const item = this.item(String(message.item_id));
item.text = String(message.transcript ?? item.text);
item.final = true;
this.emitItems();
if (this.committed && !this.items.size && !this.commits.length) this.stop();
return;
}
case "error":
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
return;
}
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
return;
default:
return;
}
}
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
let item = this.items.get(id);
if (!item) {
item = { text: "", final: false, paragraphBreakAfter: false };
this.items.set(id, item);
}
return item;
}
private emitItems(): void {
// Committed notifications precede transcription events, so Map insertion
// order is audio order even when completion events arrive out of order.
for (const [id, item] of this.items) {
if (!item.final) break;
this.items.delete(id);
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
}
if (this.items.size) {
const items = [...this.items.values()];
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
const transcribeText = items.map((item, index) =>
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
).join("");
this.options.onTranscript({ text, transcribeText, isFinal: false });
}
}
}
function sessionConfig(): object {
+141
View File
@@ -0,0 +1,141 @@
import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
import { SpeechPauseDetector } from "./speech-pause";
// Ten seconds of unsent PCM. Never replay audio already sent: that would
// duplicate words already shown. An extended outage drops the oldest audio.
const MAX_PENDING_BYTES = 16000 * 2 * 10;
/** Owns retries and audio buffering independently of the provider protocol. */
export class ReconnectingSttClient implements CloudSttClient {
private client: CloudSttClient | null = null;
private generation = 0;
private stopped = false;
private finishing = false;
private ready = false;
private retry: ReturnType<typeof setTimeout> | null = null;
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
private attempt = 0;
private pending: Uint8Array[] = [];
private pendingBytes = 0;
private latest: CloudSttTranscriptEvent | null = null;
private readonly pause = new SpeechPauseDetector();
constructor(
private readonly create: (options: CloudSttOptions) => CloudSttClient,
private readonly options: CloudSttOptions,
private readonly continuous: () => boolean,
) {}
start(): void {
if (this.stopped || this.client || this.retry !== null) return;
const generation = ++this.generation;
const current = () => !this.stopped && generation === this.generation;
this.client = this.create({
...this.options,
onReady: () => {
if (!current()) return;
this.ready = true;
const chunks = this.pending;
this.pending = [];
this.pendingBytes = 0;
for (const chunk of chunks) this.acceptPcm(chunk);
if (this.finishing) this.client?.finish();
},
onTranscript: (event) => {
if (!current()) return;
// A handshake alone can immediately be followed by another failure.
this.attempt = 0;
this.latest = event.isFinal ? null : event;
this.options.onTranscript(event);
},
onStatus: (status) => { if (current()) this.options.onStatus(status); },
onError: (message) => {
if (!current()) return;
this.stop();
this.options.onError(message);
},
onDisconnected: (message) => {
if (current()) this.disconnected(message);
},
});
try {
this.client.start();
} catch (error) {
this.disconnected(`Transcription connection failed: ${String(error)}`);
}
}
acceptPcm(pcm: Uint8Array): void {
if (this.stopped || !pcm.length) return;
if (!this.ready) {
const copy = pcm.slice(-MAX_PENDING_BYTES);
this.pending.push(copy);
this.pendingBytes += copy.length;
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
return;
}
this.client?.acceptPcm(pcm);
// A rejected send synchronously reports disconnection. Keep that packet
// with the unsent audio instead of losing it at the outage boundary.
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
}
finish(): void {
if (this.stopped || this.finishing) return;
this.finishing = true;
// Releasing the last mic holder must not leave a reconnect loop alive.
if (this.retry !== null) {
this.preservePartial();
this.stop();
return;
}
this.finishTimeout = setTimeout(() => this.stop(), 5000);
if (this.ready) this.client?.finish();
}
stop(): void {
this.stopped = true;
++this.generation;
if (this.retry !== null) clearTimeout(this.retry);
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
this.retry = this.finishTimeout = null;
this.client?.stop();
this.client = null;
this.ready = false;
this.pending = [];
this.pendingBytes = 0;
this.latest = null;
}
private preservePartial(): void {
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
this.latest = null;
}
private disconnected(message: string): void {
if (this.stopped) return;
++this.generation; // Ignore late finals / closes from the abandoned socket.
this.ready = false;
this.client?.stop();
this.client = null;
this.pause.reset();
this.preservePartial();
if (this.stopped) return; // A transcript listener can release capture.
if (/\b(401|402|403)\b/.test(message)) {
this.stop();
this.options.onError(message);
return;
}
if (this.finishing) {
this.stop();
return;
}
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
this.retry = setTimeout(() => {
this.retry = null;
this.start();
}, delay);
}
}
+48 -13
View File
@@ -1,4 +1,5 @@
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt";
import { TimedTranscript } from "./transcript-format";
declare const com: any;
@@ -30,15 +31,18 @@ export class SonioxSttClient implements CloudSttClient {
// PCM queued until the socket opens and the config message is sent.
private readonly pendingPcm: Uint8Array[] = [];
private pendingFinish = false;
private finishing = false;
// Concatenation of all final tokens so far.
private finalText = "";
private formatted = new TimedTranscript();
constructor(private readonly options: SonioxSttOptions) {}
start(): void {
if (this.ws) return;
if (this.closed || this.ws) return;
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.closed) return;
// The API key rides in the config message; there is no auth header.
this.trySendText(
JSON.stringify({
@@ -47,8 +51,10 @@ export class SonioxSttClient implements CloudSttClient {
audio_format: "pcm_s16le",
sample_rate: CLOUD_STT_SAMPLE_RATE,
num_channels: 1,
enable_speaker_diarization: true,
}),
);
if (this.closed) return;
this.open = true;
for (const chunk of this.pendingPcm.splice(0)) {
this.sendPcm(chunk);
@@ -58,21 +64,25 @@ export class SonioxSttClient implements CloudSttClient {
this.finish();
}
this.options.onStatus("Listening (Soniox)...");
this.options.onReady?.();
},
onTextMessage: (message: string) => {
if (!this.closed) this.handleMessage(String(message));
},
onTextMessage: (message: string) => this.handleMessage(String(message)),
onClosed: () => {
this.open = false;
if (!this.closed) this.options.onDisconnected?.("Soniox connection closed.");
},
onFailure: (message: string) => {
if (this.closed) return;
this.options.onError(`Soniox connection failed: ${String(message)}`);
this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`);
},
});
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null);
this.options.onStatus("Connecting to Soniox...");
} catch (error) {
this.options.onError(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`);
}
}
@@ -89,6 +99,7 @@ export class SonioxSttClient implements CloudSttClient {
/** End of utterance: an empty text frame tells Soniox the audio is done. */
finish(): void {
if (this.closed) return;
this.finishing = true;
if (this.open) {
this.trySendText("");
} else {
@@ -96,8 +107,13 @@ export class SonioxSttClient implements CloudSttClient {
}
}
commitSegment(): void {
if (!this.closed && this.open) this.trySendText(JSON.stringify({ type: "finalize" }));
}
stop(): void {
this.closed = true;
this.open = false;
if (this.ws) {
try {
this.ws.close(1000, "bye");
@@ -107,21 +123,26 @@ export class SonioxSttClient implements CloudSttClient {
this.ws = null;
}
this.listenerProxy = null;
this.pendingPcm.length = 0;
}
private sendPcm(pcm: Uint8Array): void {
try {
this.ws?.sendBinary(toJavaBytes(pcm));
if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) {
this.options.onDisconnected?.("Soniox audio send failed.");
}
} catch (error) {
console.warn("soniox send failed", error);
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
}
}
private trySendText(message: string): void {
try {
this.ws?.sendText(message);
if (this.ws?.sendText(message) === false) {
this.options.onDisconnected?.("Soniox send failed.");
}
} catch (error) {
console.warn("soniox send failed", error);
this.options.onDisconnected?.(`Soniox send failed: ${String(error)}`);
}
}
@@ -133,28 +154,42 @@ export class SonioxSttClient implements CloudSttClient {
return;
}
if (message?.error_code != null) {
this.options.onError(`Soniox: ${String(message.error_message ?? message.error_code)}`);
const error = `Soniox: ${String(message.error_message ?? message.error_code)}`;
if (Number(message.error_code) === 429 || Number(message.error_code) >= 500) this.options.onDisconnected?.(error);
else this.options.onError(error);
return;
}
const tokens = Array.isArray(message?.tokens) ? message.tokens : [];
let nonFinal = "";
let preview: TimedTranscript | null = null;
for (const token of tokens) {
const tokenText = String(token?.text ?? "");
// Markers emitted by endpoint detection / manual finalize; not speech.
if (tokenText === "<end>" || tokenText === "<fin>") continue;
if (tokenText === "<end>" || tokenText === "<fin>") {
if (token?.is_final && this.finalText) {
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true, paragraphBreakAfter: true });
this.finalText = "";
this.formatted = new TimedTranscript();
}
continue;
}
if (token?.is_final) {
this.finalText += tokenText;
this.formatted.append({ ...token, text: tokenText });
} else {
nonFinal += tokenText;
preview ??= this.formatted.copy();
preview.append({ ...token, text: tokenText });
}
}
if (message?.finished) {
this.options.onTranscript({ text: this.finalText, isFinal: true });
this.stop();
this.options.onTranscript({ text: this.finalText, transcribeText: this.formatted.text, isFinal: true });
if (this.finishing) this.stop();
else this.options.onDisconnected?.("Soniox session ended; restarting transcription.");
return;
}
if (tokens.length > 0) {
this.options.onTranscript({ text: this.finalText + nonFinal, isFinal: false });
this.options.onTranscript({ text: this.finalText + nonFinal, transcribeText: (preview ?? this.formatted).text, isFinal: false });
}
}
}
+44
View File
@@ -0,0 +1,44 @@
/** Sample-clock pause detection: BLE/network delivery timing is not speech timing. */
export const TRANSCRIPT_PAUSE_MS = 1500;
export class SpeechPauseDetector {
private noiseFloor = 220;
private speaking = false;
private quietSamples = 0;
reset(): void {
this.noiseFloor = 220;
this.speaking = false;
this.quietSamples = 0;
}
/** PCM16LE, 16 kHz. Returns true once per pause after speech. */
accept(pcm: Uint8Array): boolean {
const count = pcm.length >> 1;
if (!count) return false;
let squares = 0;
for (let i = 0; i < count; i++) {
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
squares += sample * sample;
}
const rms = Math.sqrt(squares / count);
const threshold = this.noiseFloor * (this.speaking ? 1.8 : 3);
if (rms >= threshold) {
this.speaking = true;
this.quietSamples = 0;
} else {
// Slowly follow background levels without learning speech as noise.
this.noiseFloor = Math.max(220, this.noiseFloor * 0.98 + rms * 0.02);
if (this.speaking) {
this.quietSamples += count;
if (this.quietSamples >= 16 * TRANSCRIPT_PAUSE_MS) {
this.speaking = false;
this.quietSamples = 0;
return true;
}
}
}
return false;
}
}
+32
View File
@@ -0,0 +1,32 @@
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
export type TranscriptToken = { text: string; start_ms?: number; end_ms?: number; speaker?: string };
/** A separate presentation buffer keeps paragraph breaks out of dictation. */
export class TimedTranscript {
text = "";
private endMs: number | undefined;
private speaker: string | undefined;
copy(): TimedTranscript {
const copy = new TimedTranscript();
copy.text = this.text;
copy.endMs = this.endMs;
copy.speaker = this.speaker;
return copy;
}
append(token: TranscriptToken): void {
const changedSpeaker = token.speaker != null && this.speaker != null && token.speaker !== this.speaker;
const paused = token.start_ms != null && this.endMs != null && token.start_ms - this.endMs >= TRANSCRIPT_PAUSE_MS;
if (token.text.trim() && this.text && (changedSpeaker || paused)) {
this.text = `${this.text.trimEnd()}\n${token.text.trimStart()}`;
} else {
this.text += token.text;
}
if (token.text.trim()) {
if (token.end_ms != null) this.endMs = token.end_ms;
if (token.speaker != null) this.speaker = token.speaker;
}
}
}
+28 -9
View File
@@ -1,6 +1,8 @@
import { Utils } from "@nativescript/core";
import { CloudSttClient } from "./cloud-stt";
import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SpeechPauseDetector } from "./speech-pause";
import { ElevenLabsSttClient } from "./elevenlabs-stt";
import { OpenAiRealtimeSttClient } from "./openai-stt";
import { SonioxSttClient } from "./soniox-stt";
@@ -14,7 +16,7 @@ export type VoiceControlState = {
export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox";
export type VoiceTranscriptEvent = {
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
* Complete best transcript of the current utterance. REPLACE semantics —
* render as-is, replacing any previous partial. Not a delta.
@@ -80,6 +82,8 @@ export type FrameMetaListener = (meta: MicFrameMeta) => void;
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
private readonly speechPauseListeners = new Set<() => void>();
private readonly speechPause = new SpeechPauseDetector();
private readonly speechEndListeners = new Set<() => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
@@ -132,6 +136,12 @@ export class FaceclawVoiceControlBridge {
return () => this.speechEndListeners.delete(listener);
}
/** Pauses in on-device recognition, for Transcribe's presentation only. */
onSpeechPause(listener: () => void): () => void {
this.speechPauseListeners.add(listener);
return () => this.speechPauseListeners.delete(listener);
}
/** Begin push-to-talk capture (momentary; released with stopPushToTalk). */
startPushToTalk(options: PushToTalkOptions): void {
this.acquireCapture("ptt", options);
@@ -251,6 +261,9 @@ export class FaceclawVoiceControlBridge {
this.controller?.clearSpeakerVerification();
}
this.speechPause.reset();
// A previous push-to-talk commit may still be awaiting its final result.
this.cloudClient?.stop();
const cloudClient = this.createCloudClient(options);
if (cloudClient) {
this.cloudClient = cloudClient;
@@ -274,18 +287,20 @@ export class FaceclawVoiceControlBridge {
if (options.provider === "onboard") return null;
const sttOptions = {
apiKey: "",
onTranscript: (event: { text: string; isFinal: boolean }) =>
this.emitTranscript(event.text, event.isFinal),
onTranscript: (event: CloudSttTranscriptEvent) =>
this.emitTranscript(event.text, event.isFinal, event),
onStatus: (status: string) => this.setStatus(status),
onError: (message: string) => this.setStatus(message),
};
const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) =>
new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous"));
if (options.provider === "elevenlabs") {
const apiKey = options.elevenLabsApiKey.trim();
if (!apiKey) {
this.setStatus("No ElevenLabs key set; using on-device voice.");
return null;
}
return new ElevenLabsSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new ElevenLabsSttClient(config), apiKey);
}
if (options.provider === "soniox") {
const apiKey = options.sonioxApiKey.trim();
@@ -293,14 +308,14 @@ export class FaceclawVoiceControlBridge {
this.setStatus("No Soniox key set; using on-device voice.");
return null;
}
return new SonioxSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new SonioxSttClient(config), apiKey);
}
const apiKey = options.openAiApiKey.trim();
if (!apiKey) {
this.setStatus("No OpenAI key set; using on-device voice.");
return null;
}
return new OpenAiRealtimeSttClient({ ...sttOptions, apiKey });
return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey);
}
private releaseCapture(holder: CaptureHolder, commit: boolean): void {
@@ -432,6 +447,9 @@ export class FaceclawVoiceControlBridge {
onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener();
}
if (this.rawPcmListeners.size > 0) {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
@@ -459,15 +477,16 @@ export class FaceclawVoiceControlBridge {
this.controller.setListener(this.listenerProxy);
}
private emitTranscript(text: string, isFinal: boolean): void {
private emitTranscript(text: string, isFinal: boolean, metadata?: CloudSttTranscriptEvent): void {
// "My voice only": a rejected session's final transcript is emptied so
// nothing downstream (assistant, dictation) acts on another speaker's
// words. The verification verdict is posted before the final, and cloud
// finals arrive over the network later still, so the flag is settled.
if (isFinal && this.verificationActive && this.verificationRejected) {
text = "";
metadata = undefined;
}
const event = { text, isFinal };
const event = { ...metadata, text, isFinal };
for (const listener of this.transcriptListeners) {
listener(event);
}