Voice: self-hosted Whisper server provider; Transcribe on iOS

- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible
  Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests,
  with URL, model, language and key settings (Voice). Pause-ended spans are
  one request each, finalized in audio order, with provisional text while a
  span is open. See notes/whisper-server.md.
- Transcribe (and other continuous capture) works on iOS with a cloud or
  Whisper-server provider; push-to-talk shares a running capture.
- Transcribe saves each session's audio as transcribe-<time>.wav in Downloads
  (setting "Save Transcribe audio", on by default), written as it is captured
  so a crash leaves a playable file; a saved transcript uses the same stamp.
- Calculator stops continuous listening when its window closes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
FilipAlexanderandClaude Opus 5.5 committed 2026-10-17 16:43:18 +02:00
1 parent ecb6c7f465
commit e2caf28392
24 files changed
+1065 -43

No files matched your search

+40 -7
View File
@@ -4,6 +4,7 @@ import { OpenAiRealtimeSttClient } from "./openai-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { SonioxSttClient } from "./soniox-stt";
import type { VoiceProviderKind } from "./voice-control";
import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServerSttClient } from "./whisper-server-stt";
/**
* The transcription-provider setting plus the keys the cloud providers need.
@@ -14,12 +15,22 @@ export type VoiceProviderSelection = {
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
};
type CloudProvider = {
name: string;
apiKey: (selection: VoiceProviderSelection) => string;
create: (options: CloudSttOptions) => CloudSttClient;
/** Whether the provider can run; defaults to having a key. */
configured?: (selection: VoiceProviderSelection) => boolean;
/** What to tell the user when it can't. */
missing?: string;
create: (options: CloudSttOptions, selection: VoiceProviderSelection) => CloudSttClient;
finalTimeoutMs?: number;
};
// Providers absent here ("onboard", "onboard-whisper") transcribe on-device.
@@ -39,11 +50,28 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
apiKey: (selection) => selection.openAiApiKey,
create: (options) => new OpenAiRealtimeSttClient(options),
},
"whisper-server": {
name: "Whisper server",
apiKey: (selection) => selection.whisperServerApiKey ?? "",
configured: (selection) => Boolean(normalizeWhisperServerUrl(selection.whisperServerUrl ?? "")),
missing: "No Whisper server URL set; using on-device voice.",
create: (options, selection) => new WhisperServerSttClient(options, {
baseUrl: selection.whisperServerUrl ?? "",
model: selection.whisperServerModel ?? "",
language: selection.whisperServerLanguage ?? "",
}),
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
},
};
function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection): boolean {
return provider.configured ? provider.configured(selection) : Boolean(provider.apiKey(selection).trim());
}
/** Whether createCloudSttClient would return a cloud client for this selection. */
export function usesCloudStt(selection: VoiceProviderSelection): boolean {
return Boolean(CLOUD_PROVIDERS[selection.provider]?.apiKey(selection).trim());
const provider = CLOUD_PROVIDERS[selection.provider];
return Boolean(provider && isConfigured(provider, selection));
}
/**
@@ -56,13 +84,18 @@ export function createCloudSttClient(
selection: VoiceProviderSelection,
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): CloudSttClient | null {
): ReconnectingSttClient | null {
const provider = CLOUD_PROVIDERS[selection.provider];
if (!provider) return null;
const apiKey = provider.apiKey(selection).trim();
if (!apiKey) {
options.onStatus(`No ${provider.name} key set; using on-device voice.`);
if (!isConfigured(provider, selection)) {
options.onStatus(provider.missing ?? `No ${provider.name} key set; using on-device voice.`);
return null;
}
return new ReconnectingSttClient(provider.create, { ...options, apiKey }, continuous);
const apiKey = provider.apiKey(selection).trim();
return new ReconnectingSttClient(
(clientOptions) => provider.create(clientOptions, selection),
{ ...options, apiKey },
continuous,
provider.finalTimeoutMs,
);
}
+3 -1
View File
@@ -24,6 +24,8 @@ export class ReconnectingSttClient implements CloudSttClient {
private readonly create: (options: CloudSttOptions) => CloudSttClient,
private readonly options: CloudSttOptions,
private readonly continuous: () => boolean,
/** How long finish() waits for the provider's final transcript. */
readonly finalTimeoutMs = 5000,
) {}
start(): void {
@@ -90,7 +92,7 @@ export class ReconnectingSttClient implements CloudSttClient {
this.stop();
return;
}
this.finishTimeout = setTimeout(() => this.stop(), 5000);
this.finishTimeout = setTimeout(() => this.stop(), this.finalTimeoutMs);
if (this.ready) this.client?.finish();
}
+56 -10
View File
@@ -1,6 +1,8 @@
import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
import type { CloudSttClient, CloudSttTranscriptEvent } from './cloud-stt'
import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { WavRecorder } from './wav-recorder.ios'
declare const FaceclawSpeech: any
export type IosMicrophoneSession = {
@@ -12,6 +14,8 @@ const CLOUD_FINAL_TIMEOUT_MS = 5000
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
* the PCM capture runs open-ended, and pauses commit its segments.
*/
export class IosVoiceControlBridge {
private native: any = null
@@ -32,9 +36,14 @@ export class IosVoiceControlBridge {
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
// provider was asked to finalize; it is the status shown when it does.
private cloud: CloudSttClient | null = null
private cloud: ReconnectingSttClient | null = null
private cloudEndStatus: string | null = null
private cloudTimer: ReturnType<typeof setTimeout> | null = null
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
// joins it instead of restarting the mic (transcripts reach every listener).
private continuous = false
// The capture's audio as a WAV in Downloads, while recordTo asked for one.
private recorder: WavRecorder | null = null
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) }
@@ -49,7 +58,11 @@ export class IosVoiceControlBridge {
if (!this.native) {
this.native = FaceclawSpeech.new()
this.native.eventHandler = (json: string) => this.receive(JSON.parse(json))
this.native.pcmHandler = (pcm: NSData) => this.cloud?.acceptPcm(new Uint8Array(interop.bufferFromData(pcm)))
this.native.pcmHandler = (pcm: NSData) => {
const bytes = new Uint8Array(interop.bufferFromData(pcm))
this.cloud?.acceptPcm(bytes)
this.recorder?.append(bytes)
}
}
return this.native
}
@@ -74,9 +87,12 @@ export class IosVoiceControlBridge {
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.stop()
const generation = ++this.generation
this.continuous = continuous
this.startRecording(recordTo, log)
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const startError = this.startNative(voice, endpointing, false)
@@ -101,8 +117,11 @@ export class IosVoiceControlBridge {
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false): Promise<void> {
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.stop()
this.continuous = continuous
this.startRecording(recordTo, log)
this.log = log; this.source = 'phone'; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = this.startNative(voice, endpointing, true)
@@ -118,7 +137,7 @@ export class IosVoiceControlBridge {
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
onStatus: status => { if (current()) this.setStatus(status) },
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}, () => false)
}, () => this.continuous)
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
if (error) return error
@@ -130,7 +149,26 @@ export class IosVoiceControlBridge {
if (this.source !== 'phone') return
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
}
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
/** End continuous capture, transcribing the audio since the last pause. */
stopContinuousCapture(): Promise<void> {
if (!this.continuous) return Promise.resolve()
return this.finalizeCapture()
}
/** Push-to-talk shares a listening continuous capture (not one that is only finalizing). */
private joinsContinuous(continuous: boolean): boolean {
if (continuous || !this.continuous || !this.capturing) return false
this.log('Voice: push-to-talk joins the running continuous capture')
return true
}
stopPushToTalk(): Promise<void> {
// The continuous capture owns the mic and keeps it.
if (this.continuous) return Promise.resolve()
return this.finalizeCapture()
}
private finalizeCapture(): Promise<void> {
const completion = this.completion
if (!this.capturing) return completion
this.capturing = false; this.finalizing = true
@@ -140,7 +178,7 @@ export class IosVoiceControlBridge {
}
stop(): void {
++this.generation
this.capturing = this.finalizing = false
this.capturing = this.finalizing = this.continuous = false
this.source = null
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
this.clearCloud()
@@ -155,7 +193,14 @@ export class IosVoiceControlBridge {
const session = this.session; this.session = null
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
/** Record this capture's audio as Downloads/<name>.wav (only cloud captures supply PCM). */
private startRecording(name: string, log: (message: string) => void): void {
if (!name) return
this.recorder = WavRecorder.open(name)
log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
}
private clearCloud(): void {
this.recorder?.close(); this.recorder = null
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
this.cloudTimer = null
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
@@ -166,13 +211,13 @@ export class IosVoiceControlBridge {
this.cloudEndStatus = status
this.setStatus('Recognizing…')
// Armed first: finish() can deliver the final synchronously.
this.cloudTimer = setTimeout(() => this.completeCloud(), CLOUD_FINAL_TIMEOUT_MS)
this.cloudTimer = setTimeout(() => this.completeCloud(), this.cloud?.finalTimeoutMs ?? CLOUD_FINAL_TIMEOUT_MS)
this.cloud?.finish()
}
private completeCloud(): void {
const status = this.cloudEndStatus ?? 'Ready to send'
this.clearCloud()
this.capturing = this.finalizing = false
this.capturing = this.finalizing = this.continuous = false
this.finishCompletion()
this.setStatus(status); this.emitEnd()
}
@@ -180,7 +225,8 @@ export class IosVoiceControlBridge {
if (!this.capturing && !this.finalizing) return
for (const fn of [...this.transcripts]) fn(event)
// Push-to-talk commits once, so the first final after it is the last.
if (event.isFinal && this.cloudEndStatus !== null) this.completeCloud()
// In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive.
if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud()
}
private emitEnd(): void { for (const fn of [...this.ends]) fn() }
private finishCompletion(): void {
+7 -1
View File
@@ -28,7 +28,9 @@ export type VoiceControlState = {
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
// renamed, since it's a persisted settings value on real installs already.
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP
// (whisper-server-stt.ts).
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
/**
@@ -52,6 +54,10 @@ export type PushToTalkOptions = {
elevenLabsApiKey: string;
openAiApiKey: string;
sonioxApiKey: string;
whisperServerUrl?: string;
whisperServerModel?: string;
whisperServerLanguage?: string;
whisperServerApiKey?: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
+22
View File
@@ -0,0 +1,22 @@
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
export function wavHeader(dataBytes: number): Uint8Array {
const header = new Uint8Array(44);
const view = new DataView(header.buffer);
const ascii = (offset: number, text: string) => {
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
};
ascii(0, "RIFF");
view.setUint32(4, 36 + dataBytes, true);
ascii(8, "WAVE");
ascii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, 16000, true);
view.setUint32(28, 32000, true); // byte rate
view.setUint16(32, 2, true); // block align
view.setUint16(34, 16, true); // bits per sample
ascii(36, "data");
view.setUint32(40, dataBytes, true);
return header;
}
+75
View File
@@ -0,0 +1,75 @@
import { knownFolders, path as paths } from '@nativescript/core'
import { wavHeader } from './wav-header'
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
const FLUSH_BYTES = 32000
/**
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private pending: Uint8Array[] = []
private pendingBytes = 0
private dataBytes = 0
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
const fm = NSFileManager.defaultManager
const dir = paths.join(knownFolders.documents().path, 'Downloads')
try {
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
let path = paths.join(dir, `${name}.wav`)
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
return new WavRecorder(handle, path)
} catch {
return null
}
}
append(pcm: Uint8Array): void {
if (!this.handle || !pcm.length) return
this.pending.push(pcm.slice())
this.pendingBytes += pcm.length
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
}
close(): void {
this.flush()
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
this.handle = null
}
private flush(): void {
const handle = this.handle
if (!handle || !this.pendingBytes) return
const chunk = new Uint8Array(this.pendingBytes)
let offset = 0
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
this.pending = []
this.pendingBytes = 0
try {
// NativeScript throws a failed call's NSError (storage full, file removed).
handle.writeDataError(toData(chunk))
handle.seekToOffsetError(0)
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
handle.seekToEndReturningOffsetError(null)
this.dataBytes += chunk.length
return
} catch {
// Stop recording rather than retry.
}
try { handle.closeAndReturnError() } catch { /* ignore */ }
this.handle = null
}
}
function toData(bytes: Uint8Array): NSData {
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
}
+386
View File
@@ -0,0 +1,386 @@
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
/**
* Speech-to-text against a self-hosted OpenAI-compatible Whisper server (vLLM,
* speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file.
* Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause)
* is one request, finalized in audio order; an open segment is re-sent every
* INTERIM_STEP_BYTES for provisional text. start() lists {baseUrl}/models to check
* reachability and pick a model when none is configured.
*/
export type WhisperServerConfig = {
/** OpenAI-style API root, e.g. http://host:8004/v1. */
baseUrl: string;
/** Model id to request; empty uses the first one the server lists. */
model: string;
/** ISO-639-1 language code; empty lets Whisper detect it. */
language: string;
};
export type WhisperHttpRequest = {
url: string;
method: "GET" | "POST";
headers: Record<string, string>;
body?: ArrayBuffer;
timeoutMs: number;
};
export type WhisperHttpResponse = { status: number; text: string };
export type WhisperHttp = (request: WhisperHttpRequest) => Promise<WhisperHttpResponse>;
const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2;
// Provisional transcripts: first after one second of audio, then every 1.5s,
// and not for segments past Whisper's 30s window (each would re-send it all).
const INTERIM_MIN_BYTES = BYTES_PER_SECOND;
const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5;
const INTERIM_MAX_BYTES = BYTES_PER_SECOND * 30;
const MODELS_TIMEOUT_MS = 5000;
const INTERIM_TIMEOUT_MS = 10000;
const FINAL_TIMEOUT_MS = 12000;
const FINAL_RETRY_DELAY_MS = 750;
// Whisper invents text ("Thank you.") for silence, so a segment whose loudest
// 50ms frame stays below this RMS is not sent at all. Normal speech on the G2
// mic measures in the thousands; the pause detector's noise floor starts at 220.
const SILENCE_RMS = 300;
/** Time the voice bridges should allow a final transcript before giving up. */
export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETRY_DELAY_MS + 2000;
type Segment = {
chunks: Uint8Array[];
bytes: number;
peakRms: number;
text: string;
final: boolean;
closed: boolean;
paragraphBreakAfter: boolean;
/** Emit a final even when empty: the end of push-to-talk needs one. */
required: boolean;
interimInFlight: boolean;
interimBytes: number;
};
/**
* The API root from what a user typed: tolerates a trailing slash or the full
* transcription endpoint, and adds /v1 to a bare host. Empty when unusable.
*/
export function normalizeWhisperServerUrl(input: string): string {
let url = input.trim().replace(/\/+$/, "");
if (!/^https?:\/\/[^/\s]+/i.test(url)) return "";
url = url.replace(/\/audio\/transcriptions$/i, "");
if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1";
return url;
}
export class WhisperServerSttClient implements CloudSttClient {
private readonly baseUrl: string;
private model: string;
private ready = false;
private closed = false;
private readonly segments: Segment[] = [];
constructor(
private readonly options: CloudSttOptions,
private readonly config: WhisperServerConfig,
private readonly http: WhisperHttp = nativeHttp,
) {
this.baseUrl = normalizeWhisperServerUrl(config.baseUrl);
this.model = config.model.trim();
}
start(): void {
if (this.closed || this.ready) return;
if (!this.baseUrl) {
this.options.onError("Whisper server URL is not a valid http(s) address.");
return;
}
this.options.onStatus("Connecting to Whisper server...");
this.http({ url: `${this.baseUrl}/models`, method: "GET", headers: this.authHeaders(), timeoutMs: MODELS_TIMEOUT_MS })
.then((response) => {
if (this.closed) return;
if (response.status < 200 || response.status >= 300) {
this.options.onDisconnected?.(`Whisper server returned ${response.status} for /models.`);
return;
}
if (!this.model) this.model = firstModelId(response.text);
if (!this.model) {
this.options.onError("Whisper server lists no models; set the model in Settings > Voice.");
return;
}
this.ready = true;
this.options.onStatus("Listening (Whisper server)...");
this.options.onReady?.();
})
.catch((error) => {
if (!this.closed) this.options.onDisconnected?.(`Whisper server unreachable: ${describeError(error)}`);
});
}
/** PCM16LE at 16 kHz, appended to the open segment. */
acceptPcm(pcm: Uint8Array): void {
if (this.closed || !pcm.length) return;
const segment = this.openSegment();
segment.chunks.push(pcm.slice());
segment.bytes += pcm.length;
segment.peakRms = Math.max(segment.peakRms, rms(pcm));
this.maybeRequestInterim(segment);
}
/** A pause in continuous capture: transcribe what came before it. */
commitSegment(): void {
this.closeSegment(true, false);
}
/** End of the utterance: transcribe the rest and deliver a final. */
finish(): void {
this.closeSegment(false, true);
}
stop(): void {
this.closed = true;
this.segments.length = 0;
}
private openSegment(): Segment {
const last = this.segments[this.segments.length - 1];
if (last && !last.closed) return last;
const segment: Segment = {
chunks: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
paragraphBreakAfter: false, required: false, interimInFlight: false, interimBytes: 0,
};
this.segments.push(segment);
return segment;
}
private closeSegment(paragraphBreakAfter: boolean, required: boolean): void {
if (this.closed) return;
const last = this.segments[this.segments.length - 1];
if (!last || last.closed) {
// Nothing new since the last boundary. Push-to-talk still needs a final.
if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required);
return;
}
if (last.peakRms < SILENCE_RMS) {
this.finalize(last, "", paragraphBreakAfter, required);
return;
}
last.closed = true;
last.paragraphBreakAfter = paragraphBreakAfter;
last.required = required;
void this.requestFinal(last, 0);
}
private async requestFinal(segment: Segment, attempt: number): Promise<void> {
try {
const text = await this.transcribe(segment, segment.bytes, FINAL_TIMEOUT_MS);
if (!this.closed) this.finalize(segment, text, segment.paragraphBreakAfter, segment.required);
} catch (error) {
if (this.closed) return;
if (error instanceof WhisperAuthError) {
this.options.onError(error.message);
return;
}
if (attempt === 0) {
setTimeout(() => { void this.requestFinal(segment, 1); }, FINAL_RETRY_DELAY_MS);
return;
}
// Keep the capture alive and keep whatever the interim pass heard.
this.options.onStatus(`Whisper server request failed: ${describeError(error)}`);
this.finalize(segment, segment.text, segment.paragraphBreakAfter, segment.required);
}
}
private maybeRequestInterim(segment: Segment): void {
if (!this.ready || segment.interimInFlight || segment.closed) return;
if (segment.bytes < INTERIM_MIN_BYTES || segment.bytes > INTERIM_MAX_BYTES) return;
if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return;
if (segment.peakRms < SILENCE_RMS) return;
segment.interimInFlight = true;
segment.interimBytes = segment.bytes;
this.transcribe(segment, segment.bytes, INTERIM_TIMEOUT_MS)
.then((text) => {
if (this.closed || segment.final) return;
segment.text = text;
this.emit();
})
.catch(() => {
// Provisional only; the final request reports real failures.
})
.finally(() => {
segment.interimInFlight = false;
});
}
private finalize(segment: Segment, text: string, paragraphBreakAfter: boolean, required: boolean): void {
segment.closed = segment.final = true;
segment.text = text;
segment.paragraphBreakAfter = paragraphBreakAfter;
segment.required = required;
segment.chunks = [];
this.emit();
}
/**
* Finals leave in audio order, even when a later request answers first;
* whatever is not final yet is shown as one provisional transcript.
*/
private emit(): void {
while (this.segments.length && this.segments[0]!.final) {
const segment = this.segments.shift()!;
if (!segment.text && !segment.required) continue;
this.options.onTranscript({ text: segment.text, isFinal: true, paragraphBreakAfter: segment.paragraphBreakAfter });
}
const pending = this.segments.filter((segment) => segment.text.trim());
if (!pending.length) return;
const text = pending.map((segment) => segment.text.trim()).join(" ");
const transcribeText = pending.map((segment, index) =>
(index && pending[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + segment.text.trim(),
).join("");
this.options.onTranscript({ text, transcribeText, isFinal: false });
}
private async transcribe(segment: Segment, bytes: number, timeoutMs: number): Promise<string> {
const fields: Record<string, string> = { model: this.model, response_format: "json", temperature: "0" };
const language = this.config.language.trim().toLowerCase();
if (language) fields.language = language;
const boundary = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
const body = multipartBody(boundary, fields, wavFile(segment.chunks, bytes));
const response = await this.http({
url: `${this.baseUrl}/audio/transcriptions`,
method: "POST",
headers: { ...this.authHeaders(), "Content-Type": `multipart/form-data; boundary=${boundary}` },
body,
timeoutMs,
});
if (response.status === 401 || response.status === 403) {
throw new WhisperAuthError(`Whisper server rejected the key (${response.status}).`);
}
if (response.status < 200 || response.status >= 300) {
throw new Error(`HTTP ${response.status}: ${response.text.slice(0, 200)}`);
}
return parseTranscript(response.text);
}
private authHeaders(): Record<string, string> {
const key = this.options.apiKey.trim();
return key ? { Authorization: `Bearer ${key}` } : {};
}
}
class WhisperAuthError extends Error {}
/** The transcript from a JSON response; tolerates servers that send plain text. */
export function parseTranscript(body: string): string {
const raw = body.trim();
try {
const parsed = JSON.parse(raw);
if (parsed && typeof parsed.text === "string") return parsed.text.trim();
} catch {
// Not JSON: a server that answered response_format=text anyway.
}
return raw;
}
function firstModelId(body: string): string {
try {
const parsed = JSON.parse(body);
const id = parsed?.data?.[0]?.id;
return typeof id === "string" ? id : "";
} catch {
return "";
}
}
/** RMS of a PCM16LE chunk. */
function rms(pcm: Uint8Array): number {
const count = pcm.length >> 1;
if (!count) return 0;
let squares = 0;
for (let i = 0; i < count; i++) {
const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned;
squares += sample * sample;
}
return Math.sqrt(squares / count);
}
/** The first `bytes` of the chunks as a 16 kHz mono 16-bit WAV file. */
export function wavFile(chunks: Uint8Array[], bytes: number): Uint8Array {
const length = bytes & ~1;
const out = new Uint8Array(44 + length);
const view = new DataView(out.buffer);
const ascii = (offset: number, text: string) => {
for (let i = 0; i < text.length; i++) out[offset + i] = text.charCodeAt(i);
};
ascii(0, "RIFF");
view.setUint32(4, 36 + length, true);
ascii(8, "WAVE");
ascii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, CLOUD_STT_SAMPLE_RATE, true);
view.setUint32(28, BYTES_PER_SECOND, true);
view.setUint16(32, 2, true);
view.setUint16(34, 16, true);
ascii(36, "data");
view.setUint32(40, length, true);
let offset = 44;
for (const chunk of chunks) {
const take = Math.min(chunk.length, out.length - offset);
if (take <= 0) break;
out.set(chunk.subarray(0, take), offset);
offset += take;
}
return out;
}
/** multipart/form-data with text fields and one WAV part named "file". */
export function multipartBody(boundary: string, fields: Record<string, string>, wav: Uint8Array): ArrayBuffer {
let head = "";
for (const [name, value] of Object.entries(fields)) {
head += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`;
}
head += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`;
const headBytes = utf8(head);
const tailBytes = utf8(`\r\n--${boundary}--\r\n`);
const out = new Uint8Array(headBytes.length + wav.length + tailBytes.length);
out.set(headBytes, 0);
out.set(wav, headBytes.length);
out.set(tailBytes, headBytes.length + wav.length);
return out.buffer;
}
function utf8(text: string): Uint8Array {
const out: number[] = [];
for (const char of text) {
const code = char.codePointAt(0)!;
if (code < 0x80) out.push(code);
else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 63));
else if (code < 0x10000) out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
else out.push(0xf0 | (code >> 18), 0x80 | ((code >> 12) & 63), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
}
return Uint8Array.from(out);
}
function describeError(error: unknown): string {
return String((error as Error)?.message ?? error);
}
/** NativeScript's HTTP module: sends an ArrayBuffer body as-is on both platforms. */
const nativeHttp: WhisperHttp = async (request) => {
// Required lazily so Node tests can load this module without NativeScript.
const { withUserAgent } = require("../util/http") as typeof import("../util/http");
const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http");
const response = await send({
url: request.url,
method: request.method,
headers: withUserAgent(request.headers),
content: request.body,
timeout: request.timeoutMs,
});
const content = response.content as any;
let text = "";
if (content) text = global.isAndroid ? String(content.raw.toString("UTF-8")) : String(content.toString());
return { status: response.statusCode, text };
};