Bugfixes in iOS Transcribe and Whisper server integration

This commit is contained in:
James Babcock committed 2026-10-17 13:43:52 -07:00
1 parent 33eef6538c
commit 5bf809d733
12 files changed
+553 -73

No files matched your search

+91 -20
View File
@@ -148,7 +148,16 @@ export class IosPreviewController {
private acknowledgedFrames = 0
private logLines: string[] = []
private logTimer: ReturnType<typeof setTimeout> | null = null
// Windows that want continuous capture, each with the recording it asked for
// ('' for none). They share one capture, which runs while any of them wants it.
private readonly continuousClients = new Map<string, string>()
// Continuous capture should (re)start once audio is available: a window
// asked, the glasses connected or disconnected, or the glasses lock or the
// app lifecycle cut it short. A capture that fails doesn't set it, so a
// broken start is retried on the next such change rather than in a loop.
private continuousPending = false
private continuousCaptureToken = 0
private sessionConnected = false
private readonly actions: LayerActions = {
...noopLayerActions,
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
@@ -156,8 +165,8 @@ export class IosPreviewController {
disconnect: () => this.disconnect(),
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
startContinuousVoiceCapture: options => this.startContinuousVoiceCapture(options?.recordAs),
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options?.recordAs),
stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client),
startTextSettingEdit: async setting => { await this.editSetting(setting) },
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
@@ -244,6 +253,8 @@ export class IosPreviewController {
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
registerNavigateTools(id => this.launchApp(id))
registerRoamTools(id => this.launchApp(id))
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
}
@@ -299,10 +310,12 @@ export class IosPreviewController {
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
this.syncEvenHubPhoneUi()
this.syncContinuousCapture()
}
pause(): void {
if (!this.active) return
this.active = false
if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true
iosVoiceInput.stopPhoneCapture()
// The phone preview is hidden; the glasses still own their screen and input.
this.syncRuntime()
@@ -421,8 +434,10 @@ export class IosPreviewController {
this.glassesLocked = locked
if (locked) {
this.glance.dismiss()
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.')
}
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
'Glasses locked. Transcription resumes when you unlock your phone.')
} else this.syncContinuousCapture()
void (async () => {
await this.displayReady
const display = this.display
@@ -720,6 +735,7 @@ export class IosPreviewController {
this.maybeLaunchOnboarding(this.state)
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
this.syncContinuousSession()
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
this.syncPresence()
this.syncRuntime()
@@ -947,33 +963,88 @@ export class IosPreviewController {
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
}
}
/** Open-ended capture (Transcribe, Teleprompter); holds voiceActivity like Android. */
private async startContinuousVoiceCapture(recordAs?: string): Promise<void> {
if (this.glassesLocked) return
const voice = this.voiceProvider()
if (!iosVoiceInput.supportsContinuous(voice)) {
/**
* Continuous capture on behalf of `client` (one id per window: Transcribe,
* Teleprompter, Calculator); it runs while any client wants it, as on Android.
* `recordAs` names a WAV of the audio in Downloads when Save Transcribe audio is on.
*/
private startContinuousVoiceCapture(client: string, recordAs = ''): void {
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
return
}
this.continuousClients.set(client, recordAs)
this.syncContinuousRecording()
// A listening capture is shared as it is; otherwise start one (after any drain).
if (!iosVoiceInput.continuousSource) this.continuousPending = true
this.syncContinuousCapture()
}
private stopContinuousVoiceCapture(client: string): void {
if (!this.continuousClients.delete(client)) return
this.syncContinuousRecording()
if (this.continuousClients.size > 0) return
// A start still awaiting permission or the mic backs off.
++this.continuousCaptureToken
this.continuousPending = false
this.syncContinuousClaim()
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
}
/** Connecting or losing the glasses moves continuous capture to the new audio source. */
private syncContinuousSession(): void {
const connected = this.state.phase === 'connected'
if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true
this.sessionConnected = connected
this.syncContinuousCapture()
}
/**
* Keep continuous capture running while a window wants it: start a pending
* one once audio is available, and move a phone capture to the glasses once
* they connect. A glasses session supplies the audio, even while it
* reconnects (as on Android); the phone mic stands in only without one, while
* Faceclaw is in the foreground. Cheap enough to call on every state change.
*/
private syncContinuousCapture(): void {
this.syncContinuousClaim()
if (this.continuousClients.size === 0 || this.glassesLocked) return
const glassesReady = !!this.communicator && this.state.phase === 'connected'
if (iosVoiceInput.continuousActive) {
// Draining or listening already. A phone capture from before the glasses
// connected is drained here, and its end restarts it on the glasses.
if (glassesReady && iosVoiceInput.continuousSource === 'phone') {
this.continuousPending = true
void iosVoiceInput.stopContinuousCapture()
}
return
}
if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return
this.continuousPending = false
void this.beginContinuousCapture()
}
private async beginContinuousCapture(): Promise<void> {
const voice = this.voiceProvider()
if (!iosVoiceInput.supportsContinuous(voice)) return
// A stop during an await below bumps the token; this start then backs off.
const token = ++this.continuousCaptureToken
const current = () => token === this.continuousCaptureToken
const log = (message: string) => this.logBluetooth(message)
const record = recordAs && saveTranscribeAudioSetting.get() ? recordAs : ''
// Claimed before the capture starts so other mic users stand down first, as on Android.
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
const communicator = this.communicator
if (communicator && this.state.phase === 'connected') {
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true, record)
} else if (this.active && await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked) {
await iosVoiceInput.startPhoneCapture(voice, log, false, true, record)
if (communicator) {
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true)
} else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) {
await iosVoiceInput.startPhoneCapture(voice, log, false, true)
}
if (!current()) return
if (!iosVoiceInput.continuousActive) voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
if (current()) this.syncContinuousClaim()
}
private stopContinuousVoiceCapture(): void {
++this.continuousCaptureToken
voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
private syncContinuousClaim(): void {
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
}
/** Record to the first client's requested name while Save Transcribe audio is on. */
private syncContinuousRecording(): void {
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].find(Boolean) ?? '' : ''
iosVoiceInput.recordTo(name)
}
/** The transcription provider setting and the keys the cloud ones need. */
private voiceProvider(): VoiceProviderSelection {
+7 -1
View File
@@ -1,6 +1,6 @@
import { Http } from '@nativescript/core';
import { withUserAgent } from '../util/http';
import type { BinaryResponse } from './binary-http';
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
@@ -8,3 +8,9 @@ export async function downloadBinary(url: string, timeoutMs: number): Promise<Bi
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
}
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
export async function postBinary(url: string, body: ArrayBuffer, headers: Record<string, string>, timeoutMs: number): Promise<BinaryPostResponse> {
const response = await Http.request({ url, method: 'POST', headers: withUserAgent(headers), content: body, timeout: timeoutMs });
return { status: response.statusCode, text: response.content ? String(response.content.toString()) : '' };
}
+23
View File
@@ -2,6 +2,7 @@ declare const com: any;
declare const java: any;
export type BinaryResponse = { status: number; bytes: Uint8Array; contentLength: number };
export type BinaryPostResponse = { status: number; text: string };
let nextDownloadSerial = 0;
@@ -35,3 +36,25 @@ export function downloadBinary(url: string, timeoutMs: number): Promise<BinaryRe
new com.faceclaw.app.FaceclawFileDownload(url, path, timeoutMs, listener).start();
});
}
/**
* POST a binary body and read the response as text. Not NativeScript's http
* module: it hands an ArrayBuffer body to Java one bridge call per byte
* (ByteBuffer.wrap(Array.from(...))), which for a ~1MB WAV upload stalls the
* main thread. FaceclawBinaryPost takes it as one direct-buffer copy instead.
*/
export function postBinary(url: string, body: ArrayBuffer, headers: Record<string, string>, timeoutMs: number): Promise<BinaryPostResponse> {
// Required lazily so Node tests can load this module's importers.
const { copyToJavaByteBuffer } = require("./java-direct-buffer") as typeof import("./java-direct-buffer");
const entries = Object.entries(headers);
const nativeHeaders = Array.create("java.lang.String", entries.length * 2);
entries.forEach(([name, value], index) => { nativeHeaders[index * 2] = name; nativeHeaders[index * 2 + 1] = value; });
return new Promise((resolve, reject) => {
const listener = new com.faceclaw.app.FaceclawBinaryPostListener({
onResponse: (status: number, text: string | null) => resolve({ status, text: String(text ?? "") }),
onFailure: (message: string | null) => reject(new Error(String(message ?? "Request failed."))),
});
// The User-Agent comes from the okhttp client's interceptor.
new com.faceclaw.app.FaceclawBinaryPost(url, copyToJavaByteBuffer(new Uint8Array(body)), nativeHeaders, timeoutMs, listener).start();
});
}
+48 -20
View File
@@ -15,7 +15,9 @@ const CLOUD_FINAL_TIMEOUT_MS = 5000
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
* the PCM capture runs open-ended, and pauses commit its segments.
* the PCM capture runs open-ended, and pauses commit its segments. When the
* session or the app lifecycle cuts one short it is drained, not discarded;
* restarting it is the controller's call.
*/
export class IosVoiceControlBridge {
private native: any = null
@@ -42,7 +44,12 @@ export class IosVoiceControlBridge {
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
// joins it instead of restarting the mic (transcripts reach every listener).
private continuous = false
// The capture's audio as a WAV in Downloads, while recordTo asked for one.
// Shown instead of the native "ended" status once a drain the bridge started completes.
private drainStatus: string | null = null
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = ''
private recorder: WavRecorder | null = null
onStatus(listener: (state: VoiceControlState) => void): () => void {
@@ -61,7 +68,7 @@ export class IosVoiceControlBridge {
this.native.pcmHandler = (pcm: NSData) => {
const bytes = new Uint8Array(interop.bufferFromData(pcm))
this.cloud?.acceptPcm(bytes)
this.recorder?.append(bytes)
if (this.continuous) this.record(bytes)
}
}
return this.native
@@ -87,12 +94,11 @@ export class IosVoiceControlBridge {
return true
}
get statusText(): string { return this.status }
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.stop()
const generation = ++this.generation
this.continuous = continuous
this.startRecording(recordTo, log)
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const startError = this.startNative(voice, endpointing, false)
@@ -117,11 +123,10 @@ export class IosVoiceControlBridge {
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
}
}
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
if (this.joinsContinuous(continuous)) return
this.stop()
this.continuous = continuous
this.startRecording(recordTo, log)
this.log = log; this.source = 'phone'; this.capturing = true
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = this.startNative(voice, endpointing, true)
@@ -147,11 +152,32 @@ export class IosVoiceControlBridge {
}
stopPhoneCapture(): void {
if (this.source !== 'phone') return
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
}
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* until called with another name or ''. Only cloud captures supply PCM.
*/
recordTo(name: string): void {
if (name === this.recordingName) return
this.recorder?.close(); this.recorder = null
this.recordingName = name
}
private record(pcm: Uint8Array): void {
if (!this.recorder && this.recordingName) {
this.recorder = WavRecorder.open(this.recordingName)
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
// One attempt per recording, not one per packet.
if (!this.recorder) this.recordingName = ''
}
this.recorder?.append(pcm)
}
/** End continuous capture, transcribing the audio since the last pause. */
stopContinuousCapture(): Promise<void> {
if (!this.continuous) return Promise.resolve()
@@ -168,10 +194,11 @@ export class IosVoiceControlBridge {
if (this.continuous) return Promise.resolve()
return this.finalizeCapture()
}
private finalizeCapture(): Promise<void> {
private finalizeCapture(status: string | null = null): Promise<void> {
const completion = this.completion
if (!this.capturing) return completion
this.capturing = false; this.finalizing = true
this.drainStatus = status
this.clearAudioTimer(); this.releaseMicrophone()
this.setStatus('Recognizing…'); this.native?.finish()
return completion
@@ -179,13 +206,20 @@ export class IosVoiceControlBridge {
stop(): void {
++this.generation
this.capturing = this.finalizing = this.continuous = false
this.source = null
this.source = null; this.drainStatus = null
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
this.clearCloud()
this.finishCompletion()
}
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
/**
* The glasses session went away (or the glasses locked). Push-to-talk ends
* with `message`; continuous capture transcribes what it already heard and
* shows `continuousMessage`.
*/
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.',
continuousMessage = 'Glasses disconnected. Transcription resumes when they reconnect.'): void {
if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
this.stop(); this.setStatus(message); this.emitEnd()
}
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
@@ -193,14 +227,7 @@ export class IosVoiceControlBridge {
const session = this.session; this.session = null
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
}
/** Record this capture's audio as Downloads/<name>.wav (only cloud captures supply PCM). */
private startRecording(name: string, log: (message: string) => void): void {
if (!name) return
this.recorder = WavRecorder.open(name)
log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
}
private clearCloud(): void {
this.recorder?.close(); this.recorder = null
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
this.cloudTimer = null
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
@@ -245,12 +272,13 @@ export class IosVoiceControlBridge {
} else if (event.kind === 'finishing') {
void this.stopPushToTalk(); this.emitEnd()
} else if (event.kind === 'ended') {
this.source = null
const status = this.drainStatus ?? event.message ?? 'Ready to send'
this.source = null; this.drainStatus = null
this.clearAudioTimer(); this.releaseMicrophone()
if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return }
if (this.cloud) { this.finishCloud(status); return }
this.capturing = this.finalizing = false
this.finishCompletion()
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
this.setStatus(status); this.emitEnd()
}
}
}
+88 -18
View File
@@ -5,8 +5,10 @@ import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from
* speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file.
* Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause)
* is one request, finalized in audio order; an open segment is re-sent every
* INTERIM_STEP_BYTES for provisional text. start() lists {baseUrl}/models to check
* reachability and pick a model when none is configured.
* INTERIM_STEP_BYTES for provisional text. A segment that runs SEGMENT_MAX_BYTES
* without a pause is cut into requests that still leave as one final. start()
* lists {baseUrl}/models to check reachability and pick a model when none is
* configured.
*/
export type WhisperServerConfig = {
@@ -29,11 +31,15 @@ export type WhisperHttpResponse = { status: number; text: string };
export type WhisperHttp = (request: WhisperHttpRequest) => Promise<WhisperHttpResponse>;
const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2;
// Provisional transcripts: first after one second of audio, then every 1.5s,
// and not for segments past Whisper's 30s window (each would re-send it all).
// Provisional transcripts: first after one second of audio, then every 1.5s.
const INTERIM_MIN_BYTES = BYTES_PER_SECOND;
const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5;
const INTERIM_MAX_BYTES = BYTES_PER_SECOND * 30;
// Steady speech or background noise can keep the pause detector from ever
// ending a span. Such a segment is cut at this length, inside Whisper's 30s
// window and well within FINAL_TIMEOUT_MS, after its quietest chunk among the
// last SPLIT_SEARCH_BYTES so a word is less likely to straddle the cut.
const SEGMENT_MAX_BYTES = BYTES_PER_SECOND * 25;
const SPLIT_SEARCH_BYTES = BYTES_PER_SECOND * 5;
const MODELS_TIMEOUT_MS = 5000;
const INTERIM_TIMEOUT_MS = 10000;
const FINAL_TIMEOUT_MS = 12000;
@@ -48,6 +54,8 @@ export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETR
type Segment = {
chunks: Uint8Array[];
/** RMS of each chunk. */
levels: number[];
bytes: number;
peakRms: number;
text: string;
@@ -56,6 +64,8 @@ type Segment = {
paragraphBreakAfter: boolean;
/** Emit a final even when empty: the end of push-to-talk needs one. */
required: boolean;
/** Cut for length: its final leaves joined with the next segment's. */
continues: boolean;
interimInFlight: boolean;
interimBytes: number;
};
@@ -120,10 +130,13 @@ export class WhisperServerSttClient implements CloudSttClient {
acceptPcm(pcm: Uint8Array): void {
if (this.closed || !pcm.length) return;
const segment = this.openSegment();
const level = rms(pcm);
segment.chunks.push(pcm.slice());
segment.levels.push(level);
segment.bytes += pcm.length;
segment.peakRms = Math.max(segment.peakRms, rms(pcm));
this.maybeRequestInterim(segment);
segment.peakRms = Math.max(segment.peakRms, level);
if (segment.bytes >= SEGMENT_MAX_BYTES) this.splitSegment(segment);
this.maybeRequestInterim(this.segments[this.segments.length - 1]!);
}
/** A pause in continuous capture: transcribe what came before it. */
@@ -145,16 +158,54 @@ export class WhisperServerSttClient implements CloudSttClient {
const last = this.segments[this.segments.length - 1];
if (last && !last.closed) return last;
const segment: Segment = {
chunks: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
paragraphBreakAfter: false, required: false, interimInFlight: false, interimBytes: 0,
chunks: [], levels: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
paragraphBreakAfter: false, required: false, continues: false, interimInFlight: false, interimBytes: 0,
};
this.segments.push(segment);
return segment;
}
/**
* Close the open segment at SEGMENT_MAX_BYTES and carry the audio after its
* cut into a new one. The cut is after the quietest chunk among the last
* SPLIT_SEARCH_BYTES; the first chunk always stays.
*/
private splitSegment(segment: Segment): void {
let cut = segment.chunks.length - 1;
let quietest = Infinity;
let carried = 0;
for (let i = segment.chunks.length - 1; i > 0 && carried <= SPLIT_SEARCH_BYTES; i--) {
if (segment.levels[i]! < quietest) {
quietest = segment.levels[i]!;
cut = i;
}
carried += segment.chunks[i]!.length;
}
const chunks = segment.chunks.splice(cut + 1);
const levels = segment.levels.splice(cut + 1);
for (const chunk of chunks) segment.bytes -= chunk.length;
segment.peakRms = peak(segment.levels);
segment.continues = true;
this.closeSegment(false, false);
if (!chunks.length) return;
const next = this.openSegment();
next.chunks = chunks;
next.levels = levels;
for (const chunk of chunks) next.bytes += chunk.length;
next.peakRms = peak(levels);
}
private closeSegment(paragraphBreakAfter: boolean, required: boolean): void {
if (this.closed) return;
const last = this.segments[this.segments.length - 1];
if (last?.closed && last.continues) {
// Cut for length with nothing after it yet: this boundary ends its run.
last.continues = false;
last.paragraphBreakAfter = paragraphBreakAfter;
last.required = required;
if (last.final) this.emit();
return;
}
if (!last || last.closed) {
// Nothing new since the last boundary. Push-to-talk still needs a final.
if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required);
@@ -192,7 +243,7 @@ export class WhisperServerSttClient implements CloudSttClient {
private maybeRequestInterim(segment: Segment): void {
if (!this.ready || segment.interimInFlight || segment.closed) return;
if (segment.bytes < INTERIM_MIN_BYTES || segment.bytes > INTERIM_MAX_BYTES) return;
if (segment.bytes < INTERIM_MIN_BYTES) return;
if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return;
if (segment.peakRms < SILENCE_RMS) return;
segment.interimInFlight = true;
@@ -221,14 +272,20 @@ export class WhisperServerSttClient implements CloudSttClient {
}
/**
* Finals leave in audio order, even when a later request answers first;
* whatever is not final yet is shown as one provisional transcript.
* Finals leave in audio order, even when a later request answers first, and
* segments cut for length leave as one final with the segment that ends
* their run; whatever is not final yet is shown as one provisional transcript.
*/
private emit(): void {
while (this.segments.length && this.segments[0]!.final) {
const segment = this.segments.shift()!;
if (!segment.text && !segment.required) continue;
this.options.onTranscript({ text: segment.text, isFinal: true, paragraphBreakAfter: segment.paragraphBreakAfter });
for (;;) {
let end = 0;
while (this.segments[end]?.final && this.segments[end]!.continues) end++;
if (!this.segments[end]?.final) break;
const run = this.segments.splice(0, end + 1);
const last = run[run.length - 1]!;
const text = run.map((segment) => segment.text.trim()).filter(Boolean).join(" ");
if (!text && !last.required) continue;
this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: last.paragraphBreakAfter });
}
const pending = this.segments.filter((segment) => segment.text.trim());
if (!pending.length) return;
@@ -291,6 +348,12 @@ function firstModelId(body: string): string {
}
}
function peak(levels: number[]): number {
let max = 0;
for (const level of levels) max = Math.max(max, level);
return max;
}
/** RMS of a PCM16LE chunk. */
function rms(pcm: Uint8Array): number {
const count = pcm.length >> 1;
@@ -367,16 +430,23 @@ function describeError(error: unknown): string {
return String((error as Error)?.message ?? error);
}
/** NativeScript's HTTP module: sends an ArrayBuffer body as-is on both platforms. */
/**
* Uploads go through postBinary (on Android NativeScript's HTTP module would
* copy each body into Java a byte at a time); the model list through
* NativeScript's HTTP module.
*/
const nativeHttp: WhisperHttp = async (request) => {
// Required lazily so Node tests can load this module without NativeScript.
if (request.body) {
const { postBinary } = require("./binary-http") as typeof import("./binary-http");
return postBinary(request.url, request.body, request.headers, request.timeoutMs);
}
const { withUserAgent } = require("../util/http") as typeof import("../util/http");
const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http");
const response = await send({
url: request.url,
method: request.method,
headers: withUserAgent(request.headers),
content: request.body,
timeout: request.timeoutMs,
});
const content = response.content as any;