diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBinaryPost.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBinaryPost.kt new file mode 100644 index 00000000..a2b358a9 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBinaryPost.kt @@ -0,0 +1,103 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper + +import java.io.IOException +import java.nio.ByteBuffer +import java.util.concurrent.TimeUnit + +import okhttp3.Call +import okhttp3.Callback +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response + +/** + * One POST of a binary body (a Whisper server WAV upload), sent by start(). + * NativeScript's http module hands an ArrayBuffer body to Java one bridge call + * per byte, which for a ~1 MB upload stalls the JS thread; here the body + * arrives as a direct ByteBuffer (copyToJavaByteBuffer) and is copied in bulk. + * headers is a flat [name, value, ...] array, as for FaceclawSseRequest, and + * may carry the Content-Type. The response text and status reach the listener + * on the constructing thread's Looper. + */ +class FaceclawBinaryPost( + url: String?, + body: ByteBuffer?, + headers: Array?, + timeoutMs: Long, + listener: FaceclawBinaryPostListener? +) { + private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper()) + private val listener: FaceclawBinaryPostListener = + listener ?: throw IllegalArgumentException("listener is required") + private val call: Call + + @Volatile + private var cancelled = false + + init { + if (url == null || url.trim().isEmpty()) { + throw IllegalArgumentException("url is required") + } + val bytes = ByteArray(body?.remaining() ?: 0) + body?.duplicate()?.get(bytes) + // No media type on the body, so okhttp keeps the caller's Content-Type header. + val builder = Request.Builder().url(url.trim()).post(bytes.toRequestBody()) + for ((name, value) in SseStream.unpackHeaders(headers)) builder.addHeader(name, value) + call = getClient().newCall(builder.build()) + call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS) + } + + fun start() { + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + post { listener.onFailure(e.message ?: e.toString()) } + } + + override fun onResponse(call: Call, response: Response) { + val text = try { + response.use { it.body?.string() ?: "" } + } catch (e: IOException) { + post { listener.onFailure(e.message ?: e.toString()) } + return + } + post { listener.onResponse(response.code, text) } + } + }) + } + + /** Aborts the request; no callback follows. */ + fun cancel() { + cancelled = true + call.cancel() + } + + private fun post(runnable: () -> Unit) { + callbackHandler.post { + if (!cancelled) runnable() + } + } + + companion object { + @Volatile + private var sharedClient: OkHttpClient? = null + + private fun getClient(): OkHttpClient { + if (sharedClient == null) { + synchronized(FaceclawBinaryPost::class.java) { + if (sharedClient == null) { + sharedClient = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .addInterceptor(FaceclawHttp.userAgentInterceptor()) + .build() + } + } + } + return sharedClient!! + } + } +} diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 6a2c1d7c..aca308db 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -148,7 +148,16 @@ export class IosPreviewController { private acknowledgedFrames = 0 private logLines: string[] = [] private logTimer: ReturnType | null = null + // Windows that want continuous capture, each with the recording it asked for + // ('' for none). They share one capture, which runs while any of them wants it. + private readonly continuousClients = new Map() + // Continuous capture should (re)start once audio is available: a window + // asked, the glasses connected or disconnected, or the glasses lock or the + // app lifecycle cut it short. A capture that fails doesn't set it, so a + // broken start is retried on the next such change rather than in a loop. + private continuousPending = false private continuousCaptureToken = 0 + private sessionConnected = false private readonly actions: LayerActions = { ...noopLayerActions, playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload), @@ -156,8 +165,8 @@ export class IosPreviewController { disconnect: () => this.disconnect(), startVoiceCapture: endpointing => this.startVoiceCapture(endpointing), stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(), - startContinuousVoiceCapture: options => this.startContinuousVoiceCapture(options?.recordAs), - stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options?.recordAs), + stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client), startTextSettingEdit: async setting => { await this.editSetting(setting) }, startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => { for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return } @@ -244,6 +253,8 @@ export class IosPreviewController { launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() }) registerNavigateTools(id => this.launchApp(id)) registerRoamTools(id => this.launchApp(id)) + // Every capture end (drained, failed, replaced) may leave continuous capture to restart. + iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture()) for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app)) } @@ -299,10 +310,12 @@ export class IosPreviewController { shell.foregroundWindow()?.requestRender() this.requestShellRender() this.syncEvenHubPhoneUi() + this.syncContinuousCapture() } pause(): void { if (!this.active) return this.active = false + if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true iosVoiceInput.stopPhoneCapture() // The phone preview is hidden; the glasses still own their screen and input. this.syncRuntime() @@ -421,8 +434,10 @@ export class IosPreviewController { this.glassesLocked = locked if (locked) { this.glance.dismiss() - iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.') - } + if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true + iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.', + 'Glasses locked. Transcription resumes when you unlock your phone.') + } else this.syncContinuousCapture() void (async () => { await this.displayReady const display = this.display @@ -720,6 +735,7 @@ export class IosPreviewController { this.maybeLaunchOnboarding(this.state) if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() + this.syncContinuousSession() shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging }) this.syncPresence() this.syncRuntime() @@ -947,33 +963,88 @@ export class IosPreviewController { await iosVoiceInput.startPhoneCapture(voice, log, endpointing) } } - /** Open-ended capture (Transcribe, Teleprompter); holds voiceActivity like Android. */ - private async startContinuousVoiceCapture(recordAs?: string): Promise { - if (this.glassesLocked) return - const voice = this.voiceProvider() - if (!iosVoiceInput.supportsContinuous(voice)) { + /** + * Continuous capture on behalf of `client` (one id per window: Transcribe, + * Teleprompter, Calculator); it runs while any client wants it, as on Android. + * `recordAs` names a WAV of the audio in Downloads when Save Transcribe audio is on. + */ + private startContinuousVoiceCapture(client: string, recordAs = ''): void { + if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) { this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).') return } + this.continuousClients.set(client, recordAs) + this.syncContinuousRecording() + // A listening capture is shared as it is; otherwise start one (after any drain). + if (!iosVoiceInput.continuousSource) this.continuousPending = true + this.syncContinuousCapture() + } + private stopContinuousVoiceCapture(client: string): void { + if (!this.continuousClients.delete(client)) return + this.syncContinuousRecording() + if (this.continuousClients.size > 0) return + // A start still awaiting permission or the mic backs off. + ++this.continuousCaptureToken + this.continuousPending = false + this.syncContinuousClaim() + void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`)) + } + /** Connecting or losing the glasses moves continuous capture to the new audio source. */ + private syncContinuousSession(): void { + const connected = this.state.phase === 'connected' + if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true + this.sessionConnected = connected + this.syncContinuousCapture() + } + /** + * Keep continuous capture running while a window wants it: start a pending + * one once audio is available, and move a phone capture to the glasses once + * they connect. A glasses session supplies the audio, even while it + * reconnects (as on Android); the phone mic stands in only without one, while + * Faceclaw is in the foreground. Cheap enough to call on every state change. + */ + private syncContinuousCapture(): void { + this.syncContinuousClaim() + if (this.continuousClients.size === 0 || this.glassesLocked) return + const glassesReady = !!this.communicator && this.state.phase === 'connected' + if (iosVoiceInput.continuousActive) { + // Draining or listening already. A phone capture from before the glasses + // connected is drained here, and its end restarts it on the glasses. + if (glassesReady && iosVoiceInput.continuousSource === 'phone') { + this.continuousPending = true + void iosVoiceInput.stopContinuousCapture() + } + return + } + if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return + this.continuousPending = false + void this.beginContinuousCapture() + } + private async beginContinuousCapture(): Promise { + const voice = this.voiceProvider() + if (!iosVoiceInput.supportsContinuous(voice)) return // A stop during an await below bumps the token; this start then backs off. const token = ++this.continuousCaptureToken const current = () => token === this.continuousCaptureToken const log = (message: string) => this.logBluetooth(message) - const record = recordAs && saveTranscribeAudioSetting.get() ? recordAs : '' + // Claimed before the capture starts so other mic users stand down first, as on Android. voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER) const communicator = this.communicator - if (communicator && this.state.phase === 'connected') { - await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true, record) - } else if (this.active && await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked) { - await iosVoiceInput.startPhoneCapture(voice, log, false, true, record) + if (communicator) { + await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true) + } else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) { + await iosVoiceInput.startPhoneCapture(voice, log, false, true) } - if (!current()) return - if (!iosVoiceInput.continuousActive) voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER) + if (current()) this.syncContinuousClaim() } - private stopContinuousVoiceCapture(): void { - ++this.continuousCaptureToken - voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER) - void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`)) + /** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */ + private syncContinuousClaim(): void { + voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) + } + /** Record to the first client's requested name while Save Transcribe audio is on. */ + private syncContinuousRecording(): void { + const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].find(Boolean) ?? '' : '' + iosVoiceInput.recordTo(name) } /** The transcription provider setting and the keys the cloud ones need. */ private voiceProvider(): VoiceProviderSelection { diff --git a/app/native/binary-http.ios.ts b/app/native/binary-http.ios.ts index d96d9aad..e65172ab 100644 --- a/app/native/binary-http.ios.ts +++ b/app/native/binary-http.ios.ts @@ -1,6 +1,6 @@ import { Http } from '@nativescript/core'; import { withUserAgent } from '../util/http'; -import type { BinaryResponse } from './binary-http'; +import type { BinaryPostResponse, BinaryResponse } from './binary-http'; /** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */ export async function downloadBinary(url: string, timeoutMs: number): Promise { @@ -8,3 +8,9 @@ export async function downloadBinary(url: string, timeoutMs: number): Promise name.toLowerCase() === 'content-length')?.[1]; return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) }; } + +/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */ +export async function postBinary(url: string, body: ArrayBuffer, headers: Record, timeoutMs: number): Promise { + const response = await Http.request({ url, method: 'POST', headers: withUserAgent(headers), content: body, timeout: timeoutMs }); + return { status: response.statusCode, text: response.content ? String(response.content.toString()) : '' }; +} diff --git a/app/native/binary-http.ts b/app/native/binary-http.ts index 12a79d32..27833ea6 100644 --- a/app/native/binary-http.ts +++ b/app/native/binary-http.ts @@ -2,6 +2,7 @@ declare const com: any; declare const java: any; export type BinaryResponse = { status: number; bytes: Uint8Array; contentLength: number }; +export type BinaryPostResponse = { status: number; text: string }; let nextDownloadSerial = 0; @@ -35,3 +36,25 @@ export function downloadBinary(url: string, timeoutMs: number): Promise, timeoutMs: number): Promise { + // Required lazily so Node tests can load this module's importers. + const { copyToJavaByteBuffer } = require("./java-direct-buffer") as typeof import("./java-direct-buffer"); + const entries = Object.entries(headers); + const nativeHeaders = Array.create("java.lang.String", entries.length * 2); + entries.forEach(([name, value], index) => { nativeHeaders[index * 2] = name; nativeHeaders[index * 2 + 1] = value; }); + return new Promise((resolve, reject) => { + const listener = new com.faceclaw.app.FaceclawBinaryPostListener({ + onResponse: (status: number, text: string | null) => resolve({ status, text: String(text ?? "") }), + onFailure: (message: string | null) => reject(new Error(String(message ?? "Request failed."))), + }); + // The User-Agent comes from the okhttp client's interceptor. + new com.faceclaw.app.FaceclawBinaryPost(url, copyToJavaByteBuffer(new Uint8Array(body)), nativeHeaders, timeoutMs, listener).start(); + }); +} diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 832543f1..38dd996f 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -15,7 +15,9 @@ const CLOUD_FINAL_TIMEOUT_MS = 5000 * from the glasses when connected, or the default phone input in preview mode, * and is transcribed by Apple's on-device recognizer or a cloud provider. * Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only - * the PCM capture runs open-ended, and pauses commit its segments. + * the PCM capture runs open-ended, and pauses commit its segments. When the + * session or the app lifecycle cuts one short it is drained, not discarded; + * restarting it is the controller's call. */ export class IosVoiceControlBridge { private native: any = null @@ -42,7 +44,12 @@ export class IosVoiceControlBridge { // Open-ended capture: runs until stopContinuousCapture, and push-to-talk // joins it instead of restarting the mic (transcripts reach every listener). private continuous = false - // The capture's audio as a WAV in Downloads, while recordTo asked for one. + // Shown instead of the native "ended" status once a drain the bridge started completes. + private drainStatus: string | null = null + // Continuous captures' audio goes to Downloads/.wav while it is + // set. The file outlives a capture, so a restarted one keeps appending to it, + // and it is created by the first audio, so a failed start leaves no file. + private recordingName = '' private recorder: WavRecorder | null = null onStatus(listener: (state: VoiceControlState) => void): () => void { @@ -61,7 +68,7 @@ export class IosVoiceControlBridge { this.native.pcmHandler = (pcm: NSData) => { const bytes = new Uint8Array(interop.bufferFromData(pcm)) this.cloud?.acceptPcm(bytes) - this.recorder?.append(bytes) + if (this.continuous) this.record(bytes) } } return this.native @@ -87,12 +94,11 @@ export class IosVoiceControlBridge { return true } get statusText(): string { return this.status } - async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise { + async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { if (this.joinsContinuous(continuous)) return this.stop() const generation = ++this.generation this.continuous = continuous - this.startRecording(recordTo, log) this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) const startError = this.startNative(voice, endpointing, false) @@ -117,11 +123,10 @@ export class IosVoiceControlBridge { this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd() } } - async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise { + async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { if (this.joinsContinuous(continuous)) return this.stop() this.continuous = continuous - this.startRecording(recordTo, log) this.log = log; this.source = 'phone'; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) const error = this.startNative(voice, endpointing, true) @@ -147,11 +152,32 @@ export class IosVoiceControlBridge { } stopPhoneCapture(): void { if (this.source !== 'phone') return + if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return } this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd() } /** Whether continuous capture can run with this provider (Apple's recognizer can't). */ supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) } get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) } + /** Where a listening continuous capture gets its audio; null when none is listening. */ + get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null } + /** + * Record continuous captures' audio to Downloads/.wav, across restarts, + * until called with another name or ''. Only cloud captures supply PCM. + */ + recordTo(name: string): void { + if (name === this.recordingName) return + this.recorder?.close(); this.recorder = null + this.recordingName = name + } + private record(pcm: Uint8Array): void { + if (!this.recorder && this.recordingName) { + this.recorder = WavRecorder.open(this.recordingName) + this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') + // One attempt per recording, not one per packet. + if (!this.recorder) this.recordingName = '' + } + this.recorder?.append(pcm) + } /** End continuous capture, transcribing the audio since the last pause. */ stopContinuousCapture(): Promise { if (!this.continuous) return Promise.resolve() @@ -168,10 +194,11 @@ export class IosVoiceControlBridge { if (this.continuous) return Promise.resolve() return this.finalizeCapture() } - private finalizeCapture(): Promise { + private finalizeCapture(status: string | null = null): Promise { const completion = this.completion if (!this.capturing) return completion this.capturing = false; this.finalizing = true + this.drainStatus = status this.clearAudioTimer(); this.releaseMicrophone() this.setStatus('Recognizing…'); this.native?.finish() return completion @@ -179,13 +206,20 @@ export class IosVoiceControlBridge { stop(): void { ++this.generation this.capturing = this.finalizing = this.continuous = false - this.source = null + this.source = null; this.drainStatus = null this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() this.clearCloud() this.finishCompletion() } - handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void { + /** + * The glasses session went away (or the glasses locked). Push-to-talk ends + * with `message`; continuous capture transcribes what it already heard and + * shows `continuousMessage`. + */ + handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.', + continuousMessage = 'Glasses disconnected. Transcription resumes when they reconnect.'): void { if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return + if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return } this.stop(); this.setStatus(message); this.emitEnd() } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } @@ -193,14 +227,7 @@ export class IosVoiceControlBridge { const session = this.session; this.session = null if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`)) } - /** Record this capture's audio as Downloads/.wav (only cloud captures supply PCM). */ - private startRecording(name: string, log: (message: string) => void): void { - if (!name) return - this.recorder = WavRecorder.open(name) - log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') - } private clearCloud(): void { - this.recorder?.close(); this.recorder = null if (this.cloudTimer !== null) clearTimeout(this.cloudTimer) this.cloudTimer = null this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null @@ -245,12 +272,13 @@ export class IosVoiceControlBridge { } else if (event.kind === 'finishing') { void this.stopPushToTalk(); this.emitEnd() } else if (event.kind === 'ended') { - this.source = null + const status = this.drainStatus ?? event.message ?? 'Ready to send' + this.source = null; this.drainStatus = null this.clearAudioTimer(); this.releaseMicrophone() - if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return } + if (this.cloud) { this.finishCloud(status); return } this.capturing = this.finalizing = false this.finishCompletion() - this.setStatus(event.message ?? 'Ready to send'); this.emitEnd() + this.setStatus(status); this.emitEnd() } } } diff --git a/app/native/whisper-server-stt.ts b/app/native/whisper-server-stt.ts index f2caac29..fe115df0 100644 --- a/app/native/whisper-server-stt.ts +++ b/app/native/whisper-server-stt.ts @@ -5,8 +5,10 @@ import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from * speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file. * Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause) * is one request, finalized in audio order; an open segment is re-sent every - * INTERIM_STEP_BYTES for provisional text. start() lists {baseUrl}/models to check - * reachability and pick a model when none is configured. + * INTERIM_STEP_BYTES for provisional text. A segment that runs SEGMENT_MAX_BYTES + * without a pause is cut into requests that still leave as one final. start() + * lists {baseUrl}/models to check reachability and pick a model when none is + * configured. */ export type WhisperServerConfig = { @@ -29,11 +31,15 @@ export type WhisperHttpResponse = { status: number; text: string }; export type WhisperHttp = (request: WhisperHttpRequest) => Promise; const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2; -// Provisional transcripts: first after one second of audio, then every 1.5s, -// and not for segments past Whisper's 30s window (each would re-send it all). +// Provisional transcripts: first after one second of audio, then every 1.5s. const INTERIM_MIN_BYTES = BYTES_PER_SECOND; const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5; -const INTERIM_MAX_BYTES = BYTES_PER_SECOND * 30; +// Steady speech or background noise can keep the pause detector from ever +// ending a span. Such a segment is cut at this length, inside Whisper's 30s +// window and well within FINAL_TIMEOUT_MS, after its quietest chunk among the +// last SPLIT_SEARCH_BYTES so a word is less likely to straddle the cut. +const SEGMENT_MAX_BYTES = BYTES_PER_SECOND * 25; +const SPLIT_SEARCH_BYTES = BYTES_PER_SECOND * 5; const MODELS_TIMEOUT_MS = 5000; const INTERIM_TIMEOUT_MS = 10000; const FINAL_TIMEOUT_MS = 12000; @@ -48,6 +54,8 @@ export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETR type Segment = { chunks: Uint8Array[]; + /** RMS of each chunk. */ + levels: number[]; bytes: number; peakRms: number; text: string; @@ -56,6 +64,8 @@ type Segment = { paragraphBreakAfter: boolean; /** Emit a final even when empty: the end of push-to-talk needs one. */ required: boolean; + /** Cut for length: its final leaves joined with the next segment's. */ + continues: boolean; interimInFlight: boolean; interimBytes: number; }; @@ -120,10 +130,13 @@ export class WhisperServerSttClient implements CloudSttClient { acceptPcm(pcm: Uint8Array): void { if (this.closed || !pcm.length) return; const segment = this.openSegment(); + const level = rms(pcm); segment.chunks.push(pcm.slice()); + segment.levels.push(level); segment.bytes += pcm.length; - segment.peakRms = Math.max(segment.peakRms, rms(pcm)); - this.maybeRequestInterim(segment); + segment.peakRms = Math.max(segment.peakRms, level); + if (segment.bytes >= SEGMENT_MAX_BYTES) this.splitSegment(segment); + this.maybeRequestInterim(this.segments[this.segments.length - 1]!); } /** A pause in continuous capture: transcribe what came before it. */ @@ -145,16 +158,54 @@ export class WhisperServerSttClient implements CloudSttClient { const last = this.segments[this.segments.length - 1]; if (last && !last.closed) return last; const segment: Segment = { - chunks: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false, - paragraphBreakAfter: false, required: false, interimInFlight: false, interimBytes: 0, + chunks: [], levels: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false, + paragraphBreakAfter: false, required: false, continues: false, interimInFlight: false, interimBytes: 0, }; this.segments.push(segment); return segment; } + /** + * Close the open segment at SEGMENT_MAX_BYTES and carry the audio after its + * cut into a new one. The cut is after the quietest chunk among the last + * SPLIT_SEARCH_BYTES; the first chunk always stays. + */ + private splitSegment(segment: Segment): void { + let cut = segment.chunks.length - 1; + let quietest = Infinity; + let carried = 0; + for (let i = segment.chunks.length - 1; i > 0 && carried <= SPLIT_SEARCH_BYTES; i--) { + if (segment.levels[i]! < quietest) { + quietest = segment.levels[i]!; + cut = i; + } + carried += segment.chunks[i]!.length; + } + const chunks = segment.chunks.splice(cut + 1); + const levels = segment.levels.splice(cut + 1); + for (const chunk of chunks) segment.bytes -= chunk.length; + segment.peakRms = peak(segment.levels); + segment.continues = true; + this.closeSegment(false, false); + if (!chunks.length) return; + const next = this.openSegment(); + next.chunks = chunks; + next.levels = levels; + for (const chunk of chunks) next.bytes += chunk.length; + next.peakRms = peak(levels); + } + private closeSegment(paragraphBreakAfter: boolean, required: boolean): void { if (this.closed) return; const last = this.segments[this.segments.length - 1]; + if (last?.closed && last.continues) { + // Cut for length with nothing after it yet: this boundary ends its run. + last.continues = false; + last.paragraphBreakAfter = paragraphBreakAfter; + last.required = required; + if (last.final) this.emit(); + return; + } if (!last || last.closed) { // Nothing new since the last boundary. Push-to-talk still needs a final. if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required); @@ -192,7 +243,7 @@ export class WhisperServerSttClient implements CloudSttClient { private maybeRequestInterim(segment: Segment): void { if (!this.ready || segment.interimInFlight || segment.closed) return; - if (segment.bytes < INTERIM_MIN_BYTES || segment.bytes > INTERIM_MAX_BYTES) return; + if (segment.bytes < INTERIM_MIN_BYTES) return; if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return; if (segment.peakRms < SILENCE_RMS) return; segment.interimInFlight = true; @@ -221,14 +272,20 @@ export class WhisperServerSttClient implements CloudSttClient { } /** - * Finals leave in audio order, even when a later request answers first; - * whatever is not final yet is shown as one provisional transcript. + * Finals leave in audio order, even when a later request answers first, and + * segments cut for length leave as one final with the segment that ends + * their run; whatever is not final yet is shown as one provisional transcript. */ private emit(): void { - while (this.segments.length && this.segments[0]!.final) { - const segment = this.segments.shift()!; - if (!segment.text && !segment.required) continue; - this.options.onTranscript({ text: segment.text, isFinal: true, paragraphBreakAfter: segment.paragraphBreakAfter }); + for (;;) { + let end = 0; + while (this.segments[end]?.final && this.segments[end]!.continues) end++; + if (!this.segments[end]?.final) break; + const run = this.segments.splice(0, end + 1); + const last = run[run.length - 1]!; + const text = run.map((segment) => segment.text.trim()).filter(Boolean).join(" "); + if (!text && !last.required) continue; + this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: last.paragraphBreakAfter }); } const pending = this.segments.filter((segment) => segment.text.trim()); if (!pending.length) return; @@ -291,6 +348,12 @@ function firstModelId(body: string): string { } } +function peak(levels: number[]): number { + let max = 0; + for (const level of levels) max = Math.max(max, level); + return max; +} + /** RMS of a PCM16LE chunk. */ function rms(pcm: Uint8Array): number { const count = pcm.length >> 1; @@ -367,16 +430,23 @@ function describeError(error: unknown): string { return String((error as Error)?.message ?? error); } -/** NativeScript's HTTP module: sends an ArrayBuffer body as-is on both platforms. */ +/** + * Uploads go through postBinary (on Android NativeScript's HTTP module would + * copy each body into Java a byte at a time); the model list through + * NativeScript's HTTP module. + */ const nativeHttp: WhisperHttp = async (request) => { // Required lazily so Node tests can load this module without NativeScript. + if (request.body) { + const { postBinary } = require("./binary-http") as typeof import("./binary-http"); + return postBinary(request.url, request.body, request.headers, request.timeoutMs); + } const { withUserAgent } = require("../util/http") as typeof import("../util/http"); const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http"); const response = await send({ url: request.url, method: request.method, headers: withUserAgent(request.headers), - content: request.body, timeout: request.timeoutMs, }); const content = response.content as any; diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBinaryPostListener.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBinaryPostListener.kt new file mode 100644 index 00000000..573ee3b1 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBinaryPostListener.kt @@ -0,0 +1,10 @@ +package com.faceclaw.app + +/** Callbacks for FaceclawBinaryPost, delivered on the constructing thread's Looper. */ +interface FaceclawBinaryPostListener { + /** The server answered, with any status; body is the response text. */ + fun onResponse(status: Int, body: String?): Unit + + /** Network-level failure (connect, TLS, timeout, mid-body drop); no response arrived. */ + fun onFailure(message: String?): Unit +} diff --git a/notes/whisper-server.md b/notes/whisper-server.md index 20ad4a5b..a17357c8 100644 --- a/notes/whisper-server.md +++ b/notes/whisper-server.md @@ -6,7 +6,11 @@ any OpenAI-compatible Whisper server instead of a cloud service: vLLM, speaches `POST {url}/audio/transcriptions` with a 16 kHz mono WAV. Push-to-talk utterances are one request each; in Transcribe, each span ended by a pause is one request. While a span is open its audio so far is re-sent every 1.5 s, so -text appears while you are still talking. +text appears while you are still talking. A span that runs 25 s without a pause +(nonstop speech, music) is cut at its quietest moment in the last 5 s, so no +request outgrows Whisper's 30 s window; the pieces still arrive as one final. +On Android the WAV goes out through `FaceclawBinaryPost` (okhttp), since +NativeScript's http module copies a binary body into Java a byte at a time. ## Server @@ -39,6 +43,11 @@ before building: ## Transcribe on iOS Transcribe needs a cloud or Whisper-server provider on iOS (Apple's recognizer -only handles short utterances). With "Save Transcribe audio" on (Voice +only handles short utterances). Its capture follows the glasses: a disconnect, +a glasses lock or backgrounding (for phone-mic capture) transcribes what was +already heard, and listening resumes once the glasses reconnect, unlock or the +app returns to the foreground. Without a glasses session the phone mic stands in +while Faceclaw is in the foreground. With "Save Transcribe audio" on (Voice settings), each session's audio is kept as `transcribe-