forked from Blendman974/faceclaw
Bugfixes in iOS Transcribe and Whisper server integration
This commit is contained in:
1 parent
33eef6538c
commit
5bf809d733
12 files changed
+553
-73
No files matched your search
@@ -0,0 +1,103 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
import android.os.Handler
|
||||||
|
import android.os.Looper
|
||||||
|
|
||||||
|
import java.io.IOException
|
||||||
|
import java.nio.ByteBuffer
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
|
import okhttp3.Call
|
||||||
|
import okhttp3.Callback
|
||||||
|
import okhttp3.OkHttpClient
|
||||||
|
import okhttp3.Request
|
||||||
|
import okhttp3.RequestBody.Companion.toRequestBody
|
||||||
|
import okhttp3.Response
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One POST of a binary body (a Whisper server WAV upload), sent by start().
|
||||||
|
* NativeScript's http module hands an ArrayBuffer body to Java one bridge call
|
||||||
|
* per byte, which for a ~1 MB upload stalls the JS thread; here the body
|
||||||
|
* arrives as a direct ByteBuffer (copyToJavaByteBuffer) and is copied in bulk.
|
||||||
|
* headers is a flat [name, value, ...] array, as for FaceclawSseRequest, and
|
||||||
|
* may carry the Content-Type. The response text and status reach the listener
|
||||||
|
* on the constructing thread's Looper.
|
||||||
|
*/
|
||||||
|
class FaceclawBinaryPost(
|
||||||
|
url: String?,
|
||||||
|
body: ByteBuffer?,
|
||||||
|
headers: Array<String?>?,
|
||||||
|
timeoutMs: Long,
|
||||||
|
listener: FaceclawBinaryPostListener?
|
||||||
|
) {
|
||||||
|
private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper())
|
||||||
|
private val listener: FaceclawBinaryPostListener =
|
||||||
|
listener ?: throw IllegalArgumentException("listener is required")
|
||||||
|
private val call: Call
|
||||||
|
|
||||||
|
@Volatile
|
||||||
|
private var cancelled = false
|
||||||
|
|
||||||
|
init {
|
||||||
|
if (url == null || url.trim().isEmpty()) {
|
||||||
|
throw IllegalArgumentException("url is required")
|
||||||
|
}
|
||||||
|
val bytes = ByteArray(body?.remaining() ?: 0)
|
||||||
|
body?.duplicate()?.get(bytes)
|
||||||
|
// No media type on the body, so okhttp keeps the caller's Content-Type header.
|
||||||
|
val builder = Request.Builder().url(url.trim()).post(bytes.toRequestBody())
|
||||||
|
for ((name, value) in SseStream.unpackHeaders(headers)) builder.addHeader(name, value)
|
||||||
|
call = getClient().newCall(builder.build())
|
||||||
|
call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun start() {
|
||||||
|
call.enqueue(object : Callback {
|
||||||
|
override fun onFailure(call: Call, e: IOException) {
|
||||||
|
post { listener.onFailure(e.message ?: e.toString()) }
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun onResponse(call: Call, response: Response) {
|
||||||
|
val text = try {
|
||||||
|
response.use { it.body?.string() ?: "" }
|
||||||
|
} catch (e: IOException) {
|
||||||
|
post { listener.onFailure(e.message ?: e.toString()) }
|
||||||
|
return
|
||||||
|
}
|
||||||
|
post { listener.onResponse(response.code, text) }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Aborts the request; no callback follows. */
|
||||||
|
fun cancel() {
|
||||||
|
cancelled = true
|
||||||
|
call.cancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun post(runnable: () -> Unit) {
|
||||||
|
callbackHandler.post {
|
||||||
|
if (!cancelled) runnable()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
@Volatile
|
||||||
|
private var sharedClient: OkHttpClient? = null
|
||||||
|
|
||||||
|
private fun getClient(): OkHttpClient {
|
||||||
|
if (sharedClient == null) {
|
||||||
|
synchronized(FaceclawBinaryPost::class.java) {
|
||||||
|
if (sharedClient == null) {
|
||||||
|
sharedClient = OkHttpClient.Builder()
|
||||||
|
.connectTimeout(30, TimeUnit.SECONDS)
|
||||||
|
.readTimeout(60, TimeUnit.SECONDS)
|
||||||
|
.addInterceptor(FaceclawHttp.userAgentInterceptor())
|
||||||
|
.build()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sharedClient!!
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -148,7 +148,16 @@ export class IosPreviewController {
|
|||||||
private acknowledgedFrames = 0
|
private acknowledgedFrames = 0
|
||||||
private logLines: string[] = []
|
private logLines: string[] = []
|
||||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||||
|
// Windows that want continuous capture, each with the recording it asked for
|
||||||
|
// ('' for none). They share one capture, which runs while any of them wants it.
|
||||||
|
private readonly continuousClients = new Map<string, string>()
|
||||||
|
// Continuous capture should (re)start once audio is available: a window
|
||||||
|
// asked, the glasses connected or disconnected, or the glasses lock or the
|
||||||
|
// app lifecycle cut it short. A capture that fails doesn't set it, so a
|
||||||
|
// broken start is retried on the next such change rather than in a loop.
|
||||||
|
private continuousPending = false
|
||||||
private continuousCaptureToken = 0
|
private continuousCaptureToken = 0
|
||||||
|
private sessionConnected = false
|
||||||
private readonly actions: LayerActions = {
|
private readonly actions: LayerActions = {
|
||||||
...noopLayerActions,
|
...noopLayerActions,
|
||||||
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
||||||
@@ -156,8 +165,8 @@ export class IosPreviewController {
|
|||||||
disconnect: () => this.disconnect(),
|
disconnect: () => this.disconnect(),
|
||||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||||
startContinuousVoiceCapture: options => this.startContinuousVoiceCapture(options?.recordAs),
|
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options?.recordAs),
|
||||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client),
|
||||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||||
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
||||||
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
|
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
|
||||||
@@ -244,6 +253,8 @@ export class IosPreviewController {
|
|||||||
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
|
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
|
||||||
registerNavigateTools(id => this.launchApp(id))
|
registerNavigateTools(id => this.launchApp(id))
|
||||||
registerRoamTools(id => this.launchApp(id))
|
registerRoamTools(id => this.launchApp(id))
|
||||||
|
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
|
||||||
|
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
|
||||||
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
|
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -299,10 +310,12 @@ export class IosPreviewController {
|
|||||||
shell.foregroundWindow()?.requestRender()
|
shell.foregroundWindow()?.requestRender()
|
||||||
this.requestShellRender()
|
this.requestShellRender()
|
||||||
this.syncEvenHubPhoneUi()
|
this.syncEvenHubPhoneUi()
|
||||||
|
this.syncContinuousCapture()
|
||||||
}
|
}
|
||||||
pause(): void {
|
pause(): void {
|
||||||
if (!this.active) return
|
if (!this.active) return
|
||||||
this.active = false
|
this.active = false
|
||||||
|
if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true
|
||||||
iosVoiceInput.stopPhoneCapture()
|
iosVoiceInput.stopPhoneCapture()
|
||||||
// The phone preview is hidden; the glasses still own their screen and input.
|
// The phone preview is hidden; the glasses still own their screen and input.
|
||||||
this.syncRuntime()
|
this.syncRuntime()
|
||||||
@@ -421,8 +434,10 @@ export class IosPreviewController {
|
|||||||
this.glassesLocked = locked
|
this.glassesLocked = locked
|
||||||
if (locked) {
|
if (locked) {
|
||||||
this.glance.dismiss()
|
this.glance.dismiss()
|
||||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.')
|
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
|
||||||
}
|
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
||||||
|
'Glasses locked. Transcription resumes when you unlock your phone.')
|
||||||
|
} else this.syncContinuousCapture()
|
||||||
void (async () => {
|
void (async () => {
|
||||||
await this.displayReady
|
await this.displayReady
|
||||||
const display = this.display
|
const display = this.display
|
||||||
@@ -720,6 +735,7 @@ export class IosPreviewController {
|
|||||||
this.maybeLaunchOnboarding(this.state)
|
this.maybeLaunchOnboarding(this.state)
|
||||||
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
||||||
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||||
|
this.syncContinuousSession()
|
||||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
||||||
this.syncPresence()
|
this.syncPresence()
|
||||||
this.syncRuntime()
|
this.syncRuntime()
|
||||||
@@ -947,33 +963,88 @@ export class IosPreviewController {
|
|||||||
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/** Open-ended capture (Transcribe, Teleprompter); holds voiceActivity like Android. */
|
/**
|
||||||
private async startContinuousVoiceCapture(recordAs?: string): Promise<void> {
|
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||||
if (this.glassesLocked) return
|
* Teleprompter, Calculator); it runs while any client wants it, as on Android.
|
||||||
const voice = this.voiceProvider()
|
* `recordAs` names a WAV of the audio in Downloads when Save Transcribe audio is on.
|
||||||
if (!iosVoiceInput.supportsContinuous(voice)) {
|
*/
|
||||||
|
private startContinuousVoiceCapture(client: string, recordAs = ''): void {
|
||||||
|
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
this.continuousClients.set(client, recordAs)
|
||||||
|
this.syncContinuousRecording()
|
||||||
|
// A listening capture is shared as it is; otherwise start one (after any drain).
|
||||||
|
if (!iosVoiceInput.continuousSource) this.continuousPending = true
|
||||||
|
this.syncContinuousCapture()
|
||||||
|
}
|
||||||
|
private stopContinuousVoiceCapture(client: string): void {
|
||||||
|
if (!this.continuousClients.delete(client)) return
|
||||||
|
this.syncContinuousRecording()
|
||||||
|
if (this.continuousClients.size > 0) return
|
||||||
|
// A start still awaiting permission or the mic backs off.
|
||||||
|
++this.continuousCaptureToken
|
||||||
|
this.continuousPending = false
|
||||||
|
this.syncContinuousClaim()
|
||||||
|
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
|
||||||
|
}
|
||||||
|
/** Connecting or losing the glasses moves continuous capture to the new audio source. */
|
||||||
|
private syncContinuousSession(): void {
|
||||||
|
const connected = this.state.phase === 'connected'
|
||||||
|
if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true
|
||||||
|
this.sessionConnected = connected
|
||||||
|
this.syncContinuousCapture()
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Keep continuous capture running while a window wants it: start a pending
|
||||||
|
* one once audio is available, and move a phone capture to the glasses once
|
||||||
|
* they connect. A glasses session supplies the audio, even while it
|
||||||
|
* reconnects (as on Android); the phone mic stands in only without one, while
|
||||||
|
* Faceclaw is in the foreground. Cheap enough to call on every state change.
|
||||||
|
*/
|
||||||
|
private syncContinuousCapture(): void {
|
||||||
|
this.syncContinuousClaim()
|
||||||
|
if (this.continuousClients.size === 0 || this.glassesLocked) return
|
||||||
|
const glassesReady = !!this.communicator && this.state.phase === 'connected'
|
||||||
|
if (iosVoiceInput.continuousActive) {
|
||||||
|
// Draining or listening already. A phone capture from before the glasses
|
||||||
|
// connected is drained here, and its end restarts it on the glasses.
|
||||||
|
if (glassesReady && iosVoiceInput.continuousSource === 'phone') {
|
||||||
|
this.continuousPending = true
|
||||||
|
void iosVoiceInput.stopContinuousCapture()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return
|
||||||
|
this.continuousPending = false
|
||||||
|
void this.beginContinuousCapture()
|
||||||
|
}
|
||||||
|
private async beginContinuousCapture(): Promise<void> {
|
||||||
|
const voice = this.voiceProvider()
|
||||||
|
if (!iosVoiceInput.supportsContinuous(voice)) return
|
||||||
// A stop during an await below bumps the token; this start then backs off.
|
// A stop during an await below bumps the token; this start then backs off.
|
||||||
const token = ++this.continuousCaptureToken
|
const token = ++this.continuousCaptureToken
|
||||||
const current = () => token === this.continuousCaptureToken
|
const current = () => token === this.continuousCaptureToken
|
||||||
const log = (message: string) => this.logBluetooth(message)
|
const log = (message: string) => this.logBluetooth(message)
|
||||||
const record = recordAs && saveTranscribeAudioSetting.get() ? recordAs : ''
|
// Claimed before the capture starts so other mic users stand down first, as on Android.
|
||||||
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
|
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
|
||||||
const communicator = this.communicator
|
const communicator = this.communicator
|
||||||
if (communicator && this.state.phase === 'connected') {
|
if (communicator) {
|
||||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true, record)
|
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true)
|
||||||
} else if (this.active && await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked) {
|
} else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) {
|
||||||
await iosVoiceInput.startPhoneCapture(voice, log, false, true, record)
|
await iosVoiceInput.startPhoneCapture(voice, log, false, true)
|
||||||
}
|
}
|
||||||
if (!current()) return
|
if (current()) this.syncContinuousClaim()
|
||||||
if (!iosVoiceInput.continuousActive) voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
|
|
||||||
}
|
}
|
||||||
private stopContinuousVoiceCapture(): void {
|
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
||||||
++this.continuousCaptureToken
|
private syncContinuousClaim(): void {
|
||||||
voiceActivity.setActive(false, CONTINUOUS_CAPTURE_HOLDER)
|
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||||
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
|
}
|
||||||
|
/** Record to the first client's requested name while Save Transcribe audio is on. */
|
||||||
|
private syncContinuousRecording(): void {
|
||||||
|
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].find(Boolean) ?? '' : ''
|
||||||
|
iosVoiceInput.recordTo(name)
|
||||||
}
|
}
|
||||||
/** The transcription provider setting and the keys the cloud ones need. */
|
/** The transcription provider setting and the keys the cloud ones need. */
|
||||||
private voiceProvider(): VoiceProviderSelection {
|
private voiceProvider(): VoiceProviderSelection {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { Http } from '@nativescript/core';
|
import { Http } from '@nativescript/core';
|
||||||
import { withUserAgent } from '../util/http';
|
import { withUserAgent } from '../util/http';
|
||||||
import type { BinaryResponse } from './binary-http';
|
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
|
||||||
|
|
||||||
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
|
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
|
||||||
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
|
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
|
||||||
@@ -8,3 +8,9 @@ export async function downloadBinary(url: string, timeoutMs: number): Promise<Bi
|
|||||||
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
|
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
|
||||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
|
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
|
||||||
|
export async function postBinary(url: string, body: ArrayBuffer, headers: Record<string, string>, timeoutMs: number): Promise<BinaryPostResponse> {
|
||||||
|
const response = await Http.request({ url, method: 'POST', headers: withUserAgent(headers), content: body, timeout: timeoutMs });
|
||||||
|
return { status: response.statusCode, text: response.content ? String(response.content.toString()) : '' };
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ declare const com: any;
|
|||||||
declare const java: any;
|
declare const java: any;
|
||||||
|
|
||||||
export type BinaryResponse = { status: number; bytes: Uint8Array; contentLength: number };
|
export type BinaryResponse = { status: number; bytes: Uint8Array; contentLength: number };
|
||||||
|
export type BinaryPostResponse = { status: number; text: string };
|
||||||
|
|
||||||
let nextDownloadSerial = 0;
|
let nextDownloadSerial = 0;
|
||||||
|
|
||||||
@@ -35,3 +36,25 @@ export function downloadBinary(url: string, timeoutMs: number): Promise<BinaryRe
|
|||||||
new com.faceclaw.app.FaceclawFileDownload(url, path, timeoutMs, listener).start();
|
new com.faceclaw.app.FaceclawFileDownload(url, path, timeoutMs, listener).start();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST a binary body and read the response as text. Not NativeScript's http
|
||||||
|
* module: it hands an ArrayBuffer body to Java one bridge call per byte
|
||||||
|
* (ByteBuffer.wrap(Array.from(...))), which for a ~1MB WAV upload stalls the
|
||||||
|
* main thread. FaceclawBinaryPost takes it as one direct-buffer copy instead.
|
||||||
|
*/
|
||||||
|
export function postBinary(url: string, body: ArrayBuffer, headers: Record<string, string>, timeoutMs: number): Promise<BinaryPostResponse> {
|
||||||
|
// Required lazily so Node tests can load this module's importers.
|
||||||
|
const { copyToJavaByteBuffer } = require("./java-direct-buffer") as typeof import("./java-direct-buffer");
|
||||||
|
const entries = Object.entries(headers);
|
||||||
|
const nativeHeaders = Array.create("java.lang.String", entries.length * 2);
|
||||||
|
entries.forEach(([name, value], index) => { nativeHeaders[index * 2] = name; nativeHeaders[index * 2 + 1] = value; });
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const listener = new com.faceclaw.app.FaceclawBinaryPostListener({
|
||||||
|
onResponse: (status: number, text: string | null) => resolve({ status, text: String(text ?? "") }),
|
||||||
|
onFailure: (message: string | null) => reject(new Error(String(message ?? "Request failed."))),
|
||||||
|
});
|
||||||
|
// The User-Agent comes from the okhttp client's interceptor.
|
||||||
|
new com.faceclaw.app.FaceclawBinaryPost(url, copyToJavaByteBuffer(new Uint8Array(body)), nativeHeaders, timeoutMs, listener).start();
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -15,7 +15,9 @@ const CLOUD_FINAL_TIMEOUT_MS = 5000
|
|||||||
* from the glasses when connected, or the default phone input in preview mode,
|
* from the glasses when connected, or the default phone input in preview mode,
|
||||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||||
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
||||||
* the PCM capture runs open-ended, and pauses commit its segments.
|
* the PCM capture runs open-ended, and pauses commit its segments. When the
|
||||||
|
* session or the app lifecycle cuts one short it is drained, not discarded;
|
||||||
|
* restarting it is the controller's call.
|
||||||
*/
|
*/
|
||||||
export class IosVoiceControlBridge {
|
export class IosVoiceControlBridge {
|
||||||
private native: any = null
|
private native: any = null
|
||||||
@@ -42,7 +44,12 @@ export class IosVoiceControlBridge {
|
|||||||
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
||||||
// joins it instead of restarting the mic (transcripts reach every listener).
|
// joins it instead of restarting the mic (transcripts reach every listener).
|
||||||
private continuous = false
|
private continuous = false
|
||||||
// The capture's audio as a WAV in Downloads, while recordTo asked for one.
|
// Shown instead of the native "ended" status once a drain the bridge started completes.
|
||||||
|
private drainStatus: string | null = null
|
||||||
|
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||||
|
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||||
|
// and it is created by the first audio, so a failed start leaves no file.
|
||||||
|
private recordingName = ''
|
||||||
private recorder: WavRecorder | null = null
|
private recorder: WavRecorder | null = null
|
||||||
|
|
||||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||||
@@ -61,7 +68,7 @@ export class IosVoiceControlBridge {
|
|||||||
this.native.pcmHandler = (pcm: NSData) => {
|
this.native.pcmHandler = (pcm: NSData) => {
|
||||||
const bytes = new Uint8Array(interop.bufferFromData(pcm))
|
const bytes = new Uint8Array(interop.bufferFromData(pcm))
|
||||||
this.cloud?.acceptPcm(bytes)
|
this.cloud?.acceptPcm(bytes)
|
||||||
this.recorder?.append(bytes)
|
if (this.continuous) this.record(bytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return this.native
|
return this.native
|
||||||
@@ -87,12 +94,11 @@ export class IosVoiceControlBridge {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
get statusText(): string { return this.status }
|
get statusText(): string { return this.status }
|
||||||
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
|
async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||||
if (this.joinsContinuous(continuous)) return
|
if (this.joinsContinuous(continuous)) return
|
||||||
this.stop()
|
this.stop()
|
||||||
const generation = ++this.generation
|
const generation = ++this.generation
|
||||||
this.continuous = continuous
|
this.continuous = continuous
|
||||||
this.startRecording(recordTo, log)
|
|
||||||
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
|
this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true
|
||||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||||
const startError = this.startNative(voice, endpointing, false)
|
const startError = this.startNative(voice, endpointing, false)
|
||||||
@@ -117,11 +123,10 @@ export class IosVoiceControlBridge {
|
|||||||
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false, recordTo = ''): Promise<void> {
|
async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise<void> {
|
||||||
if (this.joinsContinuous(continuous)) return
|
if (this.joinsContinuous(continuous)) return
|
||||||
this.stop()
|
this.stop()
|
||||||
this.continuous = continuous
|
this.continuous = continuous
|
||||||
this.startRecording(recordTo, log)
|
|
||||||
this.log = log; this.source = 'phone'; this.capturing = true
|
this.log = log; this.source = 'phone'; this.capturing = true
|
||||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||||
const error = this.startNative(voice, endpointing, true)
|
const error = this.startNative(voice, endpointing, true)
|
||||||
@@ -147,11 +152,32 @@ export class IosVoiceControlBridge {
|
|||||||
}
|
}
|
||||||
stopPhoneCapture(): void {
|
stopPhoneCapture(): void {
|
||||||
if (this.source !== 'phone') return
|
if (this.source !== 'phone') return
|
||||||
|
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
|
||||||
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
||||||
}
|
}
|
||||||
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
||||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
||||||
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
||||||
|
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||||
|
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||||
|
/**
|
||||||
|
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||||
|
* until called with another name or ''. Only cloud captures supply PCM.
|
||||||
|
*/
|
||||||
|
recordTo(name: string): void {
|
||||||
|
if (name === this.recordingName) return
|
||||||
|
this.recorder?.close(); this.recorder = null
|
||||||
|
this.recordingName = name
|
||||||
|
}
|
||||||
|
private record(pcm: Uint8Array): void {
|
||||||
|
if (!this.recorder && this.recordingName) {
|
||||||
|
this.recorder = WavRecorder.open(this.recordingName)
|
||||||
|
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||||
|
// One attempt per recording, not one per packet.
|
||||||
|
if (!this.recorder) this.recordingName = ''
|
||||||
|
}
|
||||||
|
this.recorder?.append(pcm)
|
||||||
|
}
|
||||||
/** End continuous capture, transcribing the audio since the last pause. */
|
/** End continuous capture, transcribing the audio since the last pause. */
|
||||||
stopContinuousCapture(): Promise<void> {
|
stopContinuousCapture(): Promise<void> {
|
||||||
if (!this.continuous) return Promise.resolve()
|
if (!this.continuous) return Promise.resolve()
|
||||||
@@ -168,10 +194,11 @@ export class IosVoiceControlBridge {
|
|||||||
if (this.continuous) return Promise.resolve()
|
if (this.continuous) return Promise.resolve()
|
||||||
return this.finalizeCapture()
|
return this.finalizeCapture()
|
||||||
}
|
}
|
||||||
private finalizeCapture(): Promise<void> {
|
private finalizeCapture(status: string | null = null): Promise<void> {
|
||||||
const completion = this.completion
|
const completion = this.completion
|
||||||
if (!this.capturing) return completion
|
if (!this.capturing) return completion
|
||||||
this.capturing = false; this.finalizing = true
|
this.capturing = false; this.finalizing = true
|
||||||
|
this.drainStatus = status
|
||||||
this.clearAudioTimer(); this.releaseMicrophone()
|
this.clearAudioTimer(); this.releaseMicrophone()
|
||||||
this.setStatus('Recognizing…'); this.native?.finish()
|
this.setStatus('Recognizing…'); this.native?.finish()
|
||||||
return completion
|
return completion
|
||||||
@@ -179,13 +206,20 @@ export class IosVoiceControlBridge {
|
|||||||
stop(): void {
|
stop(): void {
|
||||||
++this.generation
|
++this.generation
|
||||||
this.capturing = this.finalizing = this.continuous = false
|
this.capturing = this.finalizing = this.continuous = false
|
||||||
this.source = null
|
this.source = null; this.drainStatus = null
|
||||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||||
this.clearCloud()
|
this.clearCloud()
|
||||||
this.finishCompletion()
|
this.finishCompletion()
|
||||||
}
|
}
|
||||||
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
|
/**
|
||||||
|
* The glasses session went away (or the glasses locked). Push-to-talk ends
|
||||||
|
* with `message`; continuous capture transcribes what it already heard and
|
||||||
|
* shows `continuousMessage`.
|
||||||
|
*/
|
||||||
|
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.',
|
||||||
|
continuousMessage = 'Glasses disconnected. Transcription resumes when they reconnect.'): void {
|
||||||
if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return
|
if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return
|
||||||
|
if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return }
|
||||||
this.stop(); this.setStatus(message); this.emitEnd()
|
this.stop(); this.setStatus(message); this.emitEnd()
|
||||||
}
|
}
|
||||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||||
@@ -193,14 +227,7 @@ export class IosVoiceControlBridge {
|
|||||||
const session = this.session; this.session = null
|
const session = this.session; this.session = null
|
||||||
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`))
|
||||||
}
|
}
|
||||||
/** Record this capture's audio as Downloads/<name>.wav (only cloud captures supply PCM). */
|
|
||||||
private startRecording(name: string, log: (message: string) => void): void {
|
|
||||||
if (!name) return
|
|
||||||
this.recorder = WavRecorder.open(name)
|
|
||||||
log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
|
||||||
}
|
|
||||||
private clearCloud(): void {
|
private clearCloud(): void {
|
||||||
this.recorder?.close(); this.recorder = null
|
|
||||||
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
|
if (this.cloudTimer !== null) clearTimeout(this.cloudTimer)
|
||||||
this.cloudTimer = null
|
this.cloudTimer = null
|
||||||
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
|
this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null
|
||||||
@@ -245,12 +272,13 @@ export class IosVoiceControlBridge {
|
|||||||
} else if (event.kind === 'finishing') {
|
} else if (event.kind === 'finishing') {
|
||||||
void this.stopPushToTalk(); this.emitEnd()
|
void this.stopPushToTalk(); this.emitEnd()
|
||||||
} else if (event.kind === 'ended') {
|
} else if (event.kind === 'ended') {
|
||||||
this.source = null
|
const status = this.drainStatus ?? event.message ?? 'Ready to send'
|
||||||
|
this.source = null; this.drainStatus = null
|
||||||
this.clearAudioTimer(); this.releaseMicrophone()
|
this.clearAudioTimer(); this.releaseMicrophone()
|
||||||
if (this.cloud) { this.finishCloud(event.message ?? 'Ready to send'); return }
|
if (this.cloud) { this.finishCloud(status); return }
|
||||||
this.capturing = this.finalizing = false
|
this.capturing = this.finalizing = false
|
||||||
this.finishCompletion()
|
this.finishCompletion()
|
||||||
this.setStatus(event.message ?? 'Ready to send'); this.emitEnd()
|
this.setStatus(status); this.emitEnd()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from
|
|||||||
* speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file.
|
* speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file.
|
||||||
* Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause)
|
* Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause)
|
||||||
* is one request, finalized in audio order; an open segment is re-sent every
|
* is one request, finalized in audio order; an open segment is re-sent every
|
||||||
* INTERIM_STEP_BYTES for provisional text. start() lists {baseUrl}/models to check
|
* INTERIM_STEP_BYTES for provisional text. A segment that runs SEGMENT_MAX_BYTES
|
||||||
* reachability and pick a model when none is configured.
|
* without a pause is cut into requests that still leave as one final. start()
|
||||||
|
* lists {baseUrl}/models to check reachability and pick a model when none is
|
||||||
|
* configured.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export type WhisperServerConfig = {
|
export type WhisperServerConfig = {
|
||||||
@@ -29,11 +31,15 @@ export type WhisperHttpResponse = { status: number; text: string };
|
|||||||
export type WhisperHttp = (request: WhisperHttpRequest) => Promise<WhisperHttpResponse>;
|
export type WhisperHttp = (request: WhisperHttpRequest) => Promise<WhisperHttpResponse>;
|
||||||
|
|
||||||
const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2;
|
const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2;
|
||||||
// Provisional transcripts: first after one second of audio, then every 1.5s,
|
// Provisional transcripts: first after one second of audio, then every 1.5s.
|
||||||
// and not for segments past Whisper's 30s window (each would re-send it all).
|
|
||||||
const INTERIM_MIN_BYTES = BYTES_PER_SECOND;
|
const INTERIM_MIN_BYTES = BYTES_PER_SECOND;
|
||||||
const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5;
|
const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5;
|
||||||
const INTERIM_MAX_BYTES = BYTES_PER_SECOND * 30;
|
// Steady speech or background noise can keep the pause detector from ever
|
||||||
|
// ending a span. Such a segment is cut at this length, inside Whisper's 30s
|
||||||
|
// window and well within FINAL_TIMEOUT_MS, after its quietest chunk among the
|
||||||
|
// last SPLIT_SEARCH_BYTES so a word is less likely to straddle the cut.
|
||||||
|
const SEGMENT_MAX_BYTES = BYTES_PER_SECOND * 25;
|
||||||
|
const SPLIT_SEARCH_BYTES = BYTES_PER_SECOND * 5;
|
||||||
const MODELS_TIMEOUT_MS = 5000;
|
const MODELS_TIMEOUT_MS = 5000;
|
||||||
const INTERIM_TIMEOUT_MS = 10000;
|
const INTERIM_TIMEOUT_MS = 10000;
|
||||||
const FINAL_TIMEOUT_MS = 12000;
|
const FINAL_TIMEOUT_MS = 12000;
|
||||||
@@ -48,6 +54,8 @@ export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETR
|
|||||||
|
|
||||||
type Segment = {
|
type Segment = {
|
||||||
chunks: Uint8Array[];
|
chunks: Uint8Array[];
|
||||||
|
/** RMS of each chunk. */
|
||||||
|
levels: number[];
|
||||||
bytes: number;
|
bytes: number;
|
||||||
peakRms: number;
|
peakRms: number;
|
||||||
text: string;
|
text: string;
|
||||||
@@ -56,6 +64,8 @@ type Segment = {
|
|||||||
paragraphBreakAfter: boolean;
|
paragraphBreakAfter: boolean;
|
||||||
/** Emit a final even when empty: the end of push-to-talk needs one. */
|
/** Emit a final even when empty: the end of push-to-talk needs one. */
|
||||||
required: boolean;
|
required: boolean;
|
||||||
|
/** Cut for length: its final leaves joined with the next segment's. */
|
||||||
|
continues: boolean;
|
||||||
interimInFlight: boolean;
|
interimInFlight: boolean;
|
||||||
interimBytes: number;
|
interimBytes: number;
|
||||||
};
|
};
|
||||||
@@ -120,10 +130,13 @@ export class WhisperServerSttClient implements CloudSttClient {
|
|||||||
acceptPcm(pcm: Uint8Array): void {
|
acceptPcm(pcm: Uint8Array): void {
|
||||||
if (this.closed || !pcm.length) return;
|
if (this.closed || !pcm.length) return;
|
||||||
const segment = this.openSegment();
|
const segment = this.openSegment();
|
||||||
|
const level = rms(pcm);
|
||||||
segment.chunks.push(pcm.slice());
|
segment.chunks.push(pcm.slice());
|
||||||
|
segment.levels.push(level);
|
||||||
segment.bytes += pcm.length;
|
segment.bytes += pcm.length;
|
||||||
segment.peakRms = Math.max(segment.peakRms, rms(pcm));
|
segment.peakRms = Math.max(segment.peakRms, level);
|
||||||
this.maybeRequestInterim(segment);
|
if (segment.bytes >= SEGMENT_MAX_BYTES) this.splitSegment(segment);
|
||||||
|
this.maybeRequestInterim(this.segments[this.segments.length - 1]!);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A pause in continuous capture: transcribe what came before it. */
|
/** A pause in continuous capture: transcribe what came before it. */
|
||||||
@@ -145,16 +158,54 @@ export class WhisperServerSttClient implements CloudSttClient {
|
|||||||
const last = this.segments[this.segments.length - 1];
|
const last = this.segments[this.segments.length - 1];
|
||||||
if (last && !last.closed) return last;
|
if (last && !last.closed) return last;
|
||||||
const segment: Segment = {
|
const segment: Segment = {
|
||||||
chunks: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
|
chunks: [], levels: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false,
|
||||||
paragraphBreakAfter: false, required: false, interimInFlight: false, interimBytes: 0,
|
paragraphBreakAfter: false, required: false, continues: false, interimInFlight: false, interimBytes: 0,
|
||||||
};
|
};
|
||||||
this.segments.push(segment);
|
this.segments.push(segment);
|
||||||
return segment;
|
return segment;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Close the open segment at SEGMENT_MAX_BYTES and carry the audio after its
|
||||||
|
* cut into a new one. The cut is after the quietest chunk among the last
|
||||||
|
* SPLIT_SEARCH_BYTES; the first chunk always stays.
|
||||||
|
*/
|
||||||
|
private splitSegment(segment: Segment): void {
|
||||||
|
let cut = segment.chunks.length - 1;
|
||||||
|
let quietest = Infinity;
|
||||||
|
let carried = 0;
|
||||||
|
for (let i = segment.chunks.length - 1; i > 0 && carried <= SPLIT_SEARCH_BYTES; i--) {
|
||||||
|
if (segment.levels[i]! < quietest) {
|
||||||
|
quietest = segment.levels[i]!;
|
||||||
|
cut = i;
|
||||||
|
}
|
||||||
|
carried += segment.chunks[i]!.length;
|
||||||
|
}
|
||||||
|
const chunks = segment.chunks.splice(cut + 1);
|
||||||
|
const levels = segment.levels.splice(cut + 1);
|
||||||
|
for (const chunk of chunks) segment.bytes -= chunk.length;
|
||||||
|
segment.peakRms = peak(segment.levels);
|
||||||
|
segment.continues = true;
|
||||||
|
this.closeSegment(false, false);
|
||||||
|
if (!chunks.length) return;
|
||||||
|
const next = this.openSegment();
|
||||||
|
next.chunks = chunks;
|
||||||
|
next.levels = levels;
|
||||||
|
for (const chunk of chunks) next.bytes += chunk.length;
|
||||||
|
next.peakRms = peak(levels);
|
||||||
|
}
|
||||||
|
|
||||||
private closeSegment(paragraphBreakAfter: boolean, required: boolean): void {
|
private closeSegment(paragraphBreakAfter: boolean, required: boolean): void {
|
||||||
if (this.closed) return;
|
if (this.closed) return;
|
||||||
const last = this.segments[this.segments.length - 1];
|
const last = this.segments[this.segments.length - 1];
|
||||||
|
if (last?.closed && last.continues) {
|
||||||
|
// Cut for length with nothing after it yet: this boundary ends its run.
|
||||||
|
last.continues = false;
|
||||||
|
last.paragraphBreakAfter = paragraphBreakAfter;
|
||||||
|
last.required = required;
|
||||||
|
if (last.final) this.emit();
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!last || last.closed) {
|
if (!last || last.closed) {
|
||||||
// Nothing new since the last boundary. Push-to-talk still needs a final.
|
// Nothing new since the last boundary. Push-to-talk still needs a final.
|
||||||
if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required);
|
if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required);
|
||||||
@@ -192,7 +243,7 @@ export class WhisperServerSttClient implements CloudSttClient {
|
|||||||
|
|
||||||
private maybeRequestInterim(segment: Segment): void {
|
private maybeRequestInterim(segment: Segment): void {
|
||||||
if (!this.ready || segment.interimInFlight || segment.closed) return;
|
if (!this.ready || segment.interimInFlight || segment.closed) return;
|
||||||
if (segment.bytes < INTERIM_MIN_BYTES || segment.bytes > INTERIM_MAX_BYTES) return;
|
if (segment.bytes < INTERIM_MIN_BYTES) return;
|
||||||
if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return;
|
if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return;
|
||||||
if (segment.peakRms < SILENCE_RMS) return;
|
if (segment.peakRms < SILENCE_RMS) return;
|
||||||
segment.interimInFlight = true;
|
segment.interimInFlight = true;
|
||||||
@@ -221,14 +272,20 @@ export class WhisperServerSttClient implements CloudSttClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Finals leave in audio order, even when a later request answers first;
|
* Finals leave in audio order, even when a later request answers first, and
|
||||||
* whatever is not final yet is shown as one provisional transcript.
|
* segments cut for length leave as one final with the segment that ends
|
||||||
|
* their run; whatever is not final yet is shown as one provisional transcript.
|
||||||
*/
|
*/
|
||||||
private emit(): void {
|
private emit(): void {
|
||||||
while (this.segments.length && this.segments[0]!.final) {
|
for (;;) {
|
||||||
const segment = this.segments.shift()!;
|
let end = 0;
|
||||||
if (!segment.text && !segment.required) continue;
|
while (this.segments[end]?.final && this.segments[end]!.continues) end++;
|
||||||
this.options.onTranscript({ text: segment.text, isFinal: true, paragraphBreakAfter: segment.paragraphBreakAfter });
|
if (!this.segments[end]?.final) break;
|
||||||
|
const run = this.segments.splice(0, end + 1);
|
||||||
|
const last = run[run.length - 1]!;
|
||||||
|
const text = run.map((segment) => segment.text.trim()).filter(Boolean).join(" ");
|
||||||
|
if (!text && !last.required) continue;
|
||||||
|
this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: last.paragraphBreakAfter });
|
||||||
}
|
}
|
||||||
const pending = this.segments.filter((segment) => segment.text.trim());
|
const pending = this.segments.filter((segment) => segment.text.trim());
|
||||||
if (!pending.length) return;
|
if (!pending.length) return;
|
||||||
@@ -291,6 +348,12 @@ function firstModelId(body: string): string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function peak(levels: number[]): number {
|
||||||
|
let max = 0;
|
||||||
|
for (const level of levels) max = Math.max(max, level);
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
|
||||||
/** RMS of a PCM16LE chunk. */
|
/** RMS of a PCM16LE chunk. */
|
||||||
function rms(pcm: Uint8Array): number {
|
function rms(pcm: Uint8Array): number {
|
||||||
const count = pcm.length >> 1;
|
const count = pcm.length >> 1;
|
||||||
@@ -367,16 +430,23 @@ function describeError(error: unknown): string {
|
|||||||
return String((error as Error)?.message ?? error);
|
return String((error as Error)?.message ?? error);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** NativeScript's HTTP module: sends an ArrayBuffer body as-is on both platforms. */
|
/**
|
||||||
|
* Uploads go through postBinary (on Android NativeScript's HTTP module would
|
||||||
|
* copy each body into Java a byte at a time); the model list through
|
||||||
|
* NativeScript's HTTP module.
|
||||||
|
*/
|
||||||
const nativeHttp: WhisperHttp = async (request) => {
|
const nativeHttp: WhisperHttp = async (request) => {
|
||||||
// Required lazily so Node tests can load this module without NativeScript.
|
// Required lazily so Node tests can load this module without NativeScript.
|
||||||
|
if (request.body) {
|
||||||
|
const { postBinary } = require("./binary-http") as typeof import("./binary-http");
|
||||||
|
return postBinary(request.url, request.body, request.headers, request.timeoutMs);
|
||||||
|
}
|
||||||
const { withUserAgent } = require("../util/http") as typeof import("../util/http");
|
const { withUserAgent } = require("../util/http") as typeof import("../util/http");
|
||||||
const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http");
|
const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http");
|
||||||
const response = await send({
|
const response = await send({
|
||||||
url: request.url,
|
url: request.url,
|
||||||
method: request.method,
|
method: request.method,
|
||||||
headers: withUserAgent(request.headers),
|
headers: withUserAgent(request.headers),
|
||||||
content: request.body,
|
|
||||||
timeout: request.timeoutMs,
|
timeout: request.timeoutMs,
|
||||||
});
|
});
|
||||||
const content = response.content as any;
|
const content = response.content as any;
|
||||||
|
|||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
package com.faceclaw.app
|
||||||
|
|
||||||
|
/** Callbacks for FaceclawBinaryPost, delivered on the constructing thread's Looper. */
|
||||||
|
interface FaceclawBinaryPostListener {
|
||||||
|
/** The server answered, with any status; body is the response text. */
|
||||||
|
fun onResponse(status: Int, body: String?): Unit
|
||||||
|
|
||||||
|
/** Network-level failure (connect, TLS, timeout, mid-body drop); no response arrived. */
|
||||||
|
fun onFailure(message: String?): Unit
|
||||||
|
}
|
||||||
+12
-3
@@ -6,7 +6,11 @@ any OpenAI-compatible Whisper server instead of a cloud service: vLLM, speaches
|
|||||||
`POST {url}/audio/transcriptions` with a 16 kHz mono WAV. Push-to-talk
|
`POST {url}/audio/transcriptions` with a 16 kHz mono WAV. Push-to-talk
|
||||||
utterances are one request each; in Transcribe, each span ended by a pause is
|
utterances are one request each; in Transcribe, each span ended by a pause is
|
||||||
one request. While a span is open its audio so far is re-sent every 1.5 s, so
|
one request. While a span is open its audio so far is re-sent every 1.5 s, so
|
||||||
text appears while you are still talking.
|
text appears while you are still talking. A span that runs 25 s without a pause
|
||||||
|
(nonstop speech, music) is cut at its quietest moment in the last 5 s, so no
|
||||||
|
request outgrows Whisper's 30 s window; the pieces still arrive as one final.
|
||||||
|
On Android the WAV goes out through `FaceclawBinaryPost` (okhttp), since
|
||||||
|
NativeScript's http module copies a binary body into Java a byte at a time.
|
||||||
|
|
||||||
## Server
|
## Server
|
||||||
|
|
||||||
@@ -39,6 +43,11 @@ before building:
|
|||||||
## Transcribe on iOS
|
## Transcribe on iOS
|
||||||
|
|
||||||
Transcribe needs a cloud or Whisper-server provider on iOS (Apple's recognizer
|
Transcribe needs a cloud or Whisper-server provider on iOS (Apple's recognizer
|
||||||
only handles short utterances). With "Save Transcribe audio" on (Voice
|
only handles short utterances). Its capture follows the glasses: a disconnect,
|
||||||
|
a glasses lock or backgrounding (for phone-mic capture) transcribes what was
|
||||||
|
already heard, and listening resumes once the glasses reconnect, unlock or the
|
||||||
|
app returns to the foreground. Without a glasses session the phone mic stands in
|
||||||
|
while Faceclaw is in the foreground. With "Save Transcribe audio" on (Voice
|
||||||
settings), each session's audio is kept as `transcribe-<time>.wav` in Downloads,
|
settings), each session's audio is kept as `transcribe-<time>.wav` in Downloads,
|
||||||
next to `transcript-<time>.txt` when the text is saved.
|
next to `transcript-<time>.txt` when the text is saved; a resumed capture keeps
|
||||||
|
appending to the same file.
|
||||||
@@ -338,6 +338,81 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
|
|||||||
assert.deepEqual(calls, []);
|
assert.deepEqual(calls, []);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
|
||||||
|
const calls = [], claims = [];
|
||||||
|
// Models the bridge's continuous state: listening on a source, or draining after a stop.
|
||||||
|
const bridge = { source: null, draining: false, recording: '',
|
||||||
|
supportsContinuous: voice => voice.provider !== 'onboard',
|
||||||
|
get continuousSource() { return this.source; },
|
||||||
|
get continuousActive() { return !!this.source || this.draining; },
|
||||||
|
prepare: async () => true,
|
||||||
|
startGlassesCapture: async (_session, _voice, _log, _endpointing, continuous) => { calls.push(['glasses', continuous]); bridge.source = 'glasses'; },
|
||||||
|
startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; },
|
||||||
|
drain() { if (this.source) { this.source = null; this.draining = true; } },
|
||||||
|
stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); },
|
||||||
|
handleSessionEnded() { if (bridge.source === 'glasses') bridge.drain(); },
|
||||||
|
stopPhoneCapture() { if (bridge.source === 'phone') bridge.drain(); },
|
||||||
|
recordTo: name => { bridge.recording = name; } };
|
||||||
|
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||||
|
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, saveTranscribeAudioSetting: { get: () => true } };
|
||||||
|
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||||
|
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||||
|
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } : {},
|
||||||
|
});
|
||||||
|
const host = Object.create(IosPreviewController.prototype);
|
||||||
|
Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true,
|
||||||
|
continuousClients: new Map(), continuousPending: false, continuousCaptureToken: 0, displayReady: Promise.resolve(),
|
||||||
|
glance: { dismiss() {} }, onError: message => calls.push(['error', message]), logBluetooth() {}, fail() {}, requestShellRender() {},
|
||||||
|
syncRuntime() {}, flushBluetoothLog() {} });
|
||||||
|
const settle = () => new Promise(resolve => setImmediate(resolve));
|
||||||
|
// The bridge's end-of-capture callback, once a drain has finished.
|
||||||
|
const drained = () => { bridge.draining = false; host.syncContinuousCapture(); };
|
||||||
|
const session = phase => { host.state = { phase }; if (phase !== 'connected') bridge.handleSessionEnded(); host.syncContinuousSession(); };
|
||||||
|
|
||||||
|
host.startContinuousVoiceCapture('transcribe', 'transcribe-1'); await settle();
|
||||||
|
host.startContinuousVoiceCapture('calculator'); await settle();
|
||||||
|
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
|
||||||
|
assert.equal(bridge.recording, 'transcribe-1');
|
||||||
|
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
|
||||||
|
host.stopContinuousVoiceCapture('calculator');
|
||||||
|
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
|
||||||
|
|
||||||
|
// A reconnect drains the capture, waits for the glasses, and resumes on them.
|
||||||
|
calls.length = 0; session('retrying'); drained(); await settle();
|
||||||
|
assert.deepEqual(calls, []);
|
||||||
|
assert.deepEqual(claims.at(-1), [false, 'continuous-capture'], 'other mic users may run meanwhile');
|
||||||
|
session('connected'); await settle();
|
||||||
|
assert.deepEqual(calls, [['glasses', true]]);
|
||||||
|
|
||||||
|
// Locking drains; unlocking resumes.
|
||||||
|
calls.length = 0; host.setGlassesLocked(true); drained(); await settle();
|
||||||
|
assert.deepEqual(calls, []);
|
||||||
|
host.setGlassesLocked(false); await settle();
|
||||||
|
assert.deepEqual(calls, [['glasses', true]]);
|
||||||
|
|
||||||
|
// A failed capture is not retried in a loop, only on the next session.
|
||||||
|
calls.length = 0; bridge.source = null; drained(); await settle();
|
||||||
|
assert.deepEqual(calls, []);
|
||||||
|
|
||||||
|
// Without glasses the phone mic stands in while in the foreground, and moves to the glasses when they connect.
|
||||||
|
host.communicator = null; session('disconnected'); await settle();
|
||||||
|
assert.deepEqual(calls, [['phone', true]]);
|
||||||
|
calls.length = 0; host.pause(); drained(); await settle();
|
||||||
|
assert.deepEqual(calls, [], 'no phone capture in the background');
|
||||||
|
host.active = true; host.syncContinuousCapture(); await settle();
|
||||||
|
assert.deepEqual(calls, [['phone', true]]);
|
||||||
|
calls.length = 0; host.communicator = {}; session('connected'); await settle();
|
||||||
|
assert.deepEqual(calls, [['stop']]);
|
||||||
|
drained(); await settle();
|
||||||
|
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
|
||||||
|
|
||||||
|
// The last window closing stops it for good.
|
||||||
|
calls.length = 0; host.stopContinuousVoiceCapture('transcribe'); drained(); session('retrying'); session('connected'); await settle();
|
||||||
|
assert.deepEqual(calls, [['stop']]);
|
||||||
|
assert.equal(bridge.recording, '');
|
||||||
|
assert.deepEqual(claims.at(-1), [false, 'continuous-capture']);
|
||||||
|
});
|
||||||
|
|
||||||
test('iOS keyboard action opens the shared destination session and remains blocked while locked', () => {
|
test('iOS keyboard action opens the shared destination session and remains blocked while locked', () => {
|
||||||
let opened = 0;
|
let opened = 0;
|
||||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ function fixture() {
|
|||||||
"../native/compass.ios": { bindCompassSession() {}, receiveCompassEvent() {} },
|
"../native/compass.ios": { bindCompassSession() {}, receiveCompassEvent() {} },
|
||||||
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
|
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
|
||||||
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
||||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
|
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {}, onSpeechEnd: () => () => {} } },
|
||||||
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
|
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
|
||||||
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
|
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
|
||||||
'../native/preview-display.ios': { PreviewDisplayTarget },
|
'../native/preview-display.ios': { PreviewDisplayTarget },
|
||||||
|
|||||||
+59
-10
@@ -260,18 +260,67 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
|
|||||||
assert.equal(h.clouds[1].continuous(), false);
|
assert.equal(h.clouds[1].continuous(), false);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a continuous capture asked to record writes every PCM chunk to its WAV and closes it when the capture ends', async () => {
|
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
|
||||||
const h = harness();
|
const h = harness();
|
||||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true, 'transcribe-2026-10-17-120000'); h.release(); await start;
|
h.bridge.recordTo('transcribe-2026-10-17-120000');
|
||||||
assert.equal(h.recordings.length, 1);
|
// A start that fails before any audio leaves no file behind.
|
||||||
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
|
h.native.error = 'No phone microphone is available.';
|
||||||
|
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
||||||
|
delete h.native.error;
|
||||||
|
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||||
|
assert.equal(h.recordings.length, 0);
|
||||||
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
|
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
|
||||||
h.native.pcmHandler(new Uint8Array([5, 6]));
|
h.native.pcmHandler(new Uint8Array([5, 6]));
|
||||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
|
|
||||||
assert.deepEqual(h.clouds[0].pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
|
|
||||||
h.bridge.stop();
|
|
||||||
assert.equal(h.recordings[0].closed, true);
|
|
||||||
// Captures that don't ask (push-to-talk, Teleprompter) are not recorded.
|
|
||||||
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
|
|
||||||
assert.equal(h.recordings.length, 1);
|
assert.equal(h.recordings.length, 1);
|
||||||
|
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
|
||||||
|
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
|
||||||
|
assert.deepEqual(h.clouds.at(-1).pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
|
||||||
|
// A restarted capture (after a disconnect, say) keeps appending to the same file.
|
||||||
|
h.bridge.stop();
|
||||||
|
assert.equal(h.recordings[0].closed, false);
|
||||||
|
const again = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await again;
|
||||||
|
h.native.pcmHandler(new Uint8Array([7]));
|
||||||
|
assert.equal(h.recordings.length, 1);
|
||||||
|
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
||||||
|
// Push-to-talk on a capture of its own is not recorded.
|
||||||
|
h.bridge.stop();
|
||||||
|
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
|
||||||
|
h.native.pcmHandler(new Uint8Array([8]));
|
||||||
|
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
||||||
|
h.bridge.recordTo('');
|
||||||
|
assert.equal(h.recordings[0].closed, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
|
||||||
|
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||||
|
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||||
|
const [cloud] = h.clouds;
|
||||||
|
assert.equal(h.bridge.continuousSource, 'glasses');
|
||||||
|
h.bridge.handleSessionEnded();
|
||||||
|
assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
|
||||||
|
assert.equal(cloud.stopped, false, 'the open segment is still transcribed');
|
||||||
|
assert.equal(h.bridge.continuousSource, null); assert.equal(h.bridge.continuousActive, true);
|
||||||
|
// The controller reports the session state again while it reconnects.
|
||||||
|
h.bridge.handleSessionEnded();
|
||||||
|
h.event({ kind: 'ended', message: 'Ready to send' });
|
||||||
|
assert.equal(cloud.finished, 1);
|
||||||
|
assert.equal(ends, 0);
|
||||||
|
cloud.options.onTranscript({ text: 'Last words.', isFinal: true });
|
||||||
|
assert.equal(cloud.stopped, true); assert.equal(h.bridge.continuousActive, false);
|
||||||
|
assert.equal(h.text.at(-1).text, 'Last words.');
|
||||||
|
assert.match(h.bridge.statusText, /resumes when they reconnect/);
|
||||||
|
assert.equal(ends, 1, 'the end lets the controller restart the capture');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('backgrounding drains a continuous phone capture', async () => {
|
||||||
|
const h = harness();
|
||||||
|
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
||||||
|
assert.equal(h.bridge.continuousSource, 'phone');
|
||||||
|
h.bridge.stopPhoneCapture();
|
||||||
|
assert.equal(h.native.finished, true); assert.equal(h.clouds[0].stopped, false);
|
||||||
|
h.event({ kind: 'ended', message: 'Ready to send' });
|
||||||
|
h.clouds[0].options.onTranscript({ text: 'Tail.', isFinal: true });
|
||||||
|
assert.equal(h.bridge.continuousActive, false);
|
||||||
|
assert.equal(h.text.at(-1).text, 'Tail.');
|
||||||
|
assert.match(h.bridge.statusText, /background/);
|
||||||
});
|
});
|
||||||
@@ -141,6 +141,42 @@ test('continuous segments finalize in audio order even when answered out of orde
|
|||||||
[['first', true], ['second', true]]);
|
[['first', true], ['second', true]]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/** Seconds of audio in a transcription request's WAV part. */
|
||||||
|
function postSeconds(post) {
|
||||||
|
const body = Buffer.from(post.body);
|
||||||
|
return body.readUInt32LE(body.indexOf('RIFF') + 40) / 32000;
|
||||||
|
}
|
||||||
|
|
||||||
|
test('nonstop speech is cut at its quietest moment into bounded requests that still leave as one final', async () => {
|
||||||
|
const h = client();
|
||||||
|
h.c.start(); await flush();
|
||||||
|
// 25 s of 50 ms chunks with no pause; chunk 440 (22 s in) is the quietest.
|
||||||
|
for (let i = 0; i < 500; i++) h.c.acceptPcm(speech(50, i === 440 ? 400 : 3000));
|
||||||
|
const [, head, tailInterim] = h.s.posts();
|
||||||
|
assert.equal(postSeconds(head), 22.05, 'the cut follows the quietest chunk');
|
||||||
|
assert.equal(postSeconds(tailInterim), 2.95, 'the audio after it carries over');
|
||||||
|
h.c.acceptPcm(speech(1000)); h.c.finish();
|
||||||
|
const tail = h.s.posts().at(-1);
|
||||||
|
assert.equal(postSeconds(tail), 3.95);
|
||||||
|
tail.resolve({ status: 200, text: '{"text":"and the rest."}' }); await flush();
|
||||||
|
assert.deepEqual(h.events.filter(event => event.isFinal), [], 'the tail waits for the head');
|
||||||
|
head.resolve({ status: 200, text: '{"text":"The long part"}' }); await flush();
|
||||||
|
assert.deepEqual(h.events.filter(event => event.isFinal),
|
||||||
|
[{ text: 'The long part and the rest.', isFinal: true, paragraphBreakAfter: false }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a cut with nothing after it yet ends at the next boundary', async () => {
|
||||||
|
const h = client();
|
||||||
|
h.c.start(); await flush();
|
||||||
|
for (let i = 0; i < 500; i++) h.c.acceptPcm(speech(50, i === 499 ? 400 : 3000));
|
||||||
|
const head = h.s.posts().at(-1);
|
||||||
|
assert.equal(postSeconds(head), 25);
|
||||||
|
head.resolve({ status: 200, text: '{"text":"All of it."}' }); await flush();
|
||||||
|
assert.deepEqual(h.events.filter(event => event.isFinal), []);
|
||||||
|
h.c.commitSegment();
|
||||||
|
assert.deepEqual(h.events.filter(event => event.isFinal), [{ text: 'All of it.', isFinal: true, paragraphBreakAfter: true }]);
|
||||||
|
});
|
||||||
|
|
||||||
test('a failed final is retried once, then reported without ending the capture', async () => {
|
test('a failed final is retried once, then reported without ending the capture', async () => {
|
||||||
const h = client();
|
const h = client();
|
||||||
h.c.start(); await flush();
|
h.c.start(); await flush();
|
||||||
|
|||||||
Reference in new issue
Block a user