Use iOS SpeechAnalyzer API to support on-device transcription with the Transcribe app

This commit is contained in:
James Babcock committed 2026-10-31 17:03:56 -07:00
1 parent 210b111e27
commit 6e5864f981
6 files changed
+709 -18

No files matched your search

@@ -0,0 +1,273 @@
@preconcurrency import AVFoundation
import CoreMedia
import Foundation
import Speech
/// Long-form on-device transcription with SpeechAnalyzer and SpeechTranscriber
/// (iOS 26+), for continuous capture (Transcribe, Teleprompter, Calculator):
/// FaceclawSpeech's SFSpeechRecognizer only handles short dictations. It takes
/// the same 16 kHz mono PCM16LE the cloud providers get (see
/// app/native/speech-analyzer-stt.ios.ts) and reports JSON events on the main
/// queue through eventHandler:
///
/// {"kind":"status","message":String}
/// {"kind":"ready","locale":String} audio is accepted from here on
/// {"kind":"transcript","text":String,"final":Bool,
/// "start":Double?,"end":Double?, the result's audio range
/// "speechEnd":Double?} when its last word ends
/// {"kind":"finished"} after finish(), once the last result is in
/// {"kind":"error","message":String}
///
/// Times are seconds of audio since start. Volatile results (final: false)
/// cover all audio not finalized yet, replacing the previous volatile one;
/// their word times are only as fine as the result's range. A final's first
/// word reaches back into the silence before it, so only word ends are
/// reported.
///
/// There is no way to settle text at a pause: on macOS 26.4 any mid-stream
/// SpeechAnalyzer.finalize(through:) left the next phrase's final as dots,
/// while the model splits its finals at pauses on its own.
/// Called from JavaScript, so on the main thread.
@MainActor
@objc(FaceclawSpeechAnalyzer)
@objcMembers
public class FaceclawSpeechAnalyzer: NSObject {
public var eventHandler: ((String) -> Void)?
// An AnalyzerEngine, typed loosely so the class itself loads before iOS 26.
private var engine: AnyObject?
/// Whether this device can transcribe with SpeechTranscriber (iOS 26 and supported hardware).
public static func isSupported() -> Bool {
if #available(iOS 26.0, *) { return SpeechTranscriber.isAvailable }
return false
}
/// `language` is a locale identifier ("en", "en-US"), or "" for the device's preferred language.
public func start(language: String) {
cancel()
guard #available(iOS 26.0, *) else {
emit(["kind": "error", "message": "On-device transcription needs iOS 26 or later."])
return
}
let engine = AnalyzerEngine { [weak self] event in self?.emit(event) }
self.engine = engine
engine.start(language: language)
}
/// 16 kHz mono PCM16LE; ignored until "ready".
public func acceptPcm(_ pcm: Data) {
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.accept(pcm) }
}
/// End of audio: finalize the rest, then report "finished".
public func finish() {
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.finish() }
}
/// Abandon the session without further events.
public func cancel() {
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.cancel() }
engine = nil
}
private func emit(_ event: [String: Any]) {
// JSONSerialization raises (not throws) on NaN and other invalid values.
guard JSONSerialization.isValidJSONObject(event),
let data = try? JSONSerialization.data(withJSONObject: event),
let json = String(data: data, encoding: .utf8) else { return }
eventHandler?(json)
}
}
@available(iOS 26.0, *)
@MainActor
private final class AnalyzerEngine {
private static let sampleRate = 16000.0
private let emit: ([String: Any]) -> Void
private let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: AnalyzerEngine.sampleRate, channels: 1, interleaved: true)!
private var analyzer: SpeechAnalyzer?
private var input: AsyncStream<AnalyzerInput>.Continuation?
private var analyzerFormat: AVAudioFormat?
// Nil when the analyzer takes the input format as it is.
private var converter: AVAudioConverter?
private var ready = false
private var finishing = false
private var closed = false
private var setup: Task<Void, Never>?
private var results: Task<Void, Never>?
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
func start(language: String) {
setup = Task { [weak self] in await self?.prepare(language: language) }
}
/// Events stop once the session is cancelled, even from work still winding down.
private func send(_ event: [String: Any]) {
if !closed { emit(event) }
}
private func prepare(language: String) async {
send(["kind": "status", "message": "Preparing on-device transcription…"])
let identifier = language.isEmpty ? (Locale.preferredLanguages.first ?? "en-US") : language
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: identifier)) else {
fail("On-device transcription doesn't support \(identifier). Choose a cloud or Whisper server transcription provider (Settings > Voice).")
return
}
if closed { return }
let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [],
reportingOptions: [.volatileResults], attributeOptions: [.audioTimeRange])
do {
// Keep the model installed between sessions (best effort).
if !(await AssetInventory.reservedLocales).contains(locale) {
_ = try? await AssetInventory.reserve(locale: locale)
}
if closed { return }
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
if closed { return }
try await download(request)
}
if closed { return }
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else {
fail("On-device transcription has no usable audio format.")
return
}
if closed { return }
if format != pcmFormat {
guard let converter = AVAudioConverter(from: pcmFormat, to: format) else {
fail("On-device transcription can't convert the microphone audio.")
return
}
self.converter = converter
}
analyzerFormat = format
let analyzer = SpeechAnalyzer(modules: [transcriber])
let (stream, input) = AsyncStream<AnalyzerInput>.makeStream()
self.analyzer = analyzer
self.input = input
results = Task { [weak self] in await self?.collect(transcriber) }
try await analyzer.prepareToAnalyze(in: format)
try await analyzer.start(inputSequence: stream)
if closed { return }
ready = true
send(["kind": "ready", "locale": locale.identifier])
if finishing { await finishAnalysis() }
} catch {
fail("On-device transcription: \(error.localizedDescription)")
}
}
/// Fetch the locale's model, reporting progress while it downloads.
private func download(_ request: AssetInstallationRequest) async throws {
let progress = request.progress
let reporter = Task { [weak self] in
while !Task.isCancelled {
let percent = Int((progress.fractionCompleted * 100).rounded())
self?.send(["kind": "status", "message": "Downloading on-device speech model… \(percent)%"])
try? await Task.sleep(nanoseconds: 1_000_000_000)
}
}
defer { reporter.cancel() }
try await request.downloadAndInstall()
}
func accept(_ pcm: Data) {
guard ready, !finishing, !closed, let input, let analyzerFormat else { return }
let frames = AVAudioFrameCount(pcm.count / 2)
guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames),
let channel = buffer.int16ChannelData else { return }
buffer.frameLength = frames
pcm.withUnsafeBytes { raw in
if let base = raw.baseAddress { memcpy(channel[0], base, Int(frames) * 2) }
}
guard let converter else {
input.yield(AnalyzerInput(buffer: buffer))
return
}
let capacity = AVAudioFrameCount((Double(frames) * analyzerFormat.sampleRate / AnalyzerEngine.sampleRate).rounded(.up)) + 32
guard let converted = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return }
var consumed = false
var error: NSError?
// noDataNow, not endOfStream: the converter keeps its resampling state for the next chunk.
converter.convert(to: converted, error: &error) { _, status in
if consumed {
status.pointee = .noDataNow
return nil
}
consumed = true
status.pointee = .haveData
return buffer
}
if error == nil && converted.frameLength > 0 { input.yield(AnalyzerInput(buffer: converted)) }
}
func finish() {
guard !finishing, !closed else { return }
finishing = true
// Before "ready", prepare() finishes once the analyzer is running.
if ready { Task { await finishAnalysis() } }
}
private func finishAnalysis() async {
input?.finish()
input = nil
do {
try await analyzer?.finalizeAndFinishThroughEndOfInput()
} catch {
fail("On-device transcription: \(error.localizedDescription)")
}
// collect() reports "finished" once the results sequence ends.
}
func cancel() {
guard !closed else { return }
closed = true
input?.finish()
input = nil
setup?.cancel()
results?.cancel()
if let analyzer { Task { await analyzer.cancelAndFinishNow() } }
analyzer = nil
}
private func collect(_ transcriber: SpeechTranscriber) async {
do {
for try await result in transcriber.results {
if closed { return }
send(event(for: result))
}
send(["kind": "finished"])
} catch {
fail("On-device transcription: \(error.localizedDescription)")
}
}
private func event(for result: SpeechTranscriber.Result) -> [String: Any] {
let text = result.text
var speechEnd: Double?
for run in text.runs {
guard let range = run.audioTimeRange,
!String(text[run.range].characters).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue }
speechEnd = finite(range.end) ?? speechEnd
}
return [
"kind": "transcript",
"text": String(text.characters),
"final": result.isFinal,
"start": finite(result.range.start) ?? NSNull(),
"end": finite(result.range.end) ?? NSNull(),
"speechEnd": speechEnd ?? NSNull(),
]
}
private func finite(_ time: CMTime) -> Double? {
let seconds = time.seconds
return seconds.isFinite ? seconds : nil
}
private func fail(_ message: String) {
guard !closed else { return }
emit(["kind": "error", "message": message])
cancel()
}
}
+1 -1
View File
@@ -1054,7 +1054,7 @@ export class IosPreviewController {
this.continuousClients.set(client, options)
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
this.continuousClients.delete(client)
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
return
}
this.syncContinuousRecording()
+193
View File
@@ -0,0 +1,193 @@
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
declare const FaceclawSpeechAnalyzer: any;
/**
* How long finish() waits for the last results: the model only has to settle
* the few seconds of audio it hadn't finalized yet.
*/
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
/**
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
* whose last word ends within this much before that (the model's word ends
* against the detector's level threshold) leaves nothing before the pause
* unfinalized, so its paragraph is complete.
*/
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
type AnalyzerEvent = {
kind: string;
message?: string;
locale?: string;
text?: string;
final?: boolean;
start?: number | null;
end?: number | null;
speechEnd?: number | null;
};
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
export function speechAnalyzerSupported(): boolean {
try {
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
} catch {
return false;
}
}
/**
* On-device long-form transcription for continuous capture, wrapped like a
* cloud provider: it buffers audio until the model is ready (downloading it on
* first use) and gets pause commits.
*/
export function createSpeechAnalyzerClient(
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): ReconnectingSttClient {
return new ReconnectingSttClient(
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
continuous,
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
);
}
/**
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
* The model finalizes text in its own pieces, a few seconds behind, and can't
* be made to settle at a pause; this gathers its pieces into paragraphs ended
* by pauses, like the cloud providers' segments. A paragraph is shown as
* provisional text and sent as one final (paragraphBreakAfter) once a final
* reaches its pause, and the last one as a plain final when the audio ends.
*
* Only finals place breaks, by when their last word ends (the model's
* word-end times are accurate; first words' starts reach back into the
* silence before them, and volatile results only carry the range of all audio
* not finalized yet).
*/
export class SpeechAnalyzerSttClient implements CloudSttClient {
private native: any = null;
private samples = 0;
private finishing = false;
// Audio times (s) of pauses whose paragraph hasn't ended yet.
private pauses: number[] = [];
// Finalized text since the last paragraph break, when its last word ends,
// and the model's provisional rest.
private paragraph: string[] = [];
private paragraphEnd: number | null = null;
private volatile = "";
constructor(private readonly options: CloudSttOptions) {}
start(): void {
if (this.native) return;
const native = FaceclawSpeechAnalyzer.new();
native.eventHandler = (json: string) => {
if (this.native === native) this.receive(JSON.parse(json));
};
this.native = native;
native.startWithLanguage(this.options.language ?? "");
}
acceptPcm(pcm: Uint8Array): void {
if (!this.native || this.finishing || !pcm.length) return;
this.samples += pcm.length >> 1;
const copy = pcm.slice();
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
}
/** A pause: it ends the paragraph once the words before it are final. */
commitSegment(): void {
if (!this.native || this.finishing) return;
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
}
finish(): void {
if (!this.native || this.finishing) return;
this.finishing = true;
this.native.finish();
}
stop(): void {
const native = this.native;
this.native = null;
native?.cancel();
}
private receive(event: AnalyzerEvent): void {
switch (event.kind) {
case "status":
this.options.onStatus(event.message ?? "");
break;
case "ready":
this.options.onStatus("Listening (on-device)...");
this.options.onReady?.();
break;
case "transcript":
this.transcript(event);
break;
case "finished": {
const text = this.liveText();
this.paragraph = [];
this.volatile = "";
this.stop();
this.options.onTranscript({ text, isFinal: true });
break;
}
case "error":
this.stop();
this.options.onError(event.message ?? "On-device transcription failed.");
break;
}
}
private transcript(event: AnalyzerEvent): void {
const text = (event.text ?? "").trim();
let ended = false;
if (!event.final) {
this.volatile = text;
} else {
this.volatile = "";
if (text) {
const end = event.speechEnd ?? event.end ?? null;
// Words ending after a pause come after it. (One whose words run
// across it, if the model didn't split there, breaks a little early.)
if (end !== null && this.pauses.some((pause) => end > pause)) {
this.pauses = this.pauses.filter((pause) => pause >= end);
this.endParagraph();
}
this.paragraph.push(text);
this.paragraphEnd = end;
ended = this.endCompleteParagraph();
}
}
// The paragraph's final already cleared the provisional text.
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
}
/** End the paragraph if its last word ends just before a pause. */
private endCompleteParagraph(): boolean {
const end = this.paragraphEnd;
if (!this.paragraph.length || end === null) return false;
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
if (!this.pauses.some(reached)) return false;
this.pauses = this.pauses.filter((pause) => !reached(pause));
this.endParagraph();
return true;
}
private endParagraph(): void {
const text = this.paragraph.join(" ");
this.paragraph = [];
this.paragraphEnd = null;
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
}
private liveText(): string {
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
}
}
+32 -14
View File
@@ -2,6 +2,7 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
import { WavRecorder } from './wav-recorder.ios'
import { pcmLevel } from '../util/audio-level'
@@ -18,10 +19,12 @@ const RAW_RETRY_LIMIT = 5
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
* the PCM capture runs open-ended, and pauses commit its segments. When the
* session or the app lifecycle cuts one short it is drained, not discarded;
* restarting it is the controller's call.
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
* is open-ended and has pauses commit its segments, so it needs a cloud
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
* recognizer only handles short dictations. When the session or the app
* lifecycle cuts one short it is drained, not discarded; restarting it is the
* controller's call.
*/
export class IosVoiceControlBridge {
private native: any = null
@@ -40,10 +43,13 @@ export class IosVoiceControlBridge {
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
// on-device continuous capture) owns the transcript; FaceclawSpeech then
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
// provider was asked to finalize; it is the status shown when it does.
private cloud: ReconnectingSttClient | null = null
// What transcribes the current capture, for the log.
private recognition = 'on-device'
private cloudEndStatus: string | null = null
private cloudTimer: ReturnType<typeof setTimeout> | null = null
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
@@ -137,7 +143,7 @@ export class IosVoiceControlBridge {
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.generation || !this.capturing) return
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
this.audioTimer = setTimeout(() => {
this.audioTimer = null
if (generation === this.generation && this.capturing && !this.packets) {
@@ -158,18 +164,27 @@ export class IosVoiceControlBridge {
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = this.startNative(voice, endpointing, true)
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
}
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
const native = this.ensureNative()
const generation = this.generation
const current = () => generation === this.generation
const cloud = createCloudSttClient(voice, {
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
onStatus: status => { if (current()) this.setStatus(status) },
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}, () => this.continuous)
const options = {
language: voice.language,
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
onStatus: (status: string) => { if (current()) this.setStatus(status) },
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}
const continuous = () => this.continuous
let cloud = createCloudSttClient(voice, options, continuous)
this.recognition = cloud ? 'cloud' : 'on-device'
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
if (!cloud && this.continuous && speechAnalyzerSupported()) {
cloud = createSpeechAnalyzerClient(options, continuous)
this.recognition = 'on-device (SpeechAnalyzer)'
}
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
if (error) return error
@@ -182,8 +197,11 @@ export class IosVoiceControlBridge {
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
}
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
/**
* Whether continuous capture can run with this provider: a cloud one, or
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
*/
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
+48 -3
View File
@@ -7,7 +7,7 @@ function harness() {
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; },
startPcmCaptureWithEndpointingPhone(endpointing, phone) { this.pcmCapture = { endpointing, phone }; return this.error ?? ''; },
cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const audio = [], timers = new Map(), commands = [], text = [], clouds = [];
const audio = [], timers = new Map(), commands = [], text = [], clouds = [], analyzers = [];
// Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one.
const provider = {
usesCloudStt: voice => voice.provider !== 'onboard',
@@ -19,6 +19,17 @@ function harness() {
clouds.push(client); return client;
},
};
// Stands in for speech-analyzer-stt: SpeechAnalyzer (iOS 26+) when the test says the device has it.
let analyzerSupported = false;
const analyzer = {
speechAnalyzerSupported: () => analyzerSupported,
createSpeechAnalyzerClient(options, continuous) {
const client = { options, continuous, pcm: [], started: false, finished: 0, stopped: false,
start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); },
finish() { this.finished++; }, stop() { this.stopped = true; } };
analyzers.push(client); return client;
},
};
const recordings = [];
class FakeRecorder {
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
@@ -32,7 +43,8 @@ function harness() {
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
require: name => name === './cloud-stt-provider' ? provider : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
require: name => name === './cloud-stt-provider' ? provider : name === './speech-analyzer-stt.ios' ? analyzer
: name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
const bridge = new sandbox.exports.IosVoiceControlBridge();
@@ -43,7 +55,8 @@ function harness() {
return Promise.resolve();
} };
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
return { bridge, native, session, audio, commands, text, timers, clouds, recordings, release: () => release(),
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
analyzerSupported: value => { analyzerSupported = value; },
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
@@ -248,6 +261,38 @@ test('continuous capture needs a cloud provider, shares its mic with push-to-tal
assert.ok(ends >= 1);
});
test('on-device continuous capture runs on SpeechAnalyzer where supported; push-to-talk keeps the dictation recognizer', async () => {
const h = harness();
assert.equal(h.bridge.supportsContinuous(h.onboard), false, 'before iOS 26 on-device needs a cloud provider');
h.analyzerSupported(true);
assert.equal(h.bridge.supportsContinuous(h.onboard), true);
const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}, false, true); h.release(); await start;
assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: false, phone: false }, 'native only decodes PCM');
assert.equal(h.native.endpointing, undefined, "Apple's dictation recognizer stays idle");
const [analyzer] = h.analyzers;
assert.equal(analyzer.started, true); assert.equal(analyzer.continuous(), true);
assert.equal(h.clouds.length, 0);
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
assert.deepEqual(analyzer.pcm, [[1, 2, 3, 4]]);
analyzer.options.onTranscript({ text: 'First thought.', isFinal: true, paragraphBreakAfter: true });
const stopped = h.bridge.stopContinuousCapture();
h.event({ kind: 'ended', message: 'Transcription stopped.' });
assert.equal(analyzer.finished, 1);
analyzer.options.onTranscript({ text: 'The tail.', isFinal: true });
await stopped;
assert.equal(analyzer.stopped, true);
assert.deepEqual(h.text.map(event => event.text), ['First thought.', 'The tail.']);
// Push-to-talk (not continuous) still uses FaceclawSpeech's own recognizer.
const ptt = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await ptt;
assert.equal(h.analyzers.length, 1); assert.equal(h.native.endpointing, false);
});
test('a cloud provider keeps continuous capture even where SpeechAnalyzer is supported', async () => {
const h = harness(); h.analyzerSupported(true);
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
assert.equal(h.clouds.length, 1); assert.equal(h.analyzers.length, 0);
});
test('push-to-talk after Transcribe closed starts its own capture instead of joining the finalizing one', async () => {
const h = harness();
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
+162
View File
@@ -0,0 +1,162 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
/** FaceclawSpeechAnalyzer.swift's surface; `emit` plays its JSON events. */
class FakeAnalyzer {
static supported = true;
static instances = [];
static new() { const native = new FakeAnalyzer(); FakeAnalyzer.instances.push(native); return native; }
static isSupported() { return FakeAnalyzer.supported; }
constructor() { this.pcm = []; this.finished = 0; this.cancelled = 0; this.language = null; }
startWithLanguage(language) { this.language = language; }
acceptPcm(data) { this.pcm.push([...data]); }
finish() { this.finished++; }
cancel() { this.cancelled++; }
emit(event) { this.eventHandler(JSON.stringify(event)); }
}
function load() {
FakeAnalyzer.instances = [];
FakeAnalyzer.supported = true;
const timers = new Set();
const modules = loader({
FaceclawSpeechAnalyzer: FakeAnalyzer,
NSData: { dataWithBytesLength: (buffer, length) => new Uint8Array(buffer, 0, length) },
interop: { handleof: buffer => buffer },
setTimeout(fn) { timers.add(fn); return fn; },
clearTimeout(fn) { timers.delete(fn); },
}, {});
return { ...modules('app/native/speech-analyzer-stt.ios.ts'), timers };
}
function client(language = '') {
const { SpeechAnalyzerSttClient } = load();
const events = [], statuses = [], errors = [];
let ready = 0;
const c = new SpeechAnalyzerSttClient({
apiKey: '', language, onTranscript: event => events.push({ ...event }), onStatus: status => statuses.push(status),
onError: error => errors.push(error), onReady: () => ready++,
});
c.start();
const native = FakeAnalyzer.instances[0];
return { c, native, events, statuses, errors, ready: () => ready };
}
/** `seconds` of 16 kHz PCM16LE silence. */
const pcm = seconds => new Uint8Array(Math.round(seconds * 16000) * 2);
/** A result as FaceclawSpeechAnalyzer.swift reports it: its audio range and when its last word ends. */
const result = (text, final, start, end, speechEnd = end) => ({ kind: 'transcript', text, final, start, end, speechEnd });
const finals = h => h.events.filter(event => event.isFinal);
test('starts in the requested language and reports readiness as a provider would', () => {
const h = client('es');
assert.equal(h.native.language, 'es');
h.native.emit({ kind: 'status', message: 'Downloading on-device speech model… 40%' });
h.native.emit({ kind: 'ready', locale: 'es_ES' });
assert.deepEqual(h.statuses, ['Downloading on-device speech model… 40%', 'Listening (on-device)...']);
assert.equal(h.ready(), 1);
});
// Recorded from SpeechAnalyzer on macOS 26.4 (scripted speech, 1.5 s quiet, a
// pause, more speech): volatile ranges start where the last final ended, and
// the second final's first word reaches back into the silence (5.04 s; the
// speech starts at 6.18 s).
test('real model output: the first paragraph ends as soon as its last words are final', () => {
const h = client();
h.native.emit({ kind: 'ready', locale: 'en_US' });
h.native.emit(result('Hello there.', false, 0, 4.1));
h.native.emit(result('Hello there.', true, 0, 0.66, 0.6));
h.native.emit(result(' This is the firstst paragraph of the test spoken ', false, 0.66, 4.1815));
h.c.acceptPcm(pcm(5.6815));
h.c.commitSegment();
assert.equal(finals(h).length, 0, 'words before the pause are still provisional');
h.native.emit(result(' This is the firstst paragraph of the test spoken before a pause.', false, 0.66, 7.7815));
h.native.emit(result(' This is the first paragraph of the test, spoken before a pause.', true, 0.66, 4.38, 4.32));
assert.deepEqual(finals(h), [{ text: 'Hello there. This is the first paragraph of the test, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]);
h.native.emit(result(' And this is a second', false, 4.38, 8.1815));
assert.deepEqual(h.events.at(-1), { text: 'And this is a second', isFinal: false }, 'new speech shows in the new paragraph');
h.native.emit(result(' And this is a second paragraph, which comes after the pause.', true, 5.04, 9.18, 9.12));
h.c.finish();
assert.equal(h.native.finished, 1);
h.native.emit({ kind: 'finished' });
assert.deepEqual(h.events.at(-1), { text: 'And this is a second paragraph, which comes after the pause.', isFinal: true });
assert.equal(finals(h).length, 2);
assert.equal(h.native.cancelled, 1, 'the finished session is released');
h.native.emit(result('late', false, 9.2, 9.5));
assert.equal(h.events.at(-1).text, 'And this is a second paragraph, which comes after the pause.', 'events after stop are ignored');
});
test('a pause after words already final ends the paragraph at once, keeping later provisional text', () => {
const h = client();
h.native.emit({ kind: 'ready', locale: 'en_US' });
h.native.emit(result('Hello there.', true, 0, 1.0));
h.native.emit(result('How are you?', true, 1.0, 1.8));
h.c.acceptPcm(pcm(3.4));
h.native.emit(result('Fine', false, 1.8, 3.4));
h.c.commitSegment();
assert.deepEqual(h.events.slice(-2), [
{ text: 'Hello there. How are you?', isFinal: true, paragraphBreakAfter: true },
{ text: 'Fine', isFinal: false },
]);
});
test('a final ending mid-sentence before the pause waits for the rest of the sentence', () => {
const h = client();
h.native.emit({ kind: 'ready', locale: 'en_US' });
h.c.acceptPcm(pcm(5.7));
h.c.commitSegment();
h.native.emit(result('This is the first paragraph, spoken', true, 0.5, 3.6));
assert.equal(finals(h).length, 0);
h.native.emit(result('before a pause.', true, 3.6, 4.3));
assert.deepEqual(finals(h), [{ text: 'This is the first paragraph, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]);
});
test('without a final near the pause, a final ending after it starts the next paragraph', () => {
const h = client();
h.native.emit({ kind: 'ready', locale: 'en_US' });
h.native.emit(result('Before.', true, 0, 2.0));
// Background noise kept the pause detector from firing until well after the words.
h.c.acceptPcm(pcm(8));
h.c.commitSegment();
assert.equal(finals(h).length, 0);
h.native.emit(result('After.', true, 2.0, 9.5));
assert.deepEqual(finals(h), [{ text: 'Before.', isFinal: true, paragraphBreakAfter: true }]);
assert.deepEqual(h.events.at(-1), { text: 'After.', isFinal: false });
});
test('a pause with nothing before it adds no break', () => {
const h = client();
h.native.emit({ kind: 'ready', locale: 'en_US' });
h.c.acceptPcm(pcm(3));
h.c.commitSegment();
h.native.emit(result('First words.', true, 0, 4.2));
h.native.emit(result('More.', true, 4.2, 5.0));
assert.equal(finals(h).length, 0);
assert.deepEqual(h.events.at(-1), { text: 'First words. More.', isFinal: false });
});
test('errors end the session and reach the provider\'s error path', () => {
const h = client();
h.native.emit({ kind: 'error', message: "On-device transcription doesn't support xx." });
assert.deepEqual(h.errors, ["On-device transcription doesn't support xx."]);
assert.equal(h.native.cancelled, 1);
});
test('the wrapped client holds audio until the model is ready, then streams it', () => {
const { createSpeechAnalyzerClient, speechAnalyzerSupported } = load();
assert.equal(speechAnalyzerSupported(), true);
FakeAnalyzer.supported = false;
assert.equal(speechAnalyzerSupported(), false);
const events = [];
const wrapped = createSpeechAnalyzerClient({ onTranscript: event => events.push({ ...event }), onStatus() {}, onError() {} }, () => true);
wrapped.start();
const native = FakeAnalyzer.instances[0];
wrapped.acceptPcm(new Uint8Array([1, 0, 2, 0]));
assert.equal(native.pcm.length, 0, 'buffered while the model loads');
native.emit({ kind: 'ready', locale: 'en_US' });
wrapped.acceptPcm(new Uint8Array([3, 0]));
assert.deepEqual(native.pcm, [[1, 0, 2, 0], [3, 0]]);
wrapped.stop();
assert.equal(native.cancelled, 1);
});