Use iOS SpeechAnalyzer API to support on-device transcription with the Transcribe app
This commit is contained in:
1 parent
210b111e27
commit
6e5864f981
6 files changed
+709
-18
No files matched your search
@@ -0,0 +1,273 @@
|
||||
@preconcurrency import AVFoundation
|
||||
import CoreMedia
|
||||
import Foundation
|
||||
import Speech
|
||||
|
||||
/// Long-form on-device transcription with SpeechAnalyzer and SpeechTranscriber
|
||||
/// (iOS 26+), for continuous capture (Transcribe, Teleprompter, Calculator):
|
||||
/// FaceclawSpeech's SFSpeechRecognizer only handles short dictations. It takes
|
||||
/// the same 16 kHz mono PCM16LE the cloud providers get (see
|
||||
/// app/native/speech-analyzer-stt.ios.ts) and reports JSON events on the main
|
||||
/// queue through eventHandler:
|
||||
///
|
||||
/// {"kind":"status","message":String}
|
||||
/// {"kind":"ready","locale":String} audio is accepted from here on
|
||||
/// {"kind":"transcript","text":String,"final":Bool,
|
||||
/// "start":Double?,"end":Double?, the result's audio range
|
||||
/// "speechEnd":Double?} when its last word ends
|
||||
/// {"kind":"finished"} after finish(), once the last result is in
|
||||
/// {"kind":"error","message":String}
|
||||
///
|
||||
/// Times are seconds of audio since start. Volatile results (final: false)
|
||||
/// cover all audio not finalized yet, replacing the previous volatile one;
|
||||
/// their word times are only as fine as the result's range. A final's first
|
||||
/// word reaches back into the silence before it, so only word ends are
|
||||
/// reported.
|
||||
///
|
||||
/// There is no way to settle text at a pause: on macOS 26.4 any mid-stream
|
||||
/// SpeechAnalyzer.finalize(through:) left the next phrase's final as dots,
|
||||
/// while the model splits its finals at pauses on its own.
|
||||
/// Called from JavaScript, so on the main thread.
|
||||
@MainActor
|
||||
@objc(FaceclawSpeechAnalyzer)
|
||||
@objcMembers
|
||||
public class FaceclawSpeechAnalyzer: NSObject {
|
||||
public var eventHandler: ((String) -> Void)?
|
||||
// An AnalyzerEngine, typed loosely so the class itself loads before iOS 26.
|
||||
private var engine: AnyObject?
|
||||
|
||||
/// Whether this device can transcribe with SpeechTranscriber (iOS 26 and supported hardware).
|
||||
public static func isSupported() -> Bool {
|
||||
if #available(iOS 26.0, *) { return SpeechTranscriber.isAvailable }
|
||||
return false
|
||||
}
|
||||
|
||||
/// `language` is a locale identifier ("en", "en-US"), or "" for the device's preferred language.
|
||||
public func start(language: String) {
|
||||
cancel()
|
||||
guard #available(iOS 26.0, *) else {
|
||||
emit(["kind": "error", "message": "On-device transcription needs iOS 26 or later."])
|
||||
return
|
||||
}
|
||||
let engine = AnalyzerEngine { [weak self] event in self?.emit(event) }
|
||||
self.engine = engine
|
||||
engine.start(language: language)
|
||||
}
|
||||
|
||||
/// 16 kHz mono PCM16LE; ignored until "ready".
|
||||
public func acceptPcm(_ pcm: Data) {
|
||||
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.accept(pcm) }
|
||||
}
|
||||
|
||||
/// End of audio: finalize the rest, then report "finished".
|
||||
public func finish() {
|
||||
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.finish() }
|
||||
}
|
||||
|
||||
/// Abandon the session without further events.
|
||||
public func cancel() {
|
||||
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.cancel() }
|
||||
engine = nil
|
||||
}
|
||||
|
||||
private func emit(_ event: [String: Any]) {
|
||||
// JSONSerialization raises (not throws) on NaN and other invalid values.
|
||||
guard JSONSerialization.isValidJSONObject(event),
|
||||
let data = try? JSONSerialization.data(withJSONObject: event),
|
||||
let json = String(data: data, encoding: .utf8) else { return }
|
||||
eventHandler?(json)
|
||||
}
|
||||
}
|
||||
|
||||
@available(iOS 26.0, *)
|
||||
@MainActor
|
||||
private final class AnalyzerEngine {
|
||||
private static let sampleRate = 16000.0
|
||||
private let emit: ([String: Any]) -> Void
|
||||
private let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: AnalyzerEngine.sampleRate, channels: 1, interleaved: true)!
|
||||
private var analyzer: SpeechAnalyzer?
|
||||
private var input: AsyncStream<AnalyzerInput>.Continuation?
|
||||
private var analyzerFormat: AVAudioFormat?
|
||||
// Nil when the analyzer takes the input format as it is.
|
||||
private var converter: AVAudioConverter?
|
||||
private var ready = false
|
||||
private var finishing = false
|
||||
private var closed = false
|
||||
private var setup: Task<Void, Never>?
|
||||
private var results: Task<Void, Never>?
|
||||
|
||||
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
|
||||
|
||||
func start(language: String) {
|
||||
setup = Task { [weak self] in await self?.prepare(language: language) }
|
||||
}
|
||||
|
||||
/// Events stop once the session is cancelled, even from work still winding down.
|
||||
private func send(_ event: [String: Any]) {
|
||||
if !closed { emit(event) }
|
||||
}
|
||||
|
||||
private func prepare(language: String) async {
|
||||
send(["kind": "status", "message": "Preparing on-device transcription…"])
|
||||
let identifier = language.isEmpty ? (Locale.preferredLanguages.first ?? "en-US") : language
|
||||
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: identifier)) else {
|
||||
fail("On-device transcription doesn't support \(identifier). Choose a cloud or Whisper server transcription provider (Settings > Voice).")
|
||||
return
|
||||
}
|
||||
if closed { return }
|
||||
let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [],
|
||||
reportingOptions: [.volatileResults], attributeOptions: [.audioTimeRange])
|
||||
do {
|
||||
// Keep the model installed between sessions (best effort).
|
||||
if !(await AssetInventory.reservedLocales).contains(locale) {
|
||||
_ = try? await AssetInventory.reserve(locale: locale)
|
||||
}
|
||||
if closed { return }
|
||||
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
||||
if closed { return }
|
||||
try await download(request)
|
||||
}
|
||||
if closed { return }
|
||||
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else {
|
||||
fail("On-device transcription has no usable audio format.")
|
||||
return
|
||||
}
|
||||
if closed { return }
|
||||
if format != pcmFormat {
|
||||
guard let converter = AVAudioConverter(from: pcmFormat, to: format) else {
|
||||
fail("On-device transcription can't convert the microphone audio.")
|
||||
return
|
||||
}
|
||||
self.converter = converter
|
||||
}
|
||||
analyzerFormat = format
|
||||
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
||||
let (stream, input) = AsyncStream<AnalyzerInput>.makeStream()
|
||||
self.analyzer = analyzer
|
||||
self.input = input
|
||||
results = Task { [weak self] in await self?.collect(transcriber) }
|
||||
try await analyzer.prepareToAnalyze(in: format)
|
||||
try await analyzer.start(inputSequence: stream)
|
||||
if closed { return }
|
||||
ready = true
|
||||
send(["kind": "ready", "locale": locale.identifier])
|
||||
if finishing { await finishAnalysis() }
|
||||
} catch {
|
||||
fail("On-device transcription: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the locale's model, reporting progress while it downloads.
|
||||
private func download(_ request: AssetInstallationRequest) async throws {
|
||||
let progress = request.progress
|
||||
let reporter = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
let percent = Int((progress.fractionCompleted * 100).rounded())
|
||||
self?.send(["kind": "status", "message": "Downloading on-device speech model… \(percent)%"])
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
}
|
||||
}
|
||||
defer { reporter.cancel() }
|
||||
try await request.downloadAndInstall()
|
||||
}
|
||||
|
||||
func accept(_ pcm: Data) {
|
||||
guard ready, !finishing, !closed, let input, let analyzerFormat else { return }
|
||||
let frames = AVAudioFrameCount(pcm.count / 2)
|
||||
guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames),
|
||||
let channel = buffer.int16ChannelData else { return }
|
||||
buffer.frameLength = frames
|
||||
pcm.withUnsafeBytes { raw in
|
||||
if let base = raw.baseAddress { memcpy(channel[0], base, Int(frames) * 2) }
|
||||
}
|
||||
guard let converter else {
|
||||
input.yield(AnalyzerInput(buffer: buffer))
|
||||
return
|
||||
}
|
||||
let capacity = AVAudioFrameCount((Double(frames) * analyzerFormat.sampleRate / AnalyzerEngine.sampleRate).rounded(.up)) + 32
|
||||
guard let converted = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return }
|
||||
var consumed = false
|
||||
var error: NSError?
|
||||
// noDataNow, not endOfStream: the converter keeps its resampling state for the next chunk.
|
||||
converter.convert(to: converted, error: &error) { _, status in
|
||||
if consumed {
|
||||
status.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
consumed = true
|
||||
status.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
if error == nil && converted.frameLength > 0 { input.yield(AnalyzerInput(buffer: converted)) }
|
||||
}
|
||||
|
||||
func finish() {
|
||||
guard !finishing, !closed else { return }
|
||||
finishing = true
|
||||
// Before "ready", prepare() finishes once the analyzer is running.
|
||||
if ready { Task { await finishAnalysis() } }
|
||||
}
|
||||
|
||||
private func finishAnalysis() async {
|
||||
input?.finish()
|
||||
input = nil
|
||||
do {
|
||||
try await analyzer?.finalizeAndFinishThroughEndOfInput()
|
||||
} catch {
|
||||
fail("On-device transcription: \(error.localizedDescription)")
|
||||
}
|
||||
// collect() reports "finished" once the results sequence ends.
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
guard !closed else { return }
|
||||
closed = true
|
||||
input?.finish()
|
||||
input = nil
|
||||
setup?.cancel()
|
||||
results?.cancel()
|
||||
if let analyzer { Task { await analyzer.cancelAndFinishNow() } }
|
||||
analyzer = nil
|
||||
}
|
||||
|
||||
private func collect(_ transcriber: SpeechTranscriber) async {
|
||||
do {
|
||||
for try await result in transcriber.results {
|
||||
if closed { return }
|
||||
send(event(for: result))
|
||||
}
|
||||
send(["kind": "finished"])
|
||||
} catch {
|
||||
fail("On-device transcription: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
private func event(for result: SpeechTranscriber.Result) -> [String: Any] {
|
||||
let text = result.text
|
||||
var speechEnd: Double?
|
||||
for run in text.runs {
|
||||
guard let range = run.audioTimeRange,
|
||||
!String(text[run.range].characters).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue }
|
||||
speechEnd = finite(range.end) ?? speechEnd
|
||||
}
|
||||
return [
|
||||
"kind": "transcript",
|
||||
"text": String(text.characters),
|
||||
"final": result.isFinal,
|
||||
"start": finite(result.range.start) ?? NSNull(),
|
||||
"end": finite(result.range.end) ?? NSNull(),
|
||||
"speechEnd": speechEnd ?? NSNull(),
|
||||
]
|
||||
}
|
||||
|
||||
private func finite(_ time: CMTime) -> Double? {
|
||||
let seconds = time.seconds
|
||||
return seconds.isFinite ? seconds : nil
|
||||
}
|
||||
|
||||
private func fail(_ message: String) {
|
||||
guard !closed else { return }
|
||||
emit(["kind": "error", "message": message])
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
@@ -1054,7 +1054,7 @@ export class IosPreviewController {
|
||||
this.continuousClients.set(client, options)
|
||||
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||
this.continuousClients.delete(client)
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
this.syncContinuousRecording()
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
declare const FaceclawSpeechAnalyzer: any;
|
||||
|
||||
/**
|
||||
* How long finish() waits for the last results: the model only has to settle
|
||||
* the few seconds of audio it hadn't finalized yet.
|
||||
*/
|
||||
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
|
||||
|
||||
/**
|
||||
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
|
||||
* whose last word ends within this much before that (the model's word ends
|
||||
* against the detector's level threshold) leaves nothing before the pause
|
||||
* unfinalized, so its paragraph is complete.
|
||||
*/
|
||||
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
|
||||
|
||||
type AnalyzerEvent = {
|
||||
kind: string;
|
||||
message?: string;
|
||||
locale?: string;
|
||||
text?: string;
|
||||
final?: boolean;
|
||||
start?: number | null;
|
||||
end?: number | null;
|
||||
speechEnd?: number | null;
|
||||
};
|
||||
|
||||
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
|
||||
export function speechAnalyzerSupported(): boolean {
|
||||
try {
|
||||
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* On-device long-form transcription for continuous capture, wrapped like a
|
||||
* cloud provider: it buffers audio until the model is ready (downloading it on
|
||||
* first use) and gets pause commits.
|
||||
*/
|
||||
export function createSpeechAnalyzerClient(
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): ReconnectingSttClient {
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
|
||||
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
|
||||
continuous,
|
||||
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
|
||||
* The model finalizes text in its own pieces, a few seconds behind, and can't
|
||||
* be made to settle at a pause; this gathers its pieces into paragraphs ended
|
||||
* by pauses, like the cloud providers' segments. A paragraph is shown as
|
||||
* provisional text and sent as one final (paragraphBreakAfter) once a final
|
||||
* reaches its pause, and the last one as a plain final when the audio ends.
|
||||
*
|
||||
* Only finals place breaks, by when their last word ends (the model's
|
||||
* word-end times are accurate; first words' starts reach back into the
|
||||
* silence before them, and volatile results only carry the range of all audio
|
||||
* not finalized yet).
|
||||
*/
|
||||
export class SpeechAnalyzerSttClient implements CloudSttClient {
|
||||
private native: any = null;
|
||||
private samples = 0;
|
||||
private finishing = false;
|
||||
// Audio times (s) of pauses whose paragraph hasn't ended yet.
|
||||
private pauses: number[] = [];
|
||||
// Finalized text since the last paragraph break, when its last word ends,
|
||||
// and the model's provisional rest.
|
||||
private paragraph: string[] = [];
|
||||
private paragraphEnd: number | null = null;
|
||||
private volatile = "";
|
||||
|
||||
constructor(private readonly options: CloudSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.native) return;
|
||||
const native = FaceclawSpeechAnalyzer.new();
|
||||
native.eventHandler = (json: string) => {
|
||||
if (this.native === native) this.receive(JSON.parse(json));
|
||||
};
|
||||
this.native = native;
|
||||
native.startWithLanguage(this.options.language ?? "");
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (!this.native || this.finishing || !pcm.length) return;
|
||||
this.samples += pcm.length >> 1;
|
||||
const copy = pcm.slice();
|
||||
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
|
||||
}
|
||||
|
||||
/** A pause: it ends the paragraph once the words before it are final. */
|
||||
commitSegment(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
|
||||
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.finishing = true;
|
||||
this.native.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
const native = this.native;
|
||||
this.native = null;
|
||||
native?.cancel();
|
||||
}
|
||||
|
||||
private receive(event: AnalyzerEvent): void {
|
||||
switch (event.kind) {
|
||||
case "status":
|
||||
this.options.onStatus(event.message ?? "");
|
||||
break;
|
||||
case "ready":
|
||||
this.options.onStatus("Listening (on-device)...");
|
||||
this.options.onReady?.();
|
||||
break;
|
||||
case "transcript":
|
||||
this.transcript(event);
|
||||
break;
|
||||
case "finished": {
|
||||
const text = this.liveText();
|
||||
this.paragraph = [];
|
||||
this.volatile = "";
|
||||
this.stop();
|
||||
this.options.onTranscript({ text, isFinal: true });
|
||||
break;
|
||||
}
|
||||
case "error":
|
||||
this.stop();
|
||||
this.options.onError(event.message ?? "On-device transcription failed.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private transcript(event: AnalyzerEvent): void {
|
||||
const text = (event.text ?? "").trim();
|
||||
let ended = false;
|
||||
if (!event.final) {
|
||||
this.volatile = text;
|
||||
} else {
|
||||
this.volatile = "";
|
||||
if (text) {
|
||||
const end = event.speechEnd ?? event.end ?? null;
|
||||
// Words ending after a pause come after it. (One whose words run
|
||||
// across it, if the model didn't split there, breaks a little early.)
|
||||
if (end !== null && this.pauses.some((pause) => end > pause)) {
|
||||
this.pauses = this.pauses.filter((pause) => pause >= end);
|
||||
this.endParagraph();
|
||||
}
|
||||
this.paragraph.push(text);
|
||||
this.paragraphEnd = end;
|
||||
ended = this.endCompleteParagraph();
|
||||
}
|
||||
}
|
||||
// The paragraph's final already cleared the provisional text.
|
||||
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
/** End the paragraph if its last word ends just before a pause. */
|
||||
private endCompleteParagraph(): boolean {
|
||||
const end = this.paragraphEnd;
|
||||
if (!this.paragraph.length || end === null) return false;
|
||||
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
|
||||
if (!this.pauses.some(reached)) return false;
|
||||
this.pauses = this.pauses.filter((pause) => !reached(pause));
|
||||
this.endParagraph();
|
||||
return true;
|
||||
}
|
||||
|
||||
private endParagraph(): void {
|
||||
const text = this.paragraph.join(" ");
|
||||
this.paragraph = [];
|
||||
this.paragraphEnd = null;
|
||||
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
|
||||
}
|
||||
|
||||
private liveText(): string {
|
||||
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
@@ -18,10 +19,12 @@ const RAW_RETRY_LIMIT = 5
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
||||
* the PCM capture runs open-ended, and pauses commit its segments. When the
|
||||
* session or the app lifecycle cuts one short it is drained, not discarded;
|
||||
* restarting it is the controller's call.
|
||||
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
|
||||
* is open-ended and has pauses commit its segments, so it needs a cloud
|
||||
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
|
||||
* recognizer only handles short dictations. When the session or the app
|
||||
* lifecycle cuts one short it is drained, not discarded; restarting it is the
|
||||
* controller's call.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
@@ -40,10 +43,13 @@ export class IosVoiceControlBridge {
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
|
||||
// on-device continuous capture) owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: ReconnectingSttClient | null = null
|
||||
// What transcribes the current capture, for the log.
|
||||
private recognition = 'on-device'
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
||||
@@ -137,7 +143,7 @@ export class IosVoiceControlBridge {
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
@@ -158,18 +164,27 @@ export class IosVoiceControlBridge {
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
|
||||
}
|
||||
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
|
||||
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
|
||||
const native = this.ensureNative()
|
||||
const generation = this.generation
|
||||
const current = () => generation === this.generation
|
||||
const cloud = createCloudSttClient(voice, {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => this.continuous)
|
||||
const options = {
|
||||
language: voice.language,
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: (status: string) => { if (current()) this.setStatus(status) },
|
||||
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}
|
||||
const continuous = () => this.continuous
|
||||
let cloud = createCloudSttClient(voice, options, continuous)
|
||||
this.recognition = cloud ? 'cloud' : 'on-device'
|
||||
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
|
||||
if (!cloud && this.continuous && speechAnalyzerSupported()) {
|
||||
cloud = createSpeechAnalyzerClient(options, continuous)
|
||||
this.recognition = 'on-device (SpeechAnalyzer)'
|
||||
}
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
@@ -182,8 +197,11 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
|
||||
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
||||
}
|
||||
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
||||
/**
|
||||
* Whether continuous capture can run with this provider: a cloud one, or
|
||||
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
|
||||
*/
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
|
||||
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
|
||||
@@ -7,7 +7,7 @@ function harness() {
|
||||
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; },
|
||||
startPcmCaptureWithEndpointingPhone(endpointing, phone) { this.pcmCapture = { endpointing, phone }; return this.error ?? ''; },
|
||||
cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const audio = [], timers = new Map(), commands = [], text = [], clouds = [];
|
||||
const audio = [], timers = new Map(), commands = [], text = [], clouds = [], analyzers = [];
|
||||
// Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one.
|
||||
const provider = {
|
||||
usesCloudStt: voice => voice.provider !== 'onboard',
|
||||
@@ -19,6 +19,17 @@ function harness() {
|
||||
clouds.push(client); return client;
|
||||
},
|
||||
};
|
||||
// Stands in for speech-analyzer-stt: SpeechAnalyzer (iOS 26+) when the test says the device has it.
|
||||
let analyzerSupported = false;
|
||||
const analyzer = {
|
||||
speechAnalyzerSupported: () => analyzerSupported,
|
||||
createSpeechAnalyzerClient(options, continuous) {
|
||||
const client = { options, continuous, pcm: [], started: false, finished: 0, stopped: false,
|
||||
start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); },
|
||||
finish() { this.finished++; }, stop() { this.stopped = true; } };
|
||||
analyzers.push(client); return client;
|
||||
},
|
||||
};
|
||||
const recordings = [];
|
||||
class FakeRecorder {
|
||||
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
|
||||
@@ -32,7 +43,8 @@ function harness() {
|
||||
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
|
||||
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
|
||||
require: name => name === './cloud-stt-provider' ? provider : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
|
||||
require: name => name === './cloud-stt-provider' ? provider : name === './speech-analyzer-stt.ios' ? analyzer
|
||||
: name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
|
||||
const bridge = new sandbox.exports.IosVoiceControlBridge();
|
||||
@@ -43,7 +55,8 @@ function harness() {
|
||||
return Promise.resolve();
|
||||
} };
|
||||
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, recordings, release: () => release(),
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
|
||||
analyzerSupported: value => { analyzerSupported = value; },
|
||||
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
||||
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
||||
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
|
||||
@@ -248,6 +261,38 @@ test('continuous capture needs a cloud provider, shares its mic with push-to-tal
|
||||
assert.ok(ends >= 1);
|
||||
});
|
||||
|
||||
test('on-device continuous capture runs on SpeechAnalyzer where supported; push-to-talk keeps the dictation recognizer', async () => {
|
||||
const h = harness();
|
||||
assert.equal(h.bridge.supportsContinuous(h.onboard), false, 'before iOS 26 on-device needs a cloud provider');
|
||||
h.analyzerSupported(true);
|
||||
assert.equal(h.bridge.supportsContinuous(h.onboard), true);
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}, false, true); h.release(); await start;
|
||||
assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: false, phone: false }, 'native only decodes PCM');
|
||||
assert.equal(h.native.endpointing, undefined, "Apple's dictation recognizer stays idle");
|
||||
const [analyzer] = h.analyzers;
|
||||
assert.equal(analyzer.started, true); assert.equal(analyzer.continuous(), true);
|
||||
assert.equal(h.clouds.length, 0);
|
||||
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
|
||||
assert.deepEqual(analyzer.pcm, [[1, 2, 3, 4]]);
|
||||
analyzer.options.onTranscript({ text: 'First thought.', isFinal: true, paragraphBreakAfter: true });
|
||||
const stopped = h.bridge.stopContinuousCapture();
|
||||
h.event({ kind: 'ended', message: 'Transcription stopped.' });
|
||||
assert.equal(analyzer.finished, 1);
|
||||
analyzer.options.onTranscript({ text: 'The tail.', isFinal: true });
|
||||
await stopped;
|
||||
assert.equal(analyzer.stopped, true);
|
||||
assert.deepEqual(h.text.map(event => event.text), ['First thought.', 'The tail.']);
|
||||
// Push-to-talk (not continuous) still uses FaceclawSpeech's own recognizer.
|
||||
const ptt = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await ptt;
|
||||
assert.equal(h.analyzers.length, 1); assert.equal(h.native.endpointing, false);
|
||||
});
|
||||
|
||||
test('a cloud provider keeps continuous capture even where SpeechAnalyzer is supported', async () => {
|
||||
const h = harness(); h.analyzerSupported(true);
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
assert.equal(h.clouds.length, 1); assert.equal(h.analyzers.length, 0);
|
||||
});
|
||||
|
||||
test('push-to-talk after Transcribe closed starts its own capture instead of joining the finalizing one', async () => {
|
||||
const h = harness();
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
/** FaceclawSpeechAnalyzer.swift's surface; `emit` plays its JSON events. */
|
||||
class FakeAnalyzer {
|
||||
static supported = true;
|
||||
static instances = [];
|
||||
static new() { const native = new FakeAnalyzer(); FakeAnalyzer.instances.push(native); return native; }
|
||||
static isSupported() { return FakeAnalyzer.supported; }
|
||||
constructor() { this.pcm = []; this.finished = 0; this.cancelled = 0; this.language = null; }
|
||||
startWithLanguage(language) { this.language = language; }
|
||||
acceptPcm(data) { this.pcm.push([...data]); }
|
||||
finish() { this.finished++; }
|
||||
cancel() { this.cancelled++; }
|
||||
emit(event) { this.eventHandler(JSON.stringify(event)); }
|
||||
}
|
||||
|
||||
function load() {
|
||||
FakeAnalyzer.instances = [];
|
||||
FakeAnalyzer.supported = true;
|
||||
const timers = new Set();
|
||||
const modules = loader({
|
||||
FaceclawSpeechAnalyzer: FakeAnalyzer,
|
||||
NSData: { dataWithBytesLength: (buffer, length) => new Uint8Array(buffer, 0, length) },
|
||||
interop: { handleof: buffer => buffer },
|
||||
setTimeout(fn) { timers.add(fn); return fn; },
|
||||
clearTimeout(fn) { timers.delete(fn); },
|
||||
}, {});
|
||||
return { ...modules('app/native/speech-analyzer-stt.ios.ts'), timers };
|
||||
}
|
||||
|
||||
function client(language = '') {
|
||||
const { SpeechAnalyzerSttClient } = load();
|
||||
const events = [], statuses = [], errors = [];
|
||||
let ready = 0;
|
||||
const c = new SpeechAnalyzerSttClient({
|
||||
apiKey: '', language, onTranscript: event => events.push({ ...event }), onStatus: status => statuses.push(status),
|
||||
onError: error => errors.push(error), onReady: () => ready++,
|
||||
});
|
||||
c.start();
|
||||
const native = FakeAnalyzer.instances[0];
|
||||
return { c, native, events, statuses, errors, ready: () => ready };
|
||||
}
|
||||
|
||||
/** `seconds` of 16 kHz PCM16LE silence. */
|
||||
const pcm = seconds => new Uint8Array(Math.round(seconds * 16000) * 2);
|
||||
/** A result as FaceclawSpeechAnalyzer.swift reports it: its audio range and when its last word ends. */
|
||||
const result = (text, final, start, end, speechEnd = end) => ({ kind: 'transcript', text, final, start, end, speechEnd });
|
||||
const finals = h => h.events.filter(event => event.isFinal);
|
||||
|
||||
test('starts in the requested language and reports readiness as a provider would', () => {
|
||||
const h = client('es');
|
||||
assert.equal(h.native.language, 'es');
|
||||
h.native.emit({ kind: 'status', message: 'Downloading on-device speech model… 40%' });
|
||||
h.native.emit({ kind: 'ready', locale: 'es_ES' });
|
||||
assert.deepEqual(h.statuses, ['Downloading on-device speech model… 40%', 'Listening (on-device)...']);
|
||||
assert.equal(h.ready(), 1);
|
||||
});
|
||||
|
||||
// Recorded from SpeechAnalyzer on macOS 26.4 (scripted speech, 1.5 s quiet, a
|
||||
// pause, more speech): volatile ranges start where the last final ended, and
|
||||
// the second final's first word reaches back into the silence (5.04 s; the
|
||||
// speech starts at 6.18 s).
|
||||
test('real model output: the first paragraph ends as soon as its last words are final', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.native.emit(result('Hello there.', false, 0, 4.1));
|
||||
h.native.emit(result('Hello there.', true, 0, 0.66, 0.6));
|
||||
h.native.emit(result(' This is the firstst paragraph of the test spoken ', false, 0.66, 4.1815));
|
||||
h.c.acceptPcm(pcm(5.6815));
|
||||
h.c.commitSegment();
|
||||
assert.equal(finals(h).length, 0, 'words before the pause are still provisional');
|
||||
h.native.emit(result(' This is the firstst paragraph of the test spoken before a pause.', false, 0.66, 7.7815));
|
||||
h.native.emit(result(' This is the first paragraph of the test, spoken before a pause.', true, 0.66, 4.38, 4.32));
|
||||
assert.deepEqual(finals(h), [{ text: 'Hello there. This is the first paragraph of the test, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
h.native.emit(result(' And this is a second', false, 4.38, 8.1815));
|
||||
assert.deepEqual(h.events.at(-1), { text: 'And this is a second', isFinal: false }, 'new speech shows in the new paragraph');
|
||||
h.native.emit(result(' And this is a second paragraph, which comes after the pause.', true, 5.04, 9.18, 9.12));
|
||||
h.c.finish();
|
||||
assert.equal(h.native.finished, 1);
|
||||
h.native.emit({ kind: 'finished' });
|
||||
assert.deepEqual(h.events.at(-1), { text: 'And this is a second paragraph, which comes after the pause.', isFinal: true });
|
||||
assert.equal(finals(h).length, 2);
|
||||
assert.equal(h.native.cancelled, 1, 'the finished session is released');
|
||||
h.native.emit(result('late', false, 9.2, 9.5));
|
||||
assert.equal(h.events.at(-1).text, 'And this is a second paragraph, which comes after the pause.', 'events after stop are ignored');
|
||||
});
|
||||
|
||||
test('a pause after words already final ends the paragraph at once, keeping later provisional text', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.native.emit(result('Hello there.', true, 0, 1.0));
|
||||
h.native.emit(result('How are you?', true, 1.0, 1.8));
|
||||
h.c.acceptPcm(pcm(3.4));
|
||||
h.native.emit(result('Fine', false, 1.8, 3.4));
|
||||
h.c.commitSegment();
|
||||
assert.deepEqual(h.events.slice(-2), [
|
||||
{ text: 'Hello there. How are you?', isFinal: true, paragraphBreakAfter: true },
|
||||
{ text: 'Fine', isFinal: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test('a final ending mid-sentence before the pause waits for the rest of the sentence', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.c.acceptPcm(pcm(5.7));
|
||||
h.c.commitSegment();
|
||||
h.native.emit(result('This is the first paragraph, spoken', true, 0.5, 3.6));
|
||||
assert.equal(finals(h).length, 0);
|
||||
h.native.emit(result('before a pause.', true, 3.6, 4.3));
|
||||
assert.deepEqual(finals(h), [{ text: 'This is the first paragraph, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
});
|
||||
|
||||
test('without a final near the pause, a final ending after it starts the next paragraph', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.native.emit(result('Before.', true, 0, 2.0));
|
||||
// Background noise kept the pause detector from firing until well after the words.
|
||||
h.c.acceptPcm(pcm(8));
|
||||
h.c.commitSegment();
|
||||
assert.equal(finals(h).length, 0);
|
||||
h.native.emit(result('After.', true, 2.0, 9.5));
|
||||
assert.deepEqual(finals(h), [{ text: 'Before.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
assert.deepEqual(h.events.at(-1), { text: 'After.', isFinal: false });
|
||||
});
|
||||
|
||||
test('a pause with nothing before it adds no break', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.c.acceptPcm(pcm(3));
|
||||
h.c.commitSegment();
|
||||
h.native.emit(result('First words.', true, 0, 4.2));
|
||||
h.native.emit(result('More.', true, 4.2, 5.0));
|
||||
assert.equal(finals(h).length, 0);
|
||||
assert.deepEqual(h.events.at(-1), { text: 'First words. More.', isFinal: false });
|
||||
});
|
||||
|
||||
test('errors end the session and reach the provider\'s error path', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'error', message: "On-device transcription doesn't support xx." });
|
||||
assert.deepEqual(h.errors, ["On-device transcription doesn't support xx."]);
|
||||
assert.equal(h.native.cancelled, 1);
|
||||
});
|
||||
|
||||
test('the wrapped client holds audio until the model is ready, then streams it', () => {
|
||||
const { createSpeechAnalyzerClient, speechAnalyzerSupported } = load();
|
||||
assert.equal(speechAnalyzerSupported(), true);
|
||||
FakeAnalyzer.supported = false;
|
||||
assert.equal(speechAnalyzerSupported(), false);
|
||||
const events = [];
|
||||
const wrapped = createSpeechAnalyzerClient({ onTranscript: event => events.push({ ...event }), onStatus() {}, onError() {} }, () => true);
|
||||
wrapped.start();
|
||||
const native = FakeAnalyzer.instances[0];
|
||||
wrapped.acceptPcm(new Uint8Array([1, 0, 2, 0]));
|
||||
assert.equal(native.pcm.length, 0, 'buffered while the model loads');
|
||||
native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
wrapped.acceptPcm(new Uint8Array([3, 0]));
|
||||
assert.deepEqual(native.pcm, [[1, 0, 2, 0], [3, 0]]);
|
||||
wrapped.stop();
|
||||
assert.equal(native.cancelled, 1);
|
||||
});
|
||||
Reference in new issue
Block a user