Use iOS SpeechAnalyzer API to support on-device transcription with the Transcribe app

This commit is contained in:
James Babcock committed 2026-10-31 17:03:56 -07:00
1 parent 210b111e27
commit 6e5864f981
6 files changed
+709 -18

No files matched your search

+193
View File
@@ -0,0 +1,193 @@
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
import { ReconnectingSttClient } from "./reconnecting-stt";
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
declare const FaceclawSpeechAnalyzer: any;
/**
* How long finish() waits for the last results: the model only has to settle
* the few seconds of audio it hadn't finalized yet.
*/
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
/**
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
* whose last word ends within this much before that (the model's word ends
* against the detector's level threshold) leaves nothing before the pause
* unfinalized, so its paragraph is complete.
*/
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
type AnalyzerEvent = {
kind: string;
message?: string;
locale?: string;
text?: string;
final?: boolean;
start?: number | null;
end?: number | null;
speechEnd?: number | null;
};
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
export function speechAnalyzerSupported(): boolean {
try {
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
} catch {
return false;
}
}
/**
* On-device long-form transcription for continuous capture, wrapped like a
* cloud provider: it buffers audio until the model is ready (downloading it on
* first use) and gets pause commits.
*/
export function createSpeechAnalyzerClient(
options: Omit<CloudSttOptions, "apiKey">,
continuous: () => boolean,
): ReconnectingSttClient {
return new ReconnectingSttClient(
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
continuous,
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
);
}
/**
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
* The model finalizes text in its own pieces, a few seconds behind, and can't
* be made to settle at a pause; this gathers its pieces into paragraphs ended
* by pauses, like the cloud providers' segments. A paragraph is shown as
* provisional text and sent as one final (paragraphBreakAfter) once a final
* reaches its pause, and the last one as a plain final when the audio ends.
*
* Only finals place breaks, by when their last word ends (the model's
* word-end times are accurate; first words' starts reach back into the
* silence before them, and volatile results only carry the range of all audio
* not finalized yet).
*/
export class SpeechAnalyzerSttClient implements CloudSttClient {
private native: any = null;
private samples = 0;
private finishing = false;
// Audio times (s) of pauses whose paragraph hasn't ended yet.
private pauses: number[] = [];
// Finalized text since the last paragraph break, when its last word ends,
// and the model's provisional rest.
private paragraph: string[] = [];
private paragraphEnd: number | null = null;
private volatile = "";
constructor(private readonly options: CloudSttOptions) {}
start(): void {
if (this.native) return;
const native = FaceclawSpeechAnalyzer.new();
native.eventHandler = (json: string) => {
if (this.native === native) this.receive(JSON.parse(json));
};
this.native = native;
native.startWithLanguage(this.options.language ?? "");
}
acceptPcm(pcm: Uint8Array): void {
if (!this.native || this.finishing || !pcm.length) return;
this.samples += pcm.length >> 1;
const copy = pcm.slice();
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
}
/** A pause: it ends the paragraph once the words before it are final. */
commitSegment(): void {
if (!this.native || this.finishing) return;
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
}
finish(): void {
if (!this.native || this.finishing) return;
this.finishing = true;
this.native.finish();
}
stop(): void {
const native = this.native;
this.native = null;
native?.cancel();
}
private receive(event: AnalyzerEvent): void {
switch (event.kind) {
case "status":
this.options.onStatus(event.message ?? "");
break;
case "ready":
this.options.onStatus("Listening (on-device)...");
this.options.onReady?.();
break;
case "transcript":
this.transcript(event);
break;
case "finished": {
const text = this.liveText();
this.paragraph = [];
this.volatile = "";
this.stop();
this.options.onTranscript({ text, isFinal: true });
break;
}
case "error":
this.stop();
this.options.onError(event.message ?? "On-device transcription failed.");
break;
}
}
private transcript(event: AnalyzerEvent): void {
const text = (event.text ?? "").trim();
let ended = false;
if (!event.final) {
this.volatile = text;
} else {
this.volatile = "";
if (text) {
const end = event.speechEnd ?? event.end ?? null;
// Words ending after a pause come after it. (One whose words run
// across it, if the model didn't split there, breaks a little early.)
if (end !== null && this.pauses.some((pause) => end > pause)) {
this.pauses = this.pauses.filter((pause) => pause >= end);
this.endParagraph();
}
this.paragraph.push(text);
this.paragraphEnd = end;
ended = this.endCompleteParagraph();
}
}
// The paragraph's final already cleared the provisional text.
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
}
/** End the paragraph if its last word ends just before a pause. */
private endCompleteParagraph(): boolean {
const end = this.paragraphEnd;
if (!this.paragraph.length || end === null) return false;
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
if (!this.pauses.some(reached)) return false;
this.pauses = this.pauses.filter((pause) => !reached(pause));
this.endParagraph();
return true;
}
private endParagraph(): void {
const text = this.paragraph.join(" ");
this.paragraph = [];
this.paragraphEnd = null;
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
}
private liveText(): string {
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
}
}
+32 -14
View File
@@ -2,6 +2,7 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
import { WavRecorder } from './wav-recorder.ios'
import { pcmLevel } from '../util/audio-level'
@@ -18,10 +19,12 @@ const RAW_RETRY_LIMIT = 5
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
* from the glasses when connected, or the default phone input in preview mode,
* and is transcribed by Apple's on-device recognizer or a cloud provider.
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
* the PCM capture runs open-ended, and pauses commit its segments. When the
* session or the app lifecycle cuts one short it is drained, not discarded;
* restarting it is the controller's call.
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
* is open-ended and has pauses commit its segments, so it needs a cloud
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
* recognizer only handles short dictations. When the session or the app
* lifecycle cuts one short it is drained, not discarded; restarting it is the
* controller's call.
*/
export class IosVoiceControlBridge {
private native: any = null
@@ -40,10 +43,13 @@ export class IosVoiceControlBridge {
private audioTimer: ReturnType<typeof setTimeout> | null = null
private packets = 0
private log: (message: string) => void = () => {}
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
// on-device continuous capture) owns the transcript; FaceclawSpeech then
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
// provider was asked to finalize; it is the status shown when it does.
private cloud: ReconnectingSttClient | null = null
// What transcribes the current capture, for the log.
private recognition = 'on-device'
private cloudEndStatus: string | null = null
private cloudTimer: ReturnType<typeof setTimeout> | null = null
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
@@ -137,7 +143,7 @@ export class IosVoiceControlBridge {
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
})
if (generation !== this.generation || !this.capturing) return
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
this.audioTimer = setTimeout(() => {
this.audioTimer = null
if (generation === this.generation && this.capturing && !this.packets) {
@@ -158,18 +164,27 @@ export class IosVoiceControlBridge {
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
const error = this.startNative(voice, endpointing, true)
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
}
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
const native = this.ensureNative()
const generation = this.generation
const current = () => generation === this.generation
const cloud = createCloudSttClient(voice, {
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
onStatus: status => { if (current()) this.setStatus(status) },
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}, () => this.continuous)
const options = {
language: voice.language,
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
onStatus: (status: string) => { if (current()) this.setStatus(status) },
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
}
const continuous = () => this.continuous
let cloud = createCloudSttClient(voice, options, continuous)
this.recognition = cloud ? 'cloud' : 'on-device'
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
if (!cloud && this.continuous && speechAnalyzerSupported()) {
cloud = createSpeechAnalyzerClient(options, continuous)
this.recognition = 'on-device (SpeechAnalyzer)'
}
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
if (error) return error
@@ -182,8 +197,11 @@ export class IosVoiceControlBridge {
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
}
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
/**
* Whether continuous capture can run with this provider: a cloud one, or
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
*/
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }