forked from Blendman974/faceclaw
Use iOS SpeechAnalyzer API to support on-device transcription with the Transcribe app
This commit is contained in:
1 parent
210b111e27
commit
6e5864f981
6 files changed
+709
-18
No files matched your search
@@ -1054,7 +1054,7 @@ export class IosPreviewController {
|
||||
this.continuousClients.set(client, options)
|
||||
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||
this.continuousClients.delete(client)
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
this.syncContinuousRecording()
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
||||
|
||||
declare const FaceclawSpeechAnalyzer: any;
|
||||
|
||||
/**
|
||||
* How long finish() waits for the last results: the model only has to settle
|
||||
* the few seconds of audio it hadn't finalized yet.
|
||||
*/
|
||||
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
|
||||
|
||||
/**
|
||||
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
|
||||
* whose last word ends within this much before that (the model's word ends
|
||||
* against the detector's level threshold) leaves nothing before the pause
|
||||
* unfinalized, so its paragraph is complete.
|
||||
*/
|
||||
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
|
||||
|
||||
type AnalyzerEvent = {
|
||||
kind: string;
|
||||
message?: string;
|
||||
locale?: string;
|
||||
text?: string;
|
||||
final?: boolean;
|
||||
start?: number | null;
|
||||
end?: number | null;
|
||||
speechEnd?: number | null;
|
||||
};
|
||||
|
||||
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
|
||||
export function speechAnalyzerSupported(): boolean {
|
||||
try {
|
||||
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* On-device long-form transcription for continuous capture, wrapped like a
|
||||
* cloud provider: it buffers audio until the model is ready (downloading it on
|
||||
* first use) and gets pause commits.
|
||||
*/
|
||||
export function createSpeechAnalyzerClient(
|
||||
options: Omit<CloudSttOptions, "apiKey">,
|
||||
continuous: () => boolean,
|
||||
): ReconnectingSttClient {
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
|
||||
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
|
||||
continuous,
|
||||
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
|
||||
* The model finalizes text in its own pieces, a few seconds behind, and can't
|
||||
* be made to settle at a pause; this gathers its pieces into paragraphs ended
|
||||
* by pauses, like the cloud providers' segments. A paragraph is shown as
|
||||
* provisional text and sent as one final (paragraphBreakAfter) once a final
|
||||
* reaches its pause, and the last one as a plain final when the audio ends.
|
||||
*
|
||||
* Only finals place breaks, by when their last word ends (the model's
|
||||
* word-end times are accurate; first words' starts reach back into the
|
||||
* silence before them, and volatile results only carry the range of all audio
|
||||
* not finalized yet).
|
||||
*/
|
||||
export class SpeechAnalyzerSttClient implements CloudSttClient {
|
||||
private native: any = null;
|
||||
private samples = 0;
|
||||
private finishing = false;
|
||||
// Audio times (s) of pauses whose paragraph hasn't ended yet.
|
||||
private pauses: number[] = [];
|
||||
// Finalized text since the last paragraph break, when its last word ends,
|
||||
// and the model's provisional rest.
|
||||
private paragraph: string[] = [];
|
||||
private paragraphEnd: number | null = null;
|
||||
private volatile = "";
|
||||
|
||||
constructor(private readonly options: CloudSttOptions) {}
|
||||
|
||||
start(): void {
|
||||
if (this.native) return;
|
||||
const native = FaceclawSpeechAnalyzer.new();
|
||||
native.eventHandler = (json: string) => {
|
||||
if (this.native === native) this.receive(JSON.parse(json));
|
||||
};
|
||||
this.native = native;
|
||||
native.startWithLanguage(this.options.language ?? "");
|
||||
}
|
||||
|
||||
acceptPcm(pcm: Uint8Array): void {
|
||||
if (!this.native || this.finishing || !pcm.length) return;
|
||||
this.samples += pcm.length >> 1;
|
||||
const copy = pcm.slice();
|
||||
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
|
||||
}
|
||||
|
||||
/** A pause: it ends the paragraph once the words before it are final. */
|
||||
commitSegment(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
|
||||
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
finish(): void {
|
||||
if (!this.native || this.finishing) return;
|
||||
this.finishing = true;
|
||||
this.native.finish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
const native = this.native;
|
||||
this.native = null;
|
||||
native?.cancel();
|
||||
}
|
||||
|
||||
private receive(event: AnalyzerEvent): void {
|
||||
switch (event.kind) {
|
||||
case "status":
|
||||
this.options.onStatus(event.message ?? "");
|
||||
break;
|
||||
case "ready":
|
||||
this.options.onStatus("Listening (on-device)...");
|
||||
this.options.onReady?.();
|
||||
break;
|
||||
case "transcript":
|
||||
this.transcript(event);
|
||||
break;
|
||||
case "finished": {
|
||||
const text = this.liveText();
|
||||
this.paragraph = [];
|
||||
this.volatile = "";
|
||||
this.stop();
|
||||
this.options.onTranscript({ text, isFinal: true });
|
||||
break;
|
||||
}
|
||||
case "error":
|
||||
this.stop();
|
||||
this.options.onError(event.message ?? "On-device transcription failed.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private transcript(event: AnalyzerEvent): void {
|
||||
const text = (event.text ?? "").trim();
|
||||
let ended = false;
|
||||
if (!event.final) {
|
||||
this.volatile = text;
|
||||
} else {
|
||||
this.volatile = "";
|
||||
if (text) {
|
||||
const end = event.speechEnd ?? event.end ?? null;
|
||||
// Words ending after a pause come after it. (One whose words run
|
||||
// across it, if the model didn't split there, breaks a little early.)
|
||||
if (end !== null && this.pauses.some((pause) => end > pause)) {
|
||||
this.pauses = this.pauses.filter((pause) => pause >= end);
|
||||
this.endParagraph();
|
||||
}
|
||||
this.paragraph.push(text);
|
||||
this.paragraphEnd = end;
|
||||
ended = this.endCompleteParagraph();
|
||||
}
|
||||
}
|
||||
// The paragraph's final already cleared the provisional text.
|
||||
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
||||
}
|
||||
|
||||
/** End the paragraph if its last word ends just before a pause. */
|
||||
private endCompleteParagraph(): boolean {
|
||||
const end = this.paragraphEnd;
|
||||
if (!this.paragraph.length || end === null) return false;
|
||||
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
|
||||
if (!this.pauses.some(reached)) return false;
|
||||
this.pauses = this.pauses.filter((pause) => !reached(pause));
|
||||
this.endParagraph();
|
||||
return true;
|
||||
}
|
||||
|
||||
private endParagraph(): void {
|
||||
const text = this.paragraph.join(" ");
|
||||
this.paragraph = [];
|
||||
this.paragraphEnd = null;
|
||||
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
|
||||
}
|
||||
|
||||
private liveText(): string {
|
||||
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control'
|
||||
import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
@@ -18,10 +19,12 @@ const RAW_RETRY_LIMIT = 5
|
||||
/** iOS implementation of the shared voice dialog's event bridge. Audio comes
|
||||
* from the glasses when connected, or the default phone input in preview mode,
|
||||
* and is transcribed by Apple's on-device recognizer or a cloud provider.
|
||||
* Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only
|
||||
* the PCM capture runs open-ended, and pauses commit its segments. When the
|
||||
* session or the app lifecycle cuts one short it is drained, not discarded;
|
||||
* restarting it is the controller's call.
|
||||
* Continuous capture (Transcribe, Teleprompter) runs on the PCM capture, which
|
||||
* is open-ended and has pauses commit its segments, so it needs a cloud
|
||||
* provider or, on-device, SpeechAnalyzer (iOS 26+): FaceclawSpeech's
|
||||
* recognizer only handles short dictations. When the session or the app
|
||||
* lifecycle cuts one short it is drained, not discarded; restarting it is the
|
||||
* controller's call.
|
||||
*/
|
||||
export class IosVoiceControlBridge {
|
||||
private native: any = null
|
||||
@@ -40,10 +43,13 @@ export class IosVoiceControlBridge {
|
||||
private audioTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private packets = 0
|
||||
private log: (message: string) => void = () => {}
|
||||
// Non-null while a cloud provider owns the transcript; FaceclawSpeech then
|
||||
// Non-null while a PCM transcriber (a cloud provider, or SpeechAnalyzer for
|
||||
// on-device continuous capture) owns the transcript; FaceclawSpeech then
|
||||
// only supplies PCM. cloudEndStatus is set once the audio has ended and the
|
||||
// provider was asked to finalize; it is the status shown when it does.
|
||||
private cloud: ReconnectingSttClient | null = null
|
||||
// What transcribes the current capture, for the log.
|
||||
private recognition = 'on-device'
|
||||
private cloudEndStatus: string | null = null
|
||||
private cloudTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Open-ended capture: runs until stopContinuousCapture, and push-to-talk
|
||||
@@ -137,7 +143,7 @@ export class IosVoiceControlBridge {
|
||||
this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength))
|
||||
})
|
||||
if (generation !== this.generation || !this.capturing) return
|
||||
log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: glasses microphone enabled; using ${this.recognition} recognition`)
|
||||
this.audioTimer = setTimeout(() => {
|
||||
this.audioTimer = null
|
||||
if (generation === this.generation && this.capturing && !this.packets) {
|
||||
@@ -158,18 +164,27 @@ export class IosVoiceControlBridge {
|
||||
this.completion = new Promise(resolve => { this.resolveCompletion = resolve })
|
||||
const error = this.startNative(voice, endpointing, true)
|
||||
if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return }
|
||||
log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`)
|
||||
log(`Voice: phone default microphone enabled; using ${this.recognition} recognition`)
|
||||
}
|
||||
/** Starts the native capture for the chosen provider; returns an actionable error, or ''. */
|
||||
private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string {
|
||||
const native = this.ensureNative()
|
||||
const generation = this.generation
|
||||
const current = () => generation === this.generation
|
||||
const cloud = createCloudSttClient(voice, {
|
||||
onTranscript: event => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: status => { if (current()) this.setStatus(status) },
|
||||
onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}, () => this.continuous)
|
||||
const options = {
|
||||
language: voice.language,
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => { if (current()) this.receiveCloudTranscript(event) },
|
||||
onStatus: (status: string) => { if (current()) this.setStatus(status) },
|
||||
onError: (message: string) => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } },
|
||||
}
|
||||
const continuous = () => this.continuous
|
||||
let cloud = createCloudSttClient(voice, options, continuous)
|
||||
this.recognition = cloud ? 'cloud' : 'on-device'
|
||||
// Continuous capture outlasts FaceclawSpeech's recognizer; SpeechAnalyzer takes the PCM instead.
|
||||
if (!cloud && this.continuous && speechAnalyzerSupported()) {
|
||||
cloud = createSpeechAnalyzerClient(options, continuous)
|
||||
this.recognition = 'on-device (SpeechAnalyzer)'
|
||||
}
|
||||
if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '')
|
||||
const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '')
|
||||
if (error) return error
|
||||
@@ -182,8 +197,11 @@ export class IosVoiceControlBridge {
|
||||
if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return }
|
||||
this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd()
|
||||
}
|
||||
/** Whether continuous capture can run with this provider (Apple's recognizer can't). */
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) }
|
||||
/**
|
||||
* Whether continuous capture can run with this provider: a cloud one, or
|
||||
* on-device with SpeechAnalyzer (FaceclawSpeech's recognizer can't).
|
||||
*/
|
||||
supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) || speechAnalyzerSupported() }
|
||||
get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) }
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
|
||||
Reference in new issue
Block a user