forked from Blendman974/faceclaw
274 lines
11 KiB
Swift
274 lines
11 KiB
Swift
@preconcurrency import AVFoundation
|
|
import CoreMedia
|
|
import Foundation
|
|
import Speech
|
|
|
|
/// Long-form on-device transcription with SpeechAnalyzer and SpeechTranscriber
|
|
/// (iOS 26+), for continuous capture (Transcribe, Teleprompter, Calculator):
|
|
/// FaceclawSpeech's SFSpeechRecognizer only handles short dictations. It takes
|
|
/// the same 16 kHz mono PCM16LE the cloud providers get (see
|
|
/// app/native/speech-analyzer-stt.ios.ts) and reports JSON events on the main
|
|
/// queue through eventHandler:
|
|
///
|
|
/// {"kind":"status","message":String}
|
|
/// {"kind":"ready","locale":String} audio is accepted from here on
|
|
/// {"kind":"transcript","text":String,"final":Bool,
|
|
/// "start":Double?,"end":Double?, the result's audio range
|
|
/// "speechEnd":Double?} when its last word ends
|
|
/// {"kind":"finished"} after finish(), once the last result is in
|
|
/// {"kind":"error","message":String}
|
|
///
|
|
/// Times are seconds of audio since start. Volatile results (final: false)
|
|
/// cover all audio not finalized yet, replacing the previous volatile one;
|
|
/// their word times are only as fine as the result's range. A final's first
|
|
/// word reaches back into the silence before it, so only word ends are
|
|
/// reported.
|
|
///
|
|
/// There is no way to settle text at a pause: on macOS 26.4 any mid-stream
|
|
/// SpeechAnalyzer.finalize(through:) left the next phrase's final as dots,
|
|
/// while the model splits its finals at pauses on its own.
|
|
/// Called from JavaScript, so on the main thread.
|
|
@MainActor
|
|
@objc(FaceclawSpeechAnalyzer)
|
|
@objcMembers
|
|
public class FaceclawSpeechAnalyzer: NSObject {
|
|
public var eventHandler: ((String) -> Void)?
|
|
// An AnalyzerEngine, typed loosely so the class itself loads before iOS 26.
|
|
private var engine: AnyObject?
|
|
|
|
/// Whether this device can transcribe with SpeechTranscriber (iOS 26 and supported hardware).
|
|
public static func isSupported() -> Bool {
|
|
if #available(iOS 26.0, *) { return SpeechTranscriber.isAvailable }
|
|
return false
|
|
}
|
|
|
|
/// `language` is a locale identifier ("en", "en-US"), or "" for the device's preferred language.
|
|
public func start(language: String) {
|
|
cancel()
|
|
guard #available(iOS 26.0, *) else {
|
|
emit(["kind": "error", "message": "On-device transcription needs iOS 26 or later."])
|
|
return
|
|
}
|
|
let engine = AnalyzerEngine { [weak self] event in self?.emit(event) }
|
|
self.engine = engine
|
|
engine.start(language: language)
|
|
}
|
|
|
|
/// 16 kHz mono PCM16LE; ignored until "ready".
|
|
public func acceptPcm(_ pcm: Data) {
|
|
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.accept(pcm) }
|
|
}
|
|
|
|
/// End of audio: finalize the rest, then report "finished".
|
|
public func finish() {
|
|
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.finish() }
|
|
}
|
|
|
|
/// Abandon the session without further events.
|
|
public func cancel() {
|
|
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.cancel() }
|
|
engine = nil
|
|
}
|
|
|
|
private func emit(_ event: [String: Any]) {
|
|
// JSONSerialization raises (not throws) on NaN and other invalid values.
|
|
guard JSONSerialization.isValidJSONObject(event),
|
|
let data = try? JSONSerialization.data(withJSONObject: event),
|
|
let json = String(data: data, encoding: .utf8) else { return }
|
|
eventHandler?(json)
|
|
}
|
|
}
|
|
|
|
@available(iOS 26.0, *)
|
|
@MainActor
|
|
private final class AnalyzerEngine {
|
|
private static let sampleRate = 16000.0
|
|
private let emit: ([String: Any]) -> Void
|
|
private let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: AnalyzerEngine.sampleRate, channels: 1, interleaved: true)!
|
|
private var analyzer: SpeechAnalyzer?
|
|
private var input: AsyncStream<AnalyzerInput>.Continuation?
|
|
private var analyzerFormat: AVAudioFormat?
|
|
// Nil when the analyzer takes the input format as it is.
|
|
private var converter: AVAudioConverter?
|
|
private var ready = false
|
|
private var finishing = false
|
|
private var closed = false
|
|
private var setup: Task<Void, Never>?
|
|
private var results: Task<Void, Never>?
|
|
|
|
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
|
|
|
|
func start(language: String) {
|
|
setup = Task { [weak self] in await self?.prepare(language: language) }
|
|
}
|
|
|
|
/// Events stop once the session is cancelled, even from work still winding down.
|
|
private func send(_ event: [String: Any]) {
|
|
if !closed { emit(event) }
|
|
}
|
|
|
|
private func prepare(language: String) async {
|
|
send(["kind": "status", "message": "Preparing on-device transcription…"])
|
|
let identifier = language.isEmpty ? (Locale.preferredLanguages.first ?? "en-US") : language
|
|
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: identifier)) else {
|
|
fail("On-device transcription doesn't support \(identifier). Choose a cloud or Whisper server transcription provider (Settings > Voice).")
|
|
return
|
|
}
|
|
if closed { return }
|
|
let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [],
|
|
reportingOptions: [.volatileResults], attributeOptions: [.audioTimeRange])
|
|
do {
|
|
// Keep the model installed between sessions (best effort).
|
|
if !(await AssetInventory.reservedLocales).contains(locale) {
|
|
_ = try? await AssetInventory.reserve(locale: locale)
|
|
}
|
|
if closed { return }
|
|
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
|
if closed { return }
|
|
try await download(request)
|
|
}
|
|
if closed { return }
|
|
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else {
|
|
fail("On-device transcription has no usable audio format.")
|
|
return
|
|
}
|
|
if closed { return }
|
|
if format != pcmFormat {
|
|
guard let converter = AVAudioConverter(from: pcmFormat, to: format) else {
|
|
fail("On-device transcription can't convert the microphone audio.")
|
|
return
|
|
}
|
|
self.converter = converter
|
|
}
|
|
analyzerFormat = format
|
|
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
|
let (stream, input) = AsyncStream<AnalyzerInput>.makeStream()
|
|
self.analyzer = analyzer
|
|
self.input = input
|
|
results = Task { [weak self] in await self?.collect(transcriber) }
|
|
try await analyzer.prepareToAnalyze(in: format)
|
|
try await analyzer.start(inputSequence: stream)
|
|
if closed { return }
|
|
ready = true
|
|
send(["kind": "ready", "locale": locale.identifier])
|
|
if finishing { await finishAnalysis() }
|
|
} catch {
|
|
fail("On-device transcription: \(error.localizedDescription)")
|
|
}
|
|
}
|
|
|
|
/// Fetch the locale's model, reporting progress while it downloads.
|
|
private func download(_ request: AssetInstallationRequest) async throws {
|
|
let progress = request.progress
|
|
let reporter = Task { [weak self] in
|
|
while !Task.isCancelled {
|
|
let percent = Int((progress.fractionCompleted * 100).rounded())
|
|
self?.send(["kind": "status", "message": "Downloading on-device speech model… \(percent)%"])
|
|
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
|
}
|
|
}
|
|
defer { reporter.cancel() }
|
|
try await request.downloadAndInstall()
|
|
}
|
|
|
|
func accept(_ pcm: Data) {
|
|
guard ready, !finishing, !closed, let input, let analyzerFormat else { return }
|
|
let frames = AVAudioFrameCount(pcm.count / 2)
|
|
guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames),
|
|
let channel = buffer.int16ChannelData else { return }
|
|
buffer.frameLength = frames
|
|
pcm.withUnsafeBytes { raw in
|
|
if let base = raw.baseAddress { memcpy(channel[0], base, Int(frames) * 2) }
|
|
}
|
|
guard let converter else {
|
|
input.yield(AnalyzerInput(buffer: buffer))
|
|
return
|
|
}
|
|
let capacity = AVAudioFrameCount((Double(frames) * analyzerFormat.sampleRate / AnalyzerEngine.sampleRate).rounded(.up)) + 32
|
|
guard let converted = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return }
|
|
var consumed = false
|
|
var error: NSError?
|
|
// noDataNow, not endOfStream: the converter keeps its resampling state for the next chunk.
|
|
converter.convert(to: converted, error: &error) { _, status in
|
|
if consumed {
|
|
status.pointee = .noDataNow
|
|
return nil
|
|
}
|
|
consumed = true
|
|
status.pointee = .haveData
|
|
return buffer
|
|
}
|
|
if error == nil && converted.frameLength > 0 { input.yield(AnalyzerInput(buffer: converted)) }
|
|
}
|
|
|
|
func finish() {
|
|
guard !finishing, !closed else { return }
|
|
finishing = true
|
|
// Before "ready", prepare() finishes once the analyzer is running.
|
|
if ready { Task { await finishAnalysis() } }
|
|
}
|
|
|
|
private func finishAnalysis() async {
|
|
input?.finish()
|
|
input = nil
|
|
do {
|
|
try await analyzer?.finalizeAndFinishThroughEndOfInput()
|
|
} catch {
|
|
fail("On-device transcription: \(error.localizedDescription)")
|
|
}
|
|
// collect() reports "finished" once the results sequence ends.
|
|
}
|
|
|
|
func cancel() {
|
|
guard !closed else { return }
|
|
closed = true
|
|
input?.finish()
|
|
input = nil
|
|
setup?.cancel()
|
|
results?.cancel()
|
|
if let analyzer { Task { await analyzer.cancelAndFinishNow() } }
|
|
analyzer = nil
|
|
}
|
|
|
|
private func collect(_ transcriber: SpeechTranscriber) async {
|
|
do {
|
|
for try await result in transcriber.results {
|
|
if closed { return }
|
|
send(event(for: result))
|
|
}
|
|
send(["kind": "finished"])
|
|
} catch {
|
|
fail("On-device transcription: \(error.localizedDescription)")
|
|
}
|
|
}
|
|
|
|
private func event(for result: SpeechTranscriber.Result) -> [String: Any] {
|
|
let text = result.text
|
|
var speechEnd: Double?
|
|
for run in text.runs {
|
|
guard let range = run.audioTimeRange,
|
|
!String(text[run.range].characters).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue }
|
|
speechEnd = finite(range.end) ?? speechEnd
|
|
}
|
|
return [
|
|
"kind": "transcript",
|
|
"text": String(text.characters),
|
|
"final": result.isFinal,
|
|
"start": finite(result.range.start) ?? NSNull(),
|
|
"end": finite(result.range.end) ?? NSNull(),
|
|
"speechEnd": speechEnd ?? NSNull(),
|
|
]
|
|
}
|
|
|
|
private func finite(_ time: CMTime) -> Double? {
|
|
let seconds = time.seconds
|
|
return seconds.isFinite ? seconds : nil
|
|
}
|
|
|
|
private func fail(_ message: String) {
|
|
guard !closed else { return }
|
|
emit(["kind": "error", "message": message])
|
|
cancel()
|
|
}
|
|
}
|