forked from Blendman974/faceclaw
Use iOS SpeechAnalyzer API to support on-device transcription with the Transcribe app
This commit is contained in:
1 parent
210b111e27
commit
6e5864f981
6 files changed
+709
-18
No files matched your search
@@ -7,7 +7,7 @@ function harness() {
|
||||
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; },
|
||||
startPcmCaptureWithEndpointingPhone(endpointing, phone) { this.pcmCapture = { endpointing, phone }; return this.error ?? ''; },
|
||||
cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
|
||||
const audio = [], timers = new Map(), commands = [], text = [], clouds = [];
|
||||
const audio = [], timers = new Map(), commands = [], text = [], clouds = [], analyzers = [];
|
||||
// Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one.
|
||||
const provider = {
|
||||
usesCloudStt: voice => voice.provider !== 'onboard',
|
||||
@@ -19,6 +19,17 @@ function harness() {
|
||||
clouds.push(client); return client;
|
||||
},
|
||||
};
|
||||
// Stands in for speech-analyzer-stt: SpeechAnalyzer (iOS 26+) when the test says the device has it.
|
||||
let analyzerSupported = false;
|
||||
const analyzer = {
|
||||
speechAnalyzerSupported: () => analyzerSupported,
|
||||
createSpeechAnalyzerClient(options, continuous) {
|
||||
const client = { options, continuous, pcm: [], started: false, finished: 0, stopped: false,
|
||||
start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); },
|
||||
finish() { this.finished++; }, stop() { this.stopped = true; } };
|
||||
analyzers.push(client); return client;
|
||||
},
|
||||
};
|
||||
const recordings = [];
|
||||
class FakeRecorder {
|
||||
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
|
||||
@@ -32,7 +43,8 @@ function harness() {
|
||||
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
|
||||
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
|
||||
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
|
||||
require: name => name === './cloud-stt-provider' ? provider : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
|
||||
require: name => name === './cloud-stt-provider' ? provider : name === './speech-analyzer-stt.ios' ? analyzer
|
||||
: name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
|
||||
const bridge = new sandbox.exports.IosVoiceControlBridge();
|
||||
@@ -43,7 +55,8 @@ function harness() {
|
||||
return Promise.resolve();
|
||||
} };
|
||||
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, recordings, release: () => release(),
|
||||
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
|
||||
analyzerSupported: value => { analyzerSupported = value; },
|
||||
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
||||
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
||||
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
|
||||
@@ -248,6 +261,38 @@ test('continuous capture needs a cloud provider, shares its mic with push-to-tal
|
||||
assert.ok(ends >= 1);
|
||||
});
|
||||
|
||||
test('on-device continuous capture runs on SpeechAnalyzer where supported; push-to-talk keeps the dictation recognizer', async () => {
|
||||
const h = harness();
|
||||
assert.equal(h.bridge.supportsContinuous(h.onboard), false, 'before iOS 26 on-device needs a cloud provider');
|
||||
h.analyzerSupported(true);
|
||||
assert.equal(h.bridge.supportsContinuous(h.onboard), true);
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}, false, true); h.release(); await start;
|
||||
assert.deepEqual({ ...h.native.pcmCapture }, { endpointing: false, phone: false }, 'native only decodes PCM');
|
||||
assert.equal(h.native.endpointing, undefined, "Apple's dictation recognizer stays idle");
|
||||
const [analyzer] = h.analyzers;
|
||||
assert.equal(analyzer.started, true); assert.equal(analyzer.continuous(), true);
|
||||
assert.equal(h.clouds.length, 0);
|
||||
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
|
||||
assert.deepEqual(analyzer.pcm, [[1, 2, 3, 4]]);
|
||||
analyzer.options.onTranscript({ text: 'First thought.', isFinal: true, paragraphBreakAfter: true });
|
||||
const stopped = h.bridge.stopContinuousCapture();
|
||||
h.event({ kind: 'ended', message: 'Transcription stopped.' });
|
||||
assert.equal(analyzer.finished, 1);
|
||||
analyzer.options.onTranscript({ text: 'The tail.', isFinal: true });
|
||||
await stopped;
|
||||
assert.equal(analyzer.stopped, true);
|
||||
assert.deepEqual(h.text.map(event => event.text), ['First thought.', 'The tail.']);
|
||||
// Push-to-talk (not continuous) still uses FaceclawSpeech's own recognizer.
|
||||
const ptt = h.bridge.startGlassesCapture(h.session, h.onboard, () => {}); h.release(); await ptt;
|
||||
assert.equal(h.analyzers.length, 1); assert.equal(h.native.endpointing, false);
|
||||
});
|
||||
|
||||
test('a cloud provider keeps continuous capture even where SpeechAnalyzer is supported', async () => {
|
||||
const h = harness(); h.analyzerSupported(true);
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
assert.equal(h.clouds.length, 1); assert.equal(h.analyzers.length, 0);
|
||||
});
|
||||
|
||||
test('push-to-talk after Transcribe closed starts its own capture instead of joining the finalizing one', async () => {
|
||||
const h = harness();
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
/** FaceclawSpeechAnalyzer.swift's surface; `emit` plays its JSON events. */
|
||||
class FakeAnalyzer {
|
||||
static supported = true;
|
||||
static instances = [];
|
||||
static new() { const native = new FakeAnalyzer(); FakeAnalyzer.instances.push(native); return native; }
|
||||
static isSupported() { return FakeAnalyzer.supported; }
|
||||
constructor() { this.pcm = []; this.finished = 0; this.cancelled = 0; this.language = null; }
|
||||
startWithLanguage(language) { this.language = language; }
|
||||
acceptPcm(data) { this.pcm.push([...data]); }
|
||||
finish() { this.finished++; }
|
||||
cancel() { this.cancelled++; }
|
||||
emit(event) { this.eventHandler(JSON.stringify(event)); }
|
||||
}
|
||||
|
||||
function load() {
|
||||
FakeAnalyzer.instances = [];
|
||||
FakeAnalyzer.supported = true;
|
||||
const timers = new Set();
|
||||
const modules = loader({
|
||||
FaceclawSpeechAnalyzer: FakeAnalyzer,
|
||||
NSData: { dataWithBytesLength: (buffer, length) => new Uint8Array(buffer, 0, length) },
|
||||
interop: { handleof: buffer => buffer },
|
||||
setTimeout(fn) { timers.add(fn); return fn; },
|
||||
clearTimeout(fn) { timers.delete(fn); },
|
||||
}, {});
|
||||
return { ...modules('app/native/speech-analyzer-stt.ios.ts'), timers };
|
||||
}
|
||||
|
||||
function client(language = '') {
|
||||
const { SpeechAnalyzerSttClient } = load();
|
||||
const events = [], statuses = [], errors = [];
|
||||
let ready = 0;
|
||||
const c = new SpeechAnalyzerSttClient({
|
||||
apiKey: '', language, onTranscript: event => events.push({ ...event }), onStatus: status => statuses.push(status),
|
||||
onError: error => errors.push(error), onReady: () => ready++,
|
||||
});
|
||||
c.start();
|
||||
const native = FakeAnalyzer.instances[0];
|
||||
return { c, native, events, statuses, errors, ready: () => ready };
|
||||
}
|
||||
|
||||
/** `seconds` of 16 kHz PCM16LE silence. */
|
||||
const pcm = seconds => new Uint8Array(Math.round(seconds * 16000) * 2);
|
||||
/** A result as FaceclawSpeechAnalyzer.swift reports it: its audio range and when its last word ends. */
|
||||
const result = (text, final, start, end, speechEnd = end) => ({ kind: 'transcript', text, final, start, end, speechEnd });
|
||||
const finals = h => h.events.filter(event => event.isFinal);
|
||||
|
||||
test('starts in the requested language and reports readiness as a provider would', () => {
|
||||
const h = client('es');
|
||||
assert.equal(h.native.language, 'es');
|
||||
h.native.emit({ kind: 'status', message: 'Downloading on-device speech model… 40%' });
|
||||
h.native.emit({ kind: 'ready', locale: 'es_ES' });
|
||||
assert.deepEqual(h.statuses, ['Downloading on-device speech model… 40%', 'Listening (on-device)...']);
|
||||
assert.equal(h.ready(), 1);
|
||||
});
|
||||
|
||||
// Recorded from SpeechAnalyzer on macOS 26.4 (scripted speech, 1.5 s quiet, a
|
||||
// pause, more speech): volatile ranges start where the last final ended, and
|
||||
// the second final's first word reaches back into the silence (5.04 s; the
|
||||
// speech starts at 6.18 s).
|
||||
test('real model output: the first paragraph ends as soon as its last words are final', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.native.emit(result('Hello there.', false, 0, 4.1));
|
||||
h.native.emit(result('Hello there.', true, 0, 0.66, 0.6));
|
||||
h.native.emit(result(' This is the firstst paragraph of the test spoken ', false, 0.66, 4.1815));
|
||||
h.c.acceptPcm(pcm(5.6815));
|
||||
h.c.commitSegment();
|
||||
assert.equal(finals(h).length, 0, 'words before the pause are still provisional');
|
||||
h.native.emit(result(' This is the firstst paragraph of the test spoken before a pause.', false, 0.66, 7.7815));
|
||||
h.native.emit(result(' This is the first paragraph of the test, spoken before a pause.', true, 0.66, 4.38, 4.32));
|
||||
assert.deepEqual(finals(h), [{ text: 'Hello there. This is the first paragraph of the test, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
h.native.emit(result(' And this is a second', false, 4.38, 8.1815));
|
||||
assert.deepEqual(h.events.at(-1), { text: 'And this is a second', isFinal: false }, 'new speech shows in the new paragraph');
|
||||
h.native.emit(result(' And this is a second paragraph, which comes after the pause.', true, 5.04, 9.18, 9.12));
|
||||
h.c.finish();
|
||||
assert.equal(h.native.finished, 1);
|
||||
h.native.emit({ kind: 'finished' });
|
||||
assert.deepEqual(h.events.at(-1), { text: 'And this is a second paragraph, which comes after the pause.', isFinal: true });
|
||||
assert.equal(finals(h).length, 2);
|
||||
assert.equal(h.native.cancelled, 1, 'the finished session is released');
|
||||
h.native.emit(result('late', false, 9.2, 9.5));
|
||||
assert.equal(h.events.at(-1).text, 'And this is a second paragraph, which comes after the pause.', 'events after stop are ignored');
|
||||
});
|
||||
|
||||
test('a pause after words already final ends the paragraph at once, keeping later provisional text', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.native.emit(result('Hello there.', true, 0, 1.0));
|
||||
h.native.emit(result('How are you?', true, 1.0, 1.8));
|
||||
h.c.acceptPcm(pcm(3.4));
|
||||
h.native.emit(result('Fine', false, 1.8, 3.4));
|
||||
h.c.commitSegment();
|
||||
assert.deepEqual(h.events.slice(-2), [
|
||||
{ text: 'Hello there. How are you?', isFinal: true, paragraphBreakAfter: true },
|
||||
{ text: 'Fine', isFinal: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test('a final ending mid-sentence before the pause waits for the rest of the sentence', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.c.acceptPcm(pcm(5.7));
|
||||
h.c.commitSegment();
|
||||
h.native.emit(result('This is the first paragraph, spoken', true, 0.5, 3.6));
|
||||
assert.equal(finals(h).length, 0);
|
||||
h.native.emit(result('before a pause.', true, 3.6, 4.3));
|
||||
assert.deepEqual(finals(h), [{ text: 'This is the first paragraph, spoken before a pause.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
});
|
||||
|
||||
test('without a final near the pause, a final ending after it starts the next paragraph', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.native.emit(result('Before.', true, 0, 2.0));
|
||||
// Background noise kept the pause detector from firing until well after the words.
|
||||
h.c.acceptPcm(pcm(8));
|
||||
h.c.commitSegment();
|
||||
assert.equal(finals(h).length, 0);
|
||||
h.native.emit(result('After.', true, 2.0, 9.5));
|
||||
assert.deepEqual(finals(h), [{ text: 'Before.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
assert.deepEqual(h.events.at(-1), { text: 'After.', isFinal: false });
|
||||
});
|
||||
|
||||
test('a pause with nothing before it adds no break', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
h.c.acceptPcm(pcm(3));
|
||||
h.c.commitSegment();
|
||||
h.native.emit(result('First words.', true, 0, 4.2));
|
||||
h.native.emit(result('More.', true, 4.2, 5.0));
|
||||
assert.equal(finals(h).length, 0);
|
||||
assert.deepEqual(h.events.at(-1), { text: 'First words. More.', isFinal: false });
|
||||
});
|
||||
|
||||
test('errors end the session and reach the provider\'s error path', () => {
|
||||
const h = client();
|
||||
h.native.emit({ kind: 'error', message: "On-device transcription doesn't support xx." });
|
||||
assert.deepEqual(h.errors, ["On-device transcription doesn't support xx."]);
|
||||
assert.equal(h.native.cancelled, 1);
|
||||
});
|
||||
|
||||
test('the wrapped client holds audio until the model is ready, then streams it', () => {
|
||||
const { createSpeechAnalyzerClient, speechAnalyzerSupported } = load();
|
||||
assert.equal(speechAnalyzerSupported(), true);
|
||||
FakeAnalyzer.supported = false;
|
||||
assert.equal(speechAnalyzerSupported(), false);
|
||||
const events = [];
|
||||
const wrapped = createSpeechAnalyzerClient({ onTranscript: event => events.push({ ...event }), onStatus() {}, onError() {} }, () => true);
|
||||
wrapped.start();
|
||||
const native = FakeAnalyzer.instances[0];
|
||||
wrapped.acceptPcm(new Uint8Array([1, 0, 2, 0]));
|
||||
assert.equal(native.pcm.length, 0, 'buffered while the model loads');
|
||||
native.emit({ kind: 'ready', locale: 'en_US' });
|
||||
wrapped.acceptPcm(new Uint8Array([3, 0]));
|
||||
assert.deepEqual(native.pcm, [[1, 0, 2, 0], [3, 0]]);
|
||||
wrapped.stop();
|
||||
assert.equal(native.cancelled, 1);
|
||||
});
|
||||
Reference in new issue
Block a user