Bugfixes in iOS Transcribe and Whisper server integration

This commit is contained in:
James Babcock committed 2026-10-17 13:43:52 -07:00
1 parent 33eef6538c
commit 5bf809d733
12 files changed
+553 -73

No files matched your search

+75
View File
@@ -338,6 +338,81 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
assert.deepEqual(calls, []);
});
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
const calls = [], claims = [];
// Models the bridge's continuous state: listening on a source, or draining after a stop.
const bridge = { source: null, draining: false, recording: '',
supportsContinuous: voice => voice.provider !== 'onboard',
get continuousSource() { return this.source; },
get continuousActive() { return !!this.source || this.draining; },
prepare: async () => true,
startGlassesCapture: async (_session, _voice, _log, _endpointing, continuous) => { calls.push(['glasses', continuous]); bridge.source = 'glasses'; },
startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; },
drain() { if (this.source) { this.source = null; this.draining = true; } },
stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); },
handleSessionEnded() { if (bridge.source === 'glasses') bridge.drain(); },
stopPhoneCapture() { if (bridge.source === 'phone') bridge.drain(); },
recordTo: name => { bridge.recording = name; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, saveTranscribeAudioSetting: { get: () => true } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } : {},
});
const host = Object.create(IosPreviewController.prototype);
Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true,
continuousClients: new Map(), continuousPending: false, continuousCaptureToken: 0, displayReady: Promise.resolve(),
glance: { dismiss() {} }, onError: message => calls.push(['error', message]), logBluetooth() {}, fail() {}, requestShellRender() {},
syncRuntime() {}, flushBluetoothLog() {} });
const settle = () => new Promise(resolve => setImmediate(resolve));
// The bridge's end-of-capture callback, once a drain has finished.
const drained = () => { bridge.draining = false; host.syncContinuousCapture(); };
const session = phase => { host.state = { phase }; if (phase !== 'connected') bridge.handleSessionEnded(); host.syncContinuousSession(); };
host.startContinuousVoiceCapture('transcribe', 'transcribe-1'); await settle();
host.startContinuousVoiceCapture('calculator'); await settle();
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
assert.equal(bridge.recording, 'transcribe-1');
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
host.stopContinuousVoiceCapture('calculator');
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
// A reconnect drains the capture, waits for the glasses, and resumes on them.
calls.length = 0; session('retrying'); drained(); await settle();
assert.deepEqual(calls, []);
assert.deepEqual(claims.at(-1), [false, 'continuous-capture'], 'other mic users may run meanwhile');
session('connected'); await settle();
assert.deepEqual(calls, [['glasses', true]]);
// Locking drains; unlocking resumes.
calls.length = 0; host.setGlassesLocked(true); drained(); await settle();
assert.deepEqual(calls, []);
host.setGlassesLocked(false); await settle();
assert.deepEqual(calls, [['glasses', true]]);
// A failed capture is not retried in a loop, only on the next session.
calls.length = 0; bridge.source = null; drained(); await settle();
assert.deepEqual(calls, []);
// Without glasses the phone mic stands in while in the foreground, and moves to the glasses when they connect.
host.communicator = null; session('disconnected'); await settle();
assert.deepEqual(calls, [['phone', true]]);
calls.length = 0; host.pause(); drained(); await settle();
assert.deepEqual(calls, [], 'no phone capture in the background');
host.active = true; host.syncContinuousCapture(); await settle();
assert.deepEqual(calls, [['phone', true]]);
calls.length = 0; host.communicator = {}; session('connected'); await settle();
assert.deepEqual(calls, [['stop']]);
drained(); await settle();
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
// The last window closing stops it for good.
calls.length = 0; host.stopContinuousVoiceCapture('transcribe'); drained(); session('retrying'); session('connected'); await settle();
assert.deepEqual(calls, [['stop']]);
assert.equal(bridge.recording, '');
assert.deepEqual(claims.at(-1), [false, 'continuous-capture']);
});
test('iOS keyboard action opens the shared destination session and remains blocked while locked', () => {
let opened = 0;
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
+1 -1
View File
@@ -138,7 +138,7 @@ function fixture() {
"../native/compass.ios": { bindCompassSession() {}, receiveCompassEvent() {} },
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {}, onSpeechEnd: () => () => {} } },
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
'../native/preview-display.ios': { PreviewDisplayTarget },
+59 -10
View File
@@ -260,18 +260,67 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
assert.equal(h.clouds[1].continuous(), false);
});
test('a continuous capture asked to record writes every PCM chunk to its WAV and closes it when the capture ends', async () => {
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
const h = harness();
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true, 'transcribe-2026-10-17-120000'); h.release(); await start;
assert.equal(h.recordings.length, 1);
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
h.bridge.recordTo('transcribe-2026-10-17-120000');
// A start that fails before any audio leaves no file behind.
h.native.error = 'No phone microphone is available.';
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
delete h.native.error;
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
assert.equal(h.recordings.length, 0);
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
h.native.pcmHandler(new Uint8Array([5, 6]));
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
assert.deepEqual(h.clouds[0].pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
h.bridge.stop();
assert.equal(h.recordings[0].closed, true);
// Captures that don't ask (push-to-talk, Teleprompter) are not recorded.
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
assert.equal(h.recordings.length, 1);
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
assert.deepEqual(h.clouds.at(-1).pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
// A restarted capture (after a disconnect, say) keeps appending to the same file.
h.bridge.stop();
assert.equal(h.recordings[0].closed, false);
const again = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await again;
h.native.pcmHandler(new Uint8Array([7]));
assert.equal(h.recordings.length, 1);
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
// Push-to-talk on a capture of its own is not recorded.
h.bridge.stop();
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
h.native.pcmHandler(new Uint8Array([8]));
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
h.bridge.recordTo('');
assert.equal(h.recordings[0].closed, true);
});
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
const [cloud] = h.clouds;
assert.equal(h.bridge.continuousSource, 'glasses');
h.bridge.handleSessionEnded();
assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
assert.equal(cloud.stopped, false, 'the open segment is still transcribed');
assert.equal(h.bridge.continuousSource, null); assert.equal(h.bridge.continuousActive, true);
// The controller reports the session state again while it reconnects.
h.bridge.handleSessionEnded();
h.event({ kind: 'ended', message: 'Ready to send' });
assert.equal(cloud.finished, 1);
assert.equal(ends, 0);
cloud.options.onTranscript({ text: 'Last words.', isFinal: true });
assert.equal(cloud.stopped, true); assert.equal(h.bridge.continuousActive, false);
assert.equal(h.text.at(-1).text, 'Last words.');
assert.match(h.bridge.statusText, /resumes when they reconnect/);
assert.equal(ends, 1, 'the end lets the controller restart the capture');
});
test('backgrounding drains a continuous phone capture', async () => {
const h = harness();
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
assert.equal(h.bridge.continuousSource, 'phone');
h.bridge.stopPhoneCapture();
assert.equal(h.native.finished, true); assert.equal(h.clouds[0].stopped, false);
h.event({ kind: 'ended', message: 'Ready to send' });
h.clouds[0].options.onTranscript({ text: 'Tail.', isFinal: true });
assert.equal(h.bridge.continuousActive, false);
assert.equal(h.text.at(-1).text, 'Tail.');
assert.match(h.bridge.statusText, /background/);
});
+36
View File
@@ -141,6 +141,42 @@ test('continuous segments finalize in audio order even when answered out of orde
[['first', true], ['second', true]]);
});
/** Seconds of audio in a transcription request's WAV part. */
function postSeconds(post) {
const body = Buffer.from(post.body);
return body.readUInt32LE(body.indexOf('RIFF') + 40) / 32000;
}
test('nonstop speech is cut at its quietest moment into bounded requests that still leave as one final', async () => {
const h = client();
h.c.start(); await flush();
// 25 s of 50 ms chunks with no pause; chunk 440 (22 s in) is the quietest.
for (let i = 0; i < 500; i++) h.c.acceptPcm(speech(50, i === 440 ? 400 : 3000));
const [, head, tailInterim] = h.s.posts();
assert.equal(postSeconds(head), 22.05, 'the cut follows the quietest chunk');
assert.equal(postSeconds(tailInterim), 2.95, 'the audio after it carries over');
h.c.acceptPcm(speech(1000)); h.c.finish();
const tail = h.s.posts().at(-1);
assert.equal(postSeconds(tail), 3.95);
tail.resolve({ status: 200, text: '{"text":"and the rest."}' }); await flush();
assert.deepEqual(h.events.filter(event => event.isFinal), [], 'the tail waits for the head');
head.resolve({ status: 200, text: '{"text":"The long part"}' }); await flush();
assert.deepEqual(h.events.filter(event => event.isFinal),
[{ text: 'The long part and the rest.', isFinal: true, paragraphBreakAfter: false }]);
});
test('a cut with nothing after it yet ends at the next boundary', async () => {
const h = client();
h.c.start(); await flush();
for (let i = 0; i < 500; i++) h.c.acceptPcm(speech(50, i === 499 ? 400 : 3000));
const head = h.s.posts().at(-1);
assert.equal(postSeconds(head), 25);
head.resolve({ status: 200, text: '{"text":"All of it."}' }); await flush();
assert.deepEqual(h.events.filter(event => event.isFinal), []);
h.c.commitSegment();
assert.deepEqual(h.events.filter(event => event.isFinal), [{ text: 'All of it.', isFinal: true, paragraphBreakAfter: true }]);
});
test('a failed final is retried once, then reported without ending the capture', async () => {
const h = client();
h.c.start(); await flush();