forked from Blendman974/faceclaw
Bugfixes in iOS Transcribe and Whisper server integration
This commit is contained in:
1 parent
33eef6538c
commit
5bf809d733
12 files changed
+553
-73
No files matched your search
@@ -338,6 +338,81 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
|
||||
const calls = [], claims = [];
|
||||
// Models the bridge's continuous state: listening on a source, or draining after a stop.
|
||||
const bridge = { source: null, draining: false, recording: '',
|
||||
supportsContinuous: voice => voice.provider !== 'onboard',
|
||||
get continuousSource() { return this.source; },
|
||||
get continuousActive() { return !!this.source || this.draining; },
|
||||
prepare: async () => true,
|
||||
startGlassesCapture: async (_session, _voice, _log, _endpointing, continuous) => { calls.push(['glasses', continuous]); bridge.source = 'glasses'; },
|
||||
startPhoneCapture: async (_voice, _log, _endpointing, continuous) => { calls.push(['phone', continuous]); bridge.source = 'phone'; },
|
||||
drain() { if (this.source) { this.source = null; this.draining = true; } },
|
||||
stopContinuousCapture: async () => { calls.push(['stop']); bridge.drain(); },
|
||||
handleSessionEnded() { if (bridge.source === 'glasses') bridge.drain(); },
|
||||
stopPhoneCapture() { if (bridge.source === 'phone') bridge.drain(); },
|
||||
recordTo: name => { bridge.recording = name; } };
|
||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, saveTranscribeAudioSetting: { get: () => true } };
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } : {},
|
||||
});
|
||||
const host = Object.create(IosPreviewController.prototype);
|
||||
Object.assign(host, { active: true, glassesLocked: false, communicator: {}, state: { phase: 'connected' }, sessionConnected: true,
|
||||
continuousClients: new Map(), continuousPending: false, continuousCaptureToken: 0, displayReady: Promise.resolve(),
|
||||
glance: { dismiss() {} }, onError: message => calls.push(['error', message]), logBluetooth() {}, fail() {}, requestShellRender() {},
|
||||
syncRuntime() {}, flushBluetoothLog() {} });
|
||||
const settle = () => new Promise(resolve => setImmediate(resolve));
|
||||
// The bridge's end-of-capture callback, once a drain has finished.
|
||||
const drained = () => { bridge.draining = false; host.syncContinuousCapture(); };
|
||||
const session = phase => { host.state = { phase }; if (phase !== 'connected') bridge.handleSessionEnded(); host.syncContinuousSession(); };
|
||||
|
||||
host.startContinuousVoiceCapture('transcribe', 'transcribe-1'); await settle();
|
||||
host.startContinuousVoiceCapture('calculator'); await settle();
|
||||
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
|
||||
assert.equal(bridge.recording, 'transcribe-1');
|
||||
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
|
||||
host.stopContinuousVoiceCapture('calculator');
|
||||
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
|
||||
|
||||
// A reconnect drains the capture, waits for the glasses, and resumes on them.
|
||||
calls.length = 0; session('retrying'); drained(); await settle();
|
||||
assert.deepEqual(calls, []);
|
||||
assert.deepEqual(claims.at(-1), [false, 'continuous-capture'], 'other mic users may run meanwhile');
|
||||
session('connected'); await settle();
|
||||
assert.deepEqual(calls, [['glasses', true]]);
|
||||
|
||||
// Locking drains; unlocking resumes.
|
||||
calls.length = 0; host.setGlassesLocked(true); drained(); await settle();
|
||||
assert.deepEqual(calls, []);
|
||||
host.setGlassesLocked(false); await settle();
|
||||
assert.deepEqual(calls, [['glasses', true]]);
|
||||
|
||||
// A failed capture is not retried in a loop, only on the next session.
|
||||
calls.length = 0; bridge.source = null; drained(); await settle();
|
||||
assert.deepEqual(calls, []);
|
||||
|
||||
// Without glasses the phone mic stands in while in the foreground, and moves to the glasses when they connect.
|
||||
host.communicator = null; session('disconnected'); await settle();
|
||||
assert.deepEqual(calls, [['phone', true]]);
|
||||
calls.length = 0; host.pause(); drained(); await settle();
|
||||
assert.deepEqual(calls, [], 'no phone capture in the background');
|
||||
host.active = true; host.syncContinuousCapture(); await settle();
|
||||
assert.deepEqual(calls, [['phone', true]]);
|
||||
calls.length = 0; host.communicator = {}; session('connected'); await settle();
|
||||
assert.deepEqual(calls, [['stop']]);
|
||||
drained(); await settle();
|
||||
assert.deepEqual(calls, [['stop'], ['glasses', true]]);
|
||||
|
||||
// The last window closing stops it for good.
|
||||
calls.length = 0; host.stopContinuousVoiceCapture('transcribe'); drained(); session('retrying'); session('connected'); await settle();
|
||||
assert.deepEqual(calls, [['stop']]);
|
||||
assert.equal(bridge.recording, '');
|
||||
assert.deepEqual(claims.at(-1), [false, 'continuous-capture']);
|
||||
});
|
||||
|
||||
test('iOS keyboard action opens the shared destination session and remains blocked while locked', () => {
|
||||
let opened = 0;
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
|
||||
@@ -138,7 +138,7 @@ function fixture() {
|
||||
"../native/compass.ios": { bindCompassSession() {}, receiveCompassEvent() {} },
|
||||
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
|
||||
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
|
||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {}, onSpeechEnd: () => () => {} } },
|
||||
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
|
||||
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
|
||||
'../native/preview-display.ios': { PreviewDisplayTarget },
|
||||
|
||||
+59
-10
@@ -260,18 +260,67 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
|
||||
assert.equal(h.clouds[1].continuous(), false);
|
||||
});
|
||||
|
||||
test('a continuous capture asked to record writes every PCM chunk to its WAV and closes it when the capture ends', async () => {
|
||||
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
|
||||
const h = harness();
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true, 'transcribe-2026-10-17-120000'); h.release(); await start;
|
||||
assert.equal(h.recordings.length, 1);
|
||||
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
|
||||
h.bridge.recordTo('transcribe-2026-10-17-120000');
|
||||
// A start that fails before any audio leaves no file behind.
|
||||
h.native.error = 'No phone microphone is available.';
|
||||
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
||||
delete h.native.error;
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
assert.equal(h.recordings.length, 0);
|
||||
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
|
||||
h.native.pcmHandler(new Uint8Array([5, 6]));
|
||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
|
||||
assert.deepEqual(h.clouds[0].pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
|
||||
h.bridge.stop();
|
||||
assert.equal(h.recordings[0].closed, true);
|
||||
// Captures that don't ask (push-to-talk, Teleprompter) are not recorded.
|
||||
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
|
||||
assert.equal(h.recordings.length, 1);
|
||||
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
|
||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
|
||||
assert.deepEqual(h.clouds.at(-1).pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
|
||||
// A restarted capture (after a disconnect, say) keeps appending to the same file.
|
||||
h.bridge.stop();
|
||||
assert.equal(h.recordings[0].closed, false);
|
||||
const again = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await again;
|
||||
h.native.pcmHandler(new Uint8Array([7]));
|
||||
assert.equal(h.recordings.length, 1);
|
||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
||||
// Push-to-talk on a capture of its own is not recorded.
|
||||
h.bridge.stop();
|
||||
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
|
||||
h.native.pcmHandler(new Uint8Array([8]));
|
||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
||||
h.bridge.recordTo('');
|
||||
assert.equal(h.recordings[0].closed, true);
|
||||
});
|
||||
|
||||
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
|
||||
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
|
||||
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||
const [cloud] = h.clouds;
|
||||
assert.equal(h.bridge.continuousSource, 'glasses');
|
||||
h.bridge.handleSessionEnded();
|
||||
assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
|
||||
assert.equal(cloud.stopped, false, 'the open segment is still transcribed');
|
||||
assert.equal(h.bridge.continuousSource, null); assert.equal(h.bridge.continuousActive, true);
|
||||
// The controller reports the session state again while it reconnects.
|
||||
h.bridge.handleSessionEnded();
|
||||
h.event({ kind: 'ended', message: 'Ready to send' });
|
||||
assert.equal(cloud.finished, 1);
|
||||
assert.equal(ends, 0);
|
||||
cloud.options.onTranscript({ text: 'Last words.', isFinal: true });
|
||||
assert.equal(cloud.stopped, true); assert.equal(h.bridge.continuousActive, false);
|
||||
assert.equal(h.text.at(-1).text, 'Last words.');
|
||||
assert.match(h.bridge.statusText, /resumes when they reconnect/);
|
||||
assert.equal(ends, 1, 'the end lets the controller restart the capture');
|
||||
});
|
||||
|
||||
test('backgrounding drains a continuous phone capture', async () => {
|
||||
const h = harness();
|
||||
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
||||
assert.equal(h.bridge.continuousSource, 'phone');
|
||||
h.bridge.stopPhoneCapture();
|
||||
assert.equal(h.native.finished, true); assert.equal(h.clouds[0].stopped, false);
|
||||
h.event({ kind: 'ended', message: 'Ready to send' });
|
||||
h.clouds[0].options.onTranscript({ text: 'Tail.', isFinal: true });
|
||||
assert.equal(h.bridge.continuousActive, false);
|
||||
assert.equal(h.text.at(-1).text, 'Tail.');
|
||||
assert.match(h.bridge.statusText, /background/);
|
||||
});
|
||||
@@ -141,6 +141,42 @@ test('continuous segments finalize in audio order even when answered out of orde
|
||||
[['first', true], ['second', true]]);
|
||||
});
|
||||
|
||||
/** Seconds of audio in a transcription request's WAV part. */
|
||||
function postSeconds(post) {
|
||||
const body = Buffer.from(post.body);
|
||||
return body.readUInt32LE(body.indexOf('RIFF') + 40) / 32000;
|
||||
}
|
||||
|
||||
test('nonstop speech is cut at its quietest moment into bounded requests that still leave as one final', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
// 25 s of 50 ms chunks with no pause; chunk 440 (22 s in) is the quietest.
|
||||
for (let i = 0; i < 500; i++) h.c.acceptPcm(speech(50, i === 440 ? 400 : 3000));
|
||||
const [, head, tailInterim] = h.s.posts();
|
||||
assert.equal(postSeconds(head), 22.05, 'the cut follows the quietest chunk');
|
||||
assert.equal(postSeconds(tailInterim), 2.95, 'the audio after it carries over');
|
||||
h.c.acceptPcm(speech(1000)); h.c.finish();
|
||||
const tail = h.s.posts().at(-1);
|
||||
assert.equal(postSeconds(tail), 3.95);
|
||||
tail.resolve({ status: 200, text: '{"text":"and the rest."}' }); await flush();
|
||||
assert.deepEqual(h.events.filter(event => event.isFinal), [], 'the tail waits for the head');
|
||||
head.resolve({ status: 200, text: '{"text":"The long part"}' }); await flush();
|
||||
assert.deepEqual(h.events.filter(event => event.isFinal),
|
||||
[{ text: 'The long part and the rest.', isFinal: true, paragraphBreakAfter: false }]);
|
||||
});
|
||||
|
||||
test('a cut with nothing after it yet ends at the next boundary', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
for (let i = 0; i < 500; i++) h.c.acceptPcm(speech(50, i === 499 ? 400 : 3000));
|
||||
const head = h.s.posts().at(-1);
|
||||
assert.equal(postSeconds(head), 25);
|
||||
head.resolve({ status: 200, text: '{"text":"All of it."}' }); await flush();
|
||||
assert.deepEqual(h.events.filter(event => event.isFinal), []);
|
||||
h.c.commitSegment();
|
||||
assert.deepEqual(h.events.filter(event => event.isFinal), [{ text: 'All of it.', isFinal: true, paragraphBreakAfter: true }]);
|
||||
});
|
||||
|
||||
test('a failed final is retried once, then reported without ending the capture', async () => {
|
||||
const h = client();
|
||||
h.c.start(); await flush();
|
||||
|
||||
Reference in new issue
Block a user