Voice: self-hosted Whisper server provider; Transcribe on iOS

- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible
  Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests,
  with URL, model, language and key settings (Voice). Pause-ended spans are
  one request each, finalized in audio order, with provisional text while a
  span is open. See notes/whisper-server.md.
- Transcribe (and other continuous capture) works on iOS with a cloud or
  Whisper-server provider; push-to-talk shares a running capture.
- Transcribe saves each session's audio as transcribe-<time>.wav in Downloads
  (setting "Save Transcribe audio", on by default), written as it is captured
  so a crash leaves a playable file; a saved transcript uses the same stamp.
- Calculator stops continuous listening when its window closes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
FilipAlexanderandClaude Opus 5.5 committed 2026-10-17 16:43:18 +02:00
1 parent ecb6c7f465
commit e2caf28392
24 files changed
+1065 -43

No files matched your search

+3 -1
View File
@@ -6,7 +6,9 @@ const ts = require('typescript');
const path = require('node:path');
// The provider settings ios-preview-controller hands the voice bridge.
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' } };
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
function load(file, context) {
const sandbox = { exports: {}, ...context };
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
+1
View File
@@ -139,6 +139,7 @@ function fixture() {
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
'../ui/shell/voice-activity': { voiceActivity: { setActive() {}, isActive: () => false, subscribe: () => () => {} } },
'../native/faceclaw-communicator.ios': { FaceclawCommunicatorBridge, resolveIosPeripherals: async addresses => addresses },
'../native/preview-display.ios': { PreviewDisplayTarget },
'./ancs-client': { ANCS_FIRMWARE_VERSION: 16, AncsClient: class { state = 'disconnected'; start() {} stop() {} stopCommand() { return new Uint8Array(); } receive() { return false; } } },
+66 -4
View File
@@ -11,14 +11,20 @@ function harness() {
// Stands in for cloud-stt-provider: any provider but "onboard" is a keyed cloud one.
const provider = {
usesCloudStt: voice => voice.provider !== 'onboard',
createCloudSttClient(voice, options) {
createCloudSttClient(voice, options, continuous) {
if (voice.provider === 'onboard') return null;
const client = { options, pcm: [], started: false, finished: 0, stopped: false,
const client = { options, continuous, pcm: [], started: false, finished: 0, stopped: false,
start() { this.started = true; }, acceptPcm(bytes) { this.pcm.push([...bytes]); },
finish() { this.finished++; }, stop() { this.stopped = true; } };
clouds.push(client); return client;
},
};
const recordings = [];
class FakeRecorder {
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
append(bytes) { this.pcm.push(...bytes); }
close() { this.closed = true; }
}
let authorization = 3, microphoneAuthorization = 3, microphoneRequests = 0, permissionRequests = 0, release;
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
requestAuthorization: done => { permissionRequests++; done(3); },
@@ -26,7 +32,7 @@ function harness() {
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b, bufferFromData: d => d },
require: name => name === './cloud-stt-provider' ? provider : {} };
require: name => name === './cloud-stt-provider' ? provider : name === './wav-recorder.ios' ? { WavRecorder: FakeRecorder } : {} };
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 } }).outputText, sandbox);
const bridge = new sandbox.exports.IosVoiceControlBridge();
@@ -37,7 +43,7 @@ function harness() {
return Promise.resolve();
} };
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
return { bridge, native, session, audio, commands, text, timers, clouds, release: () => release(),
return { bridge, native, session, audio, commands, text, timers, clouds, recordings, release: () => release(),
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
@@ -213,3 +219,59 @@ test('cloud capture times out a missing final, keeps native errors, and ends on
assert.equal(h.clouds[2].started, false, 'no connection without audio');
assert.match(h.bridge.statusText, /No phone microphone/);
});
test('continuous capture needs a cloud provider, shares its mic with push-to-talk, and drains pause segments on stop', async () => {
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
assert.equal(h.bridge.supportsContinuous(h.onboard), false);
assert.equal(h.bridge.supportsContinuous(h.cloud), true);
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
assert.deepEqual(h.native.pcmCapture, { endpointing: false, phone: false });
const [cloud] = h.clouds;
assert.equal(cloud.continuous(), true, 'pauses commit segments while continuous');
// Push-to-talk joins instead of restarting the mic, and releasing it keeps the capture.
await h.bridge.startGlassesCapture(h.session, h.cloud, () => {});
await h.bridge.stopPushToTalk();
assert.equal(h.clouds.length, 1); assert.deepEqual(h.commands, [true]); assert.equal(h.native.finished, undefined);
cloud.options.onTranscript({ text: 'First thought.', isFinal: true, paragraphBreakAfter: true });
const stopped = h.bridge.stopContinuousCapture();
assert.deepEqual(h.commands, [true, false]); assert.equal(h.native.finished, true);
h.event({ kind: 'ended', message: 'Transcription stopped.' });
assert.equal(cloud.finished, 1);
// An earlier pause segment answering late must not end the drain.
cloud.options.onTranscript({ text: 'Second thought.', isFinal: true, paragraphBreakAfter: true });
assert.equal(cloud.stopped, false);
cloud.options.onTranscript({ text: 'The tail.', isFinal: true, paragraphBreakAfter: false });
await stopped;
assert.equal(cloud.stopped, true);
assert.deepEqual(h.text.map(event => event.text), ['First thought.', 'Second thought.', 'The tail.']);
assert.equal(cloud.continuous(), false);
assert.ok(ends >= 1);
});
test('push-to-talk after Transcribe closed starts its own capture instead of joining the finalizing one', async () => {
const h = harness();
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
void h.bridge.stopContinuousCapture();
h.event({ kind: 'ended', message: 'Transcription stopped.' });
assert.equal(h.clouds[0].finished, 1, 'the closed window is still finalizing');
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
assert.equal(h.clouds.length, 2, 'push-to-talk got a capture of its own');
assert.equal(h.clouds[0].stopped, true);
assert.equal(h.clouds[1].continuous(), false);
});
test('a continuous capture asked to record writes every PCM chunk to its WAV and closes it when the capture ends', async () => {
const h = harness();
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true, 'transcribe-2026-10-17-120000'); h.release(); await start;
assert.equal(h.recordings.length, 1);
assert.equal(h.recordings[0].name, 'transcribe-2026-10-17-120000');
h.native.pcmHandler(new Uint8Array([1, 2, 3, 4]));
h.native.pcmHandler(new Uint8Array([5, 6]));
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6]);
assert.deepEqual(h.clouds[0].pcm, [[1, 2, 3, 4], [5, 6]], 'transcription still gets the audio');
h.bridge.stop();
assert.equal(h.recordings[0].closed, true);
// Captures that don't ask (push-to-talk, Teleprompter) are not recorded.
const ptt = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}); h.release(); await ptt;
assert.equal(h.recordings.length, 1);
});
+19
View File
@@ -0,0 +1,19 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
test('the WAV header describes 16 kHz mono PCM16 and its data size', () => {
const { wavHeader } = loader({ DataView }, {})('app/native/wav-header.ts');
const header = Buffer.from(wavHeader(32000));
assert.equal(header.length, 44);
assert.equal(header.toString('ascii', 0, 4), 'RIFF');
assert.equal(header.readUInt32LE(4), 36 + 32000);
assert.equal(header.toString('ascii', 8, 16), 'WAVEfmt ');
assert.equal(header.readUInt16LE(20), 1);
assert.equal(header.readUInt16LE(22), 1);
assert.equal(header.readUInt32LE(24), 16000);
assert.equal(header.readUInt32LE(28), 32000);
assert.equal(header.readUInt16LE(34), 16);
assert.equal(header.toString('ascii', 36, 40), 'data');
assert.equal(header.readUInt32LE(40), 32000);
});
+192
View File
@@ -0,0 +1,192 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function load() {
const timers = [];
const modules = loader({
setTimeout(fn, delay) { timers.push({ fn, delay }); return timers.length; },
clearTimeout() {},
DataView, Math, Date, JSON, Object, String, Error, Promise,
}, {});
return { ...modules('app/native/whisper-server-stt.ts'), timers };
}
/** An http stub whose requests resolve only when the test says so. */
function server(models = '{"data":[{"id":"openai/whisper-large-v3"}]}') {
const requests = [];
const http = (request) => new Promise((resolve, reject) => {
if (request.method === 'GET') { requests.push({ ...request, kind: 'models' }); resolve({ status: 200, text: models }); return; }
requests.push({ ...request, kind: 'transcribe', resolve, reject, fields: fields(request.body) });
});
return { http, requests, posts: () => requests.filter(request => request.kind === 'transcribe') };
}
function fields(body) {
const text = Buffer.from(body).toString('latin1');
const out = {};
for (const match of text.matchAll(/name="([^"]+)"\r\n\r\n([^\r]*)\r\n/g)) out[match[1]] = match[2];
return out;
}
function client(config = {}, s = server()) {
const { WhisperServerSttClient, timers } = load();
const events = [], statuses = [], errors = [];
let ready = 0;
const c = new WhisperServerSttClient({
apiKey: config.apiKey ?? '', onTranscript: event => events.push({ ...event }), onStatus: status => statuses.push(status),
onError: error => errors.push(error), onReady: () => ready++, onDisconnected: message => errors.push(`disconnected: ${message}`),
}, { baseUrl: 'http://whisper-host:8004/v1', model: '', language: '', ...config }, s.http);
return { c, s, events, statuses, errors, timers, ready: () => ready };
}
const flush = () => new Promise(resolve => setImmediate(resolve));
function speech(ms, amplitude = 3000) {
const bytes = new Uint8Array(ms * 32);
for (let i = 0; i < bytes.length; i += 2) {
const sample = i % 4 ? amplitude : -amplitude;
bytes[i] = sample & 255; bytes[i + 1] = (sample >> 8) & 255;
}
return bytes;
}
test('URL normalization accepts a bare host, a /v1 root and the full endpoint', () => {
const { normalizeWhisperServerUrl } = load();
assert.equal(normalizeWhisperServerUrl('http://192.0.2.10:8004'), 'http://192.0.2.10:8004/v1');
assert.equal(normalizeWhisperServerUrl(' http://192.0.2.10:8004/v1/ '), 'http://192.0.2.10:8004/v1');
assert.equal(normalizeWhisperServerUrl('https://stt.example/v1/audio/transcriptions'), 'https://stt.example/v1');
assert.equal(normalizeWhisperServerUrl('192.0.2.10:8004'), '');
assert.equal(normalizeWhisperServerUrl(''), '');
});
test('the WAV header describes 16 kHz mono PCM16 and the multipart body carries the fields', () => {
const { wavFile, multipartBody } = load();
const wav = wavFile([new Uint8Array([1, 2, 3, 4]), new Uint8Array([5, 6])], 6);
const view = Buffer.from(wav);
assert.equal(view.toString('ascii', 0, 4), 'RIFF');
assert.equal(view.readUInt32LE(24), 16000);
assert.equal(view.readUInt16LE(22), 1);
assert.equal(view.readUInt32LE(40), 6);
assert.deepEqual([...wav.subarray(44)], [1, 2, 3, 4, 5, 6]);
const body = Buffer.from(multipartBody('B', { model: 'm', language: 'sk' }, wav)).toString('latin1');
assert.match(body, /^--B\r\nContent-Disposition: form-data; name="model"\r\n\r\nm\r\n/);
assert.match(body, /name="file"; filename="audio.wav"\r\nContent-Type: audio\/wav\r\n\r\nRIFF/);
assert.ok(body.endsWith('\r\n--B--\r\n'));
});
test('start lists models, picks the first id, and push-to-talk delivers one final', async () => {
const h = client();
h.c.start(); await flush();
assert.equal(h.ready(), 1);
assert.equal(h.s.requests[0].url, 'http://whisper-host:8004/v1/models');
h.c.acceptPcm(speech(500));
h.c.finish();
const [post] = h.s.posts();
assert.equal(post.url, 'http://whisper-host:8004/v1/audio/transcriptions');
assert.equal(post.fields.model, 'openai/whisper-large-v3');
assert.equal(post.fields.response_format, 'json');
assert.equal('language' in post.fields, false, 'empty language lets Whisper auto-detect');
assert.equal(post.headers.Authorization, undefined);
post.resolve({ status: 200, text: '{"text":" Remind me to call the bank."}' }); await flush();
assert.deepEqual(h.events, [{ text: 'Remind me to call the bank.', isFinal: true, paragraphBreakAfter: false }]);
});
test('a configured model, language and key are sent as given', async () => {
const h = client({ model: 'whisper-1', language: 'SK', apiKey: 'secret' });
h.c.start(); await flush();
h.c.acceptPcm(speech(300)); h.c.finish();
const [post] = h.s.posts();
assert.equal(post.fields.model, 'whisper-1');
assert.equal(post.fields.language, 'sk');
assert.equal(post.headers.Authorization, 'Bearer secret');
});
test('silence is not sent (Whisper would invent text) but push-to-talk still gets an empty final', async () => {
const h = client();
h.c.start(); await flush();
h.c.acceptPcm(speech(2000, 40));
h.c.finish();
assert.equal(h.s.posts().length, 0);
assert.deepEqual(h.events, [{ text: '', isFinal: true, paragraphBreakAfter: false }]);
});
test('interim requests show provisional text and never overwrite the final', async () => {
const h = client();
h.c.start(); await flush();
h.c.acceptPcm(speech(1600));
const [interim] = h.s.posts();
assert.ok(interim, 'an interim request goes out once enough audio arrived');
h.c.acceptPcm(speech(1600));
assert.equal(h.s.posts().length, 1, 'one interim in flight at a time');
interim.resolve({ status: 200, text: '{"text":"hello"}' }); await flush();
assert.deepEqual(h.events.at(-1), { text: 'hello', transcribeText: 'hello', isFinal: false });
h.c.finish();
const final = h.s.posts().at(-1);
final.resolve({ status: 200, text: '{"text":"hello world"}' }); await flush();
assert.deepEqual(h.events.at(-1), { text: 'hello world', isFinal: true, paragraphBreakAfter: false });
});
test('continuous segments finalize in audio order even when answered out of order', async () => {
const h = client();
h.c.start(); await flush();
h.c.acceptPcm(speech(500)); h.c.commitSegment();
h.c.acceptPcm(speech(500)); h.c.commitSegment();
const [first, second] = h.s.posts();
second.resolve({ status: 200, text: '{"text":"second"}' }); await flush();
assert.deepEqual(h.events.map(event => event.isFinal), [false]);
assert.equal(h.events[0].text, 'second');
first.resolve({ status: 200, text: '{"text":"first"}' }); await flush();
assert.deepEqual(h.events.filter(event => event.isFinal).map(event => [event.text, event.paragraphBreakAfter]),
[['first', true], ['second', true]]);
});
test('a failed final is retried once, then reported without ending the capture', async () => {
const h = client();
h.c.start(); await flush();
h.c.acceptPcm(speech(500)); h.c.commitSegment();
h.s.posts()[0].reject(new Error('timed out')); await flush();
assert.equal(h.timers.at(-1).delay, 750);
h.timers.at(-1).fn();
h.s.posts()[1].resolve({ status: 502, text: 'bad gateway' }); await flush();
assert.match(h.statuses.at(-1), /Whisper server request failed: HTTP 502/);
assert.deepEqual(h.errors, []);
h.c.acceptPcm(speech(500)); h.c.finish();
h.s.posts()[2].resolve({ status: 200, text: '{"text":"still here"}' }); await flush();
assert.equal(h.events.at(-1).text, 'still here');
});
test('an auth rejection is an error, an unreachable server is a disconnect', async () => {
const auth = client();
auth.c.start(); await flush();
auth.c.acceptPcm(speech(500)); auth.c.finish();
auth.s.posts()[0].resolve({ status: 401, text: '' }); await flush();
assert.deepEqual(auth.errors, ['Whisper server rejected the key (401).']);
const down = client({}, { http: () => Promise.reject(new Error('connection refused')) });
down.c.start(); await flush();
assert.equal(down.ready(), 0);
assert.deepEqual(down.errors, ['disconnected: Whisper server unreachable: connection refused']);
});
test('provider selection needs a URL, not a key, and passes the server settings through', () => {
const sockets = [];
const modules = loader({
setTimeout() {}, clearTimeout() {}, console: { log() {}, warn() {} }, DataView,
com: { faceclaw: { app: {
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
FaceclawWebSocket: class { constructor(url) { sockets.push(url); } },
} } },
}, { './cloud-stt': { CLOUD_STT_SAMPLE_RATE: 16000, encodeBase64: () => '', toJavaBytes: bytes => bytes } });
const { createCloudSttClient, usesCloudStt } = modules('app/native/cloud-stt-provider.ts');
const statuses = [];
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
const selection = { provider: 'whisper-server', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
assert.equal(usesCloudStt(selection), false);
assert.equal(createCloudSttClient(selection, options, () => false), null);
assert.deepEqual(statuses, ['No Whisper server URL set; using on-device voice.']);
const configured = { ...selection, whisperServerUrl: 'http://192.0.2.10:8004' };
assert.equal(usesCloudStt(configured), true);
const client = createCloudSttClient(configured, options, () => false);
assert.ok(client.finalTimeoutMs > 5000, 'batch requests get longer than the 5s realtime default');
assert.equal(sockets.length, 0);
});