iOS porting: Main screen and compositor

This commit is contained in:
James Babcock committed 2026-09-05 22:59:37 -07:00
1 parent 61b3bf8b7c
commit 99793a37a7
24 files changed
+1124 -70

No files matched your search

+195
View File
@@ -0,0 +1,195 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function harness(provider = 'elevenlabs', continuous = true) {
const timers = new Map();
let timerId = 0;
const sockets = [];
const events = [], statuses = [], errors = [];
const load = loader({
setTimeout(fn, delay) { const id = ++timerId; timers.set(id, { fn, delay }); return id; },
clearTimeout(id) { timers.delete(id); },
console: { log() {}, warn() {} },
com: { faceclaw: { app: {
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
FaceclawWebSocket: class {
constructor(url, listener) { Object.assign(this, { url, listener, sent: [], closed: false, reject: false }); sockets.push(this); }
sendText(text) { if (this.reject) return false; this.sent.push(text); return true; }
sendBinary(bytes) { if (this.reject) return false; this.sent.push(bytes); return true; }
close() { this.closed = true; }
}, } }, },
}, {
'./cloud-stt': {
CLOUD_STT_SAMPLE_RATE: 16000,
encodeBase64: bytes => Buffer.from(bytes).toString('base64'),
toJavaBytes: bytes => bytes,
},
});
const names = { elevenlabs: 'ElevenLabsSttClient', openai: 'OpenAiRealtimeSttClient', soniox: 'SonioxSttClient' };
const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]];
const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts');
const client = new ReconnectingSttClient(options => new Provider(options), {
apiKey: 'test', onTranscript: event => events.push(event),
onStatus: status => statuses.push(status), onError: error => errors.push(error),
}, () => continuous);
client.start();
return { client, sockets, events, statuses, errors, timers,
open() { sockets.at(-1).listener.onOpen(); },
message(message, socket = sockets.at(-1)) { socket.listener.onTextMessage(JSON.stringify(message)); },
tick() { const [id, timer] = timers.entries().next().value; timers.delete(id); timer.fn(); return timer.delay; },
};
}
function pcm(amplitude, ms = 50) {
const bytes = new Uint8Array(ms * 32);
for (let i = 0; i < bytes.length; i += 2) { bytes[i] = amplitude & 255; bytes[i + 1] = amplitude >> 8; }
return bytes;
}
for (const provider of ['elevenlabs', 'openai', 'soniox']) {
test(`${provider}: failure and close reconnect, stale callbacks are ignored, stop cancels retry`, () => {
const h = harness(provider);
h.open();
const old = h.sockets[0];
old.listener.onFailure('network unavailable');
old.listener.onClosed(); // Both callbacks must schedule only one retry.
assert.equal(h.timers.size, 1);
assert.equal(old.closed, true);
h.client.acceptPcm(pcm(2000));
assert.equal(h.tick(), 1000);
assert.equal(h.sockets.length, 2);
h.open();
assert.ok(h.sockets[1].sent.length > 0);
const eventCount = h.events.length;
old.listener.onTextMessage(JSON.stringify({ message_type: 'partial_transcript', text: 'stale' }));
old.listener.onOpen();
assert.equal(h.events.length, eventCount);
h.sockets[1].listener.onClosed();
assert.equal(h.timers.size, 1);
h.client.stop();
assert.equal(h.timers.size, 0);
});
test(`${provider}: rejected audio send triggers recovery and requeues the packet`, () => {
const h = harness(provider);
h.open();
h.sockets[0].reject = true;
h.client.acceptPcm(pcm(2000));
assert.equal(h.timers.size, 1);
h.tick(); h.open();
assert.ok(h.sockets[1].sent.length > (provider === 'elevenlabs' ? 0 : 1));
h.client.stop();
});
test(`${provider}: auth failure stops instead of retrying indefinitely`, () => {
const h = harness(provider);
h.sockets[0].listener.onFailure("Expected HTTP 101 response but was '401 Unauthorized'");
assert.equal(h.timers.size, 0);
assert.equal(h.errors.length, 1);
});
}
test('reconnect preserves visible partials once and does not replay already-sent audio', () => {
const h = harness(); h.open();
h.client.acceptPcm(pcm(2000));
h.message({ message_type: 'partial_transcript', text: 'Before outage' });
h.sockets[0].listener.onFailure('offline');
assert.equal(h.events.at(-1).text, 'Before outage');
assert.equal(h.events.at(-1).isFinal, true);
h.tick(); h.open();
assert.equal(h.sockets[1].sent.length, 0);
h.message({ message_type: 'partial_transcript', text: 'After outage' });
assert.equal(h.events.at(-1).text, 'After outage');
h.client.stop();
});
test('pending audio is bounded to ten seconds and backoff is capped', () => {
const h = harness();
for (let i = 0; i < 20; i++) h.client.acceptPcm(pcm(i, 1000));
h.open();
const sentBytes = h.sockets[0].sent.reduce((sum, text) => sum + Buffer.from(JSON.parse(text).audio_base_64, 'base64').length, 0);
assert.equal(sentBytes, 320000);
for (const delay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) {
h.sockets.at(-1).listener.onFailure('offline');
assert.equal(h.tick(), delay);
}
h.client.stop();
});
test('finishing while disconnected cancels retries; finishing on an open socket times out cleanly', () => {
const h = harness(); h.open();
h.sockets[0].listener.onClosed();
h.client.finish();
assert.equal(h.timers.size, 0);
const live = harness(); live.open();
live.client.finish();
assert.equal(live.tick(), 5000);
assert.equal(live.sockets[0].closed, true);
live.sockets[0].listener.onFailure('late');
assert.equal(live.timers.size, 0);
});
test('only continuous capture commits at pauses, keeping ElevenLabs connected', () => {
for (const continuous of [false, true]) {
const h = harness('elevenlabs', continuous); h.open();
h.client.acceptPcm(pcm(3000));
h.client.acceptPcm(pcm(0, 1500));
const commits = h.sockets[0].sent.map(JSON.parse).filter(m => m.commit);
assert.equal(commits.length, continuous ? 1 : 0);
if (continuous) {
h.message({ message_type: 'committed_transcript', text: 'First.' });
h.message({ message_type: 'committed_transcript_with_timestamps', text: 'First.' });
assert.equal(h.events.length, 1);
assert.equal(h.events[0].paragraphBreakAfter, true);
assert.equal(h.sockets[0].closed, false);
}
h.client.stop();
}
});
test('OpenAI keeps successive items separate and drains out-of-order completions in audio order', () => {
const h = harness('openai'); h.open();
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
h.message({ type: 'input_audio_buffer.committed', item_id: 'a' });
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'a', delta: 'Fir' });
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
h.message({ type: 'input_audio_buffer.committed', item_id: 'b' });
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'b', transcript: 'Second.' });
assert.equal(h.events.some(e => e.isFinal), false);
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'a', transcript: 'First.' });
assert.deepEqual(h.events.filter(e => e.isFinal).map(e => e.text), ['First.', 'Second.']);
assert.equal(h.sockets[0].closed, false);
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'c', delta: 'Third.' });
assert.equal(h.events.at(-1).text, 'Third.');
h.client.stop();
});
test('Soniox speaker and pause formatting remains separate from dictation text and resets on finalize', () => {
const h = harness('soniox'); h.open();
h.message({ tokens: [
{ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1', is_final: true },
{ text: ' Hi.', start_ms: 550, end_ms: 800, speaker: '2', is_final: false },
] });
assert.equal(h.events.at(-1).text, 'Hello. Hi.');
assert.equal(h.events.at(-1).transcribeText, 'Hello.\nHi.');
h.message({ tokens: [
{ text: ' Yes.', start_ms: 550, end_ms: 800, speaker: '2', is_final: true },
{ text: '<fin>', is_final: true },
] });
assert.equal(h.events.find(e => e.isFinal).transcribeText, 'Hello.\nYes.');
h.message({ tokens: [{ text: 'New.', start_ms: 3000, end_ms: 3400, speaker: '2', is_final: false }] });
assert.equal(h.events.at(-1).text, 'New.');
h.client.stop();
});
test('an unsolicited Soniox session end finalizes visible text and reconnects', () => {
const h = harness('soniox'); h.open();
h.message({ tokens: [{ text: 'Before expiry.', is_final: true }], finished: true });
assert.equal(h.events.at(-1).isFinal, true);
assert.equal(h.events.at(-1).text, 'Before expiry.');
assert.equal(h.timers.size, 1);
h.tick(); h.open();
assert.equal(h.sockets.length, 2);
h.client.stop();
});
+28
View File
@@ -0,0 +1,28 @@
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
// Execute production TypeScript with only the native boundary and clock stubbed.
exports.loader = (globals = {}, mocks = {}) => {
const cache = new Map();
function load(file) {
file = path.resolve(__dirname, '../..', file);
if (cache.has(file)) return cache.get(file);
const output = {};
cache.set(file, output);
const source = ts.transpileModule(fs.readFileSync(file, 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText;
vm.runInNewContext(source, {
exports: output, console, Uint8Array, ...globals,
require(name) {
if (name in mocks) return mocks[name];
if (!name.startsWith('.')) throw new Error(`Unmocked module: ${name}`);
return load(path.resolve(path.dirname(file), name + '.ts'));
},
}, { filename: file });
return output;
}
return load;
};
+56
View File
@@ -0,0 +1,56 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { PhoneGestureRecognizer } = require('../.test-build/app/phone-ui/phone-gestures.js');
function setup() {
let now = 0, id = 0;
const timers = new Map(), events = [];
const r = new PhoneGestureRecognizer((type, x, y) => events.push({ type, x, y }), {
set(fn, ms) { timers.set(++id, { fn, at: now + ms }); return id; },
clear(id) { timers.delete(id); },
});
return {
r, events,
touch: (action, x = 20, y = 30, pointers = 1) => r.touch({ action, x, y, pointers }),
advance(ms) {
now += ms;
for (const [id, timer] of [...timers]) if (timer.at <= now && timers.delete(id)) timer.fn();
},
types: () => events.map(e => e.type),
};
}
test('single tap retains mirror coordinates and waits to disambiguate double tap', () => {
const s = setup(); s.touch('down'); s.touch('up');
assert.deepEqual(s.types(), []); s.advance(280);
assert.deepEqual(s.events, [{ type: 'tap', x: 20, y: 30 }]);
});
test('double tap emits back without accidental selection', () => {
const s = setup(); s.touch('down'); s.touch('up'); s.advance(150);
s.touch('down'); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['double-tap']);
});
test('a hold emits a press and release, never a tap', () => {
const s = setup(); s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
});
test('tap then hold emits the app-menu gesture without an earlier selection', () => {
const s = setup(); s.touch('down'); s.touch('up'); s.advance(100);
s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['short-then-long-press', 'long-press-release']);
});
test('four directional swipes cancel pending holds and taps', () => {
for (const [x, y, expected] of [[80, 30, 'swipe-right'], [-40, 30, 'swipe-left'], [20, 90, 'swipe-down'], [20, -30, 'swipe-up']]) {
const s = setup(); s.touch('down'); s.touch('move', x, y); s.advance(500); s.touch('up', x, y); s.advance(500);
assert.deepEqual(s.types(), [expected]);
}
});
test('two fingers send back exactly once and cancel the long-press timer', () => {
const s = setup(); s.touch('down'); s.touch('down', 21, 30, 2); s.advance(500);
s.touch('up', 21, 30, 2); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['double-tap']);
});
test('cancellation prevents delayed selections and releases active holds', () => {
const s = setup(); s.touch('down'); s.touch('up'); s.r.cancel(); s.advance(1000);
assert.deepEqual(s.types(), []);
s.touch('down'); s.advance(500); s.touch('cancel'); s.advance(1000);
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
});
+49
View File
@@ -0,0 +1,49 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js');
const rect = (x, y, width, height) => ({ x, y, width, height });
function surface(c, id, box, values, zOrder = 0, transparency = 'opaque') {
c.configureSurface(id, { ...box, zOrder, transparency });
c.submitSurfaceFrame(id, Uint8Array.from(values), rect(0, 0, box.width, box.height));
}
test('shell color key preserves underlying text while opaque black covers it', () => {
const c = new SurfaceCompositor(3, 2);
surface(c, 'app', rect(0, 0, 3, 2), [200, 200, 200, 200, 200, 200]);
surface(c, 'shell', rect(0, 0, 3, 2), [0, 1, 255, 0, 0, 0], 1, 'color-key');
c.setUnderlayDim(1, 0.5);
assert.deepEqual([...c.composite()], [100, 1, 255, 100, 100, 100]);
c.setSurfaceVisible('shell', false);
assert.deepEqual([...c.composite()], [100, 100, 100, 100, 100, 100]);
surface(c, 'opaque', rect(1, 0, 1, 2), [0, 0], 2);
assert.deepEqual([...c.composite()], [100, 0, 100, 100, 0, 100]);
});
test('partial frames clip in surface coordinates, then surfaces clip to the screen', () => {
const c = new SurfaceCompositor(3, 2);
surface(c, 'app', rect(-1, 0, 3, 2), [1, 2, 3, 4, 5, 6]);
const source = Uint8Array.from([99, 10, 20, 99, 30, 40, 99]);
c.submitSurfaceFrame('app', source.subarray(1, 5), rect(1, 1, 2, 2));
source.fill(0);
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
c.submitSurfaceFrame('app', Uint8Array.from([50, 60, 70, 80]), rect(-1, -1, 2, 2));
// The clipped update changed the offscreen column, leaving visible pixels intact.
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
});
test('moving a surface retains its frame; resizing clears it; blanking is reversible', () => {
const c = new SurfaceCompositor(3, 1);
surface(c, 'app', rect(0, 0, 2, 1), [7, 9]);
c.configureSurface('app', { ...rect(1, 0, 2, 1), zOrder: 0, transparency: 'opaque' });
assert.deepEqual([...c.composite()], [0, 7, 9]);
c.setScreenBlanked(true); assert.deepEqual([...c.composite()], [0, 0, 0]);
c.setScreenBlanked(false); assert.deepEqual([...c.composite()], [0, 7, 9]);
c.configureSurface('app', { ...rect(0, 0, 3, 1), zOrder: 0, transparency: 'opaque' });
assert.deepEqual([...c.composite()], [0, 0, 0]);
});
test('layer order is independent of creation order; removal reveals lower surfaces', () => {
const c = new SurfaceCompositor(1, 1);
surface(c, 'top', rect(0, 0, 1, 1), [20], 10);
surface(c, 'bottom', rect(0, 0, 1, 1), [80], -1);
assert.equal(c.composite()[0], 20);
c.removeSurface('top'); assert.equal(c.composite()[0], 80);
assert.throws(() => c.submitSurfaceFrame('missing', new Uint8Array(1), rect(0, 0, 1, 1)));
assert.throws(() => c.submitSurfaceFrame('bottom', new Uint8Array(2), rect(0, 0, 1, 1)));
});
+94
View File
@@ -0,0 +1,94 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
const load = loader();
const { TranscriptModel, TranscriptScroll, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts');
const { SpeechPauseDetector } = load('app/native/speech-pause.ts');
const { TimedTranscript } = load('app/native/transcript-format.ts');
function pcm(amplitude, ms = 50) {
const bytes = new Uint8Array(ms * 16 * 2);
for (let i = 0; i < bytes.length; i += 2) {
bytes[i] = amplitude & 255;
bytes[i + 1] = (amplitude >> 8) & 255;
}
return bytes;
}
test('partials replace, finals correct, and pause breaks survive display and saving', () => {
const m = new TranscriptModel();
m.accept({ text: 'Hello wor', isFinal: false });
m.accept({ text: 'Hello world.', isFinal: true, paragraphBreakAfter: true });
m.accept({ text: 'Next', isFinal: false });
m.accept({ text: 'Next paragraph.', isFinal: false });
assert.equal(m.text, 'Hello world.\nNext paragraph.');
m.accept({ text: 'Next paragraph!', isFinal: true });
assert.equal(m.text, 'Hello world.\nNext paragraph!');
m.accept({ text: '', isFinal: true });
assert.equal(m.text, 'Hello world.\nNext paragraph!');
});
test('onboard pauses can precede or follow finalization', () => {
for (const before of [true, false]) {
const m = new TranscriptModel();
m.accept({ text: 'first', isFinal: false });
if (before) m.pause();
m.accept({ text: 'First.', isFinal: true });
if (!before) m.pause();
m.accept({ text: 'Second.', isFinal: false });
assert.equal(m.text, 'First.\nSecond.');
}
});
test('formatting metadata is used only by Transcribe, with revisable live speaker tokens', () => {
const final = new TimedTranscript();
final.append({ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1' });
const preview = final.copy();
preview.append({ text: ' Wrong', start_ms: 600, end_ms: 900, speaker: '2' });
assert.equal(preview.text, 'Hello.\nWrong');
assert.equal(final.text, 'Hello.');
final.append({ text: ' Right.', start_ms: 600, end_ms: 900, speaker: '1' });
final.append({ text: ' Later.', start_ms: 2400, end_ms: 2800, speaker: '1' });
assert.equal(final.text, 'Hello. Right.\nLater.');
const event = { text: 'Hello. Right. Later.', transcribeText: final.text, isFinal: false };
const m = new TranscriptModel();
m.accept(event);
assert.equal(m.text, final.text);
assert.equal(event.text.includes('\n'), false);
});
test('wrapping preserves explicit newlines, blank paragraphs, and long unspaced words', () => {
assert.deepEqual([...wrapTranscribeText(s => s.length, 'one two\nthree\n\nabcdefghij', 5)],
['one', 'two', 'three', '', 'abcde', 'fghij']);
});
test('scroll follows the tail until scrolled back, then resumes only at the bottom', () => {
const s = new TranscriptScroll();
assert.equal(s.layout(30, 10), 20);
assert.equal(s.layout(33, 10), 23);
s.scroll(-3);
assert.equal(s.layout(50, 10), 20);
assert.equal(s.layout(51, 10), 20);
s.scroll(100);
assert.equal(s.layout(55, 10), 45);
s.scroll(-100);
assert.equal(s.layout(60, 10), 0);
});
test('short transcripts become scrollable without disabling automatic following', () => {
const s = new TranscriptScroll();
assert.equal(s.layout(3, 10), 0);
s.scroll(-3);
assert.equal(s.layout(12, 10), 2);
});
test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores leading silence', () => {
const d = new SpeechPauseDetector();
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
d.accept(pcm(3000));
for (let i = 0; i < 29; i++) assert.equal(d.accept(pcm(0)), false);
assert.equal(d.accept(pcm(0)), true);
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
d.accept(pcm(3000));
assert.equal(d.accept(pcm(0, 1500)), true);
});
+2
View File
@@ -11,6 +11,8 @@
"outDir": "../.test-build"
},
"include": [
"../app/graphics/surface-compositor.ts",
"../app/phone-ui/phone-gestures.ts",
"../app/g2/glasses-hardware-identity.ts",
"../app/g2/ble-proximity.ts",
"../app/g2/even-advertisement.ts",