Make iOS app main screen match the Android UI

This commit is contained in:
James Babcock committed 2026-09-19 21:54:18 -07:00
1 parent 9f53847bb7
commit 6c3144cc31
30 files changed
+1013 -439

No files matched your search

+44 -4
View File
@@ -185,11 +185,12 @@ test('background glasses input still composites frames; phone resume preserves t
constructor(_transport, onState, onInput) { session = this; this.onState = onState; this.onInput = onInput; }
async start() { starts++; this.state = { phase: 'connected' }; this.onState(this.state); }
async stop() { stops++; this.state = { phase: 'disconnected' }; this.onState(this.state); }
async setBrightness() {}
setFrame(pixels) { if (this.state.phase === 'connected') frames.push(pixels); }
playBuzzerSequence(payload) { sounds.push([...payload]); }
wake() {}
}
const settings = { lockScreenEnabledSetting: { get: () => true }, onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } };
const settings = { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, lockScreenEnabledSetting: { get: () => true }, onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } };
const modules = {
"../ui/input-monitor": load("app/ui/input-monitor.ts", {}),
'../assistant/system-tools': { registerSystemTools() {} },
@@ -201,7 +202,7 @@ test('background glasses input still composites frames; phone resume preserves t
'../native/notification-sources': { shouldShowNotificationOnGlasses: () => true },
'../native/compass.ios': { bindCompassSession() {}, receiveCompassEvent() {} },
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) }, knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
'../native/ios-bluetooth': { iosBluetooth: () => ({}) }, './glasses-session': { GlassesSession: Session },
'../native/nightscout-bridge': { nightscoutBridge: { async start() { pollStarts++; }, async stop() { pollStops++; } } },
'./glance-host': { GlanceHost: class { dismiss() {} isVisible() { return false; } } },
@@ -266,12 +267,15 @@ test('iOS bandwidth footer toggles live, polls only in foreground and resets its
constructor() { views.push(this); }
on(event, fn) { this.events.set(event, fn); }
addChild(view) { this.children.push(view); }
set(key, value) { this[key] = value; }
notifyPropertyChange() {}
getActualSize() { return { width: 0, height: 0 }; }
getViewById() { return null; }
static setRow() {} static setColumn() {}
}
const core = { Application: { suspendEvent: 'suspend', resumeEvent: 'resume',
on: (key, fn) => appEvents.set(key, fn), off: key => appEvents.delete(key) },
Button: View, Color: class {}, Dialogs: {}, GridLayout: View, Image: View, Label: View, Page: View, StackLayout: View };
Builder: { load: () => new View() }, Observable: View, Button: View, Color: class {}, Dialogs: {}, GridLayout: View, Image: View, Label: View, Page: View, StackLayout: View };
const modules = {
"../ui/input-monitor": load("app/ui/input-monitor.ts", {}),
'../assistant/system-tools': { registerSystemTools() {} },
@@ -289,15 +293,19 @@ test('iOS bandwidth footer toggles live, polls only in foreground and resets its
'./phone-gestures': { PhoneGestureRecognizer: class { cancel() {} } },
'./ble-bandwidth-meter': require('../.test-build/app/phone-ui/ble-bandwidth-meter.js'),
'../native/ble-traffic': { sampleBleTraffic: () => totals },
'../ui/dashboard-settings': { showBleBandwidthSetting: { get: () => enabled },
'../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, displayModeSetting: { get: () => 'full' }, displayModeLabel: () => 'Full', showBleBandwidthSetting: { get: () => enabled },
onAnySettingChanged: fn => { settingListeners.add(fn); return () => settingListeners.delete(fn); } },
'../g2/device-addresses': { loadDeviceAddresses: () => ({}) },
'../g2/ios-peripheral-identity': { deviceAddressError: () => 'no test devices' },
'./onboarding-state': { isPreviewOnlyMode: () => false },
'../g2/reconnect-policy': { isAutoReconnectSuppressed: () => false },
};
modules['./keyboard-input-view-model'] = load('app/phone-ui/keyboard-input-view-model.ts', { require: id => modules[id] });
modules['./remote-controls-view-model'] = load('app/phone-ui/remote-controls-view-model.ts', { require: id => modules[id] });
const { createMainPage } = load('app/phone-ui/main-page.ios.ts', {
require: id => modules[id] ?? {}, Date: { now: () => atMs },
NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock: () => ({}), removeObserver() {} } },
UIKeyboardWillChangeFrameNotification: 'keyboard', NSOperationQueue: { mainQueue: {} },
setInterval: fn => { const id = ++nextId; intervals.set(id, fn); return id; }, clearInterval: id => intervals.delete(id),
setTimeout: () => 1, clearTimeout() {},
});
@@ -397,3 +405,35 @@ test('iOS welcome sound waits for a new acknowledged frame and is consumed once'
host.maybePlayWelcomeSound({ phase: 'connected', frames: 2 });
assert.equal(played.length, count);
});
test('iOS preview voice uses the phone microphone, while connected capture stays on glasses', async () => {
const calls = [];
const bridge = { prepare: async (foreground, phone) => { calls.push(['prepare', foreground, phone]); return true; },
startPhoneCapture: async (_log, endpointing) => calls.push(['phone', endpointing]),
startGlassesCapture: async (_session, _log, endpointing) => calls.push(['glasses', endpointing]) };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : {},
});
const host = Object.create(IosPreviewController.prototype);
host.active = true; host.glassesLocked = false;
assert.equal(await host.prepareVoiceCapture(), true);
await host.startVoiceCapture(true);
assert.deepEqual(calls, [['prepare', true, true], ['prepare', true, true], ['phone', true]]);
calls.length = 0; host.session = { state: { phase: 'connected' } };
assert.equal(await host.prepareVoiceCapture(), true); await host.startVoiceCapture(false);
assert.deepEqual(calls, [['prepare', true, false], ['glasses', false]]);
calls.length = 0; host.active = false; host.session = null;
assert.equal(await host.prepareVoiceCapture(), false); await host.startVoiceCapture();
assert.deepEqual(calls, []);
});
test('iOS keyboard action opens the shared destination session and remains blocked while locked', () => {
let opened = 0;
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../ui/shell/shell' ? { shell: { startKeyboardInput: () => opened++ } } : {},
});
const host = Object.create(IosPreviewController.prototype);
host.active = true; host.glassesLocked = false; host.typeIntoApp(); assert.equal(opened, 1);
host.glassesLocked = true; host.typeIntoApp(); assert.equal(opened, 1);
host.glassesLocked = false; host.active = false; host.typeIntoApp(); assert.equal(opened, 1);
});
+17
View File
@@ -667,3 +667,20 @@ test('ANCS app-name lookup uses the right-link write limit, including after reau
assert.equal(h.session.notifications.read()[0].appName,'Pushover Notifications');
}
});
test('brightness sends the Android settings payload to the right temple, including explicit zero and Auto', async () => {
const transport = new FakeTransport(), session = new GlassesSession(transport, () => {}, () => {});
await session.setBrightness(50);
assert.equal(transport.sent.length, 0, 'preview mode must not send Bluetooth commands');
const requests = [];
session.state.phase = 'connected';
session.request = async (role, sid, build) => { requests.push({ role, sid, payload: hex(build(105)) }); };
await session.setBrightness(0);
await session.setBrightness(80);
await session.setBrightness(null);
assert.deepEqual(requests, [
{ role: 'right', sid: 9, payload: '080110691a060a0408001000' },
{ role: 'right', sid: 9, payload: '080110691a060a0408001050' },
{ role: 'right', sid: 9, payload: '080110691a040a020801' },
]);
});
+23 -3
View File
@@ -60,6 +60,7 @@ function fixture() {
async enableWearDetectionAndRequestState() {}
async start() { this.state = { phase: 'connected' }; this.onState(this.state); }
async stop() { this.state = { phase: 'disconnected' }; this.onState(this.state); }
async setBrightness() {}
setFrame(pixels) { if (this.state.phase === 'connected') sent.push(pixels); }
wake() {}
}
@@ -84,7 +85,7 @@ function fixture() {
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-bluetooth': { iosBluetooth: () => ({}) },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {}, stopPhoneCapture() {} } },
'./glasses-session': { GlassesSession: Session },
'../native/nightscout-bridge': { nightscoutBridge: { async start() {}, async stop() {} } },
'./glance-host': { GlanceHost }, './events': events,
@@ -97,7 +98,7 @@ function fixture() {
'../phone-ui/onboarding-state': { isWelcomeSoundPending: () => false },
'../ui/sound-effects': {},
'../ui/shell/worker-window': {}, '../ui/shell/in-process-window': {},
'../ui/dashboard-settings': { lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } },
'../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } },
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../apps/ios-availability': { iosAppUnavailableReason: () => null },
'../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images,
@@ -138,7 +139,7 @@ function fixture() {
await render();
}
async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); }
return { controller, shell, settings, phoneState, observers, wear: wearing => session.onWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render };
return { controller, shell, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, wear: wearing => session.onWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render };
}
const assertBlank = pixels => assert.ok(pixels.every(p => p === 0));
const assertBoard = pixels => {
@@ -299,3 +300,22 @@ test('iOS deduplicates original ring ticks before apps, lock toggles and Glanceb
f.controller.session.onState({ phase:'connected' });
await f.hardware(3,2,2051); assert.equal(f.shell.isScreenOn(),false);
});
test('keyboard sessions cannot deliver text after the glasses lock or phone leaves the foreground', () => {
const f = fixture();
let editor;
f.controller.onKeyboardInputChanged = session => { editor = session; };
const calls = [];
f.keyboardChanged({ targets: [{ id: 'app', label: 'Type into App' }],
setText: text => calls.push(['text', text]), send: () => calls.push(['send']),
sendTo: id => calls.push(['target', id]), discard: () => calls.push(['discard']) });
editor.setText('hello'); editor.sendTo('app');
assert.deepEqual(calls, [['text', 'hello'], ['target', 'app']]);
f.controller.glassesLocked = true;
editor.setText('locked'); editor.send(); editor.sendTo('app');
assert.equal(calls.length, 2);
f.controller.glassesLocked = false; f.controller.pause();
editor.setText('background'); editor.sendTo('app');
assert.equal(calls.length, 2);
editor.discard(); assert.deepEqual(calls.at(-1), ['discard']);
});
+58 -3
View File
@@ -4,11 +4,13 @@ const fs = require('node:fs');
const vm = require('node:vm');
const ts = require('typescript');
function harness() {
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const native = { startWithEndpointing(endpointing) { this.endpointing = endpointing; return ''; }, startPhoneWithEndpointing(endpointing) { this.endpointing = endpointing; this.phone = true; return this.error ?? ''; }, cancel() { this.cancels++; }, cancels: 0, finish() { this.finished = true; }, acceptPacket: p => audio.push(p) };
const audio = [], timers = new Map(), commands = [], text = [];
let authorization = 3, permissionRequests = 0, release;
let authorization = 3, microphoneAuthorization = 3, microphoneRequests = 0, permissionRequests = 0, release;
const sandbox = { exports: {}, FaceclawSpeech: { new: () => native, authorizationStatus: () => authorization,
requestAuthorization: done => { permissionRequests++; done(3); } },
requestAuthorization: done => { permissionRequests++; done(3); },
microphoneAuthorizationStatus: () => microphoneAuthorization,
requestMicrophoneAuthorization: done => { microphoneRequests++; done(3); } },
setTimeout: fn => { timers.set(fn, fn); return fn; }, clearTimeout: fn => timers.delete(fn),
NSData: { dataWithBytesLength: (b, n) => Buffer.from(b, 0, n) }, interop: { handleof: b => b } };
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/native/voice-control.ios.ts', 'utf8'), {
@@ -21,6 +23,7 @@ function harness() {
return Promise.resolve();
} };
return { bridge, native, session, audio, commands, text, timers, release: () => release(),
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
authorization: value => { authorization = value; }, permissionRequests: () => permissionRequests,
event: event => native.eventHandler(JSON.stringify(event)) };
}
@@ -81,3 +84,55 @@ test('hands-free endpointing releases the microphone and waits for the final tra
assert.equal(h.native.endpointing, false);
const cancelled = h.bridge.stopPushToTalk(); h.bridge.stop(); await cancelled;
});
test('phone microphone permission is requested only for foreground preview capture', async () => {
const h = harness(); h.microphoneAuthorization(0);
assert.equal(await h.bridge.prepare(true), true);
assert.equal(h.microphoneRequests(), 0, 'glasses input does not need phone microphone permission');
assert.equal(await h.bridge.prepare(false, true), false);
assert.equal(h.microphoneRequests(), 0);
assert.equal(await h.bridge.prepare(true, true), true);
assert.equal(h.microphoneRequests(), 1);
h.microphoneAuthorization(2);
assert.equal(await h.bridge.prepare(true, true), false);
assert.match(h.bridge.statusText, /Microphone.*Settings/);
});
test('phone capture forwards endpointing and drains final text without touching Bluetooth', async () => {
const h = harness();
await h.bridge.startPhoneCapture(() => {}, true);
assert.equal(h.native.phone, true); assert.equal(h.native.endpointing, true);
h.bridge.handleSessionEnded();
h.event({ kind: 'transcript', text: 'First line\nSecond line', final: false });
assert.equal(h.text.length, 1, 'BLE state changes must not stop phone capture');
h.event({ kind: 'finishing' });
let finished = false;
const completion = h.bridge.stopPushToTalk().then(() => { finished = true; });
await Promise.resolve(); assert.equal(finished, false);
h.bridge.handleSessionEnded();
h.event({ kind: 'transcript', text: 'First line\nSecond line.', final: true });
h.event({ kind: 'ended' }); await completion;
assert.equal(h.text.at(-1).text, 'First line\nSecond line.');
assert.deepEqual(h.commands, []);
});
test('background cleanup stops phone capture but preserves glasses capture', async () => {
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
await h.bridge.startPhoneCapture(() => {});
h.bridge.stopPhoneCapture();
assert.equal(ends, 1);
h.event({ kind: 'transcript', text: 'stale', final: true }); assert.equal(h.text.length, 0);
const start = h.bridge.startGlassesCapture(h.session, () => {}); h.release(); await start;
h.bridge.stopPhoneCapture();
h.event({ kind: 'transcript', text: 'on glasses', final: false });
assert.equal(h.text.at(-1).text, 'on glasses');
h.bridge.stop();
});
test('phone microphone startup errors end capture with an actionable status', async () => {
const h = harness(); let ends = 0; h.bridge.onSpeechEnd(() => ends++);
h.native.error = 'No phone microphone is available.';
await h.bridge.startPhoneCapture(() => {});
assert.equal(ends, 1); assert.match(h.bridge.statusText, /No phone microphone/);
await h.bridge.stopPushToTalk();
});
+32
View File
@@ -0,0 +1,32 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function fixture() {
class Observable { get(key) { return this[key]; } notifyPropertyChange() {} }
const { KeyboardInputViewModel } = loader({}, { '@nativescript/core': { Observable } })('app/phone-ui/keyboard-input-view-model.ts');
const model = new KeyboardInputViewModel(), sent = [], text = [];
const session = { targets: [{ id: 'assistant', label: 'Send to Assistant' }, { id: 'app', label: 'Type Into App' }],
setText: value => text.push(value), sendTo: id => sent.push(id), discard: () => model.setSession(null) };
model.setSession(session);
return { model, session, sent, text };
}
test('multiline keyboard editor routes Agent and App buttons to distinct shell destinations', () => {
const h = fixture();
assert.equal(h.model.keyboardInputPrimaryTargetLabel, 'Send to Agent');
assert.equal(h.model.keyboardInputSecondaryTargetLabel, 'Type into App');
h.model.onKeyboardInputTextChange({ object: { text: 'First line\nSecond line' } });
h.model.onKeyboardInputPrimarySendTap();
h.model.onKeyboardInputSecondarySendTap();
assert.deepEqual(h.sent, ['assistant', 'app']);
assert.equal(h.text.at(-1), 'First line\nSecond line');
});
test('discard and shell-side closure reset the editor; unsupported destinations stay hidden', () => {
const h = fixture();
h.model.keyboardInputText = 'draft'; h.model.onKeyboardInputDiscardTap();
assert.equal(h.model.keyboardInputText, '');
assert.equal(h.model.keyboardInputVisibility, 'collapse');
h.model.setSession({ ...h.session, targets: [{ id: 'assistant', label: 'Send' }] });
assert.equal(h.model.keyboardInputPrimaryTargetLabel, 'Send to Agent');
assert.equal(h.model.keyboardInputSecondaryTargetVisibility, 'collapse');
h.model.onKeyboardInputSecondarySendTap(); assert.deepEqual(h.sent, []);
});
+61
View File
@@ -0,0 +1,61 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function fixture() {
let brightness = '40', mode = 'small', picked;
const changes = [], writes = [];
class Observable {
notifyPropertyChange(name, value) { changes.push([name, value]); }
}
const settings = {
brightnessSetting: { get: () => brightness, set: value => { brightness = value; writes.push(value); } },
displayModeSetting: { get: () => mode, set: value => { mode = value; } },
DISPLAY_MODE_VALUES: ['small', 'large'], displayModeLabel: value => value === 'small' ? 'Small' : 'Large',
};
const { RemoteControlsViewModel } = loader({}, {
'@nativescript/core': { Observable, Dialogs: { action: async () => picked } },
'../ui/dashboard-settings': settings,
})('app/phone-ui/remote-controls-view-model.ts');
return { model: new RemoteControlsViewModel(), create: () => new RemoteControlsViewModel(), changes, writes,
brightness: () => brightness, mode: () => mode, pick: value => { picked = value; } };
}
test('shared controls preserve the selected tab across page visits and update all tab bindings', () => {
const h = fixture();
assert.equal(h.model.watchTabVisibility, 'visible');
h.model.onRingTabTap();
assert.equal(h.model.watchTabVisibility, 'collapse');
assert.equal(h.model.ringTabVisibility, 'visible');
assert.match(h.model.ringTabClass, /tab-button-selected/);
assert.equal(h.changes.length, 6);
assert.equal(h.create().ringTabVisibility, 'visible');
h.model.onSettingsTabTap();
assert.equal(h.model.settingsTabVisibility, 'visible');
});
test('shared brightness controls snap manual levels and restore the manual value after Auto', () => {
const h = fixture();
h.model.onBrightnessChange({ value: 76 });
assert.equal(h.brightness(), '80');
h.model.onBrightnessAutoChange({ value: true });
assert.equal(h.brightness(), 'auto');
assert.equal(h.model.brightnessSliderEnabled, false);
h.model.onBrightnessChange({ value: 10 });
assert.equal(h.brightness(), 'auto');
h.model.onBrightnessAutoChange({ value: false });
assert.equal(h.brightness(), '80');
h.model.onBrightnessChange({ value: -10 });
assert.equal(h.brightness(), '0');
h.model.onBrightnessChange({ value: NaN });
assert.equal(h.brightness(), '0');
});
test('shared screen-size picker preserves cancellation and applies the selected mode', async () => {
const h = fixture();
h.pick('Cancel'); await h.model.onDisplayModeTap();
assert.equal(h.mode(), 'small');
h.pick('Large'); await h.model.onDisplayModeTap();
assert.equal(h.mode(), 'large');
assert.equal(h.model.displayModeLabel, 'Large ▾');
});