iOS porting: Glanceboard wake gestures

This commit is contained in:
James Babcock committed 2026-09-17 23:50:14 -07:00
1 parent 6b2e0375e6
commit 14fbbe66ff
6 files changed
+263 -12

No files matched your search

+4 -2
View File
@@ -1,4 +1,5 @@
/** G2 wire protocol, ported from g2protocol/BleProtocol.java. No platform APIs. */
import { OsEventTypeList } from './events'
export const G2_WRITE = '00002760-08c2-11e1-9073-0e8ac72e5401'
export const G2_NOTIFY = '00002760-08c2-11e1-9073-0e8ac72e5402'
export const G2_RENDER_NOTIFY = '00002760-08c2-11e1-9073-0e8ac72e6402'
@@ -125,13 +126,14 @@ export function readString(data: Uint8Array, number: number): string { return Ar
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
}
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null {
if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null
const events = readBytes(message.payload, 13); if (!events) return null
for (const [field, kind, typeField] of [[1, 'list-click', 5], [2, 'text-click', 3], [3, 'sys-event', 1]] as const) {
const event = readBytes(events, field)
if (event) return { kind, eventType: readInteger(event, typeField), eventSource: field === 3 ? readInteger(event, 2) : 0,
if (event) return { kind: kind === 'sys-event' && readInteger(event, typeField) === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : kind,
eventType: readInteger(event, typeField), eventSource: field === 3 ? readInteger(event, 2) : 0,
systemExitReasonCode: field === 3 ? readInteger(event, 4) : 0, containerName: field === 3 ? '' : readString(event, 2), frameId: 0 }
}
return null
+7 -3
View File
@@ -21,8 +21,12 @@ export const GLANCE_SURFACE_ID = "glance";
/** Above the shell (1) and every window (0); below the lock screen (1000). */
export const GLANCE_SURFACE_Z_ORDER = 900;
/** Surface operations needed by the board, shared by Android and iOS. */
export type GlanceDisplay = Pick<DisplayTarget,
"configureSurface" | "setSurfaceVisible" | "setScreenBlanked" | "submitSurfaceFrame">;
export type GlanceHostOptions = {
getDisplay: () => DisplayTarget | null;
getDisplay: () => GlanceDisplay | null;
getProvider: () => GlanceboardProvider | null;
/** Whether a board may show right now (connected or preview, not locked). */
canShow: () => boolean;
@@ -47,7 +51,7 @@ export class GlanceHost {
/** Whether the surface is currently visible on the compositor. */
private shown = false;
private hideTimer: ReturnType<typeof setTimeout> | null = null;
private configuredFor: DisplayTarget | null = null;
private configuredFor: GlanceDisplay | null = null;
/** Serializes show/hide against each other and against in-flight renders. */
private queue: Promise<void> = Promise.resolve();
private rendering = false;
@@ -144,7 +148,7 @@ export class GlanceHost {
}, Math.max(0, hideAtMs - Date.now()));
}
private async configureSurface(display: DisplayTarget): Promise<void> {
private async configureSurface(display: GlanceDisplay): Promise<void> {
await display.configureSurface(GLANCE_SURFACE_ID, {
x: 0,
y: 0,
+56 -6
View File
@@ -2,6 +2,8 @@ import { Dialogs, File, knownFolders, path, type ImageSource } from '@nativescri
import { iosBluetooth } from '../native/ios-bluetooth'
import { iosVoiceInput } from '../native/ios-voice-input'
import { GlassesSession, type SessionState } from './glasses-session'
import { GlanceHost, type GlanceDisplay } from './glance-host'
import { OsEventTypeList } from './events'
import { loadDeviceAddresses } from './device-addresses'
import { deviceAddressError } from './ios-peripheral-identity'
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
@@ -16,7 +18,7 @@ import { SurfaceCompositor } from '../graphics/surface-compositor'
import { flattenPlanes, type Plane } from '../graphics/plane'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { previewPixels } from '../native/ios-graphics'
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
import { noopLayerActions, type LayerActions } from '../ui/layers'
import { TextViewerLayer } from '../apps/files/text-viewer'
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
@@ -28,6 +30,36 @@ import type { PhoneGesture } from '../phone-ui/phone-gestures'
/** iOS host for the shared app registry, shell, compositor and BLE session. */
export class IosPreviewController {
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
private readonly glanceDisplay: GlanceDisplay = {
configureSurface: async (id, options) => {
this.compositor.configureSurface(id, options)
},
setSurfaceVisible: async (id, visible) => {
this.compositor.setSurfaceVisible(id, visible); this.scheduleFrame()
},
setScreenBlanked: async blanked => {
this.compositor.setScreenBlanked(blanked); this.scheduleFrame()
},
submitSurfaceFrame: async (id, pixels, rect) => {
this.compositor.submitSurfaceFrame(id, pixels, rect); this.scheduleFrame()
},
}
private readonly glance = new GlanceHost({
getDisplay: () => this.glanceDisplay,
getProvider: () => ALL_APPS.find(app => app.glanceboard)?.glanceboard ?? null,
canShow: () => this.runtimeNeeded,
// iOS retains the EvenHub session while the shell sleeps. The board's
// opaque first frame is ready before we unblank the compositor.
ensureSessionActive: async () => {
this.compositor.setScreenBlanked(false); this.scheduleFrame(); return true
},
onHiddenWhileAsleep: () => {
if (!shell.isScreenOn()) this.compositor.setScreenBlanked(true)
this.scheduleFrame()
},
onVisibilityChanged: () => this.scheduleFrame(),
appendLog: message => this.logBluetooth(message),
})
private renderTimer: ReturnType<typeof setTimeout> | null = null
private clockTimer: ReturnType<typeof setInterval> | null = null
private offSettings: (() => void) | null = null
@@ -69,7 +101,10 @@ export class IosPreviewController {
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
onScreenStateChanged: on => {
if (on) this.glance.dismiss()
this.compositor.setScreenBlanked(!on); this.requestShellRender()
},
})
const launcher = createLauncherWindow({
actions: this.actions,
@@ -117,6 +152,8 @@ export class IosPreviewController {
this.runtimeRunning = running
for (const window of shell.getWindows()) window.setScreenOn?.(running && shell.isScreenOn())
if (!running) {
this.glance.dismiss()
this.compositor.setScreenBlanked(!shell.isScreenOn())
this.offSettings?.(); this.offSettings = null
for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer)
UIDevice.currentDevice.batteryMonitoringEnabled = false
@@ -180,7 +217,8 @@ export class IosPreviewController {
this.session?.setFrame(pixels)
if (this.active) {
const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green')
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
this.onFrame(image, this.glance.isVisible() ? 'Glanceboard'
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
}
} catch (error) { this.fail(error) }
}, 33)
@@ -203,12 +241,23 @@ export class IosPreviewController {
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
}
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
await shell.receiveInput(event)
this.requestShellRender()
await this.receiveInput(event)
}
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
}).catch(error => this.fail(error))
}
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
if (!shell.isScreenOn()) {
const glanceEvent = this.glance.eventForGesture(headTilt ? 'head-tilt' : event.type)
if (glanceEvent?.type === 'dismiss') this.glance.dismiss()
else if (glanceEvent) {
await this.glance.handleEvent(glanceEvent, 0)
return
}
}
await shell.receiveInput(event)
this.requestShellRender()
}
private async mirrorTap(nx: number, ny: number): Promise<void> {
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
@@ -317,7 +366,8 @@ export class IosPreviewController {
}, input => {
this.inputQueue = this.inputQueue.then(async () => {
if (this.session?.state.phase !== 'connected') return
await shell.receiveInput(rawInputEventToInputEvent(input)); this.requestShellRender()
await this.receiveInput(rawInputEventToInputEvent(input),
input.kind === 'display-wake' && input.eventType === OsEventTypeList.HEAD_UP_EVENT)
this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`)
}).catch(error => this.fail(error))
}, message => this.logBluetooth(message), () => this.refreshClock())
+2 -1
View File
@@ -92,12 +92,13 @@ test('background glasses input still composites frames; phone resume preserves t
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) }, knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
'../native/ios-bluetooth': { iosBluetooth: () => ({}) }, './glasses-session': { GlassesSession: Session },
'./glance-host': { GlanceHost: class { dismiss() {} isVisible() { return false; } } },
'./device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null },
'../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' },
'../apps/all-apps': { ALL_APPS: [] }, '../ui/dashboard-settings': settings,
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../graphics/surface-compositor': { SurfaceCompositor: class {
configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {}
configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {} setScreenBlanked() {}
composite() { return new Uint8Array([1, 2]); }
} },
'../graphics/plane': { flattenPlanes: () => ({ pixels: new Uint8Array([1, 2]), width: 2, height: 1 }) },
+5
View File
@@ -87,6 +87,11 @@ test('ring and arm input decoders preserve gesture and source', () => {
const payload = p.bytes(13, p.bytes(3, p.concat(p.integer(1, 9), p.integer(2, 3))));
assert.equal(p.decodeGlassesInput({ sid: 224, flag: 1, payload }).eventSource, 3);
assert.equal(p.decodeGlassesInput({ sid: 224, flag: 32, payload }), null);
const headUp = p.bytes(13, p.bytes(3, p.concat(p.integer(1, 12), p.integer(2, 1))));
const wake = p.decodeGlassesInput({ sid: 224, flag: 1, payload: headUp });
assert.equal(wake.kind, 'display-wake');
assert.equal(wake.eventType, 12);
assert.equal(wake.eventSource, 1);
});
class FakeTransport {
+189
View File
@@ -0,0 +1,189 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
const images = require('../.test-build/app/graphics/image.js');
const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js');
const protocol = require('../.test-build/app/g2/ble-protocol.js');
const events = require('../.test-build/app/g2/events.js');
function load(file, modules, globals = {}) {
const context = { exports: {}, require: id => {
if (!(id in modules)) throw new Error(`Unstubbed import: ${id}`);
return modules[id];
}, ...globals };
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText, context);
return context.exports;
}
const planes = load('app/graphics/plane.ts', { './image': images });
const timings = load('app/native/frame-timings.ts', {});
function fixture() {
let now = 1000, nextTask = 0, screenOn = true, shellOptions, session;
const tasks = new Map(), sent = [], previews = [], received = [], errors = [];
const boardStats = { starts: 0, stops: 0, paints: 0 };
const settings = { enabled: true, tap: true, hold: true, tilt: true, duration: 3000 };
const clock = { Date: class extends Date { static now() { return now; } },
setTimeout: (fn, ms) => { tasks.set(++nextTask, { fn, at: now + ms }); return nextTask; },
clearTimeout: id => tasks.delete(id), setInterval: () => ++nextTask, clearInterval() {} };
const { GlanceHost } = load('app/g2/glance-host.ts', {
'../graphics/image': images, '../graphics/plane': planes,
'../graphics/glyph-wire': { prepareFrameDraws: () => null }, '../native/frame-timings': timings,
'../ui/shell/geometry': { minWindowTop: () => 48 },
'../util/render-freshness': { beginRenderPass() {}, endRenderPass: () => false },
'./glance-state': require('../.test-build/app/g2/glance-state.js'),
}, clock);
const window = { windowId: 'launcher', surfaceId: 'launcher', appId: 'launcher', title: 'Apps',
requestRender() {}, setScreenOn() {} };
const shell = {
configure: options => { shellOptions = options; }, registerWindow() {}, focusWindow() {},
wake: () => { screenOn = true; shellOptions.onScreenStateChanged(true); },
sleep: () => { screenOn = false; shellOptions.onScreenStateChanged(false); },
isScreenOn: () => screenOn, getWindows: () => [window], foregroundWindow: () => window,
setBatteryLevels() {}, underlayDim: () => 1, getFocus: () => 'app', hasOverlay: () => false,
describeInputTarget: () => 'test app',
paintSurface: () => [{ image: new images.GrayImage(640, 480, 0), x: 0, y: 0 }],
receiveInput: async input => {
received.push(input);
if (input.type === 'display-wake' || !screenOn && input.type === 'double-click') shell.wake();
},
};
class Session {
state = { phase: 'disconnected' };
constructor(_transport, onState, onInput) { session = this; this.onState = onState; this.onInput = onInput; }
async start() { this.state = { phase: 'connected' }; this.onState(this.state); }
async stop() { this.state = { phase: 'disconnected' }; this.onState(this.state); }
setFrame(pixels) { if (this.state.phase === 'connected') sent.push(pixels); }
wake() {}
}
const provider = {
isEnabled: () => settings.enabled, showOnTap: () => settings.tap,
showOnLongPress: () => settings.hold, showOnHeadTilt: () => settings.tilt,
tapTimeoutMs: () => settings.duration,
createBoard: () => ({ start: () => boardStats.starts++, stop: () => boardStats.stops++,
paint: () => { boardStats.paints++; return new images.GrayImage(100, 80, 200); } }),
};
const modules = {
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
'../native/ios-bluetooth': { iosBluetooth: () => ({}) },
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
'./glasses-session': { GlassesSession: Session }, './glance-host': { GlanceHost }, './events': events,
'./device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null },
'../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' },
'../apps/all-apps': { ALL_APPS: [{ appId: 'glanceboard', glanceboard: provider }] },
'../ui/shell/worker-window': {}, '../ui/shell/in-process-window': {},
'../ui/dashboard-settings': { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } },
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../apps/ios-availability': { iosAppUnavailableReason: () => null },
'../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images,
'../native/ios-graphics': { previewPixels: pixels => pixels },
'../ui/gestures': { makeInputEvent: event => event }, '../ui/layers': { noopLayerActions: {} },
'../apps/files/text-viewer': {},
'../ui/shell/shell': { shell, rawInputEventToInputEvent: event => ({
type: event.kind === 'display-wake' ? 'display-wake'
: ({ 0: 'click', 1: 'scroll-up', 2: 'scroll-down', 3: 'double-click', 9: 'long-press',
10: 'long-press-release', 11: 'short-then-long-press' })[event.eventType] ?? 'unknown', source: 'ring',
}) },
'../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 640, height: 480 }) },
};
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', modules, {
...clock, console: { log() {}, warn() {}, error: text => errors.push(text) },
UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } },
UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state',
NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } },
NSOperationQueue: { mainQueue: {} },
});
const controller = new IosPreviewController((pixels, title) => previews.push({ pixels, title }), message => errors.push(message));
controller.compositor.submitSurfaceFrame('launcher', new Uint8Array(640 * 480).fill(75), { x: 0, y: 0, width: 640, height: 480 });
controller.resume();
async function drain() { await controller.inputQueue; await controller.glance.queue; }
async function advance(ms) {
now += ms;
for (const [id, task] of [...tasks]) if (task.at <= now && tasks.delete(id)) task.fn();
await drain();
}
async function render() { await drain(); await advance(33); assert.deepEqual(errors, []); }
async function hardware(type, source = 2) {
const payload = protocol.bytes(13, protocol.bytes(3, protocol.concat(protocol.integer(1, type), protocol.integer(2, source))));
session.onInput(protocol.decodeGlassesInput({ sid: protocol.SID.hub, flag: 1, payload }));
await render();
}
async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); }
return { controller, shell, settings, sent, previews, received, boardStats, hardware, phone, advance, render };
}
const assertBlank = pixels => assert.ok(pixels.every(p => p === 0));
const assertBoard = pixels => {
assert.equal(pixels[48 * 640 + 270], 200);
assert.equal(pixels[0], 0, 'opaque Glanceboard hides the retained app and shell');
};
test('iOS sleep tap shows the shared board; repeated tap renews timeout and hides to black', async () => {
const f = fixture(); await f.controller.connect(); f.shell.sleep(); await f.render();
assertBlank(f.sent.at(-1));
await f.hardware(0); assertBoard(f.sent.at(-1)); assert.equal(f.shell.isScreenOn(), false);
assert.equal(f.previews.at(-1).title, 'Glanceboard'); assert.equal(f.received.length, 0);
await f.advance(2000); await f.hardware(0); await f.advance(1100);
assert.equal(f.controller.glance.isVisible(), true);
const offset = f.sent.length;
await f.advance(2000); await f.render();
assert.equal(f.controller.glance.isVisible(), false);
for (const pixels of f.sent.slice(offset)) assertBlank(pixels);
assert.equal(f.boardStats.starts, 1); assert.equal(f.boardStats.stops, 1);
});
test('iOS holds and tap-then-holds last until release, including with phone backgrounded', async () => {
const f = fixture(); await f.controller.connect(); f.shell.sleep(); f.controller.pause();
const previews = f.previews.length;
for (const type of [9, 11]) {
await f.hardware(type, 1); assertBoard(f.sent.at(-1));
await f.advance(10000); assert.equal(f.controller.glance.isVisible(), true);
await f.hardware(10, 1); assertBlank(f.sent.at(-1));
}
assert.equal(f.previews.length, previews); assert.equal(f.received.length, 0);
});
test('double-tap and other shell wakes replace Glanceboard without waking it again', async () => {
const f = fixture(); await f.controller.connect(); f.shell.sleep();
await f.hardware(9); await f.hardware(3);
assert.equal(f.shell.isScreenOn(), true); assert.equal(f.controller.glance.isVisible(), false);
assert.equal(f.sent.at(-1)[0], 75); assert.equal(f.boardStats.stops, 1);
f.shell.sleep(); await f.hardware(0); f.shell.wake(); await f.render();
assert.equal(f.controller.glance.isVisible(), false); assert.equal(f.sent.at(-1)[0], 75);
await f.advance(5000); await f.render(); assert.equal(f.sent.at(-1)[0], 75);
});
test('head tilt uses Glanceboard when enabled and otherwise wakes the regular UI', async () => {
const f = fixture(); await f.controller.connect(); f.shell.sleep();
await f.hardware(12, 1); assertBoard(f.sent.at(-1)); assert.equal(f.shell.isScreenOn(), false);
await f.hardware(3); f.shell.sleep(); f.settings.tilt = false;
await f.hardware(12, 1); assert.equal(f.shell.isScreenOn(), true); assert.equal(f.sent.at(-1)[0], 75);
});
test('disabled triggers and scrolls do not show the board; awake taps still reach the shell', async () => {
const f = fixture(); await f.controller.connect();
await f.hardware(0); assert.equal(f.received.at(-1).type, 'click'); assert.equal(f.boardStats.starts, 0);
f.shell.sleep(); f.settings.tap = false; f.settings.hold = false;
for (const type of [0, 9, 11, 10, 1, 2]) await f.hardware(type);
assert.equal(f.boardStats.starts, 0); assertBlank(f.sent.at(-1));
f.settings.enabled = false; f.settings.tap = f.settings.hold = true;
for (const type of [0, 9, 11]) await f.hardware(type);
assert.equal(f.boardStats.starts, 0);
await f.hardware(3); assert.equal(f.shell.isScreenOn(), true);
});
test('phone preview gestures use Glanceboard; losing runtime cleans up holds and timers', async () => {
const f = fixture(); f.shell.sleep();
await f.phone('tap', 'mirror'); assertBoard(f.previews.at(-1).pixels);
await f.phone('double-tap', 'mirror'); assert.equal(f.shell.isScreenOn(), true);
f.shell.sleep(); await f.phone('long-press', 'ring');
await f.phone('long-press-release', 'ring'); assertBlank(f.previews.at(-1).pixels);
await f.phone('short-then-long-press', 'ring'); assert.equal(f.controller.glance.isVisible(), true);
f.controller.pause(); await f.render(); assert.equal(f.controller.glance.isVisible(), false);
f.controller.resume(); await f.render(); assertBlank(f.previews.at(-1).pixels);
assert.equal(f.boardStats.starts, f.boardStats.stops);
});