forked from Blendman974/faceclaw
iOS porting: Glanceboard wake gestures
This commit is contained in:
1 parent
6b2e0375e6
commit
14fbbe66ff
6 files changed
+263
-12
No files matched your search
@@ -92,12 +92,13 @@ test('background glasses input still composites frames; phone resume preserves t
|
||||
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) }, knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
|
||||
'../native/ios-bluetooth': { iosBluetooth: () => ({}) }, './glasses-session': { GlassesSession: Session },
|
||||
'./glance-host': { GlanceHost: class { dismiss() {} isVisible() { return false; } } },
|
||||
'./device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null },
|
||||
'../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' },
|
||||
'../apps/all-apps': { ALL_APPS: [] }, '../ui/dashboard-settings': settings,
|
||||
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
|
||||
'../graphics/surface-compositor': { SurfaceCompositor: class {
|
||||
configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {}
|
||||
configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {} setScreenBlanked() {}
|
||||
composite() { return new Uint8Array([1, 2]); }
|
||||
} },
|
||||
'../graphics/plane': { flattenPlanes: () => ({ pixels: new Uint8Array([1, 2]), width: 2, height: 1 }) },
|
||||
|
||||
@@ -87,6 +87,11 @@ test('ring and arm input decoders preserve gesture and source', () => {
|
||||
const payload = p.bytes(13, p.bytes(3, p.concat(p.integer(1, 9), p.integer(2, 3))));
|
||||
assert.equal(p.decodeGlassesInput({ sid: 224, flag: 1, payload }).eventSource, 3);
|
||||
assert.equal(p.decodeGlassesInput({ sid: 224, flag: 32, payload }), null);
|
||||
const headUp = p.bytes(13, p.bytes(3, p.concat(p.integer(1, 12), p.integer(2, 1))));
|
||||
const wake = p.decodeGlassesInput({ sid: 224, flag: 1, payload: headUp });
|
||||
assert.equal(wake.kind, 'display-wake');
|
||||
assert.equal(wake.eventType, 12);
|
||||
assert.equal(wake.eventSource, 1);
|
||||
});
|
||||
|
||||
class FakeTransport {
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
const images = require('../.test-build/app/graphics/image.js');
|
||||
const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js');
|
||||
const protocol = require('../.test-build/app/g2/ble-protocol.js');
|
||||
const events = require('../.test-build/app/g2/events.js');
|
||||
|
||||
function load(file, modules, globals = {}) {
|
||||
const context = { exports: {}, require: id => {
|
||||
if (!(id in modules)) throw new Error(`Unstubbed import: ${id}`);
|
||||
return modules[id];
|
||||
}, ...globals };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, context);
|
||||
return context.exports;
|
||||
}
|
||||
const planes = load('app/graphics/plane.ts', { './image': images });
|
||||
const timings = load('app/native/frame-timings.ts', {});
|
||||
|
||||
function fixture() {
|
||||
let now = 1000, nextTask = 0, screenOn = true, shellOptions, session;
|
||||
const tasks = new Map(), sent = [], previews = [], received = [], errors = [];
|
||||
const boardStats = { starts: 0, stops: 0, paints: 0 };
|
||||
const settings = { enabled: true, tap: true, hold: true, tilt: true, duration: 3000 };
|
||||
const clock = { Date: class extends Date { static now() { return now; } },
|
||||
setTimeout: (fn, ms) => { tasks.set(++nextTask, { fn, at: now + ms }); return nextTask; },
|
||||
clearTimeout: id => tasks.delete(id), setInterval: () => ++nextTask, clearInterval() {} };
|
||||
const { GlanceHost } = load('app/g2/glance-host.ts', {
|
||||
'../graphics/image': images, '../graphics/plane': planes,
|
||||
'../graphics/glyph-wire': { prepareFrameDraws: () => null }, '../native/frame-timings': timings,
|
||||
'../ui/shell/geometry': { minWindowTop: () => 48 },
|
||||
'../util/render-freshness': { beginRenderPass() {}, endRenderPass: () => false },
|
||||
'./glance-state': require('../.test-build/app/g2/glance-state.js'),
|
||||
}, clock);
|
||||
const window = { windowId: 'launcher', surfaceId: 'launcher', appId: 'launcher', title: 'Apps',
|
||||
requestRender() {}, setScreenOn() {} };
|
||||
const shell = {
|
||||
configure: options => { shellOptions = options; }, registerWindow() {}, focusWindow() {},
|
||||
wake: () => { screenOn = true; shellOptions.onScreenStateChanged(true); },
|
||||
sleep: () => { screenOn = false; shellOptions.onScreenStateChanged(false); },
|
||||
isScreenOn: () => screenOn, getWindows: () => [window], foregroundWindow: () => window,
|
||||
setBatteryLevels() {}, underlayDim: () => 1, getFocus: () => 'app', hasOverlay: () => false,
|
||||
describeInputTarget: () => 'test app',
|
||||
paintSurface: () => [{ image: new images.GrayImage(640, 480, 0), x: 0, y: 0 }],
|
||||
receiveInput: async input => {
|
||||
received.push(input);
|
||||
if (input.type === 'display-wake' || !screenOn && input.type === 'double-click') shell.wake();
|
||||
},
|
||||
};
|
||||
class Session {
|
||||
state = { phase: 'disconnected' };
|
||||
constructor(_transport, onState, onInput) { session = this; this.onState = onState; this.onInput = onInput; }
|
||||
async start() { this.state = { phase: 'connected' }; this.onState(this.state); }
|
||||
async stop() { this.state = { phase: 'disconnected' }; this.onState(this.state); }
|
||||
setFrame(pixels) { if (this.state.phase === 'connected') sent.push(pixels); }
|
||||
wake() {}
|
||||
}
|
||||
const provider = {
|
||||
isEnabled: () => settings.enabled, showOnTap: () => settings.tap,
|
||||
showOnLongPress: () => settings.hold, showOnHeadTilt: () => settings.tilt,
|
||||
tapTimeoutMs: () => settings.duration,
|
||||
createBoard: () => ({ start: () => boardStats.starts++, stop: () => boardStats.stops++,
|
||||
paint: () => { boardStats.paints++; return new images.GrayImage(100, 80, 200); } }),
|
||||
};
|
||||
const modules = {
|
||||
'@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) },
|
||||
knownFolders: { documents: () => ({ path: '/tmp' }) }, path },
|
||||
'../native/ios-bluetooth': { iosBluetooth: () => ({}) },
|
||||
'../native/ios-voice-input': { iosVoiceInput: { handleSessionEnded() {} } },
|
||||
'./glasses-session': { GlassesSession: Session }, './glance-host': { GlanceHost }, './events': events,
|
||||
'./device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null },
|
||||
'../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' },
|
||||
'../apps/all-apps': { ALL_APPS: [{ appId: 'glanceboard', glanceboard: provider }] },
|
||||
'../ui/shell/worker-window': {}, '../ui/shell/in-process-window': {},
|
||||
'../ui/dashboard-settings': { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } },
|
||||
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
|
||||
'../apps/ios-availability': { iosAppUnavailableReason: () => null },
|
||||
'../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images,
|
||||
'../native/ios-graphics': { previewPixels: pixels => pixels },
|
||||
'../ui/gestures': { makeInputEvent: event => event }, '../ui/layers': { noopLayerActions: {} },
|
||||
'../apps/files/text-viewer': {},
|
||||
'../ui/shell/shell': { shell, rawInputEventToInputEvent: event => ({
|
||||
type: event.kind === 'display-wake' ? 'display-wake'
|
||||
: ({ 0: 'click', 1: 'scroll-up', 2: 'scroll-down', 3: 'double-click', 9: 'long-press',
|
||||
10: 'long-press-release', 11: 'short-then-long-press' })[event.eventType] ?? 'unknown', source: 'ring',
|
||||
}) },
|
||||
'../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 640, height: 480 }) },
|
||||
};
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', modules, {
|
||||
...clock, console: { log() {}, warn() {}, error: text => errors.push(text) },
|
||||
UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } },
|
||||
UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state',
|
||||
NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } },
|
||||
NSOperationQueue: { mainQueue: {} },
|
||||
});
|
||||
const controller = new IosPreviewController((pixels, title) => previews.push({ pixels, title }), message => errors.push(message));
|
||||
controller.compositor.submitSurfaceFrame('launcher', new Uint8Array(640 * 480).fill(75), { x: 0, y: 0, width: 640, height: 480 });
|
||||
controller.resume();
|
||||
async function drain() { await controller.inputQueue; await controller.glance.queue; }
|
||||
async function advance(ms) {
|
||||
now += ms;
|
||||
for (const [id, task] of [...tasks]) if (task.at <= now && tasks.delete(id)) task.fn();
|
||||
await drain();
|
||||
}
|
||||
async function render() { await drain(); await advance(33); assert.deepEqual(errors, []); }
|
||||
async function hardware(type, source = 2) {
|
||||
const payload = protocol.bytes(13, protocol.bytes(3, protocol.concat(protocol.integer(1, type), protocol.integer(2, source))));
|
||||
session.onInput(protocol.decodeGlassesInput({ sid: protocol.SID.hub, flag: 1, payload }));
|
||||
await render();
|
||||
}
|
||||
async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); }
|
||||
return { controller, shell, settings, sent, previews, received, boardStats, hardware, phone, advance, render };
|
||||
}
|
||||
const assertBlank = pixels => assert.ok(pixels.every(p => p === 0));
|
||||
const assertBoard = pixels => {
|
||||
assert.equal(pixels[48 * 640 + 270], 200);
|
||||
assert.equal(pixels[0], 0, 'opaque Glanceboard hides the retained app and shell');
|
||||
};
|
||||
|
||||
test('iOS sleep tap shows the shared board; repeated tap renews timeout and hides to black', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.shell.sleep(); await f.render();
|
||||
assertBlank(f.sent.at(-1));
|
||||
await f.hardware(0); assertBoard(f.sent.at(-1)); assert.equal(f.shell.isScreenOn(), false);
|
||||
assert.equal(f.previews.at(-1).title, 'Glanceboard'); assert.equal(f.received.length, 0);
|
||||
await f.advance(2000); await f.hardware(0); await f.advance(1100);
|
||||
assert.equal(f.controller.glance.isVisible(), true);
|
||||
const offset = f.sent.length;
|
||||
await f.advance(2000); await f.render();
|
||||
assert.equal(f.controller.glance.isVisible(), false);
|
||||
for (const pixels of f.sent.slice(offset)) assertBlank(pixels);
|
||||
assert.equal(f.boardStats.starts, 1); assert.equal(f.boardStats.stops, 1);
|
||||
});
|
||||
|
||||
test('iOS holds and tap-then-holds last until release, including with phone backgrounded', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.shell.sleep(); f.controller.pause();
|
||||
const previews = f.previews.length;
|
||||
for (const type of [9, 11]) {
|
||||
await f.hardware(type, 1); assertBoard(f.sent.at(-1));
|
||||
await f.advance(10000); assert.equal(f.controller.glance.isVisible(), true);
|
||||
await f.hardware(10, 1); assertBlank(f.sent.at(-1));
|
||||
}
|
||||
assert.equal(f.previews.length, previews); assert.equal(f.received.length, 0);
|
||||
});
|
||||
|
||||
test('double-tap and other shell wakes replace Glanceboard without waking it again', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.shell.sleep();
|
||||
await f.hardware(9); await f.hardware(3);
|
||||
assert.equal(f.shell.isScreenOn(), true); assert.equal(f.controller.glance.isVisible(), false);
|
||||
assert.equal(f.sent.at(-1)[0], 75); assert.equal(f.boardStats.stops, 1);
|
||||
f.shell.sleep(); await f.hardware(0); f.shell.wake(); await f.render();
|
||||
assert.equal(f.controller.glance.isVisible(), false); assert.equal(f.sent.at(-1)[0], 75);
|
||||
await f.advance(5000); await f.render(); assert.equal(f.sent.at(-1)[0], 75);
|
||||
});
|
||||
|
||||
test('head tilt uses Glanceboard when enabled and otherwise wakes the regular UI', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.shell.sleep();
|
||||
await f.hardware(12, 1); assertBoard(f.sent.at(-1)); assert.equal(f.shell.isScreenOn(), false);
|
||||
await f.hardware(3); f.shell.sleep(); f.settings.tilt = false;
|
||||
await f.hardware(12, 1); assert.equal(f.shell.isScreenOn(), true); assert.equal(f.sent.at(-1)[0], 75);
|
||||
});
|
||||
|
||||
test('disabled triggers and scrolls do not show the board; awake taps still reach the shell', async () => {
|
||||
const f = fixture(); await f.controller.connect();
|
||||
await f.hardware(0); assert.equal(f.received.at(-1).type, 'click'); assert.equal(f.boardStats.starts, 0);
|
||||
f.shell.sleep(); f.settings.tap = false; f.settings.hold = false;
|
||||
for (const type of [0, 9, 11, 10, 1, 2]) await f.hardware(type);
|
||||
assert.equal(f.boardStats.starts, 0); assertBlank(f.sent.at(-1));
|
||||
f.settings.enabled = false; f.settings.tap = f.settings.hold = true;
|
||||
for (const type of [0, 9, 11]) await f.hardware(type);
|
||||
assert.equal(f.boardStats.starts, 0);
|
||||
await f.hardware(3); assert.equal(f.shell.isScreenOn(), true);
|
||||
});
|
||||
|
||||
test('phone preview gestures use Glanceboard; losing runtime cleans up holds and timers', async () => {
|
||||
const f = fixture(); f.shell.sleep();
|
||||
await f.phone('tap', 'mirror'); assertBoard(f.previews.at(-1).pixels);
|
||||
await f.phone('double-tap', 'mirror'); assert.equal(f.shell.isScreenOn(), true);
|
||||
f.shell.sleep(); await f.phone('long-press', 'ring');
|
||||
await f.phone('long-press-release', 'ring'); assertBlank(f.previews.at(-1).pixels);
|
||||
await f.phone('short-then-long-press', 'ring'); assert.equal(f.controller.glance.isVisible(), true);
|
||||
f.controller.pause(); await f.render(); assert.equal(f.controller.glance.isVisible(), false);
|
||||
f.controller.resume(); await f.render(); assertBlank(f.previews.at(-1).pixels);
|
||||
assert.equal(f.boardStats.starts, f.boardStats.stops);
|
||||
});
|
||||
Reference in new issue
Block a user