diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index a70a7c55..d4187363 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -51,8 +51,7 @@ import { voiceControlBridge } from "../native/voice-control"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; import { prepareFrameDraws } from "../graphics/glyph-wire"; -import { getDefaultMediumFont } from "../graphics/ui-fonts"; -import { wrapText } from "../graphics/textwrap"; +import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen"; import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; import { type InputEvent } from "../ui/gestures"; import { registerSystemTools } from "../assistant/system-tools"; @@ -141,8 +140,6 @@ type DashboardListener = (snapshot: DashboardSnapshot) => void; const SHELL_SURFACE_ID = "shell"; /** The shell chrome composites above every window surface (zOrder 0). */ const SHELL_SURFACE_Z_ORDER = 1; -const LOCK_SCREEN_SURFACE_ID = "lock-screen"; -const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glasses."; // Top-bar clock refresh; the phone-side preview polls the Java composite so // it reflects every app (including worker apps the TS side never renders). const SHELL_REFRESH_INTERVAL_MS = 60_000; @@ -172,24 +169,6 @@ function createInitialDisplayPreview(): ImageSource | null { return grayImageToPreviewSource(new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0)); } -function createLockScreenImage(): GrayImage { - const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - const font = getDefaultMediumFont(); - const boxWidth = 480; - const boxHeight = 150; - const boxX = Math.round((G2_LENS_WIDTH - boxWidth) / 2); - const boxY = Math.round((G2_LENS_HEIGHT - boxHeight) / 2); - image.drawRoundedRect(boxX, boxY, boxWidth, boxHeight, 150, 12); - const lines = wrapText(font, LOCK_SCREEN_MESSAGE, boxWidth - 64); - const textHeight = lines.length * font.lineHeight; - const firstY = boxY + Math.round((boxHeight - textHeight) / 2); - lines.forEach((line, index) => { - const x = Math.round((G2_LENS_WIDTH - font.measureText(line)) / 2); - image.drawText(font, x, firstY + index * font.lineHeight, line, 230); - }); - return image; -} - function formatTimestamp(date: Date): string { return date.toISOString().slice(11, 23); } diff --git a/app/g2/glasses-session.ts b/app/g2/glasses-session.ts index d3aeca16..e7502923 100644 --- a/app/g2/glasses-session.ts +++ b/app/g2/glasses-session.ts @@ -6,6 +6,7 @@ import { CfwTransport, CFW_MAX_MESSAGE, parseCfwAcks } from './cfw-transport' import { hasCompatibleFirmware, firmwareIncompatibilityMessage, REQUIRED_FACECLAW_FIRMWARE_VERSION } from './firmware-compat' import { hexToBytes } from '../util/hex-util' import { deviceAddressError } from './ios-peripheral-identity' +import { decodeWearState, enableWearDetection, queryWearState } from './wear-protocol' declare function setTimeout(callback: () => void, ms: number): number declare function clearTimeout(id: number): void @@ -84,7 +85,8 @@ export class GlassesSession { private readonly onInput: (input: protocol.GlassesInput) => void, private readonly log: (message: string) => void = () => {}, private readonly onActivity: () => void = () => {}, - private readonly onCompass: (event: CompassEvent) => void = () => {}) { + private readonly onCompass: (event: CompassEvent) => void = () => {}, + private readonly onWearState: (wearing: boolean) => void = () => {}) { this.off = transport.onEvent(event => this.receive(event)) } private update(phase: SessionState['phase'], status: string): void { @@ -134,6 +136,7 @@ export class GlassesSession { await this.lease(true) await this.request('right', protocol.SID.hub, protocol.createLayout, 'Create display layout') this.check(generation); this.layoutCreated = true + await this.enableWearDetectionAndRequestState(); this.check(generation) this.lastHeartbeat = this.lastLease = this.lastSettings = Date.now() this.displayed = null; this.retryCount = 0 this.update('connected', 'Glasses connected') @@ -353,6 +356,8 @@ export class GlassesSession { } } if (message.sid === protocol.SID.settings) this.applySettings(message) + const wearing = decodeWearState(message) + if (wearing !== null) this.onWearState(wearing) if (event.identifier === this.ids.right) { if (this.state.phase === 'connected') { const compass = protocol.decodeCompassInput(message) @@ -389,6 +394,16 @@ export class GlassesSession { } this.onState({ ...this.state }) } + async enableWearDetectionAndRequestState(): Promise { + const generation = this.generation + // Like Android, a missing enable ACK must not suppress the current-state query. + try { await this.request('right', protocol.SID.settings, enableWearDetection, 'Enable wear detection') } + catch (error) { if (!(error instanceof AckTimeout)) throw error; this.log(this.message(error)) } + this.check(generation) + await this.send('right', protocol.SID.settings, 0x20, queryWearState()) + this.check(generation) + await this.send('left', protocol.SID.settings, 0x20, queryWearState()) + } setFrame(gray: Uint8Array): void { if (this.state.phase !== 'connected') return this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0) diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index f61fed99..ac7be2db 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -16,6 +16,7 @@ import { iosVoiceInput } from '../native/ios-voice-input' import { nightscoutBridge } from '../native/nightscout-bridge' import { GlassesSession, type SessionState } from './glasses-session' import { GlanceHost, type GlanceDisplay } from './glance-host' +import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen' import { OsEventTypeList } from './events' import { loadDeviceAddresses } from './device-addresses' import { deviceAddressError } from './ios-peripheral-identity' @@ -37,7 +38,7 @@ import { TextViewerLayer } from '../apps/files/text-viewer' import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell' import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry' import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged, - previewColorSetting } from '../ui/dashboard-settings' + previewColorSetting, lockScreenEnabledSetting } from '../ui/dashboard-settings' import type { PhoneGesture } from '../phone-ui/phone-gestures' /** iOS host for the shared app registry, shell, compositor and BLE session. */ @@ -60,7 +61,7 @@ export class IosPreviewController { private readonly glance = new GlanceHost({ getDisplay: () => this.glanceDisplay, getProvider: () => ALL_APPS.find(app => app.glanceboard)?.glanceboard ?? null, - canShow: () => this.runtimeNeeded, + canShow: () => this.runtimeNeeded && !this.glassesLocked, // iOS retains the EvenHub session while the shell sleeps. The board's // opaque first frame is ready before we unblank the compositor. ensureSessionActive: async () => { @@ -86,6 +87,11 @@ export class IosPreviewController { private readonly appHosts = new Map() private readonly inProcessApps = new Map() private readonly batteryObservers: any[] = [] + private readonly lockObservers: any[] = [] + private phoneLocked = false + private glassesWorn: boolean | null = null + private glassesLocked = false + private lockEnabled = lockScreenEnabledSetting.get() private session: GlassesSession | null = null private logLines: string[] = [] private logTimer: ReturnType | null = null @@ -112,6 +118,10 @@ export class IosPreviewController { private readonly onError: (message: string) => void, private readonly onConnectionState: (state: SessionState) => void = () => {}) { this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' }) + this.compositor.configureSurface(LOCK_SCREEN_SURFACE_ID, { + x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: 1000, transparency: 'opaque', + }) + this.compositor.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, false) shell.configure({ actions: this.actions, voiceInputEnabled: true, @@ -148,6 +158,7 @@ export class IosPreviewController { if (this.active) return this.active = true this.syncRuntime() + this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable) this.logBluetooth('Phone foreground') if (this.session) this.onConnectionState({ ...this.session.state }) this.session?.wake() @@ -178,6 +189,7 @@ export class IosPreviewController { this.compositor.setScreenBlanked(!shell.isScreenOn()) this.offSettings?.(); this.offSettings = null for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer) + for (const observer of this.lockObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer) UIDevice.currentDevice.batteryMonitoringEnabled = false if (this.clockTimer !== null) clearInterval(this.clockTimer) if (this.renderTimer !== null) clearTimeout(this.renderTimer) @@ -185,6 +197,16 @@ export class IosPreviewController { return } UIDevice.currentDevice.batteryMonitoringEnabled = true + // Keep these observers alive while BLE owns the runtime, including background. + for (const [name, locked] of [ + [UIApplicationProtectedDataWillBecomeUnavailable, true], + [UIApplicationProtectedDataDidBecomeAvailable, false], + ] as const) { + this.lockObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock( + name, null, NSOperationQueue.mainQueue, () => this.handlePhoneLockState(locked))) + } + this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable) + this.syncLockSetting() void nightscoutBridge.start().catch(error => this.fail(error)) for (const name of [UIDeviceBatteryLevelDidChangeNotification, UIDeviceBatteryStateDidChangeNotification]) { this.batteryObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(name, null, NSOperationQueue.mainQueue, () => this.requestShellRender())) @@ -192,12 +214,53 @@ export class IosPreviewController { const phone = readPhoneBatteryState() this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.offSettings = onAnySettingChanged(() => { + this.syncLockSetting() this.relayout() shell.foregroundWindow()?.requestRender() this.requestShellRender() }) this.clockTimer = setInterval(() => this.refreshClock(), 60_000) } + private syncLockSetting(): void { + const enabled = lockScreenEnabledSetting.get() + if (enabled === this.lockEnabled) return + this.lockEnabled = enabled + if (!enabled) this.setGlassesLocked(false) + else { + if (this.phoneLocked && this.glassesWorn === false) this.setGlassesLocked(true) + if (this.session?.state.phase === 'connected') + void this.session.enableWearDetectionAndRequestState().catch(error => this.fail(error)) + } + } + private handlePhoneLockState(locked: boolean): void { + this.phoneLocked = locked + if (!locked) this.setGlassesLocked(false) + else if (this.lockEnabled && this.glassesWorn === false) this.setGlassesLocked(true) + } + private handleWearState(wearing: boolean): void { + // Sample false to catch a notification missed during suspension. Do not + // sample true here: WillBecomeUnavailable precedes the property transition. + if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true) + this.glassesWorn = wearing + this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`) + if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true) + } + private setGlassesLocked(locked: boolean): void { + if (locked === this.glassesLocked) return + if (locked) { + const image = createLockScreenImage() + this.compositor.submitSurfaceFrame(LOCK_SCREEN_SURFACE_ID, image.to8bppBuffer(), + { x: 0, y: 0, width: image.width, height: image.height }) + } + this.glassesLocked = locked + if (locked) { + this.glance.dismiss() + iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.') + } + this.compositor.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, locked) + this.logBluetooth(`Glasses ${locked ? 'locked' : 'unlocked'}`) + this.requestShellRender() + } private refreshClock(): void { const minute = Math.floor(Date.now() / 60_000) if (minute !== this.clockMinute) { this.clockMinute = minute; this.requestShellRender() } @@ -240,7 +303,7 @@ export class IosPreviewController { this.session?.setFrame(pixels) if (this.active) { const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green') - this.onFrame(image, this.glance.isVisible() ? 'Glanceboard' + this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard' : `${shell.foregroundWindow()?.title ?? 'Apps'} ยท ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) } } catch (error) { this.fail(error) } @@ -255,7 +318,7 @@ export class IosPreviewController { gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void { this.inputQueue = this.inputQueue.then(async () => { if (!this.active) return - if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) { + if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn() && !this.glassesLocked) { await this.mirrorTap(nx, ny) } else { let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record)[gesture] ?? gesture @@ -270,6 +333,16 @@ export class IosPreviewController { }).catch(error => this.fail(error)) } private async receiveInput(event: InputEvent, headTilt = false): Promise { + if (this.glassesLocked) { + // Preserve the locked display's sleep/wake controls without dispatching + // gestures to apps, shell menus, voice input or the Glanceboard. + if (event.type === 'double-click' && (event.source === 'ring' || event.source === 'watch')) { + if (shell.isScreenOn()) shell.sleep() + else shell.wake('sidebar') + } else if (event.type === 'display-wake' && !shell.isScreenOn()) shell.wake('sidebar') + this.requestShellRender() + return + } if (!shell.isScreenOn()) { const glanceEvent = this.glance.eventForGesture(headTilt ? 'head-tilt' : event.type) if (glanceEvent?.type === 'dismiss') this.glance.dismiss() @@ -409,6 +482,7 @@ export class IosPreviewController { if (error) { this.onError(error); return } if (!this.session) { this.session = new GlassesSession(iosBluetooth(), state => { + if (state.phase !== 'connected' && state.phase !== 'connecting') this.glassesWorn = null if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded() shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging }) this.syncRuntime() @@ -421,24 +495,28 @@ export class IosPreviewController { input.kind === 'display-wake' && input.eventType === OsEventTypeList.HEAD_UP_EVENT) this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`) }).catch(error => this.fail(error)) - }, message => this.logBluetooth(message), () => this.refreshClock(), receiveCompassEvent) + }, message => this.logBluetooth(message), () => { + if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true) + this.refreshClock() + }, receiveCompassEvent, wearing => this.handleWearState(wearing)) bindCompassSession(this.session) } await this.session.start(addresses) } async disconnect(): Promise { await this.session?.stop() } - startVoiceInput(): void { shell.startVoiceInput() } + startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() } private async prepareVoiceCapture(): Promise { + if (this.glassesLocked) return false if (this.session?.state.phase !== 'connected') { if (this.active) this.onError('Connect the glasses to use their microphone.') return false } const ready = await iosVoiceInput.prepare(this.active) if (!ready && this.active) this.onError(iosVoiceInput.statusText) - return ready && this.session?.state.phase === 'connected' + return ready && !this.glassesLocked && this.session?.state.phase === 'connected' } private async startVoiceCapture(endpointing = false): Promise { - if (!this.session || this.session.state.phase !== 'connected') return + if (this.glassesLocked || !this.session || this.session.state.phase !== 'connected') return await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing) } private logBluetooth(message: string): void { @@ -458,13 +536,14 @@ export class IosPreviewController { catch (error) { console.warn(`Bluetooth log: ${error}`) } } async typeIntoApp(): Promise { + if (this.glassesLocked) return if (!this.active) { this.logBluetooth('Text input requires opening Faceclaw on the phone'); return } if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return this.prompting = true try { const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title, message: 'Enter text to send to this app.', okButtonText: 'Send', cancelButtonText: 'Cancel' }) - if (result.result) shell.sendTextToForegroundWindow(result.text) + if (result.result && !this.glassesLocked) shell.sendTextToForegroundWindow(result.text) } catch (error) { this.fail(error) } finally { this.prompting = false } } diff --git a/app/g2/lock-screen.ts b/app/g2/lock-screen.ts new file mode 100644 index 00000000..485504bd --- /dev/null +++ b/app/g2/lock-screen.ts @@ -0,0 +1,24 @@ +import { GrayImage, G2_LENS_WIDTH, G2_LENS_HEIGHT } from "../graphics/image"; +import { getDefaultMediumFont } from "../graphics/ui-fonts"; +import { wrapText } from "../graphics/textwrap"; + +export const LOCK_SCREEN_SURFACE_ID = "lock-screen"; +const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glasses."; + +export function createLockScreenImage(): GrayImage { + const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + const font = getDefaultMediumFont(); + const boxWidth = 480; + const boxHeight = 150; + const boxX = Math.round((G2_LENS_WIDTH - boxWidth) / 2); + const boxY = Math.round((G2_LENS_HEIGHT - boxHeight) / 2); + image.drawRoundedRect(boxX, boxY, boxWidth, boxHeight, 150, 12); + const lines = wrapText(font, LOCK_SCREEN_MESSAGE, boxWidth - 64); + const textHeight = lines.length * font.lineHeight; + const firstY = boxY + Math.round((boxHeight - textHeight) / 2); + lines.forEach((line, index) => { + const x = Math.round((G2_LENS_WIDTH - font.measureText(line)) / 2); + image.drawText(font, x, firstY + index * font.lineHeight, line, 230); + }); + return image; +} diff --git a/app/g2/wear-protocol.ts b/app/g2/wear-protocol.ts new file mode 100644 index 00000000..4e06c1ef --- /dev/null +++ b/app/g2/wear-protocol.ts @@ -0,0 +1,15 @@ +import { bytes, concat, integer, readBytes, readInteger, type ProtocolMessage } from './ble-protocol' + +// Matches BleProtocol.buildSetWearDetection/buildFaceclawWakeControl on Android. +export const enableWearDetection = (magic: number): Uint8Array => concat( + integer(1, 1), integer(2, magic), bytes(3, bytes(5, integer(1, 1)))) +export const queryWearState = (): Uint8Array => concat( + integer(1, 1), integer(2, 0), bytes(101, new Uint8Array([70, 67, 1, 7, 0, 0]))) + +export function decodeWearState(message: ProtocolMessage): boolean | null { + if (message.sid !== 0x10 || message.command !== 3) return null + const event = readBytes(message.payload, 5) + if (!event || readInteger(event, 1, -1) !== 1) return null + const state = readInteger(event, 2, -1) + return state === 0 ? false : state === 1 ? true : null +} diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 62c9df51..d6333bc3 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -91,9 +91,9 @@ export class IosVoiceControlBridge { this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() this.finishCompletion() } - handleSessionEnded(): void { + handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void { if (!this.capturing && !this.finalizing) return - this.stop(); this.setStatus('Glasses disconnected. Start voice input again after reconnecting.'); this.emitEnd() + this.stop(); this.setStatus(message); this.emitEnd() } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } private releaseMicrophone(): void { diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 95557747..aed240dc 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -341,7 +341,8 @@ export const lockScreenEnabledSetting = new ConfigSettingBoolean({ storageKey: "display.lockScreenEnabled", defaultValue: true, description: - "Lock the glasses after they are taken off while the phone is locked. Unlocking the phone unlocks the glasses.", + "Lock the glasses after they are taken off while the phone is locked. Unlocking the phone unlocks the glasses." + + (global.isIOS ? " On iPhone, this requires a device passcode and follows iOS data-protection notifications, which may be delayed after the screen locks." : ""), }); // Phone display: the phone app's mirror of the glasses screen and the diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index c06aef99..938ba9df 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -254,7 +254,7 @@ function settingsSections(): SettingsSection[] { description: `${section.label} integration has not been ported to iOS.`, }] }; if (section.label === "Display") return { ...section, items: section.items.filter(item => - ![brightnessSetting.label, screenTimeoutSetting.label, lockScreenEnabledSetting.label].includes(item.label)) }; + ![brightnessSetting.label, screenTimeoutSetting.label].includes(item.label)) }; if (section.label === "Phone display") return { ...section, items: section.items.filter(item => item.label !== mirrorTouchSetting.label) }; return section; }); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index f67fcdcc..657e7596 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -80,6 +80,7 @@ for (const isIOS of [true, false]) { uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) }; } }); const sections = menus.createSettingsPanelLayer().sections; + assert.ok(sections.find(section => section.label === 'Display').items.some(item => item.label === settings.lockScreenEnabledSetting.label)); const assistant = sections.find(section => section.label === 'Assistant'); assert.equal(assistant.items.some(item => item.disabled === true), false); const ctx = { stack: { pop() {} }, actions: { requestRender() {} } }; @@ -186,7 +187,7 @@ test('background glasses input still composites frames; phone resume preserves t setFrame(pixels) { if (this.state.phase === 'connected') frames.push(pixels); } wake() {} } - const settings = { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } }; + const settings = { lockScreenEnabledSetting: { get: () => true }, onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } }; const modules = { '../assistant/system-tools': { registerSystemTools() {} }, '../assistant/window-tools': { registerWindowTools() {} }, @@ -219,6 +220,7 @@ test('background glasses input still composites frames; phone resume preserves t setTimeout: fn => { tasks.set(++nextTask, fn); return nextTask; }, clearTimeout: id => tasks.delete(id), setInterval: () => ++nextTask, clearInterval() {}, UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } }, + UIApplicationProtectedDataWillBecomeUnavailable: 'lock', UIApplicationProtectedDataDidBecomeAvailable: 'unlock', UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state', NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } }, NSOperationQueue: { mainQueue: {} }, diff --git a/tests/ios-ble.test.cjs b/tests/ios-ble.test.cjs index 5d28078a..6ef67cb3 100644 --- a/tests/ios-ble.test.cjs +++ b/tests/ios-ble.test.cjs @@ -159,10 +159,10 @@ async function until(predicate) { assert.fail('Timed out waiting for test state'); } function harness(t, transport = new FakeTransport()) { - const states = [], input = [], compass = []; - const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event)); + const states = [], input = [], compass = [], wear = []; + const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing)); t.after(async () => { transport.hold = false; await session.stop(); }); - return { transport, session, states, input, compass }; + return { transport, session, states, input, compass, wear }; } test('session authenticates both arms, checks firmware and sends acknowledged display data to L', async t => { const h = harness(t); await h.session.start({ ...addresses, ring: 'AA:BB:CC:DD:EE:03' }); @@ -511,3 +511,42 @@ test('compass notifications only come from the right arm and preserve diagnostic magneticAccuracy: 3, magneticAnomalies: 2, orientationSource: 3, flags: 140, sampleTimeMs: 0xfedcba98, } }, { command: 16, headingDegrees: -1 }, { command: 17, headingDegrees: -1 }]); }); + + +test('wear detection matches Android wire format and queries both arms on every connection', async t => { + const wear = require('../.test-build/app/g2/wear-protocol.js'); + assert.equal(hex(wear.enableWearDetection(42)), '0801102a1a042a020801'); + assert.equal(hex(wear.queryWearState()), '08011000aa0606464301070000'); + const h = harness(t); await h.session.start(addresses); + const queries = () => h.transport.sent.filter(({ message }) => hex(message.payload) === hex(wear.queryWearState())); + assert.deepEqual(queries().map(q => q.id).sort(), ['L', 'R']); + const enable = h.transport.sent.find(({ message }) => { + const body = p.readBytes(message.payload, 3); + return message.sid === p.SID.settings && message.command === 1 && body && p.readBytes(body, 5); + }); + assert.equal(enable.id, 'R'); assert.equal(enable.message.flag, 0x20); + await h.session.stop(); await h.session.start(addresses); + assert.equal(queries().length, 4); +}); + +test('wear notifications validate the event and peer and work during initial connection', async t => { + const h = harness(t); + const payload = state => p.concat(p.integer(1, 3), p.bytes(5, p.concat(p.integer(1, 1), p.integer(2, state)))); + const emit = (id, data, sid = 0x10) => h.transport.emit({ kind: 'notification', identifier: id, + characteristic: p.G2_NOTIFY, data: hex(p.concat(...p.frameMessage(data, sid, 1, 12))) }); + const originalWrite = h.transport.write.bind(h.transport); + h.transport.write = async (...args) => { + await originalWrite(...args); + if (h.transport.sent.at(-1)?.message.magic === 0) emit(args[0], payload(0)); + }; + await h.session.start(addresses); + assert.ok(h.wear.length >= 2); assert.ok(h.wear.every(value => value === false)); + h.wear.length = 0; + emit('stranger', payload(0)); emit('ring', payload(0)); emit('R', payload(2)); + emit('R', payload(0), p.SID.hub); emit('R', p.integer(1, 3)); + assert.deepEqual(h.wear, []); + emit('L', payload(0)); emit('R', payload(1)); + assert.deepEqual(h.wear, [false, true]); + await h.session.stop(); emit('R', payload(0)); + assert.deepEqual(h.wear, [false, true]); +}); diff --git a/tests/ios-glanceboard.test.cjs b/tests/ios-glanceboard.test.cjs index 0eb9e85e..75a16bc9 100644 --- a/tests/ios-glanceboard.test.cjs +++ b/tests/ios-glanceboard.test.cjs @@ -23,10 +23,12 @@ const planes = load('app/graphics/plane.ts', { './image': images }); const timings = load('app/native/frame-timings.ts', {}); function fixture() { + const observers = new Map(), phoneState = { protectedDataAvailable: true }; + let settingsChanged; let now = 1000, nextTask = 0, screenOn = true, shellOptions, session; const tasks = new Map(), sent = [], previews = [], received = [], errors = []; const boardStats = { starts: 0, stops: 0, paints: 0 }; - const settings = { enabled: true, tap: true, hold: true, tilt: true, duration: 3000 }; + const settings = { lock: true, enabled: true, tap: true, hold: true, tilt: true, duration: 3000 }; const clock = { Date: class extends Date { static now() { return now; } }, setTimeout: (fn, ms) => { tasks.set(++nextTask, { fn, at: now + ms }); return nextTask; }, clearTimeout: id => tasks.delete(id), setInterval: () => ++nextTask, clearInterval() {} }; @@ -54,7 +56,8 @@ function fixture() { }; class Session { state = { phase: 'disconnected' }; - constructor(_transport, onState, onInput) { session = this; this.onState = onState; this.onInput = onInput; } + constructor(_transport, onState, onInput, _log, _activity, _compass, onWear) { session = this; this.onState = onState; this.onInput = onInput; this.onWear = onWear; } + async enableWearDetectionAndRequestState() {} async start() { this.state = { phase: 'connected' }; this.onState(this.state); } async stop() { this.state = { phase: 'disconnected' }; this.onState(this.state); } setFrame(pixels) { if (this.state.phase === 'connected') sent.push(pixels); } @@ -81,13 +84,14 @@ function fixture() { './glasses-session': { GlassesSession: Session }, '../native/nightscout-bridge': { nightscoutBridge: { async start() {}, async stop() {} } }, './glance-host': { GlanceHost }, './events': events, + './lock-screen': { LOCK_SCREEN_SURFACE_ID: 'lock-screen', createLockScreenImage: () => new images.GrayImage(640, 480, 123) }, './device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null }, '../apps/launcher': { launcherEntries: () => [] }, '../apps/evenhub/installed-apps': {}, '../apps/evenhub/manager': {}, '../apps/evenhub/updates': {}, '../apps/evenhub': {}, '../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' }, '../apps/all-apps': { ALL_APPS: [{ appId: 'glanceboard', glanceboard: provider }] }, '../ui/shell/worker-window': {}, '../ui/shell/in-process-window': {}, - '../ui/dashboard-settings': { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } }, + '../ui/dashboard-settings': { lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } }, '../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) }, '../apps/ios-availability': { iosAppUnavailableReason: () => null }, '../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images, @@ -103,9 +107,10 @@ function fixture() { }; const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', modules, { ...clock, console: { log() {}, warn() {}, error: text => errors.push(text) }, - UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } }, + UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: phoneState }, + UIApplicationProtectedDataWillBecomeUnavailable: 'lock', UIApplicationProtectedDataDidBecomeAvailable: 'unlock', UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state', - NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } }, + NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock(name, _object, _queue, fn) { observers.set(name, fn); return name; }, removeObserver(name) { observers.delete(name); } } }, NSOperationQueue: { mainQueue: {} }, }); const controller = new IosPreviewController((pixels, title) => previews.push({ pixels, title }), message => errors.push(message)); @@ -124,7 +129,7 @@ function fixture() { await render(); } async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); } - return { controller, shell, settings, sent, previews, received, boardStats, hardware, phone, advance, render }; + return { controller, shell, settings, phoneState, observers, wear: wearing => session.onWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render }; } const assertBlank = pixels => assert.ok(pixels.every(p => p === 0)); const assertBoard = pixels => { @@ -197,3 +202,70 @@ test('phone preview gestures use Glanceboard; losing runtime cleans up holds and f.controller.resume(); await f.render(); assertBlank(f.previews.at(-1).pixels); assert.equal(f.boardStats.starts, f.boardStats.stops); }); + + +test('iOS lock hides apps and Glanceboard, blocks input, and stays locked when put back on', async () => { + const f = fixture(); await f.controller.connect(); f.shell.sleep(); await f.hardware(0); + f.controller.pause(); + // The notification precedes the property change: do not immediately undo it. + f.observers.get('lock')(); f.wear(false); await f.render(); + assert.equal(f.controller.glassesLocked, true); + assert.equal(f.controller.glance.isVisible(), false); + await f.hardware(3); // wake the lock screen + assert.ok(f.sent.at(-1).every(p => p === 123)); + for (const type of [0, 1, 2, 9, 11]) await f.hardware(type); + f.wear(true); await f.render(); + assert.equal(f.received.length, 0); + assert.ok(f.sent.at(-1).every(p => p === 123)); + await f.hardware(3); assertBlank(f.sent.at(-1)); + await f.hardware(12, 1); assert.ok(f.sent.at(-1).every(p => p === 123)); + f.observers.get('unlock')(); await f.render(); + assert.equal(f.controller.glassesLocked, false); + assert.equal(f.sent.at(-1)[0], 75); + await f.hardware(0); assert.equal(f.received.length, 1); +}); + +test('iOS removal before phone lock, setting changes, and observer lifetime', async () => { + const f = fixture(); await f.controller.connect(); f.wear(false); await f.render(); + assert.equal(f.controller.glassesLocked, false); + f.observers.get('lock')(); await f.render(); + assert.equal(f.controller.glassesLocked, true); + f.settings.lock = false; f.settingsChanged(); await f.render(); + assert.equal(f.controller.glassesLocked, false); + f.settings.lock = true; f.settingsChanged(); await f.render(); + assert.equal(f.controller.glassesLocked, true); + f.controller.pause(); assert.ok(f.observers.has('unlock')); + await f.controller.disconnect(); assert.equal(f.observers.has('unlock'), false); + f.controller.resume(); await f.render(); + assert.equal(f.controller.glassesLocked, false); + assert.equal(f.observers.size, 4); +}); + +test('iOS wear notifications catch protected-data changes missed while suspended', async () => { + const f = fixture(); await f.controller.connect(); f.controller.pause(); + f.phoneState.protectedDataAvailable = false; f.wear(false); await f.render(); + assert.equal(f.controller.glassesLocked, true); + f.controller.resume(); await f.render(); + assert.equal(f.controller.glassesLocked, true); + f.controller.pause(); f.phoneState.protectedDataAvailable = true; + f.controller.resume(); await f.render(); + assert.equal(f.controller.glassesLocked, false); +}); + + +test('iOS locked phone controls cannot dispatch mirror, voice, or keyboard input', async () => { + const f = fixture(); await f.controller.connect(); f.wear(false); + f.observers.get('lock')(); await f.render(); + await f.phone('tap', 'mirror'); await f.phone('long-press', 'watch'); + f.controller.startVoiceInput(); + assert.equal(await f.controller.prepareVoiceCapture(), false); + await f.controller.startVoiceCapture(); await f.controller.typeIntoApp(); + assert.equal(f.received.length, 0); + assert.equal(f.controller.glassesLocked, true); + // Locked state also survives transport recovery until a phone unlock. + f.controller.session.onState({ phase: 'retrying' }); + f.controller.session.onState({ phase: 'connecting' }); + f.controller.session.onState({ phase: 'connected' }); + await f.render(); + assert.ok(f.sent.at(-1).every(p => p === 123)); +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 13a97140..037e5c2d 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -18,6 +18,7 @@ "../app/g2/ios-peripheral-identity.ts", "../app/g2/glasses-session.ts", "../app/graphics/surface-compositor.ts", + "../app/graphics/image.ts", "../app/phone-ui/phone-gestures.ts", "../app/g2/firmware-compat.ts", "../app/g2/glasses-hardware-identity.ts",