iOS porting: Glanceboard wake gestures

This commit is contained in:
James Babcock committed 2026-09-17 23:50:14 -07:00
1 parent 6b2e0375e6
commit 14fbbe66ff
6 files changed
+263 -12

No files matched your search

+4 -2
View File
@@ -1,4 +1,5 @@
/** G2 wire protocol, ported from g2protocol/BleProtocol.java. No platform APIs. */
import { OsEventTypeList } from './events'
export const G2_WRITE = '00002760-08c2-11e1-9073-0e8ac72e5401'
export const G2_NOTIFY = '00002760-08c2-11e1-9073-0e8ac72e5402'
export const G2_RENDER_NOTIFY = '00002760-08c2-11e1-9073-0e8ac72e6402'
@@ -125,13 +126,14 @@ export function readString(data: Uint8Array, number: number): string { return Ar
export function authenticationSucceeded(message: ProtocolMessage, magic: number): boolean {
return message.sid === SID.auth && message.command === 4 && message.magic === magic && readBytes(message.payload, 3)?.length === 0
}
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export type GlassesInput = { kind: 'sys-event' | 'list-click' | 'text-click' | 'display-wake'; eventType: number; eventSource: number; systemExitReasonCode: number; containerName: string; frameId: number }
export function decodeGlassesInput(message: ProtocolMessage): GlassesInput | null {
if (message.sid !== SID.hub || ![1, 6].includes(message.flag)) return null
const events = readBytes(message.payload, 13); if (!events) return null
for (const [field, kind, typeField] of [[1, 'list-click', 5], [2, 'text-click', 3], [3, 'sys-event', 1]] as const) {
const event = readBytes(events, field)
if (event) return { kind, eventType: readInteger(event, typeField), eventSource: field === 3 ? readInteger(event, 2) : 0,
if (event) return { kind: kind === 'sys-event' && readInteger(event, typeField) === OsEventTypeList.HEAD_UP_EVENT ? 'display-wake' : kind,
eventType: readInteger(event, typeField), eventSource: field === 3 ? readInteger(event, 2) : 0,
systemExitReasonCode: field === 3 ? readInteger(event, 4) : 0, containerName: field === 3 ? '' : readString(event, 2), frameId: 0 }
}
return null
+7 -3
View File
@@ -21,8 +21,12 @@ export const GLANCE_SURFACE_ID = "glance";
/** Above the shell (1) and every window (0); below the lock screen (1000). */
export const GLANCE_SURFACE_Z_ORDER = 900;
/** Surface operations needed by the board, shared by Android and iOS. */
export type GlanceDisplay = Pick<DisplayTarget,
"configureSurface" | "setSurfaceVisible" | "setScreenBlanked" | "submitSurfaceFrame">;
export type GlanceHostOptions = {
getDisplay: () => DisplayTarget | null;
getDisplay: () => GlanceDisplay | null;
getProvider: () => GlanceboardProvider | null;
/** Whether a board may show right now (connected or preview, not locked). */
canShow: () => boolean;
@@ -47,7 +51,7 @@ export class GlanceHost {
/** Whether the surface is currently visible on the compositor. */
private shown = false;
private hideTimer: ReturnType<typeof setTimeout> | null = null;
private configuredFor: DisplayTarget | null = null;
private configuredFor: GlanceDisplay | null = null;
/** Serializes show/hide against each other and against in-flight renders. */
private queue: Promise<void> = Promise.resolve();
private rendering = false;
@@ -144,7 +148,7 @@ export class GlanceHost {
}, Math.max(0, hideAtMs - Date.now()));
}
private async configureSurface(display: DisplayTarget): Promise<void> {
private async configureSurface(display: GlanceDisplay): Promise<void> {
await display.configureSurface(GLANCE_SURFACE_ID, {
x: 0,
y: 0,
+56 -6
View File
@@ -2,6 +2,8 @@ import { Dialogs, File, knownFolders, path, type ImageSource } from '@nativescri
import { iosBluetooth } from '../native/ios-bluetooth'
import { iosVoiceInput } from '../native/ios-voice-input'
import { GlassesSession, type SessionState } from './glasses-session'
import { GlanceHost, type GlanceDisplay } from './glance-host'
import { OsEventTypeList } from './events'
import { loadDeviceAddresses } from './device-addresses'
import { deviceAddressError } from './ios-peripheral-identity'
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
@@ -16,7 +18,7 @@ import { SurfaceCompositor } from '../graphics/surface-compositor'
import { flattenPlanes, type Plane } from '../graphics/plane'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { previewPixels } from '../native/ios-graphics'
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
import { noopLayerActions, type LayerActions } from '../ui/layers'
import { TextViewerLayer } from '../apps/files/text-viewer'
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
@@ -28,6 +30,36 @@ import type { PhoneGesture } from '../phone-ui/phone-gestures'
/** iOS host for the shared app registry, shell, compositor and BLE session. */
export class IosPreviewController {
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
private readonly glanceDisplay: GlanceDisplay = {
configureSurface: async (id, options) => {
this.compositor.configureSurface(id, options)
},
setSurfaceVisible: async (id, visible) => {
this.compositor.setSurfaceVisible(id, visible); this.scheduleFrame()
},
setScreenBlanked: async blanked => {
this.compositor.setScreenBlanked(blanked); this.scheduleFrame()
},
submitSurfaceFrame: async (id, pixels, rect) => {
this.compositor.submitSurfaceFrame(id, pixels, rect); this.scheduleFrame()
},
}
private readonly glance = new GlanceHost({
getDisplay: () => this.glanceDisplay,
getProvider: () => ALL_APPS.find(app => app.glanceboard)?.glanceboard ?? null,
canShow: () => this.runtimeNeeded,
// iOS retains the EvenHub session while the shell sleeps. The board's
// opaque first frame is ready before we unblank the compositor.
ensureSessionActive: async () => {
this.compositor.setScreenBlanked(false); this.scheduleFrame(); return true
},
onHiddenWhileAsleep: () => {
if (!shell.isScreenOn()) this.compositor.setScreenBlanked(true)
this.scheduleFrame()
},
onVisibilityChanged: () => this.scheduleFrame(),
appendLog: message => this.logBluetooth(message),
})
private renderTimer: ReturnType<typeof setTimeout> | null = null
private clockTimer: ReturnType<typeof setInterval> | null = null
private offSettings: (() => void) | null = null
@@ -69,7 +101,10 @@ export class IosPreviewController {
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
onScreenStateChanged: on => {
if (on) this.glance.dismiss()
this.compositor.setScreenBlanked(!on); this.requestShellRender()
},
})
const launcher = createLauncherWindow({
actions: this.actions,
@@ -117,6 +152,8 @@ export class IosPreviewController {
this.runtimeRunning = running
for (const window of shell.getWindows()) window.setScreenOn?.(running && shell.isScreenOn())
if (!running) {
this.glance.dismiss()
this.compositor.setScreenBlanked(!shell.isScreenOn())
this.offSettings?.(); this.offSettings = null
for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer)
UIDevice.currentDevice.batteryMonitoringEnabled = false
@@ -180,7 +217,8 @@ export class IosPreviewController {
this.session?.setFrame(pixels)
if (this.active) {
const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green')
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
this.onFrame(image, this.glance.isVisible() ? 'Glanceboard'
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
}
} catch (error) { this.fail(error) }
}, 33)
@@ -203,12 +241,23 @@ export class IosPreviewController {
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
}
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
await shell.receiveInput(event)
this.requestShellRender()
await this.receiveInput(event)
}
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
}).catch(error => this.fail(error))
}
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
if (!shell.isScreenOn()) {
const glanceEvent = this.glance.eventForGesture(headTilt ? 'head-tilt' : event.type)
if (glanceEvent?.type === 'dismiss') this.glance.dismiss()
else if (glanceEvent) {
await this.glance.handleEvent(glanceEvent, 0)
return
}
}
await shell.receiveInput(event)
this.requestShellRender()
}
private async mirrorTap(nx: number, ny: number): Promise<void> {
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
@@ -317,7 +366,8 @@ export class IosPreviewController {
}, input => {
this.inputQueue = this.inputQueue.then(async () => {
if (this.session?.state.phase !== 'connected') return
await shell.receiveInput(rawInputEventToInputEvent(input)); this.requestShellRender()
await this.receiveInput(rawInputEventToInputEvent(input),
input.kind === 'display-wake' && input.eventType === OsEventTypeList.HEAD_UP_EVENT)
this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`)
}).catch(error => this.fail(error))
}, message => this.logBluetooth(message), () => this.refreshClock())