import { startRemoteInput } from "../remote/service" import type { KeyboardInputSession } from '../ui/shell/keyboard-input' import { acceptInput, resetRingInputFilter } from "../ui/input-monitor"; import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup, readActiveNotifications } from '../native/notification-icons.ios' import { launcherEntries } from '../apps/launcher' import { discardZeroedEvenHubPackage, getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps' import { isInstalledPackagePresent } from '../apps/evenhub/updates' import { openEvenHubStoreForPackage } from '../apps/evenhub' import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager' import { registerSystemTools } from '../assistant/system-tools' import { registerWindowTools } from '../assistant/window-tools' import { registerNavigateTools } from '../assistant/navigate-tools' import { registerRoamTools } from '../assistant/roam-tools' import { IosNavigationSensors } from '../native/ios-navigation-sensors' import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios' import { bindBandwidthBenchmarkSession } from '../native/bandwidth-benchmark.ios' import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core' import { iosVoiceInput } from '../native/ios-voice-input' import { voiceActivity } from '../ui/shell/voice-activity' import { nightscoutBridge } from '../native/nightscout-bridge' import { weatherBridge } from '../native/weather' import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios' import { PreviewDisplayTarget, type DisplayTarget } from '../native/preview-display.ios' import { AncsClient, ANCS_FIRMWARE_VERSION } from './ancs-client' import { G1V0_IMAGE } from '../g1/firmware/g1v0-image' import { GlanceHost, type GlanceDisplay } from './glance-host' import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen' import { OsEventTypeList } from './events' import { loadDeviceAddresses } from './device-addresses' import { updateGlassesPresence } from './glasses-presence' import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat' import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy' import { WearRemote, type WearRemoteInputKind } from './wear-remote' import { deviceAddressError } from './ios-peripheral-identity' import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app' import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app' import { ALL_APPS } from '../apps/all-apps' import type { AppContext, AppDefinition, AppLaunchParams } from '../apps/app-definition' import { WorkerAppHost } from '../ui/shell/worker-window' import { createInProcessWindow, YieldAtRootLayer, type InProcessAppOptions, type InProcessWindow } from '../ui/shell/in-process-window' import { readPhoneBatteryState } from '../native/phone-battery' import { iosAppUnavailableReason } from '../apps/ios-availability' import { flattenPlanesWithDraws, planesFingerprint, type Plane } from '../graphics/plane' import { prepareFrameDraws } from '../graphics/glyph-wire' import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image' import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures' import { noopLayerActions, type ContinuousCaptureOptions, type LayerActions } from '../ui/layers' import { TextViewerLayer } from '../apps/files/text-viewer' import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell' import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, windowEdges, type ScreenRect } from '../ui/shell/geometry' import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' const SHELL_SURFACE_ID = 'shell' // voiceActivity holder for continuous capture (Transcribe), as on Android. const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture' const SHELL_SURFACE_Z_ORDER = 1 const LOCK_SURFACE_Z_ORDER = 1000 /** The shell render loop waits this long for the glasses to take a frame before moving on. */ const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 2000 export type SessionPhase = 'disconnected' | 'connecting' | 'connected' | 'retrying' | 'disconnecting' | 'error' export type SessionState = { phase: SessionPhase; status: string; battery: number | null; charging: boolean | null; ring: boolean; ringBattery: number | null; ringCharging: boolean | null; frames: number; capabilities: string; leftVersion: string; rightVersion: string; /** Why the connected firmware can't run Faceclaw (firmware-compat.ts), or '' when it can or is unknown. */ firmwareWarning: string } function ancsCapable(capabilities: string): boolean { return /^Faceclaw\/(\d+)/.test(capabilities) && Number(capabilities.split('/')[1]) >= ANCS_FIRMWARE_VERSION } /** * iOS host for the shared app registry, shell and BLE session. The session, * compositor and transport live in shared Kotlin (FaceclawCommunicatorBridge * over FaceclawKitIosGlassesSession); with no glasses connected the same * compositor runs headless (PreviewDisplayTarget) so the phone mirror works. */ export class IosPreviewController { private previewTarget: PreviewDisplayTarget | null = null private communicator: FaceclawCommunicatorBridge | null = null private readonly startedCommunicators = new WeakSet() state: SessionState = { phase: 'disconnected', status: 'Preview only', battery: null, charging: null, ring: false, ringBattery: null, ringCharging: null, frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' } private notifications: AncsClient | null = null private readonly sessionOffs: (() => void)[] = [] private stopping: Promise | null = null /** Resolves once the current display target has every surface configured. */ private displayReady: Promise = Promise.resolve() private connecting = false private sessionCreated = false private wearDetectionRequested = false /** The last firmware report was Faceclaw's firmware at the required revision. */ private firmwareCompatible = false private readonly glanceDisplay: GlanceDisplay = { configureSurface: async (id, options) => { await this.display?.configureSurface(id, options) }, setSurfaceVisible: async (id, visible) => { await this.display?.setSurfaceVisible(id, visible); this.schedulePreviewUpdate() }, setSurfaceDepth: async (id, depth) => { await this.display?.setSurfaceDepth(id, depth) }, setScreenBlanked: async blanked => { await this.display?.setScreenBlanked(blanked); this.schedulePreviewUpdate() }, submitSurfaceFrame: async (id, pixels, rect, fingerprint, paintMs, frameId, draws) => { await this.display?.submitSurfaceFrame(id, pixels, rect, fingerprint, paintMs, frameId, draws); this.schedulePreviewUpdate() }, } private readonly glance = new GlanceHost({ getDisplay: () => this.display ? this.glanceDisplay : null, getProvider: () => ALL_APPS.find(app => app.glanceboard)?.glanceboard ?? null, canShow: () => this.runtimeNeeded && !this.glassesLocked, // iOS retains the EvenHub session while the shell sleeps. The board's // opaque first frame is ready before we unblank the compositor. ensureSessionActive: async () => { await this.display?.setScreenBlanked(false); this.schedulePreviewUpdate(); return true }, onHiddenWhileAsleep: () => { if (!shell.isScreenOn()) void this.display?.setScreenBlanked(true).then(() => this.schedulePreviewUpdate()) else this.schedulePreviewUpdate() }, onVisibilityChanged: () => this.schedulePreviewUpdate(), appendLog: message => this.logBluetooth(message), }) private previewTimer: ReturnType | null = null /** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */ private previewCrop: ScreenRect | null = null private clockTimer: ReturnType | null = null private offSettings: (() => void) | null = null private lastBrightness: string | null = null private active = false private runtimeRunning = false private clockMinute = -1 private shellRenderInProgress = false private shellRenderQueued = false /** The shell woke; the next shell render unblanks the compositor after submitting. */ private unblankAfterShellRender = false private nextFrameId = 1 private nextWorkerFrame = 0 private inputQueue: Promise = Promise.resolve() private lastLayout = '' /** The phone-UI button last pushed to the page (onEvenHubPhoneUiChanged). */ private evenHubPhoneUi: PhoneUiButton | null = null private readonly appHosts = new Map() private readonly inProcessApps = new Map() private readonly batteryObservers: any[] = [] private readonly lockObservers: any[] = [] private phoneLocked = false private glassesWorn: boolean | null = null private glassesLocked = false private lockEnabled = lockScreenEnabledSetting.get() private acknowledgedFrames = 0 private logLines: string[] = [] private logTimer: ReturnType | null = null // Windows that want continuous capture, and what each needs of it (a // recording, a provider and language). They share one capture, which runs // while any of them wants it. private readonly continuousClients = new Map() // transcriptionKey of the running continuous capture (provider and // language); one that no longer matches what its windows need is drained // and restarted. private continuousTranscription = '' // Continuous capture should (re)start once audio is available: a window // asked, the glasses connected or disconnected, or the glasses lock or the // app lifecycle cut it short. A capture that fails doesn't set it, so a // broken start is retried on the next such change rather than in a loop. private continuousPending = false private continuousCaptureToken = 0 private sessionConnected = false /** The Apple Watch remote; constructed last so it sees a fully wired shell. */ private wearRemote: WearRemote | null = null private readonly actions: LayerActions = { ...noopLayerActions, playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload), requestRender: () => this.requestShellRender(), disconnect: () => this.disconnect(), startVoiceCapture: endpointing => this.startVoiceCapture(endpointing), stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(), startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options), stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client), startTextSettingEdit: async setting => { await this.editSetting(setting) }, startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => { for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return } if (toggle) { const choice = await Dialogs.action({ title, cancelButtonText: 'Cancel', actions: [toggle.label, 'This session only'] }) if (choice === 'Cancel') { cancelled?.(); return } toggle.setting.set(choice === toggle.label) } finished?.() }, } constructor(private readonly onFrame: (image: ImageSource, focus: string) => void, private readonly onError: (message: string) => void, private readonly onConnectionState: (state: SessionState) => void = () => {}, private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {}, private readonly onEvenHubPhoneUiChanged: (button: PhoneUiButton | null) => void = () => {}) { this.ensurePreviewDisplay() shell.configure({ actions: this.actions, voiceInputEnabled: true, onKeyboardInputChanged: session => this.onKeyboardInputChanged(session ? { targets: session.targets, setText: text => { if (this.active && !this.glassesLocked) session.setText(text) }, send: () => { if (this.active && !this.glassesLocked) session.send() }, sendTo: id => { if (this.active && !this.glassesLocked) session.sendTo(id) }, discard: () => session.discard(), get visibility() { return session.visibility }, setVisibility: visibility => { if (this.active && !this.glassesLocked) session.setVisibility(visibility) }, get history() { return session.history }, } : null), prepareVoiceCapture: () => this.prepareVoiceCapture(), getScreenTimeoutMs: () => null, requestShellRender: () => this.requestShellRender(), onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi(); this.wearRemote?.schedulePublish() }, openNotificationsWindow: () => this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, options => createNotificationsAppWindow(options, section => { void this.launchApp('settings', { section }) })), onSwitcherBarHeightChanged: () => { this.relayout() shell.foregroundWindow()?.requestRender() this.requestShellRender() }, onScreenStateChanged: on => { if (on) this.glance.dismiss() // A wake unblanks only once the next shell render has staged an awake // scene: unblanking composites the retained one, painted while asleep // with no chrome and depth 0, and that first frame is the one the // brightness fade-in starts on. this.unblankAfterShellRender = on if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error)) this.requestShellRender() this.wearRemote?.schedulePublish() }, }) const launcher = createLauncherWindow({ actions: this.actions, apps: () => launcherEntries(ALL_APPS), launchApp: id => this.launchApp(id), uninstallApp: id => this.uninstallApp(id), submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes), setSurfaceVisible: visible => this.setWindowSurfaceVisible(LAUNCHER_SURFACE_ID, visible), }) void this.configureWindow(launcher) shell.registerWindow(launcher) shell.wake('window') shell.focusWindow(launcher.windowId) onIosNotificationPopup(key => { const notification = readActiveNotifications(128).find(n => n.key === key) if (this.glassesLocked || !notification || !shouldPopUpNewNotification(notification.packageName)) return const woke = !shell.isScreenOn() && shell.wake('sidebar') shell.openNotificationModal(key, woke) this.requestShellRender() }) startRemoteInput({ ready: () => this.runtimeRunning, locked: () => this.glassesLocked, input: (gesture, source) => { // Share the controller queue with physical input. A discrete hold includes its release. const task = this.inputQueue.then(async () => { await this.receiveInput(makeInputEvent({ type: gesture, source } as InputEventPayload)) if (gesture === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source })) }) this.inputQueue = task.catch(error => this.fail(error)) return task }, acceptsText: () => !!shell.foregroundWindow()?.receiveTextInput, text: (text, submit) => { if (!shell.isScreenOn()) shell.wake('window'); shell.sendTextToForegroundWindow(text, { submit }); this.requestShellRender() }, assistantAvailable: () => shell.isAssistantAvailable(), assistant: text => shell.sendToAssistant(text), }) // The header's EvenHub phone-UI button hides while that UI is showing. onPhoneShownChanged(() => this.syncEvenHubPhoneUi()) registerSystemTools() registerWindowTools({ apps: () => ALL_APPS.filter(app => app.isSupported?.() !== false && !iosAppUnavailableReason(app.appId)), launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() }) registerNavigateTools(id => this.launchApp(id)) registerRoamTools(id => this.launchApp(id)) // Every capture end (drained, failed, replaced) may leave continuous capture to restart. iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture()) for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app)) this.wearRemote = new WearRemote({ apps: () => ALL_APPS.filter(app => app.showInLauncher !== false && app.isSupported?.() !== false && !iosAppUnavailableReason(app.appId)), injectInput: kind => this.injectWatchInput(kind), launchApp: id => this.launchApp(id), // The watch's Connect/Disconnect are explicit user choices, like the phone's own buttons. connect: () => { resumeAutoReconnect(); return this.connect(true) }, disconnect: () => { suppressAutoReconnect(); return this.disconnect() }, setGlassesLocked: locked => this.setGlassesLocked(locked), requestShellRender: () => this.requestShellRender(), previewInputActive: () => !this.communicator && this.runtimeRunning, getState: () => ({ phase: this.state.phase, status: this.state.status, glassesLocked: this.glassesLocked, glassesWorn: this.glassesWorn, silentMode: false, lastHeadsetBattery: this.state.battery }), appendLog: line => this.logBluetooth(line), }) } /** The compositor frames go to: the live session, else the headless preview. */ private get display(): DisplayTarget | null { return this.communicator ?? this.previewTarget } private ensurePreviewDisplay(): void { if (this.communicator || this.previewTarget) return const target = new PreviewDisplayTarget() this.previewTarget = target target.activate(() => this.schedulePreviewUpdate()) this.displayReady = this.registerSurfaces(target).catch(error => this.fail(error)) } private teardownPreviewDisplay(): void { const target = this.previewTarget if (!target) return this.previewTarget = null this.glance.reset() target.release() } /** (Re)create every compositor surface on a fresh display target. */ private async registerSurfaces(target: DisplayTarget): Promise { await target.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT) await target.configureSurface(SHELL_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: SHELL_SURFACE_Z_ORDER, transparency: 'color-key' }) await target.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1) await target.configureSurface(LOCK_SCREEN_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: LOCK_SURFACE_Z_ORDER, transparency: 'opaque', }) if (this.glassesLocked) await this.submitLockImage(target) await target.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, this.glassesLocked) await target.setScreenBlanked(!shell.isScreenOn()) this.lastLayout = '' for (const window of shell.getWindows()) await this.configureWindow(window, target) } private async submitLockImage(target: DisplayTarget): Promise { const image = createLockScreenImage() await target.submitSurfaceFrame(LOCK_SCREEN_SURFACE_ID, image.to8bppBuffer(), { x: 0, y: 0, width: image.width, height: image.height }, 'lock-screen', 0, 0) } resume(): void { if (this.active) return this.active = true this.syncRuntime() this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable) this.logBluetooth('Phone foreground') if (this.communicator) this.onConnectionState({ ...this.state }) this.relayout() shell.foregroundWindow()?.requestRender() this.requestShellRender() this.syncEvenHubPhoneUi() this.syncContinuousCapture() } pause(): void { if (!this.active) return this.active = false if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true iosVoiceInput.stopPhoneCapture() // The phone preview is hidden; the glasses still own their screen and input. this.syncRuntime() this.logBluetooth(`Phone background; glasses ${this.state.phase}`) this.flushBluetoothLog() } /** Overlay an EvenHub app's phone UI (the header's app-icon button). */ showEvenHubPhoneUi(windowId: string): void { showOnPhone(windowId) } /** * Tell the page when the foreground window's EvenHub phone-UI button * changes. A paused controller skips this and catches up on resume. */ private syncEvenHubPhoneUi(): void { if (!this.active) return const button = phoneUiButton() const current = this.evenHubPhoneUi if (current?.windowId === button?.windowId && current?.icon === button?.icon) return this.evenHubPhoneUi = button this.onEvenHubPhoneUiChanged(button) } private get runtimeNeeded(): boolean { return this.active || ['connected', 'connecting', 'retrying', 'disconnecting'].includes(this.state.phase) } private syncRuntime(): void { const running = this.runtimeNeeded if (running === this.runtimeRunning) return this.runtimeRunning = running for (const window of shell.getWindows()) window.setScreenOn?.(running && shell.isScreenOn()) if (!running) { void nightscoutBridge.stop().catch(error => this.fail(error)) weatherBridge.setSessionActive(false) this.glance.dismiss() void this.display?.setScreenBlanked(!shell.isScreenOn()).catch(error => this.fail(error)) this.offSettings?.(); this.offSettings = null for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer) for (const observer of this.lockObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer) UIDevice.currentDevice.batteryMonitoringEnabled = false if (this.clockTimer !== null) clearInterval(this.clockTimer) if (this.previewTimer !== null) clearTimeout(this.previewTimer) this.clockTimer = this.previewTimer = null return } UIDevice.currentDevice.batteryMonitoringEnabled = true // Keep these observers alive while BLE owns the runtime, including background. for (const [name, locked] of [ [UIApplicationProtectedDataWillBecomeUnavailable, true], [UIApplicationProtectedDataDidBecomeAvailable, false], ] as const) { this.lockObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock( name, null, NSOperationQueue.mainQueue, () => this.handlePhoneLockState(locked))) } this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable) this.syncLockSetting() void nightscoutBridge.start().catch(error => this.fail(error)) weatherBridge.setSessionActive(true) for (const name of [UIDeviceBatteryLevelDidChangeNotification, UIDeviceBatteryStateDidChangeNotification]) { this.batteryObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(name, null, NSOperationQueue.mainQueue, () => this.requestShellRender())) } const phone = readPhoneBatteryState() this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.offSettings = onAnySettingChanged(() => { this.syncLockSetting() this.pushBrightness() this.relayout() shell.foregroundWindow()?.requestRender() this.requestShellRender() }) this.clockTimer = setInterval(() => this.refreshClock(), 60_000) } private pushBrightness(): void { const communicator = this.communicator if (!communicator || this.state.phase !== 'connected') { this.lastBrightness = null; return } const preferences = getBrightnessPreferences() const value = JSON.stringify(preferences) if (value === this.lastBrightness) return this.lastBrightness = value void communicator.configureBrightness(preferences).catch(error => { this.lastBrightness = null this.logBluetooth(`Brightness: ${String(error)}`) }) } private syncLockSetting(): void { const enabled = lockScreenEnabledSetting.get() if (enabled === this.lockEnabled) return this.lockEnabled = enabled if (!enabled) this.setGlassesLocked(false) else { if (this.phoneLocked && this.glassesWorn === false) this.setGlassesLocked(true) if (this.communicator && this.state.phase === 'connected' && this.firmwareCompatible) void this.communicator.enableWearDetectionAndRequestState().catch(error => this.fail(error)) } } private handlePhoneLockState(locked: boolean): void { this.phoneLocked = locked this.communicator?.onPhoneLockSignal() if (!locked) this.setGlassesLocked(false) else if (this.lockEnabled && this.glassesWorn === false) this.setGlassesLocked(true) } private handleWearState(wearing: boolean): void { // Sample false to catch a notification missed during suspension. Do not // sample true here: WillBecomeUnavailable precedes the property transition. if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true) // Putting the glasses on (an observed OFF_HEAD -> ON_HEAD transition, not // a session's first snapshot) wakes the screen, to the lock notice if locked. const putOn = wearing && this.glassesWorn === false this.glassesWorn = wearing this.wearRemote?.schedulePublish() this.syncPresence() this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`) if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true) if (putOn && this.runtimeRunning && !shell.isScreenOn()) shell.wake('sidebar') } private setGlassesLocked(locked: boolean): void { if (locked === this.glassesLocked) return this.glassesLocked = locked this.wearRemote?.schedulePublish() if (locked) { this.glance.dismiss() if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.', 'Glasses locked. Transcription resumes when you unlock your phone.') } else this.syncContinuousCapture() this.syncRawMicSession() void (async () => { await this.displayReady const display = this.display if (!display) return { if (locked) await this.submitLockImage(display) await display.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, locked) this.schedulePreviewUpdate() } })().catch(error => this.fail(error)) this.logBluetooth(`Glasses ${locked ? 'locked' : 'unlocked'}`) this.requestShellRender() } private refreshClock(): void { const minute = Math.floor(Date.now() / 60_000) if (minute !== this.clockMinute) { this.clockMinute = minute; this.requestShellRender() } } private async configureWindow(window: ShellWindow, target: DisplayTarget | null = this.display): Promise { if (!target) return await target.configureSurface(window.surfaceId, { ...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque', }) await target.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId) } private relayout(): void { // Each window's rect, and what its edges touch (which sets its padding). const layout = shell.getWindows().map(window => JSON.stringify([appViewportRect(window.heightMode, window.appId), windowEdges(window.appId)])).join(";") if (layout === this.lastLayout) return this.lastLayout = layout for (const window of shell.getWindows()) { void this.configureWindow(window).catch(error => this.fail(error)) window.relayout?.() } } /** Submit a painted frame for an in-process window (e.g. the launcher). */ private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise { await this.displayReady const display = this.display // The window can close (removing its surface) while this frame waits. if (!display || !live()) return const fingerprint = planesFingerprint(planes) const { image, draws } = flattenPlanesWithDraws(planes) await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, fingerprint, -1, 0, prepareFrameDraws(draws)) this.schedulePreviewUpdate() } /** Worker windows ship baked pixels; every submit is new content to the compositor. */ private async submitWorkerPixels(id: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null): Promise { await this.displayReady const display = this.display if (!display) return await display.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }, `${id}#${++this.nextWorkerFrame}`, -1, 0, draws) this.schedulePreviewUpdate() } private setWindowSurfaceVisible(id: string, visible: boolean): void { void this.display?.setSurfaceVisible(id, visible).then(() => this.schedulePreviewUpdate()).catch(error => this.fail(error)) } private removeWindowSurface(id: string): void { void this.display?.removeSurface(id).then(() => this.schedulePreviewUpdate()).catch(error => this.fail(error)) } /** Re-render and resubmit the shell surface; one render in flight, at most one queued. */ private requestShellRender(): void { if (this.shellRenderInProgress) { this.shellRenderQueued = true; return } this.shellRenderInProgress = true void (async () => { try { do { this.shellRenderQueued = false await this.renderShell() } while (this.shellRenderQueued) } catch (error) { this.fail(error) } finally { this.shellRenderInProgress = false } })() } private async renderShell(): Promise { await this.displayReady const display = this.display if (!display) return // Taken with the paint, so a render that painted before the wake can't unblank. const unblank = this.unblankAfterShellRender this.unblankAfterShellRender = false const frameId = this.allocateFrameId() try { const startedAt = Date.now() const bytes = shell.paintScene() await display.submitShellScene(bytes, Date.now() - startedAt, frameId) } finally { // Even if the render failed, so the screen doesn't stay dark; not if it // went back to sleep meanwhile (that sleep re-blanked). if (unblank && shell.isScreenOn()) await display.setScreenBlanked(false) } // Backpressure: the next shell render waits for this one to reach the // glasses (the preview target resolves immediately; nothing transmits). await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS) this.schedulePreviewUpdate() } private allocateFrameId(): number { const id = this.nextFrameId this.nextFrameId = this.nextFrameId >= 0x7fffffff ? 1 : this.nextFrameId + 1 return id } /** Refresh the phone mirror from the composited screen, coalesced at 30 fps. */ private schedulePreviewUpdate(): void { if (!this.active || this.previewTimer !== null) return this.previewTimer = setTimeout(() => { this.previewTimer = null if (!this.active) return try { const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect()) const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop) if (!image) return this.previewCrop = crop this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard' : `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) } catch (error) { this.fail(error) } }, 33) } private fail(error: unknown): void { const message = error instanceof Error ? error.stack ?? error.message : String(error) console.error(`[ios-preview] ${message}`) this.logBluetooth(`App error: ${message}`) if (this.active) this.onError(error instanceof Error ? error.message : String(error)) } gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void { this.inputQueue = this.inputQueue.then(async () => { if (!this.active) return // A mirror tap may land on a hit target rather than being a gesture. if (gesture === 'press' && origin === 'mirror') return if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn() && !this.glassesLocked) { await this.mirrorTap(nx, ny) } else { let type: string = ({ press: 'ring-press', tap: 'click', 'double-tap': 'double-click' } as Record)[gesture] ?? gesture if (origin === 'ring' && type.startsWith('swipe-')) { if (type === 'swipe-left' || type === 'swipe-right') return type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down' } const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload) await this.receiveInput(event) } console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`) }).catch(error => this.fail(error)) } /** Input from the Apple Watch: shares the controller queue with physical input. */ private injectWatchInput(kind: WearRemoteInputKind): Promise { const task = this.inputQueue.then(async () => { this.onActivity() // As on Android, and because the watch may be acting on a stale state snapshot: a dark // display takes a double-click (wake) or a tap/hold (Glanceboard), and scrolls mean nothing. if (!shell.isScreenOn() && (kind === 'scroll-up' || kind === 'scroll-down' || kind === 'short-then-long-press')) return // A real hold releases when the finger lifts; a discrete long-press releases at once. const type = kind === 'long-press-start' ? 'long-press' : kind await this.receiveInput(makeInputEvent({ type, source: 'watch' } as InputEventPayload)) if (kind === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: 'watch' })) console.log(`[ios-preview] apple-watch ${kind}: ${shell.describeInputTarget()}`) }) this.inputQueue = task.catch(error => this.fail(error)) return task } private async receiveInput(event: InputEvent, headTilt = false): Promise { // Input can change the foreground app or the display state, and the shell doesn't announce those. this.wearRemote?.schedulePublish() if (!acceptInput(event)) return if (this.glassesLocked) { // Preserve the locked display's sleep/wake controls without dispatching // gestures to apps, shell menus, voice input or the Glanceboard. if (event.type === 'double-click' && (event.source === 'ring' || event.source === 'watch')) { if (shell.isScreenOn()) shell.sleep() else shell.wake('sidebar') } else if (event.type === 'display-wake' && !shell.isScreenOn()) shell.wake('sidebar') this.requestShellRender() return } if (!shell.isScreenOn()) { const glanceEvent = this.glance.eventForGesture(headTilt ? 'head-tilt' : event.type) if (glanceEvent?.type === 'dismiss') this.glance.dismiss() else if (glanceEvent) { await this.glance.handleEvent(glanceEvent, 0) return } } await shell.receiveInput(event) this.requestShellRender() } private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise { const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy) const x = Math.max(0, Math.min(639, Math.floor(nx * 640))) const y = Math.max(0, Math.min(479, Math.floor(ny * 480))) if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) { this.requestShellRender(); return } const window = shell.foregroundWindow() if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && isOnSwitcherEdge(x, y, window?.heightMode ?? 'min', window?.appId)) { const target = shell.windowAtSidebarPoint(x, y) if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() } return } if (window && !shell.hasOverlay()) { const rect = appViewportRect(window.heightMode, window.appId) if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return shell.focusWindow(window.windowId) if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return } } await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' })) this.requestShellRender() } async launchApp(id: string, params?: AppLaunchParams): Promise { const app = ALL_APPS.find(candidate => candidate.appId === id) if (!app) { const installed = getInstalledEvenHubAppById(id) const host = ALL_APPS.find(candidate => candidate.appId === 'evenhub') if (installed && host) { try { if (isInstalledPackagePresent(installed.packageId)) { try { await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed); return } catch (error) { // Checked only after a failure: it reads the whole package again. if (!discardZeroedEvenHubPackage(installed.packageId)) throw error console.log(`[ios-preview] Discarded zeroed package for ${installed.packageId}; opening store page`) } } await openEvenHubStoreForPackage(this.buildAppContext(host), installed) } catch (error) { this.fail(error) } } return } try { const reason = iosAppUnavailableReason(id) if (reason) { await this.launchInProcessApp(id, `window:${id}`, options => createInProcessWindow({ ...options, appId: id, windowId: id, title: app.title, icon: app.icon, iconLetter: app.title[0], closeable: true, baseLayer: new YieldAtRootLayer(new TextViewerLayer(reason, app.title)), })) } else await app.launch(this.buildAppContext(app), params) console.log(`[ios-preview] Launched ${id}`) } catch (error) { this.fail(error) } } private async uninstallApp(id: string): Promise { const packageId = installedEvenHubPackageId(id) if (!packageId) return closeRunningPackage(packageId) uninstallEvenHubPackage(packageId) this.requestShellRender() } private buildAppContext(app: AppDefinition): AppContext { return { appId: app.appId, apps: ALL_APPS, actions: this.actions, launchApp: (id, params) => this.launchApp(id, params), uninstallApp: id => this.uninstallApp(id), launchInProcessApp: (id, surface, create) => this.launchInProcessApp(id, surface, create), openInProcessAppInBackground: (id, surface, create) => this.openInProcessAppInBackground(id, surface, create), ensureWorkerHost: create => this.ensureWorkerHost(app.appId, create), submitWindowFrame: (id, planes) => this.submit(id, planes), setWindowSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible), requestShellRender: () => this.requestShellRender(), appendLog: message => console.log(`[ios-app] ${message}`), setTextEditorHost: () => {}, } } private async launchInProcessApp(windowId: string, surfaceId: string, create: (options: InProcessAppOptions) => InProcessWindow): Promise { const existing = this.inProcessApps.get(windowId) if (existing) { shell.focusWindow(windowId); existing.requestRender(); this.requestShellRender(); return } const { app, surface } = this.createInProcessApp(windowId, surfaceId, create) await this.configureWindow(app.window) surface.ready = true shell.registerWindow(app.window); shell.focusWindow(windowId) app.requestRender(); this.requestShellRender() } /** * An in-process singleton app's window, opened in the background (neither * foregrounded nor focused) unless it is open already, for the shell to * bring forward itself: the Notifications window under a notification * selected in the switcher, or an app widget's window under the widget. * Display calls run in order and frames wait on displayReady before * queueing, so the surface configure queued here lands ahead of any frame * and of the shell foregrounding the window, which it may do at once. */ private openInProcessAppInBackground(windowId: string, surfaceId: string, create: (options: InProcessAppOptions) => InProcessWindow): { window: InProcessWindow; opened: boolean } { const existing = this.inProcessApps.get(windowId) if (existing) return { window: existing, opened: false } const { app, surface } = this.createInProcessApp(windowId, surfaceId, create) void this.configureWindow(app.window).catch(error => this.fail(error)) surface.ready = true shell.registerWindow(app.window) return { window: app, opened: true } } /** * Create an in-process singleton app's window (not yet registered with the * shell). Its frames reach the compositor only while `surface.ready`: apps * can request a render while their factory runs (Nightscout's tray * subscription does), before the caller has configured the surface. */ private createInProcessApp(windowId: string, surfaceId: string, create: (options: InProcessAppOptions) => InProcessWindow): { app: InProcessWindow; surface: { ready: boolean } } { const surface = { ready: false } const app = create({ actions: this.actions, submitFrame: planes => surface.ready ? this.submit(surfaceId, planes, () => surface.ready) : Promise.resolve(), setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible), removeSurface: () => { surface.ready = false; this.removeWindowSurface(surfaceId) }, reconfigureSurface: () => { const window = shell.getWindows().find(w => w.windowId === windowId) if (window) void this.configureWindow(window).catch(error => this.fail(error)) this.requestShellRender() }, onClosed: () => { this.inProcessApps.delete(windowId) }, }) this.inProcessApps.set(windowId, app) return { app, surface } } private ensureWorkerHost(appId: string, create: () => Worker): WorkerAppHost { const existing = this.appHosts.get(appId) if (existing) return existing const worker = create() const navigationSensors = appId === 'navigate' ? new IosNavigationSensors(event => worker.postMessage({ type: 'navigation-sensors', event })) : undefined const host = new WorkerAppHost({ appId, worker, navigationSensors, onStopping: () => { if (this.appHosts.get(appId) === host) this.appHosts.delete(appId) }, playBuzzerSequence: payload => this.actions.playBuzzerSequence(payload), openUrl: url => { void Utils.openUrl(url) }, configureSurface: async (id, visible, heightMode) => { const display = this.display if (!display) return await display.configureSurface(id, { ...appViewportRect(heightMode, appId), zOrder: 0, transparency: 'opaque' }) await display.setSurfaceVisible(id, typeof visible === 'function' ? visible() : visible) }, setSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible), removeSurface: id => this.removeWindowSurface(id), submitPixels: (id, pixels, width, height, draws) => { void this.submitWorkerPixels(id, pixels, width, height, draws).catch(error => this.fail(error)) }, requestShellRender: () => this.requestShellRender(), openSettings: section => { void this.launchApp('settings', { section }) }, startTextSettingEdit: id => { const setting = getStringSettingById(id); if (setting) void this.editSetting(setting) }, endTextSettingEdit: () => {}, startTextInput: () => shell.startVoiceInput(), }) this.appHosts.set(appId, host) return host } get connectionState(): SessionState | null { return this.sessionCreated ? this.state : null } get connectionDetails(): string { const state = this.connectionState return state ? `${state.status}\nL: ${state.leftVersion || 'unknown'}\nR: ${state.rightVersion || 'unknown'}\nBattery: ${state.battery ?? '?'}%\nRing: ${state.ring ? 'connected' : 'disconnected'}\nFrames acknowledged: ${state.frames}\n${state.capabilities}\n\n${this.logLines.slice(-12).join('\n')}` : 'Preview only. Configure devices, then connect.' } private update(phase: SessionPhase, status: string): void { this.state = { ...this.state, phase, status } this.logBluetooth(status) this.emitState() } /** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */ private syncPresence(): void { updateGlassesPresence({ connected: this.state.phase === 'connected', worn: this.glassesWorn, charging: this.state.charging ?? false }) } private emitState(): void { if (this.state.phase !== 'connected') resetRingInputFilter() this.pushBrightness() this.maybeLaunchOnboarding(this.state) if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null // Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session. this.syncRawMicSession() if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() this.syncContinuousSession() this.pushBatteryLevels() this.syncPresence() this.syncRuntime() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() this.wearRemote?.schedulePublish() } /** The ring level arrives through the glasses, so like Android it is shown only while connected. */ private pushBatteryLevels(): void { const connected = this.state.phase === 'connected' shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging, ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null }) } /** `fromBackground`: the watch asked, and the phone may be in a pocket with the app suspended. */ async connect(fromBackground = false): Promise { if ((!this.active && !fromBackground) || this.connecting || this.stopping) return if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return const addresses = loadDeviceAddresses(), error = deviceAddressError(addresses) if (error) { if (this.active) this.onError(error) else this.logBluetooth(error) return } this.connecting = true this.sessionCreated = true try { this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false, ringBattery: null, ringCharging: null } this.firmwareCompatible = false this.update('connecting', 'Finding configured devices…') let identifiers // As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect // fails and retries every few seconds, starving the glasses link. const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : '' try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) } catch (failure) { this.update('error', failure instanceof Error ? failure.message : String(failure)) return } if (this.communicator) await this.closeCommunicator() const communicator = new FaceclawCommunicatorBridge({ ...identifiers, glassesType: glassesTypeSetting.get() }) this.teardownPreviewDisplay() this.communicator = communicator this.wearDetectionRequested = false this.bindCommunicator(communicator) this.displayReady = this.registerSurfaces(communicator) await this.displayReady this.update('connecting', 'Connecting to the glasses…') await communicator.configureBrightness(getBrightnessPreferences()) this.startedCommunicators.add(communicator) await communicator.start() for (const window of shell.getWindows()) window.requestRender() this.requestShellRender() } catch (failure) { this.fail(failure) await this.closeCommunicator().catch(() => {}) this.ensurePreviewDisplay() this.update('error', failure instanceof Error ? failure.message : String(failure)) } finally { this.connecting = false } } private bindCommunicator(communicator: FaceclawCommunicatorBridge): void { const notifications = new AncsClient(packet => communicator.writeRawToRight(packet), (key, popup) => { iosNotificationsChanged(key, popup); this.requestShellRender() }, message => this.logBluetooth('ANCS: ' + message)) this.notifications = notifications bindIosNotifications(notifications) bindCompassSession(communicator) bindBandwidthBenchmarkSession(communicator) communicator.setRequiresAncs(true) this.sessionOffs.push( communicator.onStateChange(state => this.handleSessionState(communicator, state)), communicator.onRingEvent(input => this.handleRingEvent(input)), communicator.onBatteryState(state => { if (this.communicator !== communicator) return // Unlike the headset level, an unavailable ring reading (-1: ring away or old // firmware) clears the indicator, as on Android. const ringBattery = state.ringBattery ?? -1, hasRingBattery = Number.isInteger(ringBattery) && ringBattery >= 0 && ringBattery <= 100 this.state = { ...this.state, battery: state.battery >= 0 && state.battery <= 100 ? state.battery : this.state.battery, charging: state.chargingStatus >= 0 ? state.chargingStatus > 0 : this.state.charging, ring: hasRingBattery || this.state.ring, ringBattery: hasRingBattery ? ringBattery : null, ringCharging: hasRingBattery ? state.ringChargingStatus === 1 : null } this.pushBatteryLevels() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() this.wearRemote?.schedulePublish() }), communicator.onCaseState(state => { if (this.communicator !== communicator) return // As on Android: keep the last level when a reading has none (out of the case); // the case's own charging is only known while the glasses are docked. shell.setBatteryLevels({ ...(state.battery !== null ? { case: state.battery } : {}), caseCharging: state.docked ? state.charging : null }) this.wearRemote?.schedulePublish() }), communicator.onFirmwareInfo(info => { if (this.communicator !== communicator) return this.logBluetooth(`Firmware: L=${info.leftVersion || '?'} R=${info.rightVersion || '?'}` + (info.extension ? ` ext="${info.extension}"` : ' (no firmware extension string)')) // The shared session halts on incompatible firmware by itself ("incompatible-firmware" // phase, handled in handleSessionState); this only records why, for the warning. const firmwareWarning = firmwareIncompatibilityMessage(info) ?? '' if (firmwareWarning && firmwareWarning !== this.state.firmwareWarning) this.logBluetooth(`Firmware compatibility warning: ${firmwareWarning}`) this.firmwareCompatible = !firmwareWarning && hasCompatibleFirmware(info) this.state = { ...this.state, leftVersion: info.leftVersion || this.state.leftVersion, rightVersion: info.rightVersion || this.state.rightVersion, capabilities: info.extension || this.state.capabilities, firmwareWarning } if (this.active) this.onConnectionState({ ...this.state }) this.syncNotifications(communicator) this.ensureWearDetection(communicator) }), communicator.onFrameMetrics(() => { if (this.communicator !== communicator) return this.state = { ...this.state, frames: this.state.frames + 1 } this.onActivity() this.maybeLaunchOnboarding(this.state) if (this.active) this.onConnectionState({ ...this.state }) this.schedulePreviewUpdate() }), communicator.onPreviewAnimationFrame(() => { if (this.communicator === communicator) this.schedulePreviewUpdate() }), communicator.onWearState(wearing => { if (this.communicator === communicator) this.handleWearState(wearing) }), communicator.addCompassListener(receiveCompassEvent), communicator.onAncsRelayFrame(frame => { if (this.communicator === communicator) notifications.receive(frame) }), communicator.onAncsAuthorization(authorized => { if (this.communicator !== communicator) return if (authorized) { this.syncNotifications(communicator); return } const active = notifications.state !== 'disconnected' const command = notifications.stopCommand() notifications.stop('Enable Share System Notifications in iPhone Settings → Bluetooth → right lens.') if (active && this.state.phase === 'connected') void communicator.writeRawToRight(command).catch(() => {}) }), ) } private handleSessionState(communicator: FaceclawCommunicatorBridge, state: CommunicatorState): void { if (this.communicator !== communicator) return // Registering the listener makes the Kotlin session report its current state, which is // "disconnected" before start(); that must not be mistaken for the session ending. if (!this.startedCommunicators.has(communicator)) return // The shared session parked its retry loop (an arm unpaired, or firmware Faceclaw can't // run): a redial would fail the same way, so stay down until the user connects explicitly // (or re-pairs / installs the custom firmware, which lift the suppression). if (isHaltedSessionPhase(state.phase)) suppressAutoReconnect() const phase: SessionPhase = state.phase === 'charging' ? 'connected' : isHaltedSessionPhase(state.phase) ? 'error' : state.phase // The relay ends with the link, so a reconnected session has to ask for it again (syncNotifications). if (phase !== 'connected' && this.notifications && this.notifications.state !== 'disconnected') this.notifications.stop() // Wear state is a per-transport snapshot: ask again on every new session. if (phase !== 'connected') this.wearDetectionRequested = false this.update(phase, state.status) if (phase === 'connected') { this.syncNotifications(communicator) this.ensureWearDetection(communicator) } else if ((phase === 'disconnected' || phase === 'error') && !this.stopping) { // The session ended on its own (an arm unpaired, incompatible firmware): release it and go headless. void this.disconnect(state.status) } } /** * Wear reports come from the custom firmware, so like Android only ask once the session * is up and the firmware is confirmed compatible (stock firmware never hears the query). */ private ensureWearDetection(communicator: FaceclawCommunicatorBridge): void { if (this.wearDetectionRequested || !this.firmwareCompatible || this.state.phase !== 'connected') return this.wearDetectionRequested = true void communicator.enableWearDetectionAndRequestState().catch(error => this.logBluetooth(`Wear detection: ${String(error)}`)) } private handleRingEvent(input: RawInputEvent): void { this.inputQueue = this.inputQueue.then(async () => { if (this.state.phase !== 'connected') return this.onActivity() if (input.ringInput) this.state = { ...this.state, ring: true } await this.receiveInput(rawInputEventToInputEvent(input), input.kind === 'display-wake' && input.eventType === OsEventTypeList.HEAD_UP_EVENT) this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`) }).catch(error => this.fail(error)) } /** Any glasses traffic: CoreBluetooth can wake a suspended process, so refresh what timers would have. */ private onActivity(): void { if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true) this.refreshClock() } /** Start iPhone notification relay once the firmware is known to support it and ANCS is authorized. */ private syncNotifications(communicator: FaceclawCommunicatorBridge): void { const notifications = this.notifications if (!notifications || this.state.phase !== 'connected') return if (!ancsCapable(this.state.capabilities)) { if (this.state.capabilities) notifications.stop(`Update glasses to Faceclaw firmware ${ANCS_FIRMWARE_VERSION} or newer for iPhone notifications.`) return } // A G1 reports no revision that says so: its session knows from what the right arm answered. if (!communicator.ancsRelayAvailable()) { notifications.stop(`Update the glasses to Faceclaw's firmware ${G1V0_IMAGE.version} for iPhone notifications, then reconnect.`) return } if (notifications.state !== 'disconnected') return notifications.start((Date.now() ^ Math.floor(Math.random() * 0xffffffff)) >>> 0, communicator.rightWriteLimit()) } async disconnect(reason?: string): Promise { if (this.stopping) return this.stopping const communicator = this.communicator if (!communicator) return // Marked before any bridge call: the session may report "disconnected" from inside disconnect(). let finished!: () => void this.stopping = new Promise(resolve => { finished = resolve }) try { const notifications = this.notifications if (notifications && notifications.state !== 'disconnected') { const command = notifications.stopCommand() notifications.stop() if (this.state.phase === 'connected') { try { await communicator.writeRawToRight(command) } catch {} } } if (this.state.phase === 'connected' || this.state.phase === 'connecting' || this.state.phase === 'retrying') this.update('disconnecting', 'Disconnecting…') await communicator.disconnect().catch(error => this.logBluetooth(`Disconnect: ${String(error)}`)) } finally { await this.closeCommunicator() this.ensurePreviewDisplay() this.update(reason ? 'error' : 'disconnected', reason ?? 'Preview only') this.stopping = null finished() } } private async closeCommunicator(): Promise { const communicator = this.communicator if (!communicator) return for (const off of this.sessionOffs.splice(0)) off() this.notifications = null this.communicator = null bindBandwidthBenchmarkSession(null) // Releases an EvenHub app's mic before the session closes. this.syncRawMicSession() this.glance.reset() await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`)) } private maybeLaunchOnboarding(state: SessionState): void { const firstNewFrame = state.frames > this.acknowledgedFrames this.acknowledgedFrames = state.frames // Match Android: wait for an acknowledged frame (a warm session, so the // onboarding's welcome jingle isn't dropped) before opening it. if (!firstNewFrame || state.phase !== 'connected' || !claimGlassesOnboardingAutoLaunch()) return void this.launchApp(ONBOARDING_APP_ID) } startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() } private async prepareVoiceCapture(): Promise { if (this.glassesLocked) return false const usePhoneMic = this.state.phase !== 'connected' if (usePhoneMic && !this.active) return false const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider()) if (!ready && this.active) this.onError(iosVoiceInput.statusText) return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active) } private async startVoiceCapture(endpointing = false): Promise { if (this.glassesLocked) return const log = (message: string) => this.logBluetooth(message) const communicator = this.communicator const voice = this.voiceProvider() if (communicator && this.state.phase === 'connected') { await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing) } else if (this.active) { // Recheck microphone permission if the glasses disconnected after prepare. if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return await iosVoiceInput.startPhoneCapture(voice, log, endpointing) } } /** * Continuous capture on behalf of `client` (one id per window: Transcribe, * Translate, Teleprompter, Calculator); it runs while any client wants it, as * on Android. `recordAs` names a WAV of the audio in Downloads when Save * Transcribe audio is on. */ private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { this.continuousClients.set(client, options) if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) { this.continuousClients.delete(client) this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).') return } this.syncContinuousRecording() // A listening capture is shared as it is; otherwise start one (after any drain). if (!iosVoiceInput.continuousSource) this.continuousPending = true this.syncContinuousCapture() } private stopContinuousVoiceCapture(client: string): void { if (!this.continuousClients.delete(client)) return this.syncContinuousRecording() // The window that needed its own provider may be the one that left. if (this.continuousClients.size > 0) { this.syncContinuousCapture(); return } // A start still awaiting permission or the mic backs off. ++this.continuousCaptureToken this.continuousPending = false this.syncContinuousClaim() void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`)) } /** Connecting or losing the glasses moves continuous capture to the new audio source. */ private syncContinuousSession(): void { const connected = this.state.phase === 'connected' if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true this.sessionConnected = connected this.syncContinuousCapture() } /** * Keep continuous capture running while a window wants it: start a pending * one once audio is available, and move a phone capture to the glasses once * they connect. A glasses session supplies the audio, even while it * reconnects (as on Android); the phone mic stands in only without one, while * Faceclaw is in the foreground. Cheap enough to call on every state change. */ private syncContinuousCapture(): void { this.syncContinuousClaim() if (this.continuousClients.size === 0 || this.glassesLocked) return const glassesReady = !!this.communicator && this.state.phase === 'connected' if (iosVoiceInput.continuousActive) { // Draining or listening already. A phone capture from before the glasses // connected is drained here, and its end restarts it on the glasses; so // is one transcribing with another provider than its windows now need. const retarget = this.continuousTranscription !== transcriptionKey(this.voiceProvider()) if (iosVoiceInput.continuousSource && (retarget || glassesReady && iosVoiceInput.continuousSource === 'phone')) { this.continuousPending = true void iosVoiceInput.stopContinuousCapture() } return } if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return this.continuousPending = false void this.beginContinuousCapture() } private async beginContinuousCapture(): Promise { const voice = this.voiceProvider() if (!iosVoiceInput.supportsContinuous(voice)) return this.continuousTranscription = transcriptionKey(voice) // A stop during an await below bumps the token; this start then backs off. const token = ++this.continuousCaptureToken const current = () => token === this.continuousCaptureToken const log = (message: string) => this.logBluetooth(message) // Claimed before the capture starts so other mic users stand down first, as on Android. voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER) const communicator = this.communicator if (communicator) { await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true) } else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) { await iosVoiceInput.startPhoneCapture(voice, log, false, true) } if (current()) this.syncContinuousClaim() } /** * EvenHub apps' mic (the bridge's raw tap) listens through a connected, * unlocked glasses session, as speech capture does; the EvenHub mic router * decides which app hears it and when. */ private syncRawMicSession(): void { const ready = this.state.phase === 'connected' && !this.glassesLocked iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message)) } /** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */ private syncContinuousClaim(): void { voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) } /** Record to the first client's requested name while Save Transcribe audio is on. */ private syncContinuousRecording(): void { const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : '' iosVoiceInput.recordTo(name) } /** * The transcription provider setting and the keys the cloud ones need. A * window that needs its own provider and language (Translate) sets them for * every capture while it is open; otherwise only Whisper server is told a * language (its own setting). */ private voiceProvider(): VoiceProviderSelection { const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription const provider = choice?.provider ?? voiceProviderSetting.get() return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '', elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` console.log(`[ios-ble] ${line}`); this.logLines.push(line) if (this.logLines.length > 1000) this.logLines.shift() if (this.logTimer !== null) return this.logTimer = setTimeout(() => { this.logTimer = null this.flushBluetoothLog() }, 500) } private flushBluetoothLog(): void { if (this.logTimer !== null) clearTimeout(this.logTimer) this.logTimer = null try { File.fromPath(path.join(knownFolders.documents().path, 'bluetooth.log')).writeTextSync(this.logLines.join('\n')) } catch (error) { console.warn(`Bluetooth log: ${error}`) } } typeIntoApp(): void { if (this.glassesLocked || !this.active) return shell.startKeyboardInput() } private async editSetting(setting: { editorTitle: string; inputKind?: string; get(): string; set(value: string): void; validationError?(value?: string): string | null }): Promise { if (!this.active) { this.logBluetooth('Editing text settings requires opening Faceclaw on the phone'); return false } let draft = setting.get() while (true) { const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: draft, inputType: setting.inputKind, okButtonText: 'Save', cancelButtonText: 'Cancel' }) if (!result.result) return false draft = result.text const error = setting.validationError?.(draft) if (error) { await Dialogs.alert({ title: setting.editorTitle, message: error, okButtonText: 'OK' }) continue } setting.set(draft) if (setting === nightscoutSiteUrlSetting || setting === nightscoutApiTokenSetting) await nightscoutBridge.refreshNow() return true } } }