diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 97f3fe3c..639f00dd 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -48,7 +48,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, - openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings' + openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings' import type { VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' @@ -451,10 +451,11 @@ export class IosPreviewController { } } /** Submit a painted frame for an in-process window (e.g. the launcher). */ - private async submit(id: string, planes: Plane[]): Promise { + private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise { await this.displayReady const display = this.display - if (!display) return + // The window can close (removing its surface) while this frame waits. + if (!display || !live()) return const fingerprint = planesFingerprint(planes) const { image, draws } = flattenPlanesWithDraws(planes) await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, @@ -646,7 +647,7 @@ export class IosPreviewController { // The explicit render below supplies a fresh frame after configuration. let surfaceReady = false const app = create({ - actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(), + actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(), setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible), removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) }, reconfigureSurface: () => { @@ -731,7 +732,10 @@ export class IosPreviewController { this.firmwareCompatible = false this.update('connecting', 'Finding configured devices…') let identifiers - try { identifiers = await resolveIosPeripherals(addresses) } + // As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect + // fails and retries every few seconds, starving the glasses link. + const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : '' + try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) } catch (failure) { this.update('error', failure instanceof Error ? failure.message : String(failure)) return diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index ed8937b4..f09025c8 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -141,7 +141,7 @@ class AncsListener extends NSObject { class AudioPacketListener extends NSObject { static ObjCProtocols = [FaceclawKitIosAudioPacketListener]; handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void; - onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void { + onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void { this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs)); } } @@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge { private ancsProxy: AncsListener | null = null; private nativeCallQueue: Promise = Promise.resolve(); private queuedNativeCalls = 0; + // Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface). + private readonly nativeSurfaces = new Set(); private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>(); private readonly finishedFrameOutcomes = new Map(); private readonly frameFinishedWaiters = new Map void>>(); @@ -518,15 +520,29 @@ export class FaceclawCommunicatorBridge { async configureSurface(id: string, options: SurfaceOptions): Promise { const transparency = options.transparency === "color-key" ? 1 : 0; await this.enqueueNativeCall(() => { - this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency( + const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency( id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height), Math.round(options.zOrder), transparency, ); + if (configured !== false) this.nativeSurfaces.add(id); }); } async removeSurface(id: string): Promise { - await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); }); + await this.enqueueNativeCall(() => { + this.nativeSurfaces.delete(id); + this.communicator.removeSurfaceId(id); + }); + } + + /** + * Checked inside each queued call: a render in flight when its window closed lands + * behind the removal, and Kotlin aborts on a frame for an unknown surface. + */ + private hasNativeSurface(id: string, what: string): boolean { + if (this.nativeSurfaces.has(id)) return true; + console.log(`[ios-ble] ${what} for removed surface ${id} dropped`); + return false; } async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise { @@ -541,11 +557,15 @@ export class FaceclawCommunicatorBridge { } async setSurfaceVisible(id: string, visible: boolean): Promise { - await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); }); + await this.enqueueNativeCall(() => { + if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); + }); } async setSurfaceDepth(id: string, depth: number): Promise { - await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); }); + await this.enqueueNativeCall(() => { + if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); + }); } async setScreenBlanked(blanked: boolean): Promise { @@ -572,6 +592,7 @@ export class FaceclawCommunicatorBridge { const snapshot = new Uint8Array(pixels8bpp); const draws = glyphs ? toData(new Uint8Array(glyphs)) : null; await this.enqueueNativeCall(() => { + if (!this.hasNativeSurface(surfaceId, "frame")) return; this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs( toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height), fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws, diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt index 3084b297..b326bdbd 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt @@ -15,9 +15,13 @@ interface IosAncsListener { fun onRelayFrame(data: NSData) } -/** Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue. */ +/** + * Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue. + * Not named onAudioPacket: that selector collides with FaceclawAudioPacketListener's, and the + * mangled name Kotlin/Native picks broke the TypeScript proxy ("unrecognized selector"). + */ interface IosAudioPacketListener { - fun onAudioPacket(data: NSData, arm: String, arrivalMs: Long) + fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long) } /** @@ -209,14 +213,21 @@ class IosGlassesSession internal constructor( /** Set the compositor's output frame size. Call before configuring surfaces. */ fun configureCompositorScreen(width: Int, height: Int) = core.configureCompositorScreen(width, height) - fun setSurfaceVisible(id: String, visible: Boolean) = core.setSurfaceVisible(id, visible) + // The compositor throws for an unknown surface or a bad rect, and without @Throws (which + // would rename the selectors) that aborts the process. Log and drop instead; false then. - fun setSurfaceDepth(id: String, depth: Int) = core.setSurfaceDepth(id, depth) + fun setSurfaceVisible(id: String, visible: Boolean): Boolean = surfaceCall("setSurfaceVisible $id") { + core.setSurfaceVisible(id, visible) + } + + fun setSurfaceDepth(id: String, depth: Int): Boolean = surfaceCall("setSurfaceDepth $id") { + core.setSurfaceDepth(id, depth) + } fun setScreenBlanked(blanked: Boolean) = core.setScreenBlanked(blanked) - fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) = - core.configureSurface(id, x, y, width, height, zOrder, transparency) + fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int): Boolean = + surfaceCall("configureSurface $id") { core.configureSurface(id, x, y, width, height, zOrder, transparency) } fun removeSurface(id: String) = core.removeSurface(id) @@ -235,10 +246,28 @@ class IosGlassesSession internal constructor( paintMs: Int, frameId: Int, glyphs: NSData?, - ) = core.submitSurfaceFrame( - IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId, - glyphs?.let { IosByteReader(it) }, - ) + ): Boolean { + val ok = surfaceCall("submitSurfaceFrame $surfaceId") { + core.submitSurfaceFrame( + IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId, + glyphs?.let { IosByteReader(it) }, + ) + } + if (!ok) frameTimings.finishFrame(frameId, "discarded: surface $surfaceId rejected") + return ok + } + + private inline fun surfaceCall(what: String, block: () -> Unit): Boolean = + try { + block() + true + } catch (e: IllegalArgumentException) { + core.logLine("$what dropped: ${e.message}") + false + } catch (e: IllegalStateException) { + core.logLine("$what dropped: ${e.message}") + false + } /** The current composited screen (gray, 8bpp) or null before any surface was configured. */ fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) } @@ -265,7 +294,7 @@ class IosGlassesSession internal constructor( override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { val packet = (data ?: ByteArray(0)).data() val label = arm ?: "" - dispatch_async(dispatch_get_main_queue()) { delegate.onAudioPacket(packet, label, arrivalMs) } + dispatch_async(dispatch_get_main_queue()) { delegate.onNsDataAudioPacket(packet, label, arrivalMs) } } } diff --git a/tests/ios-communicator-surfaces.test.cjs b/tests/ios-communicator-surfaces.test.cjs new file mode 100644 index 00000000..c84d629f --- /dev/null +++ b/tests/ios-communicator-surfaces.test.cjs @@ -0,0 +1,76 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +// FaceclawCommunicatorBridge over a fake Kotlin session that, like the real +// SurfaceCompositor, rejects calls for surfaces it does not hold. +function harness() { + const calls = []; + const surfaces = new Set(); + const session = { + setRequiredFirmwareRevisionRevision() {}, setListenerListener() {}, setPreviewAnimationListenerListener() {}, + configureSurfaceIdXYWidthHeightZOrderTransparency(id) { calls.push(['configure', id]); surfaces.add(id); return true; }, + removeSurfaceId(id) { calls.push(['remove', id]); surfaces.delete(id); }, + setSurfaceVisibleIdVisible(id) { calls.push(['visible', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; }, + setSurfaceDepthIdDepth(id) { calls.push(['depth', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; }, + submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(_pixels, id) { + calls.push(['frame', id]); + if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); + return true; + }, + isSurfaceCurrentSurfaceIdFingerprint: (id, fingerprint) => fingerprint === 'same', + }; + class NSObject { static new() { return new this(); } } + const timers = []; + const load = loader({ + NSObject, NativeClass: () => cls => cls, setTimeout: (fn, delay) => { timers.push({ fn, delay }); return timers.length; }, clearTimeout: id => { timers[id - 1].cleared = true; }, + FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) }, + FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {}, + FaceclawKitFaceclawCompassListener: {}, console: { log() {}, warn() {} }, + }, { + '@nativescript/core': { File: {}, knownFolders: {}, path: {} }, + '../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 }, + './kotlin-data': { toData: bytes => bytes, fromData: data => data }, + './ios-graphics': { previewPixels() {} }, + './ios-bluetooth': { iosBluetooth: {} }, + }); + const { FaceclawCommunicatorBridge } = load('app/native/faceclaw-communicator.ios.ts'); + const bridge = new FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + // Run queued native calls (each is one setTimeout(0) hop) until none are left. + async function drain() { + for (let i = 0; i < 50; i++) { + await new Promise(resolve => setImmediate(resolve)); + const next = timers.find(timer => !timer.ran && !timer.cleared && timer.delay === 0); + if (!next) { await new Promise(resolve => setImmediate(resolve)); if (!timers.some(t => !t.ran && !t.cleared && t.delay === 0)) return; continue; } + next.ran = true; next.fn(); + } + } + return { bridge, calls, drain }; +} + +const rect = { x: 0, y: 0, width: 2, height: 1 }; +const options = { x: 0, y: 0, width: 2, height: 1, zOrder: 0, transparency: 'opaque' }; + +test('a frame queued behind its surface removal is dropped instead of reaching Kotlin', async () => { + const h = harness(); + const configured = h.bridge.configureSurface('window:transcribe', options); await h.drain(); await configured; + // Close-window order: the removal is queued first, the in-flight render's frame lands behind it. + const removed = h.bridge.removeSurface('window:transcribe'); + const frame = h.bridge.submitSurfaceFrame('window:transcribe', new Uint8Array(2), rect, 'f'); + const visible = h.bridge.setSurfaceVisible('window:transcribe', true); + const depth = h.bridge.setSurfaceDepth('window:transcribe', 1); + await h.drain(); + await Promise.all([removed, frame, visible, depth]); + assert.deepEqual(h.calls, [['configure', 'window:transcribe'], ['remove', 'window:transcribe']]); +}); + +test('frames for a live surface still go through, including the idle inline path', async () => { + const h = harness(); + const configured = h.bridge.configureSurface('window:calc', options); await h.drain(); await configured; + await h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'f'); + assert.deepEqual(h.calls.at(-1), ['frame', 'window:calc']); + const reconfigured = h.bridge.configureSurface('window:calc', options); + const frame = h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'g'); + await h.drain(); await Promise.all([reconfigured, frame]); + assert.deepEqual(h.calls.slice(-2), [['configure', 'window:calc'], ['frame', 'window:calc']]); +});