iOS: fix the audio-packet and window-close crashes, stop retrying the ring

- Rename IosAudioPacketListener.onAudioPacket to onNsDataAudioPacket. The
  selector collided with FaceclawAudioPacketListener's, Kotlin/Native mangled
  one of the two depending on export order, and the first glasses audio packet
  aborted the app with "unrecognized selector".
- Closing a window (e.g. Transcribe from its system menu) aborted the app: the
  menu pop re-renders the window, that frame is queued behind the surface
  removal, and Kotlin's compositor throws on the unknown surface. The bridge now
  tracks which surfaces Kotlin holds and drops stale calls inside the queue,
  submit re-checks the window is still open, and the IosGlassesSession surface
  wrappers log and drop compositor errors instead of aborting.
- Only connect to the R1 ring from the phone in "Direct" mode, as Android does.
  iOS always tried; the failing connect retried every few seconds and starved
  the glasses link, so the glasses image lagged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
FilipAlexanderandClaude Opus 5.5 committed 2026-10-17 16:42:41 +02:00
1 parent 2bb7b3ea1a
commit ecb6c7f465
4 files changed
+151 -21

No files matched your search

+9 -5
View File
@@ -48,7 +48,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings'
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
@@ -451,10 +451,11 @@ export class IosPreviewController {
}
}
/** Submit a painted frame for an in-process window (e.g. the launcher). */
private async submit(id: string, planes: Plane[]): Promise<void> {
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
await this.displayReady
const display = this.display
if (!display) return
// The window can close (removing its surface) while this frame waits.
if (!display || !live()) return
const fingerprint = planesFingerprint(planes)
const { image, draws } = flattenPlanesWithDraws(planes)
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
@@ -646,7 +647,7 @@ export class IosPreviewController {
// The explicit render below supplies a fresh frame after configuration.
let surfaceReady = false
const app = create({
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(),
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
reconfigureSurface: () => {
@@ -731,7 +732,10 @@ export class IosPreviewController {
this.firmwareCompatible = false
this.update('connecting', 'Finding configured devices…')
let identifiers
try { identifiers = await resolveIosPeripherals(addresses) }
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
// fails and retries every few seconds, starving the glasses link.
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
catch (failure) {
this.update('error', failure instanceof Error ? failure.message : String(failure))
return
+26 -5
View File
@@ -141,7 +141,7 @@ class AncsListener extends NSObject {
class AudioPacketListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosAudioPacketListener];
handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void;
onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs));
}
}
@@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge {
private ancsProxy: AncsListener | null = null;
private nativeCallQueue: Promise<void> = Promise.resolve();
private queuedNativeCalls = 0;
// Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface).
private readonly nativeSurfaces = new Set<string>();
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
private readonly finishedFrameOutcomes = new Map<number, string>();
private readonly frameFinishedWaiters = new Map<number, Set<(outcome: string) => void>>();
@@ -518,15 +520,29 @@ export class FaceclawCommunicatorBridge {
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
const transparency = options.transparency === "color-key" ? 1 : 0;
await this.enqueueNativeCall(() => {
this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height),
Math.round(options.zOrder), transparency,
);
if (configured !== false) this.nativeSurfaces.add(id);
});
}
async removeSurface(id: string): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); });
await this.enqueueNativeCall(() => {
this.nativeSurfaces.delete(id);
this.communicator.removeSurfaceId(id);
});
}
/**
* Checked inside each queued call: a render in flight when its window closed lands
* behind the removal, and Kotlin aborts on a frame for an unknown surface.
*/
private hasNativeSurface(id: string, what: string): boolean {
if (this.nativeSurfaces.has(id)) return true;
console.log(`[ios-ble] ${what} for removed surface ${id} dropped`);
return false;
}
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
@@ -541,11 +557,15 @@ export class FaceclawCommunicatorBridge {
}
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); });
await this.enqueueNativeCall(() => {
if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible));
});
}
async setSurfaceDepth(id: string, depth: number): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); });
await this.enqueueNativeCall(() => {
if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth));
});
}
async setScreenBlanked(blanked: boolean): Promise<void> {
@@ -572,6 +592,7 @@ export class FaceclawCommunicatorBridge {
const snapshot = new Uint8Array(pixels8bpp);
const draws = glyphs ? toData(new Uint8Array(glyphs)) : null;
await this.enqueueNativeCall(() => {
if (!this.hasNativeSurface(surfaceId, "frame")) return;
this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(
toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height),
fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws,