import * as protocol from './ble-protocol' import { iosBleTraffic } from './ble-traffic-counters' import { buildBoundingBoxPayload } from './ble-image-optimizer' import { hexToBytes } from '../util/hex-util' import { deviceAddressError } from './ios-peripheral-identity' declare function setTimeout(callback: () => void, ms: number): number declare function clearTimeout(id: number): void export type SessionAddresses = { left: string; right: string; ring: string } export type TransportEvent = { kind: string; identifier?: string; characteristic?: string; data?: string; message?: string; state?: number } export interface SessionTransport { onEvent(listener: (event: TransportEvent) => void): () => void resolveDevices(addresses: SessionAddresses): Promise> connect(identifier: string): Promise<{ characteristics: string[]; maxWrite: number }> subscribe(identifier: string, characteristic: string): Promise write(identifier: string, characteristic: string, bytes: Uint8Array): Promise disconnect(identifier: string): void stopScan(): void } export type SessionState = { phase: 'disconnected' | 'connecting' | 'connected' | 'retrying' | 'disconnecting' | 'error'; status: string; battery: number | null; charging: boolean | null; ring: boolean; frames: number; capabilities: string; leftVersion: string; rightVersion: string } type PendingAck = { resolve: (message: protocol.ProtocolMessage) => void; reject: (error: Error) => void; timer: number | null; command: number; label: string } class AckTimeout extends Error {} // Match Android ConnectionOptions.WINDOW_SIZE. This bounds complete image // messages queued for writing or awaiting ACK, not individual BLE packets. const DISPLAY_WINDOW_SIZE = 3 type DisplayFrame = { packed: Uint8Array; payload: Uint8Array; offset: number; pending: number } /** G2 session independent of the phone OS. Images go to L, control to R, as on Android. */ export class GlassesSession { state: SessionState = { phase: 'disconnected', status: 'Preview only', battery: null, charging: null, ring: false, frames: 0, capabilities: '', leftVersion: '', rightVersion: '' } private generation = 0 private ids: Record = {} private limits: Record = {} private pending = new Map() private receiver = new protocol.MessageReceiver() private writes = new Map>() private magic = 100 private sequence = 0x40 private timer: number | null = null private retryTimer: number | null = null private ringRetryTimer: number | null = null private ringRetryCount = 0 private ringConnecting = false private retryCount = 0 private wanted = false private addresses: SessionAddresses | null = null private latest: Uint8Array | null = null private displayed: Uint8Array | null = null private lastEnqueued: Uint8Array | null = null // Match Android: advance only for emitted deltas, skipping 0 / 0xffff. private nextImageFrameId = 1 private displayFrames: DisplayFrame[] = [] private displaySending: DisplayFrame | null = null private displayInFlight = 0 private pumping = false private lastHeartbeat = 0 private lastLease = 0 private lastSettings = 0 private charging = false private layoutCreated = false private off: () => void private stopping: Promise | null = null private audioListener: ((packet: Uint8Array) => void) | null = null private microphoneToken = 0 private microphoneWork: Promise = Promise.resolve() private microphoneEnabled = false constructor(private readonly transport: SessionTransport, private readonly deflate: (data: Uint8Array) => Uint8Array, private readonly onState: (state: SessionState) => void, private readonly onInput: (input: protocol.GlassesInput) => void, private readonly log: (message: string) => void = () => {}, private readonly onActivity: () => void = () => {}) { this.off = transport.onEvent(event => this.receive(event)) } private update(phase: SessionState['phase'], status: string): void { this.state = { ...this.state, phase, status }; this.log(status); this.onState({ ...this.state }) } async start(addresses: SessionAddresses, retry = false): Promise { if (['connecting', 'connected', 'disconnecting'].includes(this.state.phase)) return const invalid = deviceAddressError(addresses) if (invalid) { this.update('error', invalid); return } this.addresses = { ...addresses }; this.wanted = true if (!retry) this.retryCount = 0 const generation = ++this.generation this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', battery: null, charging: null, ring: false } this.charging = false this.update('connecting', 'Finding configured devices…') try { const ids = await this.transport.resolveDevices(addresses); this.check(generation) this.ids = ids for (const role of ['right', 'left']) { this.update('connecting', `Connecting ${role} arm…`) const details = await this.transport.connect(ids[role]); this.check(generation) this.limits[role] = details.maxWrite if (!details.characteristics.includes(protocol.G2_WRITE) || !details.characteristics.includes(protocol.G2_NOTIFY)) throw new Error(`${role} arm is missing G2 communication characteristics.`) await this.transport.subscribe(ids[role], protocol.G2_NOTIFY); this.check(generation) } this.update('connecting', 'Authenticating… Accept pairing if iOS asks.') await Promise.all(['right', 'left'].map(async role => { try { const answer = await this.request(role, protocol.SID.auth, protocol.authentication, 'Security authentication', 30_000, 0) if (!protocol.authenticationSucceeded(answer, answer.magic)) throw new Error(`${role} arm rejected authentication.`) } catch (error) { if (!(error instanceof AckTimeout)) throw error // Older CFW may not implement the reply. A settings response below // still has to prove we have a working, compatible session. this.log(`${role} authentication unconfirmed; checking firmware response`) } })) this.check(generation) this.update('connecting', 'Checking glasses firmware…') const settings = await this.request('right', protocol.SID.settings, protocol.settingsQuery, 'Settings query') this.applySettings(settings) const caps = this.state.capabilities.split(/\s+/) if (!['img640', 'fbguard', 'wearnotify'].every(cap => caps.includes(cap))) throw new Error('The glasses do not report compatible Faceclaw firmware (img640, fbguard, wearnotify). Bluetooth is working, but this display requires the modified firmware. Flashing remains available on Android.') this.update('connecting', 'Starting glasses display…') await this.request('right', protocol.SID.launch, () => protocol.prelude(), 'App launch', 3500, 0x20, 156) await this.lease(true) await this.request('right', protocol.SID.hub, protocol.createLayout, 'Create display layout') this.check(generation); this.layoutCreated = true this.lastHeartbeat = this.lastLease = this.lastSettings = Date.now() this.displayed = null; this.retryCount = 0 this.update('connected', 'Glasses connected') this.schedule() if (addresses.ring && ids.ring) await this.connectRing(generation) else if (addresses.ring) this.update('connected', 'Glasses connected; configured ring was not found') } catch (error) { if (generation !== this.generation) return this.fail(error, retry || error instanceof AckTimeout) } } private async connectRing(generation: number): Promise { if (this.ringConnecting || generation !== this.generation || this.state.phase !== 'connected' || !this.ids.ring) return this.ringConnecting = true const identifier = this.ids.ring try { const ring = await this.transport.connect(identifier); this.check(generation) this.log(`R1 services discovered: ${ring.characteristics.join(', ')}`) const notifications = protocol.RING_NOTIFY.filter(uuid => ring.characteristics.includes(uuid)) if (!notifications.length) throw new Error('No R1 gesture notifications found') let subscribed = 0 for (const uuid of notifications) { this.log(`Subscribing R1 ${uuid}`) try { await this.transport.subscribe(identifier, uuid); this.check(generation); subscribed++; this.log(`R1 subscribed ${uuid}`) } catch (error) { this.check(generation); this.log(`R1 channel unavailable ${uuid}: ${this.message(error)}`) } } if (!subscribed) throw new Error('Neither R1 notification channel could be subscribed') this.check(generation); this.state.ring = true; this.ringRetryCount = 0 this.update('connected', 'Glasses and ring connected') } catch (error) { if (generation !== this.generation) return this.state.ring = false this.transport.disconnect(identifier) this.update('connected', `Glasses connected; ring unavailable: ${this.message(error)}`) } finally { if (generation === this.generation) { this.ringConnecting = false if (!this.state.ring) this.scheduleRingRetry() } } } private scheduleRingRetry(): void { if (!this.wanted || this.state.phase !== 'connected' || !this.ids.ring || this.ringConnecting || this.ringRetryTimer !== null || this.ringRetryCount >= 5) return const generation = this.generation, delay = Math.min(30_000, 2000 * 2 ** this.ringRetryCount++) this.log(`Retrying ring in ${delay / 1000}s`) this.ringRetryTimer = setTimeout(() => { this.ringRetryTimer = null if (generation === this.generation) void this.connectRing(generation) }, delay) } private check(generation: number): void { if (generation !== this.generation) throw new Error('Connection cancelled') } private nextMagic(): number { for (let i = 0; i < 156; i++) { const magic = this.magic; this.magic = this.magic === 255 ? 100 : this.magic + 1 if (magic !== 156 && ![...this.pending.keys()].some(key => key.endsWith(`:${magic}`))) return magic } throw new Error('All message identifiers are in use') } private request(role: string, sid: number, build: (magic: number) => Uint8Array, label: string, timeout = 3500, flag = 0x20, fixedMagic?: number): Promise { const magic = fixedMagic ?? this.nextMagic(), payload = build(magic), key = `${sid}:${magic}` this.log(`TX ${label} ${role} sid=${sid.toString(16)} magic=${magic}`) const generation = this.generation return new Promise((resolve, reject) => { const pending: PendingAck = { resolve, reject, timer: null, command: protocol.readInteger(payload, 1, -1), label } this.pending.set(key, pending) this.send(role, sid, flag, payload).then(() => { if (this.pending.get(key) !== pending || generation !== this.generation) return pending.timer = setTimeout(() => { this.pending.delete(key); reject(new AckTimeout(`${label} was not acknowledged by the glasses.`)) }, timeout) }).catch(error => { if (this.pending.get(key) !== pending) return this.pending.delete(key); if (pending.timer) clearTimeout(pending.timer); reject(error) }) }) } private send(role: string, sid: number, flag: number, payload: Uint8Array): Promise { const identifier = this.ids[role], generation = this.generation if (!identifier) return Promise.reject(new Error(`No ${role} device connected`)) const frames = protocol.frameMessage(payload, sid, flag, this.sequence++, this.limits[role]) const previous = this.writes.get(identifier) ?? Promise.resolve() // A failed write poisons this link's queue until reset: do not transmit // later fragments after a hole in the image stream. const work = previous.then(async () => { for (const frame of frames) { this.check(generation) await this.transport.write(identifier, protocol.G2_WRITE, frame) iosBleTraffic.recordWrite(frame.length) } iosBleTraffic.recordMessage() }) // A whole message owns the write queue; heartbeat packets cannot split an image message. this.writes.set(identifier, work) return work } private lease(acquire: boolean): Promise { return Promise.all(['right', 'left'].map(role => this.send(role, protocol.SID.settings, 0x20, protocol.framebufferLease(acquire)))) } private receive(event: TransportEvent): void { if (!Object.values(this.ids).includes(event.identifier ?? '')) return if (event.kind === 'disconnected') { if (event.identifier === this.ids.ring) { this.state.ring = false if (this.state.phase === 'connected') { this.update('connected', `Glasses connected; ring disconnected${event.message ? ': ' + event.message : ''}`) this.scheduleRingRetry() } } else if (['connected', 'connecting'].includes(this.state.phase)) this.fail(new Error(event.message || 'Glasses disconnected'), true) return } if (event.kind !== 'notification' || !event.data) return if (event.characteristic === protocol.G2_RENDER_NOTIFY && event.identifier !== this.ids.ring) { if (this.audioListener && this.state.phase === 'connected') { this.audioListener(hexToBytes(event.data)) this.onActivity(); this.wake() } return } const ring = event.identifier === this.ids.ring if (ring ? !protocol.RING_NOTIFY.includes(event.characteristic ?? '') : event.characteristic !== protocol.G2_NOTIFY) return try { const data = hexToBytes(event.data) if (ring) { const input = protocol.decodeRingInput(data) if (input) { this.log(`Direct R1 gesture ${input.eventType}`); this.onInput(input) } return } for (const message of this.receiver.receive(event.identifier!, data)) { this.log(`RX ${event.identifier === this.ids.left ? 'L' : 'R'} sid=${message.sid.toString(16)} flag=${message.flag.toString(16)} cmd=${message.command} magic=${message.magic}`) if (![1, 6].includes(message.flag)) { // A master arm can relay the other arm's ACK; magic is allocated // across BOTH links, matching Android's communicator. const key = `${message.sid}:${message.magic}`, pending = this.pending.get(key) if (pending) { this.pending.delete(key); if (pending.timer) clearTimeout(pending.timer) pending.resolve(message) } } if (message.sid === protocol.SID.settings) this.applySettings(message) if (event.identifier === this.ids.right) { const input = protocol.decodeGlassesInput(message) if (input) { if (input.kind === 'sys-event' && [5, 6, 7].includes(input.eventType) && this.state.phase === 'connected') { // Respect an explicit glasses-side exit; the user can reconnect. void this.stop(); continue } this.onInput(input) } } } } catch (error) { this.log(`Ignored malformed BLE notification: ${this.message(error)}`) } finally { // CoreBluetooth can wake a suspended iOS process for a notification. // Service wall-clock deadlines now, without waiting for a suspended timer. if (this.state.phase === 'connected') { this.onActivity(); this.wake() } } } private applySettings(message: protocol.ProtocolMessage): void { const values = protocol.readBytes(message.payload, 4) const capabilities = protocol.readString(message.payload, 100) if (capabilities) this.state.capabilities = capabilities if (values) { const left = protocol.readString(values, 5), right = protocol.readString(values, 6) if (left) this.state.leftVersion = left if (right) this.state.rightVersion = right const battery = protocol.readInteger(values, 12, -1), charging = protocol.readInteger(values, 13, -1) if (battery >= 0 && battery <= 100) this.state.battery = battery if (charging >= 0) this.state.charging = this.charging = charging > 0 } this.onState({ ...this.state }) } setFrame(gray: Uint8Array): void { if (this.state.phase !== 'connected') return this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0) } /** Serialize mic enable/disable, including a release while enable awaits ACK. */ setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise { const token = ++this.microphoneToken, generation = this.generation this.audioListener = enabled ? listener ?? null : null const current = () => generation === this.generation && token === this.microphoneToken const work = this.microphoneWork.catch(() => {}).then(async () => { if (!current()) return if (this.state.phase !== 'connected') { if (enabled) throw new Error('Connect the glasses before starting voice input.') return } if (enabled) { await Promise.all(['left', 'right'].map(role => this.transport.subscribe(this.ids[role], protocol.G2_RENDER_NOTIFY))) if (!current()) return } await this.request('right', protocol.SID.hub, magic => protocol.audioControl(magic, enabled), enabled ? 'Microphone enable' : 'Microphone disable') if (current()) { this.microphoneEnabled = enabled; this.log(`Glasses microphone ${enabled ? 'enabled' : 'disabled'}`) } }).catch(error => { if (current()) { this.audioListener = null; this.microphoneEnabled = false } throw error }) this.microphoneWork = work return work } wake(): void { if (this.timer !== null) { clearTimeout(this.timer); this.timer = null } void this.pump() } private schedule(delay = 1000): void { if (delay === 0 && this.timer !== null) { clearTimeout(this.timer); this.timer = null } if (this.timer !== null || this.pumping || this.state.phase !== 'connected') return this.timer = setTimeout(() => { this.timer = null; void this.pump() }, delay) } private canSendDisplay(): boolean { // Also bound completed frames retained behind a missing/out-of-order ACK. return this.displayInFlight < DISPLAY_WINDOW_SIZE && !!(this.displaySending || (this.latest && !this.charging && this.displayFrames.length < DISPLAY_WINDOW_SIZE)) } private fillDisplayWindow(generation: number): void { while (this.canSendDisplay()) { if (!this.displaySending) { const packed = this.latest!; this.latest = null // Compare with the last enqueued image, not the last ACKed one. In an // A -> B -> A sequence, B may still be in flight when A is requested. if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId, this.deflate) const payload = delta ?? protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(packed))) if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer') if (delta) { this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1 this.log(`Display bbox ${delta[3] * 4}x${delta[4] * 2}+${delta[1] * 4}+${delta[2] * 2} fid=${delta[5] | (delta[6] << 8)} (${payload.length} bytes)`) } else this.log(`Display full 640x480 (${payload.length} bytes)`) this.displaySending = { packed, payload, offset: 0, pending: 0 } this.displayFrames.push(this.displaySending); this.lastEnqueued = packed } const frame = this.displaySending, offset = frame.offset frame.offset += 3800; frame.pending++; this.displayInFlight++ if (frame.offset >= frame.payload.length) this.displaySending = null // send() serializes whole messages on L, but request() waits for each // magic's ACK independently. Fill the other slots without awaiting it. void this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, frame.payload, offset), 'Display frame').then(() => { if (generation !== this.generation) return frame.pending--; this.displayInFlight--; this.lastHeartbeat = Date.now() // Retire in submission order even if relayed ACKs arrive out of order. // A frame counts only after every fragment has been acknowledged. while (this.displayFrames.length) { const completed = this.displayFrames[0] if (completed.offset < completed.payload.length || completed.pending) break this.displayFrames.shift(); this.displayed = completed.packed this.state.frames++; iosBleTraffic.recordDisplayFrame(); this.onState({ ...this.state }) this.log(`Display frame ${this.state.frames} acknowledged (${completed.payload.length} bytes)`) } this.wake() }).catch(error => { if (generation === this.generation) this.fail(error, true) }) } } private async pump(): Promise { if (this.pumping || this.state.phase !== 'connected') return this.pumping = true; const generation = this.generation try { const now = Date.now() if (now - this.lastHeartbeat >= 4000) { const gap = now - this.lastHeartbeat await this.request('right', protocol.SID.hub, protocol.heartbeat, 'Heartbeat', 1500); this.lastHeartbeat = Date.now() this.log(`Heartbeat acknowledged (gap ${gap}ms)`) } if (now - this.lastLease >= 45_000) { await this.lease(true); this.lastLease = Date.now() } if (now - this.lastSettings >= (this.charging ? 30_000 : 300_000)) { const settings = await this.request('right', protocol.SID.settings, protocol.settingsQuery, 'Battery query') this.applySettings(settings); this.lastSettings = Date.now() } this.check(generation) this.fillDisplayWindow(generation) } catch (error) { if (generation === this.generation) this.fail(error, true) } finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.canSendDisplay() ? 0 : 1000) } } } private reset(): void { ++this.generation ++this.microphoneToken; this.audioListener = null; this.microphoneEnabled = false if (this.timer !== null) clearTimeout(this.timer) if (this.retryTimer !== null) clearTimeout(this.retryTimer) if (this.ringRetryTimer !== null) clearTimeout(this.ringRetryTimer) this.ringRetryTimer = null; this.ringConnecting = false; this.ringRetryCount = 0 this.timer = this.retryTimer = null; this.pumping = false const pending = [...this.pending.values()]; this.pending.clear() for (const request of pending) { if (request.timer) clearTimeout(request.timer); request.reject(new Error('Connection ended')) } this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0 } private closeLinks(): void { const ids = this.ids; this.ids = {}; this.limits = {} for (const identifier of Object.values(ids)) this.transport.disconnect(identifier) this.transport.stopScan(); this.layoutCreated = false; this.state.ring = false } private fail(error: unknown, retry: boolean): void { const message = this.message(error); this.reset(); this.closeLinks() if (retry && this.wanted && this.addresses && this.retryCount < 5) { const delay = Math.min(30_000, 2000 * 2 ** this.retryCount++) this.update('retrying', `${message} Retrying in ${delay / 1000}s…`) this.retryTimer = setTimeout(() => { this.retryTimer = null; if (this.wanted && this.addresses) void this.start(this.addresses, true) }, delay) } else { this.wanted = false; this.update('error', message) } } stop(): Promise { if (this.stopping) return this.stopping this.stopping = this.stopInternal().finally(() => { this.stopping = null }) return this.stopping } private async stopInternal(): Promise { this.wanted = false // Stop mic capture before ending the display session. Late enable ACKs // cannot leave a microphone streaming after an explicit Disconnect. if (this.audioListener || this.microphoneEnabled) { try { await this.setMicrophone(false) } catch (error) { this.log(`Microphone cleanup: ${this.message(error)}`) } } const cleanup = this.layoutCreated && this.state.phase === 'connected' this.update('disconnecting', 'Disconnecting…'); this.reset() try { if (cleanup) { if (this.state.capabilities.split(/\s+/).includes('cleanup11')) await this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, new Uint8Array([11]), 0), 'Display cleanup', 1200) else { await this.request('right', protocol.SID.hub, protocol.shutdown, 'Close display', 1200) await this.lease(false) } } } catch (error) { this.log(`Disconnect cleanup: ${this.message(error)}`) } finally { this.reset(); this.closeLinks(); this.update('disconnected', 'Preview only') } } dispose(): void { void this.stop().finally(() => this.off()) } private message(error: unknown): string { return error instanceof Error ? error.message : String(error) } }