iOS porting: Image-sending performance: Pipelining, bounding-box updates

This commit is contained in:
James Babcock committed 2026-09-08 13:18:50 -07:00
1 parent 16764359bd
commit 79b6c276f8
15 files changed
+592 -57

No files matched your search

+34
View File
@@ -0,0 +1,34 @@
import { concat, rle4 } from './ble-protocol'
/** Android BleImageOptimizer's mode-3 changed rectangle. Coordinates are in
* units of 4 columns / 2 rows; pixels are top-down packed 4bpp, high nibble first.
* null means identical, incomparable, or a full-screen box: use dedup/mode 6.
*/
export function buildBoundingBoxPayload(previous: Uint8Array | null, next: Uint8Array,
width: number, height: number, frameId: number, deflate: (data: Uint8Array) => Uint8Array): Uint8Array | null {
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0 ||
width % 4 || height % 2 || width / 4 > 255 || height / 2 > 255 ||
!previous || previous.length !== width * height / 2 || next.length !== previous.length) return null
const stride = width / 2
let minX = stride, maxX = -1, minY = height, maxY = -1
for (let y = 0; y < height; y++) {
const row = y * stride
for (let x = 0; x < stride; x++) {
if (previous[row + x] === next[row + x]) continue
minX = Math.min(minX, x); maxX = Math.max(maxX, x)
minY = Math.min(minY, y); maxY = y
}
}
if (maxY < 0) return null
const left = (minX * 2) & ~3, right = Math.min(width, ((maxX + 1) * 2 + 3) & ~3)
const top = minY & ~1, bottom = Math.min(height, (maxY + 2) & ~1)
const boxWidth = right - left, boxHeight = bottom - top
if (boxWidth === width && boxHeight === height) return null
const regionStride = boxWidth / 2, region = new Uint8Array(regionStride * boxHeight)
for (let y = 0; y < boxHeight; y++) {
const start = (top + y) * stride + left / 2
region.set(next.subarray(start, start + regionStride), y * regionStride)
}
return concat(new Uint8Array([3, left / 4, top / 2, boxWidth / 4, boxHeight / 2,
frameId & 255, (frameId >>> 8) & 255]), deflate(rle4(region)))
}
+17
View File
@@ -0,0 +1,17 @@
/** Same units as Android FaceclawBleManager: protocol messages, framed write
* bytes accepted by the transport, and fully acknowledged display updates.
* Totals survive reconnects and the setting being switched off.
*/
export class BleTrafficCounters {
private messages = 0
private bytes = 0
private frames = 0
recordWrite(bytes: number): void { this.bytes += bytes }
recordMessage(): void { this.messages++ }
recordDisplayFrame(): void { this.frames++ }
sample(): { messages: number; bytes: number; frames: number } {
return { messages: this.messages, bytes: this.bytes, frames: this.frames }
}
}
// The iOS BLE session and phone UI both run in the main isolate.
export const iosBleTraffic = new BleTrafficCounters()
+68 -14
View File
@@ -1,4 +1,6 @@
import * as protocol from './ble-protocol'
import { iosBleTraffic } from './ble-traffic-counters'
import { buildBoundingBoxPayload } from './ble-image-optimizer'
import { hexToBytes } from '../util/hex-util'
import { deviceAddressError } from './ios-peripheral-identity'
@@ -20,6 +22,10 @@ export type SessionState = { phase: 'disconnected' | 'connecting' | 'connected'
type PendingAck = { resolve: (message: protocol.ProtocolMessage) => void; reject: (error: Error) => void;
timer: number | null; command: number; label: string }
class AckTimeout extends Error {}
// Match Android ConnectionOptions.WINDOW_SIZE. This bounds complete image
// messages queued for writing or awaiting ACK, not individual BLE packets.
const DISPLAY_WINDOW_SIZE = 3
type DisplayFrame = { packed: Uint8Array; payload: Uint8Array; offset: number; pending: number }
/** G2 session independent of the phone OS. Images go to L, control to R, as on Android. */
export class GlassesSession {
@@ -43,6 +49,12 @@ export class GlassesSession {
private addresses: SessionAddresses | null = null
private latest: Uint8Array | null = null
private displayed: Uint8Array | null = null
private lastEnqueued: Uint8Array | null = null
// Match Android: advance only for emitted deltas, skipping 0 / 0xffff.
private nextImageFrameId = 1
private displayFrames: DisplayFrame[] = []
private displaySending: DisplayFrame | null = null
private displayInFlight = 0
private pumping = false
private lastHeartbeat = 0
private lastLease = 0
@@ -192,8 +204,15 @@ export class GlassesSession {
if (!identifier) return Promise.reject(new Error(`No ${role} device connected`))
const frames = protocol.frameMessage(payload, sid, flag, this.sequence++, this.limits[role])
const previous = this.writes.get(identifier) ?? Promise.resolve()
const work = previous.catch(() => {}).then(async () => {
for (const frame of frames) { this.check(generation); await this.transport.write(identifier, protocol.G2_WRITE, frame) }
// A failed write poisons this link's queue until reset: do not transmit
// later fragments after a hole in the image stream.
const work = previous.then(async () => {
for (const frame of frames) {
this.check(generation)
await this.transport.write(identifier, protocol.G2_WRITE, frame)
iosBleTraffic.recordWrite(frame.length)
}
iosBleTraffic.recordMessage()
})
// A whole message owns the write queue; heartbeat packets cannot split an image message.
this.writes.set(identifier, work)
@@ -312,6 +331,49 @@ export class GlassesSession {
if (this.timer !== null || this.pumping || this.state.phase !== 'connected') return
this.timer = setTimeout(() => { this.timer = null; void this.pump() }, delay)
}
private canSendDisplay(): boolean {
// Also bound completed frames retained behind a missing/out-of-order ACK.
return this.displayInFlight < DISPLAY_WINDOW_SIZE && !!(this.displaySending ||
(this.latest && !this.charging && this.displayFrames.length < DISPLAY_WINDOW_SIZE))
}
private fillDisplayWindow(generation: number): void {
while (this.canSendDisplay()) {
if (!this.displaySending) {
const packed = this.latest!; this.latest = null
// Compare with the last enqueued image, not the last ACKed one. In an
// A -> B -> A sequence, B may still be in flight when A is requested.
if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId, this.deflate)
const payload = delta ?? protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(packed)))
if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer')
if (delta) {
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
this.log(`Display bbox ${delta[3] * 4}x${delta[4] * 2}+${delta[1] * 4}+${delta[2] * 2} fid=${delta[5] | (delta[6] << 8)} (${payload.length} bytes)`)
} else this.log(`Display full 640x480 (${payload.length} bytes)`)
this.displaySending = { packed, payload, offset: 0, pending: 0 }
this.displayFrames.push(this.displaySending); this.lastEnqueued = packed
}
const frame = this.displaySending, offset = frame.offset
frame.offset += 3800; frame.pending++; this.displayInFlight++
if (frame.offset >= frame.payload.length) this.displaySending = null
// send() serializes whole messages on L, but request() waits for each
// magic's ACK independently. Fill the other slots without awaiting it.
void this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, frame.payload, offset), 'Display frame').then(() => {
if (generation !== this.generation) return
frame.pending--; this.displayInFlight--; this.lastHeartbeat = Date.now()
// Retire in submission order even if relayed ACKs arrive out of order.
// A frame counts only after every fragment has been acknowledged.
while (this.displayFrames.length) {
const completed = this.displayFrames[0]
if (completed.offset < completed.payload.length || completed.pending) break
this.displayFrames.shift(); this.displayed = completed.packed
this.state.frames++; iosBleTraffic.recordDisplayFrame(); this.onState({ ...this.state })
this.log(`Display frame ${this.state.frames} acknowledged (${completed.payload.length} bytes)`)
}
this.wake()
}).catch(error => { if (generation === this.generation) this.fail(error, true) })
}
}
private async pump(): Promise<void> {
if (this.pumping || this.state.phase !== 'connected') return
this.pumping = true; const generation = this.generation
@@ -327,19 +389,10 @@ export class GlassesSession {
const settings = await this.request('right', protocol.SID.settings, protocol.settingsQuery, 'Battery query')
this.applySettings(settings); this.lastSettings = Date.now()
}
const frame = this.latest; this.latest = null
if (frame && !this.charging && (!this.displayed || !frame.every((value, i) => value === this.displayed![i]))) {
const payload = protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(frame)))
if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer')
for (let offset = 0; offset < payload.length; offset += 3800) {
await this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, payload, offset), 'Display frame')
this.check(generation); this.lastHeartbeat = Date.now()
}
this.displayed = frame; this.state.frames++; this.onState({ ...this.state })
this.log(`Display frame ${this.state.frames} acknowledged (${payload.length} bytes)`)
} else if (frame && this.charging) this.latest = frame
this.check(generation)
this.fillDisplayWindow(generation)
} catch (error) { if (generation === this.generation) this.fail(error, true) }
finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.latest && !this.charging ? 0 : 1000) } }
finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.canSendDisplay() ? 0 : 1000) } }
}
private reset(): void {
++this.generation
@@ -352,6 +405,7 @@ export class GlassesSession {
const pending = [...this.pending.values()]; this.pending.clear()
for (const request of pending) { if (request.timer) clearTimeout(request.timer); request.reject(new Error('Connection ended')) }
this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null
this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0
}
private closeLinks(): void {
const ids = this.ids; this.ids = {}; this.limits = {}