forked from Blendman974/faceclaw
iOS porting: Image-sending performance: Pipelining, bounding-box updates
This commit is contained in:
1 parent
16764359bd
commit
79b6c276f8
15 files changed
+592
-57
No files matched your search
@@ -0,0 +1,34 @@
|
||||
import { concat, rle4 } from './ble-protocol'
|
||||
|
||||
/** Android BleImageOptimizer's mode-3 changed rectangle. Coordinates are in
|
||||
* units of 4 columns / 2 rows; pixels are top-down packed 4bpp, high nibble first.
|
||||
* null means identical, incomparable, or a full-screen box: use dedup/mode 6.
|
||||
*/
|
||||
export function buildBoundingBoxPayload(previous: Uint8Array | null, next: Uint8Array,
|
||||
width: number, height: number, frameId: number, deflate: (data: Uint8Array) => Uint8Array): Uint8Array | null {
|
||||
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0 ||
|
||||
width % 4 || height % 2 || width / 4 > 255 || height / 2 > 255 ||
|
||||
!previous || previous.length !== width * height / 2 || next.length !== previous.length) return null
|
||||
const stride = width / 2
|
||||
let minX = stride, maxX = -1, minY = height, maxY = -1
|
||||
for (let y = 0; y < height; y++) {
|
||||
const row = y * stride
|
||||
for (let x = 0; x < stride; x++) {
|
||||
if (previous[row + x] === next[row + x]) continue
|
||||
minX = Math.min(minX, x); maxX = Math.max(maxX, x)
|
||||
minY = Math.min(minY, y); maxY = y
|
||||
}
|
||||
}
|
||||
if (maxY < 0) return null
|
||||
const left = (minX * 2) & ~3, right = Math.min(width, ((maxX + 1) * 2 + 3) & ~3)
|
||||
const top = minY & ~1, bottom = Math.min(height, (maxY + 2) & ~1)
|
||||
const boxWidth = right - left, boxHeight = bottom - top
|
||||
if (boxWidth === width && boxHeight === height) return null
|
||||
const regionStride = boxWidth / 2, region = new Uint8Array(regionStride * boxHeight)
|
||||
for (let y = 0; y < boxHeight; y++) {
|
||||
const start = (top + y) * stride + left / 2
|
||||
region.set(next.subarray(start, start + regionStride), y * regionStride)
|
||||
}
|
||||
return concat(new Uint8Array([3, left / 4, top / 2, boxWidth / 4, boxHeight / 2,
|
||||
frameId & 255, (frameId >>> 8) & 255]), deflate(rle4(region)))
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/** Same units as Android FaceclawBleManager: protocol messages, framed write
|
||||
* bytes accepted by the transport, and fully acknowledged display updates.
|
||||
* Totals survive reconnects and the setting being switched off.
|
||||
*/
|
||||
export class BleTrafficCounters {
|
||||
private messages = 0
|
||||
private bytes = 0
|
||||
private frames = 0
|
||||
recordWrite(bytes: number): void { this.bytes += bytes }
|
||||
recordMessage(): void { this.messages++ }
|
||||
recordDisplayFrame(): void { this.frames++ }
|
||||
sample(): { messages: number; bytes: number; frames: number } {
|
||||
return { messages: this.messages, bytes: this.bytes, frames: this.frames }
|
||||
}
|
||||
}
|
||||
// The iOS BLE session and phone UI both run in the main isolate.
|
||||
export const iosBleTraffic = new BleTrafficCounters()
|
||||
+68
-14
@@ -1,4 +1,6 @@
|
||||
import * as protocol from './ble-protocol'
|
||||
import { iosBleTraffic } from './ble-traffic-counters'
|
||||
import { buildBoundingBoxPayload } from './ble-image-optimizer'
|
||||
import { hexToBytes } from '../util/hex-util'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
|
||||
@@ -20,6 +22,10 @@ export type SessionState = { phase: 'disconnected' | 'connecting' | 'connected'
|
||||
type PendingAck = { resolve: (message: protocol.ProtocolMessage) => void; reject: (error: Error) => void;
|
||||
timer: number | null; command: number; label: string }
|
||||
class AckTimeout extends Error {}
|
||||
// Match Android ConnectionOptions.WINDOW_SIZE. This bounds complete image
|
||||
// messages queued for writing or awaiting ACK, not individual BLE packets.
|
||||
const DISPLAY_WINDOW_SIZE = 3
|
||||
type DisplayFrame = { packed: Uint8Array; payload: Uint8Array; offset: number; pending: number }
|
||||
|
||||
/** G2 session independent of the phone OS. Images go to L, control to R, as on Android. */
|
||||
export class GlassesSession {
|
||||
@@ -43,6 +49,12 @@ export class GlassesSession {
|
||||
private addresses: SessionAddresses | null = null
|
||||
private latest: Uint8Array | null = null
|
||||
private displayed: Uint8Array | null = null
|
||||
private lastEnqueued: Uint8Array | null = null
|
||||
// Match Android: advance only for emitted deltas, skipping 0 / 0xffff.
|
||||
private nextImageFrameId = 1
|
||||
private displayFrames: DisplayFrame[] = []
|
||||
private displaySending: DisplayFrame | null = null
|
||||
private displayInFlight = 0
|
||||
private pumping = false
|
||||
private lastHeartbeat = 0
|
||||
private lastLease = 0
|
||||
@@ -192,8 +204,15 @@ export class GlassesSession {
|
||||
if (!identifier) return Promise.reject(new Error(`No ${role} device connected`))
|
||||
const frames = protocol.frameMessage(payload, sid, flag, this.sequence++, this.limits[role])
|
||||
const previous = this.writes.get(identifier) ?? Promise.resolve()
|
||||
const work = previous.catch(() => {}).then(async () => {
|
||||
for (const frame of frames) { this.check(generation); await this.transport.write(identifier, protocol.G2_WRITE, frame) }
|
||||
// A failed write poisons this link's queue until reset: do not transmit
|
||||
// later fragments after a hole in the image stream.
|
||||
const work = previous.then(async () => {
|
||||
for (const frame of frames) {
|
||||
this.check(generation)
|
||||
await this.transport.write(identifier, protocol.G2_WRITE, frame)
|
||||
iosBleTraffic.recordWrite(frame.length)
|
||||
}
|
||||
iosBleTraffic.recordMessage()
|
||||
})
|
||||
// A whole message owns the write queue; heartbeat packets cannot split an image message.
|
||||
this.writes.set(identifier, work)
|
||||
@@ -312,6 +331,49 @@ export class GlassesSession {
|
||||
if (this.timer !== null || this.pumping || this.state.phase !== 'connected') return
|
||||
this.timer = setTimeout(() => { this.timer = null; void this.pump() }, delay)
|
||||
}
|
||||
private canSendDisplay(): boolean {
|
||||
// Also bound completed frames retained behind a missing/out-of-order ACK.
|
||||
return this.displayInFlight < DISPLAY_WINDOW_SIZE && !!(this.displaySending ||
|
||||
(this.latest && !this.charging && this.displayFrames.length < DISPLAY_WINDOW_SIZE))
|
||||
}
|
||||
private fillDisplayWindow(generation: number): void {
|
||||
while (this.canSendDisplay()) {
|
||||
if (!this.displaySending) {
|
||||
const packed = this.latest!; this.latest = null
|
||||
// Compare with the last enqueued image, not the last ACKed one. In an
|
||||
// A -> B -> A sequence, B may still be in flight when A is requested.
|
||||
if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue
|
||||
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId, this.deflate)
|
||||
const payload = delta ?? protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(packed)))
|
||||
if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer')
|
||||
if (delta) {
|
||||
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
|
||||
this.log(`Display bbox ${delta[3] * 4}x${delta[4] * 2}+${delta[1] * 4}+${delta[2] * 2} fid=${delta[5] | (delta[6] << 8)} (${payload.length} bytes)`)
|
||||
} else this.log(`Display full 640x480 (${payload.length} bytes)`)
|
||||
this.displaySending = { packed, payload, offset: 0, pending: 0 }
|
||||
this.displayFrames.push(this.displaySending); this.lastEnqueued = packed
|
||||
}
|
||||
const frame = this.displaySending, offset = frame.offset
|
||||
frame.offset += 3800; frame.pending++; this.displayInFlight++
|
||||
if (frame.offset >= frame.payload.length) this.displaySending = null
|
||||
// send() serializes whole messages on L, but request() waits for each
|
||||
// magic's ACK independently. Fill the other slots without awaiting it.
|
||||
void this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, frame.payload, offset), 'Display frame').then(() => {
|
||||
if (generation !== this.generation) return
|
||||
frame.pending--; this.displayInFlight--; this.lastHeartbeat = Date.now()
|
||||
// Retire in submission order even if relayed ACKs arrive out of order.
|
||||
// A frame counts only after every fragment has been acknowledged.
|
||||
while (this.displayFrames.length) {
|
||||
const completed = this.displayFrames[0]
|
||||
if (completed.offset < completed.payload.length || completed.pending) break
|
||||
this.displayFrames.shift(); this.displayed = completed.packed
|
||||
this.state.frames++; iosBleTraffic.recordDisplayFrame(); this.onState({ ...this.state })
|
||||
this.log(`Display frame ${this.state.frames} acknowledged (${completed.payload.length} bytes)`)
|
||||
}
|
||||
this.wake()
|
||||
}).catch(error => { if (generation === this.generation) this.fail(error, true) })
|
||||
}
|
||||
}
|
||||
private async pump(): Promise<void> {
|
||||
if (this.pumping || this.state.phase !== 'connected') return
|
||||
this.pumping = true; const generation = this.generation
|
||||
@@ -327,19 +389,10 @@ export class GlassesSession {
|
||||
const settings = await this.request('right', protocol.SID.settings, protocol.settingsQuery, 'Battery query')
|
||||
this.applySettings(settings); this.lastSettings = Date.now()
|
||||
}
|
||||
const frame = this.latest; this.latest = null
|
||||
if (frame && !this.charging && (!this.displayed || !frame.every((value, i) => value === this.displayed![i]))) {
|
||||
const payload = protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(frame)))
|
||||
if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer')
|
||||
for (let offset = 0; offset < payload.length; offset += 3800) {
|
||||
await this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, payload, offset), 'Display frame')
|
||||
this.check(generation); this.lastHeartbeat = Date.now()
|
||||
}
|
||||
this.displayed = frame; this.state.frames++; this.onState({ ...this.state })
|
||||
this.log(`Display frame ${this.state.frames} acknowledged (${payload.length} bytes)`)
|
||||
} else if (frame && this.charging) this.latest = frame
|
||||
this.check(generation)
|
||||
this.fillDisplayWindow(generation)
|
||||
} catch (error) { if (generation === this.generation) this.fail(error, true) }
|
||||
finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.latest && !this.charging ? 0 : 1000) } }
|
||||
finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.canSendDisplay() ? 0 : 1000) } }
|
||||
}
|
||||
private reset(): void {
|
||||
++this.generation
|
||||
@@ -352,6 +405,7 @@ export class GlassesSession {
|
||||
const pending = [...this.pending.values()]; this.pending.clear()
|
||||
for (const request of pending) { if (request.timer) clearTimeout(request.timer); request.reject(new Error('Connection ended')) }
|
||||
this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null
|
||||
this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0
|
||||
}
|
||||
private closeLinks(): void {
|
||||
const ids = this.ids; this.ids = {}; this.limits = {}
|
||||
|
||||
Reference in new issue
Block a user