iOS porting: Image-sending performance: Pipelining, bounding-box updates

This commit is contained in:
James Babcock committed 2026-09-08 13:18:50 -07:00
1 parent 16764359bd
commit 79b6c276f8
15 files changed
+592 -57

No files matched your search

+34
View File
@@ -0,0 +1,34 @@
import { concat, rle4 } from './ble-protocol'
/** Android BleImageOptimizer's mode-3 changed rectangle. Coordinates are in
* units of 4 columns / 2 rows; pixels are top-down packed 4bpp, high nibble first.
* null means identical, incomparable, or a full-screen box: use dedup/mode 6.
*/
export function buildBoundingBoxPayload(previous: Uint8Array | null, next: Uint8Array,
width: number, height: number, frameId: number, deflate: (data: Uint8Array) => Uint8Array): Uint8Array | null {
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0 ||
width % 4 || height % 2 || width / 4 > 255 || height / 2 > 255 ||
!previous || previous.length !== width * height / 2 || next.length !== previous.length) return null
const stride = width / 2
let minX = stride, maxX = -1, minY = height, maxY = -1
for (let y = 0; y < height; y++) {
const row = y * stride
for (let x = 0; x < stride; x++) {
if (previous[row + x] === next[row + x]) continue
minX = Math.min(minX, x); maxX = Math.max(maxX, x)
minY = Math.min(minY, y); maxY = y
}
}
if (maxY < 0) return null
const left = (minX * 2) & ~3, right = Math.min(width, ((maxX + 1) * 2 + 3) & ~3)
const top = minY & ~1, bottom = Math.min(height, (maxY + 2) & ~1)
const boxWidth = right - left, boxHeight = bottom - top
if (boxWidth === width && boxHeight === height) return null
const regionStride = boxWidth / 2, region = new Uint8Array(regionStride * boxHeight)
for (let y = 0; y < boxHeight; y++) {
const start = (top + y) * stride + left / 2
region.set(next.subarray(start, start + regionStride), y * regionStride)
}
return concat(new Uint8Array([3, left / 4, top / 2, boxWidth / 4, boxHeight / 2,
frameId & 255, (frameId >>> 8) & 255]), deflate(rle4(region)))
}
+17
View File
@@ -0,0 +1,17 @@
/** Same units as Android FaceclawBleManager: protocol messages, framed write
* bytes accepted by the transport, and fully acknowledged display updates.
* Totals survive reconnects and the setting being switched off.
*/
export class BleTrafficCounters {
private messages = 0
private bytes = 0
private frames = 0
recordWrite(bytes: number): void { this.bytes += bytes }
recordMessage(): void { this.messages++ }
recordDisplayFrame(): void { this.frames++ }
sample(): { messages: number; bytes: number; frames: number } {
return { messages: this.messages, bytes: this.bytes, frames: this.frames }
}
}
// The iOS BLE session and phone UI both run in the main isolate.
export const iosBleTraffic = new BleTrafficCounters()
+68 -14
View File
@@ -1,4 +1,6 @@
import * as protocol from './ble-protocol'
import { iosBleTraffic } from './ble-traffic-counters'
import { buildBoundingBoxPayload } from './ble-image-optimizer'
import { hexToBytes } from '../util/hex-util'
import { deviceAddressError } from './ios-peripheral-identity'
@@ -20,6 +22,10 @@ export type SessionState = { phase: 'disconnected' | 'connecting' | 'connected'
type PendingAck = { resolve: (message: protocol.ProtocolMessage) => void; reject: (error: Error) => void;
timer: number | null; command: number; label: string }
class AckTimeout extends Error {}
// Match Android ConnectionOptions.WINDOW_SIZE. This bounds complete image
// messages queued for writing or awaiting ACK, not individual BLE packets.
const DISPLAY_WINDOW_SIZE = 3
type DisplayFrame = { packed: Uint8Array; payload: Uint8Array; offset: number; pending: number }
/** G2 session independent of the phone OS. Images go to L, control to R, as on Android. */
export class GlassesSession {
@@ -43,6 +49,12 @@ export class GlassesSession {
private addresses: SessionAddresses | null = null
private latest: Uint8Array | null = null
private displayed: Uint8Array | null = null
private lastEnqueued: Uint8Array | null = null
// Match Android: advance only for emitted deltas, skipping 0 / 0xffff.
private nextImageFrameId = 1
private displayFrames: DisplayFrame[] = []
private displaySending: DisplayFrame | null = null
private displayInFlight = 0
private pumping = false
private lastHeartbeat = 0
private lastLease = 0
@@ -192,8 +204,15 @@ export class GlassesSession {
if (!identifier) return Promise.reject(new Error(`No ${role} device connected`))
const frames = protocol.frameMessage(payload, sid, flag, this.sequence++, this.limits[role])
const previous = this.writes.get(identifier) ?? Promise.resolve()
const work = previous.catch(() => {}).then(async () => {
for (const frame of frames) { this.check(generation); await this.transport.write(identifier, protocol.G2_WRITE, frame) }
// A failed write poisons this link's queue until reset: do not transmit
// later fragments after a hole in the image stream.
const work = previous.then(async () => {
for (const frame of frames) {
this.check(generation)
await this.transport.write(identifier, protocol.G2_WRITE, frame)
iosBleTraffic.recordWrite(frame.length)
}
iosBleTraffic.recordMessage()
})
// A whole message owns the write queue; heartbeat packets cannot split an image message.
this.writes.set(identifier, work)
@@ -312,6 +331,49 @@ export class GlassesSession {
if (this.timer !== null || this.pumping || this.state.phase !== 'connected') return
this.timer = setTimeout(() => { this.timer = null; void this.pump() }, delay)
}
private canSendDisplay(): boolean {
// Also bound completed frames retained behind a missing/out-of-order ACK.
return this.displayInFlight < DISPLAY_WINDOW_SIZE && !!(this.displaySending ||
(this.latest && !this.charging && this.displayFrames.length < DISPLAY_WINDOW_SIZE))
}
private fillDisplayWindow(generation: number): void {
while (this.canSendDisplay()) {
if (!this.displaySending) {
const packed = this.latest!; this.latest = null
// Compare with the last enqueued image, not the last ACKed one. In an
// A -> B -> A sequence, B may still be in flight when A is requested.
if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId, this.deflate)
const payload = delta ?? protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(packed)))
if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer')
if (delta) {
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
this.log(`Display bbox ${delta[3] * 4}x${delta[4] * 2}+${delta[1] * 4}+${delta[2] * 2} fid=${delta[5] | (delta[6] << 8)} (${payload.length} bytes)`)
} else this.log(`Display full 640x480 (${payload.length} bytes)`)
this.displaySending = { packed, payload, offset: 0, pending: 0 }
this.displayFrames.push(this.displaySending); this.lastEnqueued = packed
}
const frame = this.displaySending, offset = frame.offset
frame.offset += 3800; frame.pending++; this.displayInFlight++
if (frame.offset >= frame.payload.length) this.displaySending = null
// send() serializes whole messages on L, but request() waits for each
// magic's ACK independently. Fill the other slots without awaiting it.
void this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, frame.payload, offset), 'Display frame').then(() => {
if (generation !== this.generation) return
frame.pending--; this.displayInFlight--; this.lastHeartbeat = Date.now()
// Retire in submission order even if relayed ACKs arrive out of order.
// A frame counts only after every fragment has been acknowledged.
while (this.displayFrames.length) {
const completed = this.displayFrames[0]
if (completed.offset < completed.payload.length || completed.pending) break
this.displayFrames.shift(); this.displayed = completed.packed
this.state.frames++; iosBleTraffic.recordDisplayFrame(); this.onState({ ...this.state })
this.log(`Display frame ${this.state.frames} acknowledged (${completed.payload.length} bytes)`)
}
this.wake()
}).catch(error => { if (generation === this.generation) this.fail(error, true) })
}
}
private async pump(): Promise<void> {
if (this.pumping || this.state.phase !== 'connected') return
this.pumping = true; const generation = this.generation
@@ -327,19 +389,10 @@ export class GlassesSession {
const settings = await this.request('right', protocol.SID.settings, protocol.settingsQuery, 'Battery query')
this.applySettings(settings); this.lastSettings = Date.now()
}
const frame = this.latest; this.latest = null
if (frame && !this.charging && (!this.displayed || !frame.every((value, i) => value === this.displayed![i]))) {
const payload = protocol.concat(new Uint8Array([6]), this.deflate(protocol.rle4(frame)))
if (payload.length > 165888) throw new Error('Compressed frame exceeds the glasses image buffer')
for (let offset = 0; offset < payload.length; offset += 3800) {
await this.request('left', protocol.SID.hub, magic => protocol.imageFragment(magic, payload, offset), 'Display frame')
this.check(generation); this.lastHeartbeat = Date.now()
}
this.displayed = frame; this.state.frames++; this.onState({ ...this.state })
this.log(`Display frame ${this.state.frames} acknowledged (${payload.length} bytes)`)
} else if (frame && this.charging) this.latest = frame
this.check(generation)
this.fillDisplayWindow(generation)
} catch (error) { if (generation === this.generation) this.fail(error, true) }
finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.latest && !this.charging ? 0 : 1000) } }
finally { if (generation === this.generation) { this.pumping = false; this.schedule(this.canSendDisplay() ? 0 : 1000) } }
}
private reset(): void {
++this.generation
@@ -352,6 +405,7 @@ export class GlassesSession {
const pending = [...this.pending.values()]; this.pending.clear()
for (const request of pending) { if (request.timer) clearTimeout(request.timer); request.reject(new Error('Connection ended')) }
this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null
this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0
}
private closeLinks(): void {
const ids = this.ids; this.ids = {}; this.limits = {}
+3
View File
@@ -0,0 +1,3 @@
import { iosBleTraffic } from '../g2/ble-traffic-counters'
export type { BleTrafficSample } from './ble-traffic'
export function sampleBleTraffic(): { messages: number; bytes: number; frames: number } { return iosBleTraffic.sample() }
+27
View File
@@ -0,0 +1,27 @@
import type { BleTrafficSample } from '../native/ble-traffic'
/** Shared Android/iOS five-second traffic readout. Poll once per second. */
export class BleBandwidthMeter {
private history: Array<{ atMs: number; bytes: number; frames: number }> = []
reset(): void { this.history = [] }
sample(sample: BleTrafficSample, atMs: number): string {
const previous = this.history[this.history.length - 1]
if (previous && (atMs <= previous.atMs || sample.bytes < previous.bytes || sample.frames < previous.frames)) this.reset()
this.history.push({ atMs, bytes: sample.bytes, frames: sample.frames })
while (this.history.length && this.history[0]!.atMs < atMs - 5000) this.history.shift()
let label = `BLE sent: ${formatCount(sample.messages)} messages, ${formatCount(sample.bytes)} bytes`
const oldest = this.history[0]!, elapsedSec = (atMs - oldest.atMs) / 1000
if (elapsedSec > 0) {
const byteDelta = sample.bytes - oldest.bytes, frameDelta = sample.frames - oldest.frames
label += ` · ${formatByteRate(byteDelta / elapsedSec)}, ${(frameDelta / elapsedSec).toFixed(1)} fps`
if (frameDelta > 0) label += `, ${formatCount(byteDelta / frameDelta)} B/frame`
}
return label
}
}
function formatCount(value: number): string { return String(Math.round(value)).replace(/\B(?=(\d{3})+(?!\d))/g, ',') }
function formatByteRate(bytesPerSec: number): string {
if (bytesPerSec >= 1e6) return (bytesPerSec / 1e6).toFixed(2) + ' MB/s'
if (bytesPerSec >= 1e3) return (bytesPerSec / 1e3).toFixed(1) + ' kB/s'
return Math.round(bytesPerSec) + ' B/s'
}
+35 -8
View File
@@ -3,7 +3,9 @@ import { createConfigureDevicesPage } from './config-page.ios'
import { ALL_APPS } from '../apps/all-apps'
import { IosPreviewController } from '../g2/ios-preview-controller'
import { PhoneGestureRecognizer } from './phone-gestures'
import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings'
import { BleBandwidthMeter } from './ble-bandwidth-meter'
import { sampleBleTraffic } from '../native/ble-traffic'
import { onAnySettingChanged, previewColorSetting, showBleBandwidthSetting } from '../ui/dashboard-settings'
import { loadDeviceAddresses } from '../g2/device-addresses'
import { deviceAddressError } from '../g2/ios-peripheral-identity'
@@ -11,7 +13,7 @@ export function createMainPage(): Page {
const page = new Page()
page.actionBarHidden = true
const root = new GridLayout()
root.rows = 'auto,*'
root.rows = 'auto,*,auto'
page.content = root
const header = new GridLayout()
header.columns = '*,auto'; header.padding = '12 20'
@@ -127,20 +129,44 @@ export function createMainPage(): Page {
view.padding = '16 8'; view.fontSize = 14
GridLayout.setColumn(view, index); footer.addChild(view)
}
const bandwidth = new Label()
bandwidth.accessibilityIdentifier = 'ble-bandwidth-indicator'
bandwidth.fontSize = 10; bandwidth.textWrap = true; bandwidth.padding = '2 8 4'
bandwidth.color = new Color('#68756c'); bandwidth.isUserInteractionEnabled = false
bandwidth.visibility = 'collapse'
GridLayout.setRow(bandwidth, 2); root.addChild(bandwidth)
const bandwidthMeter = new BleBandwidthMeter()
let bandwidthTimer: ReturnType<typeof setInterval> | null = null
let foreground = false
function stopBandwidth(): void {
if (bandwidthTimer !== null) clearInterval(bandwidthTimer)
bandwidthTimer = null; bandwidthMeter.reset()
}
function syncBandwidth(): void {
if (!foreground) { stopBandwidth(); return }
const enabled = showBleBandwidthSetting.get()
bandwidth.visibility = enabled ? 'visible' : 'collapse'
if (!enabled) { stopBandwidth(); return }
if (bandwidthTimer !== null) return
const refresh = () => { bandwidth.text = bandwidthMeter.sample(sampleBleTraffic(), Date.now()) }
refresh(); bandwidthTimer = setInterval(refresh, 1000)
}
let lastLayout = ''
function layout(): void {
const size = root.getActualSize()
if (size.width <= 0 || size.height <= 0) return
const key = `${size.width}:${size.height}:${selectedTab}`
const metricsHeight = bandwidth.visibility === 'visible' ? bandwidth.getActualSize().height : 0
const availableHeight = size.height - metricsHeight
const key = `${size.width}:${availableHeight}:${selectedTab}`
if (key === lastLayout) return
lastLayout = key; padInput.cancel(); mirrorInput.cancel()
const landscape = size.width > size.height
body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*'
GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0)
const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width
const width = landscape ? Math.min(size.width - 320, (availableHeight - 64) * 4 / 3) : size.width
mirror.width = width; mirror.height = width * 3 / 4
pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32)
pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200)))
pad.height = Math.max(120, Math.min(250, availableHeight - (landscape ? 180 : width * 3 / 4 + 200)))
}
// iOS emits layoutChanged inside its layout pass. Defer mutations so
// NativeScript does not clear our new layout request at the end of that pass.
@@ -150,17 +176,18 @@ export function createMainPage(): Page {
layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0)
})
const pause = () => {
foreground = false; stopBandwidth()
if (layoutTimer !== null) clearTimeout(layoutTimer)
layoutTimer = null
padInput.cancel(); mirrorInput.cancel(); controller.pause()
}
const resume = () => controller.resume()
const resume = () => { foreground = true; controller.resume(); syncBandwidth() }
let offSettings: (() => void) | null = null
let firstLoad = true
page.on('loaded', () => {
Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume)
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel })
selectTab(selectedTab); controller.resume()
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel; syncBandwidth() })
selectTab(selectedTab); resume()
if (firstLoad) {
firstLoad = false
// Match Android's reconnect-on-launch behavior once addresses are saved.
+4 -33
View File
@@ -1,3 +1,4 @@
import { BleBandwidthMeter } from "./ble-bandwidth-meter";
import {
Application,
Dialogs,
@@ -1268,7 +1269,7 @@ export class MainViewModel extends Observable {
private _bleBandwidthLabel = "";
private bleBandwidthTimer: ReturnType<typeof setInterval> | null = null;
// Recent counter samples, one per poll tick, for the windowed rates.
private bleRateHistory: Array<{ atMs: number; bytes: number; frames: number }> = [];
private readonly bleBandwidthMeter = new BleBandwidthMeter();
get bleBandwidthVisibility(): "visible" | "collapse" {
return showBleBandwidthSetting.get() ? "visible" : "collapse";
@@ -1297,29 +1298,13 @@ export class MainViewModel extends Observable {
this.bleBandwidthTimer = null;
}
// Don't let a later re-enable compute a rate across the disabled gap.
this.bleRateHistory = [];
this.bleBandwidthMeter.reset();
}
private refreshBleBandwidth(): void {
let label: string;
try {
const sample = sampleBleTraffic();
const atMs = Date.now();
this.bleRateHistory.push({ atMs, bytes: sample.bytes, frames: sample.frames });
while (this.bleRateHistory.length > 0 && this.bleRateHistory[0]!.atMs < atMs - BLE_RATE_WINDOW_MS) {
this.bleRateHistory.shift();
}
label = `BLE sent: ${formatCount(sample.messages)} messages, ${formatCount(sample.bytes)} bytes`;
const oldest = this.bleRateHistory[0]!;
const elapsedSec = (atMs - oldest.atMs) / 1000;
if (elapsedSec > 0) {
const byteDelta = sample.bytes - oldest.bytes;
const frameDelta = sample.frames - oldest.frames;
label += ` · ${formatByteRate(byteDelta / elapsedSec)}, ${(frameDelta / elapsedSec).toFixed(1)} fps`;
if (frameDelta > 0) {
label += `, ${formatCount(byteDelta / frameDelta)} B/frame`;
}
}
label = this.bleBandwidthMeter.sample(sampleBleTraffic(), Date.now());
} catch (error) {
label = `BLE sent: ${this.formatError(error)}`;
}
@@ -1342,20 +1327,6 @@ export class MainViewModel extends Observable {
}
}
/** 1234567 -> "1,234,567"; kept exact rather than rounded so growth is visible at a glance. */
function formatCount(value: number): string {
return String(Math.round(value)).replace(/\B(?=(\d{3})+(?!\d))/g, ",");
}
/** How far back the BLE bandwidth indicator's rates look. */
const BLE_RATE_WINDOW_MS = 5000;
function formatByteRate(bytesPerSec: number): string {
if (bytesPerSec >= 1e6) return (bytesPerSec / 1e6).toFixed(2) + " MB/s";
if (bytesPerSec >= 1e3) return (bytesPerSec / 1e3).toFixed(1) + " kB/s";
return Math.round(bytesPerSec) + " B/s";
}
/** NativeScript swipe direction -> the watch-scheme directional gesture. */
function swipeKind(direction: SwipeDirection): "swipe-up" | "swipe-down" | "swipe-left" | "swipe-right" {
switch (direction) {
+1 -1
View File
@@ -444,7 +444,7 @@ export const showBleBandwidthSetting = new ConfigSettingBoolean({
storageKey: "developer.showBleBandwidth",
defaultValue: false,
description:
"Show a running total of Bluetooth messages and bytes sent, at the bottom of the phone app's main screen.",
"Show Bluetooth messages and bytes sent, throughput, acknowledged display fps, and bytes per frame at the bottom of the phone screen. Rates use a five-second window; bytes include control traffic and protocol framing.",
});
export type RingConnectionMode = "glasses" | "direct";
+2 -1
View File
@@ -233,8 +233,9 @@ function settingsSections(): SettingsSection[] {
},
];
if (!global.isIOS) return sections;
const deferred = new Set(["Assistant", "Navigate", "Watch", "Developer"]);
const deferred = new Set(["Assistant", "Navigate", "Watch"]);
return sections.map(section => {
if (section.label === "Developer") return { ...section, items: [toggleSettingMenuItem(showBleBandwidthSetting)] };
if (section.label === "Voice") return { label: "Voice", items: [{
label: "On-device dictation (Apple)", disabled: true, onSelect: () => {},
description: "Uses the glasses microphone and your iPhone's speech language. No API key needed. Open Voice from the phone or glasses menu, speak, then click to finish and send the text into an app.",