Multi-rect optimization and pipelining

This commit is contained in:
James Babcock committed 2026-07-15 12:39:19 -07:00
1 parent ddec2a7b39
commit 0d4905ff28
3 files changed
+316 -33

No files matched your search

@@ -109,6 +109,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private String displayedFingerprint = "";
private byte[] displayedBmp = new byte[0];
// The frame the firmware shadow will hold once the current image pipeline
// drains: the most recently ENQUEUED image, which is the correct base for the
// next delta when frames are pipelined (displayedBmp only reflects the last
// ACKED frame and lags behind). Set at enqueue; cleared whenever the image
// pipeline is cleared (clearAllMessagesLocked / clearMessagesOfKindLocked
// "image"), so it is only ever read while it holds a valid current-session base.
private byte[] lastEnqueuedImageBmp = new byte[0];
private String lastEnqueuedFingerprint = "";
private final Map<Integer, BleImageOptimizer.ImageUpdateStats> imageUpdateStats = new HashMap<>();
private final Object desiredTilesLock = new Object();
@@ -328,9 +336,18 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
appContext, java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height);
}
/** Show or hide a compositor surface; takes effect at the next composite. */
/**
* Show or hide a compositor surface, immediately submitting the resulting
* frame. Recompositing here (rather than waiting for the next surface
* update) is what makes a just-foregrounded window's retained frame
* actually appear — otherwise a static window (e.g. the terminal hub) whose
* frame landed while briefly hidden would stay blank until its next repaint.
*/
public void setSurfaceVisible(String id, boolean visible) {
compositor.setSurfaceVisible(id, visible);
SurfaceCompositor.Composite composite = compositor.composite();
byte[] bmp = BmpUtil.build4bppBmp(composite.gray, composite.width, composite.height);
storeDesiredComposite(composite, bmp, 0, 0);
}
/**
@@ -956,11 +973,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
handleAckTimeoutLocked(oldest);
return 0;
}
if (sessionReady
if (connectionOptions.WINDOW_SIZE <= 1
&& sessionReady
&& prewrittenMessage == null
&& !pendingMessages.isEmpty()
&& canPrewriteCandidate(pendingMessages.peekFirst())
&& !shouldBlockPrewriteForHeartbeatLocked(now)) {
// Only the serial (window==1) path pre-sends the all-but-last
// packet; with a real window we just send the next message fully.
messageToPrewrite = pendingMessages.peekFirst();
}
}
@@ -1004,24 +1024,30 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
enqueueFirmwareDebugFlagsLocked();
}
// Up to WINDOW_SIZE messages may be in flight at once (full
// pipelining); a slot frees when an ack arrives.
boolean windowHasRoom = inFlightMessages.size() < Math.max(1, connectionOptions.WINDOW_SIZE);
// A frame ready to send right now: don't inject a fresh
// heartbeat in front of it (the image's own ack resets the
// firmware heartbeat timer, so the heartbeat is redundant).
boolean imageWaiting = !shutdownRequested && fixedLayoutCreated && warmedUp
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()
&& !hasPendingOrInflightKindLocked("heartbeat")
&& now >= imageRetryAfterMs
&& !getDesiredFingerprint().equals(displayedFingerprint);
&& !getDesiredFingerprint().equals(lastEnqueuedFingerprint);
if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) {
return ConnectionOptions.IDLE_SLEEP_MS;
}
if (messageToPrewrite == null && sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) {
OutboundMessage pending = pendingMessages.peekFirst();
if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty()) {
messageToWrite = pendingMessages.removeFirst();
Log.i(TAG, "sending pending message: " + messageToWrite.label);
} else if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated && warmedUp && pendingMessages.isEmpty() && inFlightMessages.isEmpty()
} else if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated && warmedUp
&& windowHasRoom && !hasPendingImageLocked()
&& now >= imageRetryAfterMs
&& !getDesiredFingerprint().equals(displayedFingerprint)) {
&& !getDesiredFingerprint().equals(lastEnqueuedFingerprint)) {
// Enqueue the next frame's delta against lastEnqueuedImageBmp
// (what the shadow will be), so it can pipeline behind an
// image still awaiting its ack.
Log.i(TAG, "Enqueued image update");
enqueueDesiredImageLocked();
return 0;
@@ -1064,7 +1090,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
int frameIdToFinish = 0;
synchronized (lock) {
synchronized (desiredTilesLock) {
if (desiredFrameId != 0 && !displayedFingerprint.isEmpty() && desiredFingerprint.equals(displayedFingerprint)) {
if (desiredFrameId != 0 && !lastEnqueuedFingerprint.isEmpty() && desiredFingerprint.equals(lastEnqueuedFingerprint)) {
frameIdToFinish = desiredFrameId;
desiredFrameId = 0;
}
@@ -1483,25 +1509,49 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if (bmp == null) {
bmp = new byte[0];
}
if (Arrays.equals(bmp, displayedBmp)) {
displayedFingerprint = fingerprint;
if (Arrays.equals(bmp, lastEnqueuedImageBmp)) {
lastEnqueuedFingerprint = fingerprint;
finishFrame(frameId, "discarded: image content identical to displayed");
return;
}
FrameTimings.getInstance().spanStart(frameId, "compress-and-plan");
// Incremental (mode 3 bounding box) update when the previous frame is
// known-displayed; displayedBmp is cleared on connect/reset/timeout
// paths, so a non-empty value means display state is trusted.
// Incremental (mode 3 bounding box) update against the last ENQUEUED frame
// (the base the firmware shadow will hold when this update is applied).
// lastEnqueuedImageBmp is cleared whenever the image pipeline is cleared, so
// a non-empty value means the display base is trusted.
byte[] incrementalPayload = null;
BleImageOptimizer.IncrementalPlan incrementalPlan = null;
if (connectionOptions.INCREMENTAL_FRAMES && displayedBmp.length > 0) {
incrementalPlan = BleImageOptimizer.buildIncrementalImagePayload(displayedBmp, bmp, nextImageFrameId);
if (incrementalPlan != null) {
incrementalPayload = incrementalPlan.payload;
String incrementalLog = null;
if (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedImageBmp.length > 0) {
int baseFid = nextImageFrameId;
BleImageOptimizer.IncrementalPlan single =
BleImageOptimizer.buildIncrementalImagePayload(lastEnqueuedImageBmp, bmp, baseFid);
if (single != null) {
incrementalPayload = single.payload;
// advance only when a delta is actually emitted, so consecutive
// deltas carry consecutive ids (CFW skip/reorder detection)
nextImageFrameId = nextImageFrameId >= 0xfffe ? 1 : nextImageFrameId + 1;
incrementalLog = "incremental update bbox="
+ ((single.payload[3] & 0xff) * 4) + "x" + ((single.payload[4] & 0xff) * 2)
+ "+" + ((single.payload[1] & 0xff) * 4) + "+" + ((single.payload[2] & 0xff) * 2)
+ " changed=" + single.changedBytes + "/" + single.boxBytes + "B"
+ " clusters=" + single.clusterCount;
// When the bounding box spans multiple clusters or is sizeable, try
// splitting into tight rects (CFW mode-8). Only replace the single
// box if the multi-rect message is actually smaller on the wire.
if (connectionOptions.MULTI_RECT_FRAMES
&& (single.clusterCount > 1 || single.payload.length > ConnectionOptions.MULTI_RECT_MIN_PAYLOAD)) {
BleImageOptimizer.MultiRectPlan multi = BleImageOptimizer.buildMultiRectImagePayload(
lastEnqueuedImageBmp, bmp, baseFid, ConnectionOptions.MULTI_RECT_MAX_RECTS);
if (multi != null && multi.payload.length < single.payload.length) {
incrementalPayload = multi.payload;
nextImageFrameId = multi.nextFid; // rectCount fids consumed
incrementalLog = "multi-rect update n=" + multi.rectCount
+ " covered=" + multi.coveredBytes + "B"
+ " payload=" + multi.payload.length + "B (vs bbox " + single.payload.length + "B)";
}
}
}
}
BleImageOptimizer.TileImagePlan plan = incrementalPayload != null
@@ -1509,16 +1559,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
: new BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, bmp, nextMapSessionId());
plan.fragments = BleImageOptimizer.planImageFragments(plan.payload, ConnectionOptions.IMAGE_FRAGMENT_SIZE);
FrameTimings.getInstance().spanEnd(frameId, "compress-and-plan");
if (incrementalPayload != null) {
// changed/box are packed-byte counts (2px each); clusters>1 with
// changed<<box means distant small edits merged into an oversized
// box (see BleImageOptimizer.IncrementalPlan) — the prime suspect
// for a delta spilling into a second, serialized BLE message.
FrameTimings.getInstance().log(frameId, "incremental update bbox="
+ ((incrementalPayload[3] & 0xff) * 4) + "x" + ((incrementalPayload[4] & 0xff) * 2)
+ "+" + ((incrementalPayload[1] & 0xff) * 4) + "+" + ((incrementalPayload[2] & 0xff) * 2)
+ " changed=" + incrementalPlan.changedBytes + "/" + incrementalPlan.boxBytes + "B"
+ " clusters=" + incrementalPlan.clusterCount);
if (incrementalLog != null) {
FrameTimings.getInstance().log(frameId, incrementalLog);
}
int updateId = nextImageUpdateId++;
@@ -1528,6 +1570,11 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
BleProtocol.ImageFragment fragment = plan.fragments.get(i);
enqueueImageFragmentLocked(plan, fragment, fingerprint, updateId, i + 1, messageCount, true);
}
// This frame is now the base for the next delta (it will be the firmware
// shadow once applied), even though it hasn't been acked yet — that is what
// lets the next frame pipeline behind it. plan.bmp is the full frame.
lastEnqueuedImageBmp = BmpUtil.copyTileBmp(plan.bmp);
lastEnqueuedFingerprint = fingerprint;
FrameTimings.getInstance().log(frameId, "queued image update#" + updateId
+ " messages=" + messageCount + " payload=" + plan.payload.length + "B");
@@ -1750,11 +1797,31 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
magicPool.release(message.sid, message.magic, message.label, "cleared inflight " + kind);
}
}
if ("image".equals(kind)) {
// The image pipeline was flushed (e.g. ack timeout -> keyframe resync):
// drop the pipelined delta base so the next image is a full keyframe.
lastEnqueuedImageBmp = new byte[0];
lastEnqueuedFingerprint = "";
}
}
private void clearAllMessagesLocked(String reason) {
clearPendingMessagesLocked(reason);
clearInFlightMessagesLocked(reason);
// The image pipeline is gone: drop the pipelined delta base so the next
// image is a full keyframe rather than a delta onto a stale base.
lastEnqueuedImageBmp = new byte[0];
lastEnqueuedFingerprint = "";
}
/** Any image update whose fragments are still queued (not yet sent). */
private boolean hasPendingImageLocked() {
for (OutboundMessage message : pendingMessages) {
if ("image".equals(message.kind)) {
return true;
}
}
return false;
}
private void clearPendingMessagesLocked(String reason) {
@@ -200,6 +200,20 @@ public final class BleImageOptimizer {
return null; // whole screen changed; full update is strictly better
}
byte[] out = encodeMode3Rect(next, stride, left, top, boxWidth, boxHeight, frameId);
return new IncrementalPlan(out, changedBytes, (boxWidth >> 1) * boxHeight, clusterCount);
}
/**
* Encode one CFW mode-3 rectangle delta:
* [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][zlib box pixels]
* left/width must be multiples of 4 (=> left>>1, width>>1 are whole bytes), top/
* height multiples of 2. The box pixels are top-down rows of `next` (4bpp packed,
* width>>1 bytes/row). Shared by the single-bbox path and each rect of a mode-8
* multi-rect batch.
*/
private static byte[] encodeMode3Rect(byte[] next, int stride, int left, int top,
int boxWidth, int boxHeight, int fid) {
int regionStride = boxWidth >> 1;
byte[] region = new byte[regionStride * boxHeight];
for (int y = 0; y < boxHeight; y++) {
@@ -212,10 +226,192 @@ public final class BleImageOptimizer {
out[2] = (byte) (top / 2);
out[3] = (byte) (boxWidth / 4);
out[4] = (byte) (boxHeight / 2);
out[5] = (byte) (frameId & 0xff); // fid_lo
out[6] = (byte) ((frameId >> 8) & 0xff); // fid_hi
out[5] = (byte) (fid & 0xff); // fid_lo
out[6] = (byte) ((fid >> 8) & 0xff); // fid_hi
System.arraycopy(compressed, 0, out, 7, compressed.length);
return new IncrementalPlan(out, changedBytes, region.length, clusterCount);
return out;
}
/**
* A CFW mode-8 multi-segment payload (`[8][count]` then per-rect
* `[len16][mode-3 submsg]`) carrying several tight rectangle deltas instead of
* one bounding box, plus diagnostics. Each rect is applied to the firmware
* shadow with the panel push deferred, then presented once. Used when the change
* splits into regions whose bounding box wastes bytes (distant edits, or a sparse
* box); the caller falls back to the single-rect path when this returns null or
* isn't smaller.
*/
public static final class MultiRectPlan {
public final byte[] payload;
public final int rectCount;
public final int coveredBytes; // sum of rect region bytes, pre-compression
public final int nextFid; // fid to resume from (rectCount consumed)
MultiRectPlan(byte[] payload, int rectCount, int coveredBytes, int nextFid) {
this.payload = payload;
this.rectCount = rectCount;
this.coveredBytes = coveredBytes;
this.nextFid = nextFid;
}
}
// Break-even tuning for the rect splitter. Splitting adds ~15 fixed bytes per
// rect (seglen + mode-3 header + fid + zlib framing) and loses cross-rect
// dictionary sharing, so only split across gaps big enough to pay for that.
// Gaps are in the changed-mask's native units: whole 4bpp bytes (2px) for
// columns, rows for the vertical bands. Kept above the 4px/2px box alignment so
// aligned rects never overlap.
private static final int MULTI_RECT_V_GAP_ROWS = 4;
private static final int MULTI_RECT_H_GAP_BYTES = 6;
/**
* Build a mode-8 multi-rect payload for the change between two same-size 4bpp
* frames, assigning consecutive frame ids starting at fidStart (each rect needs a
* distinct fid — the CFW skips duplicate fids). Returns null when the change is a
* single region (use the single-rect path), when it fragments past maxRects, or
* when the frames aren't comparable.
*/
public static MultiRectPlan buildMultiRectImagePayload(byte[] previousBmp, byte[] newBmp, int fidStart, int maxRects) {
int width = BmpUtil.readBmpWidth(newBmp);
int height = BmpUtil.readBmpHeight(newBmp);
if (width <= 0 || height <= 0
|| width != BmpUtil.readBmpWidth(previousBmp)
|| height != BmpUtil.readBmpHeight(previousBmp)) {
return null;
}
byte[] previous = BmpUtil.pack4bppFromBmp(previousBmp);
byte[] next = BmpUtil.pack4bppFromBmp(newBmp);
int stride = (width + 1) >> 1;
if (previous.length != stride * height || next.length != stride * height) {
return null;
}
List<int[]> rects = computeSplitRects(previous, next, stride, width, height, maxRects);
if (rects == null || rects.size() < 2) {
return null; // single region (or too fragmented): caller uses the bbox path
}
List<byte[]> subs = new ArrayList<>(rects.size());
int coveredBytes = 0;
int fid = fidStart;
int total = 2; // [8][count]
for (int[] r : rects) {
byte[] sub = encodeMode3Rect(next, stride, r[0], r[1], r[2], r[3], fid);
subs.add(sub);
coveredBytes += (r[2] >> 1) * r[3];
total += 2 + sub.length; // len16 + submsg
fid = (fid >= 0xfffe) ? 1 : fid + 1;
}
byte[] out = new byte[total];
out[0] = 8;
out[1] = (byte) rects.size();
int pos = 2;
for (byte[] sub : subs) {
out[pos] = (byte) (sub.length & 0xff);
out[pos + 1] = (byte) ((sub.length >> 8) & 0xff);
pos += 2;
System.arraycopy(sub, 0, out, pos, sub.length);
pos += sub.length;
}
return new MultiRectPlan(out, rects.size(), coveredBytes, fid);
}
/**
* Split the changed region between two packed 4bpp frames into a small set of
* aligned rectangles that together cover every changed pixel: maximal vertical
* bands of changed rows (merging gaps < V_GAP), each split into horizontal column
* clusters (merging gaps < H_GAP), with each cluster's rows tightened to where it
* actually changes. Rects are pixel-space {left,top,width,height}, left/width
* aligned to 4px and top/height to 2px. Returns null if there's no change or the
* split exceeds maxRects (caller falls back to a single bounding box).
*/
private static List<int[]> computeSplitRects(byte[] previous, byte[] next, int stride, int width, int height, int maxRects) {
boolean[] rowChanged = new boolean[height];
boolean anyChange = false;
for (int y = 0; y < height; y++) {
int off = y * stride;
for (int x = 0; x < stride; x++) {
if (previous[off + x] != next[off + x]) {
rowChanged[y] = true;
anyChange = true;
break;
}
}
}
if (!anyChange) {
return null;
}
List<int[]> rects = new ArrayList<>();
boolean[] col = new boolean[stride];
int y = 0;
while (y < height) {
if (!rowChanged[y]) { y++; continue; }
// Extend a vertical band across changed rows, hopping unchanged gaps < V_GAP.
int bandTop = y;
int bandBot = y;
int cur = y + 1;
while (cur < height) {
if (rowChanged[cur]) { bandBot = cur; cur++; continue; }
int g = cur;
while (g < height && !rowChanged[g]) g++;
if (g < height && (g - cur) < MULTI_RECT_V_GAP_ROWS) { cur = g; } else break;
}
// Columns changed anywhere in the band.
Arrays.fill(col, false);
for (int yy = bandTop; yy <= bandBot; yy++) {
if (!rowChanged[yy]) continue;
int off = yy * stride;
for (int x = 0; x < stride; x++) {
if (previous[off + x] != next[off + x]) col[x] = true;
}
}
// Split the band into horizontal clusters, hopping column gaps < H_GAP.
int x = 0;
while (x < stride) {
if (!col[x]) { x++; continue; }
int cL = x;
int cR = x;
int cx = x + 1;
while (cx < stride) {
if (col[cx]) { cR = cx; cx++; continue; }
int g = cx;
while (g < stride && !col[g]) g++;
if (g < stride && (g - cx) < MULTI_RECT_H_GAP_BYTES) { cx = g; } else break;
}
// Tighten this cluster's rows to those that change within [cL,cR].
int tTop = height;
int tBot = -1;
for (int yy = bandTop; yy <= bandBot; yy++) {
if (!rowChanged[yy]) continue;
int off = yy * stride;
for (int xx = cL; xx <= cR; xx++) {
if (previous[off + xx] != next[off + xx]) {
if (yy < tTop) tTop = yy;
tBot = yy;
break;
}
}
}
if (tBot >= 0) {
int left = (cL * 2) & ~3;
int rightExclusive = Math.min(width, (((cR + 1) * 2) + 3) & ~3);
int top = tTop & ~1;
int bottomExclusive = Math.min(height, (tBot + 2) & ~1);
rects.add(new int[]{left, top, rightExclusive - left, bottomExclusive - top});
if (rects.size() > maxRects) {
return null; // too fragmented: a single bounding box is simpler
}
}
x = cR + 1;
}
y = bandBot + 1;
}
return rects;
}
/**
@@ -30,7 +30,12 @@ public class ConnectionOptions {
final boolean sendImagesToLeft = true;
final boolean skipSessionIds = true;
final int WINDOW_SIZE = 2;
// Max messages in flight (awaiting ack) at once — full pipelining. >1 lets a
// frame be sent before the previous frame's ack returns, hiding the BLE ack
// round-trip that otherwise serialized every update. Requires firmware that
// accepts pipelined image messages (the CFW snapshot/deferred FIFO). At 1 the
// communicator behaves exactly as before (serial sends + last-packet prewrite).
final int WINDOW_SIZE = 3;
// Send changed-region (mode 3 bounding box) updates instead of full frames
// when the previous frame is known to be displayed. Requires firmware with
// the experimental bbox-incremental mode. Disabled: the firmware-side
@@ -39,4 +44,19 @@ public class ConnectionOptions {
// composite onto stale content per-lens. Re-enable once the firmware
// guarantees the compositing base.
final boolean INCREMENTAL_FRAMES = true;
// Send several tight rectangle deltas as one CFW mode-8 multi-segment message
// instead of a single bounding box, when the change splits into regions whose
// bounding box wastes bytes (distant edits, or a sparse box). Requires firmware
// with mode-8 multi-rect support; needs INCREMENTAL_FRAMES (same seeded shadow).
// Falls back to the single bounding box whenever multi-rect isn't smaller.
final boolean MULTI_RECT_FRAMES = true;
// Cap on rects per batch. Each rect consumes a distinct CFW frame id, and the
// firmware's duplicate-fid ring holds 16, so keep several batches of history
// within it. Above this the split is abandoned for a single bounding box.
public static final int MULTI_RECT_MAX_RECTS = 6;
// Don't bother splitting when the single bounding box already compresses this
// small (one tight rect); only attempt multi-rect above it or when the box
// spans multiple horizontal clusters.
public static final int MULTI_RECT_MIN_PAYLOAD = 900;
}