From 0d4905ff2839effd173eff919db6e6eaa8fd601b Mon Sep 17 00:00:00 2001 From: James Babcock Date: Wed, 15 Jul 2026 12:39:19 -0700 Subject: [PATCH] Multi-rect optimization and pipelining --- .../faceclaw/app/FaceclawBleCommunicator.java | 125 ++++++++--- .../app/g2protocol/BleImageOptimizer.java | 202 +++++++++++++++++- .../app/g2protocol/ConnectionOptions.java | 22 +- 3 files changed, 316 insertions(+), 33 deletions(-) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 9a829561..94aa1ec5 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -109,6 +109,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private String displayedFingerprint = ""; private byte[] displayedBmp = new byte[0]; + // The frame the firmware shadow will hold once the current image pipeline + // drains: the most recently ENQUEUED image, which is the correct base for the + // next delta when frames are pipelined (displayedBmp only reflects the last + // ACKED frame and lags behind). Set at enqueue; cleared whenever the image + // pipeline is cleared (clearAllMessagesLocked / clearMessagesOfKindLocked + // "image"), so it is only ever read while it holds a valid current-session base. + private byte[] lastEnqueuedImageBmp = new byte[0]; + private String lastEnqueuedFingerprint = ""; private final Map imageUpdateStats = new HashMap<>(); private final Object desiredTilesLock = new Object(); @@ -328,9 +336,18 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { appContext, java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height); } - /** Show or hide a compositor surface; takes effect at the next composite. */ + /** + * Show or hide a compositor surface, immediately submitting the resulting + * frame. Recompositing here (rather than waiting for the next surface + * update) is what makes a just-foregrounded window's retained frame + * actually appear — otherwise a static window (e.g. the terminal hub) whose + * frame landed while briefly hidden would stay blank until its next repaint. + */ public void setSurfaceVisible(String id, boolean visible) { compositor.setSurfaceVisible(id, visible); + SurfaceCompositor.Composite composite = compositor.composite(); + byte[] bmp = BmpUtil.build4bppBmp(composite.gray, composite.width, composite.height); + storeDesiredComposite(composite, bmp, 0, 0); } /** @@ -956,11 +973,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { handleAckTimeoutLocked(oldest); return 0; } - if (sessionReady + if (connectionOptions.WINDOW_SIZE <= 1 + && sessionReady && prewrittenMessage == null && !pendingMessages.isEmpty() && canPrewriteCandidate(pendingMessages.peekFirst()) && !shouldBlockPrewriteForHeartbeatLocked(now)) { + // Only the serial (window==1) path pre-sends the all-but-last + // packet; with a real window we just send the next message fully. messageToPrewrite = pendingMessages.peekFirst(); } } @@ -1004,24 +1024,30 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { enqueueFirmwareDebugFlagsLocked(); } + // Up to WINDOW_SIZE messages may be in flight at once (full + // pipelining); a slot frees when an ack arrives. + boolean windowHasRoom = inFlightMessages.size() < Math.max(1, connectionOptions.WINDOW_SIZE); // A frame ready to send right now: don't inject a fresh // heartbeat in front of it (the image's own ack resets the // firmware heartbeat timer, so the heartbeat is redundant). boolean imageWaiting = !shutdownRequested && fixedLayoutCreated && warmedUp - && pendingMessages.isEmpty() && inFlightMessages.isEmpty() + && !hasPendingOrInflightKindLocked("heartbeat") && now >= imageRetryAfterMs - && !getDesiredFingerprint().equals(displayedFingerprint); + && !getDesiredFingerprint().equals(lastEnqueuedFingerprint); if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) { return ConnectionOptions.IDLE_SLEEP_MS; } - if (messageToPrewrite == null && sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) { - OutboundMessage pending = pendingMessages.peekFirst(); + if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty()) { messageToWrite = pendingMessages.removeFirst(); Log.i(TAG, "sending pending message: " + messageToWrite.label); - } else if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated && warmedUp && pendingMessages.isEmpty() && inFlightMessages.isEmpty() + } else if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated && warmedUp + && windowHasRoom && !hasPendingImageLocked() && now >= imageRetryAfterMs - && !getDesiredFingerprint().equals(displayedFingerprint)) { + && !getDesiredFingerprint().equals(lastEnqueuedFingerprint)) { + // Enqueue the next frame's delta against lastEnqueuedImageBmp + // (what the shadow will be), so it can pipeline behind an + // image still awaiting its ack. Log.i(TAG, "Enqueued image update"); enqueueDesiredImageLocked(); return 0; @@ -1064,7 +1090,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { int frameIdToFinish = 0; synchronized (lock) { synchronized (desiredTilesLock) { - if (desiredFrameId != 0 && !displayedFingerprint.isEmpty() && desiredFingerprint.equals(displayedFingerprint)) { + if (desiredFrameId != 0 && !lastEnqueuedFingerprint.isEmpty() && desiredFingerprint.equals(lastEnqueuedFingerprint)) { frameIdToFinish = desiredFrameId; desiredFrameId = 0; } @@ -1483,25 +1509,49 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { if (bmp == null) { bmp = new byte[0]; } - if (Arrays.equals(bmp, displayedBmp)) { - displayedFingerprint = fingerprint; + if (Arrays.equals(bmp, lastEnqueuedImageBmp)) { + lastEnqueuedFingerprint = fingerprint; finishFrame(frameId, "discarded: image content identical to displayed"); return; } FrameTimings.getInstance().spanStart(frameId, "compress-and-plan"); - // Incremental (mode 3 bounding box) update when the previous frame is - // known-displayed; displayedBmp is cleared on connect/reset/timeout - // paths, so a non-empty value means display state is trusted. + // Incremental (mode 3 bounding box) update against the last ENQUEUED frame + // (the base the firmware shadow will hold when this update is applied). + // lastEnqueuedImageBmp is cleared whenever the image pipeline is cleared, so + // a non-empty value means the display base is trusted. byte[] incrementalPayload = null; - BleImageOptimizer.IncrementalPlan incrementalPlan = null; - if (connectionOptions.INCREMENTAL_FRAMES && displayedBmp.length > 0) { - incrementalPlan = BleImageOptimizer.buildIncrementalImagePayload(displayedBmp, bmp, nextImageFrameId); - if (incrementalPlan != null) { - incrementalPayload = incrementalPlan.payload; + String incrementalLog = null; + if (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedImageBmp.length > 0) { + int baseFid = nextImageFrameId; + BleImageOptimizer.IncrementalPlan single = + BleImageOptimizer.buildIncrementalImagePayload(lastEnqueuedImageBmp, bmp, baseFid); + if (single != null) { + incrementalPayload = single.payload; // advance only when a delta is actually emitted, so consecutive // deltas carry consecutive ids (CFW skip/reorder detection) nextImageFrameId = nextImageFrameId >= 0xfffe ? 1 : nextImageFrameId + 1; + incrementalLog = "incremental update bbox=" + + ((single.payload[3] & 0xff) * 4) + "x" + ((single.payload[4] & 0xff) * 2) + + "+" + ((single.payload[1] & 0xff) * 4) + "+" + ((single.payload[2] & 0xff) * 2) + + " changed=" + single.changedBytes + "/" + single.boxBytes + "B" + + " clusters=" + single.clusterCount; + + // When the bounding box spans multiple clusters or is sizeable, try + // splitting into tight rects (CFW mode-8). Only replace the single + // box if the multi-rect message is actually smaller on the wire. + if (connectionOptions.MULTI_RECT_FRAMES + && (single.clusterCount > 1 || single.payload.length > ConnectionOptions.MULTI_RECT_MIN_PAYLOAD)) { + BleImageOptimizer.MultiRectPlan multi = BleImageOptimizer.buildMultiRectImagePayload( + lastEnqueuedImageBmp, bmp, baseFid, ConnectionOptions.MULTI_RECT_MAX_RECTS); + if (multi != null && multi.payload.length < single.payload.length) { + incrementalPayload = multi.payload; + nextImageFrameId = multi.nextFid; // rectCount fids consumed + incrementalLog = "multi-rect update n=" + multi.rectCount + + " covered=" + multi.coveredBytes + "B" + + " payload=" + multi.payload.length + "B (vs bbox " + single.payload.length + "B)"; + } + } } } BleImageOptimizer.TileImagePlan plan = incrementalPayload != null @@ -1509,16 +1559,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { : new BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, bmp, nextMapSessionId()); plan.fragments = BleImageOptimizer.planImageFragments(plan.payload, ConnectionOptions.IMAGE_FRAGMENT_SIZE); FrameTimings.getInstance().spanEnd(frameId, "compress-and-plan"); - if (incrementalPayload != null) { - // changed/box are packed-byte counts (2px each); clusters>1 with - // changed< keyframe resync): + // drop the pipelined delta base so the next image is a full keyframe. + lastEnqueuedImageBmp = new byte[0]; + lastEnqueuedFingerprint = ""; + } } private void clearAllMessagesLocked(String reason) { clearPendingMessagesLocked(reason); clearInFlightMessagesLocked(reason); + // The image pipeline is gone: drop the pipelined delta base so the next + // image is a full keyframe rather than a delta onto a stale base. + lastEnqueuedImageBmp = new byte[0]; + lastEnqueuedFingerprint = ""; + } + + /** Any image update whose fragments are still queued (not yet sent). */ + private boolean hasPendingImageLocked() { + for (OutboundMessage message : pendingMessages) { + if ("image".equals(message.kind)) { + return true; + } + } + return false; } private void clearPendingMessagesLocked(String reason) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java index 4bdaa423..3b2d81de 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java @@ -200,6 +200,20 @@ public final class BleImageOptimizer { return null; // whole screen changed; full update is strictly better } + byte[] out = encodeMode3Rect(next, stride, left, top, boxWidth, boxHeight, frameId); + return new IncrementalPlan(out, changedBytes, (boxWidth >> 1) * boxHeight, clusterCount); + } + + /** + * Encode one CFW mode-3 rectangle delta: + * [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][zlib box pixels] + * left/width must be multiples of 4 (=> left>>1, width>>1 are whole bytes), top/ + * height multiples of 2. The box pixels are top-down rows of `next` (4bpp packed, + * width>>1 bytes/row). Shared by the single-bbox path and each rect of a mode-8 + * multi-rect batch. + */ + private static byte[] encodeMode3Rect(byte[] next, int stride, int left, int top, + int boxWidth, int boxHeight, int fid) { int regionStride = boxWidth >> 1; byte[] region = new byte[regionStride * boxHeight]; for (int y = 0; y < boxHeight; y++) { @@ -212,10 +226,192 @@ public final class BleImageOptimizer { out[2] = (byte) (top / 2); out[3] = (byte) (boxWidth / 4); out[4] = (byte) (boxHeight / 2); - out[5] = (byte) (frameId & 0xff); // fid_lo - out[6] = (byte) ((frameId >> 8) & 0xff); // fid_hi + out[5] = (byte) (fid & 0xff); // fid_lo + out[6] = (byte) ((fid >> 8) & 0xff); // fid_hi System.arraycopy(compressed, 0, out, 7, compressed.length); - return new IncrementalPlan(out, changedBytes, region.length, clusterCount); + return out; + } + + /** + * A CFW mode-8 multi-segment payload (`[8][count]` then per-rect + * `[len16][mode-3 submsg]`) carrying several tight rectangle deltas instead of + * one bounding box, plus diagnostics. Each rect is applied to the firmware + * shadow with the panel push deferred, then presented once. Used when the change + * splits into regions whose bounding box wastes bytes (distant edits, or a sparse + * box); the caller falls back to the single-rect path when this returns null or + * isn't smaller. + */ + public static final class MultiRectPlan { + public final byte[] payload; + public final int rectCount; + public final int coveredBytes; // sum of rect region bytes, pre-compression + public final int nextFid; // fid to resume from (rectCount consumed) + + MultiRectPlan(byte[] payload, int rectCount, int coveredBytes, int nextFid) { + this.payload = payload; + this.rectCount = rectCount; + this.coveredBytes = coveredBytes; + this.nextFid = nextFid; + } + } + + // Break-even tuning for the rect splitter. Splitting adds ~15 fixed bytes per + // rect (seglen + mode-3 header + fid + zlib framing) and loses cross-rect + // dictionary sharing, so only split across gaps big enough to pay for that. + // Gaps are in the changed-mask's native units: whole 4bpp bytes (2px) for + // columns, rows for the vertical bands. Kept above the 4px/2px box alignment so + // aligned rects never overlap. + private static final int MULTI_RECT_V_GAP_ROWS = 4; + private static final int MULTI_RECT_H_GAP_BYTES = 6; + + /** + * Build a mode-8 multi-rect payload for the change between two same-size 4bpp + * frames, assigning consecutive frame ids starting at fidStart (each rect needs a + * distinct fid — the CFW skips duplicate fids). Returns null when the change is a + * single region (use the single-rect path), when it fragments past maxRects, or + * when the frames aren't comparable. + */ + public static MultiRectPlan buildMultiRectImagePayload(byte[] previousBmp, byte[] newBmp, int fidStart, int maxRects) { + int width = BmpUtil.readBmpWidth(newBmp); + int height = BmpUtil.readBmpHeight(newBmp); + if (width <= 0 || height <= 0 + || width != BmpUtil.readBmpWidth(previousBmp) + || height != BmpUtil.readBmpHeight(previousBmp)) { + return null; + } + byte[] previous = BmpUtil.pack4bppFromBmp(previousBmp); + byte[] next = BmpUtil.pack4bppFromBmp(newBmp); + int stride = (width + 1) >> 1; + if (previous.length != stride * height || next.length != stride * height) { + return null; + } + + List rects = computeSplitRects(previous, next, stride, width, height, maxRects); + if (rects == null || rects.size() < 2) { + return null; // single region (or too fragmented): caller uses the bbox path + } + + List subs = new ArrayList<>(rects.size()); + int coveredBytes = 0; + int fid = fidStart; + int total = 2; // [8][count] + for (int[] r : rects) { + byte[] sub = encodeMode3Rect(next, stride, r[0], r[1], r[2], r[3], fid); + subs.add(sub); + coveredBytes += (r[2] >> 1) * r[3]; + total += 2 + sub.length; // len16 + submsg + fid = (fid >= 0xfffe) ? 1 : fid + 1; + } + + byte[] out = new byte[total]; + out[0] = 8; + out[1] = (byte) rects.size(); + int pos = 2; + for (byte[] sub : subs) { + out[pos] = (byte) (sub.length & 0xff); + out[pos + 1] = (byte) ((sub.length >> 8) & 0xff); + pos += 2; + System.arraycopy(sub, 0, out, pos, sub.length); + pos += sub.length; + } + return new MultiRectPlan(out, rects.size(), coveredBytes, fid); + } + + /** + * Split the changed region between two packed 4bpp frames into a small set of + * aligned rectangles that together cover every changed pixel: maximal vertical + * bands of changed rows (merging gaps < V_GAP), each split into horizontal column + * clusters (merging gaps < H_GAP), with each cluster's rows tightened to where it + * actually changes. Rects are pixel-space {left,top,width,height}, left/width + * aligned to 4px and top/height to 2px. Returns null if there's no change or the + * split exceeds maxRects (caller falls back to a single bounding box). + */ + private static List computeSplitRects(byte[] previous, byte[] next, int stride, int width, int height, int maxRects) { + boolean[] rowChanged = new boolean[height]; + boolean anyChange = false; + for (int y = 0; y < height; y++) { + int off = y * stride; + for (int x = 0; x < stride; x++) { + if (previous[off + x] != next[off + x]) { + rowChanged[y] = true; + anyChange = true; + break; + } + } + } + if (!anyChange) { + return null; + } + + List rects = new ArrayList<>(); + boolean[] col = new boolean[stride]; + int y = 0; + while (y < height) { + if (!rowChanged[y]) { y++; continue; } + // Extend a vertical band across changed rows, hopping unchanged gaps < V_GAP. + int bandTop = y; + int bandBot = y; + int cur = y + 1; + while (cur < height) { + if (rowChanged[cur]) { bandBot = cur; cur++; continue; } + int g = cur; + while (g < height && !rowChanged[g]) g++; + if (g < height && (g - cur) < MULTI_RECT_V_GAP_ROWS) { cur = g; } else break; + } + + // Columns changed anywhere in the band. + Arrays.fill(col, false); + for (int yy = bandTop; yy <= bandBot; yy++) { + if (!rowChanged[yy]) continue; + int off = yy * stride; + for (int x = 0; x < stride; x++) { + if (previous[off + x] != next[off + x]) col[x] = true; + } + } + + // Split the band into horizontal clusters, hopping column gaps < H_GAP. + int x = 0; + while (x < stride) { + if (!col[x]) { x++; continue; } + int cL = x; + int cR = x; + int cx = x + 1; + while (cx < stride) { + if (col[cx]) { cR = cx; cx++; continue; } + int g = cx; + while (g < stride && !col[g]) g++; + if (g < stride && (g - cx) < MULTI_RECT_H_GAP_BYTES) { cx = g; } else break; + } + + // Tighten this cluster's rows to those that change within [cL,cR]. + int tTop = height; + int tBot = -1; + for (int yy = bandTop; yy <= bandBot; yy++) { + if (!rowChanged[yy]) continue; + int off = yy * stride; + for (int xx = cL; xx <= cR; xx++) { + if (previous[off + xx] != next[off + xx]) { + if (yy < tTop) tTop = yy; + tBot = yy; + break; + } + } + } + if (tBot >= 0) { + int left = (cL * 2) & ~3; + int rightExclusive = Math.min(width, (((cR + 1) * 2) + 3) & ~3); + int top = tTop & ~1; + int bottomExclusive = Math.min(height, (tBot + 2) & ~1); + rects.add(new int[]{left, top, rightExclusive - left, bottomExclusive - top}); + if (rects.size() > maxRects) { + return null; // too fragmented: a single bounding box is simpler + } + } + x = cR + 1; + } + y = bandBot + 1; + } + return rects; } /** diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java index b5849d13..b63add63 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java @@ -30,7 +30,12 @@ public class ConnectionOptions { final boolean sendImagesToLeft = true; final boolean skipSessionIds = true; - final int WINDOW_SIZE = 2; + // Max messages in flight (awaiting ack) at once — full pipelining. >1 lets a + // frame be sent before the previous frame's ack returns, hiding the BLE ack + // round-trip that otherwise serialized every update. Requires firmware that + // accepts pipelined image messages (the CFW snapshot/deferred FIFO). At 1 the + // communicator behaves exactly as before (serial sends + last-packet prewrite). + final int WINDOW_SIZE = 3; // Send changed-region (mode 3 bounding box) updates instead of full frames // when the previous frame is known to be displayed. Requires firmware with // the experimental bbox-incremental mode. Disabled: the firmware-side @@ -39,4 +44,19 @@ public class ConnectionOptions { // composite onto stale content per-lens. Re-enable once the firmware // guarantees the compositing base. final boolean INCREMENTAL_FRAMES = true; + + // Send several tight rectangle deltas as one CFW mode-8 multi-segment message + // instead of a single bounding box, when the change splits into regions whose + // bounding box wastes bytes (distant edits, or a sparse box). Requires firmware + // with mode-8 multi-rect support; needs INCREMENTAL_FRAMES (same seeded shadow). + // Falls back to the single bounding box whenever multi-rect isn't smaller. + final boolean MULTI_RECT_FRAMES = true; + // Cap on rects per batch. Each rect consumes a distinct CFW frame id, and the + // firmware's duplicate-fid ring holds 16, so keep several batches of history + // within it. Above this the split is abandoned for a single bounding box. + public static final int MULTI_RECT_MAX_RECTS = 6; + // Don't bother splitting when the single bounding box already compresses this + // small (one tight rect); only attempt multi-rect above it or when the box + // spans multiple horizontal clusters. + public static final int MULTI_RECT_MIN_PAYLOAD = 900; }