diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index fe39c3c7..46004fda 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -178,6 +178,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private int desiredHeight; private int desiredPaintMs; private int desiredFrameId; + // Screen-space deferred draws (glyphs + images) whose pixels are baked + // into desiredPacked; the texture-cache planner may replay them as + // on-glasses cached draws. + private SurfaceCompositor.ScreenDraw[] desiredDraws = new SurfaceCompositor.ScreenDraw[0]; // Highest compositor sequence stored as the desired frame; composites that // lost a store race to a newer one are discarded (their content is already // included in the newer composite). @@ -185,6 +189,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private final SurfaceCompositor compositor = new SurfaceCompositor(); + // Phone-side model of the CFW's 64 KiB texture cache (modes 12/13/14). + // Reset whenever the image pipeline / EvenHub session is torn down: the + // firmware frees the cache with the fb lease, and after any resync the + // cheap safe assumption is an empty cache (glyphs re-upload lazily). + private final TextureCacheState textureCache = new TextureCacheState(); + private boolean textureCacheSupported; + private boolean textureImagesSupported; + private final ArrayDeque pendingMessages = new ArrayDeque<>(); private final ArrayDeque inFlightMessages = new ArrayDeque<>(); private OutboundMessage prewrittenMessage; @@ -684,11 +696,34 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { String contentFingerprint, int paintMs, int frameId + ) { + submitSurfaceFrame(pixels8bpp, surfaceId, rectX, rectY, rectWidth, rectHeight, + contentFingerprint, paintMs, frameId, null); + } + + /** + * As above, with the frame's glyph draws (see SurfaceCompositor's glyph + * overload for the buffer format): the surface's full text content as + * structured draws, letting the texture-cache planner ship glyphs as + * on-glasses cached draws instead of pixels. Null when the submitter has + * no glyph metadata; the pixels alone remain fully correct. + */ + public void submitSurfaceFrame( + java.nio.ByteBuffer pixels8bpp, + String surfaceId, + int rectX, + int rectY, + int rectWidth, + int rectHeight, + String contentFingerprint, + int paintMs, + int frameId, + java.nio.ByteBuffer glyphs ) { Log.i(TAG, "Received an updated frame for surface " + surfaceId); FrameTimings.getInstance().spanStart(frameId, "composite"); SurfaceCompositor.Composite composite = compositor.applyAndComposite( - surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint); + surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs); FrameTimings.getInstance().spanEnd(frameId, "composite"); // Pack the composited 8bpp buffer down to the headerless 4bpp frame // format the wire planners consume; BMP framing is added later only on @@ -717,6 +752,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { desiredFingerprint = composite.fingerprint; desiredPaintMs = paintMs; desiredFrameId = frameId; + desiredDraws = composite.draws; } } if (stale) { @@ -868,6 +904,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { * reconnecting Bluetooth here would defeat the power-saving mode. */ public boolean suspendEvenHubSession() { + // Ending the plugin task releases the fb lease, which frees the + // on-glasses texture cache; forget it phone-side either way (a lost + // ack may still have taken effect). + synchronized (lock) { + textureCache.reset(); + } if (sendShutdownInternal(0, false)) { return true; } @@ -2155,12 +2197,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { int height; int paintMs; int frameId; + SurfaceCompositor.ScreenDraw[] draws; synchronized (desiredTilesLock) { packed = desiredPacked; width = desiredWidth; height = desiredHeight; paintMs = desiredPaintMs; frameId = desiredFrameId; + draws = desiredDraws; desiredFrameId = 0; } if (packed == null) { @@ -2173,6 +2217,43 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { return; } + // Texture-cache path: ship text as cached-glyph draws, punching their + // ink out of the baked deltas. Uploads (mode 12) ride ahead of the + // image message on the ordered transport. Falls through to the plain + // paths whenever the planner has nothing to draw. + if (textureCacheSupported && connectionOptions.TEXTURE_CACHE_FRAMES + && draws != null && draws.length > 0 && packed.length > 0) { + byte[] deltaBase = (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedPacked.length > 0 + && lastEnqueuedWidth == width && lastEnqueuedHeight == height) + ? lastEnqueuedPacked : null; + FrameTimings.getInstance().spanStart(frameId, "texture-plan"); + TexturePlanner.Result tex = TexturePlanner.plan( + deltaBase, packed, width, height, draws, textureCache, + nextImageFrameId, + connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS, + textureImagesSupported); + if (tex != null) { + nextImageFrameId = tex.nextFid; + for (byte[] upload : tex.uploads) { + enqueueTextureUploadLocked(upload); + } + BleImageOptimizer.TileImagePlan plan = new BleImageOptimizer.TileImagePlan( + 0, DASHBOARD_TILE, packed, width, height, nextMapSessionId(), tex.payload); + plan.fragments = BleImageOptimizer.planImageFragments(plan.payload, ConnectionOptions.IMAGE_FRAGMENT_SIZE); + FrameTimings.getInstance().spanEnd(frameId, "texture-plan"); + String texLog = "texture update " + (tex.fullFrame ? "full" : ("rects=" + tex.rectCount)) + + " glyphs=" + tex.drawnGlyphs + " runs=" + tex.runCount + + " images=" + tex.drawnImages + + " baked=" + tex.bakedCandidates + + (tex.uploadBytes > 0 ? " upload=" + tex.uploadBytes + "B" : "") + + " payload=" + tex.payload.length + "B"; + FrameTimings.getInstance().log(frameId, texLog); + finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId); + return; + } + FrameTimings.getInstance().spanEnd(frameId, "texture-plan"); + } + FrameTimings.getInstance().spanStart(frameId, "compress-and-plan"); // Incremental (mode 3 bounding box) update against the last ENQUEUED frame // (the base the firmware shadow will hold when this update is applied). @@ -2221,7 +2302,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { if (incrementalLog != null) { FrameTimings.getInstance().log(frameId, incrementalLog); } + finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId); + } + /** Shared tail of enqueueDesiredImageLocked: enqueue fragments, advance the delta base, log. */ + private void finishEnqueueDesiredImageLocked( + BleImageOptimizer.TileImagePlan plan, String fingerprint, int paintMs, int frameId) { int updateId = nextImageUpdateId++; int messageCount = plan.fragments.size(); imageUpdateStats.put(updateId, new BleImageOptimizer.ImageUpdateStats(paintMs, 1, frameId)); @@ -2244,6 +2330,29 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { + " messages=" + messageCount); } + /** + * Enqueue one mode-12 texture-cache upload ahead of the image message that + * references its glyphs (the transport is FIFO, so no ack round trip is + * needed before use). A timeout means the on-glasses cache state is + * unknown; forget everything phone-side (glyphs re-upload lazily) and let + * the accompanying image update's own timeout drive the frame resync. + */ + private void enqueueTextureUploadLocked(byte[] payload) { + OutboundMessage message = messageBuilder.imagePayload( + "texcache", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "texture upload " + payload.length + "B", + connectionOptions.sendImagesToLeft); + message.onTimeout = () -> { + textureCache.reset(); + logLine("texture upload ack timeout; texture cache state reset"); + }; + pendingMessages.addLast(message); + logLine("queue " + message.label); + } + private void enqueueImageFragmentLocked( BleImageOptimizer.TileImagePlan plan, BleProtocol.ImageFragment fragment, @@ -2333,6 +2442,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload); if (firmwareInfo != null) { cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11"); + textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12") + && hasCapability(firmwareInfo.capabilities, "texstr14"); + textureImagesSupported = hasCapability(firmwareInfo.capabilities, "teximg13"); emitFirmwareInfo(firmwareInfo); } }; @@ -2621,6 +2733,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { // image is a full keyframe rather than a delta onto a stale base. lastEnqueuedPacked = new byte[0]; lastEnqueuedFingerprint = ""; + // Queued texture uploads (if any) were dropped with the rest, and the + // session churn behind a full clear may have freed the on-glasses + // cache; forget it phone-side so glyphs re-upload lazily. + textureCache.reset(); } /** @@ -2651,6 +2767,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { clearInFlightMessagesLocked(reason); lastEnqueuedPacked = new byte[0]; lastEnqueuedFingerprint = ""; + textureCache.reset(); } /** Any image update whose fragments are still queued (not yet sent). */ diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/GlyphAtlas.java b/App_Resources/Android/src/main/java/com/faceclaw/app/GlyphAtlas.java new file mode 100644 index 00000000..0bbd5be1 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/GlyphAtlas.java @@ -0,0 +1,192 @@ +package com.faceclaw.app; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +/** + * Process-wide registry of glyph rasters, fed from the TS side so glyph + * identity survives to the BLE encoder (see notes/texture-cache-display-list- + * design.md). A frame's glyph list references entries here by (fontId, + * encoding); the texture-cache planner uses the raster three ways: + * + * - as the ink mask for the "would this draw land correctly" check against + * the composited frame, + * - to punch the ink pixels out of the baked delta rect it replaces, + * - pre-encoded as the CFW cached-image bytes ([w][h][4bpp RLE], ink = 15) + * uploaded via mode 12 and drawn via mode 14 with a top-color LUT. + * + * Fonts are identified by a stable string key (the embedded font name), NOT a + * per-JS-context counter: worker threads register independently and must agree + * on ids. Glyph rasters are immutable once registered; re-registration of the + * same (fontId, encoding) is ignored. + * + * Cell model: a cached glyph image is bbxWidth x cellHeight (the font's line + * height), with the ink rows at inkTop — so every glyph of a line is drawn at + * the same y (the line top), which is what lets one mode-14 string carry a + * whole run. The horizontal bearing (bbxX) is applied phone-side when + * computing the draw x. Vertical padding is nearly free: it RLE-encodes to a + * couple of tokens. + */ +public final class GlyphAtlas { + private GlyphAtlas() {} + + public static final class Glyph { + /** Cached image width in pixels (the glyph's tight bbox width). */ + public final int width; + /** Cached image height in pixels (the font's line height). */ + public final int cellHeight; + /** First row of ink within the cell. */ + public final int inkTop; + /** Number of ink rows. */ + public final int inkHeight; + /** Horizontal bearing: draw x = pen x + bbxX. */ + public final int bbxX; + /** Ink bitmap rows (inkHeight entries), bit (rowBitWidth-1-col) = ink. */ + final int[] rows; + final int rowBitWidth; + /** CFW cached-image bytes: [w][cellHeight][RLE(w*cellHeight px, ink=15)]. */ + public final byte[] cachedBytes; + + Glyph(int width, int cellHeight, int inkTop, int inkHeight, int bbxX, int[] rows) { + this.width = width; + this.cellHeight = cellHeight; + this.inkTop = inkTop; + this.inkHeight = inkHeight; + this.bbxX = bbxX; + this.rows = rows; + this.rowBitWidth = ((width + 7) >> 3) << 3; + this.cachedBytes = encodeCachedImage(); + } + + /** Whether the cell pixel at (col, row) is ink (row is cell-relative). */ + public boolean inkAt(int col, int row) { + if (col < 0 || col >= width) return false; + int inkRow = row - inkTop; + if (inkRow < 0 || inkRow >= inkHeight) return false; + return ((rows[inkRow] >>> (rowBitWidth - 1 - col)) & 1) != 0; + } + + /** + * The firmware cached-image encoding: [width][height][RLE tokens] + * covering exactly width*cellHeight pixels (no row padding), ink pixels + * as color 15 so the draw-time LUT (top-color scaling) maps them to any + * requested 4bpp level. + */ + private byte[] encodeCachedImage() { + int total = width * cellHeight; + // Worst case one token per pixel (1 byte each) + 2 header bytes. + byte[] out = new byte[2 + total]; + out[0] = (byte) width; + out[1] = (byte) cellHeight; + int o = 2; + int i = 0; + while (i < total) { + int color = inkAt(i % width, i / width) ? 15 : 0; + int j = i + 1; + while (j < total && (inkAt(j % width, j / width) ? 15 : 0) == color) j++; + int run = j - i; + while (run > 0) { + int c = Math.min(run, 0xffff); + if (c <= 15) { + out[o++] = (byte) ((c << 4) | color); + } else if (c <= 255) { + // Worst case can exceed the sizing above only via escape + // forms, which occur for runs >= 16 (net win); safe. + out[o++] = (byte) color; + out[o++] = (byte) c; + } else { + out[o++] = (byte) color; + out[o++] = 0; + out[o++] = (byte) (c & 0xff); + out[o++] = (byte) (c >> 8); + } + run -= c; + } + i = j; + } + byte[] trimmed = new byte[o]; + System.arraycopy(out, 0, trimmed, 0, o); + return trimmed; + } + } + + private static final Object lock = new Object(); + private static final Map fontIds = new HashMap<>(); + private static final Map glyphs = new HashMap<>(); + private static int nextFontId = 1; + + /** Stable id for a font key; assigns one on first use. */ + public static int fontId(String key) { + synchronized (lock) { + Integer id = fontIds.get(key); + if (id == null) { + id = nextFontId++; + fontIds.put(key, id); + } + return id; + } + } + + public static Glyph get(int fontId, int encoding) { + synchronized (lock) { + return glyphs.get(key(fontId, encoding)); + } + } + + private static long key(int fontId, int encoding) { + return ((long) fontId << 32) | (encoding & 0xffffffffL); + } + + /** + * Register a batch of glyph rasters. Little-endian buffer, a sequence of + * font groups: + * [keyLen u8][key utf8][cellHeight u8][count u16] + * each followed by count glyph records: + * [encoding u32][bbxX s8][inkTop u8][width u8][inkHeight u8] + * [inkHeight x rows u32] (bit (ceil(width/8)*8 - 1 - col) = ink) + * Records for an already-registered (font, encoding) are skipped (glyph + * rasters are immutable for a given font key). Malformed buffers throw: + * they indicate a phone-side marshalling bug, never device state. + */ + public static void register(ByteBuffer buffer) { + if (buffer == null) return; + ByteBuffer in = buffer.order(ByteOrder.LITTLE_ENDIAN); + synchronized (lock) { + while (in.remaining() > 0) { + int keyLen = in.get() & 0xff; + byte[] keyBytes = new byte[keyLen]; + in.get(keyBytes); + String fontKey = new String(keyBytes, StandardCharsets.UTF_8); + int cellHeight = in.get() & 0xff; + int count = in.getShort() & 0xffff; + Integer idBoxed = fontIds.get(fontKey); + int id; + if (idBoxed == null) { + id = nextFontId++; + fontIds.put(fontKey, id); + } else { + id = idBoxed; + } + for (int g = 0; g < count; g++) { + int encoding = in.getInt(); + int bbxX = in.get(); + int inkTop = in.get() & 0xff; + int width = in.get() & 0xff; + int inkHeight = in.get() & 0xff; + int[] rows = new int[inkHeight]; + for (int r = 0; r < inkHeight; r++) { + rows[r] = in.getInt(); + } + long k = key(id, encoding); + if (!glyphs.containsKey(k) && width > 0 && cellHeight > 0 + && inkTop + inkHeight <= cellHeight) { + glyphs.put(k, new Glyph(width, cellHeight, inkTop, inkHeight, bbxX, rows)); + } + } + } + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageAtlas.java b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageAtlas.java new file mode 100644 index 00000000..206bae6b --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageAtlas.java @@ -0,0 +1,115 @@ +package com.faceclaw.app; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Process-wide registry of icon/image rasters for the texture-cache pipeline, + * the image counterpart of GlyphAtlas. Entries are content-addressed: the TS + * side keys each image by a hash of its dimensions and pixels, so the same + * icon registered from any thread or rendered at any time dedupes to one + * entry, and an icon whose content changes is simply a new entry. + * + * Unlike glyphs (1-bit ink recolored at draw time), images keep their exact + * 4bpp values: the cached bytes quantize the registered 8bpp pixels with the + * same GRAY_TO_NIBBLE table the composite is packed with, and the planner + * draws them via mode 13 with an identity LUT (top color 15) and the + * transparent bit — the draw writes exactly the nonzero-nibble pixels, which + * is what the eligibility check and hole punching are defined over. + */ +public final class ImageAtlas { + private ImageAtlas() {} + + public static final class Entry { + public final int width; + public final int height; + /** One 4bpp value per pixel, row-major. */ + final byte[] nibbles; + /** CFW cached-image bytes: [w][h][RLE(w*h pixels)]. */ + public final byte[] cachedBytes; + + Entry(int width, int height, byte[] nibbles) { + this.width = width; + this.height = height; + this.nibbles = nibbles; + this.cachedBytes = encodeCachedImage(); + } + + /** The 4bpp value the mode-13 draw would write at (col, row); 0 = skipped. */ + public int nibbleAt(int col, int row) { + return nibbles[row * width + col] & 0xff; + } + + private byte[] encodeCachedImage() { + int total = width * height; + byte[] out = new byte[2 + total]; + out[0] = (byte) width; + out[1] = (byte) height; + int o = 2; + int i = 0; + while (i < total) { + int color = nibbles[i] & 0xff; + int j = i + 1; + while (j < total && (nibbles[j] & 0xff) == color) j++; + int run = j - i; + while (run > 0) { + int c = Math.min(run, 0xffff); + if (c <= 15) { + out[o++] = (byte) ((c << 4) | color); + } else if (c <= 255) { + out[o++] = (byte) color; + out[o++] = (byte) c; + } else { + out[o++] = (byte) color; + out[o++] = 0; + out[o++] = (byte) (c & 0xff); + out[o++] = (byte) (c >> 8); + } + run -= c; + } + i = j; + } + byte[] trimmed = new byte[o]; + System.arraycopy(out, 0, trimmed, 0, o); + return trimmed; + } + } + + private static final Object lock = new Object(); + private static final Map ids = new HashMap<>(); + private static final List entries = new ArrayList<>(); + + /** + * Register an image (8bpp grayscale pixels, width*height bytes, row-major) + * under a content key, returning its id — or the existing id when the key + * is already registered (the pixels are then ignored: same key means same + * content). Ids are positive and stable for the process lifetime. + */ + public static int ensure(String key, int width, int height, ByteBuffer pixels8bpp) { + if (key == null || width <= 0 || width > 255 || height <= 0 || height > 255 + || pixels8bpp == null || pixels8bpp.remaining() < width * height) { + throw new IllegalArgumentException("bad image registration " + width + "x" + height); + } + synchronized (lock) { + Integer existing = ids.get(key); + if (existing != null) return existing; + byte[] nibbles = new byte[width * height]; + for (int i = 0; i < nibbles.length; i++) { + nibbles[i] = (byte) BmpUtil.nibbleForGray(pixels8bpp.get(i) & 0xff); + } + entries.add(new Entry(width, height, nibbles)); + int id = entries.size(); // ids start at 1 + ids.put(key, id); + return id; + } + } + + public static Entry get(int id) { + synchronized (lock) { + return id >= 1 && id <= entries.size() ? entries.get(id - 1) : null; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java index 64249af9..9c862296 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java @@ -42,6 +42,55 @@ public final class SurfaceCompositor { public static final int TRANSPARENCY_OPAQUE = 0; public static final int TRANSPARENCY_COLOR_KEY = 1; + /** + * One deferred draw (a text glyph or an icon image) within a frame, in + * screen coordinates. The draw's pixels are already baked into the + * composited gray buffer (the TS side bakes before submitting); this + * record preserves the draw's identity so the texture-cache planner can + * replay it as an on-glasses cached draw instead of image bytes. + * + * Glyphs: x/y are the pen position and line top; the raster (and its + * bearing/cell placement) comes from GlyphAtlas under (fontId, encoding). + * Images: x/y are the blit's top-left; the raster comes from ImageAtlas + * under imageId. + */ + public static final class ScreenDraw { + public static final int KIND_GLYPH = 0; + public static final int KIND_IMAGE = 1; + + public final int kind; + /** Glyph draws only. */ + public final int fontId; + /** Glyph draws only. */ + public final int encoding; + /** Image draws only. */ + public final int imageId; + public final int x; + public final int y; + /** Glyph draws only: 8-bit brightness the glyph was drawn with. */ + public final int value; + + ScreenDraw(int kind, int fontId, int encoding, int imageId, int x, int y, int value) { + this.kind = kind; + this.fontId = fontId; + this.encoding = encoding; + this.imageId = imageId; + this.x = x; + this.y = y; + this.value = value; + } + + static ScreenDraw glyph(int fontId, int encoding, int penX, int lineY, int value) { + return new ScreenDraw(KIND_GLYPH, fontId, encoding, 0, penX, lineY, value); + } + + static ScreenDraw image(int imageId, int x, int y) { + return new ScreenDraw(KIND_IMAGE, 0, 0, imageId, x, y, 0); + } + } + + private static final ScreenDraw[] NO_DRAWS = new ScreenDraw[0]; + /** One composited full-screen frame plus the metadata the pipeline needs. */ public static final class Composite { /** Full-screen 8bpp grayscale pixels, screenWidth*screenHeight bytes. */ @@ -56,13 +105,20 @@ public final class SurfaceCompositor { public final String fingerprint; /** Monotonic: a Composite with a higher seq contains strictly newer state. */ public final long seq; + /** + * Screen-space deferred draws of every visible surface, in composite + * order (surface z ascending, then each surface's draw order). Their + * pixels are already baked into gray. + */ + public final ScreenDraw[] draws; - Composite(byte[] gray, int width, int height, String fingerprint, long seq) { + Composite(byte[] gray, int width, int height, String fingerprint, long seq, ScreenDraw[] draws) { this.gray = gray; this.width = width; this.height = height; this.fingerprint = fingerprint; this.seq = seq; + this.draws = draws == null ? NO_DRAWS : draws; } } @@ -77,6 +133,8 @@ public final class SurfaceCompositor { boolean visible = true; byte[] pixels; String fingerprint = ""; + /** Surface-local deferred draws of the retained content (already baked into pixels). */ + ScreenDraw[] draws = NO_DRAWS; Surface(String id) { this.id = id; @@ -125,6 +183,7 @@ public final class SurfaceCompositor { if (surface.pixels == null || surface.width != width || surface.height != height) { surface.pixels = new byte[width * height]; surface.fingerprint = ""; + surface.draws = NO_DRAWS; } surface.x = x; surface.y = y; @@ -183,6 +242,31 @@ public final class SurfaceCompositor { int rectHeight, String contentFingerprint ) { + return applyAndComposite(surfaceId, pixels, rectX, rectY, rectWidth, rectHeight, + contentFingerprint, null); + } + + /** + * As above, with the frame's deferred draws: a little-endian buffer of + * tagged records + * [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] (glyph) + * [1][imageId u32][x s16][y s16] (image) + * in surface-local coordinates and draw order. The list describes the + * surface's FULL retained content and replaces the previous list, so it is + * only meaningful for full-surface updates (which is what every submitter + * sends); pass null to clear. Draw pixels must already be baked into pixels. + */ + public Composite applyAndComposite( + String surfaceId, + ByteBuffer pixels, + int rectX, + int rectY, + int rectWidth, + int rectHeight, + String contentFingerprint, + ByteBuffer glyphs + ) { + ScreenDraw[] parsed = parseDraws(glyphs); synchronized (lock) { requireScreenConfiguredLocked(); Surface surface = surfaces.get(surfaceId); @@ -205,10 +289,38 @@ public final class SurfaceCompositor { pixels.get(surface.pixels, dstOffset, rectWidth); } surface.fingerprint = contentFingerprint == null ? "" : contentFingerprint; + surface.draws = parsed; return compositeLocked(); } } + private static ScreenDraw[] parseDraws(ByteBuffer draws) { + if (draws == null || draws.remaining() < 1) { + return NO_DRAWS; + } + ByteBuffer in = draws.order(java.nio.ByteOrder.LITTLE_ENDIAN); + List out = new ArrayList<>(); + while (in.remaining() >= 1) { + int kind = in.get() & 0xff; + if (kind == ScreenDraw.KIND_GLYPH && in.remaining() >= 11) { + int fontId = in.getShort() & 0xffff; + int encoding = in.getInt(); + int penX = in.getShort(); + int lineY = in.getShort(); + int value = in.get() & 0xff; + out.add(ScreenDraw.glyph(fontId, encoding, penX, lineY, value)); + } else if (kind == ScreenDraw.KIND_IMAGE && in.remaining() >= 8) { + int imageId = in.getInt(); + int x = in.getShort(); + int y = in.getShort(); + out.add(ScreenDraw.image(imageId, x, y)); + } else { + break; // malformed tail: keep what parsed cleanly + } + } + return out.toArray(new ScreenDraw[0]); + } + /** Composite the current retained state without applying an update. */ public Composite composite() { synchronized (lock) { @@ -228,7 +340,7 @@ public final class SurfaceCompositor { return null; } byte[] gray = buildGrayLocked(); - return new Composite(gray, screenWidth, screenHeight, "preview", 0); + return new Composite(gray, screenWidth, screenHeight, "preview", 0, NO_DRAWS); } } @@ -253,7 +365,7 @@ public final class SurfaceCompositor { byte[] gray = new byte[screenWidth * screenHeight]; if (blanked) { return new Composite(gray, screenWidth, screenHeight, - "blanked:" + screenWidth + "x" + screenHeight, nextCompositeSeq++); + "blanked:" + screenWidth + "x" + screenHeight, nextCompositeSeq++, NO_DRAWS); } List ordered = new ArrayList<>(surfaces.values()); ordered.sort(Comparator @@ -261,9 +373,14 @@ public final class SurfaceCompositor { .thenComparing(s -> s.id)); StringBuilder fingerprint = new StringBuilder(); fingerprint.append(screenWidth).append('x').append(screenHeight); + List draws = new ArrayList<>(); for (Surface surface : ordered) { if (!surface.visible) continue; blendLocked(gray, surface); + for (ScreenDraw draw : surface.draws) { + draws.add(new ScreenDraw(draw.kind, draw.fontId, draw.encoding, draw.imageId, + draw.x + surface.x, draw.y + surface.y, draw.value)); + } fingerprint.append('|').append(surface.id) .append('@').append(surface.x).append(',').append(surface.y) .append('+').append(surface.width).append('x').append(surface.height) @@ -271,7 +388,8 @@ public final class SurfaceCompositor { .append(':').append(surface.transparency) .append(':').append(surface.fingerprint); } - return new Composite(gray, screenWidth, screenHeight, fingerprint.toString(), nextCompositeSeq++); + return new Composite(gray, screenWidth, screenHeight, fingerprint.toString(), nextCompositeSeq++, + draws.toArray(new ScreenDraw[0])); } private void blendLocked(byte[] gray, Surface surface) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java index d8115ac3..ec8eac4a 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java @@ -205,15 +205,68 @@ public final class BleImageOptimizer { return new IncrementalPlan(out, changedBytes, (boxWidth >> 1) * boxHeight, clusterCount); } + /** + * The aligned changed bounding box between two same-size packed frames, or + * null when they are identical: pixel-space {left, top, width, height} with + * left/width multiples of 4 and top/height multiples of 2 (mode-3 header + * quantization). Exposed so the texture-cache planner can reuse the exact + * rect geometry the single-bbox path would send. + */ + public static int[] computeChangedBox(byte[] previous, byte[] next, int width, int height) { + int stride = (width + 1) >> 1; + if (width <= 0 || height <= 0 || previous == null || next == null + || previous.length != stride * height || next.length != stride * height) { + return null; + } + int minByteX = stride; + int maxByteX = -1; + int minY = height; + int maxY = -1; + for (int y = 0; y < height; y++) { + int rowOffset = y * stride; + for (int x = 0; x < stride; x++) { + if (previous[rowOffset + x] != next[rowOffset + x]) { + if (x < minByteX) minByteX = x; + if (x > maxByteX) maxByteX = x; + if (y < minY) minY = y; + maxY = y; + } + } + } + if (maxY < 0) { + return null; + } + int left = (minByteX * 2) & ~3; + int rightExclusive = Math.min(width, (((maxByteX + 1) * 2) + 3) & ~3); + int top = minY & ~1; + int bottomExclusive = Math.min(height, (maxY + 2) & ~1); + return new int[]{left, top, rightExclusive - left, bottomExclusive - top}; + } + + /** + * The multi-rect split of the changed region (see computeSplitRects), or null + * when it is one region / too fragmented / the frames aren't comparable. + * Exposed for the texture-cache planner. + */ + public static List computeChangedRects(byte[] previous, byte[] next, int width, int height, int maxRects) { + int stride = (width + 1) >> 1; + if (width <= 0 || height <= 0 || previous == null || next == null + || previous.length != stride * height || next.length != stride * height) { + return null; + } + List rects = computeSplitRects(previous, next, stride, width, height, maxRects); + return (rects == null || rects.size() < 2) ? null : rects; + } + /** * Encode one CFW mode-3 rectangle delta: * [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][zlib(rle(box pixels))] * left/width must be multiples of 4 (=> left>>1, width>>1 are whole bytes), top/ * height multiples of 2. The box pixels are top-down rows of `next` (4bpp packed, * width>>1 bytes/row), run-length encoded before deflate. Shared by the single-bbox - * path and each rect of a mode-8 multi-rect batch. + * path, each rect of a mode-8 multi-rect batch, and the texture-cache planner. */ - private static byte[] encodeMode3Rect(byte[] next, int stride, int left, int top, + static byte[] encodeMode3Rect(byte[] next, int stride, int left, int top, int boxWidth, int boxHeight, int fid) { int regionStride = boxWidth >> 1; byte[] region = new byte[regionStride * boxHeight]; diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java index b63add63..6656f9a6 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java @@ -59,4 +59,9 @@ public class ConnectionOptions { // small (one tight rect); only attempt multi-rect above it or when the box // spans multiple horizontal clusters. public static final int MULTI_RECT_MIN_PAYLOAD = 900; + + // Ship text as on-glasses cached-glyph draws (CFW modes 12/14) instead of + // pixels, punching glyph ink out of the baked deltas (see TexturePlanner). + // Also gated at runtime on the firmware advertising texcache12 + texstr14. + final boolean TEXTURE_CACHE_FRAMES = true; } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TextureCacheState.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TextureCacheState.java new file mode 100644 index 00000000..ecb951cc --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TextureCacheState.java @@ -0,0 +1,167 @@ +package com.faceclaw.app; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Phone-side model of the CFW's flat 64 KiB texture cache (zlib_glue mode 12). + * The firmware holds only bytes; every structural decision — where font + * tables and glyph images live, what is resident, when to start over — is + * made here. Layout per font: a 96-entry uint16 offset table (chars 32..127) + * whose entries are written lazily as each glyph is uploaded; glyph images + * are the GlyphAtlas cached bytes ([w][h][4bpp RLE]). + * + * Allocation is a bump pointer with reset-on-full: the realistic working set + * (a few fonts' tables plus ~hundreds of glyphs at tens of bytes each) is far + * below 64 KiB, so eviction sophistication buys nothing. After a reset the + * firmware cache still holds stale bytes; that is safe because a draw only + * ever references (font table entry, glyph image) pairs uploaded in the + * current generation — stale table entries are never named in a mode-14 + * string. + * + * Upload entries accumulate as complete [offset u16][len u16][data] records + * and drain into one or more standalone mode-12 payloads, each independently + * valid, enqueued ahead of the image message that references them (the + * transport is FIFO, so no ack round-trip is needed before use). + * + * Not thread-safe by itself: the communicator calls it under its own lock. + */ +public final class TextureCacheState { + public static final int CACHE_SIZE = 65536; + private static final int FONT_TABLE_BYTES = 96 * 2; + + private int allocEnd = 0; + private final Map fontTableOffsets = new HashMap<>(); + private final Map glyphOffsets = new HashMap<>(); + private final Map imageOffsets = new HashMap<>(); + private final List pendingEntries = new ArrayList<>(); + private int pendingBytes = 0; + private int generation = 0; + + /** + * Forget everything: the on-glasses cache is gone (session teardown, lease + * loss) or no longer trustworthy (upload timeout), or we ran out of space. + * Glyphs and images re-upload lazily on next use. + */ + public void reset() { + allocEnd = 0; + fontTableOffsets.clear(); + glyphOffsets.clear(); + imageOffsets.clear(); + pendingEntries.clear(); + pendingBytes = 0; + generation++; + } + + /** Bumped by every reset; lets a planner detect a mid-plan reset. */ + public int generation() { + return generation; + } + + /** The font's 96-entry table offset in the cache, or -1 when it exists but this glyph can't fit. */ + public int fontTableOffset(int fontId) { + Integer existing = fontTableOffsets.get(fontId); + return existing == null ? -1 : existing; + } + + /** + * Ensure a glyph image (and its font table slot) is resident, queuing the + * mode-12 upload entries for anything newly placed. Returns the glyph's + * image offset, or -1 when the cache is full (caller resets and retries) + * or the encoding is outside the mode-14 table (32..127). + */ + public int ensureGlyph(int fontId, int encoding, GlyphAtlas.Glyph glyph) { + if (glyph == null || encoding < 32 || encoding > 127) return -1; + long key = ((long) fontId << 32) | (encoding & 0xffffffffL); + Integer existing = glyphOffsets.get(key); + if (existing != null) return existing; + + Integer table = fontTableOffsets.get(fontId); + int need = glyph.cachedBytes.length + (table == null ? FONT_TABLE_BYTES : 0); + if (allocEnd + need > CACHE_SIZE) return -1; + + if (table == null) { + table = allocEnd; + allocEnd += FONT_TABLE_BYTES; + fontTableOffsets.put(fontId, table); + // No upload for the table itself: entries are written per glyph, + // and only uploaded entries are ever referenced by a draw. + } + int offset = allocEnd; + allocEnd += glyph.cachedBytes.length; + glyphOffsets.put(key, offset); + + addEntry(offset, glyph.cachedBytes); + addEntry(table + 2 * (encoding - 32), + new byte[]{(byte) (offset & 0xff), (byte) ((offset >> 8) & 0xff)}); + return offset; + } + + /** + * Ensure an ImageAtlas entry is resident (mode-13 draws reference it by + * raw cache offset; no table involved). Returns the image offset, or -1 + * when the cache is full (caller resets and retries). + */ + public int ensureImage(int imageId, ImageAtlas.Entry image) { + if (image == null) return -1; + Integer existing = imageOffsets.get(imageId); + if (existing != null) return existing; + if (allocEnd + image.cachedBytes.length > CACHE_SIZE) return -1; + int offset = allocEnd; + allocEnd += image.cachedBytes.length; + imageOffsets.put(imageId, offset); + addEntry(offset, image.cachedBytes); + return offset; + } + + private void addEntry(int offset, byte[] data) { + byte[] entry = new byte[4 + data.length]; + entry[0] = (byte) (offset & 0xff); + entry[1] = (byte) ((offset >> 8) & 0xff); + entry[2] = (byte) (data.length & 0xff); + entry[3] = (byte) ((data.length >> 8) & 0xff); + System.arraycopy(data, 0, entry, 4, data.length); + pendingEntries.add(entry); + pendingBytes += entry.length; + } + + public boolean hasPendingUploads() { + return !pendingEntries.isEmpty(); + } + + /** Total queued upload bytes (entry framing included), for logging. */ + public int pendingUploadBytes() { + return pendingBytes; + } + + /** + * Drain the queued entries into complete mode-12 payloads ([12][entries]), + * each at most maxPayloadBytes so a payload fits one BLE image message + * without fragmentation. A single entry larger than the cap still gets its + * own payload (the transport can fragment it). + */ + public List drainUploadPayloads(int maxPayloadBytes) { + List payloads = new ArrayList<>(); + ByteArrayOutputStream current = null; + for (byte[] entry : pendingEntries) { + if (current != null && current.size() + entry.length > maxPayloadBytes) { + payloads.add(current.toByteArray()); + current = null; + } + if (current == null) { + current = new ByteArrayOutputStream(Math.min(maxPayloadBytes, 4 + entry.length)); + current.write(12); + } + current.write(entry, 0, entry.length); + } + if (current != null) { + payloads.add(current.toByteArray()); + } + pendingEntries.clear(); + pendingBytes = 0; + return payloads; + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java new file mode 100644 index 00000000..83e1427c --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java @@ -0,0 +1,488 @@ +package com.faceclaw.app; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * Plans a screen update that ships text and icons as on-glasses cached draws + * instead of pixels (CFW modes 12/13/14; see g2flash/patches/zlib_glue.c and + * texture_cache.c). + * + * Approach: dirty rects come from comparing the fully composited old and new + * frames, exactly as the plain incremental path does. For each deferred draw + * (glyph or image) whose pixels intersect a sent rect, check that every pixel + * the on-glasses draw would write lands correct — i.e. the new composite's + * 4bpp value at each in-panel nonzero-source pixel equals what the draw + * produces (this one check subsumes plane occlusion, surface + * occlusion/clipping, and draw-on-draw overlap). A draw that passes and is + * (or can be made) resident in the texture cache is replayed on-glasses: its + * written pixels are punched to 0 in the delta rect content — which is what + * makes text and icons nearly free to compress — and re-emitted as a mode-14 + * string or mode-13 image sub-message in the same atomic mode-8 batch, so + * the shadow after apply equals the full composite exactly. Everything else + * stays baked. + * + * Fallback shape: when nothing qualifies, plan() returns null and the caller + * uses the plain single-bbox/multi-rect/full-frame paths unchanged. + */ +public final class TexturePlanner { + private TexturePlanner() {} + + /** Cap on mode-14 string sub-messages (mode-8 count is a u8). */ + private static final int MAX_GLYPH_RUNS = 180; + /** Cap on mode-13 image sub-messages per update. */ + private static final int MAX_IMAGE_DRAWS = 60; + /** Mode-12 payload size that fits one BLE image message comfortably. */ + private static final int UPLOAD_PAYLOAD_MAX = 3600; + /** Max x-adjust control bytes between two glyphs before starting a new run. */ + private static final int MAX_ADJUST_BYTES = 4; + /** Options: identity LUT (top 15) + transparent, for mode-13 image draws. */ + private static final int IMAGE_DRAW_OPTIONS = 0x1f; + + public static final class Result { + /** Complete image payload: a mode-8 batch of rect deltas + cached draws. */ + public final byte[] payload; + /** Mode-12 upload payloads to enqueue BEFORE the image message. */ + public final List uploads; + /** Next mode-3 frame id (deltas consumed some). */ + public final int nextFid; + public final int rectCount; + public final int drawnGlyphs; + public final int runCount; + public final int drawnImages; + public final int bakedCandidates; + public final int uploadBytes; + public final boolean fullFrame; + + Result(byte[] payload, List uploads, int nextFid, int rectCount, + int drawnGlyphs, int runCount, int drawnImages, int bakedCandidates, + int uploadBytes, boolean fullFrame) { + this.payload = payload; + this.uploads = uploads; + this.nextFid = nextFid; + this.rectCount = rectCount; + this.drawnGlyphs = drawnGlyphs; + this.runCount = runCount; + this.drawnImages = drawnImages; + this.bakedCandidates = bakedCandidates; + this.uploadBytes = uploadBytes; + this.fullFrame = fullFrame; + } + } + + /** One draw selected for on-glasses replay. */ + private static final class Selected { + final SurfaceCompositor.ScreenDraw draw; + final GlyphAtlas.Glyph glyphAtlas; // glyph draws + final ImageAtlas.Entry imageAtlas; // image draws + final int gx; // screen x of the cached image (pen + bearing / blit left) + final int top; // glyph draws: 4bpp top color + + Selected(SurfaceCompositor.ScreenDraw draw, GlyphAtlas.Glyph glyphAtlas, + ImageAtlas.Entry imageAtlas, int gx, int top) { + this.draw = draw; + this.glyphAtlas = glyphAtlas; + this.imageAtlas = imageAtlas; + this.gx = gx; + this.top = top; + } + } + + /** + * Plan a cached-draw update. previous is the delta base (the frame the + * shadow currently holds), or null/mismatched for a full-frame keyframe. + * allowImages requires the teximg13 capability. Returns null when the + * plain paths should run instead (no replayable draws, or identical + * frames). + */ + public static Result plan( + byte[] previous, byte[] next, int width, int height, + SurfaceCompositor.ScreenDraw[] draws, + TextureCacheState cache, int fidStart, + boolean allowMultiRect, int maxRects, boolean allowImages) { + if (next == null || draws == null || draws.length == 0 || width <= 0 || height <= 0) { + return null; + } + int stride = (width + 1) >> 1; + if (next.length != stride * height) { + return null; + } + + // Rect geometry, matching what the plain paths would send. + boolean fullFrame = previous == null || previous.length != next.length; + List rects; + if (!fullFrame) { + int[] box = BleImageOptimizer.computeChangedBox(previous, next, width, height); + if (box == null) { + return null; // identical frames; caller's dedup normally catches this + } + if (box[2] >= width && box[3] >= height) { + fullFrame = true; + rects = Collections.singletonList(new int[]{0, 0, width, height}); + } else { + List split = allowMultiRect + ? BleImageOptimizer.computeChangedRects(previous, next, width, height, maxRects) + : null; + rects = split != null ? split : Collections.singletonList(box); + } + } else { + rects = Collections.singletonList(new int[]{0, 0, width, height}); + } + + // Candidates: draws whose written pixels intersect a sent rect, are + // representable on the wire, and land correct against the composite. + List selected = new ArrayList<>(); + int bakedCandidates = 0; + int selectedImages = 0; + for (SurfaceCompositor.ScreenDraw draw : draws) { + if (draw.kind == SurfaceCompositor.ScreenDraw.KIND_GLYPH) { + GlyphAtlas.Glyph atlas = GlyphAtlas.get(draw.fontId, draw.encoding); + if (atlas == null) continue; // unregistered: stays baked, not a "candidate" + int gx = draw.x + atlas.bbxX; + int inkTop = draw.y + atlas.inkTop; + if (!intersectsAny(rects, gx, inkTop, atlas.width, atlas.inkHeight)) { + continue; + } + if (draw.encoding < 32 || draw.encoding > 127 + || gx < 0 || draw.y < 0 || gx > 0xffff || draw.y > 0xffff) { + bakedCandidates++; + continue; + } + int top = BmpUtil.nibbleForGray(draw.value); + if (!glyphMatchesComposite(next, stride, width, height, atlas, gx, draw.y, top)) { + bakedCandidates++; + continue; + } + if (selected.size() >= MAX_GLYPH_RUNS * 10) { // soft cap before run grouping + bakedCandidates++; + continue; + } + selected.add(new Selected(draw, atlas, null, gx, top)); + } else { + if (!allowImages) continue; // firmware lacks teximg13: stays baked + ImageAtlas.Entry atlas = ImageAtlas.get(draw.imageId); + if (atlas == null) continue; + if (!intersectsAny(rects, draw.x, draw.y, atlas.width, atlas.height)) { + continue; + } + if (draw.x < 0 || draw.y < 0 || draw.x > 0xffff || draw.y > 0xffff + || selectedImages >= MAX_IMAGE_DRAWS) { + bakedCandidates++; + continue; + } + if (!imageMatchesComposite(next, stride, width, height, atlas, draw.x, draw.y)) { + bakedCandidates++; + continue; + } + selected.add(new Selected(draw, null, atlas, draw.x, 0)); + selectedImages++; + } + } + if (selected.isEmpty()) { + return null; + } + + // Residency: ensure every selected draw is in the cache, retrying once + // after a reset when the bump allocator fills. A draw that still fails + // falls back to baked. IMPORTANT: from here on, every bail-out to the + // plain path must cache.reset() — ensure* marked entries resident + // and queued uploads that only a returned Result would enqueue. + List drawable = ensureResident(cache, selected); + bakedCandidates += selected.size() - drawable.size(); + if (drawable.isEmpty()) { + cache.reset(); + return null; + } + + // Group glyphs into mode-14 runs; leftover-budget glyphs bake. + List glyphDrawable = new ArrayList<>(); + List imageDrawable = new ArrayList<>(); + for (Selected sel : drawable) { + if (sel.glyphAtlas != null) glyphDrawable.add(sel); + else imageDrawable.add(sel); + } + List runs = new ArrayList<>(); + List drawnGlyphs = buildRuns(cache, glyphDrawable, runs); + bakedCandidates += glyphDrawable.size() - drawnGlyphs.size(); + List drawn = new ArrayList<>(drawnGlyphs); + drawn.addAll(imageDrawable); + if (drawn.isEmpty() || rects.size() + runs.size() + imageDrawable.size() > 255) { + cache.reset(); + return null; + } + + // Punch the drawn pixels out of a copy of the composite, then encode + // the rect deltas from the punched content. + byte[] punched = next.clone(); + for (Selected sel : drawn) { + punch(punched, stride, width, height, sel); + } + + List subs = new ArrayList<>(); + int fid = fidStart; + if (fullFrame) { + subs.add(BleImageOptimizer.maybeCompress(punched, width, height)); + } else { + for (int[] r : rects) { + subs.add(BleImageOptimizer.encodeMode3Rect(punched, stride, r[0], r[1], r[2], r[3], fid)); + fid = (fid >= 0xfffe) ? 1 : fid + 1; + } + } + for (Selected sel : imageDrawable) { + subs.add(encodeImageDraw(cache, sel)); + } + subs.addAll(runs); + byte[] payload = assembleMode8(subs); + + int uploadBytes = cache.pendingUploadBytes(); + List uploads = cache.drainUploadPayloads(UPLOAD_PAYLOAD_MAX); + return new Result(payload, uploads, fid, rects.size(), drawnGlyphs.size(), runs.size(), + imageDrawable.size(), bakedCandidates, uploadBytes, fullFrame); + } + + private static boolean intersectsAny(List rects, int x, int y, int w, int h) { + if (w <= 0 || h <= 0) return false; + for (int[] r : rects) { + if (x < r[0] + r[2] && x + w > r[0] && y < r[1] + r[3] && y + h > r[1]) { + return true; + } + } + return false; + } + + /** + * Whether every in-panel ink pixel of the glyph equals `top` in the packed + * new frame — i.e. the draw's writes all land correct. Ink outside the + * panel is clipped by the firmware and irrelevant. + */ + private static boolean glyphMatchesComposite(byte[] packed, int stride, int width, int height, + GlyphAtlas.Glyph atlas, int gx, int lineY, int top) { + for (int row = atlas.inkTop; row < atlas.inkTop + atlas.inkHeight; row++) { + int y = lineY + row; + if (y < 0 || y >= height) continue; + for (int col = 0; col < atlas.width; col++) { + if (!atlas.inkAt(col, row)) continue; + int x = gx + col; + if (x < 0 || x >= width) continue; + if (nibbleAt(packed, stride, x, y) != top) { + return false; + } + } + } + return true; + } + + /** + * Whether every in-panel nonzero pixel of the cached image equals the + * packed new frame — the mode-13 transparent draw writes exactly those. + */ + private static boolean imageMatchesComposite(byte[] packed, int stride, int width, int height, + ImageAtlas.Entry atlas, int left, int top) { + for (int row = 0; row < atlas.height; row++) { + int y = top + row; + if (y < 0 || y >= height) continue; + for (int col = 0; col < atlas.width; col++) { + int source = atlas.nibbleAt(col, row); + if (source == 0) continue; + int x = left + col; + if (x < 0 || x >= width) continue; + if (nibbleAt(packed, stride, x, y) != source) { + return false; + } + } + } + return true; + } + + private static int nibbleAt(byte[] packed, int stride, int x, int y) { + int b = packed[y * stride + (x >> 1)] & 0xff; + return (x & 1) != 0 ? (b & 0x0f) : (b >> 4); + } + + private static void punch(byte[] packed, int stride, int width, int height, Selected sel) { + if (sel.glyphAtlas != null) { + GlyphAtlas.Glyph atlas = sel.glyphAtlas; + for (int row = atlas.inkTop; row < atlas.inkTop + atlas.inkHeight; row++) { + int y = sel.draw.y + row; + if (y < 0 || y >= height) continue; + for (int col = 0; col < atlas.width; col++) { + if (!atlas.inkAt(col, row)) continue; + punchPixel(packed, stride, width, sel.gx + col, y); + } + } + } else { + ImageAtlas.Entry atlas = sel.imageAtlas; + for (int row = 0; row < atlas.height; row++) { + int y = sel.draw.y + row; + if (y < 0 || y >= height) continue; + for (int col = 0; col < atlas.width; col++) { + if (atlas.nibbleAt(col, row) == 0) continue; + punchPixel(packed, stride, width, sel.gx + col, y); + } + } + } + } + + private static void punchPixel(byte[] packed, int stride, int width, int x, int y) { + if (x < 0 || x >= width) return; + int index = y * stride + (x >> 1); + if ((x & 1) != 0) { + packed[index] = (byte) (packed[index] & 0xf0); + } else { + packed[index] = (byte) (packed[index] & 0x0f); + } + } + + /** + * Ensure residency for all of `selected`, resetting the cache once if the + * allocator fills mid-plan (a reset invalidates offsets handed out earlier + * in the same pass, so the whole pass reruns). Returns the draws that are + * resident afterwards. + */ + private static List ensureResident(TextureCacheState cache, List selected) { + for (int attempt = 0; attempt < 2; attempt++) { + List resident = new ArrayList<>(selected.size()); + boolean full = false; + for (Selected sel : selected) { + int offset = sel.glyphAtlas != null + ? cache.ensureGlyph(sel.draw.fontId, sel.draw.encoding, sel.glyphAtlas) + : cache.ensureImage(sel.draw.imageId, sel.imageAtlas); + if (offset >= 0) { + resident.add(sel); + } else { + full = true; + } + } + if (!full || attempt == 1) { + return resident; + } + cache.reset(); + } + return Collections.emptyList(); // unreachable + } + + /** Mode-13 cached-image draw: [13][offset u16][x u16][y u16][options u8]. */ + private static byte[] encodeImageDraw(TextureCacheState cache, Selected sel) { + int offset = cache.ensureImage(sel.draw.imageId, sel.imageAtlas); // resident: returns the offset + byte[] sub = new byte[8]; + sub[0] = 13; + sub[1] = (byte) (offset & 0xff); + sub[2] = (byte) ((offset >> 8) & 0xff); + sub[3] = (byte) (sel.draw.x & 0xff); + sub[4] = (byte) ((sel.draw.x >> 8) & 0xff); + sub[5] = (byte) (sel.draw.y & 0xff); + sub[6] = (byte) ((sel.draw.y >> 8) & 0xff); + sub[7] = (byte) IMAGE_DRAW_OPTIONS; + return sub; + } + + /** + * Group selected glyphs into mode-14 string sub-messages: + * [14][fontTable u16][x u16][y u16][options u8][strlen u8][string] + * One run per (font, line y, top color) span; within a run, control bytes + * 1..31 adjust x by -10..+20 to hit each glyph's exact position, and each + * glyph advances x by its cached width. Order across runs is free: every + * drawn glyph's ink equals the composite, so overlaps write equal values. + * Returns the glyphs actually emitted (run budget can drop stragglers). + */ + private static List buildRuns(TextureCacheState cache, List drawable, List outRuns) { + List sorted = new ArrayList<>(drawable); + sorted.sort(Comparator + .comparingInt((Selected s) -> s.draw.fontId) + .thenComparingInt(s -> s.draw.y) + .thenComparingInt(s -> s.top) + .thenComparingInt(s -> s.gx)); + + List drawn = new ArrayList<>(sorted.size()); + int i = 0; + while (i < sorted.size() && outRuns.size() < MAX_GLYPH_RUNS) { + Selected first = sorted.get(i); + int fontTable = cache.fontTableOffset(first.draw.fontId); + if (fontTable < 0) { // shouldn't happen: residency ensured above + i++; + continue; + } + java.io.ByteArrayOutputStream string = new java.io.ByteArrayOutputStream(); + List runGlyphs = new ArrayList<>(); + int cursor = first.gx; // the run's wire x starts exactly at the first glyph + int j = i; + while (j < sorted.size()) { + Selected sel = sorted.get(j); + if (sel.draw.fontId != first.draw.fontId + || sel.draw.y != first.draw.y + || sel.top != first.top) { + break; + } + int delta = sel.gx - cursor; + int adjustBytes = adjustByteCount(delta); + if (adjustBytes > MAX_ADJUST_BYTES && j > i) { + break; // far gap: cheaper to start a fresh run + } + if (string.size() + adjustBytes + 1 > 255) { + break; // strlen is a u8 + } + emitAdjust(string, delta); + string.write(sel.draw.encoding); + runGlyphs.add(sel); + cursor = sel.gx + sel.glyphAtlas.width; + j++; + } + if (!runGlyphs.isEmpty()) { + byte[] stringBytes = string.toByteArray(); + byte[] run = new byte[9 + stringBytes.length]; + run[0] = 14; + run[1] = (byte) (fontTable & 0xff); + run[2] = (byte) ((fontTable >> 8) & 0xff); + run[3] = (byte) (first.gx & 0xff); + run[4] = (byte) ((first.gx >> 8) & 0xff); + run[5] = (byte) (first.draw.y & 0xff); + run[6] = (byte) ((first.draw.y >> 8) & 0xff); + run[7] = (byte) (first.top | 0x10); // top color + transparent + run[8] = (byte) stringBytes.length; + System.arraycopy(stringBytes, 0, run, 9, stringBytes.length); + outRuns.add(run); + drawn.addAll(runGlyphs); + i = j; + } else { + i++; + } + } + return drawn; + } + + /** How many 1..31 control bytes are needed to move the cursor by delta. */ + private static int adjustByteCount(int delta) { + if (delta == 0) return 0; + return delta > 0 ? (delta + 19) / 20 : (-delta + 9) / 10; + } + + private static void emitAdjust(java.io.ByteArrayOutputStream out, int delta) { + while (delta != 0) { + int step = delta > 0 ? Math.min(delta, 20) : Math.max(delta, -10); + out.write(step + 11); // byte 1..31 adjusts x by (byte - 11) + delta -= step; + } + } + + /** [8][count]([len u16][sub])* — one atomic present for deltas + draws. */ + private static byte[] assembleMode8(List subs) { + int total = 2; + for (byte[] sub : subs) { + total += 2 + sub.length; + } + byte[] out = new byte[total]; + out[0] = 8; + out[1] = (byte) subs.size(); + int pos = 2; + for (byte[] sub : subs) { + out[pos] = (byte) (sub.length & 0xff); + out[pos + 1] = (byte) ((sub.length >> 8) & 0xff); + pos += 2; + System.arraycopy(sub, 0, out, pos, sub.length); + pos += sub.length; + } + return out; + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/util/BmpUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/util/BmpUtil.java index f07f21d6..a4746c71 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/util/BmpUtil.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/util/BmpUtil.java @@ -20,6 +20,15 @@ public class BmpUtil { } } + /** + * The 4bpp level an 8-bit gray value packs to. Public so the texture-cache + * planner computes a glyph draw's top color with the exact quantization + * the composited frame was packed with. + */ + public static int nibbleForGray(int gray) { + return GRAY_TO_NIBBLE[gray & 0xff] & 0xff; + } + /** * Pack a full-resolution 8bpp grayscale buffer (row-major, top-to-bottom, * one byte per pixel) into the headerless 4bpp wire format used by CFW diff --git a/app/apps/blocks/blocks-app.worker.ts b/app/apps/blocks/blocks-app.worker.ts index a9db1260..293c6151 100644 --- a/app/apps/blocks/blocks-app.worker.ts +++ b/app/apps/blocks/blocks-app.worker.ts @@ -9,7 +9,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; import * as frameTimings from "../../native/frame-timings"; import type { DashboardInputEvent } from "../../ui/layers"; @@ -647,7 +648,7 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -659,6 +660,7 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/freecell/freecell-app.worker.ts b/app/apps/freecell/freecell-app.worker.ts index c56705f8..35eb95d8 100644 --- a/app/apps/freecell/freecell-app.worker.ts +++ b/app/apps/freecell/freecell-app.worker.ts @@ -13,7 +13,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; import * as frameTimings from "../../native/frame-timings"; import type { DashboardInputEvent } from "../../ui/layers"; @@ -752,7 +753,7 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -764,6 +765,7 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/launcher/launcher-app.ts b/app/apps/launcher/launcher-app.ts index e2845387..68ee541a 100644 --- a/app/apps/launcher/launcher-app.ts +++ b/app/apps/launcher/launcher-app.ts @@ -287,11 +287,17 @@ class LauncherGridLayer implements Layer { : renderIcon("folder-filled", ICON_SIZE); if (icon) { // Clip the icon at the grid bottom so a peeking row shows only its top. + // A fully visible icon goes through the deferred-image path (texture + // cacheable); the clipped peeking row falls back to a raster blit, + // since cached draws are whole-image only. const clipHeight = Math.min(icon.height, Math.floor(gridBottom - blockTop)); - image.bitBlt(icon, Math.round(centerX - icon.width / 2), Math.round(blockTop), { - height: clipHeight, - transparentZero: true, - }); + const iconX = Math.round(centerX - icon.width / 2); + const iconY = Math.round(blockTop); + if (clipHeight >= icon.height) { + image.drawImage(icon, iconX, iconY); + } else { + image.bitBlt(icon, iconX, iconY, { height: clipHeight, transparentZero: true }); + } } const labelY = Math.round(blockTop + ICON_SIZE + LABEL_GAP); if (labelY + font.lineHeight <= gridBottom) { diff --git a/app/apps/minesweeper/minesweeper-app.worker.ts b/app/apps/minesweeper/minesweeper-app.worker.ts index 2a4439a4..1a060fc3 100644 --- a/app/apps/minesweeper/minesweeper-app.worker.ts +++ b/app/apps/minesweeper/minesweeper-app.worker.ts @@ -13,7 +13,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; import * as frameTimings from "../../native/frame-timings"; import type { DashboardInputEvent } from "../../ui/layers"; @@ -711,7 +712,7 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -723,6 +724,7 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/navigate/navigate-app.worker.ts b/app/apps/navigate/navigate-app.worker.ts index c27edd5c..878b0eeb 100644 --- a/app/apps/navigate/navigate-app.worker.ts +++ b/app/apps/navigate/navigate-app.worker.ts @@ -8,7 +8,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; import * as frameTimings from "../../native/frame-timings"; import type { DashboardInputEvent } from "../../ui/layers"; @@ -873,7 +874,7 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -885,6 +886,7 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); win.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/pinball/pinball-app.worker.ts b/app/apps/pinball/pinball-app.worker.ts index 288804ce..41c1cfdc 100644 --- a/app/apps/pinball/pinball-app.worker.ts +++ b/app/apps/pinball/pinball-app.worker.ts @@ -19,7 +19,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; import * as frameTimings from "../../native/frame-timings"; import { getStringSetting, setStringSetting } from "../../native/settings-store"; @@ -1040,7 +1041,7 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -1052,6 +1053,7 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/roam/roam-app.worker.ts b/app/apps/roam/roam-app.worker.ts index 83541981..939abad4 100644 --- a/app/apps/roam/roam-app.worker.ts +++ b/app/apps/roam/roam-app.worker.ts @@ -7,7 +7,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont } from "../../graphics/bdffont"; import { truncateText, wrapText } from "../../graphics/textwrap"; import * as frameTimings from "../../native/frame-timings"; @@ -593,7 +594,7 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -605,6 +606,7 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); win.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/terminal/terminal-app.worker.ts b/app/apps/terminal/terminal-app.worker.ts index bcb16570..caa0fc0e 100644 --- a/app/apps/terminal/terminal-app.worker.ts +++ b/app/apps/terminal/terminal-app.worker.ts @@ -32,7 +32,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { TERMINAL_ICON_GLYPHS } from "../../graphics/icons"; import * as frameTimings from "../../native/frame-timings"; @@ -1561,7 +1562,7 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -1573,6 +1574,7 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/timer/timer-app.worker.ts b/app/apps/timer/timer-app.worker.ts index 0db4ac4c..e0b45888 100644 --- a/app/apps/timer/timer-app.worker.ts +++ b/app/apps/timer/timer-app.worker.ts @@ -5,7 +5,8 @@ */ import "@nativescript/core/globals"; import { GrayImage } from "../../graphics/image"; -import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane"; +import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; import * as frameTimings from "../../native/frame-timings"; import { @@ -660,7 +661,7 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: no active communicator"); return; } - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); communicator.submitSurfaceFrame( buffer.buffer, @@ -672,6 +673,7 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 0c3efa5c..e05f7670 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -16,7 +16,8 @@ import { isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboa import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { voiceControlBridge } from "../native/voice-control"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; -import { flattenPlanes, planesFingerprint, type Plane } from "../graphics/plane"; +import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; +import { prepareFrameDraws } from "../graphics/glyph-wire"; import { getDefaultMediumFont } from "../graphics/bdffont"; import { wrapText } from "../graphics/textwrap"; import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; @@ -1562,7 +1563,7 @@ class DashboardController { return; } const fingerprint = planesFingerprint(planes); - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = image.to8bppBuffer(); await communicator.submitSurfaceFrame( surfaceId, @@ -1571,6 +1572,7 @@ class DashboardController { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); } @@ -1630,7 +1632,7 @@ class DashboardController { return; } const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); - const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes)); + const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); await this.communicator.submitSurfaceFrame( SHELL_SURFACE_ID, @@ -1639,6 +1641,7 @@ class DashboardController { fingerprint, paintMs, frameId, + prepareFrameDraws(draws), ); await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); this.updateCompositePreview(); diff --git a/app/graphics/bdffont.ts b/app/graphics/bdffont.ts index ca34308c..563007eb 100644 --- a/app/graphics/bdffont.ts +++ b/app/graphics/bdffont.ts @@ -20,6 +20,13 @@ export class BdfFont { * bitmap). Only meaningful within one JS context; never persist it. */ readonly fingerprintId = nextFontFingerprintId++; + /** + * Stable cross-context identity for the on-glasses glyph atlas (the + * embedded font name). Worker threads and the main thread must agree on + * it, unlike fingerprintId. Fonts without one (ad-hoc parses) simply never + * participate in texture caching — their glyphs stay baked. + */ + readonly atlasKey: string | null; readonly ascent: number; readonly descent: number; readonly lineHeight: number; @@ -40,6 +47,7 @@ export class BdfFont { defaultChar: number; glyphs: Map; fallback?: BdfFont; + atlasKey?: string; }) { this.ascent = opts.ascent; this.descent = opts.descent; @@ -47,9 +55,10 @@ export class BdfFont { this.defaultChar = opts.defaultChar; this.glyphs = opts.glyphs; this.fallback = opts.fallback; + this.atlasKey = opts.atlasKey ?? null; } - static parse(source: string, opts: { fallback?: BdfFont; minEncoding?: number } = {}): BdfFont { + static parse(source: string, opts: { fallback?: BdfFont; minEncoding?: number; atlasKey?: string } = {}): BdfFont { const minEncoding = opts.minEncoding ?? 0; const lines = source.replace(/\r/g, "").split("\n"); let ascent = 10; @@ -128,7 +137,7 @@ export class BdfFont { } } - return new BdfFont({ ascent, descent, defaultChar, glyphs, fallback: opts.fallback }); + return new BdfFont({ ascent, descent, defaultChar, glyphs, fallback: opts.fallback, atlasKey: opts.atlasKey }); } /** Exact lookup for a single code point, with no default-char substitution. */ @@ -192,6 +201,7 @@ function loadEmbeddedFont(name: EmbeddedFontName): BdfFont { const font = BdfFont.parse(text, { fallback: fallbackName ? loadEmbeddedFont(fallbackName) : undefined, minEncoding: fontMinEncoding[name], + atlasKey: name, }); cachedEmbeddedFonts.set(name, font); return font; diff --git a/app/graphics/glyph-wire.ts b/app/graphics/glyph-wire.ts new file mode 100644 index 00000000..85b49562 --- /dev/null +++ b/app/graphics/glyph-wire.ts @@ -0,0 +1,225 @@ +/** + * Marshals deferred-draw identity (text glyphs and icon images) to the Java + * side for the texture-cache pipeline (CFW modes 12/13/14; see + * notes/texture-cache-display-list-design.md). + * + * Channels, all cheap ByteBuffer crossings: + * - glyph raster registration into the process-wide Java GlyphAtlas, once + * per (font, glyph) per JS context (main thread and each worker register + * independently; the atlas dedupes by the font's stable atlasKey); + * - image registration into the Java ImageAtlas, once per distinct image + * content per JS context (images are content-addressed: the key is a hash + * of dimensions + pixels, so the same icon rendered anywhere dedupes and + * a changed icon is simply a new image); + * - a per-frame draw list accompanying each submitted surface frame, which + * the Java planner may replay as on-glasses cached draws. + * + * Draws that can't participate (no atlasKey, outside the mode-14 ASCII + * table, ink outside the line cell, oversized images) are simply left out — + * their pixels are baked into the submitted frame either way, so exclusion + * only means "no wire savings for this draw". + */ +import { BdfFont, Glyph } from "./bdffont"; +import { GrayImage, type DeferredDraw, type PlacedImage } from "./image"; + +declare const com: any; + +const GLYPH_RECORD_BYTES = 12; // [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] +const IMAGE_RECORD_BYTES = 9; // [1][imageId u32][x s16][y s16] + +type FontWireState = { + fontId: number; + cellHeight: number; + /** Encodings already registered with the Java atlas from this JS context. */ + registered: Set; +}; + +const fontStates = new Map(); + +/** + * Java-side image id per source image object, or null for images that can't + * participate. Sources handed to drawImage are immutable by contract, so + * memoizing by object identity is safe; a re-rendered icon is a new object + * and re-resolves (usually to the same content-addressed id). + */ +const imageIds = new WeakMap(); + +function fontWireState(font: BdfFont): FontWireState | null { + if (!global.isAndroid || !font.atlasKey) return null; + if (font.lineHeight <= 0 || font.lineHeight > 255) return null; + let state = fontStates.get(font); + if (!state) { + state = { + fontId: com.faceclaw.app.GlyphAtlas.fontId(font.atlasKey), + cellHeight: font.lineHeight, + registered: new Set(), + }; + fontStates.set(font, state); + } + return state; +} + +/** + * Whether this glyph can be expressed as an on-glasses cached draw: within + * the mode-14 char table (32..127), a nonempty raster that fits the font's + * line cell (the cached image is bbxWidth x lineHeight with the ink placed at + * inkTop), and metrics that fit the wire fields. + */ +function representableGlyph(font: BdfFont, glyph: Glyph): boolean { + if (glyph.encoding < 32 || glyph.encoding > 127) return false; + if (glyph.bbxWidth <= 0 || glyph.bbxWidth > 255) return false; + if (glyph.bbxHeight <= 0 || glyph.bbxHeight > 255) return false; + if (glyph.bbxX < -128 || glyph.bbxX > 127) return false; + const inkTop = font.ascent - (glyph.bbxHeight + glyph.bbxY); + return inkTop >= 0 && inkTop + glyph.bbxHeight <= font.lineHeight; +} + +/** Resolve (registering on first sight) the Java image id for a drawImage source. */ +function imageId(placed: PlacedImage): number | null { + if (!global.isAndroid) return null; + const source = placed.source; + const memo = imageIds.get(source); + if (memo !== undefined) return memo; + let id: number | null = null; + if (source.width > 0 && source.width <= 255 && source.height > 0 && source.height <= 255) { + const key = `img:${source.width}x${source.height}:${source.contentHash32().toString(16)}`; + id = com.faceclaw.app.ImageAtlas.ensure(key, source.width, source.height, source.pixels.buffer); + if (!(typeof id === "number") || id <= 0) id = null; + } + imageIds.set(source, id); + return id; +} + +function inRange16(v: number): boolean { + return v >= -32768 && v <= 32767; +} + +/** + * Register any not-yet-registered glyph rasters and image contents among + * `draws` with the Java atlases, then build the per-frame draw buffer to pass + * alongside the frame's pixels: little-endian tagged records + * [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] (glyph) + * [1][imageId u32][x s16][y s16] (image) + * in draw order. Returns null when nothing is expressible (or off-Android). + */ +export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | null { + if (!global.isAndroid || draws.length === 0) return null; + + // Pass 1: resolve ids, collect unregistered glyph rasters, size the buffer. + let registration: Map | null = null; + let bytes = 0; + for (const placed of draws) { + if (placed.kind === "glyph") { + const state = fontWireState(placed.font); + if (!state || !representableGlyph(placed.font, placed.glyph)) continue; + if (!inRange16(placed.x) || !inRange16(placed.y)) continue; + bytes += GLYPH_RECORD_BYTES; + if (state.registered.has(placed.glyph.encoding)) continue; + state.registered.add(placed.glyph.encoding); + registration ??= new Map(); + let group = registration.get(state); + if (!group) { + group = { font: placed.font, glyphs: [] }; + registration.set(state, group); + } + group.glyphs.push(placed.glyph); + } else { + if (!inRange16(placed.x) || !inRange16(placed.y)) continue; + if (imageId(placed) !== null) bytes += IMAGE_RECORD_BYTES; + } + } + if (registration) { + com.faceclaw.app.GlyphAtlas.register(buildRegistrationBuffer(registration)); + } + if (bytes === 0) return null; + + // Pass 2: the frame's draw records. + const out = new DataView(new ArrayBuffer(bytes)); + let offset = 0; + for (const placed of draws) { + if (!inRange16(placed.x) || !inRange16(placed.y)) continue; + if (placed.kind === "glyph") { + const state = fontWireState(placed.font); + if (!state || !representableGlyph(placed.font, placed.glyph)) continue; + out.setUint8(offset, 0); + out.setUint16(offset + 1, state.fontId, true); + out.setUint32(offset + 3, placed.glyph.encoding, true); + out.setInt16(offset + 7, placed.x, true); + out.setInt16(offset + 9, placed.y, true); + out.setUint8(offset + 11, placed.value); + offset += GLYPH_RECORD_BYTES; + } else { + const id = imageId(placed); + if (id === null) continue; + out.setUint8(offset, 1); + out.setUint32(offset + 1, id, true); + out.setInt16(offset + 5, placed.x, true); + out.setInt16(offset + 7, placed.y, true); + offset += IMAGE_RECORD_BYTES; + } + } + return out.buffer; +} + +/** + * Registration buffer: per font group + * [keyLen u8][key utf8][cellHeight u8][count u16] + * then per glyph + * [encoding u32][bbxX s8][inkTop u8][width u8][inkHeight u8] + * [inkHeight x row u32] (bit (ceil(width/8)*8 - 1 - col) = ink) + * mirroring GlyphAtlas.register on the Java side. + */ +function buildRegistrationBuffer( + registration: Map, +): ArrayBuffer { + let total = 0; + const encodedKeys = new Map(); + registration.forEach((group, state) => { + const keyBytes = utf8Encode(group.font.atlasKey ?? ""); + encodedKeys.set(state, keyBytes); + total += 1 + keyBytes.length + 1 + 2; + for (const glyph of group.glyphs) { + total += 8 + 4 * glyph.bbxHeight; + } + }); + const buffer = new ArrayBuffer(total); + const view = new DataView(buffer); + const bytes = new Uint8Array(buffer); + let offset = 0; + registration.forEach((group, state) => { + const keyBytes = encodedKeys.get(state)!; + view.setUint8(offset, keyBytes.length); + bytes.set(keyBytes, offset + 1); + offset += 1 + keyBytes.length; + view.setUint8(offset, state.cellHeight); + view.setUint16(offset + 1, group.glyphs.length, true); + offset += 3; + for (const glyph of group.glyphs) { + const inkTop = group.font.ascent - (glyph.bbxHeight + glyph.bbxY); + view.setUint32(offset, glyph.encoding, true); + view.setInt8(offset + 4, glyph.bbxX); + view.setUint8(offset + 5, inkTop); + view.setUint8(offset + 6, glyph.bbxWidth); + view.setUint8(offset + 7, glyph.bbxHeight); + offset += 8; + for (let row = 0; row < glyph.bbxHeight; row++) { + view.setUint32(offset, (glyph.bitmapRows[row] ?? 0) >>> 0, true); + offset += 4; + } + } + }); + return buffer; +} + +function utf8Encode(text: string): Uint8Array { + // TextEncoder exists in NativeScript's JS runtime; fall back to code units + // (atlas keys are ASCII font names, so both paths agree). + if (typeof TextEncoder !== "undefined") { + return new TextEncoder().encode(text); + } + const out = new Uint8Array(text.length); + for (let i = 0; i < text.length; i++) { + out[i] = text.charCodeAt(i) & 0x7f; + } + return out; +} diff --git a/app/graphics/image.ts b/app/graphics/image.ts index 245255b2..cc115708 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -24,11 +24,12 @@ export function imageFromAsciiArt(lines: readonly string[], value = 255): GrayIm /** * One deferred glyph draw. Text drawn through drawGlyph/drawText/ * drawTextWrapped is retained here rather than baked into the pixel buffer, - * so the compositor can later ship glyphs as display-list references instead - * of pixels. (x, y) and value are the drawGlyph arguments (y is the top of + * so the compositor can ship glyphs as on-glasses cached draws instead of + * pixels. (x, y) and value are the drawGlyph arguments (y is the top of * the line; the baseline is y + font.ascent). */ export type PlacedGlyph = { + kind: "glyph"; font: BdfFont; glyph: Glyph; x: number; @@ -36,16 +37,32 @@ export type PlacedGlyph = { value: number; }; +/** + * One deferred image draw (drawImage): a color-key blit of `source` at + * (x, y), retained so icons can ship as on-glasses cached-image draws. + * `source` is flat (no deferred draws of its own) and treated as immutable + * from the moment it is placed. + */ +export type PlacedImage = { + kind: "image"; + source: GrayImage; + x: number; + y: number; +}; + +/** + * A retained draw, always rendered on top of the raster pixels regardless of + * draw order (deferred draws keep their order among themselves). An overlay + * that must cover deferred draws below it needs its own plane (see plane.ts) + * or must suppress the covered draws. + */ +export type DeferredDraw = PlacedGlyph | PlacedImage; + export class GrayImage { readonly width: number; readonly height: number; readonly pixels: Uint8Array; - /** - * Deferred glyph draws, always rendered on top of the raster pixels - * regardless of draw order. An overlay that must cover text below it needs - * its own plane (see plane.ts) or must suppress the covered draws. - */ - private readonly glyphList: PlacedGlyph[] = []; + private readonly drawList: DeferredDraw[] = []; constructor(width: number, height: number, fill = 0) { this.width = width; @@ -54,24 +71,24 @@ export class GrayImage { this.clear(fill); } - get glyphs(): readonly PlacedGlyph[] { - return this.glyphList; + get draws(): readonly DeferredDraw[] { + return this.drawList; } - hasGlyphs(): boolean { - return this.glyphList.length > 0; + hasDeferredDraws(): boolean { + return this.drawList.length > 0; } clone(): GrayImage { const clone = new GrayImage(this.width, this.height, 0); clone.pixels.set(this.pixels); - clone.glyphList.push(...this.glyphList); + clone.drawList.push(...this.drawList); return clone; } clear(value = 0): void { this.pixels.fill(clampByte(value)); - this.glyphList.length = 0; + this.drawList.length = 0; } setPixel(x: number, y: number, value: number): void { @@ -199,19 +216,19 @@ export class GrayImage { } } - // Carry the source's deferred glyphs along, translated into destination - // space. Membership in the copied rect is judged by the glyph's anchor + // Carry the source's deferred draws along, translated into destination + // space. Membership in the copied rect is judged by the draw's anchor // (fine for the whole-image blits this is used for; a sub-rect blit can - // clip a glyph's overhang imperfectly). - if (source.glyphList.length) { + // clip a draw's overhang imperfectly). + if (source.drawList.length) { const offsetX = destX - srcX; const offsetY = destY - srcY; - for (const placed of source.glyphList) { + for (const placed of source.drawList) { if ( placed.x >= srcX && placed.x < srcX + copyWidth && placed.y >= srcY && placed.y < srcY + copyHeight ) { - this.glyphList.push({ ...placed, x: placed.x + offsetX, y: placed.y + offsetY }); + this.drawList.push({ ...placed, x: placed.x + offsetX, y: placed.y + offsetY }); } } } @@ -292,63 +309,91 @@ export class GrayImage { } } - fingerprint(): string { + /** 32-bit content hash over pixels and deferred draws (fingerprint's core). */ + contentHash32(): number { let hash = 2166136261; for (let i = 0; i < this.pixels.length; i++) { hash ^= this.pixels[i]!; hash = Math.imul(hash, 16777619); } - for (const placed of this.glyphList) { - hash = mixInt(hash, placed.font.fingerprintId); - hash = mixInt(hash, placed.glyph.encoding); + for (const placed of this.drawList) { + if (placed.kind === "glyph") { + hash = mixInt(hash, placed.font.fingerprintId); + hash = mixInt(hash, placed.glyph.encoding); + hash = mixInt(hash, placed.value); + } else { + hash = mixInt(hash, 0x1c0e5); + hash = mixInt(hash, placed.source.width); + hash = mixInt(hash, placed.source.height); + hash = mixInt(hash, placed.source.contentHash32()); + } hash = mixInt(hash, placed.x); hash = mixInt(hash, placed.y); - hash = mixInt(hash, placed.value); } - return `fnv:${(hash >>> 0).toString(16)}`; + return hash >>> 0; + } + + fingerprint(): string { + return `fnv:${this.contentHash32().toString(16)}`; } /** * Snapshot of the full image as a single 8-bit-per-pixel grayscale buffer, - * row-major and top-to-bottom, with any deferred glyphs baked in. The Java + * row-major and top-to-bottom, with any deferred draws baked in. The Java * side turns this into whatever wire format the firmware currently expects * (today: a 4bpp BMP), so all framing concerns stay on one side of the * bridge. */ to8bppBuffer(): Uint8Array { - if (!this.glyphList.length) { + if (!this.drawList.length) { return Uint8Array.from(this.pixels); } - // withGlyphsBaked's buffer is a private temporary, safe to hand out. - return this.withGlyphsBaked().pixels; + // withDrawsBaked's buffer is a private temporary, safe to hand out. + return this.withDrawsBaked().pixels; } /** - * Record a glyph draw. Deferred: the glyph joins the image's glyph list + * Record a glyph draw. Deferred: the glyph joins the image's draw list * (rendered above all raster pixels) instead of being baked into the * buffer. */ drawGlyph(font: BdfFont, glyph: Glyph, x: number, y: number, value: number): void { - this.glyphList.push({ font, glyph, x, y, value: clampByte(value) }); + this.drawList.push({ kind: "glyph", font, glyph, x, y, value: clampByte(value) }); } /** - * This image with its glyph list rasterized into the pixel buffer. Returns - * this image unchanged when there are no glyphs, otherwise a baked copy - * with an empty glyph list. + * Record a color-key image blit (source value 0 = transparent), deferred + * like glyph draws so icons keep their identity to the wire encoder. The + * source is snapshot-flattened (its own deferred draws baked) and must not + * be mutated afterwards — pass long-lived rendered assets (icons), not + * scratch buffers. For a clipped or mutable blit, use bitBlt instead. */ - withGlyphsBaked(): GrayImage { - if (!this.glyphList.length) return this; + drawImage(source: GrayImage, x: number, y: number): void { + if (source.width <= 0 || source.height <= 0) return; + this.drawList.push({ kind: "image", source: source.withDrawsBaked(), x, y }); + } + + /** + * This image with its deferred draws rasterized into the pixel buffer. + * Returns this image unchanged when there are none, otherwise a baked copy + * with an empty draw list. + */ + withDrawsBaked(): GrayImage { + if (!this.drawList.length) return this; const baked = new GrayImage(this.width, this.height, 0); baked.pixels.set(this.pixels); - this.bakeGlyphsInto(baked, 0, 0); + this.bakeDrawsInto(baked, 0, 0); return baked; } - /** Rasterize this image's glyph list into target, translated by (dx, dy). */ - bakeGlyphsInto(target: GrayImage, dx: number, dy: number): void { - for (const placed of this.glyphList) { - rasterizeGlyph(target, placed.font, placed.glyph, placed.x + dx, placed.y + dy, placed.value); + /** Rasterize this image's deferred draws into target, translated by (dx, dy). */ + bakeDrawsInto(target: GrayImage, dx: number, dy: number): void { + for (const placed of this.drawList) { + if (placed.kind === "glyph") { + rasterizeGlyph(target, placed.font, placed.glyph, placed.x + dx, placed.y + dy, placed.value); + } else { + target.bitBlt(placed.source, placed.x + dx, placed.y + dy, { transparentZero: true }); + } } } } diff --git a/app/graphics/plane.ts b/app/graphics/plane.ts index ab6b7050..e29ff88f 100644 --- a/app/graphics/plane.ts +++ b/app/graphics/plane.ts @@ -1,4 +1,4 @@ -import { GrayImage } from "./image"; +import { GrayImage, type DeferredDraw } from "./image"; /** * The unit apps (and the shell) submit to the compositor: instead of one @@ -49,7 +49,7 @@ export function flattenPlanes( const only = planes.length === 1 ? planes[0]! : undefined; if (only && only.x === 0 && only.y === 0 && only.image.width === width && only.image.height === height) { - return only.image.withGlyphsBaked(); + return only.image.withDrawsBaked(); } const target = new GrayImage(width, height, 0); @@ -57,11 +57,37 @@ export function flattenPlanes( // Baking first (rather than bitBlt's glyph carry-over) is what enforces // the plane ordering: the next plane's raster must be able to cover this // plane's glyphs. - target.bitBlt(plane.image.withGlyphsBaked(), plane.x, plane.y, { transparentZero: true }); + target.bitBlt(plane.image.withDrawsBaked(), plane.x, plane.y, { transparentZero: true }); } return target; } +/** + * Flatten planes and also return the frame's deferred draws (glyphs and + * images), translated into frame coordinates and in bake order (plane order, + * then each image's draw order). The flattened image has every draw baked in, + * exactly as flattenPlanes produces; the list preserves the draws' identity + * so the texture-cache pipeline can replay them as on-glasses cached draws + * (see graphics/glyph-wire.ts). + */ +export function flattenPlanesWithDraws( + planes: readonly Plane[], + size?: { width: number; height: number }, +): { image: GrayImage; draws: DeferredDraw[] } { + const image = flattenPlanes(planes, size); + const draws: DeferredDraw[] = []; + for (const plane of planes) { + for (const placed of plane.image.draws) { + draws.push( + plane.x === 0 && plane.y === 0 + ? placed + : { ...placed, x: placed.x + plane.x, y: placed.y + plane.y }, + ); + } + } + return { image, draws }; +} + /** * Stable identifier for a plane stack's composited content: equal * fingerprints mean an equal flattened frame. Feeds the same dedupe paths diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 400e3ece..78481a92 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -515,6 +515,12 @@ export class FaceclawCommunicatorBridge { fingerprint: string, paintMs = -1, frameId = 0, + /** + * The frame's glyph draws in surface coordinates (see + * graphics/glyph-wire.ts prepareFrameGlyphs); lets the texture-cache + * pipeline ship text as on-glasses cached draws instead of pixels. + */ + glyphs: ArrayBuffer | null = null, ): Promise { // Snapshot because the Java call is deferred; the buffer is passed as an // ArrayBuffer, which NativeScript marshals to a ByteBuffer without the @@ -531,6 +537,7 @@ export class FaceclawCommunicatorBridge { fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), + glyphs, ); }); } diff --git a/app/native/gray-image-preview.ts b/app/native/gray-image-preview.ts index e3e3cff2..690c1e4b 100644 --- a/app/native/gray-image-preview.ts +++ b/app/native/gray-image-preview.ts @@ -17,7 +17,7 @@ export function grayImageToPreviewSource(image: GrayImage): ImageSource | null { return null; } - const baked = image.withGlyphsBaked(); + const baked = image.withDrawsBaked(); const bitmap = com.faceclaw.app.PreviewBitmapUtil.fromGray( baked.pixels.buffer, baked.width, diff --git a/app/ui/shell/chrome-layer.ts b/app/ui/shell/chrome-layer.ts index a8a16fba..27a7e620 100644 --- a/app/ui/shell/chrome-layer.ts +++ b/app/ui/shell/chrome-layer.ts @@ -95,6 +95,30 @@ export function makeLetterWindowIcon(letter: string): ShellChromeWindow["drawIco }; } +/** + * Inverted (black-on-white) variant of an icon raster, for the focused tab: + * coverage becomes darkness, with full coverage mapping to SHELL_OPAQUE_BLACK + * (1) rather than 0, since 0 is the surface's transparent color key. Memoized + * per source object so the variant is a stable image the texture cache can + * key on (mode-13's inverse LUT can't reproduce this mapping exactly, so it + * is simply a separate cached image). + */ +const invertedIconCache = new WeakMap(); +function invertedIcon(icon: GrayImage): GrayImage { + let inverted = invertedIconCache.get(icon); + if (!inverted) { + inverted = new GrayImage(icon.width, icon.height, 0); + for (let i = 0; i < icon.pixels.length; i++) { + const coverage = icon.pixels[i]!; + if (coverage > 0) { + inverted.pixels[i] = Math.max(SHELL_OPAQUE_BLACK, 255 - coverage); + } + } + invertedIconCache.set(icon, inverted); + } + return inverted; +} + /** Window icon rendered from an SVG (Lucide), rendered once per size and cached. */ export function makeSvgWindowIcon(name: IconName, glyph?: string): ShellChromeWindow["drawIcon"] { return (image, x, y, size, inverted) => { @@ -102,21 +126,7 @@ export function makeSvgWindowIcon(name: IconName, glyph?: string): ShellChromeWi if (!icon) return; const dx = x + Math.max(0, ((size - icon.width) / 2) | 0); const dy = y + Math.max(0, ((size - icon.height) / 2) | 0); - if (!inverted) { - image.bitBlt(icon, dx, dy, { transparentZero: true }); - return; - } - // Invert onto a white background: coverage becomes darkness. Full coverage - // maps to SHELL_OPAQUE_BLACK (1) rather than 0, since 0 is the surface's - // transparent color key. - for (let row = 0; row < icon.height; row++) { - for (let col = 0; col < icon.width; col++) { - const coverage = icon.pixels[row * icon.width + col] ?? 0; - if (coverage > 0) { - image.setPixel(dx + col, dy + row, Math.max(SHELL_OPAQUE_BLACK, 255 - coverage)); - } - } - } + image.drawImage(inverted ? invertedIcon(icon) : icon, dx, dy); }; } @@ -138,18 +148,7 @@ export function makeImageWindowIcon( } const dx = x + Math.max(0, ((size - icon.width) / 2) | 0); const dy = y + Math.max(0, ((size - icon.height) / 2) | 0); - if (!inverted) { - image.bitBlt(icon, dx, dy, { transparentZero: true }); - return; - } - for (let row = 0; row < icon.height; row++) { - for (let col = 0; col < icon.width; col++) { - const coverage = icon.pixels[row * icon.width + col] ?? 0; - if (coverage > 0) { - image.setPixel(dx + col, dy + row, Math.max(SHELL_OPAQUE_BLACK, 255 - coverage)); - } - } - } + image.drawImage(inverted ? invertedIcon(icon) : icon, dx, dy); }; } @@ -243,8 +242,10 @@ export class ShellChromeLayer implements Layer { } window.drawIcon(image, x, y, iconSize, focused); if (window.attention) { - // Black dot on the white focused tab, white dot otherwise. - image.fillRoundedRect(x + iconSize - 7, y - 1, 8, 8, focused ? SHELL_OPAQUE_BLACK : 255, 4); + // Black dot on the white focused tab, white dot otherwise. A deferred + // image (not a raster fill) so it renders above the icon, which is + // itself a deferred draw. + image.drawImage(attentionDot(focused ? SHELL_OPAQUE_BLACK : 255), x + iconSize - 7, y - 1); } } @@ -287,9 +288,7 @@ export class ShellChromeLayer implements Layer { } const iconY = barTop + (((TOP_BAR_HEIGHT - NOTIFICATION_ICON_SIZE) / 2) | 0); for (let index = 0; index < icons.length; index++) { - image.bitBlt(icons[index]!, iconsX + index * (NOTIFICATION_ICON_SIZE + 4), iconY, { - transparentZero: true, - }); + image.drawImage(icons[index]!, iconsX + index * (NOTIFICATION_ICON_SIZE + 4), iconY); } } } @@ -354,13 +353,23 @@ function drawTrayIcons(image: GrayImage, trayIcons: GrayImage[], rightEdge: numb for (let index = trayIcons.length - 1; index >= 0; index--) { const icon = trayIcons[index]!; x -= icon.width + 10; - image.bitBlt(icon, x, barTop + Math.max(0, ((TOP_BAR_HEIGHT - icon.height) / 2) | 0), { - transparentZero: true, - }); + image.drawImage(icon, x, barTop + Math.max(0, ((TOP_BAR_HEIGHT - icon.height) / 2) | 0)); } return x; } +/** The sidebar attention marker, cached per fill value (deferred-image source). */ +const attentionDots = new Map(); +function attentionDot(value: number): GrayImage { + let dot = attentionDots.get(value); + if (!dot) { + dot = new GrayImage(8, 8, 0); + dot.fillRoundedRect(0, 0, 8, 8, value, 4); + attentionDots.set(value, dot); + } + return dot; +} + const TAB_RADIUS = 6; // How far the diversion pokes past the separator into the main area. const TAB_EXTEND = 1;