Use the texture cache and builtin-font drawing firmware extensions

This commit is contained in:
James Babcock committed 2026-08-19 10:36:16 -07:00
1 parent deac81f98f
commit 90603b5aa4
9 files changed
+674 -60

No files matched your search

@@ -196,6 +196,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private final TextureCacheState textureCache = new TextureCacheState();
private boolean textureCacheSupported;
private boolean textureImagesSupported;
private boolean fwTextSupported;
private final ArrayDeque<OutboundMessage> pendingMessages = new ArrayDeque<>();
private final ArrayDeque<OutboundMessage> inFlightMessages = new ArrayDeque<>();
@@ -2231,7 +2232,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
deltaBase, packed, width, height, draws, textureCache,
nextImageFrameId,
connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS,
textureImagesSupported);
textureImagesSupported, fwTextSupported);
if (tex != null) {
nextImageFrameId = tex.nextFid;
for (byte[] upload : tex.uploads) {
@@ -2244,6 +2245,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
String texLog = "texture update " + (tex.fullFrame ? "full" : ("rects=" + tex.rectCount))
+ " glyphs=" + tex.drawnGlyphs + " runs=" + tex.runCount
+ " images=" + tex.drawnImages
+ " fw=" + tex.fwGlyphs + "/" + tex.fwRuns
+ " baked=" + tex.bakedCandidates
+ (tex.uploadBytes > 0 ? " upload=" + tex.uploadBytes + "B" : "")
+ " payload=" + tex.payload.length + "B";
@@ -2445,6 +2447,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
&& hasCapability(firmwareInfo.capabilities, "texstr14");
textureImagesSupported = hasCapability(firmwareInfo.capabilities, "teximg13");
fwTextSupported = hasCapability(firmwareInfo.capabilities, "font15");
emitFirmwareInfo(firmwareInfo);
}
};
@@ -0,0 +1,83 @@
package com.faceclaw.app;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.HashMap;
import java.util.Map;
/**
* Process-wide registry of the glasses' builtin 20 px font's glyph rasters,
* fed from the TS side (EvenHubFont's extracted firmware font data). Unlike
* GlyphAtlas/ImageAtlas entries, these are never uploaded: the firmware
* already owns the font, and mode 15 draws it directly. The planner needs
* the rasters purely phone-side — for the "would this draw land correctly"
* check against the composite and for punching drawn ink out of delta rects.
*
* Placement model matches the firmware's mode-15 math: a glyph draws at
* (penX + ofsX, lineY + inkTop) where inkTop already folds in the root
* font's line height and baseline. The TS side only registers glyphs whose
* on-glasses rendering it predicts exactly (native-table glyphs; baseline-
* shifted fallback symbols stay baked).
*/
public final class FwGlyphAtlas {
private FwGlyphAtlas() {}
public static final class Entry {
/** Draw x = pen x + ofsX. */
public final int ofsX;
/** First raster row relative to the run's line-top y. */
public final int inkTop;
public final int boxW;
public final int boxH;
/** 4bpp rows, stride ceil(boxW/2), high nibble = left pixel. */
final byte[] nibbles;
Entry(int ofsX, int inkTop, int boxW, int boxH, byte[] nibbles) {
this.ofsX = ofsX;
this.inkTop = inkTop;
this.boxW = boxW;
this.boxH = boxH;
this.nibbles = nibbles;
}
/** The 4bpp source value at (col, row); the draw skips 0 (transparent). */
public int nibbleAt(int col, int row) {
int b = nibbles[row * ((boxW + 1) >> 1) + (col >> 1)] & 0xff;
return (col & 1) != 0 ? (b & 0x0f) : (b >> 4);
}
}
private static final Object lock = new Object();
private static final Map<Integer, Entry> entries = new HashMap<>();
/**
* Register glyph rasters. Little-endian buffer of records:
* [cp u32][ofsX s16][inkTop s16][boxW u8][boxH u8]
* [boxH x ceil(boxW/2) nibble bytes]
* Re-registration of a codepoint is ignored (the builtin font is fixed).
*/
public static void register(ByteBuffer buffer) {
if (buffer == null) return;
ByteBuffer in = buffer.order(ByteOrder.LITTLE_ENDIAN);
synchronized (lock) {
while (in.remaining() >= 10) {
int cp = in.getInt();
int ofsX = in.getShort();
int inkTop = in.getShort();
int boxW = in.get() & 0xff;
int boxH = in.get() & 0xff;
byte[] nibbles = new byte[((boxW + 1) >> 1) * boxH];
in.get(nibbles);
if (boxW > 0 && boxH > 0 && !entries.containsKey(cp)) {
entries.put(cp, new Entry(ofsX, inkTop, boxW, boxH, nibbles));
}
}
}
}
public static Entry get(int cp) {
synchronized (lock) {
return entries.get(cp);
}
}
}
@@ -57,6 +57,8 @@ public final class SurfaceCompositor {
public static final class ScreenDraw {
public static final int KIND_GLYPH = 0;
public static final int KIND_IMAGE = 1;
/** A firmware-builtin-font text run (CFW mode 15). */
public static final int KIND_FWTEXT = 2;
public final int kind;
/** Glyph draws only. */
@@ -67,10 +69,17 @@ public final class SurfaceCompositor {
public final int imageId;
public final int x;
public final int y;
/** Glyph draws only: 8-bit brightness the glyph was drawn with. */
/** Glyph and fw-text draws: 8-bit brightness. */
public final int value;
/** Fw-text runs only: member codepoints, in text order. */
public final int[] fwCps;
/** Fw-text runs only: member pen offsets relative to x. */
public final int[] fwDx;
/** Fw-text runs only: whether each member has visible pixels. */
public final boolean[] fwInk;
ScreenDraw(int kind, int fontId, int encoding, int imageId, int x, int y, int value) {
ScreenDraw(int kind, int fontId, int encoding, int imageId, int x, int y, int value,
int[] fwCps, int[] fwDx, boolean[] fwInk) {
this.kind = kind;
this.fontId = fontId;
this.encoding = encoding;
@@ -78,14 +87,21 @@ public final class SurfaceCompositor {
this.x = x;
this.y = y;
this.value = value;
this.fwCps = fwCps;
this.fwDx = fwDx;
this.fwInk = fwInk;
}
static ScreenDraw glyph(int fontId, int encoding, int penX, int lineY, int value) {
return new ScreenDraw(KIND_GLYPH, fontId, encoding, 0, penX, lineY, value);
return new ScreenDraw(KIND_GLYPH, fontId, encoding, 0, penX, lineY, value, null, null, null);
}
static ScreenDraw image(int imageId, int x, int y) {
return new ScreenDraw(KIND_IMAGE, 0, 0, imageId, x, y, 0);
return new ScreenDraw(KIND_IMAGE, 0, 0, imageId, x, y, 0, null, null, null);
}
static ScreenDraw fwText(int x, int y, int value, int[] cps, int[] dx, boolean[] ink) {
return new ScreenDraw(KIND_FWTEXT, 0, 0, 0, x, y, value, cps, dx, ink);
}
}
@@ -314,6 +330,23 @@ public final class SurfaceCompositor {
int x = in.getShort();
int y = in.getShort();
out.add(ScreenDraw.image(imageId, x, y));
} else if (kind == ScreenDraw.KIND_FWTEXT && in.remaining() >= 6) {
int x = in.getShort();
int y = in.getShort();
int value = in.get() & 0xff;
int count = in.get() & 0xff;
if (in.remaining() < count * 7) {
break; // malformed tail: keep what parsed cleanly
}
int[] cps = new int[count];
int[] dx = new int[count];
boolean[] ink = new boolean[count];
for (int i = 0; i < count; i++) {
cps[i] = in.getInt();
dx[i] = in.getShort();
ink[i] = in.get() != 0;
}
out.add(ScreenDraw.fwText(x, y, value, cps, dx, ink));
} else {
break; // malformed tail: keep what parsed cleanly
}
@@ -379,7 +412,8 @@ public final class SurfaceCompositor {
blendLocked(gray, surface);
for (ScreenDraw draw : surface.draws) {
draws.add(new ScreenDraw(draw.kind, draw.fontId, draw.encoding, draw.imageId,
draw.x + surface.x, draw.y + surface.y, draw.value));
draw.x + surface.x, draw.y + surface.y, draw.value,
draw.fwCps, draw.fwDx, draw.fwInk));
}
fingerprint.append('|').append(surface.id)
.append('@').append(surface.x).append(',').append(surface.y)
@@ -32,8 +32,16 @@ public final class TexturePlanner {
/** Cap on mode-14 string sub-messages (mode-8 count is a u8). */
private static final int MAX_GLYPH_RUNS = 180;
/**
* Soft cap on cached-glyph selection per update, before run grouping.
* Sized above a full-screen 6x12 terminal repaint (~4,240 cells) so dense
* screens draw everything; the run budget is the real bound.
*/
private static final int MAX_SELECTED_GLYPHS = 4600;
/** Cap on mode-13 image sub-messages per update. */
private static final int MAX_IMAGE_DRAWS = 60;
/** Cap on mode-15 builtin-font string sub-messages per update. */
private static final int MAX_FWTEXT_RUNS = 80;
/** Mode-12 payload size that fits one BLE image message comfortably. */
private static final int UPLOAD_PAYLOAD_MAX = 3600;
/** Max x-adjust control bytes between two glyphs before starting a new run. */
@@ -52,13 +60,15 @@ public final class TexturePlanner {
public final int drawnGlyphs;
public final int runCount;
public final int drawnImages;
public final int fwGlyphs;
public final int fwRuns;
public final int bakedCandidates;
public final int uploadBytes;
public final boolean fullFrame;
Result(byte[] payload, List<byte[]> uploads, int nextFid, int rectCount,
int drawnGlyphs, int runCount, int drawnImages, int bakedCandidates,
int uploadBytes, boolean fullFrame) {
int drawnGlyphs, int runCount, int drawnImages, int fwGlyphs, int fwRuns,
int bakedCandidates, int uploadBytes, boolean fullFrame) {
this.payload = payload;
this.uploads = uploads;
this.nextFid = nextFid;
@@ -66,6 +76,8 @@ public final class TexturePlanner {
this.drawnGlyphs = drawnGlyphs;
this.runCount = runCount;
this.drawnImages = drawnImages;
this.fwGlyphs = fwGlyphs;
this.fwRuns = fwRuns;
this.bakedCandidates = bakedCandidates;
this.uploadBytes = uploadBytes;
this.fullFrame = fullFrame;
@@ -93,15 +105,15 @@ public final class TexturePlanner {
/**
* Plan a cached-draw update. previous is the delta base (the frame the
* shadow currently holds), or null/mismatched for a full-frame keyframe.
* allowImages requires the teximg13 capability. Returns null when the
* plain paths should run instead (no replayable draws, or identical
* frames).
* allowImages requires the teximg13 capability; allowFwText requires
* font15. Returns null when the plain paths should run instead (no
* replayable draws, or identical frames).
*/
public static Result plan(
byte[] previous, byte[] next, int width, int height,
SurfaceCompositor.ScreenDraw[] draws,
TextureCacheState cache, int fidStart,
boolean allowMultiRect, int maxRects, boolean allowImages) {
boolean allowMultiRect, int maxRects, boolean allowImages, boolean allowFwText) {
if (next == null || draws == null || draws.length == 0 || width <= 0 || height <= 0) {
return null;
}
@@ -136,6 +148,21 @@ public final class TexturePlanner {
List<Selected> selected = new ArrayList<>();
int bakedCandidates = 0;
int selectedImages = 0;
// Builtin-font (mode 15) runs: no cache residency, no uploads — the
// firmware owns the font. Processed first because it has no cache
// side effects; the counters feed the shared stats.
List<byte[]> fwSubs = new ArrayList<>();
List<FwPunch> fwPunches = new ArrayList<>();
int fwGlyphCount = 0;
int fwBaked = 0;
if (allowFwText) {
for (SurfaceCompositor.ScreenDraw draw : draws) {
if (draw.kind != SurfaceCompositor.ScreenDraw.KIND_FWTEXT) continue;
fwBaked += planFwRun(draw, rects, next, stride, width, height, fwSubs, fwPunches);
}
fwGlyphCount = fwPunches.size();
bakedCandidates += fwBaked;
}
for (SurfaceCompositor.ScreenDraw draw : draws) {
if (draw.kind == SurfaceCompositor.ScreenDraw.KIND_GLYPH) {
GlyphAtlas.Glyph atlas = GlyphAtlas.get(draw.fontId, draw.encoding);
@@ -155,7 +182,7 @@ public final class TexturePlanner {
bakedCandidates++;
continue;
}
if (selected.size() >= MAX_GLYPH_RUNS * 10) { // soft cap before run grouping
if (selected.size() >= MAX_SELECTED_GLYPHS) {
bakedCandidates++;
continue;
}
@@ -180,21 +207,20 @@ public final class TexturePlanner {
selectedImages++;
}
}
if (selected.isEmpty()) {
if (selected.isEmpty() && fwSubs.isEmpty()) {
return null;
}
// Residency: ensure every selected draw is in the cache, retrying once
// after a reset when the bump allocator fills. A draw that still fails
// falls back to baked. IMPORTANT: from here on, every bail-out to the
// plain path must cache.reset() — ensure* marked entries resident
// and queued uploads that only a returned Result would enqueue.
List<Selected> drawable = ensureResident(cache, selected);
// falls back to baked. IMPORTANT: once ensure* has run, every bail-out
// to the plain path must cache.reset() — ensure* marked entries
// resident and queued uploads that only a returned Result would
// enqueue. (Fw-text needs no residency, so a fw-only plan skips this.)
List<Selected> drawable = selected.isEmpty()
? Collections.<Selected>emptyList()
: ensureResident(cache, selected);
bakedCandidates += selected.size() - drawable.size();
if (drawable.isEmpty()) {
cache.reset();
return null;
}
// Group glyphs into mode-14 runs; leftover-budget glyphs bake.
List<Selected> glyphDrawable = new ArrayList<>();
@@ -208,8 +234,11 @@ public final class TexturePlanner {
bakedCandidates += glyphDrawable.size() - drawnGlyphs.size();
List<Selected> drawn = new ArrayList<>(drawnGlyphs);
drawn.addAll(imageDrawable);
if (drawn.isEmpty() || rects.size() + runs.size() + imageDrawable.size() > 255) {
cache.reset();
if ((drawn.isEmpty() && fwSubs.isEmpty())
|| rects.size() + runs.size() + imageDrawable.size() + fwSubs.size() > 255) {
if (!selected.isEmpty()) {
cache.reset();
}
return null;
}
@@ -219,6 +248,9 @@ public final class TexturePlanner {
for (Selected sel : drawn) {
punch(punched, stride, width, height, sel);
}
for (FwPunch fw : fwPunches) {
punchFw(punched, stride, width, height, fw);
}
List<byte[]> subs = new ArrayList<>();
int fid = fidStart;
@@ -234,12 +266,184 @@ public final class TexturePlanner {
subs.add(encodeImageDraw(cache, sel));
}
subs.addAll(runs);
subs.addAll(fwSubs);
byte[] payload = assembleMode8(subs);
int uploadBytes = cache.pendingUploadBytes();
List<byte[]> uploads = cache.drainUploadPayloads(UPLOAD_PAYLOAD_MAX);
return new Result(payload, uploads, fid, rects.size(), drawnGlyphs.size(), runs.size(),
imageDrawable.size(), bakedCandidates, uploadBytes, fullFrame);
imageDrawable.size(), fwGlyphCount, fwSubs.size(),
bakedCandidates, uploadBytes, fullFrame);
}
/** One punch task for an emitted firmware-font glyph. */
private static final class FwPunch {
final FwGlyphAtlas.Entry entry;
final int gx;
final int gy;
FwPunch(FwGlyphAtlas.Entry entry, int gx, int gy) {
this.entry = entry;
this.gx = gx;
this.gy = gy;
}
}
/**
* Plan one firmware-font run (mode 15): classify every member, and when
* any correct ink member intersects a sent rect, emit sub-messages over
* the maximal correct stretches. A stretch never spans an incorrect
* member — the sub-message boundary restarts the firmware's advance walk
* at an explicit pen x, so occluded or unknown glyphs simply stay baked
* without disturbing their neighbors' positions. Emitted ink members are
* queued for punching. Returns how many ink members stay baked.
*/
private static int planFwRun(
SurfaceCompositor.ScreenDraw draw, List<int[]> rects,
byte[] next, int stride, int width, int height,
List<byte[]> outSubs, List<FwPunch> outPunches) {
int n = draw.fwCps.length;
if (n == 0 || draw.y < 0 || draw.y > 0xffff) {
return countInk(draw);
}
int top = BmpUtil.nibbleForGray(draw.value);
boolean[] ok = new boolean[n];
FwGlyphAtlas.Entry[] entries = new FwGlyphAtlas.Entry[n];
boolean relevant = false;
int baked = 0;
for (int i = 0; i < n; i++) {
if (!draw.fwInk[i]) {
ok[i] = true; // inkless connector (space): draws nothing
continue;
}
FwGlyphAtlas.Entry entry = FwGlyphAtlas.get(draw.fwCps[i]);
if (entry == null) {
baked++;
continue;
}
int gx = draw.x + draw.fwDx[i] + entry.ofsX;
int gy = draw.y + entry.inkTop;
if (!fwGlyphMatchesComposite(next, stride, width, height, entry, gx, gy, top)) {
baked++;
continue;
}
ok[i] = true;
entries[i] = entry;
if (intersectsAny(rects, gx, gy, entry.boxW, entry.boxH)) {
relevant = true;
}
}
if (!relevant) {
return 0; // nothing in a sent rect: leave the run's pixels alone
}
int i = 0;
while (i < n && outSubs.size() < MAX_FWTEXT_RUNS) {
while (i < n && !(ok[i] && draw.fwInk[i])) i++;
if (i >= n) break;
int startX = draw.x + draw.fwDx[i];
if (startX < 0 || startX > 0xffff) {
baked++;
i++;
continue;
}
// Extend across correct members while the UTF-8 payload fits,
// tracking the last ink member so trailing connectors trim away.
int j = i;
int lastInk = -1;
int bytesToLastInk = 0;
int bytesSoFar = 0;
while (j < n && ok[j]) {
int encodedLength = utf8Length(draw.fwCps[j]);
if (bytesSoFar + encodedLength > 255) break;
bytesSoFar += encodedLength;
if (draw.fwInk[j]) {
lastInk = j;
bytesToLastInk = bytesSoFar;
}
j++;
}
if (lastInk < i) { // a single over-long codepoint; not realistic
baked++;
i++;
continue;
}
byte[] sub = new byte[7 + bytesToLastInk];
sub[0] = 15;
sub[1] = (byte) (startX & 0xff);
sub[2] = (byte) ((startX >> 8) & 0xff);
sub[3] = (byte) (draw.y & 0xff);
sub[4] = (byte) ((draw.y >> 8) & 0xff);
sub[5] = (byte) (top | 0x10); // top color + transparent
sub[6] = (byte) bytesToLastInk;
int pos = 7;
for (int k = i; k <= lastInk; k++) {
byte[] encoded = new StringBuilder().appendCodePoint(draw.fwCps[k])
.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8);
System.arraycopy(encoded, 0, sub, pos, encoded.length);
pos += encoded.length;
}
outSubs.add(sub);
for (int k = i; k <= lastInk; k++) {
if (draw.fwInk[k]) {
outPunches.add(new FwPunch(entries[k],
draw.x + draw.fwDx[k] + entries[k].ofsX,
draw.y + entries[k].inkTop));
}
}
i = lastInk + 1;
}
return baked;
}
private static int utf8Length(int cp) {
if (cp < 0x80) return 1;
if (cp < 0x800) return 2;
if (cp < 0x10000) return 3;
return 4;
}
private static int countInk(SurfaceCompositor.ScreenDraw draw) {
int count = 0;
for (boolean ink : draw.fwInk) {
if (ink) count++;
}
return count;
}
/**
* Whether every in-panel nonzero pixel of the builtin-font glyph lands
* correct: the composite's 4bpp value must equal the firmware's LUT
* output src*top/15 (integer division — a general LUT check, since these
* glyphs are anti-aliased).
*/
private static boolean fwGlyphMatchesComposite(byte[] packed, int stride, int width, int height,
FwGlyphAtlas.Entry entry, int gx, int gy, int top) {
for (int row = 0; row < entry.boxH; row++) {
int y = gy + row;
if (y < 0 || y >= height) continue;
for (int col = 0; col < entry.boxW; col++) {
int source = entry.nibbleAt(col, row);
if (source == 0) continue;
int x = gx + col;
if (x < 0 || x >= width) continue;
if (nibbleAt(packed, stride, x, y) != (source * top) / 15) {
return false;
}
}
}
return true;
}
private static void punchFw(byte[] packed, int stride, int width, int height, FwPunch fw) {
for (int row = 0; row < fw.entry.boxH; row++) {
int y = fw.gy + row;
if (y < 0 || y >= height) continue;
for (int col = 0; col < fw.entry.boxW; col++) {
if (fw.entry.nibbleAt(col, row) == 0) continue;
punchPixel(packed, stride, width, fw.gx + col, y);
}
}
}
private static boolean intersectsAny(List<int[]> rects, int x, int y, int w, int h) {