diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index d6d6ae06..eae419de 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -182,6 +182,11 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { // into desiredPacked; the texture-cache planner may replay them as // on-glasses cached draws. private SurfaceCompositor.ScreenDraw[] desiredDraws = new SurfaceCompositor.ScreenDraw[0]; + // (frame, reason) of the last "waiting to send" line, so a frame that + // stalls for seconds records one line per state change (see + // noteImageStallLocked). Send-loop thread only. + private int stallFrameId; + private String stallReason = ""; // Highest compositor sequence stored as the desired frame; composites that // lost a store race to a newer one are discarded (their content is already // included in the newer composite). @@ -646,10 +651,15 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { * frame landed while briefly hidden would stay blank until its next repaint. */ public void setSurfaceVisible(String id, boolean visible) { + // Its own frame: this recomposite is a real screen update with real + // latency, and without one it would show up in other frames' logs only + // as an anonymous "superseded by frame#0". + int frameId = FrameTimings.getInstance().startFrame( + "compositor:visible " + id + "=" + visible); compositor.setSurfaceVisible(id, visible); SurfaceCompositor.Composite composite = compositor.composite(); byte[] packed = BmpUtil.pack4bppFromGray8(composite.gray, composite.width, composite.height); - storeDesiredComposite(composite, packed, 0, 0); + storeDesiredComposite(composite, packed, 0, frameId); } /** @@ -658,10 +668,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { * unblanking restores the previous screen content without repaints. */ public void setScreenBlanked(boolean blanked) { + int frameId = FrameTimings.getInstance().startFrame( + "compositor:" + (blanked ? "blank" : "unblank")); compositor.setBlanked(blanked); SurfaceCompositor.Composite composite = compositor.composite(); byte[] packed = BmpUtil.pack4bppFromGray8(composite.gray, composite.width, composite.height); - storeDesiredComposite(composite, packed, 0, 0); + storeDesiredComposite(composite, packed, 0, frameId); } /** @@ -722,6 +734,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { java.nio.ByteBuffer glyphs ) { Log.i(TAG, "Received an updated frame for surface " + surfaceId); + FrameTimings.getInstance().log(frameId, "surface " + surfaceId + " updated rect=" + + rectWidth + "x" + rectHeight + "+" + rectX + "+" + rectY + + (glyphs == null ? " (no glyph draws)" : "")); FrameTimings.getInstance().spanStart(frameId, "composite"); SurfaceCompositor.Composite composite = compositor.applyAndComposite( surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs); @@ -1702,10 +1717,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { // A frame ready to send right now: don't inject a fresh // heartbeat in front of it (the image's own ack resets the // firmware heartbeat timer, so the heartbeat is redundant). + // "Ready" covers both a fresh composite still to be planned + // and a planned image already queued but not yet written -- + // enqueueing sets lastEnqueuedFingerprint, so testing only + // the desired fingerprint left the queued-but-unwritten + // window unprotected and a heartbeat that came due there + // cost the frame a full ack round trip (measured 124ms on + // frame#232 of the 2026-08-20 02:27 capture). boolean imageWaiting = !shutdownRequested && fixedLayoutCreated && warmedUp && !hasPendingOrInflightKindLocked("heartbeat") && now >= imageRetryAfterMs - && !getDesiredFingerprint().equals(lastEnqueuedFingerprint); + && (hasPendingImageLocked() + || !getDesiredFingerprint().equals(lastEnqueuedFingerprint)); + noteImageStallLocked(now, windowHasRoom); if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) { return ConnectionOptions.IDLE_SLEEP_MS; } @@ -2249,7 +2273,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { + " baked=" + tex.bakedCandidates + (tex.uploadBytes > 0 ? " upload=" + tex.uploadBytes + "B" : "") + " cache=" + textureCache.usedBytes() + "B" - + " payload=" + tex.payload.length + "B"; + + " payload=" + tex.payload.length + "B" + + " (rects " + tex.rectsMs + "ms, match " + tex.matchMs + + "ms, cache " + tex.cacheMs + "ms, punch " + tex.punchMs + + "ms, encode " + tex.encodeMs + "ms)"; FrameTimings.getInstance().log(frameId, texLog); finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId); return; @@ -2511,6 +2538,117 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { : "Glasses charging."; } + /** + * Record, into the frame that is waiting, why it did not go out on this + * pass of the send loop. Without this the export shows a bare multi-second + * jump between "image submitted as desired frame" and the first BLE + * packet, with no hint whether we were blocked on warmup, a heartbeat, the + * BLE window, or another message queued ahead. Deduped on (frame, reason), + * so a frame stalled for seconds gets one line per state change rather + * than one per loop pass. + */ + private void noteImageStallLocked(long now, boolean windowHasRoom) { + int frameId; + synchronized (desiredTilesLock) { + frameId = desiredFrameId; + } + String reason; + if (frameId != 0 && !getDesiredFingerprint().equals(lastEnqueuedFingerprint)) { + reason = describeEnqueueBlockerLocked(now, windowHasRoom); + } else { + // Nothing waiting to be planned; an already-planned image may still + // be queued behind other traffic. + OutboundMessage queuedImage = firstPendingImageLocked(); + frameId = queuedImage == null ? 0 : imageUpdateFrameIdLocked(queuedImage.imageUpdateId); + reason = queuedImage == null ? null : describeWriteBlockerLocked(now, windowHasRoom, queuedImage); + } + if (frameId == 0 || reason == null) { + stallFrameId = 0; + stallReason = ""; + return; + } + if (frameId == stallFrameId && reason.equals(stallReason)) { + return; + } + stallFrameId = frameId; + stallReason = reason; + FrameTimings.getInstance().log(frameId, "waiting to send: " + reason); + } + + /** Why the desired composite has not been turned into wire messages yet, or null. */ + private String describeEnqueueBlockerLocked(long now, boolean windowHasRoom) { + if (shutdownRequested) { + return "shutdown requested"; + } + if (!sessionReady) { + return "BLE session not ready"; + } + if (!fixedLayoutCreated) { + return "display layout not created yet"; + } + if (!warmedUp) { + return "waiting for the warmup frame"; + } + if (now < imageRetryAfterMs) { + return "image retry backoff (" + (imageRetryAfterMs - now) + "ms left)"; + } + if (hasPendingImageLocked()) { + return "an earlier image is still queued"; + } + if (!windowHasRoom) { + return "BLE window full (" + inFlightMessages.size() + " message(s) in flight)"; + } + if (hasPendingOrInflightKindLocked("heartbeat")) { + return "heartbeat in flight"; + } + if (!pendingMessages.isEmpty()) { + return pendingMessages.size() + " message(s) queued ahead, next " + + pendingMessages.peekFirst().label; + } + return null; + } + + /** Why a planned image message has not been written to BLE yet, or null. */ + private String describeWriteBlockerLocked(long now, boolean windowHasRoom, OutboundMessage queuedImage) { + if (!sessionReady) { + return "BLE session not ready"; + } + if (!windowHasRoom) { + return "BLE window full (" + inFlightMessages.size() + " message(s) in flight)"; + } + if (hasPendingOrInflightKindLocked("heartbeat")) { + return "heartbeat in flight"; + } + // handleHeartbeat is a barrier: while one is due it holds back every + // other write, so a frame queued at the wrong moment waits a heartbeat + // round trip. Reported explicitly because it is otherwise invisible -- + // heartbeats belong to no frame. + long heartbeatElapsedMs = now - lastHeartbeatAckedAtMs; + if (warmedUp && fixedLayoutCreated && !shutdownRequested + && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS) { + return "heartbeat due (" + heartbeatElapsedMs + "ms since the last one acked)"; + } + OutboundMessage head = pendingMessages.peekFirst(); + if (head != null && head != queuedImage) { + return "queued behind " + head.label; + } + return null; + } + + private OutboundMessage firstPendingImageLocked() { + for (OutboundMessage message : pendingMessages) { + if (message.imageUpdateId > 0) { + return message; + } + } + return null; + } + + private int imageUpdateFrameIdLocked(int imageUpdateId) { + BleImageOptimizer.ImageUpdateStats stats = imageUpdateStats.get(imageUpdateId); + return stats == null ? 0 : stats.frameId; + } + private void finishDesiredFrameLocked(String outcome) { int frameIdToFinish; synchronized (desiredTilesLock) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java index a0620785..4dba5ac0 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.java @@ -14,8 +14,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -27,14 +26,21 @@ import java.util.Map; * update has been fully transmitted to the glasses, is discarded unsent, or * times out because nobody reported finishing it. * - * All public methods are thread-safe and cheap enough to call from the BLE - * worker thread, the Android main thread, and the JS thread. Frame IDs are - * positive ints; 0 means "no frame" and is silently ignored everywhere, so - * callers do not need to null-check. + * Frames form a tree. One input event can fan out into several renders (the + * focused app's window plus the shell chrome, say), and each of those is its + * own frame linked back to the input frame that caused it via + * {@link #startFrame(String, int)}. The export nests descendants under their + * root and renders every line against the root's clock, so a single block + * shows the whole input-to-pixels story. Latency percentiles are measured over + * roots: root start to the first descendant that actually reached the glasses. * - * Statistics (p50/p90/p99 of event-to-last-ack for sent frames) and full log - * lines for recent and slowest frames are periodically exported to - * getExternalFilesDir()/frame-timings.txt, retrievable via + * All public methods are thread-safe and cheap enough to call from the BLE + * worker thread, the Android main thread, the JS thread, and app worker + * threads. Frame IDs are positive ints; 0 means "no frame" and is silently + * ignored everywhere, so callers do not need to null-check. + * + * Statistics and full log lines for recent and slowest frames are periodically + * exported to getExternalFilesDir()/frame-timings.txt, retrievable via * adb pull /sdcard/Android/data//files/frame-timings.txt */ public final class FrameTimings { @@ -45,31 +51,66 @@ public final class FrameTimings { private static final long FRAME_TIMEOUT_MS = 30_000; private static final long EXPORT_INTERVAL_MS = 15_000; private static final long SWEEP_INTERVAL_MS = 5_000; - private static final int RECENT_FRAMES_KEPT = 50; - private static final int SLOWEST_FRAMES_KEPT = 20; + private static final int RECENT_ROOTS_KEPT = 40; + private static final int SLOWEST_ROOTS_KEPT = 20; private static final int SENT_DURATIONS_WINDOW = 512; private static final int MAX_LINES_PER_FRAME = 200; + /** + * Frames retained for ID lookup. Well above the number any export can + * reference, so a late log/span/finish call always finds its frame; frames + * older than this are unreachable and get collected. + */ + private static final int FRAMES_RETAINED = 1024; private static final String EXPORT_FILE_NAME = "frame-timings.txt"; public static FrameTimings getInstance() { return INSTANCE; } + private static final class Line { + final long atMs; // SystemClock.elapsedRealtime() + final String thread; + final String message; + + Line(long atMs, String thread, String message) { + this.atMs = atMs; + this.thread = thread; + this.message = message; + } + } + private static final class Frame { final int id; - final String reason; final long startedAtMs; // SystemClock.elapsedRealtime() final long startedWallClockMs; - final List lines = new ArrayList<>(); - final Map openSpans = new HashMap<>(); + final Frame parent; + final List children = new ArrayList<>(0); + final List lines = new ArrayList<>(); + final Map openSpans = new LinkedHashMap<>(); + String reason; long finishedAtMs; String outcome; // null while open; "sent", "discarded: ...", "timeout: ..." + /** Set on the root once some frame in its subtree reached the glasses. */ + boolean subtreeSentRecorded; - Frame(int id, String reason, long startedAtMs, long startedWallClockMs) { + Frame(int id, String reason, long startedAtMs, long startedWallClockMs, Frame parent) { this.id = id; this.reason = reason; this.startedAtMs = startedAtMs; this.startedWallClockMs = startedWallClockMs; + this.parent = parent; + } + + Frame root() { + Frame frame = this; + while (frame.parent != null) { + frame = frame.parent; + } + return frame; + } + + boolean isOpen() { + return outcome == null; } long durationMs() { @@ -79,13 +120,45 @@ public final class FrameTimings { boolean wasSent() { return outcome != null && outcome.startsWith("sent"); } + + /** Last moment anything in this subtree happened, for whole-tree duration. */ + long subtreeEndAtMs(long nowMs) { + long end = isOpen() ? nowMs : finishedAtMs; + for (Frame child : children) { + end = Math.max(end, child.subtreeEndAtMs(nowMs)); + } + return end; + } + + boolean subtreeHasOpenFrame() { + if (isOpen()) { + return true; + } + for (Frame child : children) { + if (child.subtreeHasOpenFrame()) { + return true; + } + } + return false; + } } private final Object lock = new Object(); - private final Map activeFrames = new HashMap<>(); - private final ArrayDeque recentFrames = new ArrayDeque<>(); - private final List slowestSentFrames = new ArrayList<>(); - private final ArrayDeque sentDurations = new ArrayDeque<>(); + /** + * Every frame we still retain, keyed by ID: open frames plus enough + * history that late calls and the export can still resolve one. + */ + private final LinkedHashMap framesById = + new LinkedHashMap(64, 0.75f, false) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > FRAMES_RETAINED && !eldest.getValue().isOpen(); + } + }; + private final ArrayDeque recentRoots = new ArrayDeque<>(); + private final List slowestRoots = new ArrayList<>(); + private final ArrayDeque inputLatencies = new ArrayDeque<>(); + private final ArrayDeque renderLatencies = new ArrayDeque<>(); private int nextFrameId = 1; private long framesStarted; private long framesSent; @@ -114,22 +187,56 @@ public final class FrameTimings { } } - /** Begin a frame; reason is a short label like "input:sys-event:0". Returns the frame ID. */ + /** Begin a root frame; reason is a short label like "input:sys-event type=0". */ public int startFrame(String reason) { + return startFrame(reason, 0); + } + + /** + * Begin a frame caused by an existing one (parentFrameId; 0 for a root). + * The child is nested under its root in the export and its latency counts + * against the root, so an input event that fans out into an app render and + * a shell-chrome render reads as one timeline instead of three. + */ + public int startFrame(String reason, int parentFrameId) { long now = SystemClock.elapsedRealtime(); Frame frame; synchronized (lock) { - frame = new Frame(nextFrameId++, reason == null ? "" : reason, now, System.currentTimeMillis()); - activeFrames.put(frame.id, frame); + Frame parent = framesById.get(parentFrameId); + frame = new Frame(nextFrameId++, reason == null ? "" : reason, now, + System.currentTimeMillis(), parent); + framesById.put(frame.id, frame); framesStarted++; + if (parent != null) { + parent.children.add(frame); + addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")"); + } } return frame.id; } + /** + * Append to a frame's label, for facts only known after it started (which + * app is in the foreground, which window a render is for). Shows up in the + * frame's header line, so the export is scannable without reading bodies. + */ + public void annotate(int frameId, String text) { + if (text == null || text.isEmpty()) { + return; + } + synchronized (lock) { + Frame frame = framesById.get(frameId); + if (frame == null) { + return; + } + frame.reason = frame.reason.isEmpty() ? text : frame.reason + " " + text; + } + } + public void log(int frameId, String message) { long now = SystemClock.elapsedRealtime(); synchronized (lock) { - Frame frame = activeFrames.get(frameId); + Frame frame = openFrameLocked(frameId); if (frame == null) { return; } @@ -140,7 +247,7 @@ public final class FrameTimings { public void spanStart(int frameId, String name) { long now = SystemClock.elapsedRealtime(); synchronized (lock) { - Frame frame = activeFrames.get(frameId); + Frame frame = openFrameLocked(frameId); if (frame == null) { return; } @@ -152,7 +259,7 @@ public final class FrameTimings { public void spanEnd(int frameId, String name) { long now = SystemClock.elapsedRealtime(); synchronized (lock) { - Frame frame = activeFrames.get(frameId); + Frame frame = openFrameLocked(frameId); if (frame == null) { return; } @@ -174,7 +281,7 @@ public final class FrameTimings { long now = SystemClock.elapsedRealtime(); Frame finished; synchronized (lock) { - Frame frame = activeFrames.remove(frameId); + Frame frame = openFrameLocked(frameId); if (frame == null) { return; } @@ -188,73 +295,94 @@ public final class FrameTimings { /** One-line stats summary, e.g. for showing in the phone UI. */ public String statsSummary() { synchronized (lock) { - long[] percentiles = sentPercentilesLocked(); + long[] input = percentilesLocked(inputLatencies); return "frames started=" + framesStarted + " sent=" + framesSent + " discarded=" + framesDiscarded + " timeout=" + framesTimedOut - + (percentiles == null + + (input == null ? "" - : " | sent-latency p50=" + percentiles[0] + "ms p90=" + percentiles[1] - + "ms p99=" + percentiles[2] + "ms max=" + percentiles[3] + "ms"); + : " | input-to-display p50=" + input[0] + "ms p90=" + input[1] + + "ms p99=" + input[2] + "ms max=" + input[3] + "ms"); } } // --------------------------------------------------------------------- + /** The frame with this ID if it exists and is still open, else null. */ + private Frame openFrameLocked(int frameId) { + if (frameId <= 0) { + return null; + } + Frame frame = framesById.get(frameId); + return frame != null && frame.isOpen() ? frame : null; + } + private void addLineLocked(Frame frame, long now, String message) { if (frame.lines.size() >= MAX_LINES_PER_FRAME) { if (frame.lines.size() == MAX_LINES_PER_FRAME) { - frame.lines.add("... line cap reached"); + frame.lines.add(new Line(now, Thread.currentThread().getName(), "... line cap reached")); } return; } - frame.lines.add(String.format(Locale.US, "%+6dms [%s] %s", - now - frame.startedAtMs, Thread.currentThread().getName(), message)); + frame.lines.add(new Line(now, Thread.currentThread().getName(), message)); } private void finishFrameLocked(Frame frame, long now, String outcome) { frame.finishedAtMs = now; frame.outcome = outcome; for (Map.Entry open : frame.openSpans.entrySet()) { - frame.lines.add(String.format(Locale.US, "%+6dms span %s never ended (started at +%dms)", - now - frame.startedAtMs, open.getKey(), open.getValue() - frame.startedAtMs)); + frame.lines.add(new Line(now, Thread.currentThread().getName(), + "span " + open.getKey() + " never ended (started at +" + + (open.getValue() - frame.startedAtMs) + "ms)")); } frame.openSpans.clear(); addLineLocked(frame, now, "finished: " + outcome); + Frame root = frame.root(); if (frame.wasSent()) { framesSent++; - sentDurations.addLast(frame.durationMs()); - while (sentDurations.size() > SENT_DURATIONS_WINDOW) { - sentDurations.removeFirst(); + // The user-visible latency is input (or timer) to the first pixels + // that actually reached the glasses, wherever in the tree that was. + if (!root.subtreeSentRecorded) { + root.subtreeSentRecorded = true; + recordLatencyLocked(root, now - root.startedAtMs); + insertSlowestLocked(root); } - insertSlowestLocked(frame); } else if (outcome.startsWith("timeout")) { framesTimedOut++; } else { framesDiscarded++; } - recentFrames.addLast(frame); - while (recentFrames.size() > RECENT_FRAMES_KEPT) { - recentFrames.removeFirst(); + if (root == frame) { + recentRoots.addLast(frame); + while (recentRoots.size() > RECENT_ROOTS_KEPT) { + recentRoots.removeFirst(); + } } exportDirty = true; } - private void insertSlowestLocked(Frame frame) { - slowestSentFrames.add(frame); - Collections.sort(slowestSentFrames, (a, b) -> Long.compare(b.durationMs(), a.durationMs())); - while (slowestSentFrames.size() > SLOWEST_FRAMES_KEPT) { - slowestSentFrames.remove(slowestSentFrames.size() - 1); + private void recordLatencyLocked(Frame root, long latencyMs) { + ArrayDeque bucket = root.reason.startsWith("input:") ? inputLatencies : renderLatencies; + bucket.addLast(latencyMs); + while (bucket.size() > SENT_DURATIONS_WINDOW) { + bucket.removeFirst(); } } - /** {p50, p90, p99, max} over the rolling sent-duration window, or null if empty. */ - private long[] sentPercentilesLocked() { - if (sentDurations.isEmpty()) { + private void insertSlowestLocked(Frame root) { + if (!slowestRoots.contains(root)) { + slowestRoots.add(root); + } + // Sorted and trimmed at export time, when every subtree duration is final. + } + + /** {p50, p90, p99, max} over a rolling latency window, or null if empty. */ + private long[] percentilesLocked(ArrayDeque window) { + if (window.isEmpty()) { return null; } - List sorted = new ArrayList<>(sentDurations); + List sorted = new ArrayList<>(window); Collections.sort(sorted); return new long[] { percentileOfSorted(sorted, 50), @@ -273,11 +401,8 @@ public final class FrameTimings { long now = SystemClock.elapsedRealtime(); List timedOut = new ArrayList<>(); synchronized (lock) { - Iterator> iterator = activeFrames.entrySet().iterator(); - while (iterator.hasNext()) { - Frame frame = iterator.next().getValue(); - if (now - frame.startedAtMs >= FRAME_TIMEOUT_MS) { - iterator.remove(); + for (Frame frame : new ArrayList<>(framesById.values())) { + if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) { finishFrameLocked(frame, now, "timeout: never reported finished"); timedOut.add(frame); } @@ -340,45 +465,103 @@ public final class FrameTimings { } private String buildExportLocked() { + long now = SystemClock.elapsedRealtime(); SimpleDateFormat wallClockFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US); StringBuilder out = new StringBuilder(64 * 1024); out.append("FrameTimings export at ").append(wallClockFormat.format(new Date())).append('\n'); out.append(statsSummaryLocked()).append('\n'); out.append('\n'); + out.append("Frames are trees: an input event's own frame is the root, and the renders it\n"); + out.append("caused are indented under it with offsets measured from the root's start.\n"); + out.append('\n'); - out.append("=== slowest sent frames ===\n"); - for (Frame frame : slowestSentFrames) { - appendFrameLocked(out, frame, wallClockFormat); + Collections.sort(slowestRoots, (a, b) -> + Long.compare(b.subtreeEndAtMs(now) - b.startedAtMs, a.subtreeEndAtMs(now) - a.startedAtMs)); + while (slowestRoots.size() > SLOWEST_ROOTS_KEPT) { + slowestRoots.remove(slowestRoots.size() - 1); + } + + out.append("=== slowest frames that reached the glasses ===\n"); + for (Frame frame : slowestRoots) { + appendTreeLocked(out, frame, frame, wallClockFormat, now); } out.append("=== recent frames (oldest first) ===\n"); - for (Frame frame : recentFrames) { - appendFrameLocked(out, frame, wallClockFormat); + for (Frame frame : recentRoots) { + appendTreeLocked(out, frame, frame, wallClockFormat, now); } return out.toString(); } private String statsSummaryLocked() { - long[] percentiles = sentPercentilesLocked(); - return "frames started=" + framesStarted + " sent=" + framesSent - + " discarded=" + framesDiscarded + " timeout=" + framesTimedOut - + " open=" + activeFrames.size() - + (percentiles == null - ? "" - : "\nsent-frame latency (last " + sentDurations.size() + "): p50=" + percentiles[0] - + "ms p90=" + percentiles[1] + "ms p99=" + percentiles[2] - + "ms max=" + percentiles[3] + "ms"); + long[] input = percentilesLocked(inputLatencies); + long[] render = percentilesLocked(renderLatencies); + StringBuilder out = new StringBuilder(); + out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent) + .append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut) + .append(" open=").append(openFrameCountLocked()); + appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size(), input); + appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size(), render); + return out.toString(); } - private void appendFrameLocked(StringBuilder out, Frame frame, SimpleDateFormat wallClockFormat) { - out.append("frame#").append(frame.id) - .append(" [").append(frame.reason).append(']') - .append(" started ").append(wallClockFormat.format(new Date(frame.startedWallClockMs))) - .append(" duration ").append(frame.durationMs()).append("ms") - .append(" outcome ").append(frame.outcome) - .append('\n'); - for (String line : frame.lines) { - out.append(" ").append(line).append('\n'); + private void appendPercentilesLocked(StringBuilder out, String label, int count, long[] percentiles) { + if (percentiles == null) { + return; + } + out.append('\n').append(label).append(" (last ").append(count).append("): p50=") + .append(percentiles[0]).append("ms p90=").append(percentiles[1]) + .append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms"); + } + + private int openFrameCountLocked() { + int open = 0; + for (Frame frame : framesById.values()) { + if (frame.isOpen()) { + open++; + } + } + return open; + } + + /** Print a frame and its descendants, all timed against root's start. */ + private void appendTreeLocked( + StringBuilder out, Frame frame, Frame root, SimpleDateFormat wallClockFormat, long now) { + int depth = 0; + for (Frame walk = frame; walk != root; walk = walk.parent) { + depth++; + } + String indent = repeat(" ", depth); + long startOffsetMs = frame.startedAtMs - root.startedAtMs; + out.append(indent).append("frame#").append(frame.id) + .append(" [").append(frame.reason).append(']'); + if (frame == root) { + out.append(" started ").append(wallClockFormat.format(new Date(frame.startedWallClockMs))); + long totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs; + out.append(" duration ").append(frame.durationMs()).append("ms"); + if (!frame.children.isEmpty()) { + out.append(" (tree ").append(totalMs).append("ms") + .append(frame.subtreeHasOpenFrame() ? ", still open" : "").append(')'); + } + } else { + out.append(" started +").append(startOffsetMs).append("ms") + .append(" duration ").append(frame.isOpen() ? "open" : (frame.durationMs() + "ms")); + } + out.append(" outcome ").append(frame.outcome == null ? "(open)" : frame.outcome).append('\n'); + for (Line line : frame.lines) { + out.append(indent).append(String.format(Locale.US, " %+7dms [%s] %s", + line.atMs - root.startedAtMs, line.thread, line.message)).append('\n'); + } + for (Frame child : frame.children) { + appendTreeLocked(out, child, root, wallClockFormat, now); } } + + private static String repeat(String unit, int times) { + StringBuilder out = new StringBuilder(unit.length() * times); + for (int i = 0; i < times; i++) { + out.append(unit); + } + return out.toString(); + } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java index ba128478..4ea07987 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/TexturePlanner.java @@ -65,6 +65,19 @@ public final class TexturePlanner { public final int bakedCandidates; public final int uploadBytes; public final boolean fullFrame; + /** + * Where the planning time went, in ms. This runs on the send thread, + * directly in the input-to-display path, so the split is reported in + * the frame log: rects = finding the changed region, match = checking + * each draw against the composite, cache = residency/uploads plus run + * building, punch = clearing replayed draws out of a copy of the + * frame, encode = compressing the leftover pixels. + */ + public int rectsMs; + public int matchMs; + public int cacheMs; + public int punchMs; + public int encodeMs; Result(byte[] payload, List uploads, int nextFid, int rectCount, int drawnGlyphs, int runCount, int drawnImages, int fwGlyphs, int fwRuns, @@ -121,6 +134,7 @@ public final class TexturePlanner { if (next.length != stride * height) { return null; } + long planStartedAtMs = android.os.SystemClock.elapsedRealtime(); // Rect geometry, matching what the plain paths would send. boolean fullFrame = previous == null || previous.length != next.length; @@ -143,6 +157,8 @@ public final class TexturePlanner { rects = Collections.singletonList(new int[]{0, 0, width, height}); } + long matchStartedAtMs = android.os.SystemClock.elapsedRealtime(); + // Candidates: draws whose written pixels intersect a sent rect, are // representable on the wire, and land correct against the composite. List selected = new ArrayList<>(); @@ -210,6 +226,7 @@ public final class TexturePlanner { if (selected.isEmpty() && fwSubs.isEmpty()) { return null; } + long cacheStartedAtMs = android.os.SystemClock.elapsedRealtime(); // Residency: ensure every selected draw is in the cache, retrying once // after a reset when the bump allocator fills. A draw that still fails @@ -244,6 +261,7 @@ public final class TexturePlanner { // Punch the drawn pixels out of a copy of the composite, then encode // the rect deltas from the punched content. + long punchStartedAtMs = android.os.SystemClock.elapsedRealtime(); byte[] punched = next.clone(); for (Selected sel : drawn) { punch(punched, stride, width, height, sel); @@ -252,6 +270,7 @@ public final class TexturePlanner { punchFw(punched, stride, width, height, fw); } + long encodeStartedAtMs = android.os.SystemClock.elapsedRealtime(); List subs = new ArrayList<>(); int fid = fidStart; if (fullFrame) { @@ -271,9 +290,16 @@ public final class TexturePlanner { int uploadBytes = cache.pendingUploadBytes(); List uploads = cache.drainUploadPayloads(UPLOAD_PAYLOAD_MAX); - return new Result(payload, uploads, fid, rects.size(), drawnGlyphs.size(), runs.size(), + Result result = new Result(payload, uploads, fid, rects.size(), drawnGlyphs.size(), runs.size(), imageDrawable.size(), fwGlyphCount, fwSubs.size(), bakedCandidates, uploadBytes, fullFrame); + long doneAtMs = android.os.SystemClock.elapsedRealtime(); + result.rectsMs = (int) (matchStartedAtMs - planStartedAtMs); + result.matchMs = (int) (cacheStartedAtMs - matchStartedAtMs); + result.cacheMs = (int) (punchStartedAtMs - cacheStartedAtMs); + result.punchMs = (int) (encodeStartedAtMs - punchStartedAtMs); + result.encodeMs = (int) (doneAtMs - encodeStartedAtMs); + return result; } /** One punch task for an emitted firmware-font glyph. */ diff --git a/app/apps/blocks/blocks-app.worker.ts b/app/apps/blocks/blocks-app.worker.ts index d8a7d6bf..241b4da3 100644 --- a/app/apps/blocks/blocks-app.worker.ts +++ b/app/apps/blocks/blocks-app.worker.ts @@ -210,6 +210,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { } window.focused = message.focused; inferForeground(window, message.focused); + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } @@ -650,7 +653,7 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, window.surfaceId, @@ -661,7 +664,7 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/freecell/freecell-app.worker.ts b/app/apps/freecell/freecell-app.worker.ts index 1f4da7d9..a97e93c1 100644 --- a/app/apps/freecell/freecell-app.worker.ts +++ b/app/apps/freecell/freecell-app.worker.ts @@ -165,6 +165,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { break; } window.focused = message.focused; + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } @@ -755,7 +758,7 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, window.surfaceId, @@ -766,7 +769,7 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/minesweeper/minesweeper-app.worker.ts b/app/apps/minesweeper/minesweeper-app.worker.ts index e8989419..f23bb23e 100644 --- a/app/apps/minesweeper/minesweeper-app.worker.ts +++ b/app/apps/minesweeper/minesweeper-app.worker.ts @@ -175,6 +175,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { } window.focused = message.focused; inferForeground(window, message.focused); + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } @@ -714,7 +717,7 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, window.surfaceId, @@ -725,7 +728,7 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/navigate/navigate-app.worker.ts b/app/apps/navigate/navigate-app.worker.ts index 05d7c5f0..fd021ce6 100644 --- a/app/apps/navigate/navigate-app.worker.ts +++ b/app/apps/navigate/navigate-app.worker.ts @@ -175,6 +175,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { break; } window.focused = message.focused; + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; case "text-input": @@ -876,7 +879,7 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, win.surfaceId, @@ -887,7 +890,7 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); win.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/pinball/pinball-app.worker.ts b/app/apps/pinball/pinball-app.worker.ts index 18166ded..a4bef682 100644 --- a/app/apps/pinball/pinball-app.worker.ts +++ b/app/apps/pinball/pinball-app.worker.ts @@ -346,6 +346,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { } window.focused = message.focused; inferForeground(window, message.focused); + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } @@ -1043,7 +1046,7 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, window.surfaceId, @@ -1054,7 +1057,7 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/roam/roam-app.worker.ts b/app/apps/roam/roam-app.worker.ts index 1d9410b7..aa2e8196 100644 --- a/app/apps/roam/roam-app.worker.ts +++ b/app/apps/roam/roam-app.worker.ts @@ -176,6 +176,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { } window.focused = message.focused; inferForeground(message.focused); + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; case "text-input": @@ -595,7 +598,7 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, win.surfaceId, @@ -606,7 +609,7 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); win.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/terminal/terminal-app.worker.ts b/app/apps/terminal/terminal-app.worker.ts index 34e80de2..8b5f2fd4 100644 --- a/app/apps/terminal/terminal-app.worker.ts +++ b/app/apps/terminal/terminal-app.worker.ts @@ -320,6 +320,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { break; } window.focused = message.focused; + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } @@ -1615,7 +1618,7 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, window.surfaceId, @@ -1626,7 +1629,7 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/apps/timer/timer-app.worker.ts b/app/apps/timer/timer-app.worker.ts index 47147e42..ccf6e8b9 100644 --- a/app/apps/timer/timer-app.worker.ts +++ b/app/apps/timer/timer-app.worker.ts @@ -162,6 +162,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { break; } window.focused = message.focused; + // Marks the main-thread -> worker hop, which is otherwise an + // unexplained gap inside the shell's handle-input span. + frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`); handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } @@ -666,7 +669,7 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void { return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); communicator.submitSurfaceFrame( buffer.buffer, window.surfaceId, @@ -677,7 +680,7 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void { fingerprint, paintMs, frameId, - prepareFrameDraws(draws), + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 8a1a2099..cf83c648 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -200,6 +200,8 @@ class DashboardController { private lastSys = "none yet"; private shellRenderInProgress = false; private shellRenderQueued = false; + /** Input frame that requested the next shell render; see requestShellRender. */ + private pendingShellRenderCauseFrameId = 0; private nextShellRenderWantsFreshData = false; // One shared worker per app hosts all its windows; spawned on first launch. private readonly appHosts = new Map(); @@ -467,7 +469,14 @@ class DashboardController { * visible. Concurrent display-wake, shell, and wakeword callbacks share the * same operation. */ - private ensureEvenHubSessionActive(): Promise { + /** + * frameId (0 when the caller has none) attributes the barrier's four + * round trips to the input frame waiting on them; a wake barrier is the + * largest single input-to-display term there is, so knowing which step + * owns the second matters. Callers that join an in-flight barrier share + * the first caller's spans. + */ + private ensureEvenHubSessionActive(frameId = 0): Promise { if (this.evenHubResumePromise) return this.evenHubResumePromise; const communicator = this.communicator; if (!communicator || this.phase === "charging" || this.phase === "disconnected") { @@ -475,14 +484,19 @@ class DashboardController { } const operation = (async () => { - await communicator.setG2ScreenOn(true); - if (!(await communicator.resumeEvenHubSession())) { + await frameTimings.spanAsync(frameId, "wake:screen-on", () => communicator.setG2ScreenOn(true)); + const resumed = await frameTimings.spanAsync(frameId, "wake:resume-session", () => + communicator.resumeEvenHubSession(), + ); + if (!resumed) { return false; } // Resume first: setScreenBlanked(false) then recomposites retained state // as the desired first frame for the fresh layout. - await communicator.setScreenBlanked(false); - const ready = await communicator.awaitEvenHubSessionReady(EVENHUB_WAKE_READY_TIMEOUT_MS); + await frameTimings.spanAsync(frameId, "wake:unblank", () => communicator.setScreenBlanked(false)); + const ready = await frameTimings.spanAsync(frameId, "wake:await-ready", () => + communicator.awaitEvenHubSessionReady(EVENHUB_WAKE_READY_TIMEOUT_MS), + ); if (ready && this.communicator === communicator) { this.evenHubSessionSuspended = false; } @@ -1323,6 +1337,11 @@ class DashboardController { let frameOwned = false; try { const inputEvent = rawInputEventToInputEvent(event); + // The gesture, plus which app is on screen and whether input goes to it, + // the sidebar, or a shell overlay. Java only knows the raw event codes, + // and without the target the export says what was pressed but not who + // spent the time drawing the answer. + frameTimings.annotateFrame(frameId, `${inputEvent.type} ${shell.describeInputTarget()}`); if (this.glassesLocked) { const ringDoubleTap = inputEvent.type === "double-click" && @@ -1335,7 +1354,9 @@ class DashboardController { if (ringDoubleTap) shell.sleep(); } else { shell.wake("sidebar"); - const ready = await this.ensureEvenHubSessionActive(); + const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () => + this.ensureEvenHubSessionActive(frameId), + ); if (!ready) { this.appendLog("EvenHub wake barrier timed out for locked display"); } @@ -1356,7 +1377,9 @@ class DashboardController { shellPreWoke = shell.wake("sidebar"); } if (shellPreWoke || this.evenHubSessionSuspended || this.evenHubResumePromise) { - const ready = await this.ensureEvenHubSessionActive(); + const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () => + this.ensureEvenHubSessionActive(frameId), + ); if (!ready) { this.appendLog(`EvenHub wake barrier timed out for ${event.kind}`); } @@ -1391,14 +1414,17 @@ class DashboardController { } if (outcome.shell) { - this.requestShellRender(); + // Pass frameId as the cause, not the owner: the chrome render gets its + // own child frame under this one, so the export shows the input and + // every render it triggered on a single timeline. + this.requestShellRender(frameId); } if (outcome.window) { // The window adapter owned frameId (render or explicit finish). frameOwned = true; } else if (outcome.shell) { frameOwned = true; - frameTimings.finishFrame(frameId, "input consumed by shell; chrome render scheduled"); + frameTimings.finishFrame(frameId, "handled by shell; chrome render spawned"); } } finally { if (!frameOwned) { @@ -1607,17 +1633,20 @@ class DashboardController { frameTimings.finishFrame(frameId, "discarded: window frame with no active connection"); return; } - const fingerprint = planesFingerprint(planes); + const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); - const buffer = image.to8bppBuffer(); - await communicator.submitSurfaceFrame( - surfaceId, - buffer, - { x: 0, y: 0, width: image.width, height: image.height }, - fingerprint, - paintMs, - frameId, - prepareFrameDraws(draws), + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); + const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)); + await frameTimings.spanAsync(frameId, "submit", () => + communicator.submitSurfaceFrame( + surfaceId, + buffer, + { x: 0, y: 0, width: image.width, height: image.height }, + fingerprint, + paintMs, + frameId, + preparedDraws, + ), ); } @@ -1635,7 +1664,14 @@ class DashboardController { * overlays). Coalesces like requestRender: one render in flight, at most * one queued. */ - requestShellRender(): void { + requestShellRender(causeFrameId = 0): void { + // Remember the input frame that asked for this chrome repaint so + // renderShell can nest its frame under it; without that link an input + // event and the shell render it caused show up as two unrelated frames + // and the real input-to-pixels latency is never measured. + if (causeFrameId > 0 && this.pendingShellRenderCauseFrameId === 0) { + this.pendingShellRenderCauseFrameId = causeFrameId; + } if (this.shellRenderInProgress) { this.shellRenderQueued = true; return; @@ -1656,9 +1692,13 @@ class DashboardController { } private async renderShell(): Promise { - const frameId = frameTimings.startFrame("render:shell"); + const causeFrameId = this.pendingShellRenderCauseFrameId; + this.pendingShellRenderCauseFrameId = 0; + const frameId = frameTimings.startFrame("render:shell", causeFrameId); + frameTimings.annotateFrame(frameId, shell.describeInputTarget()); const wantFreshData = this.nextShellRenderWantsFreshData; this.nextShellRenderWantsFreshData = false; + if (wantFreshData) frameTimings.logFrame(frameId, "follow-up repaint that must not use cached data"); beginRenderPass(!wantFreshData); const paintStartedAtMs = Date.now(); const planes = frameTimings.span(frameId, "paint", () => @@ -1668,27 +1708,46 @@ class DashboardController { const paintUsedStaleData = endRenderPass(); if (paintUsedStaleData) { // Repaint with fresh data (e.g. notification icons) once this frame is - // out; mirrors renderDashboard's stale-data contract. + // out; mirrors renderDashboard's stale-data contract. Linked to this + // frame so the export shows one input costing two round trips instead of + // an unexplained extra render. this.nextShellRenderWantsFreshData = true; - this.requestShellRender(); + this.requestShellRender(frameId); } - if (!this.communicator || this.phase === "charging") { + const communicator = this.communicator; + if (!communicator || this.phase === "charging") { frameTimings.finishFrame(frameId, "discarded: shell render with no active connection"); return; } const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); - await this.communicator.submitSurfaceFrame( - SHELL_SURFACE_ID, - buffer, - { x: 0, y: 0, width: image.width, height: image.height }, - fingerprint, - paintMs, - frameId, - prepareFrameDraws(draws), + const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)); + // Spanned because the bridge serializes Java calls: a frame can sit here + // behind another surface's submission, which is otherwise an unexplained + // jump between the paint spans and the composite. + await frameTimings.spanAsync(frameId, "submit", () => + communicator.submitSurfaceFrame( + SHELL_SURFACE_ID, + buffer, + { x: 0, y: 0, width: image.width, height: image.height }, + fingerprint, + paintMs, + frameId, + preparedDraws, + ), ); - await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); + // Backpressure: the next shell render waits for this one to reach the + // glasses. Timing out here means the loop was blocked for the full timeout + // and any input arriving meanwhile had its chrome repaint delayed, so say + // so in the frame rather than leaving a silent stall. + const outcome = await communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); + if (outcome === null) { + frameTimings.logFrame( + frameId, + `still unsent after ${FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS}ms; shell render loop released`, + ); + } this.updateCompositePreview(); } diff --git a/app/graphics/image.ts b/app/graphics/image.ts index 85746e0e..4a42271d 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -141,6 +141,8 @@ export class GrayImage { readonly height: number; readonly pixels: Uint8Array; private readonly drawList: DeferredDraw[] = []; + /** See sourceContentHash32; null until this image is hashed as a draw source. */ + private memoizedSourceHash: number | null = null; constructor(width: number, height: number, fill = 0) { this.width = width; @@ -378,11 +380,7 @@ export class GrayImage { /** 32-bit content hash over pixels and deferred draws (fingerprint's core). */ contentHash32(): number { - let hash = 2166136261; - for (let i = 0; i < this.pixels.length; i++) { - hash ^= this.pixels[i]!; - hash = Math.imul(hash, 16777619); - } + let hash = this.pixelsHash32(); for (const placed of this.drawList) { if (placed.kind === "glyph") { hash = mixInt(hash, placed.font.fingerprintId); @@ -392,7 +390,7 @@ export class GrayImage { hash = mixInt(hash, 0x1c0e5); hash = mixInt(hash, placed.source.width); hash = mixInt(hash, placed.source.height); - hash = mixInt(hash, placed.source.contentHash32()); + hash = mixInt(hash, placed.source.sourceContentHash32()); } else { hash = mixInt(hash, 0xf17e); hash = mixInt(hash, placed.font.atlasTag); @@ -408,6 +406,44 @@ export class GrayImage { return hash >>> 0; } + /** + * FNV-1a over the raster pixels. Four bytes at a time where alignment + * allows: this runs on every frame's fingerprint, over the whole screen, on + * the thread that is about to hand the frame to the transport. + */ + private pixelsHash32(): number { + const pixels = this.pixels; + let hash = 2166136261; + let i = 0; + if ((pixels.byteOffset & 3) === 0) { + const words = new Uint32Array(pixels.buffer, pixels.byteOffset, pixels.length >>> 2); + for (let w = 0; w < words.length; w++) { + hash ^= words[w]!; + hash = Math.imul(hash, 16777619); + } + i = words.length << 2; + } + for (; i < pixels.length; i++) { + hash ^= pixels[i]!; + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; + } + + /** + * contentHash32 for an image being used as another image's deferred-draw + * source, memoized. drawImage/drawFwText document their source as immutable + * from the call onwards (glyph-wire's ImageAtlas id cache already depends on + * that), so the hash of a long-lived icon asset can be computed once instead + * of on every frame that draws it. + */ + sourceContentHash32(): number { + if (this.memoizedSourceHash === null) { + this.memoizedSourceHash = this.contentHash32(); + } + return this.memoizedSourceHash; + } + fingerprint(): string { return `fnv:${this.contentHash32().toString(16)}`; } @@ -421,7 +457,8 @@ export class GrayImage { */ to8bppBuffer(): Uint8Array { if (!this.drawList.length) { - return Uint8Array.from(this.pixels); + // slice() is a memcpy; Uint8Array.from walks the buffer element by element. + return this.pixels.slice(); } // withDrawsBaked's buffer is a private temporary, safe to hand out. return this.withDrawsBaked().pixels; diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 78481a92..b4327160 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -111,6 +111,8 @@ export class FaceclawCommunicatorBridge { private readonly communicator: any; private readonly listenerProxy: any; private javaCallQueue: Promise = Promise.resolve(); + /** Calls waiting on javaCallQueue; 0 means enqueueJavaCall's fast path is safe. */ + private queuedJavaCalls = 0; private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>(); // Recent frame-finished outcomes from the Java side, so waitForFrameFinished // does not race against finishes that land before the wait starts. @@ -227,10 +229,30 @@ export class FaceclawCommunicatorBridge { }, 0); } - private enqueueJavaCall(operation: () => T): Promise { + /** + * Run a Java call, ordered against every other call made through here. + * + * inlineWhenIdle runs the call on the spot when nothing is queued ahead of + * it. Ordering still holds — the fast path only fires when no earlier call + * is waiting — and it skips a task-queue hop that measured ~18ms on device + * (the gap between "span submit start" and the Java side's own log line in + * the frame timings), which is pure latency on a call the caller is already + * awaiting. Off by default: a call that might block for a while should keep + * yielding to the main looper first. + */ + private enqueueJavaCall(operation: () => T, inlineWhenIdle = false): Promise { + if (inlineWhenIdle && this.queuedJavaCalls === 0) { + try { + return Promise.resolve(operation()); + } catch (error) { + return Promise.reject(error); + } + } + this.queuedJavaCalls++; const run = () => new Promise((resolve, reject) => { setTimeout(() => { + this.queuedJavaCalls--; try { resolve(operation()); } catch (error) { @@ -526,6 +548,8 @@ export class FaceclawCommunicatorBridge { // ArrayBuffer, which NativeScript marshals to a ByteBuffer without the // ~150ms per-element copy a byte[] parameter would need. const snapshot = new Uint8Array(pixels8bpp); + // inlineWhenIdle: this is the frame path, the Java side of it measures + // ~5ms (composite + pack), and the caller awaits it either way. await this.enqueueJavaCall(() => { this.communicator.submitSurfaceFrame( snapshot.buffer, @@ -539,7 +563,7 @@ export class FaceclawCommunicatorBridge { Math.round(nonNegativeNumber(frameId)), glyphs, ); - }); + }, true); } async disconnect(): Promise { diff --git a/app/native/frame-timings.ts b/app/native/frame-timings.ts index 0b6408b3..56e2a5e1 100644 --- a/app/native/frame-timings.ts +++ b/app/native/frame-timings.ts @@ -3,13 +3,17 @@ * instrumentation; see FrameTimings.java). Frame ID 0 means "no frame" and is * ignored by every method, so callers never need to null-check. * - * Because the JS side is single-threaded, this module also tracks a "current - * frame" so that leaf drawing/IO code can record spans without threading a - * frame ID through every call signature. + * Frames form a tree: pass the causing frame's ID as `parentFrameId` when one + * frame fans out into several (an input event that repaints both the focused + * app and the shell chrome), and the export nests them under one timeline + * instead of showing unrelated frames. + * + * Because each JS context is single-threaded, this module also tracks a + * "current frame" so that leaf drawing/IO code can record spans without + * threading a frame ID through every call signature. */ declare const com: any; -declare const global: any; let javaInstance: any = null; let javaUnavailable = false; @@ -20,12 +24,11 @@ let currentFrameId = 0; function getJava(): any { if (javaInstance === null && !javaUnavailable) { + // Deliberately not gated on global.isAndroid: this runs in app workers + // too, and a worker that had not yet imported the core globals would + // silently drop every frame it renders. try { - if (typeof com !== "undefined" && global.isAndroid) { - javaInstance = com.faceclaw.app.FrameTimings.getInstance(); - } else { - javaUnavailable = true; - } + javaInstance = com.faceclaw.app.FrameTimings.getInstance(); } catch { javaUnavailable = true; } @@ -33,10 +36,23 @@ function getJava(): any { return javaInstance; } -export function startFrame(reason: string): number { +/** Begin a frame; parentFrameId links it under the frame that caused it. */ +export function startFrame(reason: string, parentFrameId = 0): number { const java = getJava(); if (!java) return fallbackNextFrameId--; - return Number(java.startFrame(reason)); + // Guarded: requestRender-style callbacks can be invoked with an unrelated + // argument, and a NaN would blow up the int marshalling. + const parent = Number.isFinite(parentFrameId) ? Math.max(0, Math.round(parentFrameId)) : 0; + return Number(java.startFrame(reason, parent)); +} + +/** + * Append a fact to the frame's label, e.g. which app was in the foreground + * when the input arrived. Shows in the frame's header line in the export. + */ +export function annotateFrame(frameId: number, text: string): void { + if (frameId <= 0 || !text) return; + getJava()?.annotate(frameId, text); } export function logFrame(frameId: number, message: string): void { @@ -68,6 +84,20 @@ export function span(frameId: number, name: string, fn: () => T): T { } } +/** + * Span around an awaited step. Most multi-second gaps in the timing export are + * awaits (an app's async input handler, a worker round trip, the Java call + * queue), so instrument those with this rather than leaving a bare jump. + */ +export async function spanAsync(frameId: number, name: string, fn: () => Promise): Promise { + spanStart(frameId, name); + try { + return await fn(); + } finally { + spanEnd(frameId, name); + } +} + /** Set the current frame while running fn, for leaf code using the *Current helpers. */ export function runWithFrame(frameId: number, fn: () => T): T { const previous = currentFrameId; diff --git a/app/ui/shell/in-process-window.ts b/app/ui/shell/in-process-window.ts index d659f231..40ba89e7 100644 --- a/app/ui/shell/in-process-window.ts +++ b/app/ui/shell/in-process-window.ts @@ -65,11 +65,21 @@ export type InProcessAppOptions = { }; export function createInProcessWindow(options: InProcessWindowOptions): InProcessWindow { - const requestRender = () => { - void render(0).catch((error) => { + /** + * Repaint under a new frame. causeFrameId links it to the frame that made it + * necessary (a stale-cache paint asking for a redo); 0 for an app-initiated + * repaint. Each repaint gets a frame either way: an app-initiated one (a + * clock tick, arriving data) is a real screen update with real latency, and + * submitting it under frame 0 made it invisible in the timing export except + * as an anonymous "superseded by frame#0" line in some other frame's log. + */ + const renderInNewFrame = (causeFrameId: number) => { + void render(frameTimings.startFrame(`render:${options.windowId}`, causeFrameId)).catch((error) => { console.error(`${options.windowId} render failed: ${error}`); }); }; + // Handed to app code as a bare callback, so it takes no arguments. + const requestRender = () => renderInNewFrame(0); let heightMode = options.heightMode ?? "min"; const stack = new LayerStack( options.baseLayer, @@ -91,16 +101,21 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces frameTimings.finishFrame(frameId, "discarded: window closed"); return; } + frameTimings.annotateFrame(frameId, `window=${options.windowId}`); const wantFreshData = nextRenderWantsFreshData; nextRenderWantsFreshData = false; beginRenderPass(!wantFreshData); const paintStartedAtMs = Date.now(); - const planes = stack.paint(); + // runWithFrame so leaf data sources (notification icons, calendar) attach + // their own spans to this frame. + const planes = frameTimings.span(frameId, "paint", () => + frameTimings.runWithFrame(frameId, () => stack.paint()), + ); const paintUsedStaleData = endRenderPass(); await options.submitFrame(planes, Date.now() - paintStartedAtMs, frameId); if (paintUsedStaleData) { nextRenderWantsFreshData = true; - requestRender(); + renderInNewFrame(frameId); } } @@ -165,7 +180,12 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces await render(frameId); return; } - await stack.handleInput(event); + // Spanned separately from the paint: an app's input handler may await + // real work (launching an app, a network call), and that used to show up + // as a bare multi-second gap inside the shell's handle-input span. + await frameTimings.spanAsync(frameId, `app-input:${options.windowId}`, () => + stack.handleInput(event), + ); await render(frameId); }, requestRender, diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index aa23000c..b5409a55 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -599,6 +599,20 @@ class Shell { }; } + /** + * Compact description of where input is currently going, for the frame + * timing export: "which app was drawing" is usually the first thing you need + * to interpret an input-to-display latency, and it is not recoverable from + * the input event itself. + */ + describeInputTarget(): string { + const foreground = this.foregroundWindow()?.windowId ?? "none"; + if (!this.screenOn) return `fg=${foreground} target=screen-off`; + if (this.activeVoiceLayer) return `fg=${foreground} target=voice`; + if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`; + return `fg=${foreground} target=${this.focus}`; + } + /** Whether a window is the current input target (foreground + focus in-window). */ isWindowFocused(windowId: string): boolean { return this.screenOn && this.focus === "window" && this.foregroundWindow()?.windowId === windowId; diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index 85ae72c5..d5e0dcef 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -3,6 +3,7 @@ import { windowIcon } from "./chrome-layer"; import { type IconName } from "../../graphics/icons"; import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/tool-registry"; import { appViewportSize, type WindowHeightMode } from "./geometry"; +import * as frameTimings from "../../native/frame-timings"; import { shell, type ShellWindow } from "./shell"; /** @@ -293,6 +294,7 @@ export class WorkerAppHost { }, drawIcon: windowIcon(spec.icon, spec.iconLetter, spec.iconGlyph), handleInput: (event, frameId) => { + frameTimings.logFrame(frameId, `input posted to the ${this.options.appId} worker`); this.post({ type: "input", windowId: spec.windowId,