Merge branch 'main' into microphone-fixes

This commit is contained in:
James Babcock committed 2026-09-27 23:19:03 -07:00
commit 10865fc0cb
32 files changed
+832 -160

No files matched your search

@@ -63,7 +63,7 @@ class ScenePlanner(private val cache: ResourceCacheState) {
val calls = ArrayList<ByteArray>()
for (top in 0 until height step 32) {
val rows = minOf(32, height - top)
if (old != null && (top * stride until (top + rows) * stride).all { old[it] == next[it] }) continue
if (old != null && sameBytes(old, next, top * stride, (top + rows) * stride)) continue
var left = width; var right = -1
for (y in top until top + rows) for (x in 0 until width) {
val p = y * stride + x / 2; val shift = if (x % 2 == 0) 4 else 0
@@ -75,4 +75,12 @@ class ScenePlanner(private val cache: ResourceCacheState) {
}
return calls
}
private fun sameBytes(a: ByteArray, b: ByteArray, from: Int, to: Int): Boolean {
var i = from
while (i < to) {
if (a[i] != b[i]) return false
i++
}
return true
}
}
@@ -1036,6 +1036,10 @@ class GlassesSessionCore(
storeDesiredComposite(composite, paintMs, frameId)
}
/** See [SurfaceCompositor.isSurfaceCurrent]. */
fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean =
compositor.isSurfaceCurrent(surfaceId, fingerprint)
/** Store a composite as the desired frame unless a newer one won the race. */
internal fun storeDesiredComposite(composite: SurfaceCompositor.Composite, paintMs: Int, frameId: Int) {
var supersededFrameId = 0
@@ -2,11 +2,14 @@ package com.faceclaw.app
/** A retained selection or image, replayed at stereo depth over the screen and shell surfaces. */
class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val radius: Int,
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List<IntArray> = emptyList()) : RetainedDrawing {
val resource = CachedResource(DrawProtocol.rawImage(width, height, packed))
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List<IntArray> = emptyList(),
/** [packed] as a raw-image resource; translated copies share it (and its hash) instead of rebuilding it. */
val resource: CachedResource = CachedResource(DrawProtocol.rawImage(width, height, packed))) : RetainedDrawing {
override val resources = listOf(resource)
override val fingerprint = "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
override fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask)
override val fingerprint: String by lazy {
"$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
}
override fun translated(dx: Int, dy: Int) = if (dx == 0 && dy == 0) this else MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask, resource)
override fun calls(ids: IntArray, nowMs: Long) = calls(ids.single())
fun calls(id: Int): List<ByteArray> {
val image = when (kind) {
@@ -43,11 +43,19 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
return maxOf(1, (((value * dim) + 128) shr 8))
}
/**
* The rest of [reader] as a heap-array reader. Parsing reads field by field, and through a
* platform buffer (Android's direct ByteBuffer) each byte was its own native peek; one bulk
* copy first is far cheaper.
*/
private fun heapReader(reader: ByteReader): ByteReader =
if (reader is ArrayByteReader) reader else ArrayByteReader(ByteArray(reader.remaining()).also { reader.get(it) })
private fun parseDraws(draws: ByteReader?): Array<ScreenDraw> {
if (((draws == null) || (draws.remaining() < 1))) {
return NO_DRAWS
}
val cursor: ByteReader = draws
val cursor: ByteReader = heapReader(draws)
var out: MutableList<ScreenDraw> = ArrayList()
while ((cursor.remaining() >= 1)) {
var kind: Int = (cursor.get() and 0xff)
@@ -253,8 +261,11 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private var shellScene: ShellScene? = null
fun setShellScene(reader: ByteReader) {
val scene = ShellScene.decode(reader)
lock.withLock { shellScene = scene }
val scene = ShellScene.decode(heapReader(reader))
lock.withLock {
shellScene = scene
changedSinceComposite = true
}
}
private class Surface {
@@ -284,6 +295,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
/** Surface-local deferred draws of the retained content (already baked into pixels). */
@JvmField var draws: Array<ScreenDraw> = NO_DRAWS
/**
* [draws]' retained selections in screen coordinates, as built for [placedFor] at
* ([placedX], [placedY]). Reused by every composite until either changes, so the shell
* scene built from them (and its fingerprint) can be reused too.
*/
@JvmField var placedSelections: List<RetainedDrawing> = emptyList()
@JvmField var placedFor: Array<ScreenDraw>? = null
@JvmField var placedX: Int = 0
@JvmField var placedY: Int = 0
constructor(id: String) {
this.id = id
}
@@ -291,6 +312,8 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private val lock = protocolPlatform().createLock()
private var animatedPreview: ShellScene.AnimatedPreview? = null
/** The last composite's scene; reused while its inputs are unchanged (see compositeLocked). */
private var lastScene: ShellScene? = null
private var previewKey: String? = null
private fun previewScene(scene: ShellScene, gray: ByteArray, key: String): ByteArray {
@@ -318,6 +341,13 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private var nextCompositeSeq: Long = 1
/**
* Set by changes that take effect at the next composite (geometry, dim, depth, blanking,
* visibility, a new shell scene, content retained without compositing) and cleared by a
* composite that is kept (not a preview); see [isSurfaceCurrent].
*/
private var changedSinceComposite: Boolean = false
/** Set the output frame size. Must be called before any surface work. */
fun configureScreen(width: Int, height: Int): Unit {
if (((width <= 0) || (height <= 0))) {
@@ -376,12 +406,27 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
surface.height = height
surface.zOrder = zOrder
surface.transparency = transparency
changedSinceComposite = true
}
}
fun removeSurface(id: String): Unit {
lock.withLock {
surfaces.remove(id)
changedSinceComposite = true
}
}
/**
* Whether a full-surface update carrying [fingerprint] would change nothing: the surface
* already retains that content and every other change has reached a kept composite. A
* submitter can then drop an identical repaint without flattening or sending it.
*/
fun isSurfaceCurrent(id: String, fingerprint: String): Boolean {
lock.withLock {
if (changedSinceComposite) return false
val surface = surfaces.get(id) ?: return false
return surface.fingerprint.isNotEmpty() && surface.fingerprint == fingerprint
}
}
@@ -398,6 +443,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
lock.withLock {
underlayDimBelowZOrder = belowZOrder
underlayDim = maxOf(0, minOf(256, factor256))
changedSinceComposite = true
}
}
@@ -419,6 +465,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
throw IllegalArgumentException(("unknown surface " + id))
}
surface.visible = visible
changedSinceComposite = true
}
}
@@ -432,6 +479,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
lock.withLock {
val surface = surfaces.get(id) ?: throw IllegalArgumentException(("unknown surface " + id))
surface.depth = depth
changedSinceComposite = true
}
}
@@ -450,6 +498,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
fun setBlanked(blanked: Boolean): Unit {
lock.withLock {
this.blanked = blanked
changedSinceComposite = true
}
}
@@ -596,6 +645,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
surface.fingerprint = (if ((contentFingerprint == null)) "" else contentFingerprint)
surface.draws = parsed
if (!composeAfter) changedSinceComposite = true
return if (composeAfter) compositeLocked() else null
}
}
@@ -618,8 +668,10 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
return null
}
val seq = nextCompositeSeq
val changed = changedSinceComposite
val result = compositeLocked(includePreview = true)
nextCompositeSeq = seq
changedSinceComposite = changed
return result
}
}
@@ -640,6 +692,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
private fun compositeLocked(includePreview: Boolean = includePreviewInFrames): Composite {
changedSinceComposite = false
var gray: ByteArray = ByteArray((screenWidth * screenHeight))
if (blanked) {
animatedPreview?.player?.stop()
@@ -666,9 +719,9 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
blendLocked(gray, surface)
var dim: Int = dimForLocked(surface)
selections.addAll(placedSelectionsLocked(surface))
for (draw in surface.draws) {
val selected = draw.selection
if (selected != null) { selections.add(selected.translated(surface.x, surface.y)); continue }
if (draw.selection != null) continue
if (((dim < 256) && (draw.kind == ScreenDraw.KIND_IMAGE))) {
continue
}
@@ -710,8 +763,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
}
val screenDepth = screenDepthLocked(ordered)
val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene(emptyList(), screenDepth = screenDepth)
else ShellScene(shellScene?.layers ?: emptyList(), selections, screenDepth)
val overlaid = surfaces.values.any { it.visible && it.zOrder > 1 }
val sceneLayers = if (overlaid) emptyList() else shellScene?.layers ?: emptyList()
val sceneSelections: List<RetainedDrawing> = if (overlaid) emptyList() else selections
// Built per composite, a scene rebuilt its fingerprint and resource lists every time even
// though its inputs (the decoded shell layers, cached placed selections) rarely change.
val previous = lastScene
val scene = if (previous != null && previous.layers === sceneLayers && previous.screenDepth == screenDepth &&
sameElements(previous.selections, sceneSelections)) previous
else ShellScene(sceneLayers, sceneSelections, screenDepth)
lastScene = scene
val sceneKey = fingerprint.append("|shell:").append(scene.fingerprint).toString()
val preview = if (includePreview && (shellScene != null || selections.isNotEmpty())) {
previewScene(scene, gray, sceneKey)
@@ -727,6 +788,22 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
nextCompositeSeq++, draws.toTypedArray()).also { it.screenGray = gray; it.shellScene = scene }
}
private fun placedSelectionsLocked(surface: Surface): List<RetainedDrawing> {
if (surface.placedFor !== surface.draws || surface.placedX != surface.x || surface.placedY != surface.y) {
surface.placedSelections = surface.draws.mapNotNull { it.selection?.translated(surface.x, surface.y) }
surface.placedFor = surface.draws
surface.placedX = surface.x
surface.placedY = surface.y
}
return surface.placedSelections
}
private fun sameElements(a: List<RetainedDrawing>, b: List<RetainedDrawing>): Boolean {
if (a.size != b.size) return false
for (i in a.indices) if (a[i] !== b[i]) return false
return true
}
private fun blendLocked(gray: ByteArray, surface: Surface): Unit {
var srcX: Int = maxOf(0, -surface.x)
var srcY: Int = maxOf(0, -surface.y)
@@ -268,13 +268,16 @@ class FrameTimingsCore(
get() = lock.withLock { exportDirty }
/** The full export text (does not clear the dirty flag). */
fun buildExport(): String = lock.withLock { buildExportLocked() }
fun buildExport(): String = formatExport(lock.withLock { exportSnapshotLocked() })
/** The export text if anything changed since the last take, else null; clears the dirty flag. */
fun takeExport(): String? = lock.withLock {
if (!exportDirty) return null
exportDirty = false
buildExportLocked()
fun takeExport(): String? {
val snapshot = lock.withLock {
if (!exportDirty) return null
exportDirty = false
exportSnapshotLocked()
}
return formatExport(snapshot)
}
// ---------------------------------------------------------------------
@@ -360,30 +363,72 @@ class FrameTimingsCore(
)
}
private fun buildExportLocked(): String {
/**
* Frame state as of one moment, formatted into the export text outside the lock. Formatting
* the full export takes tens of milliseconds, and holding the lock for it stalled every
* thread recording a span meanwhile, the main thread included.
*/
private class FrameSnapshot(
val id: Int,
val reason: String,
val startedAtMs: Long,
val startedWallClockMs: Long,
val durationMs: Long,
val open: Boolean,
val outcome: String?,
val subtreeEndAtMs: Long,
val subtreeHasOpenFrame: Boolean,
val lines: List<Line>,
val children: List<FrameSnapshot>,
)
private class ExportSnapshot(
val wallClockMs: Long,
val statsSummary: String,
val slowest: List<FrameSnapshot>,
val recent: List<FrameSnapshot>,
)
private fun snapshotLocked(frame: Frame, now: Long): FrameSnapshot = FrameSnapshot(
frame.id, frame.reason, frame.startedAtMs, frame.startedWallClockMs, frame.durationMs(), frame.isOpen(),
frame.outcome, frame.subtreeEndAtMs(now), frame.subtreeHasOpenFrame(), frame.lines.toList(),
frame.children.map { snapshotLocked(it, now) },
)
private fun exportSnapshotLocked(): ExportSnapshot {
val now = platform.elapsedRealtimeMs()
// Stable sort, descending by whole-tree duration (matches Collections.sort).
val treeDurations = HashMap<Frame, Long>()
for (frame in slowestRoots) treeDurations[frame] = frame.subtreeEndAtMs(now) - frame.startedAtMs
slowestRoots.sortWith(compareByDescending { treeDurations.getValue(it) })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
return ExportSnapshot(
wallClockMs(),
statsSummaryLocked(),
slowestRoots.map { snapshotLocked(it, now) },
recentRoots.map { snapshotLocked(it, now) },
)
}
private fun formatExport(snapshot: ExportSnapshot): String {
val out = StringBuilder(64 * 1024)
out.append("FrameTimings export at ").append(formatWallClock(wallClockMs())).append('\n')
out.append(statsSummaryLocked()).append('\n')
out.append("FrameTimings export at ").append(formatWallClock(snapshot.wallClockMs)).append('\n')
out.append(snapshot.statsSummary).append('\n')
out.append('\n')
out.append("Frames are trees: an input event's own frame is the root, and the renders it\n")
out.append("caused are indented under it with offsets measured from the root's start.\n")
out.append('\n')
// Stable sort, descending by whole-tree duration (matches Collections.sort).
slowestRoots.sortWith(compareByDescending { it.subtreeEndAtMs(now) - it.startedAtMs })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
out.append("=== slowest frames that reached the glasses ===\n")
for (frame in slowestRoots) {
appendTreeLocked(out, frame, frame, now)
for (frame in snapshot.slowest) {
appendTree(out, frame, frame, 0)
}
out.append("=== recent frames (oldest first) ===\n")
for (frame in recentRoots) {
appendTreeLocked(out, frame, frame, now)
for (frame in snapshot.recent) {
appendTree(out, frame, frame, 0)
}
return out.toString()
}
@@ -416,28 +461,22 @@ class FrameTimingsCore(
}
/** Print a frame and its descendants, all timed against root's start. */
private fun appendTreeLocked(out: StringBuilder, frame: Frame, root: Frame, now: Long) {
var depth = 0
var walk: Frame? = frame
while (walk !== root) {
depth++
walk = walk!!.parent
}
private fun appendTree(out: StringBuilder, frame: FrameSnapshot, root: FrameSnapshot, depth: Int) {
val indent = " ".repeat(depth)
val startOffsetMs = frame.startedAtMs - root.startedAtMs
out.append(indent).append("frame#").append(frame.id)
.append(" [").append(frame.reason).append(']')
if (frame === root) {
out.append(" started ").append(formatWallClock(frame.startedWallClockMs))
val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs
out.append(" duration ").append(frame.durationMs()).append("ms")
val totalMs = frame.subtreeEndAtMs - frame.startedAtMs
out.append(" duration ").append(frame.durationMs).append("ms")
if (frame.children.isNotEmpty()) {
out.append(" (tree ").append(totalMs).append("ms")
.append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')')
.append(if (frame.subtreeHasOpenFrame) ", still open" else "").append(')')
}
} else {
out.append(" started +").append(startOffsetMs).append("ms")
.append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms"))
.append(" duration ").append(if (frame.open) "open" else (frame.durationMs.toString() + "ms"))
}
out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n')
for (line in frame.lines) {
@@ -445,7 +484,7 @@ class FrameTimingsCore(
.append("ms [").append(line.thread).append("] ").append(line.message).append('\n')
}
for (child in frame.children) {
appendTreeLocked(out, child, root, now)
appendTree(out, child, root, depth + 1)
}
}
}