Depth effects

This commit is contained in:
James Babcock committed 2026-09-22 10:59:51 -07:00
1 parent 4320021222
commit 392554dd37
26 files changed
+623 -136

No files matched your search

@@ -1,6 +1,6 @@
package com.faceclaw.app
/** Revision 25 wire grammar, shared by scene planning and the local renderer. */
/** Revision 26 wire grammar, shared by scene planning and the local renderer. */
object DrawProtocol {
const val DRAW = 26
const val ROOT = 27
@@ -15,8 +15,19 @@ object DrawProtocol {
const val CURRENT = 65534
fun u16(b: ByteArray, p: Int): Int = (b[p].toInt() and 255) or ((b[p + 1].toInt() and 255) shl 8)
fun word(n: Int) = byteArrayOf(n.toByte(), (n ushr 8).toByte())
fun call(op: Int, args: ByteArray, target: Int? = null): ByteArray =
byteArrayOf(op.toByte(), if (target == null) 0 else 1) + (target?.let { word(it) } ?: byteArrayOf()) + args
fun call(op: Int, args: ByteArray, target: Int? = null, depth: Int? = null): ByteArray {
require(depth == null || depth in -128..127)
val flags = (if (target == null) 0 else 1) or (if (depth == null) 0 else 2)
return byteArrayOf(op.toByte(), flags.toByte()) + (target?.let { word(it) } ?: byteArrayOf()) +
(depth?.let { byteArrayOf(it.toByte()) } ?: byteArrayOf()) + args
}
fun depthOffset(depth: Int, right: Boolean): Int =
if (right) -floorHalf(depth + 1) else floorHalf(depth)
private fun floorHalf(n: Int): Int = if (n < 0) (n - 1) / 2 else n / 2
fun roundedRect(x: Int, y: Int, width: Int, height: Int, radius: Int, background: Int, border: Int = 16, depth: Int? = null): ByteArray {
require(width in 1..640 && height in 1..480 && radius in 0..65535 && background in 0..15 && border in 0..16)
return call(8, word(x) + word(y) + word(width) + word(height) + word(radius) + byteArrayOf(background.toByte(), border.toByte()), depth = depth)
}
fun sequence(calls: List<ByteArray>): ByteArray {
require(calls.size <= 4096)
val out = ByteSink(); out.write(word(calls.size))
@@ -26,8 +37,8 @@ object DrawProtocol {
fun message(calls: List<ByteArray>) = byteArrayOf(DRAW.toByte()) + sequence(calls)
fun displayList(calls: List<ByteArray>) = byteArrayOf(LIST.toByte()) + sequence(calls)
fun root(id: Int) = byteArrayOf(ROOT.toByte()) + word(id)
fun image(id: Int, x: Int, y: Int, options: Int = 15, target: Int? = null) =
call(4, word(id) + word(x) + word(y) + byteArrayOf(options.toByte()), target)
fun image(id: Int, x: Int, y: Int, options: Int = 15, target: Int? = null, depth: Int? = null) =
call(4, word(id) + word(x) + word(y) + byteArrayOf(options.toByte()), target, depth)
fun lut(width: Int, height: Int, factor: Int): ByteArray {
val table = ByteArray(8) { i ->
(((i * 2 * factor / 256).coerceIn(0, 15) shl 4) or ((i * 2 + 1) * factor / 256).coerceIn(0, 15)).toByte()
@@ -60,7 +71,7 @@ object DrawProtocol {
flush()
return call(1, out.toByteArray(), target)
}
/** Internal optimizer formats are translated here; they are never sent to revision 25. */
/** Internal optimizer formats are translated here; they are never sent to revision 26. */
fun fromOptimized(payload: ByteArray, width: Int = 640, height: Int = 480): List<ByteArray> {
return when (val mode = payload[0].toInt() and 127) {
8 -> {
@@ -25,18 +25,19 @@ class ScenePlanner(private val cache: ResourceCacheState) {
CachedResource(DrawProtocol.rawImage(layer.width, layer.height), keys[i])
} }
owned.keys.retainAll(keys.toSet()); pixels.keys.retainAll(keys.toSet())
cache.pin(surfaces)
val selectedResources = scene.allSelections.map { it.resource }
cache.pin(surfaces + selectedResources)
val absent = surfaces.map { cache.resourceId(it) < 0 }
val ids = cache.prepare(surfaces)
check(ids.all { it >= 0 }) { "Shell surfaces exceed the resource budget" }
val prepared = cache.prepare(surfaces + selectedResources)
val ids = prepared.copyOfRange(0, surfaces.size)
val selectedIds = prepared.copyOfRange(surfaces.size, prepared.size)
check(prepared.all { it >= 0 }) { "Shell surfaces exceed the resource budget" }
commands.addAll(cache.drainCommands(3600))
val rootCalls = ArrayList<ByteArray>()
val rootCalls = scene.calls(width, height, ids, selectedIds)
for ((i, layer) in scene.layers.withIndex()) {
val old = if (absent[i]) null else pixels[keys[i]]
commands.addAll(DrawProtocol.messages(pixelCalls(old, layer.packed, layer.width, layer.height, ids[i])))
pixels[keys[i]] = layer.packed
if (layer.dim < 256) rootCalls.add(DrawProtocol.lut(width, height, layer.dim))
rootCalls.add(DrawProtocol.image(ids[i], layer.x, layer.y))
}
val base = if (incremental) previous else null
val texture = if (textures) TexturePlanner.plan(base, screen, width, height, draws, cache, firstId, multiRect, maxRects) else null
@@ -48,7 +49,7 @@ class ScenePlanner(private val cache: ResourceCacheState) {
val rootId = cache.prepare(listOf(list))[0]
check(rootId >= 0) { "No space for root display list" }
commands.addAll(cache.drainCommands(3600))
cache.pin(surfaces + list)
cache.pin(surfaces + selectedResources + list)
val evicted = commands.any { it[0].toInt() == ResourceCacheState.EVICT_MODE }
if (evicted) commands.add(0, DrawProtocol.root(DrawProtocol.SCREEN))
if (root != rootId || evicted) commands.add(DrawProtocol.root(rootId))
@@ -1,18 +1,27 @@
package com.faceclaw.app
/** Reference interpreter. It uses the firmware's packed rows, clipping, LUT and call grammar. */
class DisplayListRenderer(private val resources: Map<Int, ByteArray>, private val builtin: ((Int, Int) -> BuiltinGlyph)? = null) {
class DisplayListRenderer(private val resources: Map<Int, ByteArray>, private val builtin: ((Int, Int) -> BuiltinGlyph)? = null, private val rightLens: Boolean = false) {
class BuiltinGlyph(val image: ByteArray, val advance: Int, val x: Int = 0, val y: Int = 0)
class Target(val bytes: ByteArray, val width: Int, val height: Int, val offset: Int = 0) {
class Target(val bytes: ByteArray, val width: Int, val height: Int, val offset: Int = 0, val shiftX: Int = 0) {
val stride = (width + 1) / 2
init { require(width > 0 && height > 0 && offset >= 0 && bytes.size - offset >= stride * height) }
fun get(x: Int, y: Int) = (bytes[offset + y * stride + x / 2].toInt() ushr (if (x % 2 == 0) 4 else 0)) and 15
fun put(x: Int, y: Int, v: Int) {
fun shifted(dx: Int) = Target(bytes, width, height, offset, shiftX + dx)
fun put(localX: Int, y: Int, v: Int) {
val x = localX + shiftX
if (x !in 0 until width || y !in 0 until height) return
val index = offset + y * stride + x / 2; val old = bytes[index].toInt()
bytes[index] = (if (x % 2 == 0) (old and 15) or (v shl 4) else (old and 240) or v).toByte()
}
}
private fun roundedContains(x: Int, y: Int, w: Int, h: Int, radius: Int): Boolean {
if (x !in 0 until w || y !in 0 until h) return false
val r = minOf(radius, w / 2, h / 2)
val dx = maxOf(0, 2 * r - (2 * minOf(x, w - 1 - x) + 1))
val dy = maxOf(0, 2 * r - (2 * minOf(y, h - 1 - y) + 1))
return dx * dx + dy * dy <= 4 * r * r
}
private class Image(val width: Int, val height: Int, val pixels: IntArray)
private fun word(b: ByteArray, p: Int): Int { require(p >= 0 && p + 2 <= b.size); return DrawProtocol.u16(b, p) }
private fun signed(b: ByteArray, p: Int) = word(b, p).toShort().toInt()
@@ -77,9 +86,10 @@ class DisplayListRenderer(private val resources: Map<Int, ByteArray>, private va
repeat(count) {
val n = word(b, pos); pos += 2; require(n >= 2 && n <= b.size - pos && budget[0]-- > 0)
val call = b.copyOfRange(pos, pos + n); pos += n
val flags = call[1].toInt() and 255; require(flags <= 1)
val flags = call[1].toInt() and 255; require(flags <= 3)
var t = inherited; var header = 2
if (flags == 1) { val id = word(call, 2); refs.add(id); t = target(id); header += 2 }
if (flags and 1 != 0) { val id = word(call, 2); refs.add(id); t = target(id).shifted(inherited.shiftX); header += 2 }
if (flags and 2 != 0) { require(header < call.size); t = t.shifted(DrawProtocol.depthOffset(call[header++].toInt(), rightLens)) }
val p = call.copyOfRange(header, call.size)
when (call[0].toInt()) {
1 -> {
@@ -132,9 +142,24 @@ class DisplayListRenderer(private val resources: Map<Int, ByteArray>, private va
6 -> {
require(p.size == 16); val x = word(p, 0); val y = word(p, 2); val w = word(p, 4); val h = word(p, 6)
require(w > 0 && h > 0 && x + w <= t.width && y + h <= t.height)
if (apply) for(yy in y until y + h) for(xx in x until x + w) { val v = t.get(xx, yy)
if (apply) for(yy in y until y + h) for(xx in x until x + w) {
if (xx + t.shiftX !in 0 until t.width) continue
val v = t.get(xx + t.shiftX, yy)
t.put(xx, yy, (p[8 + v / 2].toInt() ushr (if(v % 2 == 0) 4 else 0)) and 15) }
}
8 -> {
require(p.size == 12)
val x = signed(p, 0); val y = signed(p, 2); val w = word(p, 4); val h = word(p, 6)
val radius = word(p, 8); val fill = p[10].toInt() and 255; val border = p[11].toInt() and 255
require(w in 1..640 && h in 1..480 && fill <= 15 && border <= 16)
if (apply) for (yy in 0 until h) for (xx in 0 until w) {
if (!roundedContains(xx, yy, w, h, radius)) continue
val tx = x + xx + t.shiftX; val ty = y + yy
if (tx !in 0 until t.width || ty !in 0 until t.height) continue
val edge = !roundedContains(xx - 1, yy - 1, w - 2, h - 2, maxOf(0, radius - 1))
t.put(x + xx, ty, if (edge && border < 16) border else maxOf(fill, t.get(tx, ty)))
}
}
7 -> { require(p.size == 2); executeList(word(p, 0), t, screen, apply, stack, budget, refs) }
else -> error("Unknown draw opcode")
}
@@ -0,0 +1,56 @@
package com.faceclaw.app
/** A retained selection or image, replayed at stereo depth over the screen and shell surfaces. */
class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val radius: Int,
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = 3, val mask: List<IntArray> = emptyList()) {
val resource = CachedResource(DrawProtocol.rawImage(width, height, packed))
val fingerprint = "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask)
fun calls(id: Int): List<ByteArray> {
val image = when (kind) {
3 -> listOf(DrawProtocol.roundedRect(x, y, width, height, radius, background, border, depth),
DrawProtocol.image(id, x, y, 31, depth = depth))
4 -> listOf(DrawProtocol.image(id, x, y, 31, depth = depth))
// Gray8 zero is transparent but gray8 one is opaque black. Rect copies preserve
// that distinction after packing, including the transparent corners of app menus.
5 -> mask.map { r -> DrawProtocol.call(2, DrawProtocol.word(id) +
r.flatMap { DrawProtocol.word(it).toList() }.toByteArray() +
DrawProtocol.word(x + r[0]) + DrawProtocol.word(y + r[1]), depth = depth) }
else -> error("Invalid presentation kind")
}
return image + occlusions.map { r -> DrawProtocol.call(2, DrawProtocol.word(DrawProtocol.SCREEN) + r.flatMap { DrawProtocol.word(it).toList() }.toByteArray() + DrawProtocol.word(r[0]) + DrawProtocol.word(r[1])) }
}
companion object {
/** The caller has consumed the presentation tag (3, 4 or 5). Colors cross the bridge in gray8, pixels in gray8 rows. */
fun read(reader: ByteReader, kind: Int = 3): MenuSelection {
require(kind in 3..5)
require(reader.remaining() >= 15)
val x = reader.getShort().toInt(); val y = reader.getShort().toInt()
val w = reader.getShort().toInt() and 65535; val h = reader.getShort().toInt() and 65535
val radius = reader.getShort().toInt() and 65535
val background = BmpUtil.nibbleForGray(reader.get() and 255)
val border = BmpUtil.nibbleForGray(reader.get() and 255)
val depth = reader.get().toByte().toInt()
val count = reader.getShort().toInt() and 65535
require(count <= 2048)
require(w in 1..640 && h in 1..480 && 5 + (w + 1) / 2 * h <= 65536 && reader.remaining() >= w * h)
val gray = ByteArray(w * h); reader.get(gray)
require(reader.remaining() >= count * 8)
val occlusions = List(count) { IntArray(4) { reader.getShort().toInt() and 65535 } }
val mask = ArrayList<IntArray>()
if (kind == 5) for (yy in 0 until h) {
var start = -1
for (xx in 0..w) {
val covered = xx < w && gray[yy * w + xx].toInt() != 0
if (covered && start < 0) start = xx
if (!covered && start >= 0) {
val previous = mask.lastOrNull { it[0] == start && it[2] == xx - start && it[1] + it[3] == yy }
if (previous != null) previous[3]++ else mask.add(intArrayOf(start, yy, xx - start, 1))
start = -1
}
}
}
return MenuSelection(x, y, w, h, radius, background, border, depth, BmpUtil.pack4bppFromGray8(gray, w, h), occlusions, kind, mask)
}
}
}
@@ -1,19 +1,29 @@
package com.faceclaw.app
/** Immutable shell snapshot. Keys identify writable surfaces across repaints, not their pixels. */
class ShellScene(val layers: List<Layer>) {
class Layer(val key: Int, val x: Int, val y: Int, val width: Int, val height: Int, val dim: Int, val packed: ByteArray)
val fingerprint: String = layers.joinToString(";") { "${it.key},${it.x},${it.y},${it.width},${it.height},${it.dim},${CachedResource(it.packed).hash}" }
fun preview(screenGray: ByteArray, width: Int, height: Int): ByteArray {
val screen = BmpUtil.pack4bppFromGray8(screenGray, width, height)
val output = ByteArray(screen.size); val resources = HashMap<Int, ByteArray>(); val calls = ArrayList<ByteArray>()
for ((id, layer) in layers.withIndex()) {
class ShellScene(val layers: List<Layer>, val selections: List<MenuSelection> = emptyList()) {
class Layer(val key: Int, val x: Int, val y: Int, val width: Int, val height: Int, val dim: Int, val packed: ByteArray, val selections: List<MenuSelection> = emptyList(), val depth: Int = 0)
val allSelections = selections + layers.flatMap { it.selections }
val fingerprint: String = layers.joinToString(";") { "${it.key},${it.x},${it.y},${it.width},${it.height},${it.dim},${it.depth},${CachedResource(it.packed).hash},${it.selections.joinToString { row -> row.fingerprint }}" } + selections.joinToString { it.fingerprint }
fun calls(width: Int, height: Int, surfaces: IntArray, selected: IntArray): List<ByteArray> {
val calls = ArrayList<ByteArray>(); var rowId = 0
for (row in selections) calls.addAll(row.calls(selected[rowId++]))
for ((index, layer) in layers.withIndex()) {
if (layer.dim < 256) calls.add(DrawProtocol.lut(width, height, layer.dim))
resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed)
calls.add(DrawProtocol.image(id, layer.x, layer.y))
calls.add(DrawProtocol.image(surfaces[index], layer.x, layer.y, depth = layer.depth))
for (row in layer.selections) calls.addAll(row.calls(selected[rowId++]))
}
resources[511] = DrawProtocol.displayList(calls)
DisplayListRenderer(resources).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height))
return calls
}
fun preview(screenGray: ByteArray, width: Int, height: Int, rightLens: Boolean = false): ByteArray {
val screen = BmpUtil.pack4bppFromGray8(screenGray, width, height)
val output = ByteArray(screen.size); val resources = HashMap<Int, ByteArray>()
val surfaces = IntArray(layers.size) { id ->
val layer = layers[id]; resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed); id
}
val rows = IntArray(allSelections.size) { id -> resources[id + layers.size] = allSelections[id].resource.bytes; id + layers.size }
resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows))
DisplayListRenderer(resources, rightLens = rightLens).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height))
val target = DisplayListRenderer.Target(output, width, height)
return ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() }
}
@@ -23,17 +33,20 @@ class ShellScene(val layers: List<Layer>) {
val count = reader.getShort().toInt() and 65535; require(count <= 64)
val layers = ArrayList<Layer>()
repeat(count) {
require(reader.remaining() >= 14)
require(reader.remaining() >= 16)
val key = reader.getShort().toInt() and 65535
val x = reader.getShort().toInt(); val y = reader.getShort().toInt()
val w = reader.getShort().toInt() and 65535; val h = reader.getShort().toInt() and 65535
val dim = reader.getShort().toInt() and 65535
val reserved = reader.getShort().toInt(); require(reserved == 0 && dim <= 256)
val selectionCount = reader.getShort().toInt() and 65535; require(selectionCount <= 64 && dim <= 256)
val depth = reader.getShort().toInt(); require(depth in -128..127)
require(w in 1..640 && h in 1..480 && 5 + (w + 1) / 2 * h <= 65536 && reader.remaining() >= w * h)
val gray = ByteArray(w * h); reader.get(gray)
layers.add(Layer(key, x, y, w, h, dim, BmpUtil.pack4bppFromGray8(gray, w, h)))
val selections = List(selectionCount) { MenuSelection.read(reader, reader.get().toInt()) }
layers.add(Layer(key, x, y, w, h, dim, BmpUtil.pack4bppFromGray8(gray, w, h), selections, depth))
}
require(reader.remaining() == 0)
require(layers.sumOf { it.selections.size } + layers.size < 511)
require(layers.map { it.key }.toSet().size == layers.size)
return ShellScene(layers)
}
@@ -50,6 +50,7 @@ class SurfaceCompositor {
var out: MutableList<ScreenDraw> = ArrayList()
while ((cursor.remaining() >= 1)) {
var kind: Int = (cursor.get() and 0xff)
if (kind in 3..5) { out.add(ScreenDraw.image(0, 0, 0).also { it.selection = MenuSelection.read(cursor, kind) }); continue }
if (((kind == ScreenDraw.KIND_GLYPH) && (cursor.remaining() >= 11))) {
var fontId: Int = (cursor.getShort().toInt() and 0xffff)
var encoding: Int = cursor.getInt()
@@ -106,6 +107,7 @@ class SurfaceCompositor {
* raster comes from ImageAtlas under imageId.
*/
class ScreenDraw {
@JvmField var selection: MenuSelection? = null
companion object {
const val KIND_GLYPH: Int = 0
@@ -609,6 +611,7 @@ class SurfaceCompositor {
var fingerprint: StringBuilder = StringBuilder()
fingerprint.append(screenWidth).append('x').append(screenHeight)
var draws: MutableList<ScreenDraw> = ArrayList()
val selections = ArrayList<MenuSelection>()
for (surface in ordered) {
if (!surface.visible || (shellScene != null && surface.id == "shell")) {
continue
@@ -616,6 +619,8 @@ class SurfaceCompositor {
blendLocked(gray, surface)
var dim: Int = dimForLocked(surface)
for (draw in surface.draws) {
val selected = draw.selection
if (selected != null) { selections.add(selected.translated(surface.x, surface.y)); continue }
if (((dim < 256) && (draw.kind == ScreenDraw.KIND_IMAGE))) {
continue
}
@@ -656,8 +661,8 @@ class SurfaceCompositor {
fingerprint.append(":dim").append(dim)
}
}
val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene.EMPTY else shellScene ?: ShellScene.EMPTY
val preview = if (shellScene != null) scene.preview(gray, screenWidth, screenHeight) else gray
val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene.EMPTY else ShellScene(shellScene?.layers ?: emptyList(), selections)
val preview = if (shellScene != null || selections.isNotEmpty()) scene.preview(gray, screenWidth, screenHeight) else gray
return Composite(preview, screenWidth, screenHeight, fingerprint.append("|shell:").append(scene.fingerprint).toString(),
nextCompositeSeq++, draws.toTypedArray()).also { it.screenGray = gray; it.shellScene = scene }
}