Refactor texture cache usage to shared Kotlin for iOS

This commit is contained in:
James Babcock committed 2026-09-20 15:12:13 -07:00
1 parent ccb320c66f
commit 07828db0bf
23 files changed
+552 -74

No files matched your search

+17 -4
View File
@@ -45,9 +45,21 @@ On `ios-port`, platform adapters route the existing TypeScript session and UI
logic through shared Kotlin for protocol builders, CRC/framing/reassembly,
event decoding, CFW compression/ACKs, changed-image regions, surface composition,
and LVGL glyphs. The original pure TypeScript implementations remain as Node
test references; NativeScript selects the `.ios.ts` native adapters. Features
not yet present in the iOS app can use the shared audio/recording and texture
classes without another Java port.
test references; NativeScript selects the `.ios.ts` native adapters.
Texture rendering uses the same `GlyphAtlas`, `ImageAtlas`, `TextureCacheState`
and `TexturePlanner` on both platforms. iOS's `IosTextureAtlas` and
`IosTexturePlanner` expose bulk `NSData` bridges. Surface updates retain draw
identities through composition, including worker replies; immutable composite
snapshots keep those identities paired with the pixels while frames coalesce.
The BLE session plans only frames it will enqueue, sends mode-18 uploads before
the mode-19/20 draw batch, and counts a frame only after all uploads and draws
are acknowledged by both lenses. Session failure/teardown resets residency;
resuming near the firmware's 90-second lease deadline restarts the session
before queued draws can reference a freed cache. In either case,
the next frame reuploads lazily. Plans over the CFW message limit fall back to
pixel bands and invalidate any unsent allocations. The 256 KiB bump allocator
and reset-on-full policy are unchanged; there is no pinning or individual eviction.
## Build and validation
@@ -71,7 +83,8 @@ npm run test:kotlin:ios
and iOS: wire vectors, framing/corruption/expiry, streaming compression, replay,
composition, PNG pixels, font glyphs, and Java SHA-256 golden outputs for DSP
and GIF. Android additionally runs the existing Java protocol fixtures and
concurrency checks; iOS exercises `NSData` and compositor adapters. Debug app
concurrency checks; common texture tests cover reuse, overflow, reset and 32-bit
offsets, and iOS exercises `NSData`, atlas, compositor and planner adapters. Debug app
startup checks real NativeScript metadata and bidirectional callbacks, logging
`FACECLAW_KOTLIN_BRIDGE_PASS` and `FACECLAW_KOTLIN_PROTOCOL_PASS`.
@@ -1,6 +1,7 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmOverloads
import kotlin.jvm.JvmStatic
/**
@@ -54,6 +55,7 @@ class TexturePlanner {
* should run instead (no replayable draws, or identical frames).
*/
@JvmStatic
@JvmOverloads
fun plan(
previous: ByteArray?,
next: ByteArray?,
@@ -64,6 +66,7 @@ class TexturePlanner {
fidStart: Int,
allowMultiRect: Boolean,
maxRects: Int,
platform: ProtocolPlatform = protocolPlatform(),
): Result? {
if (
(((((next == null) || (draws == null)) || (draws.size == 0)) || (width <= 0)) ||
@@ -75,7 +78,7 @@ class TexturePlanner {
if ((next.size != (stride * height))) {
return null
}
var planStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var planStartedAtMs: Long = platform.elapsedRealtimeMs()
var fullFrame: Boolean = ((previous == null) || (previous.size != next.size))
var rects: MutableList<IntArray>
if (!fullFrame) {
@@ -103,7 +106,7 @@ class TexturePlanner {
} else {
rects = mutableListOf(intArrayOf(0, 0, width, height))
}
var matchStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var matchStartedAtMs: Long = platform.elapsedRealtimeMs()
var selected: MutableList<Selected> = ArrayList()
var bakedCandidates: Int = 0
var selectedImages: Int = 0
@@ -177,7 +180,7 @@ class TexturePlanner {
if ((selected.isEmpty() && fwSubs.isEmpty())) {
return null
}
var cacheStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var cacheStartedAtMs: Long = platform.elapsedRealtimeMs()
var drawable: MutableList<Selected> =
(if (selected.isEmpty()) mutableListOf() else ensureResident(cache, selected))
bakedCandidates += (selected.size - drawable.size)
@@ -204,7 +207,7 @@ class TexturePlanner {
}
return null
}
var punchStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var punchStartedAtMs: Long = platform.elapsedRealtimeMs()
var punched: ByteArray = next.copyOf()
for (sel in drawn) {
punch(punched, stride, width, height, sel)
@@ -212,7 +215,7 @@ class TexturePlanner {
for (fw in fwPunches) {
punchFw(punched, stride, width, height, fw)
}
var encodeStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var encodeStartedAtMs: Long = platform.elapsedRealtimeMs()
var subs: MutableList<ByteArray> = ArrayList()
var fid: Int = fidStart
if (fullFrame) {
@@ -256,7 +259,7 @@ class TexturePlanner {
uploadBytes,
fullFrame,
)
var doneAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var doneAtMs: Long = platform.elapsedRealtimeMs()
result.rectsMs = ((matchStartedAtMs - planStartedAtMs)).toInt()
result.matchMs = ((cacheStartedAtMs - matchStartedAtMs)).toInt()
result.cacheMs = ((punchStartedAtMs - cacheStartedAtMs)).toInt()
@@ -466,6 +466,18 @@ class SurfaceCompositor {
rectWidth: Int,
rectHeight: Int,
fingerprint: String,
) = submitSurface(surfaceId, pixels, rectX, rectY, rectWidth, rectHeight, fingerprint, null)
/** Retain the full surface's draw identities along with its already-baked pixels. */
fun submitSurface(
surfaceId: String,
pixels: ByteReader,
rectX: Int,
rectY: Int,
rectWidth: Int,
rectHeight: Int,
fingerprint: String,
draws: ByteReader?,
) {
updateSurface(
surfaceId,
@@ -475,7 +487,7 @@ class SurfaceCompositor {
rectWidth,
rectHeight,
fingerprint,
null,
draws,
false,
)
}
@@ -230,7 +230,10 @@ class IosSurfaceCompositor(width: Int, height: Int) {
fun blank(blanked: Boolean) = compositor.setBlanked(blanked)
fun submit(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int) {
fun submit(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int) =
submitDraws(id, data, x, y, width, height, null)
fun submitDraws(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int, draws: NSData?) {
val (sw, sh) = sizes[id] ?: error("Unknown surface: $id")
require(width > 0 && height > 0 && data.length == width.toULong() * height.toULong())
val left = maxOf(0, x)
@@ -260,8 +263,14 @@ class IosSurfaceCompositor(width: Int, height: Int) {
right - left,
bottom - top,
(++sequence).toString(),
// Draw records describe a full surface. Partial/clipped updates must discard
// old identities; their pixels remain available through the baked fallback.
if (x == 0 && y == 0 && width == sw && height == sh) draws?.let(::IosByteReader)
else null,
)
}
fun composite(): NSData = compositor.composite().gray.data()
fun compositeFrame(): IosTextureFrame = IosTextureFrame(compositor.composite())
}
@@ -0,0 +1,50 @@
package com.faceclaw.app
import platform.Foundation.NSData
/** Binary adapters only: atlas identity, RLE, residency and planning stay in commonMain. */
class IosTextureAtlas {
fun fontId(key: String): Int = GlyphAtlas.fontId(key)
fun registerGlyphs(data: NSData) = GlyphAtlas.register(IosByteReader(data))
fun registerAaGlyphs(data: NSData) = GlyphAtlas.registerAa(IosByteReader(data))
fun registerFirmwareGlyphs(data: NSData) = FwGlyphAtlas.register(IosByteReader(data))
fun imageId(key: String, width: Int, height: Int, data: NSData): Int =
ImageAtlas.ensure(key, width, height, IosByteReader(data))
}
/** A snapshot keeps pixels and their screen-space draw identities together while JS coalesces. */
class IosTextureFrame internal constructor(internal val composite: SurfaceCompositor.Composite) {
val pixels: NSData = composite.gray.data()
}
class IosTexturePlan internal constructor(result: TexturePlanner.Result, val usedBytes: Int) {
val payload: NSData = result.payload.data()
val uploads: List<NSData> = result.uploads.map { it.data() }
val nextFid: Int = result.nextFid
}
/** One instance per BLE session, called on the session's serial JS thread. */
class IosTexturePlanner {
private val cache = TextureCacheState()
fun reset() = cache.reset()
fun plan(previous: NSData?, next: NSData, frame: IosTextureFrame, firstId: Int): IosTexturePlan? {
val composite = frame.composite
val result = TexturePlanner.plan(
previous?.byteArray(), next.byteArray(), composite.width, composite.height,
composite.draws, cache, firstId, true, ConnectionOptions.MULTI_RECT_MAX_RECTS,
) ?: return null
// A plan is atomic. If it cannot fit the CFW transport, discard its uploads
// and residency too, before the session falls back to ordinary pixel bands.
if (result.payload.size > 65535 || result.uploads.any { it.size > 65535 }) {
cache.reset()
return null
}
return IosTexturePlan(result, cache.usedBytes())
}
}