Refactor texture cache usage to shared Kotlin for iOS

This commit is contained in:
James Babcock committed 2026-09-20 15:12:13 -07:00
1 parent ccb320c66f
commit 07828db0bf
23 files changed
+552 -74

No files matched your search

+2 -1
View File
@@ -149,13 +149,14 @@ test('heading updates reuse the texture; brightness, layout and clipping changes
test('the background registers once and uses a nine-byte image reference on later frames', () => {
const registered = [];
const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', {}, '', {
const atlas = load('app/native/texture-atlas.ts', {}, '', {
global: { isAndroid: true },
com: { faceclaw: { app: { AndroidByteReader: class { constructor(buffer) { this.buffer = buffer; } }, ImageAtlas: { ensure: (...args) => {
registered.push(args);
return 1;
} } } } },
});
const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', { '../native/texture-atlas': atlas });
const background = compass.createCompassBackground(576, 260, 256, 193.25, 98, 140, 1);
for (const heading of [0, 1, 2, 45, 90]) {
const image = background.clone();
+40 -4
View File
@@ -11,6 +11,39 @@ function load(file, context) {
return sandbox.exports;
}
test('iOS worker replies keep cached draw references paired with their baked pixels', () => {
const replies = [], received = [];
const wrap = buffer => ({ length: buffer.byteLength, buffer,
base64EncodedStringWithOptions: () => Buffer.from(buffer).toString('base64') });
const NSData = {
dataWithBytesLength: buffer => wrap(buffer),
alloc: () => ({ initWithBase64EncodedStringOptions: text => {
const bytes = Uint8Array.from(Buffer.from(text, 'base64')); return wrap(bytes.buffer);
} }),
};
const interop = { handleof: buffer => buffer, bufferFromData: data => data.buffer };
const display = load('app/native/active-display.ios.ts', {
NSData, interop, global: { postMessage: message => replies.push(message) },
}).getActiveDisplay();
const pixels = new Uint8Array([255, 0, 0, 255]), draws = new Uint8Array([1, 7, 0, 0, 0, 0, 0, 0, 0]);
display.submitSurfaceFrame(pixels.buffer, 'window:test', 0, 0, 2, 2, 'frame', 0, 1, draws.buffer);
const worker = {};
const { WorkerAppHost } = load('app/ui/shell/worker-window.ts', {
require: () => ({}), global: { isIOS: true }, NSData, interop,
});
const host = new WorkerAppHost({ appId: 'test', worker, submitPixels: (...args) => received.push(args) });
host.openWindows.add('test');
worker.onmessage({ data: replies[0] });
assert.equal(received.length, 1);
assert.deepEqual([...received[0][1]], [...pixels]);
assert.deepEqual([...new Uint8Array(received[0][4])], [...draws]);
display.submitSurfaceFrame(pixels.buffer, 'window:test', 0, 0, 2, 2);
worker.onmessage({ data: replies[1] });
assert.equal(received[1][4], null, 'a pixel-only update must clear previous draw identities');
host.openWindows.clear(); worker.onmessage({ data: replies[0] });
assert.equal(received.length, 2, 'closed windows cannot deliver stale cached draws');
});
test('iOS installed apps launch, recover missing packages, and close before uninstall', async () => {
const calls = [], app = { packageId: 'test.app', name: 'Test' };
let present = true;
@@ -215,9 +248,11 @@ test('background glasses input still composites frames; phone resume preserves t
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../graphics/surface-compositor': { SurfaceCompositor: class {
configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {} setScreenBlanked() {}
composite() { return new Uint8Array([1, 2]); }
compositeFrame() { return { pixels: new Uint8Array([1, 2]) }; }
} },
'../graphics/plane': { flattenPlanes: () => ({ pixels: new Uint8Array([1, 2]), width: 2, height: 1 }) },
'../graphics/plane': { flattenPlanesWithDraws: () => ({ image: { pixels: new Uint8Array([1, 2]), width: 2, height: 1 }, draws: [] }) },
'../graphics/glyph-wire': { prepareFrameDraws: () => null },
'../native/texture-planner.ios': { IosTexturePlanner: class {} },
'../native/ios-graphics': { previewPixels: pixels => pixels },
'../ui/shell/shell': { shell, rawInputEventToInputEvent: input => input },
'../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 640, height: 480 }) },
@@ -333,7 +368,8 @@ test('iOS configures an in-process surface before submitting constructor-trigger
require: name => ({
'../ui/shell/shell': { shell },
'../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 2, height: 1 }) },
'../graphics/plane': { flattenPlanes: planes => planes[0] },
'../graphics/plane': { flattenPlanesWithDraws: planes => ({ image: planes[0], draws: [] }) },
'../graphics/glyph-wire': { prepareFrameDraws: () => null },
})[name] ?? {},
console,
});
@@ -365,7 +401,7 @@ test('iOS configures an in-process surface before submitting constructor-trigger
test('iOS compositor rejects missing and removed surfaces before crossing into Kotlin', () => {
let nativeSubmissions = 0;
const native = { configureIdXYWidthHeightZOrderTransparent() {}, removeId() {}, submitIdDataXYWidthHeight() { nativeSubmissions++; } };
const native = { configureIdXYWidthHeightZOrderTransparent() {}, removeId() {}, submitDrawsIdDataXYWidthHeightDraws() { nativeSubmissions++; } };
const api = load('app/graphics/surface-compositor.ios.ts', {
require: () => ({ toData: data => data }),
FaceclawKitIosSurfaceCompositor: { alloc: () => ({ initWithWidthHeight: () => native }) },
+63 -2
View File
@@ -165,9 +165,9 @@ async function until(predicate) {
for (let i = 0; i < 100; i++) { if (predicate()) return; await new Promise(resolve => setTimeout(resolve, 5)); }
assert.fail('Timed out waiting for test state');
}
function harness(t, transport = new FakeTransport()) {
function harness(t, transport = new FakeTransport(), textures) {
const states = [], input = [], compass = [], wear = [];
const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing));
const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing), undefined, textures);
t.after(async () => { transport.hold = false; await session.stop(); });
return { transport, session, states, input, compass, wear };
}
@@ -306,6 +306,67 @@ const ack = (transport, sent, overrides) => transport.ack(sent.id, sent.message,
const tick = () => new Promise(resolve => setImmediate(resolve));
const noisy = () => Uint8Array.from({ length: 640 * 480 }, (_, i) => (i % 2) * 255);
test('texture uploads precede their frame, share ACK accounting, and coalesce pixels with draw identities', async t => {
const plans = [];
let resets = 0;
const textures = {
reset() { resets++; },
plan(previous, next, frame, firstId) {
plans.push({ previous, next, frame, firstId });
return { uploads: plans.length === 1 ? [new Uint8Array([18, 1]), new Uint8Array([18, 2])] : [],
payload: new Uint8Array([8, plans.length]), nextFid: firstId + 2, usedBytes: 100 };
},
};
const h = harness(t, new FakeTransport(), textures);
await h.session.start(addresses); h.transport.hold = true;
const a = { native: { id: 'a' } }, b = { native: { id: 'b' } }, c = { native: { id: 'c' } };
h.session.setFrame(new Uint8Array(640 * 480), a);
await until(() => imageMessages(h.transport).length === 3);
const sent = imageMessages(h.transport);
assert.deepEqual(sent.map(s => s.message.payload[0]), [18, 18, 8]);
h.session.setFrame(new Uint8Array(640 * 480).fill(80), b);
h.session.setFrame(new Uint8Array(640 * 480).fill(160), c);
assert.equal(plans.length, 1, 'coalesced frames must not allocate textures');
ack(h.transport, sent[2]); ack(h.transport, sent[1]); await tick();
assert.equal(h.session.state.frames, 0, 'a draw ACK cannot retire an unacknowledged upload');
ack(h.transport, sent[0]); await until(() => imageMessages(h.transport).length === 4);
assert.equal(h.session.state.frames, 1);
assert.equal(plans.length, 2); assert.equal(plans[1].frame, c);
assert.deepEqual(plans[1].previous, plans[0].next);
assert.equal(plans[1].firstId, plans[0].firstId + 2);
assert.ok(plans[1].next.every(value => value === 0xaa));
ack(h.transport, imageMessages(h.transport)[3]); await until(() => h.session.state.frames === 2);
h.transport.hold = false; await h.session.stop();
assert.ok(resets > 0);
});
test('a failed texture upload invalidates residency and the pipelined frame base', async t => {
let resets = 0;
const textures = {
reset() { resets++; },
plan() { return { uploads: [new Uint8Array([18])], payload: new Uint8Array([8]), nextFid: 1, usedBytes: 4 }; },
};
const h = harness(t, new FakeTransport(), textures); await h.session.start(addresses);
const original = h.session.requestCfw.bind(h.session);
h.session.requestCfw = (role, payload) => payload[0] === 18 ? Promise.reject(new Error('texture upload failed')) : original(role, payload);
h.session.setFrame(new Uint8Array(640 * 480), { native: {} });
await until(() => h.session.state.phase === 'retrying');
assert.equal(resets, 1); assert.equal(h.session.lastEnqueued, null);
assert.equal(h.session.latestTextures, null); assert.equal(h.session.state.frames, 0);
});
test('resuming near framebuffer lease expiry discards cached draws before transmission', async t => {
let resets = 0, plans = 0;
const textures = { reset() { resets++; }, plan() { plans++; return null; } };
const h = harness(t, new FakeTransport(), textures); await h.session.start(addresses);
h.session.lastLease = Date.now() - 85_000;
h.session.setFrame(new Uint8Array(640 * 480), { native: {} });
await h.session.pump();
assert.equal(h.session.state.phase, 'retrying');
assert.equal(resets, 1); assert.equal(plans, 0);
assert.equal(imageMessages(h.transport).length, 0);
});
// Both eyes must finish; wrong ingress, size, CRC, ordinal and legacy replies do not count.
test('CFW ACK identity and both-lens completion gate display metrics', async t => {
const { iosBleTraffic } = require('../.test-build/app/g2/ble-traffic-counters.js');
+2
View File
@@ -102,6 +102,8 @@ function fixture() {
'../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } },
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../apps/ios-availability': { iosAppUnavailableReason: () => null },
'../graphics/glyph-wire': { prepareFrameDraws: () => null },
'../native/texture-planner.ios': { IosTexturePlanner: class {} },
'../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images,
'../native/ios-graphics': { previewPixels: pixels => pixels },
'../ui/gestures': { makeInputEvent: event => event }, '../ui/layers': { noopLayerActions: {} },
@@ -0,0 +1,99 @@
package com.faceclaw.app
import kotlin.test.*
class TextureCacheTest {
private fun u32(bytes: ByteArray, offset: Int): Int =
(0..3).fold(0) { n, i -> n or ((bytes[offset + i].toInt() and 255) shl (i * 8)) }
@Test
fun offsetsAbove64KiBAndLazyFontSlotsRoundTripOnBothPlatforms() {
val cache = TextureCacheState()
val large = ImageAtlas.Entry(255, 255, ByteArray(255 * 255) { (it % 15 + 1).toByte() })
repeat(3) { assertTrue(cache.ensureImage(it, large) >= 0) }
val glyph = GlyphAtlas.Glyph(1, 1, 0, 1, 0, null, byteArrayOf(15))
val offset = cache.ensureGlyph(42, 65, glyph)
val table = cache.fontTableOffset(42)
assertTrue(table > 65535)
assertEquals(table + 384, offset)
val memory = ByteArray(TextureCacheState.CACHE_SIZE)
for (upload in cache.drainUploadPayloads(3600)) {
assertEquals(18, upload[0].toInt())
var pos = 1
while (pos < upload.size) {
val address = u32(upload, pos)
val length = (upload[pos + 4].toInt() and 255) or ((upload[pos + 5].toInt() and 255) shl 8)
upload.copyInto(memory, address, pos + 6, pos + 6 + length)
pos += 6 + length
}
assertEquals(upload.size, pos)
}
assertEquals(offset, u32(memory, table + (65 - 32) * 4))
assertContentEquals(glyph.cachedBytes, memory.copyOfRange(offset, offset + glyph.cachedBytes.size))
assertEquals(offset, cache.ensureGlyph(42, 65, glyph))
assertFalse(cache.hasPendingUploads())
cache.reset()
assertEquals(0, cache.usedBytes())
assertEquals(384, cache.ensureGlyph(42, 65, glyph))
assertTrue(cache.hasPendingUploads())
}
@Test
fun plannerReusesResidencyAndReuploadsAfterReset() {
val cache = TextureCacheState()
val id = ImageAtlas.ensure("texture-test-white", 2, 2, ArrayByteReader(ByteArray(4) { -1 }))
val pixels = ByteArray(8 * 4)
for (y in 0..1) for (x in 0..1) pixels[y * 8 + x] = -1
val packed = BmpUtil.pack4bppFromGray8(pixels, 8, 4)
val draws = arrayOf(SurfaceCompositor.ScreenDraw.image(id, 0, 0))
fun plan() = assertNotNull(TexturePlanner.plan(null, packed, 8, 4, draws, cache, 1, true, 8, testPlatform()))
val cold = plan()
assertEquals(1, cold.drawnImages)
assertTrue(cold.uploads.isNotEmpty())
val warm = plan()
assertTrue(warm.uploads.isEmpty())
assertContentEquals(cold.payload, warm.payload)
cache.reset()
assertTrue(plan().uploads.isNotEmpty())
// Occlusion fails the correctness check and leaves the image baked.
assertNull(TexturePlanner.plan(null, ByteArray(packed.size), 8, 4, draws, cache, 1, true, 8, testPlatform()))
}
@Test
fun fullCacheResetsAndRetriesTheCurrentFrame() {
val cache = TextureCacheState()
val large = ImageAtlas.Entry(255, 255, ByteArray(255 * 255) { (it % 15 + 1).toByte() })
var fillerId = -1
while (cache.ensureImage(fillerId--, large) >= 0) {}
// Fill the remaining tail, forcing the next real allocation to fail.
val remaining = TextureCacheState.CACHE_SIZE - cache.usedBytes()
while (cache.ensureImage(fillerId--, ImageAtlas.Entry(1, 1, byteArrayOf(15))) >= 0) {}
assertTrue(remaining > 0)
val generation = cache.generation()
val id = ImageAtlas.ensure("texture-test-full", 2, 2, ArrayByteReader(ByteArray(4) { -1 }))
val result = assertNotNull(TexturePlanner.plan(null, ByteArray(4) { -1 }, 4, 2,
arrayOf(SurfaceCompositor.ScreenDraw.image(id, 0, 0)), cache, 1, true, 8, testPlatform()))
assertEquals(generation + 1, cache.generation())
assertEquals(1, result.drawnImages)
assertTrue(cache.usedBytes() < 100)
assertTrue(result.uploads.isNotEmpty())
}
@Test
fun workingSetLargerThanTheCacheFallsBackWithoutAResetLoop() {
val cache = TextureCacheState()
val raster = ByteArray(255 * 255) { if (it % 2 == 0) 16 else 32 }
val draws = (0..4).map {
val id = ImageAtlas.ensure("oversized-working-set-$it", 255, 255, ArrayByteReader(raster))
SurfaceCompositor.ScreenDraw.image(id, 0, 0)
}.toTypedArray()
val gray = ByteArray(256 * 256)
repeat(255) { raster.copyInto(gray, it * 256, it * 255, (it + 1) * 255) }
val result = assertNotNull(TexturePlanner.plan(null,
BmpUtil.pack4bppFromGray8(gray, 256, 256), 256, 256, draws, cache, 1, true, 8, testPlatform()))
assertEquals(1, cache.generation())
assertEquals(4, result.drawnImages)
assertEquals(1, result.bakedCandidates)
assertTrue(cache.usedBytes() <= TextureCacheState.CACHE_SIZE)
}
}
@@ -0,0 +1,86 @@
package com.faceclaw.app
import kotlin.test.*
class IosTextureApiTest {
private fun imageDraw(id: Int): ByteArray = byteArrayOf(
1, id.toByte(), (id shr 8).toByte(), (id shr 16).toByte(), (id shr 24).toByte(), 0, 0, 0, 0,
)
@Test
fun nativeSnapshotsPreserveDrawsAcrossCoalescingAndSessionResets() {
val atlas = IosTextureAtlas()
val white = ByteArray(4) { -1 }.data()
val id = atlas.imageId("ios-texture-api-white", 2, 2, white)
assertEquals(id, IosTextureAtlas().imageId("ios-texture-api-white", 2, 2, white))
val compositor = IosSurfaceCompositor(8, 4)
compositor.configure("app", 2, 0, 2, 2, 0, false)
compositor.submitDraws("app", white, 0, 0, 2, 2, imageDraw(id).data())
val frame = compositor.compositeFrame()
assertEquals(2, frame.composite.draws!!.single().x)
// A later update must not change the previously submitted snapshot.
compositor.submit("app", ByteArray(4).data(), 0, 0, 2, 2)
assertTrue(compositor.compositeFrame().composite.draws!!.isEmpty())
val next = IosProtocol().pack(frame.pixels, 8, 4)
val planner = IosTexturePlanner()
val cold = assertNotNull(planner.plan(null, next, frame, 1))
assertEquals(18, cold.uploads.first().byteArray()[0].toInt())
assertEquals(8, cold.payload.byteArray()[0].toInt())
val warm = assertNotNull(planner.plan(null, next, frame, 1))
assertTrue(warm.uploads.isEmpty())
assertContentEquals(cold.payload.byteArray(), warm.payload.byteArray())
planner.reset()
assertTrue(assertNotNull(planner.plan(null, next, frame, 1)).uploads.isNotEmpty())
// A partial update clears identities even if the caller supplies stale draws.
compositor.submitDraws("app", byteArrayOf(-1).data(), 0, 0, 1, 1, imageDraw(id).data())
assertTrue(compositor.compositeFrame().composite.draws!!.isEmpty())
}
@Test
fun glyphRegistrationAndDrawingUseTheSharedAsciiTable() {
val atlas = IosTextureAtlas()
val key = "ios-aa-texture-test"
val bytes = key.encodeToByteArray()
atlas.registerAaGlyphs((byteArrayOf(bytes.size.toByte()) + bytes +
byteArrayOf(2, 1, 0, 65, 0, 0, 0, 0, 0, 2, 2, -1, -1)).data())
val font = atlas.fontId(key)
val draws = byteArrayOf(0, font.toByte(), (font shr 8).toByte(), 65, 0, 0, 0, 0, 0, 0, 0, -1)
val compositor = IosSurfaceCompositor(4, 2)
compositor.configure("text", 0, 0, 2, 2, 0, false)
compositor.submitDraws("text", ByteArray(4) { -1 }.data(), 0, 0, 2, 2, draws.data())
val frame = compositor.compositeFrame()
val plan = assertNotNull(IosTexturePlanner().plan(null, IosProtocol().pack(frame.pixels, 4, 2), frame, 1))
assertTrue(plan.usedBytes > 384)
val payload = plan.payload.byteArray()
// The last mode-8 submessage is the shared mode-20 cached glyph run.
assertEquals(20, payload[payload.size - 12].toInt())
assertEquals(65, payload.last().toInt())
}
@Test
fun oversizedPlanDiscardsUnsentResidencyBeforePixelFallback() {
val atlas = IosTextureAtlas()
val white = ByteArray(4) { -1 }.data()
val id = atlas.imageId("ios-oversized-plan", 2, 2, white)
var random = 12345
val gray = ByteArray(640 * 480) {
random = random xor (random shl 13)
random = random xor (random ushr 17)
random = random xor (random shl 5)
random.toByte()
}
gray[0] = -1; gray[1] = -1; gray[640] = -1; gray[641] = -1
val compositor = IosSurfaceCompositor(640, 480)
compositor.configure("noise", 0, 0, 640, 480, 0, false)
compositor.submitDraws("noise", gray.data(), 0, 0, 640, 480, imageDraw(id).data())
val frame = compositor.compositeFrame()
val planner = IosTexturePlanner()
assertNull(planner.plan(null, IosProtocol().pack(frame.pixels, 640, 480), frame, 1))
val small = IosSurfaceCompositor(4, 2)
small.configure("icon", 0, 0, 2, 2, 0, false)
small.submitDraws("icon", white, 0, 0, 2, 2, imageDraw(id).data())
val next = small.compositeFrame()
val retry = assertNotNull(planner.plan(null, IosProtocol().pack(next.pixels, 4, 2), next, 1))
assertTrue(retry.uploads.isNotEmpty(), "the rejected plan's image was never uploaded")
}
}