Refactor texture cache usage to shared Kotlin for iOS

This commit is contained in:
James Babcock committed 2026-09-20 15:12:13 -07:00
1 parent ccb320c66f
commit 07828db0bf
23 files changed
+552 -74

No files matched your search

+31 -10
View File
@@ -9,6 +9,7 @@ import { hasCompatibleFirmware, firmwareIncompatibilityMessage, REQUIRED_FACECLA
import { hexToBytes } from '../util/hex-util'
import { deviceAddressError } from './ios-peripheral-identity'
import { decodeWearState, enableWearDetection, queryWearState } from './wear-protocol'
import type { SessionTexturePlanner, TextureFrame } from './texture-planner'
declare function setTimeout(callback: () => void, ms: number): number
declare function clearTimeout(id: number): void
@@ -60,6 +61,7 @@ export class GlassesSession {
private wanted = false
private addresses: SessionAddresses | null = null
private latest: Uint8Array | null = null
private latestTextures: TextureFrame | null = null
private displayed: Uint8Array | null = null
private lastEnqueued: Uint8Array | null = null
// Match Android: advance only for emitted deltas, skipping 0 / 0xffff.
@@ -90,7 +92,8 @@ export class GlassesSession {
private readonly onActivity: () => void = () => {},
private readonly onCompass: (event: CompassEvent) => void = () => {},
private readonly onWearState: (wearing: boolean) => void = () => {},
onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {}) {
onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {},
private readonly textures?: SessionTexturePlanner) {
this.notifications = new AncsClient(packet => this.writePackets('right', () => [packet]), onNotificationsChanged, message => this.log('ANCS: ' + message))
this.off = transport.onEvent(event => this.receive(event))
}
@@ -428,9 +431,11 @@ export class GlassesSession {
if (this.state.phase !== 'connected') return
await this.request('right', protocol.SID.settings, magic => setBrightness(magic, level), 'Brightness')
}
setFrame(gray: Uint8Array): void {
setFrame(gray: Uint8Array, textures?: TextureFrame): void {
if (this.state.phase !== 'connected') return
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
this.latest = protocol.packGray4(gray, 640, 480)
this.latestTextures = textures ?? null
this.schedule(0)
}
/** Same mode-5 kind-4 sequencer payload and acknowledged image path as Android. */
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
@@ -499,16 +504,27 @@ export class GlassesSession {
while (this.canSendDisplay()) {
if (!this.displaySending) {
const packed = this.latest!; this.latest = null
const textureFrame = this.latestTextures; this.latestTextures = null
// Compare with the last enqueued image, not the last ACKed one. In an
// A -> B -> A sequence, B may still be in flight when A is requested.
if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId)
const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed))
if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
const commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId)
if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands)
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`)
const texturePlan = textureFrame ? this.textures?.plan(this.lastEnqueued, packed, textureFrame, this.nextImageFrameId) : null
let commands: Uint8Array[]
if (texturePlan) {
// Uploads participate in the same ordered, acknowledged command window as
// the image. Coalescing only happens before planning mutates residency.
commands = [...texturePlan.uploads, texturePlan.payload]
this.nextImageFrameId = texturePlan.nextFid
this.log(`Display textures (${texturePlan.uploads.length} uploads, cache=${texturePlan.usedBytes}B)`)
} else {
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId)
const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed))
if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId)
if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands)
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`)
}
this.displaySending = { packed, commands, offset: 0, pending: 0 }
this.displayFrames.push(this.displaySending); this.lastEnqueued = packed
}
@@ -538,6 +554,10 @@ export class GlassesSession {
this.pumping = true; const generation = this.generation
try {
const now = Date.now()
// Firmware frees textures when its 90-second framebuffer lease expires.
// After a long iOS suspension, reconnect rather than replaying pending
// draws against unknown memory. Leave a margin for the two arm writes.
if (now - this.lastLease >= 80_000) throw new Error('Display lease lapsed; restarting session')
if (now - this.lastHeartbeat >= 4000) {
const gap = now - this.lastHeartbeat
await this.request('right', protocol.SID.hub, protocol.heartbeat, 'Heartbeat', 1500); this.lastHeartbeat = Date.now()
@@ -571,6 +591,7 @@ export class GlassesSession {
for (const item of custom) item.reject(new Error('Connection ended'))
this.cfwTransports.clear()
this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null
this.latestTextures = null; this.textures?.reset()
this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0
}
private closeLinks(): void {
+14 -12
View File
@@ -35,7 +35,9 @@ import { getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetti
import { readPhoneBatteryState } from '../native/phone-battery'
import { iosAppUnavailableReason } from '../apps/ios-availability'
import { SurfaceCompositor } from '../graphics/surface-compositor'
import { flattenPlanes, type Plane } from '../graphics/plane'
import { flattenPlanesWithDraws, type Plane } from '../graphics/plane'
import { prepareFrameDraws } from '../graphics/glyph-wire'
import { IosTexturePlanner } from '../native/texture-planner.ios'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { previewPixels } from '../native/ios-graphics'
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
@@ -62,8 +64,8 @@ export class IosPreviewController {
setScreenBlanked: async blanked => {
this.compositor.setScreenBlanked(blanked); this.scheduleFrame()
},
submitSurfaceFrame: async (id, pixels, rect) => {
this.compositor.submitSurfaceFrame(id, pixels, rect); this.scheduleFrame()
submitSurfaceFrame: async (id, pixels, rect, _fingerprint, _paintMs, _frameId, draws) => {
this.compositor.submitSurfaceFrame(id, pixels, rect, draws); this.scheduleFrame()
},
}
private readonly glance = new GlanceHost({
@@ -336,8 +338,8 @@ export class IosPreviewController {
}
}
private async submit(id: string, planes: Plane[]): Promise<void> {
const image = flattenPlanes(planes)
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
const { image, draws } = flattenPlanesWithDraws(planes)
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, prepareFrameDraws(draws))
this.scheduleFrame()
}
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
@@ -350,12 +352,12 @@ export class IosPreviewController {
try {
if (this.shellDirty) {
this.shellDirty = false
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
const { image, draws } = flattenPlanesWithDraws(shell.paintSurface(), { width: 640, height: 480 })
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 }, prepareFrameDraws(draws))
this.compositor.setUnderlayDim(1, shell.underlayDim())
}
const pixels = this.compositor.composite()
this.session?.setFrame(pixels)
const { pixels, textures } = this.compositor.compositeFrame()
this.session?.setFrame(pixels, textures)
if (this.active) {
const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green')
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
@@ -511,8 +513,8 @@ export class IosPreviewController {
},
setSurfaceVisible: (id, visible) => { this.compositor.setSurfaceVisible(id, visible); this.scheduleFrame() },
removeSurface: id => { this.compositor.removeSurface(id); this.scheduleFrame() },
submitPixels: (id, pixels, width, height) => {
this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }); this.scheduleFrame()
submitPixels: (id, pixels, width, height, draws) => {
this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }, draws); this.scheduleFrame()
},
requestShellRender: () => this.requestShellRender(),
openSettings: section => { void this.launchApp('settings', { section }) },
@@ -560,7 +562,7 @@ export class IosPreviewController {
this.refreshClock()
}, receiveCompassEvent, wearing => this.handleWearState(wearing), (key, popup) => {
iosNotificationsChanged(key, popup); this.requestShellRender()
})
}, new IosTexturePlanner())
bindIosNotifications(this.session.notifications)
bindCompassSession(this.session)
}
+8
View File
@@ -0,0 +1,8 @@
/** Platform-owned snapshot of the same composite as the submitted pixels. */
export type TextureFrame = { readonly native: object }
export type TexturePlan = { uploads: Uint8Array[]; payload: Uint8Array; nextFid: number; usedBytes: number }
/** Sessions own residency; frame snapshots may be coalesced without changing it. */
export interface SessionTexturePlanner {
plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null
reset(): void
}
+24 -24
View File
@@ -1,20 +1,20 @@
/**
* Marshals deferred-draw identity (text glyphs and icon images) to the Java
* side for the texture-cache pipeline (CFW modes 12/13/14; see
* Marshals deferred-draw identity (text glyphs and icon images) to the Kotlin
* side for the texture-cache pipeline (CFW modes 18/19/20; see
* notes/texture-cache-display-list-design.md).
*
* Channels, all cheap ByteBuffer crossings:
* - glyph raster registration into the process-wide Java GlyphAtlas, once
* - glyph raster registration into the process-wide Kotlin GlyphAtlas, once
* per (font, glyph) per JS context (main thread and each worker register
* independently; the atlas dedupes by the font's stable atlasKey);
* - image registration into the Java ImageAtlas, once per distinct image
* - image registration into the Kotlin ImageAtlas, once per distinct image
* content per JS context (images are content-addressed: the key is a hash
* of dimensions + pixels, so the same icon rendered anywhere dedupes and
* a changed icon is simply a new image);
* - a per-frame draw list accompanying each submitted surface frame, which
* the Java planner may replay as on-glasses cached draws.
* the Kotlin planner may replay as on-glasses cached draws.
*
* Draws that can't participate (no atlasKey, outside the mode-14 ASCII
* Draws that can't participate (no atlasKey, outside the mode-20 ASCII
* table, ink outside the line cell, oversized images) are simply left out —
* their pixels are baked into the submitted frame either way, so exclusion
* only means "no wire savings for this draw".
@@ -22,7 +22,7 @@
import { Glyph } from "./bdffont";
import { GrayImage, type DeferredDraw, type GlyphFont, type PlacedFwText, type PlacedImage } from "./image";
declare const com: any;
import { textureAtlasAvailable, textureFontId, registerTextureGlyphs, registerFirmwareGlyphs, textureImageId } from "../native/texture-atlas";
const GLYPH_RECORD_BYTES = 12; // [0][fontId u16][encoding u32][penX s16][lineY s16][value u8]
const IMAGE_RECORD_BYTES = 9; // [1][imageId u32][x s16][y s16]
@@ -32,14 +32,14 @@ const FWTEXT_MEMBER_BYTES = 7; // [cp u32][dx s16][ink u8]
type FontWireState = {
fontId: number;
cellHeight: number;
/** Encodings already registered with the Java atlas from this JS context. */
/** Encodings already registered with the Kotlin atlas from this JS context. */
registered: Set<number>;
};
const fontStates = new Map<GlyphFont, FontWireState>();
/**
* Java-side image id per source image object, or null for images that can't
* Kotlin-side image id per source image object, or null for images that can't
* participate. Sources handed to drawImage are immutable by contract, so
* memoizing by object identity is safe; a re-rendered icon is a new object
* and re-resolves (usually to the same content-addressed id).
@@ -47,12 +47,12 @@ const fontStates = new Map<GlyphFont, FontWireState>();
const imageIds = new WeakMap<GrayImage, number | null>();
function fontWireState(font: GlyphFont): FontWireState | null {
if (!global.isAndroid || !font.atlasKey) return null;
if (!textureAtlasAvailable() || !font.atlasKey) return null;
if (font.lineHeight <= 0 || font.lineHeight > 255) return null;
let state = fontStates.get(font);
if (!state) {
state = {
fontId: com.faceclaw.app.GlyphAtlas.fontId(font.atlasKey),
fontId: textureFontId(font.atlasKey),
cellHeight: font.lineHeight,
registered: new Set(),
};
@@ -63,7 +63,7 @@ function fontWireState(font: GlyphFont): FontWireState | null {
/**
* Whether this glyph can be expressed as an on-glasses cached draw: within
* the mode-14 char table (32..127), a nonempty raster that fits the font's
* the mode-20 char table (32..127), a nonempty raster that fits the font's
* line cell (the cached image is bbxWidth x lineHeight with the ink placed at
* inkTop), and metrics that fit the wire fields.
*/
@@ -76,12 +76,12 @@ function representableGlyph(font: GlyphFont, glyph: Glyph): boolean {
return inkTop >= 0 && inkTop + glyph.bbxHeight <= font.lineHeight;
}
/** Firmware-font codepoints already registered with the Java FwGlyphAtlas. */
/** Firmware-font codepoints already registered with the Kotlin FwGlyphAtlas. */
const fwRegistered = new Set<string>();
/**
* Whether a firmware-text run is expressible on the wire, registering any
* unseen ink members' rasters with the Java FwGlyphAtlas as a side effect.
* unseen ink members' rasters with the Kotlin FwGlyphAtlas as a side effect.
*/
function prepareFwTextRun(placed: PlacedFwText): boolean {
if (!inRange16(placed.x) || !inRange16(placed.y)) return false;
@@ -116,21 +116,21 @@ function prepareFwTextRun(placed: PlacedFwText): boolean {
bytes.set(entry.data.nibbles, offset + 10);
offset += 10 + entry.data.nibbles.length;
}
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer));
registerFirmwareGlyphs(buffer);
}
return true;
}
/** Resolve (registering on first sight) the Java image id for a drawImage source. */
/** Resolve (registering on first sight) the Kotlin image id for a drawImage source. */
function imageId(placed: PlacedImage): number | null {
if (!global.isAndroid) return null;
if (!textureAtlasAvailable()) return null;
const source = placed.source;
const memo = imageIds.get(source);
if (memo !== undefined) return memo;
let id: number | null = null;
if (source.width > 0 && source.width <= 255 && source.height > 0 && source.height <= 255) {
const key = `img:${source.width}x${source.height}:${source.contentHash32().toString(16)}`;
id = com.faceclaw.app.ImageAtlas.ensure(key, source.width, source.height, new com.faceclaw.app.AndroidByteReader(source.pixels.buffer));
id = textureImageId(key, source.width, source.height, source.pixels);
if (!(typeof id === "number") || id <= 0) id = null;
}
imageIds.set(source, id);
@@ -143,16 +143,16 @@ function inRange16(v: number): boolean {
/**
* Register any not-yet-registered glyph rasters and image contents among
* `draws` with the Java atlases, then build the per-frame draw buffer to pass
* `draws` with the Kotlin atlases, then build the per-frame draw buffer to pass
* alongside the frame's pixels: little-endian tagged records
* [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] (glyph)
* [1][imageId u32][x s16][y s16] (image)
* [2][x s16][y s16][value u8][count u8] (fw text run)
* then count x [cp u32][dx s16][ink u8]
* in draw order. Returns null when nothing is expressible (or off-Android).
* in draw order. Returns null when nothing is expressible (or without a native atlas).
*/
export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | null {
if (!global.isAndroid || draws.length === 0) return null;
if (!textureAtlasAvailable() || draws.length === 0) return null;
// Pass 1: resolve ids, register unseen rasters, size the buffer.
let registration: Map<FontWireState, RegistrationGroup> | null = null;
@@ -184,9 +184,9 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer |
}
if (registration) {
const plainBuffer = buildRegistrationBuffer(registration);
if (plainBuffer) com.faceclaw.app.GlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(plainBuffer));
if (plainBuffer) registerTextureGlyphs(plainBuffer, false);
const aaBuffer = buildAaRegistrationBuffer(registration);
if (aaBuffer) com.faceclaw.app.GlyphAtlas.registerAa(new com.faceclaw.app.AndroidByteReader(aaBuffer));
if (aaBuffer) registerTextureGlyphs(aaBuffer, true);
}
if (bytes === 0) return null;
@@ -247,7 +247,7 @@ type RegistrationGroup = {
* then per glyph
* [encoding u32][bbxX s8][inkTop u8][width u8][inkHeight u8]
* [inkHeight x row u32] (bit (ceil(width/8)*8 - 1 - col) = ink)
* mirroring GlyphAtlas.register on the Java side. Null when the groups hold
* mirroring GlyphAtlas.register on the Kotlin side. Null when the groups hold
* no 1bpp glyphs.
*/
function buildRegistrationBuffer(
+8 -2
View File
@@ -1,4 +1,5 @@
import { fromData, toData } from '../native/kotlin-data'
import type { TextureFrame } from '../g2/texture-planner'
import type { SurfaceConfiguration, SurfaceRect } from './surface-compositor'
export type { SurfaceConfiguration, SurfaceRect } from './surface-compositor'
declare const FaceclawKitIosSurfaceCompositor: any
@@ -23,13 +24,18 @@ export class SurfaceCompositor {
this.native.dimBelowFactor(belowZOrder, factor)
}
setScreenBlanked(blanked: boolean): void { this.native.blankBlanked(blanked) }
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, draws: ArrayBuffer | null = null): void {
// Keep lifecycle errors catchable in JS; an undeclared Kotlin exception
// crossing the Objective-C boundary terminates the process.
if (!this.surfaces.has(id)) throw new Error(`Unknown surface: ${id}`)
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) throw new Error('Invalid frame buffer or rectangle')
this.native.submitIdDataXYWidthHeight(id, toData(pixels), rect.x, rect.y, rect.width, rect.height)
this.native.submitDrawsIdDataXYWidthHeightDraws(id, toData(pixels), rect.x, rect.y, rect.width, rect.height,
draws ? toData(new Uint8Array(draws)) : null)
}
composite(): Uint8Array { return fromData(this.native.composite()) }
compositeFrame(): { pixels: Uint8Array; textures: TextureFrame } {
const native = this.native.compositeFrame()
return { pixels: fromData(native.pixels), textures: { native } }
}
}
+3 -1
View File
@@ -1,4 +1,5 @@
/** Platform-independent 8bpp surface composition for the local display. */
import type { TextureFrame } from '../g2/texture-planner'
export type SurfaceRect = { x: number; y: number; width: number; height: number }
export type SurfaceConfiguration = SurfaceRect & {
zOrder: number
@@ -35,7 +36,7 @@ export class SurfaceCompositor {
this.dimFactor = Math.max(0, Math.min(1, factor))
}
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, _draws: ArrayBuffer | null = null): void {
const surface = this.surfaces.get(id)
if (!surface) throw new Error(`Unknown surface: ${id}`)
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
@@ -50,6 +51,7 @@ export class SurfaceCompositor {
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
}
}
compositeFrame(): { pixels: Uint8Array; textures?: TextureFrame } { return { pixels: this.composite() } }
composite(): Uint8Array {
const output = new Uint8Array(this.width * this.height)
if (this.blanked) return output
+7 -3
View File
@@ -1,8 +1,12 @@
/** Workers send baked grayscale frames to the main-thread iOS compositor. */
/** Workers send baked pixels and shared-Kotlin atlas references together. */
const display = {
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number): void {
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number,
_fingerprint?: string, _paintMs?: number, _frameId?: number, draws: ArrayBuffer | null = null): void {
const data = NSData.dataWithBytesLength(interop.handleof(buffer), buffer.byteLength)
global.postMessage({ type: 'surface-frame', surfaceId, width, height, pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) })
const drawData = draws ? NSData.dataWithBytesLength(interop.handleof(draws), draws.byteLength) : null
global.postMessage({ type: 'surface-frame', surfaceId, width, height,
pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions),
draws: drawData?.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) })
},
}
export function getActiveDisplay(): typeof display { return display }
+14
View File
@@ -4,6 +4,8 @@ import { CfwTransport, parseCfwAcks } from '../g2/cfw-transport.ios'
import { SurfaceCompositor } from '../graphics/surface-compositor.ios'
import { buildBoundingBoxPayload } from '../g2/ble-image-optimizer.ios'
import { lvglMetrics } from './lvgl-font.ios'
import { textureImageId } from './texture-atlas.ios'
import { IosTexturePlanner } from './texture-planner.ios'
/** Development-only checks of real NativeScript selectors and bulk data crossings. */
export function runKotlinProtocolSmokeTest(): void {
@@ -30,6 +32,18 @@ export function runKotlinProtocolSmokeTest(): void {
assert(hex(c.composite()) === '0010ff203040', 'composition')
assert(hex(protocol.packGray4(c.composite(), 3, 2)) === '01f02340', 'packing')
c.setScreenBlanked(true); assert(c.composite().every(p => p === 0), 'blanking')
const textures = new IosTexturePlanner(), textured = new SurfaceCompositor(4, 2)
textured.configureSurface('icon', { x: 0, y: 0, width: 2, height: 2, zOrder: 0, transparency: 'opaque' })
const white = new Uint8Array(4).fill(255)
const icon = textureImageId('protocol-smoke-white', 2, 2, white)
const draw = new DataView(new ArrayBuffer(9)); draw.setUint8(0, 1); draw.setUint32(1, icon, true)
textured.submitSurfaceFrame('icon', white, { x: 0, y: 0, width: 2, height: 2 }, draw.buffer)
const snapshot = textured.compositeFrame(), packed = protocol.packGray4(snapshot.pixels, 4, 2)
const cold = textures.plan(null, packed, snapshot.textures, 1)
assert(!!cold && cold.uploads[0]?.[0] === 18 && cold.payload[0] === 8, 'texture upload and draw')
assert(textures.plan(null, packed, snapshot.textures, 1)?.uploads.length === 0, 'texture reuse')
textures.reset()
assert((textures.plan(null, packed, snapshot.textures, 1)?.uploads.length ?? 0) > 0, 'texture reset')
const changed = new Uint8Array(16); changed[0] = 255
assert(buildBoundingBoxPayload(new Uint8Array(16), changed, 8, 4, 1)?.[0] === 3, 'image update')
assert(lvglMetrics('/nonexistent/faceclaw-font').length === 0, 'font bridge')
+17
View File
@@ -0,0 +1,17 @@
import { toData } from './kotlin-data'
declare const FaceclawKitIosTextureAtlas: any
const atlas = FaceclawKitIosTextureAtlas.new()
export function textureAtlasAvailable(): boolean { return true }
export function textureFontId(key: string): number { return atlas.fontIdKey(key) }
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
const data = toData(new Uint8Array(buffer))
if (aa) atlas.registerAaGlyphsData(data)
else atlas.registerGlyphsData(data)
}
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
atlas.registerFirmwareGlyphsData(toData(new Uint8Array(buffer)))
}
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
return atlas.imageIdKeyWidthHeightData(key, width, height, toData(pixels))
}
+17
View File
@@ -0,0 +1,17 @@
declare const com: any
/** Android bridge to the process-wide shared Kotlin atlases. */
export function textureAtlasAvailable(): boolean { return !!global.isAndroid }
export function textureFontId(key: string): number { return com.faceclaw.app.GlyphAtlas.fontId(key) }
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
const reader = new com.faceclaw.app.AndroidByteReader(buffer)
if (aa) com.faceclaw.app.GlyphAtlas.registerAa(reader)
else com.faceclaw.app.GlyphAtlas.register(reader)
}
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer))
}
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
const buffer = pixels.byteOffset === 0 && pixels.byteLength === pixels.buffer.byteLength ? pixels.buffer : pixels.slice().buffer
return com.faceclaw.app.ImageAtlas.ensure(key, width, height, new com.faceclaw.app.AndroidByteReader(buffer))
}
+13
View File
@@ -0,0 +1,13 @@
import { fromData, toData, nativeArray } from './kotlin-data'
import type { SessionTexturePlanner, TextureFrame, TexturePlan } from '../g2/texture-planner'
declare const FaceclawKitIosTexturePlanner: any
export class IosTexturePlanner implements SessionTexturePlanner {
private readonly native = FaceclawKitIosTexturePlanner.new()
reset(): void { this.native.reset() }
plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null {
const result = this.native.planPreviousNextFrameFirstId(previous ? toData(previous) : null, toData(next), frame.native, firstId)
return result ? { payload: fromData(result.payload), uploads: nativeArray(result.uploads, fromData),
nextFid: result.nextFid, usedBytes: result.usedBytes } : null
}
}
+5 -3
View File
@@ -37,7 +37,7 @@ export type WorkerAppReply =
| { type: "buzzer-sequence"; payload: number[] }
| { type: "navigation-sensors"; request: NavigationSensorRequest }
| { type: "open-url"; url: string }
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string }
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string; draws?: string }
| {
/**
* The worker's bundle has evaluated and its onmessage handler is
@@ -190,7 +190,7 @@ export type WorkerAppHostOptions = {
setSurfaceVisible: (surfaceId: string, visible: boolean) => void;
removeSurface: (surfaceId: string) => void;
requestShellRender: () => void;
submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number) => void;
submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null) => void;
startTextInput?: () => void;
/** Open or focus the Settings app, optionally selecting a section. */
openSettings: (section?: string) => void;
@@ -259,7 +259,9 @@ export class WorkerAppHost {
const { width, height } = message;
if (!Number.isInteger(width) || !Number.isInteger(height) || width < 1 || height < 1 || width > 640 || height > 480) break;
const data = NSData.alloc().initWithBase64EncodedStringOptions(message.pixels, 0 as NSDataBase64DecodingOptions);
if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height);
const draws = message.draws ? NSData.alloc().initWithBase64EncodedStringOptions(message.draws, 0 as NSDataBase64DecodingOptions) : null;
if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height,
draws ? interop.bufferFromData(draws) : null);
break;
}
case "worker-ready":
+17 -4
View File
@@ -45,9 +45,21 @@ On `ios-port`, platform adapters route the existing TypeScript session and UI
logic through shared Kotlin for protocol builders, CRC/framing/reassembly,
event decoding, CFW compression/ACKs, changed-image regions, surface composition,
and LVGL glyphs. The original pure TypeScript implementations remain as Node
test references; NativeScript selects the `.ios.ts` native adapters. Features
not yet present in the iOS app can use the shared audio/recording and texture
classes without another Java port.
test references; NativeScript selects the `.ios.ts` native adapters.
Texture rendering uses the same `GlyphAtlas`, `ImageAtlas`, `TextureCacheState`
and `TexturePlanner` on both platforms. iOS's `IosTextureAtlas` and
`IosTexturePlanner` expose bulk `NSData` bridges. Surface updates retain draw
identities through composition, including worker replies; immutable composite
snapshots keep those identities paired with the pixels while frames coalesce.
The BLE session plans only frames it will enqueue, sends mode-18 uploads before
the mode-19/20 draw batch, and counts a frame only after all uploads and draws
are acknowledged by both lenses. Session failure/teardown resets residency;
resuming near the firmware's 90-second lease deadline restarts the session
before queued draws can reference a freed cache. In either case,
the next frame reuploads lazily. Plans over the CFW message limit fall back to
pixel bands and invalidate any unsent allocations. The 256 KiB bump allocator
and reset-on-full policy are unchanged; there is no pinning or individual eviction.
## Build and validation
@@ -71,7 +83,8 @@ npm run test:kotlin:ios
and iOS: wire vectors, framing/corruption/expiry, streaming compression, replay,
composition, PNG pixels, font glyphs, and Java SHA-256 golden outputs for DSP
and GIF. Android additionally runs the existing Java protocol fixtures and
concurrency checks; iOS exercises `NSData` and compositor adapters. Debug app
concurrency checks; common texture tests cover reuse, overflow, reset and 32-bit
offsets, and iOS exercises `NSData`, atlas, compositor and planner adapters. Debug app
startup checks real NativeScript metadata and bidirectional callbacks, logging
`FACECLAW_KOTLIN_BRIDGE_PASS` and `FACECLAW_KOTLIN_PROTOCOL_PASS`.
@@ -1,6 +1,7 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmOverloads
import kotlin.jvm.JvmStatic
/**
@@ -54,6 +55,7 @@ class TexturePlanner {
* should run instead (no replayable draws, or identical frames).
*/
@JvmStatic
@JvmOverloads
fun plan(
previous: ByteArray?,
next: ByteArray?,
@@ -64,6 +66,7 @@ class TexturePlanner {
fidStart: Int,
allowMultiRect: Boolean,
maxRects: Int,
platform: ProtocolPlatform = protocolPlatform(),
): Result? {
if (
(((((next == null) || (draws == null)) || (draws.size == 0)) || (width <= 0)) ||
@@ -75,7 +78,7 @@ class TexturePlanner {
if ((next.size != (stride * height))) {
return null
}
var planStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var planStartedAtMs: Long = platform.elapsedRealtimeMs()
var fullFrame: Boolean = ((previous == null) || (previous.size != next.size))
var rects: MutableList<IntArray>
if (!fullFrame) {
@@ -103,7 +106,7 @@ class TexturePlanner {
} else {
rects = mutableListOf(intArrayOf(0, 0, width, height))
}
var matchStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var matchStartedAtMs: Long = platform.elapsedRealtimeMs()
var selected: MutableList<Selected> = ArrayList()
var bakedCandidates: Int = 0
var selectedImages: Int = 0
@@ -177,7 +180,7 @@ class TexturePlanner {
if ((selected.isEmpty() && fwSubs.isEmpty())) {
return null
}
var cacheStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var cacheStartedAtMs: Long = platform.elapsedRealtimeMs()
var drawable: MutableList<Selected> =
(if (selected.isEmpty()) mutableListOf() else ensureResident(cache, selected))
bakedCandidates += (selected.size - drawable.size)
@@ -204,7 +207,7 @@ class TexturePlanner {
}
return null
}
var punchStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var punchStartedAtMs: Long = platform.elapsedRealtimeMs()
var punched: ByteArray = next.copyOf()
for (sel in drawn) {
punch(punched, stride, width, height, sel)
@@ -212,7 +215,7 @@ class TexturePlanner {
for (fw in fwPunches) {
punchFw(punched, stride, width, height, fw)
}
var encodeStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var encodeStartedAtMs: Long = platform.elapsedRealtimeMs()
var subs: MutableList<ByteArray> = ArrayList()
var fid: Int = fidStart
if (fullFrame) {
@@ -256,7 +259,7 @@ class TexturePlanner {
uploadBytes,
fullFrame,
)
var doneAtMs: Long = protocolPlatform().elapsedRealtimeMs()
var doneAtMs: Long = platform.elapsedRealtimeMs()
result.rectsMs = ((matchStartedAtMs - planStartedAtMs)).toInt()
result.matchMs = ((cacheStartedAtMs - matchStartedAtMs)).toInt()
result.cacheMs = ((punchStartedAtMs - cacheStartedAtMs)).toInt()
@@ -466,6 +466,18 @@ class SurfaceCompositor {
rectWidth: Int,
rectHeight: Int,
fingerprint: String,
) = submitSurface(surfaceId, pixels, rectX, rectY, rectWidth, rectHeight, fingerprint, null)
/** Retain the full surface's draw identities along with its already-baked pixels. */
fun submitSurface(
surfaceId: String,
pixels: ByteReader,
rectX: Int,
rectY: Int,
rectWidth: Int,
rectHeight: Int,
fingerprint: String,
draws: ByteReader?,
) {
updateSurface(
surfaceId,
@@ -475,7 +487,7 @@ class SurfaceCompositor {
rectWidth,
rectHeight,
fingerprint,
null,
draws,
false,
)
}
@@ -230,7 +230,10 @@ class IosSurfaceCompositor(width: Int, height: Int) {
fun blank(blanked: Boolean) = compositor.setBlanked(blanked)
fun submit(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int) {
fun submit(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int) =
submitDraws(id, data, x, y, width, height, null)
fun submitDraws(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int, draws: NSData?) {
val (sw, sh) = sizes[id] ?: error("Unknown surface: $id")
require(width > 0 && height > 0 && data.length == width.toULong() * height.toULong())
val left = maxOf(0, x)
@@ -260,8 +263,14 @@ class IosSurfaceCompositor(width: Int, height: Int) {
right - left,
bottom - top,
(++sequence).toString(),
// Draw records describe a full surface. Partial/clipped updates must discard
// old identities; their pixels remain available through the baked fallback.
if (x == 0 && y == 0 && width == sw && height == sh) draws?.let(::IosByteReader)
else null,
)
}
fun composite(): NSData = compositor.composite().gray.data()
fun compositeFrame(): IosTextureFrame = IosTextureFrame(compositor.composite())
}
@@ -0,0 +1,50 @@
package com.faceclaw.app
import platform.Foundation.NSData
/** Binary adapters only: atlas identity, RLE, residency and planning stay in commonMain. */
class IosTextureAtlas {
fun fontId(key: String): Int = GlyphAtlas.fontId(key)
fun registerGlyphs(data: NSData) = GlyphAtlas.register(IosByteReader(data))
fun registerAaGlyphs(data: NSData) = GlyphAtlas.registerAa(IosByteReader(data))
fun registerFirmwareGlyphs(data: NSData) = FwGlyphAtlas.register(IosByteReader(data))
fun imageId(key: String, width: Int, height: Int, data: NSData): Int =
ImageAtlas.ensure(key, width, height, IosByteReader(data))
}
/** A snapshot keeps pixels and their screen-space draw identities together while JS coalesces. */
class IosTextureFrame internal constructor(internal val composite: SurfaceCompositor.Composite) {
val pixels: NSData = composite.gray.data()
}
class IosTexturePlan internal constructor(result: TexturePlanner.Result, val usedBytes: Int) {
val payload: NSData = result.payload.data()
val uploads: List<NSData> = result.uploads.map { it.data() }
val nextFid: Int = result.nextFid
}
/** One instance per BLE session, called on the session's serial JS thread. */
class IosTexturePlanner {
private val cache = TextureCacheState()
fun reset() = cache.reset()
fun plan(previous: NSData?, next: NSData, frame: IosTextureFrame, firstId: Int): IosTexturePlan? {
val composite = frame.composite
val result = TexturePlanner.plan(
previous?.byteArray(), next.byteArray(), composite.width, composite.height,
composite.draws, cache, firstId, true, ConnectionOptions.MULTI_RECT_MAX_RECTS,
) ?: return null
// A plan is atomic. If it cannot fit the CFW transport, discard its uploads
// and residency too, before the session falls back to ordinary pixel bands.
if (result.payload.size > 65535 || result.uploads.any { it.size > 65535 }) {
cache.reset()
return null
}
return IosTexturePlan(result, cache.usedBytes())
}
}
+2 -1
View File
@@ -149,13 +149,14 @@ test('heading updates reuse the texture; brightness, layout and clipping changes
test('the background registers once and uses a nine-byte image reference on later frames', () => {
const registered = [];
const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', {}, '', {
const atlas = load('app/native/texture-atlas.ts', {}, '', {
global: { isAndroid: true },
com: { faceclaw: { app: { AndroidByteReader: class { constructor(buffer) { this.buffer = buffer; } }, ImageAtlas: { ensure: (...args) => {
registered.push(args);
return 1;
} } } } },
});
const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', { '../native/texture-atlas': atlas });
const background = compass.createCompassBackground(576, 260, 256, 193.25, 98, 140, 1);
for (const heading of [0, 1, 2, 45, 90]) {
const image = background.clone();
+40 -4
View File
@@ -11,6 +11,39 @@ function load(file, context) {
return sandbox.exports;
}
test('iOS worker replies keep cached draw references paired with their baked pixels', () => {
const replies = [], received = [];
const wrap = buffer => ({ length: buffer.byteLength, buffer,
base64EncodedStringWithOptions: () => Buffer.from(buffer).toString('base64') });
const NSData = {
dataWithBytesLength: buffer => wrap(buffer),
alloc: () => ({ initWithBase64EncodedStringOptions: text => {
const bytes = Uint8Array.from(Buffer.from(text, 'base64')); return wrap(bytes.buffer);
} }),
};
const interop = { handleof: buffer => buffer, bufferFromData: data => data.buffer };
const display = load('app/native/active-display.ios.ts', {
NSData, interop, global: { postMessage: message => replies.push(message) },
}).getActiveDisplay();
const pixels = new Uint8Array([255, 0, 0, 255]), draws = new Uint8Array([1, 7, 0, 0, 0, 0, 0, 0, 0]);
display.submitSurfaceFrame(pixels.buffer, 'window:test', 0, 0, 2, 2, 'frame', 0, 1, draws.buffer);
const worker = {};
const { WorkerAppHost } = load('app/ui/shell/worker-window.ts', {
require: () => ({}), global: { isIOS: true }, NSData, interop,
});
const host = new WorkerAppHost({ appId: 'test', worker, submitPixels: (...args) => received.push(args) });
host.openWindows.add('test');
worker.onmessage({ data: replies[0] });
assert.equal(received.length, 1);
assert.deepEqual([...received[0][1]], [...pixels]);
assert.deepEqual([...new Uint8Array(received[0][4])], [...draws]);
display.submitSurfaceFrame(pixels.buffer, 'window:test', 0, 0, 2, 2);
worker.onmessage({ data: replies[1] });
assert.equal(received[1][4], null, 'a pixel-only update must clear previous draw identities');
host.openWindows.clear(); worker.onmessage({ data: replies[0] });
assert.equal(received.length, 2, 'closed windows cannot deliver stale cached draws');
});
test('iOS installed apps launch, recover missing packages, and close before uninstall', async () => {
const calls = [], app = { packageId: 'test.app', name: 'Test' };
let present = true;
@@ -215,9 +248,11 @@ test('background glasses input still composites frames; phone resume preserves t
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../graphics/surface-compositor': { SurfaceCompositor: class {
configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {} setScreenBlanked() {}
composite() { return new Uint8Array([1, 2]); }
compositeFrame() { return { pixels: new Uint8Array([1, 2]) }; }
} },
'../graphics/plane': { flattenPlanes: () => ({ pixels: new Uint8Array([1, 2]), width: 2, height: 1 }) },
'../graphics/plane': { flattenPlanesWithDraws: () => ({ image: { pixels: new Uint8Array([1, 2]), width: 2, height: 1 }, draws: [] }) },
'../graphics/glyph-wire': { prepareFrameDraws: () => null },
'../native/texture-planner.ios': { IosTexturePlanner: class {} },
'../native/ios-graphics': { previewPixels: pixels => pixels },
'../ui/shell/shell': { shell, rawInputEventToInputEvent: input => input },
'../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 640, height: 480 }) },
@@ -333,7 +368,8 @@ test('iOS configures an in-process surface before submitting constructor-trigger
require: name => ({
'../ui/shell/shell': { shell },
'../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 2, height: 1 }) },
'../graphics/plane': { flattenPlanes: planes => planes[0] },
'../graphics/plane': { flattenPlanesWithDraws: planes => ({ image: planes[0], draws: [] }) },
'../graphics/glyph-wire': { prepareFrameDraws: () => null },
})[name] ?? {},
console,
});
@@ -365,7 +401,7 @@ test('iOS configures an in-process surface before submitting constructor-trigger
test('iOS compositor rejects missing and removed surfaces before crossing into Kotlin', () => {
let nativeSubmissions = 0;
const native = { configureIdXYWidthHeightZOrderTransparent() {}, removeId() {}, submitIdDataXYWidthHeight() { nativeSubmissions++; } };
const native = { configureIdXYWidthHeightZOrderTransparent() {}, removeId() {}, submitDrawsIdDataXYWidthHeightDraws() { nativeSubmissions++; } };
const api = load('app/graphics/surface-compositor.ios.ts', {
require: () => ({ toData: data => data }),
FaceclawKitIosSurfaceCompositor: { alloc: () => ({ initWithWidthHeight: () => native }) },
+63 -2
View File
@@ -165,9 +165,9 @@ async function until(predicate) {
for (let i = 0; i < 100; i++) { if (predicate()) return; await new Promise(resolve => setTimeout(resolve, 5)); }
assert.fail('Timed out waiting for test state');
}
function harness(t, transport = new FakeTransport()) {
function harness(t, transport = new FakeTransport(), textures) {
const states = [], input = [], compass = [], wear = [];
const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing));
const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing), undefined, textures);
t.after(async () => { transport.hold = false; await session.stop(); });
return { transport, session, states, input, compass, wear };
}
@@ -306,6 +306,67 @@ const ack = (transport, sent, overrides) => transport.ack(sent.id, sent.message,
const tick = () => new Promise(resolve => setImmediate(resolve));
const noisy = () => Uint8Array.from({ length: 640 * 480 }, (_, i) => (i % 2) * 255);
test('texture uploads precede their frame, share ACK accounting, and coalesce pixels with draw identities', async t => {
const plans = [];
let resets = 0;
const textures = {
reset() { resets++; },
plan(previous, next, frame, firstId) {
plans.push({ previous, next, frame, firstId });
return { uploads: plans.length === 1 ? [new Uint8Array([18, 1]), new Uint8Array([18, 2])] : [],
payload: new Uint8Array([8, plans.length]), nextFid: firstId + 2, usedBytes: 100 };
},
};
const h = harness(t, new FakeTransport(), textures);
await h.session.start(addresses); h.transport.hold = true;
const a = { native: { id: 'a' } }, b = { native: { id: 'b' } }, c = { native: { id: 'c' } };
h.session.setFrame(new Uint8Array(640 * 480), a);
await until(() => imageMessages(h.transport).length === 3);
const sent = imageMessages(h.transport);
assert.deepEqual(sent.map(s => s.message.payload[0]), [18, 18, 8]);
h.session.setFrame(new Uint8Array(640 * 480).fill(80), b);
h.session.setFrame(new Uint8Array(640 * 480).fill(160), c);
assert.equal(plans.length, 1, 'coalesced frames must not allocate textures');
ack(h.transport, sent[2]); ack(h.transport, sent[1]); await tick();
assert.equal(h.session.state.frames, 0, 'a draw ACK cannot retire an unacknowledged upload');
ack(h.transport, sent[0]); await until(() => imageMessages(h.transport).length === 4);
assert.equal(h.session.state.frames, 1);
assert.equal(plans.length, 2); assert.equal(plans[1].frame, c);
assert.deepEqual(plans[1].previous, plans[0].next);
assert.equal(plans[1].firstId, plans[0].firstId + 2);
assert.ok(plans[1].next.every(value => value === 0xaa));
ack(h.transport, imageMessages(h.transport)[3]); await until(() => h.session.state.frames === 2);
h.transport.hold = false; await h.session.stop();
assert.ok(resets > 0);
});
test('a failed texture upload invalidates residency and the pipelined frame base', async t => {
let resets = 0;
const textures = {
reset() { resets++; },
plan() { return { uploads: [new Uint8Array([18])], payload: new Uint8Array([8]), nextFid: 1, usedBytes: 4 }; },
};
const h = harness(t, new FakeTransport(), textures); await h.session.start(addresses);
const original = h.session.requestCfw.bind(h.session);
h.session.requestCfw = (role, payload) => payload[0] === 18 ? Promise.reject(new Error('texture upload failed')) : original(role, payload);
h.session.setFrame(new Uint8Array(640 * 480), { native: {} });
await until(() => h.session.state.phase === 'retrying');
assert.equal(resets, 1); assert.equal(h.session.lastEnqueued, null);
assert.equal(h.session.latestTextures, null); assert.equal(h.session.state.frames, 0);
});
test('resuming near framebuffer lease expiry discards cached draws before transmission', async t => {
let resets = 0, plans = 0;
const textures = { reset() { resets++; }, plan() { plans++; return null; } };
const h = harness(t, new FakeTransport(), textures); await h.session.start(addresses);
h.session.lastLease = Date.now() - 85_000;
h.session.setFrame(new Uint8Array(640 * 480), { native: {} });
await h.session.pump();
assert.equal(h.session.state.phase, 'retrying');
assert.equal(resets, 1); assert.equal(plans, 0);
assert.equal(imageMessages(h.transport).length, 0);
});
// Both eyes must finish; wrong ingress, size, CRC, ordinal and legacy replies do not count.
test('CFW ACK identity and both-lens completion gate display metrics', async t => {
const { iosBleTraffic } = require('../.test-build/app/g2/ble-traffic-counters.js');
+2
View File
@@ -102,6 +102,8 @@ function fixture() {
'../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } },
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
'../apps/ios-availability': { iosAppUnavailableReason: () => null },
'../graphics/glyph-wire': { prepareFrameDraws: () => null },
'../native/texture-planner.ios': { IosTexturePlanner: class {} },
'../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images,
'../native/ios-graphics': { previewPixels: pixels => pixels },
'../ui/gestures': { makeInputEvent: event => event }, '../ui/layers': { noopLayerActions: {} },
@@ -0,0 +1,99 @@
package com.faceclaw.app
import kotlin.test.*
class TextureCacheTest {
private fun u32(bytes: ByteArray, offset: Int): Int =
(0..3).fold(0) { n, i -> n or ((bytes[offset + i].toInt() and 255) shl (i * 8)) }
@Test
fun offsetsAbove64KiBAndLazyFontSlotsRoundTripOnBothPlatforms() {
val cache = TextureCacheState()
val large = ImageAtlas.Entry(255, 255, ByteArray(255 * 255) { (it % 15 + 1).toByte() })
repeat(3) { assertTrue(cache.ensureImage(it, large) >= 0) }
val glyph = GlyphAtlas.Glyph(1, 1, 0, 1, 0, null, byteArrayOf(15))
val offset = cache.ensureGlyph(42, 65, glyph)
val table = cache.fontTableOffset(42)
assertTrue(table > 65535)
assertEquals(table + 384, offset)
val memory = ByteArray(TextureCacheState.CACHE_SIZE)
for (upload in cache.drainUploadPayloads(3600)) {
assertEquals(18, upload[0].toInt())
var pos = 1
while (pos < upload.size) {
val address = u32(upload, pos)
val length = (upload[pos + 4].toInt() and 255) or ((upload[pos + 5].toInt() and 255) shl 8)
upload.copyInto(memory, address, pos + 6, pos + 6 + length)
pos += 6 + length
}
assertEquals(upload.size, pos)
}
assertEquals(offset, u32(memory, table + (65 - 32) * 4))
assertContentEquals(glyph.cachedBytes, memory.copyOfRange(offset, offset + glyph.cachedBytes.size))
assertEquals(offset, cache.ensureGlyph(42, 65, glyph))
assertFalse(cache.hasPendingUploads())
cache.reset()
assertEquals(0, cache.usedBytes())
assertEquals(384, cache.ensureGlyph(42, 65, glyph))
assertTrue(cache.hasPendingUploads())
}
@Test
fun plannerReusesResidencyAndReuploadsAfterReset() {
val cache = TextureCacheState()
val id = ImageAtlas.ensure("texture-test-white", 2, 2, ArrayByteReader(ByteArray(4) { -1 }))
val pixels = ByteArray(8 * 4)
for (y in 0..1) for (x in 0..1) pixels[y * 8 + x] = -1
val packed = BmpUtil.pack4bppFromGray8(pixels, 8, 4)
val draws = arrayOf(SurfaceCompositor.ScreenDraw.image(id, 0, 0))
fun plan() = assertNotNull(TexturePlanner.plan(null, packed, 8, 4, draws, cache, 1, true, 8, testPlatform()))
val cold = plan()
assertEquals(1, cold.drawnImages)
assertTrue(cold.uploads.isNotEmpty())
val warm = plan()
assertTrue(warm.uploads.isEmpty())
assertContentEquals(cold.payload, warm.payload)
cache.reset()
assertTrue(plan().uploads.isNotEmpty())
// Occlusion fails the correctness check and leaves the image baked.
assertNull(TexturePlanner.plan(null, ByteArray(packed.size), 8, 4, draws, cache, 1, true, 8, testPlatform()))
}
@Test
fun fullCacheResetsAndRetriesTheCurrentFrame() {
val cache = TextureCacheState()
val large = ImageAtlas.Entry(255, 255, ByteArray(255 * 255) { (it % 15 + 1).toByte() })
var fillerId = -1
while (cache.ensureImage(fillerId--, large) >= 0) {}
// Fill the remaining tail, forcing the next real allocation to fail.
val remaining = TextureCacheState.CACHE_SIZE - cache.usedBytes()
while (cache.ensureImage(fillerId--, ImageAtlas.Entry(1, 1, byteArrayOf(15))) >= 0) {}
assertTrue(remaining > 0)
val generation = cache.generation()
val id = ImageAtlas.ensure("texture-test-full", 2, 2, ArrayByteReader(ByteArray(4) { -1 }))
val result = assertNotNull(TexturePlanner.plan(null, ByteArray(4) { -1 }, 4, 2,
arrayOf(SurfaceCompositor.ScreenDraw.image(id, 0, 0)), cache, 1, true, 8, testPlatform()))
assertEquals(generation + 1, cache.generation())
assertEquals(1, result.drawnImages)
assertTrue(cache.usedBytes() < 100)
assertTrue(result.uploads.isNotEmpty())
}
@Test
fun workingSetLargerThanTheCacheFallsBackWithoutAResetLoop() {
val cache = TextureCacheState()
val raster = ByteArray(255 * 255) { if (it % 2 == 0) 16 else 32 }
val draws = (0..4).map {
val id = ImageAtlas.ensure("oversized-working-set-$it", 255, 255, ArrayByteReader(raster))
SurfaceCompositor.ScreenDraw.image(id, 0, 0)
}.toTypedArray()
val gray = ByteArray(256 * 256)
repeat(255) { raster.copyInto(gray, it * 256, it * 255, (it + 1) * 255) }
val result = assertNotNull(TexturePlanner.plan(null,
BmpUtil.pack4bppFromGray8(gray, 256, 256), 256, 256, draws, cache, 1, true, 8, testPlatform()))
assertEquals(1, cache.generation())
assertEquals(4, result.drawnImages)
assertEquals(1, result.bakedCandidates)
assertTrue(cache.usedBytes() <= TextureCacheState.CACHE_SIZE)
}
}
@@ -0,0 +1,86 @@
package com.faceclaw.app
import kotlin.test.*
class IosTextureApiTest {
private fun imageDraw(id: Int): ByteArray = byteArrayOf(
1, id.toByte(), (id shr 8).toByte(), (id shr 16).toByte(), (id shr 24).toByte(), 0, 0, 0, 0,
)
@Test
fun nativeSnapshotsPreserveDrawsAcrossCoalescingAndSessionResets() {
val atlas = IosTextureAtlas()
val white = ByteArray(4) { -1 }.data()
val id = atlas.imageId("ios-texture-api-white", 2, 2, white)
assertEquals(id, IosTextureAtlas().imageId("ios-texture-api-white", 2, 2, white))
val compositor = IosSurfaceCompositor(8, 4)
compositor.configure("app", 2, 0, 2, 2, 0, false)
compositor.submitDraws("app", white, 0, 0, 2, 2, imageDraw(id).data())
val frame = compositor.compositeFrame()
assertEquals(2, frame.composite.draws!!.single().x)
// A later update must not change the previously submitted snapshot.
compositor.submit("app", ByteArray(4).data(), 0, 0, 2, 2)
assertTrue(compositor.compositeFrame().composite.draws!!.isEmpty())
val next = IosProtocol().pack(frame.pixels, 8, 4)
val planner = IosTexturePlanner()
val cold = assertNotNull(planner.plan(null, next, frame, 1))
assertEquals(18, cold.uploads.first().byteArray()[0].toInt())
assertEquals(8, cold.payload.byteArray()[0].toInt())
val warm = assertNotNull(planner.plan(null, next, frame, 1))
assertTrue(warm.uploads.isEmpty())
assertContentEquals(cold.payload.byteArray(), warm.payload.byteArray())
planner.reset()
assertTrue(assertNotNull(planner.plan(null, next, frame, 1)).uploads.isNotEmpty())
// A partial update clears identities even if the caller supplies stale draws.
compositor.submitDraws("app", byteArrayOf(-1).data(), 0, 0, 1, 1, imageDraw(id).data())
assertTrue(compositor.compositeFrame().composite.draws!!.isEmpty())
}
@Test
fun glyphRegistrationAndDrawingUseTheSharedAsciiTable() {
val atlas = IosTextureAtlas()
val key = "ios-aa-texture-test"
val bytes = key.encodeToByteArray()
atlas.registerAaGlyphs((byteArrayOf(bytes.size.toByte()) + bytes +
byteArrayOf(2, 1, 0, 65, 0, 0, 0, 0, 0, 2, 2, -1, -1)).data())
val font = atlas.fontId(key)
val draws = byteArrayOf(0, font.toByte(), (font shr 8).toByte(), 65, 0, 0, 0, 0, 0, 0, 0, -1)
val compositor = IosSurfaceCompositor(4, 2)
compositor.configure("text", 0, 0, 2, 2, 0, false)
compositor.submitDraws("text", ByteArray(4) { -1 }.data(), 0, 0, 2, 2, draws.data())
val frame = compositor.compositeFrame()
val plan = assertNotNull(IosTexturePlanner().plan(null, IosProtocol().pack(frame.pixels, 4, 2), frame, 1))
assertTrue(plan.usedBytes > 384)
val payload = plan.payload.byteArray()
// The last mode-8 submessage is the shared mode-20 cached glyph run.
assertEquals(20, payload[payload.size - 12].toInt())
assertEquals(65, payload.last().toInt())
}
@Test
fun oversizedPlanDiscardsUnsentResidencyBeforePixelFallback() {
val atlas = IosTextureAtlas()
val white = ByteArray(4) { -1 }.data()
val id = atlas.imageId("ios-oversized-plan", 2, 2, white)
var random = 12345
val gray = ByteArray(640 * 480) {
random = random xor (random shl 13)
random = random xor (random ushr 17)
random = random xor (random shl 5)
random.toByte()
}
gray[0] = -1; gray[1] = -1; gray[640] = -1; gray[641] = -1
val compositor = IosSurfaceCompositor(640, 480)
compositor.configure("noise", 0, 0, 640, 480, 0, false)
compositor.submitDraws("noise", gray.data(), 0, 0, 640, 480, imageDraw(id).data())
val frame = compositor.compositeFrame()
val planner = IosTexturePlanner()
assertNull(planner.plan(null, IosProtocol().pack(frame.pixels, 640, 480), frame, 1))
val small = IosSurfaceCompositor(4, 2)
small.configure("icon", 0, 0, 2, 2, 0, false)
small.submitDraws("icon", white, 0, 0, 2, 2, imageDraw(id).data())
val next = small.compositeFrame()
val retry = assertNotNull(planner.plan(null, IosProtocol().pack(next.pixels, 4, 2), next, 1))
assertTrue(retry.uploads.isNotEmpty(), "the rejected plan's image was never uploaded")
}
}