diff --git a/app/g2/glasses-session.ts b/app/g2/glasses-session.ts index 2152f327..c3dd390f 100644 --- a/app/g2/glasses-session.ts +++ b/app/g2/glasses-session.ts @@ -9,6 +9,7 @@ import { hasCompatibleFirmware, firmwareIncompatibilityMessage, REQUIRED_FACECLA import { hexToBytes } from '../util/hex-util' import { deviceAddressError } from './ios-peripheral-identity' import { decodeWearState, enableWearDetection, queryWearState } from './wear-protocol' +import type { SessionTexturePlanner, TextureFrame } from './texture-planner' declare function setTimeout(callback: () => void, ms: number): number declare function clearTimeout(id: number): void @@ -60,6 +61,7 @@ export class GlassesSession { private wanted = false private addresses: SessionAddresses | null = null private latest: Uint8Array | null = null + private latestTextures: TextureFrame | null = null private displayed: Uint8Array | null = null private lastEnqueued: Uint8Array | null = null // Match Android: advance only for emitted deltas, skipping 0 / 0xffff. @@ -90,7 +92,8 @@ export class GlassesSession { private readonly onActivity: () => void = () => {}, private readonly onCompass: (event: CompassEvent) => void = () => {}, private readonly onWearState: (wearing: boolean) => void = () => {}, - onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {}) { + onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {}, + private readonly textures?: SessionTexturePlanner) { this.notifications = new AncsClient(packet => this.writePackets('right', () => [packet]), onNotificationsChanged, message => this.log('ANCS: ' + message)) this.off = transport.onEvent(event => this.receive(event)) } @@ -428,9 +431,11 @@ export class GlassesSession { if (this.state.phase !== 'connected') return await this.request('right', protocol.SID.settings, magic => setBrightness(magic, level), 'Brightness') } - setFrame(gray: Uint8Array): void { + setFrame(gray: Uint8Array, textures?: TextureFrame): void { if (this.state.phase !== 'connected') return - this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0) + this.latest = protocol.packGray4(gray, 640, 480) + this.latestTextures = textures ?? null + this.schedule(0) } /** Same mode-5 kind-4 sequencer payload and acknowledged image path as Android. */ async playBuzzerSequence(payload: Uint8Array): Promise { @@ -499,16 +504,27 @@ export class GlassesSession { while (this.canSendDisplay()) { if (!this.displaySending) { const packed = this.latest!; this.latest = null + const textureFrame = this.latestTextures; this.latestTextures = null // Compare with the last enqueued image, not the last ACKed one. In an // A -> B -> A sequence, B may still be in flight when A is requested. if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue - const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId) - const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed)) - if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1 - const commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId) - if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands) - this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1 - this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`) + const texturePlan = textureFrame ? this.textures?.plan(this.lastEnqueued, packed, textureFrame, this.nextImageFrameId) : null + let commands: Uint8Array[] + if (texturePlan) { + // Uploads participate in the same ordered, acknowledged command window as + // the image. Coalescing only happens before planning mutates residency. + commands = [...texturePlan.uploads, texturePlan.payload] + this.nextImageFrameId = texturePlan.nextFid + this.log(`Display textures (${texturePlan.uploads.length} uploads, cache=${texturePlan.usedBytes}B)`) + } else { + const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId) + const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed)) + if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1 + commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId) + if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands) + this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1 + this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`) + } this.displaySending = { packed, commands, offset: 0, pending: 0 } this.displayFrames.push(this.displaySending); this.lastEnqueued = packed } @@ -538,6 +554,10 @@ export class GlassesSession { this.pumping = true; const generation = this.generation try { const now = Date.now() + // Firmware frees textures when its 90-second framebuffer lease expires. + // After a long iOS suspension, reconnect rather than replaying pending + // draws against unknown memory. Leave a margin for the two arm writes. + if (now - this.lastLease >= 80_000) throw new Error('Display lease lapsed; restarting session') if (now - this.lastHeartbeat >= 4000) { const gap = now - this.lastHeartbeat await this.request('right', protocol.SID.hub, protocol.heartbeat, 'Heartbeat', 1500); this.lastHeartbeat = Date.now() @@ -571,6 +591,7 @@ export class GlassesSession { for (const item of custom) item.reject(new Error('Connection ended')) this.cfwTransports.clear() this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null + this.latestTextures = null; this.textures?.reset() this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0 } private closeLinks(): void { diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 02045a01..3593cdb5 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -35,7 +35,9 @@ import { getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetti import { readPhoneBatteryState } from '../native/phone-battery' import { iosAppUnavailableReason } from '../apps/ios-availability' import { SurfaceCompositor } from '../graphics/surface-compositor' -import { flattenPlanes, type Plane } from '../graphics/plane' +import { flattenPlanesWithDraws, type Plane } from '../graphics/plane' +import { prepareFrameDraws } from '../graphics/glyph-wire' +import { IosTexturePlanner } from '../native/texture-planner.ios' import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image' import { previewPixels } from '../native/ios-graphics' import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures' @@ -62,8 +64,8 @@ export class IosPreviewController { setScreenBlanked: async blanked => { this.compositor.setScreenBlanked(blanked); this.scheduleFrame() }, - submitSurfaceFrame: async (id, pixels, rect) => { - this.compositor.submitSurfaceFrame(id, pixels, rect); this.scheduleFrame() + submitSurfaceFrame: async (id, pixels, rect, _fingerprint, _paintMs, _frameId, draws) => { + this.compositor.submitSurfaceFrame(id, pixels, rect, draws); this.scheduleFrame() }, } private readonly glance = new GlanceHost({ @@ -336,8 +338,8 @@ export class IosPreviewController { } } private async submit(id: string, planes: Plane[]): Promise { - const image = flattenPlanes(planes) - this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }) + const { image, draws } = flattenPlanesWithDraws(planes) + this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, prepareFrameDraws(draws)) this.scheduleFrame() } private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() } @@ -350,12 +352,12 @@ export class IosPreviewController { try { if (this.shellDirty) { this.shellDirty = false - const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 }) - this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 }) + const { image, draws } = flattenPlanesWithDraws(shell.paintSurface(), { width: 640, height: 480 }) + this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 }, prepareFrameDraws(draws)) this.compositor.setUnderlayDim(1, shell.underlayDim()) } - const pixels = this.compositor.composite() - this.session?.setFrame(pixels) + const { pixels, textures } = this.compositor.compositeFrame() + this.session?.setFrame(pixels, textures) if (this.active) { const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green') this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard' @@ -511,8 +513,8 @@ export class IosPreviewController { }, setSurfaceVisible: (id, visible) => { this.compositor.setSurfaceVisible(id, visible); this.scheduleFrame() }, removeSurface: id => { this.compositor.removeSurface(id); this.scheduleFrame() }, - submitPixels: (id, pixels, width, height) => { - this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }); this.scheduleFrame() + submitPixels: (id, pixels, width, height, draws) => { + this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }, draws); this.scheduleFrame() }, requestShellRender: () => this.requestShellRender(), openSettings: section => { void this.launchApp('settings', { section }) }, @@ -560,7 +562,7 @@ export class IosPreviewController { this.refreshClock() }, receiveCompassEvent, wearing => this.handleWearState(wearing), (key, popup) => { iosNotificationsChanged(key, popup); this.requestShellRender() - }) + }, new IosTexturePlanner()) bindIosNotifications(this.session.notifications) bindCompassSession(this.session) } diff --git a/app/g2/texture-planner.ts b/app/g2/texture-planner.ts new file mode 100644 index 00000000..b41af272 --- /dev/null +++ b/app/g2/texture-planner.ts @@ -0,0 +1,8 @@ +/** Platform-owned snapshot of the same composite as the submitted pixels. */ +export type TextureFrame = { readonly native: object } +export type TexturePlan = { uploads: Uint8Array[]; payload: Uint8Array; nextFid: number; usedBytes: number } +/** Sessions own residency; frame snapshots may be coalesced without changing it. */ +export interface SessionTexturePlanner { + plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null + reset(): void +} diff --git a/app/graphics/glyph-wire.ts b/app/graphics/glyph-wire.ts index adbf533f..970fe821 100644 --- a/app/graphics/glyph-wire.ts +++ b/app/graphics/glyph-wire.ts @@ -1,20 +1,20 @@ /** - * Marshals deferred-draw identity (text glyphs and icon images) to the Java - * side for the texture-cache pipeline (CFW modes 12/13/14; see + * Marshals deferred-draw identity (text glyphs and icon images) to the Kotlin + * side for the texture-cache pipeline (CFW modes 18/19/20; see * notes/texture-cache-display-list-design.md). * * Channels, all cheap ByteBuffer crossings: - * - glyph raster registration into the process-wide Java GlyphAtlas, once + * - glyph raster registration into the process-wide Kotlin GlyphAtlas, once * per (font, glyph) per JS context (main thread and each worker register * independently; the atlas dedupes by the font's stable atlasKey); - * - image registration into the Java ImageAtlas, once per distinct image + * - image registration into the Kotlin ImageAtlas, once per distinct image * content per JS context (images are content-addressed: the key is a hash * of dimensions + pixels, so the same icon rendered anywhere dedupes and * a changed icon is simply a new image); * - a per-frame draw list accompanying each submitted surface frame, which - * the Java planner may replay as on-glasses cached draws. + * the Kotlin planner may replay as on-glasses cached draws. * - * Draws that can't participate (no atlasKey, outside the mode-14 ASCII + * Draws that can't participate (no atlasKey, outside the mode-20 ASCII * table, ink outside the line cell, oversized images) are simply left out — * their pixels are baked into the submitted frame either way, so exclusion * only means "no wire savings for this draw". @@ -22,7 +22,7 @@ import { Glyph } from "./bdffont"; import { GrayImage, type DeferredDraw, type GlyphFont, type PlacedFwText, type PlacedImage } from "./image"; -declare const com: any; +import { textureAtlasAvailable, textureFontId, registerTextureGlyphs, registerFirmwareGlyphs, textureImageId } from "../native/texture-atlas"; const GLYPH_RECORD_BYTES = 12; // [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] const IMAGE_RECORD_BYTES = 9; // [1][imageId u32][x s16][y s16] @@ -32,14 +32,14 @@ const FWTEXT_MEMBER_BYTES = 7; // [cp u32][dx s16][ink u8] type FontWireState = { fontId: number; cellHeight: number; - /** Encodings already registered with the Java atlas from this JS context. */ + /** Encodings already registered with the Kotlin atlas from this JS context. */ registered: Set; }; const fontStates = new Map(); /** - * Java-side image id per source image object, or null for images that can't + * Kotlin-side image id per source image object, or null for images that can't * participate. Sources handed to drawImage are immutable by contract, so * memoizing by object identity is safe; a re-rendered icon is a new object * and re-resolves (usually to the same content-addressed id). @@ -47,12 +47,12 @@ const fontStates = new Map(); const imageIds = new WeakMap(); function fontWireState(font: GlyphFont): FontWireState | null { - if (!global.isAndroid || !font.atlasKey) return null; + if (!textureAtlasAvailable() || !font.atlasKey) return null; if (font.lineHeight <= 0 || font.lineHeight > 255) return null; let state = fontStates.get(font); if (!state) { state = { - fontId: com.faceclaw.app.GlyphAtlas.fontId(font.atlasKey), + fontId: textureFontId(font.atlasKey), cellHeight: font.lineHeight, registered: new Set(), }; @@ -63,7 +63,7 @@ function fontWireState(font: GlyphFont): FontWireState | null { /** * Whether this glyph can be expressed as an on-glasses cached draw: within - * the mode-14 char table (32..127), a nonempty raster that fits the font's + * the mode-20 char table (32..127), a nonempty raster that fits the font's * line cell (the cached image is bbxWidth x lineHeight with the ink placed at * inkTop), and metrics that fit the wire fields. */ @@ -76,12 +76,12 @@ function representableGlyph(font: GlyphFont, glyph: Glyph): boolean { return inkTop >= 0 && inkTop + glyph.bbxHeight <= font.lineHeight; } -/** Firmware-font codepoints already registered with the Java FwGlyphAtlas. */ +/** Firmware-font codepoints already registered with the Kotlin FwGlyphAtlas. */ const fwRegistered = new Set(); /** * Whether a firmware-text run is expressible on the wire, registering any - * unseen ink members' rasters with the Java FwGlyphAtlas as a side effect. + * unseen ink members' rasters with the Kotlin FwGlyphAtlas as a side effect. */ function prepareFwTextRun(placed: PlacedFwText): boolean { if (!inRange16(placed.x) || !inRange16(placed.y)) return false; @@ -116,21 +116,21 @@ function prepareFwTextRun(placed: PlacedFwText): boolean { bytes.set(entry.data.nibbles, offset + 10); offset += 10 + entry.data.nibbles.length; } - com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer)); + registerFirmwareGlyphs(buffer); } return true; } -/** Resolve (registering on first sight) the Java image id for a drawImage source. */ +/** Resolve (registering on first sight) the Kotlin image id for a drawImage source. */ function imageId(placed: PlacedImage): number | null { - if (!global.isAndroid) return null; + if (!textureAtlasAvailable()) return null; const source = placed.source; const memo = imageIds.get(source); if (memo !== undefined) return memo; let id: number | null = null; if (source.width > 0 && source.width <= 255 && source.height > 0 && source.height <= 255) { const key = `img:${source.width}x${source.height}:${source.contentHash32().toString(16)}`; - id = com.faceclaw.app.ImageAtlas.ensure(key, source.width, source.height, new com.faceclaw.app.AndroidByteReader(source.pixels.buffer)); + id = textureImageId(key, source.width, source.height, source.pixels); if (!(typeof id === "number") || id <= 0) id = null; } imageIds.set(source, id); @@ -143,16 +143,16 @@ function inRange16(v: number): boolean { /** * Register any not-yet-registered glyph rasters and image contents among - * `draws` with the Java atlases, then build the per-frame draw buffer to pass + * `draws` with the Kotlin atlases, then build the per-frame draw buffer to pass * alongside the frame's pixels: little-endian tagged records * [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] (glyph) * [1][imageId u32][x s16][y s16] (image) * [2][x s16][y s16][value u8][count u8] (fw text run) * then count x [cp u32][dx s16][ink u8] - * in draw order. Returns null when nothing is expressible (or off-Android). + * in draw order. Returns null when nothing is expressible (or without a native atlas). */ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | null { - if (!global.isAndroid || draws.length === 0) return null; + if (!textureAtlasAvailable() || draws.length === 0) return null; // Pass 1: resolve ids, register unseen rasters, size the buffer. let registration: Map | null = null; @@ -184,9 +184,9 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | } if (registration) { const plainBuffer = buildRegistrationBuffer(registration); - if (plainBuffer) com.faceclaw.app.GlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(plainBuffer)); + if (plainBuffer) registerTextureGlyphs(plainBuffer, false); const aaBuffer = buildAaRegistrationBuffer(registration); - if (aaBuffer) com.faceclaw.app.GlyphAtlas.registerAa(new com.faceclaw.app.AndroidByteReader(aaBuffer)); + if (aaBuffer) registerTextureGlyphs(aaBuffer, true); } if (bytes === 0) return null; @@ -247,7 +247,7 @@ type RegistrationGroup = { * then per glyph * [encoding u32][bbxX s8][inkTop u8][width u8][inkHeight u8] * [inkHeight x row u32] (bit (ceil(width/8)*8 - 1 - col) = ink) - * mirroring GlyphAtlas.register on the Java side. Null when the groups hold + * mirroring GlyphAtlas.register on the Kotlin side. Null when the groups hold * no 1bpp glyphs. */ function buildRegistrationBuffer( diff --git a/app/graphics/surface-compositor.ios.ts b/app/graphics/surface-compositor.ios.ts index 30fc81f7..10f869d2 100644 --- a/app/graphics/surface-compositor.ios.ts +++ b/app/graphics/surface-compositor.ios.ts @@ -1,4 +1,5 @@ import { fromData, toData } from '../native/kotlin-data' +import type { TextureFrame } from '../g2/texture-planner' import type { SurfaceConfiguration, SurfaceRect } from './surface-compositor' export type { SurfaceConfiguration, SurfaceRect } from './surface-compositor' declare const FaceclawKitIosSurfaceCompositor: any @@ -23,13 +24,18 @@ export class SurfaceCompositor { this.native.dimBelowFactor(belowZOrder, factor) } setScreenBlanked(blanked: boolean): void { this.native.blankBlanked(blanked) } - submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void { + submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, draws: ArrayBuffer | null = null): void { // Keep lifecycle errors catchable in JS; an undeclared Kotlin exception // crossing the Objective-C boundary terminates the process. if (!this.surfaces.has(id)) throw new Error(`Unknown surface: ${id}`) if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger) || rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) throw new Error('Invalid frame buffer or rectangle') - this.native.submitIdDataXYWidthHeight(id, toData(pixels), rect.x, rect.y, rect.width, rect.height) + this.native.submitDrawsIdDataXYWidthHeightDraws(id, toData(pixels), rect.x, rect.y, rect.width, rect.height, + draws ? toData(new Uint8Array(draws)) : null) } composite(): Uint8Array { return fromData(this.native.composite()) } + compositeFrame(): { pixels: Uint8Array; textures: TextureFrame } { + const native = this.native.compositeFrame() + return { pixels: fromData(native.pixels), textures: { native } } + } } diff --git a/app/graphics/surface-compositor.ts b/app/graphics/surface-compositor.ts index 1d8f80e7..d7ea2ca6 100644 --- a/app/graphics/surface-compositor.ts +++ b/app/graphics/surface-compositor.ts @@ -1,4 +1,5 @@ /** Platform-independent 8bpp surface composition for the local display. */ +import type { TextureFrame } from '../g2/texture-planner' export type SurfaceRect = { x: number; y: number; width: number; height: number } export type SurfaceConfiguration = SurfaceRect & { zOrder: number @@ -35,7 +36,7 @@ export class SurfaceCompositor { this.dimFactor = Math.max(0, Math.min(1, factor)) } setScreenBlanked(blanked: boolean): void { this.blanked = blanked } - submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void { + submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, _draws: ArrayBuffer | null = null): void { const surface = this.surfaces.get(id) if (!surface) throw new Error(`Unknown surface: ${id}`) if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger) @@ -50,6 +51,7 @@ export class SurfaceCompositor { surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left) } } + compositeFrame(): { pixels: Uint8Array; textures?: TextureFrame } { return { pixels: this.composite() } } composite(): Uint8Array { const output = new Uint8Array(this.width * this.height) if (this.blanked) return output diff --git a/app/native/active-display.ios.ts b/app/native/active-display.ios.ts index c3a3d946..9c635c29 100644 --- a/app/native/active-display.ios.ts +++ b/app/native/active-display.ios.ts @@ -1,8 +1,12 @@ -/** Workers send baked grayscale frames to the main-thread iOS compositor. */ +/** Workers send baked pixels and shared-Kotlin atlas references together. */ const display = { - submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number): void { + submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number, + _fingerprint?: string, _paintMs?: number, _frameId?: number, draws: ArrayBuffer | null = null): void { const data = NSData.dataWithBytesLength(interop.handleof(buffer), buffer.byteLength) - global.postMessage({ type: 'surface-frame', surfaceId, width, height, pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) }) + const drawData = draws ? NSData.dataWithBytesLength(interop.handleof(draws), draws.byteLength) : null + global.postMessage({ type: 'surface-frame', surfaceId, width, height, + pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions), + draws: drawData?.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) }) }, } export function getActiveDisplay(): typeof display { return display } diff --git a/app/native/kotlin-protocol-smoke.ios.ts b/app/native/kotlin-protocol-smoke.ios.ts index 390f662f..d3c41337 100644 --- a/app/native/kotlin-protocol-smoke.ios.ts +++ b/app/native/kotlin-protocol-smoke.ios.ts @@ -4,6 +4,8 @@ import { CfwTransport, parseCfwAcks } from '../g2/cfw-transport.ios' import { SurfaceCompositor } from '../graphics/surface-compositor.ios' import { buildBoundingBoxPayload } from '../g2/ble-image-optimizer.ios' import { lvglMetrics } from './lvgl-font.ios' +import { textureImageId } from './texture-atlas.ios' +import { IosTexturePlanner } from './texture-planner.ios' /** Development-only checks of real NativeScript selectors and bulk data crossings. */ export function runKotlinProtocolSmokeTest(): void { @@ -30,6 +32,18 @@ export function runKotlinProtocolSmokeTest(): void { assert(hex(c.composite()) === '0010ff203040', 'composition') assert(hex(protocol.packGray4(c.composite(), 3, 2)) === '01f02340', 'packing') c.setScreenBlanked(true); assert(c.composite().every(p => p === 0), 'blanking') + const textures = new IosTexturePlanner(), textured = new SurfaceCompositor(4, 2) + textured.configureSurface('icon', { x: 0, y: 0, width: 2, height: 2, zOrder: 0, transparency: 'opaque' }) + const white = new Uint8Array(4).fill(255) + const icon = textureImageId('protocol-smoke-white', 2, 2, white) + const draw = new DataView(new ArrayBuffer(9)); draw.setUint8(0, 1); draw.setUint32(1, icon, true) + textured.submitSurfaceFrame('icon', white, { x: 0, y: 0, width: 2, height: 2 }, draw.buffer) + const snapshot = textured.compositeFrame(), packed = protocol.packGray4(snapshot.pixels, 4, 2) + const cold = textures.plan(null, packed, snapshot.textures, 1) + assert(!!cold && cold.uploads[0]?.[0] === 18 && cold.payload[0] === 8, 'texture upload and draw') + assert(textures.plan(null, packed, snapshot.textures, 1)?.uploads.length === 0, 'texture reuse') + textures.reset() + assert((textures.plan(null, packed, snapshot.textures, 1)?.uploads.length ?? 0) > 0, 'texture reset') const changed = new Uint8Array(16); changed[0] = 255 assert(buildBoundingBoxPayload(new Uint8Array(16), changed, 8, 4, 1)?.[0] === 3, 'image update') assert(lvglMetrics('/nonexistent/faceclaw-font').length === 0, 'font bridge') diff --git a/app/native/texture-atlas.ios.ts b/app/native/texture-atlas.ios.ts new file mode 100644 index 00000000..9656982c --- /dev/null +++ b/app/native/texture-atlas.ios.ts @@ -0,0 +1,17 @@ +import { toData } from './kotlin-data' +declare const FaceclawKitIosTextureAtlas: any +const atlas = FaceclawKitIosTextureAtlas.new() + +export function textureAtlasAvailable(): boolean { return true } +export function textureFontId(key: string): number { return atlas.fontIdKey(key) } +export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void { + const data = toData(new Uint8Array(buffer)) + if (aa) atlas.registerAaGlyphsData(data) + else atlas.registerGlyphsData(data) +} +export function registerFirmwareGlyphs(buffer: ArrayBuffer): void { + atlas.registerFirmwareGlyphsData(toData(new Uint8Array(buffer))) +} +export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number { + return atlas.imageIdKeyWidthHeightData(key, width, height, toData(pixels)) +} diff --git a/app/native/texture-atlas.ts b/app/native/texture-atlas.ts new file mode 100644 index 00000000..f39d97d7 --- /dev/null +++ b/app/native/texture-atlas.ts @@ -0,0 +1,17 @@ +declare const com: any + +/** Android bridge to the process-wide shared Kotlin atlases. */ +export function textureAtlasAvailable(): boolean { return !!global.isAndroid } +export function textureFontId(key: string): number { return com.faceclaw.app.GlyphAtlas.fontId(key) } +export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void { + const reader = new com.faceclaw.app.AndroidByteReader(buffer) + if (aa) com.faceclaw.app.GlyphAtlas.registerAa(reader) + else com.faceclaw.app.GlyphAtlas.register(reader) +} +export function registerFirmwareGlyphs(buffer: ArrayBuffer): void { + com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer)) +} +export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number { + const buffer = pixels.byteOffset === 0 && pixels.byteLength === pixels.buffer.byteLength ? pixels.buffer : pixels.slice().buffer + return com.faceclaw.app.ImageAtlas.ensure(key, width, height, new com.faceclaw.app.AndroidByteReader(buffer)) +} diff --git a/app/native/texture-planner.ios.ts b/app/native/texture-planner.ios.ts new file mode 100644 index 00000000..aabbc056 --- /dev/null +++ b/app/native/texture-planner.ios.ts @@ -0,0 +1,13 @@ +import { fromData, toData, nativeArray } from './kotlin-data' +import type { SessionTexturePlanner, TextureFrame, TexturePlan } from '../g2/texture-planner' +declare const FaceclawKitIosTexturePlanner: any + +export class IosTexturePlanner implements SessionTexturePlanner { + private readonly native = FaceclawKitIosTexturePlanner.new() + reset(): void { this.native.reset() } + plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null { + const result = this.native.planPreviousNextFrameFirstId(previous ? toData(previous) : null, toData(next), frame.native, firstId) + return result ? { payload: fromData(result.payload), uploads: nativeArray(result.uploads, fromData), + nextFid: result.nextFid, usedBytes: result.usedBytes } : null + } +} diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index a2c0cf88..adf510b5 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -37,7 +37,7 @@ export type WorkerAppReply = | { type: "buzzer-sequence"; payload: number[] } | { type: "navigation-sensors"; request: NavigationSensorRequest } | { type: "open-url"; url: string } - | { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string } + | { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string; draws?: string } | { /** * The worker's bundle has evaluated and its onmessage handler is @@ -190,7 +190,7 @@ export type WorkerAppHostOptions = { setSurfaceVisible: (surfaceId: string, visible: boolean) => void; removeSurface: (surfaceId: string) => void; requestShellRender: () => void; - submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number) => void; + submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null) => void; startTextInput?: () => void; /** Open or focus the Settings app, optionally selecting a section. */ openSettings: (section?: string) => void; @@ -259,7 +259,9 @@ export class WorkerAppHost { const { width, height } = message; if (!Number.isInteger(width) || !Number.isInteger(height) || width < 1 || height < 1 || width > 640 || height > 480) break; const data = NSData.alloc().initWithBase64EncodedStringOptions(message.pixels, 0 as NSDataBase64DecodingOptions); - if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height); + const draws = message.draws ? NSData.alloc().initWithBase64EncodedStringOptions(message.draws, 0 as NSDataBase64DecodingOptions) : null; + if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height, + draws ? interop.bufferFromData(draws) : null); break; } case "worker-ready": diff --git a/native/kotlin/README.md b/native/kotlin/README.md index 441b6639..cdc76456 100644 --- a/native/kotlin/README.md +++ b/native/kotlin/README.md @@ -45,9 +45,21 @@ On `ios-port`, platform adapters route the existing TypeScript session and UI logic through shared Kotlin for protocol builders, CRC/framing/reassembly, event decoding, CFW compression/ACKs, changed-image regions, surface composition, and LVGL glyphs. The original pure TypeScript implementations remain as Node -test references; NativeScript selects the `.ios.ts` native adapters. Features -not yet present in the iOS app can use the shared audio/recording and texture -classes without another Java port. +test references; NativeScript selects the `.ios.ts` native adapters. + +Texture rendering uses the same `GlyphAtlas`, `ImageAtlas`, `TextureCacheState` +and `TexturePlanner` on both platforms. iOS's `IosTextureAtlas` and +`IosTexturePlanner` expose bulk `NSData` bridges. Surface updates retain draw +identities through composition, including worker replies; immutable composite +snapshots keep those identities paired with the pixels while frames coalesce. +The BLE session plans only frames it will enqueue, sends mode-18 uploads before +the mode-19/20 draw batch, and counts a frame only after all uploads and draws +are acknowledged by both lenses. Session failure/teardown resets residency; +resuming near the firmware's 90-second lease deadline restarts the session +before queued draws can reference a freed cache. In either case, +the next frame reuploads lazily. Plans over the CFW message limit fall back to +pixel bands and invalidate any unsent allocations. The 256 KiB bump allocator +and reset-on-full policy are unchanged; there is no pinning or individual eviction. ## Build and validation @@ -71,7 +83,8 @@ npm run test:kotlin:ios and iOS: wire vectors, framing/corruption/expiry, streaming compression, replay, composition, PNG pixels, font glyphs, and Java SHA-256 golden outputs for DSP and GIF. Android additionally runs the existing Java protocol fixtures and -concurrency checks; iOS exercises `NSData` and compositor adapters. Debug app +concurrency checks; common texture tests cover reuse, overflow, reset and 32-bit +offsets, and iOS exercises `NSData`, atlas, compositor and planner adapters. Debug app startup checks real NativeScript metadata and bidirectional callbacks, logging `FACECLAW_KOTLIN_BRIDGE_PASS` and `FACECLAW_KOTLIN_PROTOCOL_PASS`. diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt index 6effc438..d23383a9 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt @@ -1,6 +1,7 @@ package com.faceclaw.app import kotlin.jvm.JvmField +import kotlin.jvm.JvmOverloads import kotlin.jvm.JvmStatic /** @@ -54,6 +55,7 @@ class TexturePlanner { * should run instead (no replayable draws, or identical frames). */ @JvmStatic + @JvmOverloads fun plan( previous: ByteArray?, next: ByteArray?, @@ -64,6 +66,7 @@ class TexturePlanner { fidStart: Int, allowMultiRect: Boolean, maxRects: Int, + platform: ProtocolPlatform = protocolPlatform(), ): Result? { if ( (((((next == null) || (draws == null)) || (draws.size == 0)) || (width <= 0)) || @@ -75,7 +78,7 @@ class TexturePlanner { if ((next.size != (stride * height))) { return null } - var planStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs() + var planStartedAtMs: Long = platform.elapsedRealtimeMs() var fullFrame: Boolean = ((previous == null) || (previous.size != next.size)) var rects: MutableList if (!fullFrame) { @@ -103,7 +106,7 @@ class TexturePlanner { } else { rects = mutableListOf(intArrayOf(0, 0, width, height)) } - var matchStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs() + var matchStartedAtMs: Long = platform.elapsedRealtimeMs() var selected: MutableList = ArrayList() var bakedCandidates: Int = 0 var selectedImages: Int = 0 @@ -177,7 +180,7 @@ class TexturePlanner { if ((selected.isEmpty() && fwSubs.isEmpty())) { return null } - var cacheStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs() + var cacheStartedAtMs: Long = platform.elapsedRealtimeMs() var drawable: MutableList = (if (selected.isEmpty()) mutableListOf() else ensureResident(cache, selected)) bakedCandidates += (selected.size - drawable.size) @@ -204,7 +207,7 @@ class TexturePlanner { } return null } - var punchStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs() + var punchStartedAtMs: Long = platform.elapsedRealtimeMs() var punched: ByteArray = next.copyOf() for (sel in drawn) { punch(punched, stride, width, height, sel) @@ -212,7 +215,7 @@ class TexturePlanner { for (fw in fwPunches) { punchFw(punched, stride, width, height, fw) } - var encodeStartedAtMs: Long = protocolPlatform().elapsedRealtimeMs() + var encodeStartedAtMs: Long = platform.elapsedRealtimeMs() var subs: MutableList = ArrayList() var fid: Int = fidStart if (fullFrame) { @@ -256,7 +259,7 @@ class TexturePlanner { uploadBytes, fullFrame, ) - var doneAtMs: Long = protocolPlatform().elapsedRealtimeMs() + var doneAtMs: Long = platform.elapsedRealtimeMs() result.rectsMs = ((matchStartedAtMs - planStartedAtMs)).toInt() result.matchMs = ((cacheStartedAtMs - matchStartedAtMs)).toInt() result.cacheMs = ((punchStartedAtMs - cacheStartedAtMs)).toInt() diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt index 78c59e86..daf4cfed 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt @@ -466,6 +466,18 @@ class SurfaceCompositor { rectWidth: Int, rectHeight: Int, fingerprint: String, + ) = submitSurface(surfaceId, pixels, rectX, rectY, rectWidth, rectHeight, fingerprint, null) + + /** Retain the full surface's draw identities along with its already-baked pixels. */ + fun submitSurface( + surfaceId: String, + pixels: ByteReader, + rectX: Int, + rectY: Int, + rectWidth: Int, + rectHeight: Int, + fingerprint: String, + draws: ByteReader?, ) { updateSurface( surfaceId, @@ -475,7 +487,7 @@ class SurfaceCompositor { rectWidth, rectHeight, fingerprint, - null, + draws, false, ) } diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt index 2d3cb4ba..9129c3d7 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt @@ -230,7 +230,10 @@ class IosSurfaceCompositor(width: Int, height: Int) { fun blank(blanked: Boolean) = compositor.setBlanked(blanked) - fun submit(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int) { + fun submit(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int) = + submitDraws(id, data, x, y, width, height, null) + + fun submitDraws(id: String, data: NSData, x: Int, y: Int, width: Int, height: Int, draws: NSData?) { val (sw, sh) = sizes[id] ?: error("Unknown surface: $id") require(width > 0 && height > 0 && data.length == width.toULong() * height.toULong()) val left = maxOf(0, x) @@ -260,8 +263,14 @@ class IosSurfaceCompositor(width: Int, height: Int) { right - left, bottom - top, (++sequence).toString(), + // Draw records describe a full surface. Partial/clipped updates must discard + // old identities; their pixels remain available through the baked fallback. + if (x == 0 && y == 0 && width == sw && height == sh) draws?.let(::IosByteReader) + else null, ) } fun composite(): NSData = compositor.composite().gray.data() + + fun compositeFrame(): IosTextureFrame = IosTextureFrame(compositor.composite()) } diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosTextureApi.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosTextureApi.kt new file mode 100644 index 00000000..09dd5077 --- /dev/null +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosTextureApi.kt @@ -0,0 +1,50 @@ +package com.faceclaw.app + +import platform.Foundation.NSData + +/** Binary adapters only: atlas identity, RLE, residency and planning stay in commonMain. */ +class IosTextureAtlas { + fun fontId(key: String): Int = GlyphAtlas.fontId(key) + + fun registerGlyphs(data: NSData) = GlyphAtlas.register(IosByteReader(data)) + + fun registerAaGlyphs(data: NSData) = GlyphAtlas.registerAa(IosByteReader(data)) + + fun registerFirmwareGlyphs(data: NSData) = FwGlyphAtlas.register(IosByteReader(data)) + + fun imageId(key: String, width: Int, height: Int, data: NSData): Int = + ImageAtlas.ensure(key, width, height, IosByteReader(data)) +} + +/** A snapshot keeps pixels and their screen-space draw identities together while JS coalesces. */ +class IosTextureFrame internal constructor(internal val composite: SurfaceCompositor.Composite) { + val pixels: NSData = composite.gray.data() +} + +class IosTexturePlan internal constructor(result: TexturePlanner.Result, val usedBytes: Int) { + val payload: NSData = result.payload.data() + val uploads: List = result.uploads.map { it.data() } + val nextFid: Int = result.nextFid +} + +/** One instance per BLE session, called on the session's serial JS thread. */ +class IosTexturePlanner { + private val cache = TextureCacheState() + + fun reset() = cache.reset() + + fun plan(previous: NSData?, next: NSData, frame: IosTextureFrame, firstId: Int): IosTexturePlan? { + val composite = frame.composite + val result = TexturePlanner.plan( + previous?.byteArray(), next.byteArray(), composite.width, composite.height, + composite.draws, cache, firstId, true, ConnectionOptions.MULTI_RECT_MAX_RECTS, + ) ?: return null + // A plan is atomic. If it cannot fit the CFW transport, discard its uploads + // and residency too, before the session falls back to ordinary pixel bands. + if (result.payload.size > 65535 || result.uploads.any { it.size > 65535 }) { + cache.reset() + return null + } + return IosTexturePlan(result, cache.usedBytes()) + } +} diff --git a/tests/compass-texture-cache.test.cjs b/tests/compass-texture-cache.test.cjs index a544fa79..694f2083 100644 --- a/tests/compass-texture-cache.test.cjs +++ b/tests/compass-texture-cache.test.cjs @@ -149,13 +149,14 @@ test('heading updates reuse the texture; brightness, layout and clipping changes test('the background registers once and uses a nine-byte image reference on later frames', () => { const registered = []; - const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', {}, '', { + const atlas = load('app/native/texture-atlas.ts', {}, '', { global: { isAndroid: true }, com: { faceclaw: { app: { AndroidByteReader: class { constructor(buffer) { this.buffer = buffer; } }, ImageAtlas: { ensure: (...args) => { registered.push(args); return 1; } } } } }, }); + const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', { '../native/texture-atlas': atlas }); const background = compass.createCompassBackground(576, 260, 256, 193.25, 98, 140, 1); for (const heading of [0, 1, 2, 45, 90]) { const image = background.clone(); diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index 7217df57..3f8543b9 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -11,6 +11,39 @@ function load(file, context) { return sandbox.exports; } +test('iOS worker replies keep cached draw references paired with their baked pixels', () => { + const replies = [], received = []; + const wrap = buffer => ({ length: buffer.byteLength, buffer, + base64EncodedStringWithOptions: () => Buffer.from(buffer).toString('base64') }); + const NSData = { + dataWithBytesLength: buffer => wrap(buffer), + alloc: () => ({ initWithBase64EncodedStringOptions: text => { + const bytes = Uint8Array.from(Buffer.from(text, 'base64')); return wrap(bytes.buffer); + } }), + }; + const interop = { handleof: buffer => buffer, bufferFromData: data => data.buffer }; + const display = load('app/native/active-display.ios.ts', { + NSData, interop, global: { postMessage: message => replies.push(message) }, + }).getActiveDisplay(); + const pixels = new Uint8Array([255, 0, 0, 255]), draws = new Uint8Array([1, 7, 0, 0, 0, 0, 0, 0, 0]); + display.submitSurfaceFrame(pixels.buffer, 'window:test', 0, 0, 2, 2, 'frame', 0, 1, draws.buffer); + const worker = {}; + const { WorkerAppHost } = load('app/ui/shell/worker-window.ts', { + require: () => ({}), global: { isIOS: true }, NSData, interop, + }); + const host = new WorkerAppHost({ appId: 'test', worker, submitPixels: (...args) => received.push(args) }); + host.openWindows.add('test'); + worker.onmessage({ data: replies[0] }); + assert.equal(received.length, 1); + assert.deepEqual([...received[0][1]], [...pixels]); + assert.deepEqual([...new Uint8Array(received[0][4])], [...draws]); + display.submitSurfaceFrame(pixels.buffer, 'window:test', 0, 0, 2, 2); + worker.onmessage({ data: replies[1] }); + assert.equal(received[1][4], null, 'a pixel-only update must clear previous draw identities'); + host.openWindows.clear(); worker.onmessage({ data: replies[0] }); + assert.equal(received.length, 2, 'closed windows cannot deliver stale cached draws'); +}); + test('iOS installed apps launch, recover missing packages, and close before uninstall', async () => { const calls = [], app = { packageId: 'test.app', name: 'Test' }; let present = true; @@ -215,9 +248,11 @@ test('background glasses input still composites frames; phone resume preserves t '../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) }, '../graphics/surface-compositor': { SurfaceCompositor: class { configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {} setScreenBlanked() {} - composite() { return new Uint8Array([1, 2]); } + compositeFrame() { return { pixels: new Uint8Array([1, 2]) }; } } }, - '../graphics/plane': { flattenPlanes: () => ({ pixels: new Uint8Array([1, 2]), width: 2, height: 1 }) }, + '../graphics/plane': { flattenPlanesWithDraws: () => ({ image: { pixels: new Uint8Array([1, 2]), width: 2, height: 1 }, draws: [] }) }, + '../graphics/glyph-wire': { prepareFrameDraws: () => null }, + '../native/texture-planner.ios': { IosTexturePlanner: class {} }, '../native/ios-graphics': { previewPixels: pixels => pixels }, '../ui/shell/shell': { shell, rawInputEventToInputEvent: input => input }, '../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 640, height: 480 }) }, @@ -333,7 +368,8 @@ test('iOS configures an in-process surface before submitting constructor-trigger require: name => ({ '../ui/shell/shell': { shell }, '../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 2, height: 1 }) }, - '../graphics/plane': { flattenPlanes: planes => planes[0] }, + '../graphics/plane': { flattenPlanesWithDraws: planes => ({ image: planes[0], draws: [] }) }, + '../graphics/glyph-wire': { prepareFrameDraws: () => null }, })[name] ?? {}, console, }); @@ -365,7 +401,7 @@ test('iOS configures an in-process surface before submitting constructor-trigger test('iOS compositor rejects missing and removed surfaces before crossing into Kotlin', () => { let nativeSubmissions = 0; - const native = { configureIdXYWidthHeightZOrderTransparent() {}, removeId() {}, submitIdDataXYWidthHeight() { nativeSubmissions++; } }; + const native = { configureIdXYWidthHeightZOrderTransparent() {}, removeId() {}, submitDrawsIdDataXYWidthHeightDraws() { nativeSubmissions++; } }; const api = load('app/graphics/surface-compositor.ios.ts', { require: () => ({ toData: data => data }), FaceclawKitIosSurfaceCompositor: { alloc: () => ({ initWithWidthHeight: () => native }) }, diff --git a/tests/ios-ble.test.cjs b/tests/ios-ble.test.cjs index 5cc8ce06..c6647841 100644 --- a/tests/ios-ble.test.cjs +++ b/tests/ios-ble.test.cjs @@ -165,9 +165,9 @@ async function until(predicate) { for (let i = 0; i < 100; i++) { if (predicate()) return; await new Promise(resolve => setTimeout(resolve, 5)); } assert.fail('Timed out waiting for test state'); } -function harness(t, transport = new FakeTransport()) { +function harness(t, transport = new FakeTransport(), textures) { const states = [], input = [], compass = [], wear = []; - const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing)); + const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing), undefined, textures); t.after(async () => { transport.hold = false; await session.stop(); }); return { transport, session, states, input, compass, wear }; } @@ -306,6 +306,67 @@ const ack = (transport, sent, overrides) => transport.ack(sent.id, sent.message, const tick = () => new Promise(resolve => setImmediate(resolve)); const noisy = () => Uint8Array.from({ length: 640 * 480 }, (_, i) => (i % 2) * 255); +test('texture uploads precede their frame, share ACK accounting, and coalesce pixels with draw identities', async t => { + const plans = []; + let resets = 0; + const textures = { + reset() { resets++; }, + plan(previous, next, frame, firstId) { + plans.push({ previous, next, frame, firstId }); + return { uploads: plans.length === 1 ? [new Uint8Array([18, 1]), new Uint8Array([18, 2])] : [], + payload: new Uint8Array([8, plans.length]), nextFid: firstId + 2, usedBytes: 100 }; + }, + }; + const h = harness(t, new FakeTransport(), textures); + await h.session.start(addresses); h.transport.hold = true; + const a = { native: { id: 'a' } }, b = { native: { id: 'b' } }, c = { native: { id: 'c' } }; + h.session.setFrame(new Uint8Array(640 * 480), a); + await until(() => imageMessages(h.transport).length === 3); + const sent = imageMessages(h.transport); + assert.deepEqual(sent.map(s => s.message.payload[0]), [18, 18, 8]); + h.session.setFrame(new Uint8Array(640 * 480).fill(80), b); + h.session.setFrame(new Uint8Array(640 * 480).fill(160), c); + assert.equal(plans.length, 1, 'coalesced frames must not allocate textures'); + ack(h.transport, sent[2]); ack(h.transport, sent[1]); await tick(); + assert.equal(h.session.state.frames, 0, 'a draw ACK cannot retire an unacknowledged upload'); + ack(h.transport, sent[0]); await until(() => imageMessages(h.transport).length === 4); + assert.equal(h.session.state.frames, 1); + assert.equal(plans.length, 2); assert.equal(plans[1].frame, c); + assert.deepEqual(plans[1].previous, plans[0].next); + assert.equal(plans[1].firstId, plans[0].firstId + 2); + assert.ok(plans[1].next.every(value => value === 0xaa)); + ack(h.transport, imageMessages(h.transport)[3]); await until(() => h.session.state.frames === 2); + h.transport.hold = false; await h.session.stop(); + assert.ok(resets > 0); +}); + +test('a failed texture upload invalidates residency and the pipelined frame base', async t => { + let resets = 0; + const textures = { + reset() { resets++; }, + plan() { return { uploads: [new Uint8Array([18])], payload: new Uint8Array([8]), nextFid: 1, usedBytes: 4 }; }, + }; + const h = harness(t, new FakeTransport(), textures); await h.session.start(addresses); + const original = h.session.requestCfw.bind(h.session); + h.session.requestCfw = (role, payload) => payload[0] === 18 ? Promise.reject(new Error('texture upload failed')) : original(role, payload); + h.session.setFrame(new Uint8Array(640 * 480), { native: {} }); + await until(() => h.session.state.phase === 'retrying'); + assert.equal(resets, 1); assert.equal(h.session.lastEnqueued, null); + assert.equal(h.session.latestTextures, null); assert.equal(h.session.state.frames, 0); +}); + +test('resuming near framebuffer lease expiry discards cached draws before transmission', async t => { + let resets = 0, plans = 0; + const textures = { reset() { resets++; }, plan() { plans++; return null; } }; + const h = harness(t, new FakeTransport(), textures); await h.session.start(addresses); + h.session.lastLease = Date.now() - 85_000; + h.session.setFrame(new Uint8Array(640 * 480), { native: {} }); + await h.session.pump(); + assert.equal(h.session.state.phase, 'retrying'); + assert.equal(resets, 1); assert.equal(plans, 0); + assert.equal(imageMessages(h.transport).length, 0); +}); + // Both eyes must finish; wrong ingress, size, CRC, ordinal and legacy replies do not count. test('CFW ACK identity and both-lens completion gate display metrics', async t => { const { iosBleTraffic } = require('../.test-build/app/g2/ble-traffic-counters.js'); diff --git a/tests/ios-glanceboard.test.cjs b/tests/ios-glanceboard.test.cjs index 76d065a0..52572abc 100644 --- a/tests/ios-glanceboard.test.cjs +++ b/tests/ios-glanceboard.test.cjs @@ -102,6 +102,8 @@ function fixture() { '../ui/dashboard-settings': { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } }, '../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) }, '../apps/ios-availability': { iosAppUnavailableReason: () => null }, + '../graphics/glyph-wire': { prepareFrameDraws: () => null }, + '../native/texture-planner.ios': { IosTexturePlanner: class {} }, '../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images, '../native/ios-graphics': { previewPixels: pixels => pixels }, '../ui/gestures': { makeInputEvent: event => event }, '../ui/layers': { noopLayerActions: {} }, diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/TextureCacheTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/TextureCacheTest.kt new file mode 100644 index 00000000..7a02c8fe --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/TextureCacheTest.kt @@ -0,0 +1,99 @@ +package com.faceclaw.app + +import kotlin.test.* + +class TextureCacheTest { + private fun u32(bytes: ByteArray, offset: Int): Int = + (0..3).fold(0) { n, i -> n or ((bytes[offset + i].toInt() and 255) shl (i * 8)) } + + @Test + fun offsetsAbove64KiBAndLazyFontSlotsRoundTripOnBothPlatforms() { + val cache = TextureCacheState() + val large = ImageAtlas.Entry(255, 255, ByteArray(255 * 255) { (it % 15 + 1).toByte() }) + repeat(3) { assertTrue(cache.ensureImage(it, large) >= 0) } + val glyph = GlyphAtlas.Glyph(1, 1, 0, 1, 0, null, byteArrayOf(15)) + val offset = cache.ensureGlyph(42, 65, glyph) + val table = cache.fontTableOffset(42) + assertTrue(table > 65535) + assertEquals(table + 384, offset) + val memory = ByteArray(TextureCacheState.CACHE_SIZE) + for (upload in cache.drainUploadPayloads(3600)) { + assertEquals(18, upload[0].toInt()) + var pos = 1 + while (pos < upload.size) { + val address = u32(upload, pos) + val length = (upload[pos + 4].toInt() and 255) or ((upload[pos + 5].toInt() and 255) shl 8) + upload.copyInto(memory, address, pos + 6, pos + 6 + length) + pos += 6 + length + } + assertEquals(upload.size, pos) + } + assertEquals(offset, u32(memory, table + (65 - 32) * 4)) + assertContentEquals(glyph.cachedBytes, memory.copyOfRange(offset, offset + glyph.cachedBytes.size)) + assertEquals(offset, cache.ensureGlyph(42, 65, glyph)) + assertFalse(cache.hasPendingUploads()) + cache.reset() + assertEquals(0, cache.usedBytes()) + assertEquals(384, cache.ensureGlyph(42, 65, glyph)) + assertTrue(cache.hasPendingUploads()) + } + + @Test + fun plannerReusesResidencyAndReuploadsAfterReset() { + val cache = TextureCacheState() + val id = ImageAtlas.ensure("texture-test-white", 2, 2, ArrayByteReader(ByteArray(4) { -1 })) + val pixels = ByteArray(8 * 4) + for (y in 0..1) for (x in 0..1) pixels[y * 8 + x] = -1 + val packed = BmpUtil.pack4bppFromGray8(pixels, 8, 4) + val draws = arrayOf(SurfaceCompositor.ScreenDraw.image(id, 0, 0)) + fun plan() = assertNotNull(TexturePlanner.plan(null, packed, 8, 4, draws, cache, 1, true, 8, testPlatform())) + val cold = plan() + assertEquals(1, cold.drawnImages) + assertTrue(cold.uploads.isNotEmpty()) + val warm = plan() + assertTrue(warm.uploads.isEmpty()) + assertContentEquals(cold.payload, warm.payload) + cache.reset() + assertTrue(plan().uploads.isNotEmpty()) + // Occlusion fails the correctness check and leaves the image baked. + assertNull(TexturePlanner.plan(null, ByteArray(packed.size), 8, 4, draws, cache, 1, true, 8, testPlatform())) + } + + @Test + fun fullCacheResetsAndRetriesTheCurrentFrame() { + val cache = TextureCacheState() + val large = ImageAtlas.Entry(255, 255, ByteArray(255 * 255) { (it % 15 + 1).toByte() }) + var fillerId = -1 + while (cache.ensureImage(fillerId--, large) >= 0) {} + // Fill the remaining tail, forcing the next real allocation to fail. + val remaining = TextureCacheState.CACHE_SIZE - cache.usedBytes() + while (cache.ensureImage(fillerId--, ImageAtlas.Entry(1, 1, byteArrayOf(15))) >= 0) {} + assertTrue(remaining > 0) + val generation = cache.generation() + val id = ImageAtlas.ensure("texture-test-full", 2, 2, ArrayByteReader(ByteArray(4) { -1 })) + val result = assertNotNull(TexturePlanner.plan(null, ByteArray(4) { -1 }, 4, 2, + arrayOf(SurfaceCompositor.ScreenDraw.image(id, 0, 0)), cache, 1, true, 8, testPlatform())) + assertEquals(generation + 1, cache.generation()) + assertEquals(1, result.drawnImages) + assertTrue(cache.usedBytes() < 100) + assertTrue(result.uploads.isNotEmpty()) + } + + @Test + fun workingSetLargerThanTheCacheFallsBackWithoutAResetLoop() { + val cache = TextureCacheState() + val raster = ByteArray(255 * 255) { if (it % 2 == 0) 16 else 32 } + val draws = (0..4).map { + val id = ImageAtlas.ensure("oversized-working-set-$it", 255, 255, ArrayByteReader(raster)) + SurfaceCompositor.ScreenDraw.image(id, 0, 0) + }.toTypedArray() + val gray = ByteArray(256 * 256) + repeat(255) { raster.copyInto(gray, it * 256, it * 255, (it + 1) * 255) } + val result = assertNotNull(TexturePlanner.plan(null, + BmpUtil.pack4bppFromGray8(gray, 256, 256), 256, 256, draws, cache, 1, true, 8, testPlatform())) + assertEquals(1, cache.generation()) + assertEquals(4, result.drawnImages) + assertEquals(1, result.bakedCandidates) + assertTrue(cache.usedBytes() <= TextureCacheState.CACHE_SIZE) + } +} diff --git a/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosTextureApiTest.kt b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosTextureApiTest.kt new file mode 100644 index 00000000..57895601 --- /dev/null +++ b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosTextureApiTest.kt @@ -0,0 +1,86 @@ +package com.faceclaw.app + +import kotlin.test.* + +class IosTextureApiTest { + private fun imageDraw(id: Int): ByteArray = byteArrayOf( + 1, id.toByte(), (id shr 8).toByte(), (id shr 16).toByte(), (id shr 24).toByte(), 0, 0, 0, 0, + ) + + @Test + fun nativeSnapshotsPreserveDrawsAcrossCoalescingAndSessionResets() { + val atlas = IosTextureAtlas() + val white = ByteArray(4) { -1 }.data() + val id = atlas.imageId("ios-texture-api-white", 2, 2, white) + assertEquals(id, IosTextureAtlas().imageId("ios-texture-api-white", 2, 2, white)) + val compositor = IosSurfaceCompositor(8, 4) + compositor.configure("app", 2, 0, 2, 2, 0, false) + compositor.submitDraws("app", white, 0, 0, 2, 2, imageDraw(id).data()) + val frame = compositor.compositeFrame() + assertEquals(2, frame.composite.draws!!.single().x) + // A later update must not change the previously submitted snapshot. + compositor.submit("app", ByteArray(4).data(), 0, 0, 2, 2) + assertTrue(compositor.compositeFrame().composite.draws!!.isEmpty()) + val next = IosProtocol().pack(frame.pixels, 8, 4) + val planner = IosTexturePlanner() + val cold = assertNotNull(planner.plan(null, next, frame, 1)) + assertEquals(18, cold.uploads.first().byteArray()[0].toInt()) + assertEquals(8, cold.payload.byteArray()[0].toInt()) + val warm = assertNotNull(planner.plan(null, next, frame, 1)) + assertTrue(warm.uploads.isEmpty()) + assertContentEquals(cold.payload.byteArray(), warm.payload.byteArray()) + planner.reset() + assertTrue(assertNotNull(planner.plan(null, next, frame, 1)).uploads.isNotEmpty()) + // A partial update clears identities even if the caller supplies stale draws. + compositor.submitDraws("app", byteArrayOf(-1).data(), 0, 0, 1, 1, imageDraw(id).data()) + assertTrue(compositor.compositeFrame().composite.draws!!.isEmpty()) + } + + @Test + fun glyphRegistrationAndDrawingUseTheSharedAsciiTable() { + val atlas = IosTextureAtlas() + val key = "ios-aa-texture-test" + val bytes = key.encodeToByteArray() + atlas.registerAaGlyphs((byteArrayOf(bytes.size.toByte()) + bytes + + byteArrayOf(2, 1, 0, 65, 0, 0, 0, 0, 0, 2, 2, -1, -1)).data()) + val font = atlas.fontId(key) + val draws = byteArrayOf(0, font.toByte(), (font shr 8).toByte(), 65, 0, 0, 0, 0, 0, 0, 0, -1) + val compositor = IosSurfaceCompositor(4, 2) + compositor.configure("text", 0, 0, 2, 2, 0, false) + compositor.submitDraws("text", ByteArray(4) { -1 }.data(), 0, 0, 2, 2, draws.data()) + val frame = compositor.compositeFrame() + val plan = assertNotNull(IosTexturePlanner().plan(null, IosProtocol().pack(frame.pixels, 4, 2), frame, 1)) + assertTrue(plan.usedBytes > 384) + val payload = plan.payload.byteArray() + // The last mode-8 submessage is the shared mode-20 cached glyph run. + assertEquals(20, payload[payload.size - 12].toInt()) + assertEquals(65, payload.last().toInt()) + } + + @Test + fun oversizedPlanDiscardsUnsentResidencyBeforePixelFallback() { + val atlas = IosTextureAtlas() + val white = ByteArray(4) { -1 }.data() + val id = atlas.imageId("ios-oversized-plan", 2, 2, white) + var random = 12345 + val gray = ByteArray(640 * 480) { + random = random xor (random shl 13) + random = random xor (random ushr 17) + random = random xor (random shl 5) + random.toByte() + } + gray[0] = -1; gray[1] = -1; gray[640] = -1; gray[641] = -1 + val compositor = IosSurfaceCompositor(640, 480) + compositor.configure("noise", 0, 0, 640, 480, 0, false) + compositor.submitDraws("noise", gray.data(), 0, 0, 640, 480, imageDraw(id).data()) + val frame = compositor.compositeFrame() + val planner = IosTexturePlanner() + assertNull(planner.plan(null, IosProtocol().pack(frame.pixels, 640, 480), frame, 1)) + val small = IosSurfaceCompositor(4, 2) + small.configure("icon", 0, 0, 2, 2, 0, false) + small.submitDraws("icon", white, 0, 0, 2, 2, imageDraw(id).data()) + val next = small.compositeFrame() + val retry = assertNotNull(planner.plan(null, IosProtocol().pack(next.pixels, 4, 2), next, 1)) + assertTrue(retry.uploads.isNotEmpty(), "the rejected plan's image was never uploaded") + } +}