Refactor texture cache usage to shared Kotlin for iOS

This commit is contained in:
James Babcock committed 2026-09-20 15:12:13 -07:00
1 parent ccb320c66f
commit 07828db0bf
23 files changed
+552 -74

No files matched your search

+31 -10
View File
@@ -9,6 +9,7 @@ import { hasCompatibleFirmware, firmwareIncompatibilityMessage, REQUIRED_FACECLA
import { hexToBytes } from '../util/hex-util'
import { deviceAddressError } from './ios-peripheral-identity'
import { decodeWearState, enableWearDetection, queryWearState } from './wear-protocol'
import type { SessionTexturePlanner, TextureFrame } from './texture-planner'
declare function setTimeout(callback: () => void, ms: number): number
declare function clearTimeout(id: number): void
@@ -60,6 +61,7 @@ export class GlassesSession {
private wanted = false
private addresses: SessionAddresses | null = null
private latest: Uint8Array | null = null
private latestTextures: TextureFrame | null = null
private displayed: Uint8Array | null = null
private lastEnqueued: Uint8Array | null = null
// Match Android: advance only for emitted deltas, skipping 0 / 0xffff.
@@ -90,7 +92,8 @@ export class GlassesSession {
private readonly onActivity: () => void = () => {},
private readonly onCompass: (event: CompassEvent) => void = () => {},
private readonly onWearState: (wearing: boolean) => void = () => {},
onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {}) {
onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {},
private readonly textures?: SessionTexturePlanner) {
this.notifications = new AncsClient(packet => this.writePackets('right', () => [packet]), onNotificationsChanged, message => this.log('ANCS: ' + message))
this.off = transport.onEvent(event => this.receive(event))
}
@@ -428,9 +431,11 @@ export class GlassesSession {
if (this.state.phase !== 'connected') return
await this.request('right', protocol.SID.settings, magic => setBrightness(magic, level), 'Brightness')
}
setFrame(gray: Uint8Array): void {
setFrame(gray: Uint8Array, textures?: TextureFrame): void {
if (this.state.phase !== 'connected') return
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
this.latest = protocol.packGray4(gray, 640, 480)
this.latestTextures = textures ?? null
this.schedule(0)
}
/** Same mode-5 kind-4 sequencer payload and acknowledged image path as Android. */
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
@@ -499,16 +504,27 @@ export class GlassesSession {
while (this.canSendDisplay()) {
if (!this.displaySending) {
const packed = this.latest!; this.latest = null
const textureFrame = this.latestTextures; this.latestTextures = null
// Compare with the last enqueued image, not the last ACKed one. In an
// A -> B -> A sequence, B may still be in flight when A is requested.
if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId)
const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed))
if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
const commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId)
if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands)
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`)
const texturePlan = textureFrame ? this.textures?.plan(this.lastEnqueued, packed, textureFrame, this.nextImageFrameId) : null
let commands: Uint8Array[]
if (texturePlan) {
// Uploads participate in the same ordered, acknowledged command window as
// the image. Coalescing only happens before planning mutates residency.
commands = [...texturePlan.uploads, texturePlan.payload]
this.nextImageFrameId = texturePlan.nextFid
this.log(`Display textures (${texturePlan.uploads.length} uploads, cache=${texturePlan.usedBytes}B)`)
} else {
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId)
const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed))
if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId)
if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands)
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`)
}
this.displaySending = { packed, commands, offset: 0, pending: 0 }
this.displayFrames.push(this.displaySending); this.lastEnqueued = packed
}
@@ -538,6 +554,10 @@ export class GlassesSession {
this.pumping = true; const generation = this.generation
try {
const now = Date.now()
// Firmware frees textures when its 90-second framebuffer lease expires.
// After a long iOS suspension, reconnect rather than replaying pending
// draws against unknown memory. Leave a margin for the two arm writes.
if (now - this.lastLease >= 80_000) throw new Error('Display lease lapsed; restarting session')
if (now - this.lastHeartbeat >= 4000) {
const gap = now - this.lastHeartbeat
await this.request('right', protocol.SID.hub, protocol.heartbeat, 'Heartbeat', 1500); this.lastHeartbeat = Date.now()
@@ -571,6 +591,7 @@ export class GlassesSession {
for (const item of custom) item.reject(new Error('Connection ended'))
this.cfwTransports.clear()
this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null
this.latestTextures = null; this.textures?.reset()
this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0
}
private closeLinks(): void {
+14 -12
View File
@@ -35,7 +35,9 @@ import { getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetti
import { readPhoneBatteryState } from '../native/phone-battery'
import { iosAppUnavailableReason } from '../apps/ios-availability'
import { SurfaceCompositor } from '../graphics/surface-compositor'
import { flattenPlanes, type Plane } from '../graphics/plane'
import { flattenPlanesWithDraws, type Plane } from '../graphics/plane'
import { prepareFrameDraws } from '../graphics/glyph-wire'
import { IosTexturePlanner } from '../native/texture-planner.ios'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { previewPixels } from '../native/ios-graphics'
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
@@ -62,8 +64,8 @@ export class IosPreviewController {
setScreenBlanked: async blanked => {
this.compositor.setScreenBlanked(blanked); this.scheduleFrame()
},
submitSurfaceFrame: async (id, pixels, rect) => {
this.compositor.submitSurfaceFrame(id, pixels, rect); this.scheduleFrame()
submitSurfaceFrame: async (id, pixels, rect, _fingerprint, _paintMs, _frameId, draws) => {
this.compositor.submitSurfaceFrame(id, pixels, rect, draws); this.scheduleFrame()
},
}
private readonly glance = new GlanceHost({
@@ -336,8 +338,8 @@ export class IosPreviewController {
}
}
private async submit(id: string, planes: Plane[]): Promise<void> {
const image = flattenPlanes(planes)
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
const { image, draws } = flattenPlanesWithDraws(planes)
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, prepareFrameDraws(draws))
this.scheduleFrame()
}
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
@@ -350,12 +352,12 @@ export class IosPreviewController {
try {
if (this.shellDirty) {
this.shellDirty = false
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
const { image, draws } = flattenPlanesWithDraws(shell.paintSurface(), { width: 640, height: 480 })
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 }, prepareFrameDraws(draws))
this.compositor.setUnderlayDim(1, shell.underlayDim())
}
const pixels = this.compositor.composite()
this.session?.setFrame(pixels)
const { pixels, textures } = this.compositor.compositeFrame()
this.session?.setFrame(pixels, textures)
if (this.active) {
const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green')
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
@@ -511,8 +513,8 @@ export class IosPreviewController {
},
setSurfaceVisible: (id, visible) => { this.compositor.setSurfaceVisible(id, visible); this.scheduleFrame() },
removeSurface: id => { this.compositor.removeSurface(id); this.scheduleFrame() },
submitPixels: (id, pixels, width, height) => {
this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }); this.scheduleFrame()
submitPixels: (id, pixels, width, height, draws) => {
this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }, draws); this.scheduleFrame()
},
requestShellRender: () => this.requestShellRender(),
openSettings: section => { void this.launchApp('settings', { section }) },
@@ -560,7 +562,7 @@ export class IosPreviewController {
this.refreshClock()
}, receiveCompassEvent, wearing => this.handleWearState(wearing), (key, popup) => {
iosNotificationsChanged(key, popup); this.requestShellRender()
})
}, new IosTexturePlanner())
bindIosNotifications(this.session.notifications)
bindCompassSession(this.session)
}
+8
View File
@@ -0,0 +1,8 @@
/** Platform-owned snapshot of the same composite as the submitted pixels. */
export type TextureFrame = { readonly native: object }
export type TexturePlan = { uploads: Uint8Array[]; payload: Uint8Array; nextFid: number; usedBytes: number }
/** Sessions own residency; frame snapshots may be coalesced without changing it. */
export interface SessionTexturePlanner {
plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null
reset(): void
}
+24 -24
View File
@@ -1,20 +1,20 @@
/**
* Marshals deferred-draw identity (text glyphs and icon images) to the Java
* side for the texture-cache pipeline (CFW modes 12/13/14; see
* Marshals deferred-draw identity (text glyphs and icon images) to the Kotlin
* side for the texture-cache pipeline (CFW modes 18/19/20; see
* notes/texture-cache-display-list-design.md).
*
* Channels, all cheap ByteBuffer crossings:
* - glyph raster registration into the process-wide Java GlyphAtlas, once
* - glyph raster registration into the process-wide Kotlin GlyphAtlas, once
* per (font, glyph) per JS context (main thread and each worker register
* independently; the atlas dedupes by the font's stable atlasKey);
* - image registration into the Java ImageAtlas, once per distinct image
* - image registration into the Kotlin ImageAtlas, once per distinct image
* content per JS context (images are content-addressed: the key is a hash
* of dimensions + pixels, so the same icon rendered anywhere dedupes and
* a changed icon is simply a new image);
* - a per-frame draw list accompanying each submitted surface frame, which
* the Java planner may replay as on-glasses cached draws.
* the Kotlin planner may replay as on-glasses cached draws.
*
* Draws that can't participate (no atlasKey, outside the mode-14 ASCII
* Draws that can't participate (no atlasKey, outside the mode-20 ASCII
* table, ink outside the line cell, oversized images) are simply left out —
* their pixels are baked into the submitted frame either way, so exclusion
* only means "no wire savings for this draw".
@@ -22,7 +22,7 @@
import { Glyph } from "./bdffont";
import { GrayImage, type DeferredDraw, type GlyphFont, type PlacedFwText, type PlacedImage } from "./image";
declare const com: any;
import { textureAtlasAvailable, textureFontId, registerTextureGlyphs, registerFirmwareGlyphs, textureImageId } from "../native/texture-atlas";
const GLYPH_RECORD_BYTES = 12; // [0][fontId u16][encoding u32][penX s16][lineY s16][value u8]
const IMAGE_RECORD_BYTES = 9; // [1][imageId u32][x s16][y s16]
@@ -32,14 +32,14 @@ const FWTEXT_MEMBER_BYTES = 7; // [cp u32][dx s16][ink u8]
type FontWireState = {
fontId: number;
cellHeight: number;
/** Encodings already registered with the Java atlas from this JS context. */
/** Encodings already registered with the Kotlin atlas from this JS context. */
registered: Set<number>;
};
const fontStates = new Map<GlyphFont, FontWireState>();
/**
* Java-side image id per source image object, or null for images that can't
* Kotlin-side image id per source image object, or null for images that can't
* participate. Sources handed to drawImage are immutable by contract, so
* memoizing by object identity is safe; a re-rendered icon is a new object
* and re-resolves (usually to the same content-addressed id).
@@ -47,12 +47,12 @@ const fontStates = new Map<GlyphFont, FontWireState>();
const imageIds = new WeakMap<GrayImage, number | null>();
function fontWireState(font: GlyphFont): FontWireState | null {
if (!global.isAndroid || !font.atlasKey) return null;
if (!textureAtlasAvailable() || !font.atlasKey) return null;
if (font.lineHeight <= 0 || font.lineHeight > 255) return null;
let state = fontStates.get(font);
if (!state) {
state = {
fontId: com.faceclaw.app.GlyphAtlas.fontId(font.atlasKey),
fontId: textureFontId(font.atlasKey),
cellHeight: font.lineHeight,
registered: new Set(),
};
@@ -63,7 +63,7 @@ function fontWireState(font: GlyphFont): FontWireState | null {
/**
* Whether this glyph can be expressed as an on-glasses cached draw: within
* the mode-14 char table (32..127), a nonempty raster that fits the font's
* the mode-20 char table (32..127), a nonempty raster that fits the font's
* line cell (the cached image is bbxWidth x lineHeight with the ink placed at
* inkTop), and metrics that fit the wire fields.
*/
@@ -76,12 +76,12 @@ function representableGlyph(font: GlyphFont, glyph: Glyph): boolean {
return inkTop >= 0 && inkTop + glyph.bbxHeight <= font.lineHeight;
}
/** Firmware-font codepoints already registered with the Java FwGlyphAtlas. */
/** Firmware-font codepoints already registered with the Kotlin FwGlyphAtlas. */
const fwRegistered = new Set<string>();
/**
* Whether a firmware-text run is expressible on the wire, registering any
* unseen ink members' rasters with the Java FwGlyphAtlas as a side effect.
* unseen ink members' rasters with the Kotlin FwGlyphAtlas as a side effect.
*/
function prepareFwTextRun(placed: PlacedFwText): boolean {
if (!inRange16(placed.x) || !inRange16(placed.y)) return false;
@@ -116,21 +116,21 @@ function prepareFwTextRun(placed: PlacedFwText): boolean {
bytes.set(entry.data.nibbles, offset + 10);
offset += 10 + entry.data.nibbles.length;
}
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer));
registerFirmwareGlyphs(buffer);
}
return true;
}
/** Resolve (registering on first sight) the Java image id for a drawImage source. */
/** Resolve (registering on first sight) the Kotlin image id for a drawImage source. */
function imageId(placed: PlacedImage): number | null {
if (!global.isAndroid) return null;
if (!textureAtlasAvailable()) return null;
const source = placed.source;
const memo = imageIds.get(source);
if (memo !== undefined) return memo;
let id: number | null = null;
if (source.width > 0 && source.width <= 255 && source.height > 0 && source.height <= 255) {
const key = `img:${source.width}x${source.height}:${source.contentHash32().toString(16)}`;
id = com.faceclaw.app.ImageAtlas.ensure(key, source.width, source.height, new com.faceclaw.app.AndroidByteReader(source.pixels.buffer));
id = textureImageId(key, source.width, source.height, source.pixels);
if (!(typeof id === "number") || id <= 0) id = null;
}
imageIds.set(source, id);
@@ -143,16 +143,16 @@ function inRange16(v: number): boolean {
/**
* Register any not-yet-registered glyph rasters and image contents among
* `draws` with the Java atlases, then build the per-frame draw buffer to pass
* `draws` with the Kotlin atlases, then build the per-frame draw buffer to pass
* alongside the frame's pixels: little-endian tagged records
* [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] (glyph)
* [1][imageId u32][x s16][y s16] (image)
* [2][x s16][y s16][value u8][count u8] (fw text run)
* then count x [cp u32][dx s16][ink u8]
* in draw order. Returns null when nothing is expressible (or off-Android).
* in draw order. Returns null when nothing is expressible (or without a native atlas).
*/
export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | null {
if (!global.isAndroid || draws.length === 0) return null;
if (!textureAtlasAvailable() || draws.length === 0) return null;
// Pass 1: resolve ids, register unseen rasters, size the buffer.
let registration: Map<FontWireState, RegistrationGroup> | null = null;
@@ -184,9 +184,9 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer |
}
if (registration) {
const plainBuffer = buildRegistrationBuffer(registration);
if (plainBuffer) com.faceclaw.app.GlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(plainBuffer));
if (plainBuffer) registerTextureGlyphs(plainBuffer, false);
const aaBuffer = buildAaRegistrationBuffer(registration);
if (aaBuffer) com.faceclaw.app.GlyphAtlas.registerAa(new com.faceclaw.app.AndroidByteReader(aaBuffer));
if (aaBuffer) registerTextureGlyphs(aaBuffer, true);
}
if (bytes === 0) return null;
@@ -247,7 +247,7 @@ type RegistrationGroup = {
* then per glyph
* [encoding u32][bbxX s8][inkTop u8][width u8][inkHeight u8]
* [inkHeight x row u32] (bit (ceil(width/8)*8 - 1 - col) = ink)
* mirroring GlyphAtlas.register on the Java side. Null when the groups hold
* mirroring GlyphAtlas.register on the Kotlin side. Null when the groups hold
* no 1bpp glyphs.
*/
function buildRegistrationBuffer(
+8 -2
View File
@@ -1,4 +1,5 @@
import { fromData, toData } from '../native/kotlin-data'
import type { TextureFrame } from '../g2/texture-planner'
import type { SurfaceConfiguration, SurfaceRect } from './surface-compositor'
export type { SurfaceConfiguration, SurfaceRect } from './surface-compositor'
declare const FaceclawKitIosSurfaceCompositor: any
@@ -23,13 +24,18 @@ export class SurfaceCompositor {
this.native.dimBelowFactor(belowZOrder, factor)
}
setScreenBlanked(blanked: boolean): void { this.native.blankBlanked(blanked) }
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, draws: ArrayBuffer | null = null): void {
// Keep lifecycle errors catchable in JS; an undeclared Kotlin exception
// crossing the Objective-C boundary terminates the process.
if (!this.surfaces.has(id)) throw new Error(`Unknown surface: ${id}`)
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) throw new Error('Invalid frame buffer or rectangle')
this.native.submitIdDataXYWidthHeight(id, toData(pixels), rect.x, rect.y, rect.width, rect.height)
this.native.submitDrawsIdDataXYWidthHeightDraws(id, toData(pixels), rect.x, rect.y, rect.width, rect.height,
draws ? toData(new Uint8Array(draws)) : null)
}
composite(): Uint8Array { return fromData(this.native.composite()) }
compositeFrame(): { pixels: Uint8Array; textures: TextureFrame } {
const native = this.native.compositeFrame()
return { pixels: fromData(native.pixels), textures: { native } }
}
}
+3 -1
View File
@@ -1,4 +1,5 @@
/** Platform-independent 8bpp surface composition for the local display. */
import type { TextureFrame } from '../g2/texture-planner'
export type SurfaceRect = { x: number; y: number; width: number; height: number }
export type SurfaceConfiguration = SurfaceRect & {
zOrder: number
@@ -35,7 +36,7 @@ export class SurfaceCompositor {
this.dimFactor = Math.max(0, Math.min(1, factor))
}
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, _draws: ArrayBuffer | null = null): void {
const surface = this.surfaces.get(id)
if (!surface) throw new Error(`Unknown surface: ${id}`)
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
@@ -50,6 +51,7 @@ export class SurfaceCompositor {
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
}
}
compositeFrame(): { pixels: Uint8Array; textures?: TextureFrame } { return { pixels: this.composite() } }
composite(): Uint8Array {
const output = new Uint8Array(this.width * this.height)
if (this.blanked) return output
+7 -3
View File
@@ -1,8 +1,12 @@
/** Workers send baked grayscale frames to the main-thread iOS compositor. */
/** Workers send baked pixels and shared-Kotlin atlas references together. */
const display = {
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number): void {
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number,
_fingerprint?: string, _paintMs?: number, _frameId?: number, draws: ArrayBuffer | null = null): void {
const data = NSData.dataWithBytesLength(interop.handleof(buffer), buffer.byteLength)
global.postMessage({ type: 'surface-frame', surfaceId, width, height, pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) })
const drawData = draws ? NSData.dataWithBytesLength(interop.handleof(draws), draws.byteLength) : null
global.postMessage({ type: 'surface-frame', surfaceId, width, height,
pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions),
draws: drawData?.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) })
},
}
export function getActiveDisplay(): typeof display { return display }
+14
View File
@@ -4,6 +4,8 @@ import { CfwTransport, parseCfwAcks } from '../g2/cfw-transport.ios'
import { SurfaceCompositor } from '../graphics/surface-compositor.ios'
import { buildBoundingBoxPayload } from '../g2/ble-image-optimizer.ios'
import { lvglMetrics } from './lvgl-font.ios'
import { textureImageId } from './texture-atlas.ios'
import { IosTexturePlanner } from './texture-planner.ios'
/** Development-only checks of real NativeScript selectors and bulk data crossings. */
export function runKotlinProtocolSmokeTest(): void {
@@ -30,6 +32,18 @@ export function runKotlinProtocolSmokeTest(): void {
assert(hex(c.composite()) === '0010ff203040', 'composition')
assert(hex(protocol.packGray4(c.composite(), 3, 2)) === '01f02340', 'packing')
c.setScreenBlanked(true); assert(c.composite().every(p => p === 0), 'blanking')
const textures = new IosTexturePlanner(), textured = new SurfaceCompositor(4, 2)
textured.configureSurface('icon', { x: 0, y: 0, width: 2, height: 2, zOrder: 0, transparency: 'opaque' })
const white = new Uint8Array(4).fill(255)
const icon = textureImageId('protocol-smoke-white', 2, 2, white)
const draw = new DataView(new ArrayBuffer(9)); draw.setUint8(0, 1); draw.setUint32(1, icon, true)
textured.submitSurfaceFrame('icon', white, { x: 0, y: 0, width: 2, height: 2 }, draw.buffer)
const snapshot = textured.compositeFrame(), packed = protocol.packGray4(snapshot.pixels, 4, 2)
const cold = textures.plan(null, packed, snapshot.textures, 1)
assert(!!cold && cold.uploads[0]?.[0] === 18 && cold.payload[0] === 8, 'texture upload and draw')
assert(textures.plan(null, packed, snapshot.textures, 1)?.uploads.length === 0, 'texture reuse')
textures.reset()
assert((textures.plan(null, packed, snapshot.textures, 1)?.uploads.length ?? 0) > 0, 'texture reset')
const changed = new Uint8Array(16); changed[0] = 255
assert(buildBoundingBoxPayload(new Uint8Array(16), changed, 8, 4, 1)?.[0] === 3, 'image update')
assert(lvglMetrics('/nonexistent/faceclaw-font').length === 0, 'font bridge')
+17
View File
@@ -0,0 +1,17 @@
import { toData } from './kotlin-data'
declare const FaceclawKitIosTextureAtlas: any
const atlas = FaceclawKitIosTextureAtlas.new()
export function textureAtlasAvailable(): boolean { return true }
export function textureFontId(key: string): number { return atlas.fontIdKey(key) }
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
const data = toData(new Uint8Array(buffer))
if (aa) atlas.registerAaGlyphsData(data)
else atlas.registerGlyphsData(data)
}
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
atlas.registerFirmwareGlyphsData(toData(new Uint8Array(buffer)))
}
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
return atlas.imageIdKeyWidthHeightData(key, width, height, toData(pixels))
}
+17
View File
@@ -0,0 +1,17 @@
declare const com: any
/** Android bridge to the process-wide shared Kotlin atlases. */
export function textureAtlasAvailable(): boolean { return !!global.isAndroid }
export function textureFontId(key: string): number { return com.faceclaw.app.GlyphAtlas.fontId(key) }
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
const reader = new com.faceclaw.app.AndroidByteReader(buffer)
if (aa) com.faceclaw.app.GlyphAtlas.registerAa(reader)
else com.faceclaw.app.GlyphAtlas.register(reader)
}
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer))
}
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
const buffer = pixels.byteOffset === 0 && pixels.byteLength === pixels.buffer.byteLength ? pixels.buffer : pixels.slice().buffer
return com.faceclaw.app.ImageAtlas.ensure(key, width, height, new com.faceclaw.app.AndroidByteReader(buffer))
}
+13
View File
@@ -0,0 +1,13 @@
import { fromData, toData, nativeArray } from './kotlin-data'
import type { SessionTexturePlanner, TextureFrame, TexturePlan } from '../g2/texture-planner'
declare const FaceclawKitIosTexturePlanner: any
export class IosTexturePlanner implements SessionTexturePlanner {
private readonly native = FaceclawKitIosTexturePlanner.new()
reset(): void { this.native.reset() }
plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null {
const result = this.native.planPreviousNextFrameFirstId(previous ? toData(previous) : null, toData(next), frame.native, firstId)
return result ? { payload: fromData(result.payload), uploads: nativeArray(result.uploads, fromData),
nextFid: result.nextFid, usedBytes: result.usedBytes } : null
}
}
+5 -3
View File
@@ -37,7 +37,7 @@ export type WorkerAppReply =
| { type: "buzzer-sequence"; payload: number[] }
| { type: "navigation-sensors"; request: NavigationSensorRequest }
| { type: "open-url"; url: string }
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string }
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string; draws?: string }
| {
/**
* The worker's bundle has evaluated and its onmessage handler is
@@ -190,7 +190,7 @@ export type WorkerAppHostOptions = {
setSurfaceVisible: (surfaceId: string, visible: boolean) => void;
removeSurface: (surfaceId: string) => void;
requestShellRender: () => void;
submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number) => void;
submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null) => void;
startTextInput?: () => void;
/** Open or focus the Settings app, optionally selecting a section. */
openSettings: (section?: string) => void;
@@ -259,7 +259,9 @@ export class WorkerAppHost {
const { width, height } = message;
if (!Number.isInteger(width) || !Number.isInteger(height) || width < 1 || height < 1 || width > 640 || height > 480) break;
const data = NSData.alloc().initWithBase64EncodedStringOptions(message.pixels, 0 as NSDataBase64DecodingOptions);
if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height);
const draws = message.draws ? NSData.alloc().initWithBase64EncodedStringOptions(message.draws, 0 as NSDataBase64DecodingOptions) : null;
if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height,
draws ? interop.bufferFromData(draws) : null);
break;
}
case "worker-ready":