forked from Blendman974/faceclaw
Refactor texture cache usage to shared Kotlin for iOS
This commit is contained in:
1 parent
ccb320c66f
commit
07828db0bf
23 files changed
+552
-74
No files matched your search
+31
-10
@@ -9,6 +9,7 @@ import { hasCompatibleFirmware, firmwareIncompatibilityMessage, REQUIRED_FACECLA
|
||||
import { hexToBytes } from '../util/hex-util'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
import { decodeWearState, enableWearDetection, queryWearState } from './wear-protocol'
|
||||
import type { SessionTexturePlanner, TextureFrame } from './texture-planner'
|
||||
|
||||
declare function setTimeout(callback: () => void, ms: number): number
|
||||
declare function clearTimeout(id: number): void
|
||||
@@ -60,6 +61,7 @@ export class GlassesSession {
|
||||
private wanted = false
|
||||
private addresses: SessionAddresses | null = null
|
||||
private latest: Uint8Array | null = null
|
||||
private latestTextures: TextureFrame | null = null
|
||||
private displayed: Uint8Array | null = null
|
||||
private lastEnqueued: Uint8Array | null = null
|
||||
// Match Android: advance only for emitted deltas, skipping 0 / 0xffff.
|
||||
@@ -90,7 +92,8 @@ export class GlassesSession {
|
||||
private readonly onActivity: () => void = () => {},
|
||||
private readonly onCompass: (event: CompassEvent) => void = () => {},
|
||||
private readonly onWearState: (wearing: boolean) => void = () => {},
|
||||
onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {}) {
|
||||
onNotificationsChanged: (key?: string, popup?: boolean) => void = () => {},
|
||||
private readonly textures?: SessionTexturePlanner) {
|
||||
this.notifications = new AncsClient(packet => this.writePackets('right', () => [packet]), onNotificationsChanged, message => this.log('ANCS: ' + message))
|
||||
this.off = transport.onEvent(event => this.receive(event))
|
||||
}
|
||||
@@ -428,9 +431,11 @@ export class GlassesSession {
|
||||
if (this.state.phase !== 'connected') return
|
||||
await this.request('right', protocol.SID.settings, magic => setBrightness(magic, level), 'Brightness')
|
||||
}
|
||||
setFrame(gray: Uint8Array): void {
|
||||
setFrame(gray: Uint8Array, textures?: TextureFrame): void {
|
||||
if (this.state.phase !== 'connected') return
|
||||
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
|
||||
this.latest = protocol.packGray4(gray, 640, 480)
|
||||
this.latestTextures = textures ?? null
|
||||
this.schedule(0)
|
||||
}
|
||||
/** Same mode-5 kind-4 sequencer payload and acknowledged image path as Android. */
|
||||
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
|
||||
@@ -499,16 +504,27 @@ export class GlassesSession {
|
||||
while (this.canSendDisplay()) {
|
||||
if (!this.displaySending) {
|
||||
const packed = this.latest!; this.latest = null
|
||||
const textureFrame = this.latestTextures; this.latestTextures = null
|
||||
// Compare with the last enqueued image, not the last ACKed one. In an
|
||||
// A -> B -> A sequence, B may still be in flight when A is requested.
|
||||
if (this.lastEnqueued && packed.every((value, i) => value === this.lastEnqueued![i])) continue
|
||||
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId)
|
||||
const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed))
|
||||
if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
|
||||
const commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId)
|
||||
if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands)
|
||||
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
|
||||
this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`)
|
||||
const texturePlan = textureFrame ? this.textures?.plan(this.lastEnqueued, packed, textureFrame, this.nextImageFrameId) : null
|
||||
let commands: Uint8Array[]
|
||||
if (texturePlan) {
|
||||
// Uploads participate in the same ordered, acknowledged command window as
|
||||
// the image. Coalescing only happens before planning mutates residency.
|
||||
commands = [...texturePlan.uploads, texturePlan.payload]
|
||||
this.nextImageFrameId = texturePlan.nextFid
|
||||
this.log(`Display textures (${texturePlan.uploads.length} uploads, cache=${texturePlan.usedBytes}B)`)
|
||||
} else {
|
||||
const delta = buildBoundingBoxPayload(this.lastEnqueued, packed, 640, 480, this.nextImageFrameId)
|
||||
const payload = delta ?? protocol.concat(new Uint8Array([6]), protocol.rle4(packed))
|
||||
if (delta) this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
|
||||
commands = payload.length <= CFW_MAX_MESSAGE ? [payload] : buildFullFrameBands(packed, 640, 480, this.nextImageFrameId)
|
||||
if (payload.length > CFW_MAX_MESSAGE) for (const _ of commands)
|
||||
this.nextImageFrameId = this.nextImageFrameId >= 0xfffe ? 1 : this.nextImageFrameId + 1
|
||||
this.log(`Display ${delta ? 'bbox' : 'full'} (${commands.length} CFW commands)`)
|
||||
}
|
||||
this.displaySending = { packed, commands, offset: 0, pending: 0 }
|
||||
this.displayFrames.push(this.displaySending); this.lastEnqueued = packed
|
||||
}
|
||||
@@ -538,6 +554,10 @@ export class GlassesSession {
|
||||
this.pumping = true; const generation = this.generation
|
||||
try {
|
||||
const now = Date.now()
|
||||
// Firmware frees textures when its 90-second framebuffer lease expires.
|
||||
// After a long iOS suspension, reconnect rather than replaying pending
|
||||
// draws against unknown memory. Leave a margin for the two arm writes.
|
||||
if (now - this.lastLease >= 80_000) throw new Error('Display lease lapsed; restarting session')
|
||||
if (now - this.lastHeartbeat >= 4000) {
|
||||
const gap = now - this.lastHeartbeat
|
||||
await this.request('right', protocol.SID.hub, protocol.heartbeat, 'Heartbeat', 1500); this.lastHeartbeat = Date.now()
|
||||
@@ -571,6 +591,7 @@ export class GlassesSession {
|
||||
for (const item of custom) item.reject(new Error('Connection ended'))
|
||||
this.cfwTransports.clear()
|
||||
this.receiver.clear(); this.writes.clear(); this.latest = this.displayed = null
|
||||
this.latestTextures = null; this.textures?.reset()
|
||||
this.lastEnqueued = null; this.displayFrames = []; this.displaySending = null; this.displayInFlight = 0
|
||||
}
|
||||
private closeLinks(): void {
|
||||
|
||||
@@ -35,7 +35,9 @@ import { getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetti
|
||||
import { readPhoneBatteryState } from '../native/phone-battery'
|
||||
import { iosAppUnavailableReason } from '../apps/ios-availability'
|
||||
import { SurfaceCompositor } from '../graphics/surface-compositor'
|
||||
import { flattenPlanes, type Plane } from '../graphics/plane'
|
||||
import { flattenPlanesWithDraws, type Plane } from '../graphics/plane'
|
||||
import { prepareFrameDraws } from '../graphics/glyph-wire'
|
||||
import { IosTexturePlanner } from '../native/texture-planner.ios'
|
||||
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
||||
import { previewPixels } from '../native/ios-graphics'
|
||||
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
|
||||
@@ -62,8 +64,8 @@ export class IosPreviewController {
|
||||
setScreenBlanked: async blanked => {
|
||||
this.compositor.setScreenBlanked(blanked); this.scheduleFrame()
|
||||
},
|
||||
submitSurfaceFrame: async (id, pixels, rect) => {
|
||||
this.compositor.submitSurfaceFrame(id, pixels, rect); this.scheduleFrame()
|
||||
submitSurfaceFrame: async (id, pixels, rect, _fingerprint, _paintMs, _frameId, draws) => {
|
||||
this.compositor.submitSurfaceFrame(id, pixels, rect, draws); this.scheduleFrame()
|
||||
},
|
||||
}
|
||||
private readonly glance = new GlanceHost({
|
||||
@@ -336,8 +338,8 @@ export class IosPreviewController {
|
||||
}
|
||||
}
|
||||
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||
const image = flattenPlanes(planes)
|
||||
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
|
||||
const { image, draws } = flattenPlanesWithDraws(planes)
|
||||
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, prepareFrameDraws(draws))
|
||||
this.scheduleFrame()
|
||||
}
|
||||
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
|
||||
@@ -350,12 +352,12 @@ export class IosPreviewController {
|
||||
try {
|
||||
if (this.shellDirty) {
|
||||
this.shellDirty = false
|
||||
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
|
||||
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
|
||||
const { image, draws } = flattenPlanesWithDraws(shell.paintSurface(), { width: 640, height: 480 })
|
||||
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 }, prepareFrameDraws(draws))
|
||||
this.compositor.setUnderlayDim(1, shell.underlayDim())
|
||||
}
|
||||
const pixels = this.compositor.composite()
|
||||
this.session?.setFrame(pixels)
|
||||
const { pixels, textures } = this.compositor.compositeFrame()
|
||||
this.session?.setFrame(pixels, textures)
|
||||
if (this.active) {
|
||||
const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green')
|
||||
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
|
||||
@@ -511,8 +513,8 @@ export class IosPreviewController {
|
||||
},
|
||||
setSurfaceVisible: (id, visible) => { this.compositor.setSurfaceVisible(id, visible); this.scheduleFrame() },
|
||||
removeSurface: id => { this.compositor.removeSurface(id); this.scheduleFrame() },
|
||||
submitPixels: (id, pixels, width, height) => {
|
||||
this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }); this.scheduleFrame()
|
||||
submitPixels: (id, pixels, width, height, draws) => {
|
||||
this.compositor.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }, draws); this.scheduleFrame()
|
||||
},
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
openSettings: section => { void this.launchApp('settings', { section }) },
|
||||
@@ -560,7 +562,7 @@ export class IosPreviewController {
|
||||
this.refreshClock()
|
||||
}, receiveCompassEvent, wearing => this.handleWearState(wearing), (key, popup) => {
|
||||
iosNotificationsChanged(key, popup); this.requestShellRender()
|
||||
})
|
||||
}, new IosTexturePlanner())
|
||||
bindIosNotifications(this.session.notifications)
|
||||
bindCompassSession(this.session)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Platform-owned snapshot of the same composite as the submitted pixels. */
|
||||
export type TextureFrame = { readonly native: object }
|
||||
export type TexturePlan = { uploads: Uint8Array[]; payload: Uint8Array; nextFid: number; usedBytes: number }
|
||||
/** Sessions own residency; frame snapshots may be coalesced without changing it. */
|
||||
export interface SessionTexturePlanner {
|
||||
plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null
|
||||
reset(): void
|
||||
}
|
||||
+24
-24
@@ -1,20 +1,20 @@
|
||||
/**
|
||||
* Marshals deferred-draw identity (text glyphs and icon images) to the Java
|
||||
* side for the texture-cache pipeline (CFW modes 12/13/14; see
|
||||
* Marshals deferred-draw identity (text glyphs and icon images) to the Kotlin
|
||||
* side for the texture-cache pipeline (CFW modes 18/19/20; see
|
||||
* notes/texture-cache-display-list-design.md).
|
||||
*
|
||||
* Channels, all cheap ByteBuffer crossings:
|
||||
* - glyph raster registration into the process-wide Java GlyphAtlas, once
|
||||
* - glyph raster registration into the process-wide Kotlin GlyphAtlas, once
|
||||
* per (font, glyph) per JS context (main thread and each worker register
|
||||
* independently; the atlas dedupes by the font's stable atlasKey);
|
||||
* - image registration into the Java ImageAtlas, once per distinct image
|
||||
* - image registration into the Kotlin ImageAtlas, once per distinct image
|
||||
* content per JS context (images are content-addressed: the key is a hash
|
||||
* of dimensions + pixels, so the same icon rendered anywhere dedupes and
|
||||
* a changed icon is simply a new image);
|
||||
* - a per-frame draw list accompanying each submitted surface frame, which
|
||||
* the Java planner may replay as on-glasses cached draws.
|
||||
* the Kotlin planner may replay as on-glasses cached draws.
|
||||
*
|
||||
* Draws that can't participate (no atlasKey, outside the mode-14 ASCII
|
||||
* Draws that can't participate (no atlasKey, outside the mode-20 ASCII
|
||||
* table, ink outside the line cell, oversized images) are simply left out —
|
||||
* their pixels are baked into the submitted frame either way, so exclusion
|
||||
* only means "no wire savings for this draw".
|
||||
@@ -22,7 +22,7 @@
|
||||
import { Glyph } from "./bdffont";
|
||||
import { GrayImage, type DeferredDraw, type GlyphFont, type PlacedFwText, type PlacedImage } from "./image";
|
||||
|
||||
declare const com: any;
|
||||
import { textureAtlasAvailable, textureFontId, registerTextureGlyphs, registerFirmwareGlyphs, textureImageId } from "../native/texture-atlas";
|
||||
|
||||
const GLYPH_RECORD_BYTES = 12; // [0][fontId u16][encoding u32][penX s16][lineY s16][value u8]
|
||||
const IMAGE_RECORD_BYTES = 9; // [1][imageId u32][x s16][y s16]
|
||||
@@ -32,14 +32,14 @@ const FWTEXT_MEMBER_BYTES = 7; // [cp u32][dx s16][ink u8]
|
||||
type FontWireState = {
|
||||
fontId: number;
|
||||
cellHeight: number;
|
||||
/** Encodings already registered with the Java atlas from this JS context. */
|
||||
/** Encodings already registered with the Kotlin atlas from this JS context. */
|
||||
registered: Set<number>;
|
||||
};
|
||||
|
||||
const fontStates = new Map<GlyphFont, FontWireState>();
|
||||
|
||||
/**
|
||||
* Java-side image id per source image object, or null for images that can't
|
||||
* Kotlin-side image id per source image object, or null for images that can't
|
||||
* participate. Sources handed to drawImage are immutable by contract, so
|
||||
* memoizing by object identity is safe; a re-rendered icon is a new object
|
||||
* and re-resolves (usually to the same content-addressed id).
|
||||
@@ -47,12 +47,12 @@ const fontStates = new Map<GlyphFont, FontWireState>();
|
||||
const imageIds = new WeakMap<GrayImage, number | null>();
|
||||
|
||||
function fontWireState(font: GlyphFont): FontWireState | null {
|
||||
if (!global.isAndroid || !font.atlasKey) return null;
|
||||
if (!textureAtlasAvailable() || !font.atlasKey) return null;
|
||||
if (font.lineHeight <= 0 || font.lineHeight > 255) return null;
|
||||
let state = fontStates.get(font);
|
||||
if (!state) {
|
||||
state = {
|
||||
fontId: com.faceclaw.app.GlyphAtlas.fontId(font.atlasKey),
|
||||
fontId: textureFontId(font.atlasKey),
|
||||
cellHeight: font.lineHeight,
|
||||
registered: new Set(),
|
||||
};
|
||||
@@ -63,7 +63,7 @@ function fontWireState(font: GlyphFont): FontWireState | null {
|
||||
|
||||
/**
|
||||
* Whether this glyph can be expressed as an on-glasses cached draw: within
|
||||
* the mode-14 char table (32..127), a nonempty raster that fits the font's
|
||||
* the mode-20 char table (32..127), a nonempty raster that fits the font's
|
||||
* line cell (the cached image is bbxWidth x lineHeight with the ink placed at
|
||||
* inkTop), and metrics that fit the wire fields.
|
||||
*/
|
||||
@@ -76,12 +76,12 @@ function representableGlyph(font: GlyphFont, glyph: Glyph): boolean {
|
||||
return inkTop >= 0 && inkTop + glyph.bbxHeight <= font.lineHeight;
|
||||
}
|
||||
|
||||
/** Firmware-font codepoints already registered with the Java FwGlyphAtlas. */
|
||||
/** Firmware-font codepoints already registered with the Kotlin FwGlyphAtlas. */
|
||||
const fwRegistered = new Set<string>();
|
||||
|
||||
/**
|
||||
* Whether a firmware-text run is expressible on the wire, registering any
|
||||
* unseen ink members' rasters with the Java FwGlyphAtlas as a side effect.
|
||||
* unseen ink members' rasters with the Kotlin FwGlyphAtlas as a side effect.
|
||||
*/
|
||||
function prepareFwTextRun(placed: PlacedFwText): boolean {
|
||||
if (!inRange16(placed.x) || !inRange16(placed.y)) return false;
|
||||
@@ -116,21 +116,21 @@ function prepareFwTextRun(placed: PlacedFwText): boolean {
|
||||
bytes.set(entry.data.nibbles, offset + 10);
|
||||
offset += 10 + entry.data.nibbles.length;
|
||||
}
|
||||
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer));
|
||||
registerFirmwareGlyphs(buffer);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Resolve (registering on first sight) the Java image id for a drawImage source. */
|
||||
/** Resolve (registering on first sight) the Kotlin image id for a drawImage source. */
|
||||
function imageId(placed: PlacedImage): number | null {
|
||||
if (!global.isAndroid) return null;
|
||||
if (!textureAtlasAvailable()) return null;
|
||||
const source = placed.source;
|
||||
const memo = imageIds.get(source);
|
||||
if (memo !== undefined) return memo;
|
||||
let id: number | null = null;
|
||||
if (source.width > 0 && source.width <= 255 && source.height > 0 && source.height <= 255) {
|
||||
const key = `img:${source.width}x${source.height}:${source.contentHash32().toString(16)}`;
|
||||
id = com.faceclaw.app.ImageAtlas.ensure(key, source.width, source.height, new com.faceclaw.app.AndroidByteReader(source.pixels.buffer));
|
||||
id = textureImageId(key, source.width, source.height, source.pixels);
|
||||
if (!(typeof id === "number") || id <= 0) id = null;
|
||||
}
|
||||
imageIds.set(source, id);
|
||||
@@ -143,16 +143,16 @@ function inRange16(v: number): boolean {
|
||||
|
||||
/**
|
||||
* Register any not-yet-registered glyph rasters and image contents among
|
||||
* `draws` with the Java atlases, then build the per-frame draw buffer to pass
|
||||
* `draws` with the Kotlin atlases, then build the per-frame draw buffer to pass
|
||||
* alongside the frame's pixels: little-endian tagged records
|
||||
* [0][fontId u16][encoding u32][penX s16][lineY s16][value u8] (glyph)
|
||||
* [1][imageId u32][x s16][y s16] (image)
|
||||
* [2][x s16][y s16][value u8][count u8] (fw text run)
|
||||
* then count x [cp u32][dx s16][ink u8]
|
||||
* in draw order. Returns null when nothing is expressible (or off-Android).
|
||||
* in draw order. Returns null when nothing is expressible (or without a native atlas).
|
||||
*/
|
||||
export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | null {
|
||||
if (!global.isAndroid || draws.length === 0) return null;
|
||||
if (!textureAtlasAvailable() || draws.length === 0) return null;
|
||||
|
||||
// Pass 1: resolve ids, register unseen rasters, size the buffer.
|
||||
let registration: Map<FontWireState, RegistrationGroup> | null = null;
|
||||
@@ -184,9 +184,9 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer |
|
||||
}
|
||||
if (registration) {
|
||||
const plainBuffer = buildRegistrationBuffer(registration);
|
||||
if (plainBuffer) com.faceclaw.app.GlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(plainBuffer));
|
||||
if (plainBuffer) registerTextureGlyphs(plainBuffer, false);
|
||||
const aaBuffer = buildAaRegistrationBuffer(registration);
|
||||
if (aaBuffer) com.faceclaw.app.GlyphAtlas.registerAa(new com.faceclaw.app.AndroidByteReader(aaBuffer));
|
||||
if (aaBuffer) registerTextureGlyphs(aaBuffer, true);
|
||||
}
|
||||
if (bytes === 0) return null;
|
||||
|
||||
@@ -247,7 +247,7 @@ type RegistrationGroup = {
|
||||
* then per glyph
|
||||
* [encoding u32][bbxX s8][inkTop u8][width u8][inkHeight u8]
|
||||
* [inkHeight x row u32] (bit (ceil(width/8)*8 - 1 - col) = ink)
|
||||
* mirroring GlyphAtlas.register on the Java side. Null when the groups hold
|
||||
* mirroring GlyphAtlas.register on the Kotlin side. Null when the groups hold
|
||||
* no 1bpp glyphs.
|
||||
*/
|
||||
function buildRegistrationBuffer(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { fromData, toData } from '../native/kotlin-data'
|
||||
import type { TextureFrame } from '../g2/texture-planner'
|
||||
import type { SurfaceConfiguration, SurfaceRect } from './surface-compositor'
|
||||
export type { SurfaceConfiguration, SurfaceRect } from './surface-compositor'
|
||||
declare const FaceclawKitIosSurfaceCompositor: any
|
||||
@@ -23,13 +24,18 @@ export class SurfaceCompositor {
|
||||
this.native.dimBelowFactor(belowZOrder, factor)
|
||||
}
|
||||
setScreenBlanked(blanked: boolean): void { this.native.blankBlanked(blanked) }
|
||||
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
|
||||
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, draws: ArrayBuffer | null = null): void {
|
||||
// Keep lifecycle errors catchable in JS; an undeclared Kotlin exception
|
||||
// crossing the Objective-C boundary terminates the process.
|
||||
if (!this.surfaces.has(id)) throw new Error(`Unknown surface: ${id}`)
|
||||
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|
||||
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) throw new Error('Invalid frame buffer or rectangle')
|
||||
this.native.submitIdDataXYWidthHeight(id, toData(pixels), rect.x, rect.y, rect.width, rect.height)
|
||||
this.native.submitDrawsIdDataXYWidthHeightDraws(id, toData(pixels), rect.x, rect.y, rect.width, rect.height,
|
||||
draws ? toData(new Uint8Array(draws)) : null)
|
||||
}
|
||||
composite(): Uint8Array { return fromData(this.native.composite()) }
|
||||
compositeFrame(): { pixels: Uint8Array; textures: TextureFrame } {
|
||||
const native = this.native.compositeFrame()
|
||||
return { pixels: fromData(native.pixels), textures: { native } }
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
/** Platform-independent 8bpp surface composition for the local display. */
|
||||
import type { TextureFrame } from '../g2/texture-planner'
|
||||
export type SurfaceRect = { x: number; y: number; width: number; height: number }
|
||||
export type SurfaceConfiguration = SurfaceRect & {
|
||||
zOrder: number
|
||||
@@ -35,7 +36,7 @@ export class SurfaceCompositor {
|
||||
this.dimFactor = Math.max(0, Math.min(1, factor))
|
||||
}
|
||||
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
|
||||
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
|
||||
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, _draws: ArrayBuffer | null = null): void {
|
||||
const surface = this.surfaces.get(id)
|
||||
if (!surface) throw new Error(`Unknown surface: ${id}`)
|
||||
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|
||||
@@ -50,6 +51,7 @@ export class SurfaceCompositor {
|
||||
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
|
||||
}
|
||||
}
|
||||
compositeFrame(): { pixels: Uint8Array; textures?: TextureFrame } { return { pixels: this.composite() } }
|
||||
composite(): Uint8Array {
|
||||
const output = new Uint8Array(this.width * this.height)
|
||||
if (this.blanked) return output
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
/** Workers send baked grayscale frames to the main-thread iOS compositor. */
|
||||
/** Workers send baked pixels and shared-Kotlin atlas references together. */
|
||||
const display = {
|
||||
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number): void {
|
||||
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, _x: number, _y: number, width: number, height: number,
|
||||
_fingerprint?: string, _paintMs?: number, _frameId?: number, draws: ArrayBuffer | null = null): void {
|
||||
const data = NSData.dataWithBytesLength(interop.handleof(buffer), buffer.byteLength)
|
||||
global.postMessage({ type: 'surface-frame', surfaceId, width, height, pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) })
|
||||
const drawData = draws ? NSData.dataWithBytesLength(interop.handleof(draws), draws.byteLength) : null
|
||||
global.postMessage({ type: 'surface-frame', surfaceId, width, height,
|
||||
pixels: data.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions),
|
||||
draws: drawData?.base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions) })
|
||||
},
|
||||
}
|
||||
export function getActiveDisplay(): typeof display { return display }
|
||||
@@ -4,6 +4,8 @@ import { CfwTransport, parseCfwAcks } from '../g2/cfw-transport.ios'
|
||||
import { SurfaceCompositor } from '../graphics/surface-compositor.ios'
|
||||
import { buildBoundingBoxPayload } from '../g2/ble-image-optimizer.ios'
|
||||
import { lvglMetrics } from './lvgl-font.ios'
|
||||
import { textureImageId } from './texture-atlas.ios'
|
||||
import { IosTexturePlanner } from './texture-planner.ios'
|
||||
|
||||
/** Development-only checks of real NativeScript selectors and bulk data crossings. */
|
||||
export function runKotlinProtocolSmokeTest(): void {
|
||||
@@ -30,6 +32,18 @@ export function runKotlinProtocolSmokeTest(): void {
|
||||
assert(hex(c.composite()) === '0010ff203040', 'composition')
|
||||
assert(hex(protocol.packGray4(c.composite(), 3, 2)) === '01f02340', 'packing')
|
||||
c.setScreenBlanked(true); assert(c.composite().every(p => p === 0), 'blanking')
|
||||
const textures = new IosTexturePlanner(), textured = new SurfaceCompositor(4, 2)
|
||||
textured.configureSurface('icon', { x: 0, y: 0, width: 2, height: 2, zOrder: 0, transparency: 'opaque' })
|
||||
const white = new Uint8Array(4).fill(255)
|
||||
const icon = textureImageId('protocol-smoke-white', 2, 2, white)
|
||||
const draw = new DataView(new ArrayBuffer(9)); draw.setUint8(0, 1); draw.setUint32(1, icon, true)
|
||||
textured.submitSurfaceFrame('icon', white, { x: 0, y: 0, width: 2, height: 2 }, draw.buffer)
|
||||
const snapshot = textured.compositeFrame(), packed = protocol.packGray4(snapshot.pixels, 4, 2)
|
||||
const cold = textures.plan(null, packed, snapshot.textures, 1)
|
||||
assert(!!cold && cold.uploads[0]?.[0] === 18 && cold.payload[0] === 8, 'texture upload and draw')
|
||||
assert(textures.plan(null, packed, snapshot.textures, 1)?.uploads.length === 0, 'texture reuse')
|
||||
textures.reset()
|
||||
assert((textures.plan(null, packed, snapshot.textures, 1)?.uploads.length ?? 0) > 0, 'texture reset')
|
||||
const changed = new Uint8Array(16); changed[0] = 255
|
||||
assert(buildBoundingBoxPayload(new Uint8Array(16), changed, 8, 4, 1)?.[0] === 3, 'image update')
|
||||
assert(lvglMetrics('/nonexistent/faceclaw-font').length === 0, 'font bridge')
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { toData } from './kotlin-data'
|
||||
declare const FaceclawKitIosTextureAtlas: any
|
||||
const atlas = FaceclawKitIosTextureAtlas.new()
|
||||
|
||||
export function textureAtlasAvailable(): boolean { return true }
|
||||
export function textureFontId(key: string): number { return atlas.fontIdKey(key) }
|
||||
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
|
||||
const data = toData(new Uint8Array(buffer))
|
||||
if (aa) atlas.registerAaGlyphsData(data)
|
||||
else atlas.registerGlyphsData(data)
|
||||
}
|
||||
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
|
||||
atlas.registerFirmwareGlyphsData(toData(new Uint8Array(buffer)))
|
||||
}
|
||||
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
|
||||
return atlas.imageIdKeyWidthHeightData(key, width, height, toData(pixels))
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
declare const com: any
|
||||
|
||||
/** Android bridge to the process-wide shared Kotlin atlases. */
|
||||
export function textureAtlasAvailable(): boolean { return !!global.isAndroid }
|
||||
export function textureFontId(key: string): number { return com.faceclaw.app.GlyphAtlas.fontId(key) }
|
||||
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
|
||||
const reader = new com.faceclaw.app.AndroidByteReader(buffer)
|
||||
if (aa) com.faceclaw.app.GlyphAtlas.registerAa(reader)
|
||||
else com.faceclaw.app.GlyphAtlas.register(reader)
|
||||
}
|
||||
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
|
||||
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer))
|
||||
}
|
||||
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
|
||||
const buffer = pixels.byteOffset === 0 && pixels.byteLength === pixels.buffer.byteLength ? pixels.buffer : pixels.slice().buffer
|
||||
return com.faceclaw.app.ImageAtlas.ensure(key, width, height, new com.faceclaw.app.AndroidByteReader(buffer))
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { fromData, toData, nativeArray } from './kotlin-data'
|
||||
import type { SessionTexturePlanner, TextureFrame, TexturePlan } from '../g2/texture-planner'
|
||||
declare const FaceclawKitIosTexturePlanner: any
|
||||
|
||||
export class IosTexturePlanner implements SessionTexturePlanner {
|
||||
private readonly native = FaceclawKitIosTexturePlanner.new()
|
||||
reset(): void { this.native.reset() }
|
||||
plan(previous: Uint8Array | null, next: Uint8Array, frame: TextureFrame, firstId: number): TexturePlan | null {
|
||||
const result = this.native.planPreviousNextFrameFirstId(previous ? toData(previous) : null, toData(next), frame.native, firstId)
|
||||
return result ? { payload: fromData(result.payload), uploads: nativeArray(result.uploads, fromData),
|
||||
nextFid: result.nextFid, usedBytes: result.usedBytes } : null
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ export type WorkerAppReply =
|
||||
| { type: "buzzer-sequence"; payload: number[] }
|
||||
| { type: "navigation-sensors"; request: NavigationSensorRequest }
|
||||
| { type: "open-url"; url: string }
|
||||
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string }
|
||||
| { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string; draws?: string }
|
||||
| {
|
||||
/**
|
||||
* The worker's bundle has evaluated and its onmessage handler is
|
||||
@@ -190,7 +190,7 @@ export type WorkerAppHostOptions = {
|
||||
setSurfaceVisible: (surfaceId: string, visible: boolean) => void;
|
||||
removeSurface: (surfaceId: string) => void;
|
||||
requestShellRender: () => void;
|
||||
submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number) => void;
|
||||
submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null) => void;
|
||||
startTextInput?: () => void;
|
||||
/** Open or focus the Settings app, optionally selecting a section. */
|
||||
openSettings: (section?: string) => void;
|
||||
@@ -259,7 +259,9 @@ export class WorkerAppHost {
|
||||
const { width, height } = message;
|
||||
if (!Number.isInteger(width) || !Number.isInteger(height) || width < 1 || height < 1 || width > 640 || height > 480) break;
|
||||
const data = NSData.alloc().initWithBase64EncodedStringOptions(message.pixels, 0 as NSDataBase64DecodingOptions);
|
||||
if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height);
|
||||
const draws = message.draws ? NSData.alloc().initWithBase64EncodedStringOptions(message.draws, 0 as NSDataBase64DecodingOptions) : null;
|
||||
if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height,
|
||||
draws ? interop.bufferFromData(draws) : null);
|
||||
break;
|
||||
}
|
||||
case "worker-ready":
|
||||
|
||||
Reference in new issue
Block a user