Files
faceclaw/app/native/preview-display.ts
T
Chris LeeandClaude Opus 5 54eae0c6b9 Stop leaking every submitted frame's pixel buffer in native memory
Passing a JS ArrayBuffer to a Java ByteBuffer parameter leaks it in
@nativescript/android 9.0.4 through 9.1.1: JsArgConverter wraps the backing store with
NewDirectByteBuffer and takes a JNI global ref that it never deletes. The
ref keeps the ByteBuffer reachable, so makeInstanceWeakAndCheckIfAlive
always answers "alive", the JS finalizer re-arms, and V8 never frees the
ArrayBuffer. submitSurfaceFrame passed a fresh copy of each frame this way,
so every frame leaked a full surface (576x260, 640x452 or 640x480 bytes)
plus one global ref, including frames that Java then discards as unchanged.
The draw-list buffer and buzzer payloads leaked the same way.

Each payload now goes through a reused Java direct ByteBuffer
(app/native/java-direct-buffer.ts): JS writes into it through an
ArrayBuffer.from view, and the Java object itself is passed, which takes
NativeScript's ordinary object path. Every consumer (SurfaceCompositor
applyAndComposite, playBuzzerSequence) copies before returning, so one
buffer per JS context is safe.

Covered: the communicator bridge (frames, draw lists, buzzer, which also
carries EvenHub tones), the preview-only display, and all eight worker
apps. Workers submit through the Android getActiveDisplay() (frames) and
worker-buzzer (tones), so the reused buffers live there rather than in
each worker; the worker sources are shared with iOS, whose active display
takes the ArrayBuffer itself. The
smaller one-off crossings (glyph and image atlas registration, the gray
preview helper, firmware and file writes) are left as they are.

tests/java-direct-buffer.test.cjs covers the helper against a fake
ByteBuffer, and the Android worker-buzzer test stubs it.

(Rebased onto main by James Babcock: worker changes moved into
active-display.ts and worker-buzzer.ts.)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-25 22:24:22 -07:00

190 lines
6.4 KiB
TypeScript

import { ImageSource, Utils } from "@nativescript/core";
import { dimFactor256, type FaceclawCommunicatorBridge, type SurfaceOptions } from "./faceclaw-communicator";
import { JavaDirectBuffer } from "./java-direct-buffer";
declare const com: any;
declare const java: any;
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
/**
* The display subset of the communicator bridge: everything the dashboard
* controller needs to composite frames and read them back for the phone
* mirror, screenshots, and recordings. FaceclawCommunicatorBridge satisfies
* it by construction (Pick), and PreviewDisplayTarget implements it against
* a headless compositor with no BLE transport behind it.
*/
export type DisplayTarget = Pick<
FaceclawCommunicatorBridge,
| "configureCompositorScreen"
| "configureSurface"
| "removeSurface"
| "setSurfaceVisible"
| "setSurfaceDepth"
| "setUnderlayDim"
| "setScreenBlanked"
| "submitSurfaceFrame"
| "submitShellScene"
| "waitForFrameFinished"
| "getCompositePreview"
| "saveScreenshot"
| "startScreenRecording"
| "recordScreenFrame"
| "stopScreenRecording"
>;
function nonNegativeNumber(value: number): number {
const numeric = Number(value);
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
}
/**
* Headless display target for preview-only mode (onboarding's "Preview Only"
* path): hosts the same Java SurfaceCompositor a live connection uses, so
* the whole shell/app render pipeline works with no glasses paired. Frames
* end at the compositor; the phone mirror polls them back as usual.
*
* Unlike the communicator bridge, calls go straight into Java: they are all
* quick in-memory operations, so there is no call queue and no transmit
* backpressure to wait on.
*/
export class PreviewDisplayTarget implements DisplayTarget {
private readonly compositor: any;
// Reused Java-side buffers: passing a JS ArrayBuffer to Java leaks it
// (java-direct-buffer.ts).
private readonly framePixelsBuffer = new JavaDirectBuffer(640 * 480);
private readonly frameDrawsBuffer = new JavaDirectBuffer();
constructor() {
const context = Utils.android.getApplicationContext();
if (!context) throw new Error("Android application context unavailable");
this.compositor = new com.faceclaw.app.FaceclawPreviewCompositor(context);
}
/**
* Publish the compositor as the worker isolates' frame target (they locate
* it through the Java static, like the BLE communicator) and receive a
* main-thread callback per applied frame — the preview path's stand-in for
* the connected path's frame-metrics callback.
*/
activate(onFrameComposited: () => void): void {
this.compositor.setFrameListener(
new java.lang.Runnable({ run: () => onFrameComposited() }),
);
this.compositor.makeActive();
}
/** Withdraw from workers and drop the frame callback. */
release(): void {
this.compositor.release();
}
async configureCompositorScreen(width: number, height: number): Promise<void> {
this.compositor.configureCompositorScreen(Math.round(width), Math.round(height));
}
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
const transparency = options.transparency === "color-key" ? 1 : 0;
this.compositor.configureSurface(
id,
Math.round(options.x),
Math.round(options.y),
Math.round(options.width),
Math.round(options.height),
Math.round(options.zOrder),
transparency,
);
}
async removeSurface(id: string): Promise<void> {
this.compositor.removeSurface(id);
}
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
this.compositor.setSurfaceVisible(id, Boolean(visible));
}
async setSurfaceDepth(id: string, depth: number): Promise<void> {
this.compositor.setSurfaceDepth(id, Math.round(depth));
}
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
this.compositor.submitShellScene(new Uint8Array(bytes).buffer, paintMs, frameId);
}
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
this.compositor.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
}
async setScreenBlanked(blanked: boolean): Promise<void> {
this.compositor.setScreenBlanked(Boolean(blanked));
}
async submitSurfaceFrame(
surfaceId: string,
pixels8bpp: Uint8Array,
rect: { x: number; y: number; width: number; height: number },
fingerprint: string,
paintMs = -1,
frameId = 0,
glyphs: ArrayBuffer | null = null,
): Promise<void> {
// Copied into a reused Java direct buffer, sized to exactly this view's
// bytes; passing the ArrayBuffer itself would leak it
// (java-direct-buffer.ts).
this.compositor.submitSurfaceFrame(
this.framePixelsBuffer.load(pixels8bpp),
surfaceId,
Math.round(rect.x),
Math.round(rect.y),
Math.round(rect.width),
Math.round(rect.height),
fingerprint,
Math.round(nonNegativeNumber(paintMs)),
Math.round(nonNegativeNumber(frameId)),
this.frameDrawsBuffer.loadOptional(glyphs),
);
// The Java side finishes the frame: the composite is the end of the line,
// and worker isolates submit to the same object without coming through here.
}
/** Composites are synchronous and nothing transmits, so there is never backpressure. */
waitForFrameFinished(): Promise<string | null> {
return Promise.resolve("composited");
}
getCompositePreview(green = false): ImageSource | null {
if (!global.isAndroid) return null;
const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green);
return bitmap ? new ImageSource(bitmap) : null;
}
saveScreenshot(crop?: { x: number; y: number; width: number; height: number }): string {
if (!global.isAndroid) return "";
if (!crop) return String(this.compositor.saveCompositePngScreenshot());
return String(
this.compositor.saveCompositePngScreenshot(
Math.round(crop.x),
Math.round(crop.y),
Math.round(crop.width),
Math.round(crop.height),
),
);
}
startScreenRecording(): void {
if (!global.isAndroid) return;
this.compositor.startScreenRecording();
}
recordScreenFrame(): void {
if (!global.isAndroid) return;
this.compositor.recordScreenFrame();
}
stopScreenRecording(): string {
if (!global.isAndroid) return "";
return String(this.compositor.stopScreenRecording());
}
}