forked from Blendman974/faceclaw
Merge branch 'main' of https://github.com/jimrandomh/faceclaw
This commit is contained in:
commit
51edaee883
27 files changed
+634
-127
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import { Application } from "@nativescript/core";
|
||||
import { fetchWithUserAgent } from "../../util/http";
|
||||
import { copyToJavaByteBuffer } from "../../native/java-direct-buffer";
|
||||
|
||||
declare const android: any;
|
||||
declare const androidx: any;
|
||||
@@ -93,8 +94,7 @@ async function downloadAndOpenPdf(url: string, appName: string, activity: any):
|
||||
const file = new java.io.File(directory, `policy-${hash.toString(16)}.pdf`);
|
||||
const stream = new java.io.FileOutputStream(file);
|
||||
try {
|
||||
const buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
stream.getChannel().write(buffer);
|
||||
stream.getChannel().write(copyToJavaByteBuffer(bytes));
|
||||
} finally {
|
||||
stream.close();
|
||||
}
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
* Terminal app, hosted in its own worker thread. Window model:
|
||||
* - "terminal:hub": the window opened from the launcher; shows the list of
|
||||
* live g2mirror sessions across every connected host (grouped by host when
|
||||
* more than one is connected), and hosts the Manage Connections section
|
||||
* more than one is connected; selecting a host's heading launches a shell
|
||||
* there), and hosts the Manage Connections section
|
||||
* where g2mirror:// connections are added, removed, and toggled.
|
||||
* - "terminal:view:N": opened by selecting a session in the hub; each has
|
||||
* its own websocket connection (the protocol allows one attached session
|
||||
@@ -1047,10 +1048,7 @@ function windowMenuItems(window: TerminalWindow): MenuItem[] {
|
||||
label,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
launchAndOpenView(control, preset).catch((error) => {
|
||||
// The hub status line also shows the server's error message.
|
||||
console.warn(`terminal launch ${preset} failed: ${error}`);
|
||||
});
|
||||
launchFromUi(control, preset);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1180,7 +1178,10 @@ function applyHistoryReply(window: ViewWindow, reply: { start: number; oldest: n
|
||||
|
||||
type HubItem = {
|
||||
label: string;
|
||||
/** Group heading (host name): drawn differently and skipped by selection. */
|
||||
/**
|
||||
* Group heading (host name): drawn unindented and dimmer. Selectable only
|
||||
* when it has an onSelect (launch a shell on that host).
|
||||
*/
|
||||
heading?: boolean;
|
||||
/** Session with recent output: an animated indicator marks the row. */
|
||||
active?: boolean;
|
||||
@@ -1210,7 +1211,7 @@ function hubMenu(window: HubWindow): Menu<HubItem> {
|
||||
function drawHubRow({ image, item, x, y, width, selected }: MenuDrawArgs<HubItem>): void {
|
||||
const font = chromeFont();
|
||||
if (item.heading) {
|
||||
image.drawText(font, x, y + HUB_ROW_TEXT_INSET, item.label, 140);
|
||||
image.drawText(font, x, y + HUB_ROW_TEXT_INSET, item.label, selected ? 255 : 140);
|
||||
return;
|
||||
}
|
||||
// Activity gutter, open-window number column, then the label, truncated
|
||||
@@ -1282,9 +1283,14 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
];
|
||||
const connected = connectedControls();
|
||||
const multiHost = connected.length > 1;
|
||||
const canLaunch = launchPresetNames().length > 0;
|
||||
for (const control of connected) {
|
||||
if (multiHost) {
|
||||
items.push({ label: connectionDisplayName(control.config), heading: true });
|
||||
items.push({
|
||||
label: connectionDisplayName(control.config),
|
||||
heading: true,
|
||||
onSelect: canLaunch ? () => launchOnHost(window, control) : undefined,
|
||||
});
|
||||
}
|
||||
const sessions = orderedSessions(window, control);
|
||||
for (const session of sessions) {
|
||||
@@ -1568,6 +1574,37 @@ function launchPresetNames(): string[] {
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* A host heading in the hub was selected: launch a shell there. With a single
|
||||
* launch preset it starts right away; with several, a submenu titled with the
|
||||
* host's name picks which.
|
||||
*/
|
||||
function launchOnHost(window: HubWindow, control: ControlConnection): void {
|
||||
const presets = launchPresetNames();
|
||||
if (presets.length === 1) {
|
||||
launchFromUi(control, presets[0]!);
|
||||
return;
|
||||
}
|
||||
windowMenu(window).open(
|
||||
presets.map((preset) => ({
|
||||
label: `Launch ${preset}`,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
launchFromUi(control, preset);
|
||||
},
|
||||
})),
|
||||
connectionDisplayName(control.config),
|
||||
);
|
||||
}
|
||||
|
||||
/** Launch started from a menu or list row; failures only get logged. */
|
||||
function launchFromUi(control: ControlConnection, preset: string): void {
|
||||
launchAndOpenView(control, preset).catch((error) => {
|
||||
// The hub status line also shows the server's error message.
|
||||
console.warn(`terminal launch ${preset} failed: ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
/** Launch a preset on a host and open a view window on the new session. */
|
||||
async function launchAndOpenView(control: ControlConnection, preset: string): Promise<string> {
|
||||
if (!control.client) throw new Error("Not connected to the g2mirror server.");
|
||||
|
||||
@@ -510,7 +510,7 @@ class DashboardController {
|
||||
window.relayout();
|
||||
await this.configureWindowSurface(
|
||||
window.surfaceId,
|
||||
window.windowId === foregroundWindowId,
|
||||
this.isForegroundWindow(window.windowId),
|
||||
window.heightMode,
|
||||
);
|
||||
} else if (window.closeable) {
|
||||
@@ -548,7 +548,7 @@ class DashboardController {
|
||||
void (async () => {
|
||||
for (const window of Array.from(shell.getWindows())) {
|
||||
if (!changed.includes(window.appId)) continue;
|
||||
await this.configureWindowSurface(window.surfaceId, window.windowId === shell.foregroundWindow()?.windowId, window.heightMode);
|
||||
await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode);
|
||||
window.relayout?.();
|
||||
}
|
||||
shell.foregroundWindow()?.requestRender();
|
||||
@@ -560,12 +560,11 @@ class DashboardController {
|
||||
const position = verticalPositionSetting.get();
|
||||
if (position === this.lastVerticalPosition) return;
|
||||
this.lastVerticalPosition = position;
|
||||
const foregroundWindowId = shell.foregroundWindow()?.windowId;
|
||||
void (async () => {
|
||||
for (const window of shell.getWindows()) {
|
||||
await this.configureWindowSurface(
|
||||
window.surfaceId,
|
||||
window.windowId === foregroundWindowId,
|
||||
this.isForegroundWindow(window.windowId),
|
||||
window.heightMode,
|
||||
);
|
||||
}
|
||||
@@ -1021,7 +1020,13 @@ class DashboardController {
|
||||
return this.glassesDisplayLabel();
|
||||
}
|
||||
if (this.silentMode && this.phase === "connected") {
|
||||
return "Connected (Silent mode enabled)";
|
||||
// Say how to leave it. Silent mode is entered and exited only by the
|
||||
// same gesture on the glasses, the firmware ignores all input and blanks
|
||||
// the display while it is on, and nothing here can turn it off -- there
|
||||
// is no setter, only onSilentMode. So a user who triggered it by
|
||||
// accident sees a connected pair of glasses that answers nothing, with
|
||||
// no way to find out why. The instruction belongs where they are looking.
|
||||
return "Connected (Silent mode — long-press both touchpads on the glasses to exit)";
|
||||
}
|
||||
// The preview compositor keeps the mirror live (including as a black
|
||||
// frame while the simulated screen is off), so never cover it.
|
||||
@@ -1243,11 +1248,10 @@ class DashboardController {
|
||||
// loop re-applies the current dim from here.
|
||||
await target.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1);
|
||||
this.appliedUnderlayDim = 1;
|
||||
const foregroundWindowId = shell.foregroundWindow()?.windowId;
|
||||
for (const window of shell.getWindows()) {
|
||||
await this.configureWindowSurface(
|
||||
window.surfaceId,
|
||||
window.windowId === foregroundWindowId,
|
||||
this.isForegroundWindow(window.windowId),
|
||||
window.heightMode,
|
||||
target,
|
||||
);
|
||||
@@ -1532,11 +1536,10 @@ class DashboardController {
|
||||
await communicator.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1);
|
||||
this.appliedUnderlayDim = 1;
|
||||
await this.configureLockSurface(communicator);
|
||||
const foregroundWindowId = shell.foregroundWindow()?.windowId;
|
||||
for (const window of shell.getWindows()) {
|
||||
await this.configureWindowSurface(
|
||||
window.surfaceId,
|
||||
window.windowId === foregroundWindowId,
|
||||
this.isForegroundWindow(window.windowId),
|
||||
window.heightMode,
|
||||
);
|
||||
}
|
||||
@@ -2270,8 +2273,7 @@ class DashboardController {
|
||||
reconfigureSurface: (heightMode) => {
|
||||
// Resize the surface rect to the new band; a foreground window stays
|
||||
// visible. The shell re-renders so the chrome (top bar) follows.
|
||||
const visible = shell.foregroundWindow()?.windowId === windowId;
|
||||
void this.configureWindowSurface(surfaceId, visible, heightMode);
|
||||
void this.configureWindowSurface(surfaceId, this.isForegroundWindow(windowId), heightMode);
|
||||
this.requestShellRender();
|
||||
},
|
||||
onClosed: () => {
|
||||
@@ -2424,10 +2426,21 @@ class DashboardController {
|
||||
}
|
||||
}
|
||||
|
||||
/** Create/refresh a window surface on the compositor, if a display target exists. */
|
||||
/**
|
||||
* Create/refresh a window surface on the compositor, if a display target exists.
|
||||
*
|
||||
* `visible` may be a function, asked only after the surface is configured,
|
||||
* right when the visibility call is queued. Callers that loop over windows
|
||||
* or resize one in place must pass one (isForegroundWindow): the configure
|
||||
* call yields, the foreground can change meanwhile, and a visibility decided
|
||||
* before the await then lands AFTER the focus change's own calls, since the
|
||||
* bridge runs Java calls in order. At startup, restoreOpenApps focuses
|
||||
* windows while connect() registers surfaces, so a foreground read before
|
||||
* that loop left the old window visible and the new foreground hidden.
|
||||
*/
|
||||
private async configureWindowSurface(
|
||||
surfaceId: string,
|
||||
visible: boolean,
|
||||
visible: boolean | (() => boolean),
|
||||
heightMode: WindowHeightMode = "min",
|
||||
// ensurePreviewDisplay passes its not-yet-published target explicitly.
|
||||
target: DisplayTarget | null = this.display,
|
||||
@@ -2438,7 +2451,12 @@ class DashboardController {
|
||||
zOrder: 0,
|
||||
transparency: "opaque",
|
||||
});
|
||||
await target.setSurfaceVisible(surfaceId, visible);
|
||||
await target.setSurfaceVisible(surfaceId, typeof visible === "function" ? visible() : visible);
|
||||
}
|
||||
|
||||
/** A late answer to "is this window in front?", for configureWindowSurface. */
|
||||
private isForegroundWindow(windowId: string): () => boolean {
|
||||
return () => shell.foregroundWindow()?.windowId === windowId;
|
||||
}
|
||||
|
||||
private removeWindowSurface(surfaceId: string): void {
|
||||
|
||||
@@ -1,18 +1,54 @@
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
// Reused Java-side buffers for this isolate's frames: passing a JS ArrayBuffer
|
||||
// to Java leaks it (java-direct-buffer.ts). Module state is per isolate, so
|
||||
// each worker gets its own pair.
|
||||
const framePixelsBuffer = new JavaDirectBuffer(640 * 480);
|
||||
const frameDrawsBuffer = new JavaDirectBuffer();
|
||||
|
||||
/**
|
||||
* The Java object a worker isolate submits surface frames to: the live BLE
|
||||
* The display a worker isolate submits surface frames to: the live BLE
|
||||
* communicator, or the preview-only compositor standing in when no glasses
|
||||
* are paired (both expose the same submitSurfaceFrame signature). Located
|
||||
* through Java statics because those are shared across isolates, unlike any
|
||||
* JS-side state.
|
||||
* JS-side state. Null when neither exists.
|
||||
*
|
||||
* Frame submission only. Buzzer effects use worker-buzzer, which routes to
|
||||
* the platform BLE session and skips playback when disconnected.
|
||||
*/
|
||||
export function getActiveDisplay(): any {
|
||||
export function getActiveDisplay(): typeof display | null {
|
||||
return activeJavaDisplay() ? display : null;
|
||||
}
|
||||
|
||||
function activeJavaDisplay(): any {
|
||||
return (
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive() ??
|
||||
com.faceclaw.app.FaceclawPreviewCompositor.getActive()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the same ArrayBuffer arguments as the iOS active display and copies
|
||||
* them into the reused Java buffers right before the synchronous Java call.
|
||||
*/
|
||||
const display = {
|
||||
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, x: number, y: number, width: number, height: number,
|
||||
fingerprint: string, paintMs: number, frameId: number, draws: ArrayBuffer | null = null): void {
|
||||
const target = activeJavaDisplay();
|
||||
if (!target) return;
|
||||
target.submitSurfaceFrame(
|
||||
framePixelsBuffer.load(new Uint8Array(buffer)),
|
||||
surfaceId,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
fingerprint,
|
||||
paintMs,
|
||||
frameId,
|
||||
frameDrawsBuffer.loadOptional(draws),
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { RingInput } from "../g2/ring-input";
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import * as frameTimings from "./frame-timings";
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -132,6 +133,13 @@ export class FaceclawCommunicatorBridge {
|
||||
private javaCallQueue: Promise<void> = Promise.resolve();
|
||||
/** Calls waiting on javaCallQueue; 0 means enqueueJavaCall's fast path is safe. */
|
||||
private queuedJavaCalls = 0;
|
||||
// Reused Java-side buffers for byte payloads; passing a JS ArrayBuffer to
|
||||
// Java leaks it (see java-direct-buffer.ts). Loaded inside the queued call,
|
||||
// immediately before the synchronous Java call that consumes them.
|
||||
private readonly framePixelsBuffer = new JavaDirectBuffer(640 * 480);
|
||||
private readonly frameDrawsBuffer = new JavaDirectBuffer();
|
||||
private readonly shellSceneBuffer = new JavaDirectBuffer();
|
||||
private readonly buzzerBuffer = new JavaDirectBuffer();
|
||||
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
|
||||
// Recent frame-finished outcomes from the Java side, so waitForFrameFinished
|
||||
// does not race against finishes that land before the wait starts.
|
||||
@@ -538,7 +546,10 @@ export class FaceclawCommunicatorBridge {
|
||||
*/
|
||||
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
|
||||
const snapshot = new Uint8Array(bytes);
|
||||
await this.enqueueJavaCall(() => this.communicator.submitShellScene(snapshot.buffer, paintMs, frameId), true);
|
||||
await this.enqueueJavaCall(
|
||||
() => this.communicator.submitShellScene(this.shellSceneBuffer.load(snapshot), paintMs, frameId),
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
|
||||
@@ -594,15 +605,17 @@ export class FaceclawCommunicatorBridge {
|
||||
*/
|
||||
glyphs: ArrayBuffer | null = null,
|
||||
): Promise<void> {
|
||||
// Snapshot because the Java call is deferred; the buffer is passed as an
|
||||
// ArrayBuffer, which NativeScript marshals to a ByteBuffer without the
|
||||
// ~150ms per-element copy a byte[] parameter would need.
|
||||
// Snapshot because the Java call is deferred. The bytes reach Java as a
|
||||
// ByteBuffer (no ~150ms per-element copy, as a byte[] parameter would
|
||||
// need), but through a reused Java direct buffer rather than by passing
|
||||
// the ArrayBuffer itself, which NativeScript never frees
|
||||
// (java-direct-buffer.ts).
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
// inlineWhenIdle: this is the frame path, the Java side of it measures
|
||||
// ~5ms (composite + pack), and the caller awaits it either way.
|
||||
await this.enqueueJavaCall(() => {
|
||||
this.communicator.submitSurfaceFrame(
|
||||
snapshot.buffer,
|
||||
this.framePixelsBuffer.load(snapshot),
|
||||
surfaceId,
|
||||
Math.round(rect.x),
|
||||
Math.round(rect.y),
|
||||
@@ -611,7 +624,7 @@ export class FaceclawCommunicatorBridge {
|
||||
fingerprint,
|
||||
Math.round(nonNegativeNumber(paintMs)),
|
||||
Math.round(nonNegativeNumber(frameId)),
|
||||
glyphs,
|
||||
this.frameDrawsBuffer.loadOptional(glyphs),
|
||||
);
|
||||
}, true);
|
||||
}
|
||||
@@ -673,7 +686,7 @@ export class FaceclawCommunicatorBridge {
|
||||
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
|
||||
const snapshot = new Uint8Array(payload);
|
||||
await this.enqueueJavaCall(() => {
|
||||
this.communicator.playBuzzerSequence(snapshot.buffer);
|
||||
this.communicator.playBuzzerSequence(this.buzzerBuffer.load(snapshot));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* reasonable for a sideloaded personal tool).
|
||||
*/
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { copyToJavaByteBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
@@ -150,10 +151,9 @@ export function writeBinaryFile(path: string, bytes: Uint8Array): boolean {
|
||||
if (parent) parent.mkdirs();
|
||||
const stream = new java.io.FileOutputStream(file);
|
||||
try {
|
||||
// An ArrayBuffer marshals to java.nio.ByteBuffer (the same conversion
|
||||
// submitSurfaceFrame relies on), so write through the channel.
|
||||
const buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
stream.getChannel().write(buffer);
|
||||
// Write through the channel from a Java direct buffer: a bulk copy, and
|
||||
// passing the ArrayBuffer itself would leak it (java-direct-buffer.ts).
|
||||
stream.getChannel().write(copyToJavaByteBuffer(bytes));
|
||||
} finally {
|
||||
stream.close();
|
||||
}
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import { copyToJavaByteBuffer } from './java-direct-buffer'
|
||||
|
||||
declare const com: any
|
||||
export function firmwareSha256(buffer: ArrayBuffer): string { return String(com.faceclaw.app.FaceclawFirmwareUtil.sha256Hex(buffer)) }
|
||||
export function writeFirmwareFile(path: string, buffer: ArrayBuffer): void { com.faceclaw.app.FaceclawFirmwareUtil.writeFile(path, buffer) }
|
||||
|
||||
// The image crosses as a fresh Java buffer: passing a JS ArrayBuffer to Java
|
||||
// leaks it (java-direct-buffer.ts).
|
||||
export function firmwareSha256(buffer: ArrayBuffer): string { return String(com.faceclaw.app.FaceclawFirmwareUtil.sha256Hex(copyToJavaByteBuffer(new Uint8Array(buffer)))) }
|
||||
export function writeFirmwareFile(path: string, buffer: ArrayBuffer): void { com.faceclaw.app.FaceclawFirmwareUtil.writeFile(path, copyToJavaByteBuffer(new Uint8Array(buffer))) }
|
||||
@@ -1,16 +1,17 @@
|
||||
import { ImageSource } from "@nativescript/core";
|
||||
|
||||
import { type GrayImage } from "../graphics/image";
|
||||
import { copyToJavaByteBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
|
||||
|
||||
/**
|
||||
* Build the phone-UI preview for a frame. The pixel buffer is passed to Java
|
||||
* as an ArrayBuffer (marshalled to a ByteBuffer) and expanded to ARGB there;
|
||||
* doing the per-pixel work in JS or copying element-by-element across the
|
||||
* bridge costs ~150ms per frame.
|
||||
* Build the phone-UI preview for a frame. The pixels are copied into a Java
|
||||
* direct ByteBuffer and expanded to ARGB there; doing the per-pixel work in JS
|
||||
* or copying element-by-element across the bridge costs ~150ms per frame, and
|
||||
* passing the ArrayBuffer itself would leak it (java-direct-buffer.ts).
|
||||
*/
|
||||
export function grayImageToPreviewSource(image: GrayImage): ImageSource | null {
|
||||
if (!global.isAndroid) {
|
||||
@@ -19,7 +20,7 @@ export function grayImageToPreviewSource(image: GrayImage): ImageSource | null {
|
||||
|
||||
const baked = image.withDrawsBaked();
|
||||
const bitmap = com.faceclaw.app.PreviewBitmapUtil.fromGray(
|
||||
baked.pixels.buffer,
|
||||
copyToJavaByteBuffer(baked.pixels),
|
||||
baked.width,
|
||||
baked.height,
|
||||
PREVIEW_BRIGHTEN_GAMMA,
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { grayImageFromPacket } from './image-files'
|
||||
import { type GrayImage } from '../graphics/image'
|
||||
import { copyToJavaByteBuffer } from './java-direct-buffer'
|
||||
declare const com: any
|
||||
|
||||
/** The encoded bytes cross as a fresh Java buffer; passing the ArrayBuffer leaks it (java-direct-buffer.ts). */
|
||||
export function decodeImageBytes(bytes: ArrayBuffer, width: number, height: number): GrayImage | null {
|
||||
return grayImageFromPacket(com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(bytes, width, height))
|
||||
return grayImageFromPacket(com.faceclaw.app.ImageFileLoader.loadGrayFromBytes(copyToJavaByteBuffer(new Uint8Array(bytes)), width, height))
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* Hands bytes to a Java `java.nio.ByteBuffer` parameter without leaking them.
|
||||
*
|
||||
* Why this exists: NativeScript Android (9.0.4 through at least 9.1.1)
|
||||
* converts a JS ArrayBuffer or typed array passed to a Java method by wrapping
|
||||
* its backing store in `NewDirectByteBuffer` and taking a JNI global ref that is
|
||||
* never deleted (JsArgConverter.cpp, `buffer = env.NewGlobalRef(buffer)`). The
|
||||
* ByteBuffer is also linked to the JS ArrayBuffer as its native counterpart.
|
||||
* The leaked global ref keeps the ByteBuffer reachable forever, so the
|
||||
* runtime's GC handshake (Runtime.makeInstanceWeakAndCheckIfAlive) always says
|
||||
* "alive", the JS finalizer re-arms, and V8 never frees the ArrayBuffer. Every
|
||||
* call leaked its whole buffer plus one global ref.
|
||||
*
|
||||
* On the frame path that meant a full surface (576x260, 640x452 or 640x480
|
||||
* bytes) per submitted frame, including frames the Java side then discards as
|
||||
* unchanged. Measured on a Galaxy Z Fold7, native-heap growth matched frame
|
||||
* bytes submitted to within a few percent, and Samsung's Heimdall memory
|
||||
* manager killed the app at about 6.4 GB. ART also aborts the process once
|
||||
* 51,200 global refs are held, so the ref leak is a second ceiling behind the
|
||||
* memory one.
|
||||
*
|
||||
* The workaround keeps the conversion out of the call: the bytes are copied
|
||||
* into a Java-allocated direct ByteBuffer that is reused, through an
|
||||
* ArrayBuffer view made once with `ArrayBuffer.from` (an external backing
|
||||
* store over the Java memory, no global ref), and the Java ByteBuffer object
|
||||
* itself is passed. A JS proxy for an existing Java object takes the ordinary
|
||||
* object path, which does not leak.
|
||||
*
|
||||
* Contract: the buffer returned by load() is only valid until the next load()
|
||||
* on the same instance, so the Java callee must consume it synchronously and
|
||||
* not keep a reference. The Kotlin consumers (SurfaceCompositor via
|
||||
* AndroidByteReader, playBuzzerSequence, ShellScene.decode) copy or parse
|
||||
* before returning, which satisfies this. One instance belongs to one JS
|
||||
* context (main thread or one worker); instances are not shared across
|
||||
* isolates.
|
||||
*/
|
||||
|
||||
declare const java: any;
|
||||
|
||||
/** Minimal surface of java.nio.ByteBuffer this module uses. */
|
||||
export interface JavaByteBufferLike {
|
||||
clear(): unknown;
|
||||
limit(newLimit: number): unknown;
|
||||
}
|
||||
|
||||
/** How to allocate a direct buffer and view its memory; injectable for tests. */
|
||||
export interface DirectBufferBackend {
|
||||
allocateDirect(capacity: number): JavaByteBufferLike;
|
||||
viewOf(buffer: JavaByteBufferLike): Uint8Array;
|
||||
}
|
||||
|
||||
const nativeScriptBackend: DirectBufferBackend = {
|
||||
allocateDirect: (capacity) => java.nio.ByteBuffer.allocateDirect(capacity),
|
||||
viewOf: (buffer) => new Uint8Array((ArrayBuffer as any).from(buffer)),
|
||||
};
|
||||
|
||||
/** Capacity grows in steps of this many bytes, so a slowly growing payload does not reallocate every call. */
|
||||
const CAPACITY_STEP_BYTES = 4096;
|
||||
|
||||
export class JavaDirectBuffer {
|
||||
private javaBuffer: JavaByteBufferLike | null = null;
|
||||
private view: Uint8Array | null = null;
|
||||
/** Number of Java buffers allocated so far (a test and diagnostics hook). */
|
||||
allocations = 0;
|
||||
|
||||
constructor(
|
||||
private readonly initialCapacity = 0,
|
||||
private readonly backend: DirectBufferBackend = nativeScriptBackend,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Copy `bytes` into the reused Java buffer and return it with position 0 and
|
||||
* limit `bytes.length`, so `remaining()` equals the byte count. Valid until
|
||||
* the next load() on this instance.
|
||||
*/
|
||||
load(bytes: Uint8Array): JavaByteBufferLike {
|
||||
if (!this.javaBuffer || !this.view || this.view.length < bytes.length) {
|
||||
const wanted = Math.max(bytes.length, this.initialCapacity, 1);
|
||||
const capacity = Math.ceil(wanted / CAPACITY_STEP_BYTES) * CAPACITY_STEP_BYTES;
|
||||
this.javaBuffer = this.backend.allocateDirect(capacity);
|
||||
this.view = this.backend.viewOf(this.javaBuffer);
|
||||
this.allocations++;
|
||||
}
|
||||
this.view.set(bytes);
|
||||
this.javaBuffer.clear();
|
||||
this.javaBuffer.limit(bytes.length);
|
||||
return this.javaBuffer;
|
||||
}
|
||||
|
||||
/** As load(), for an optional ArrayBuffer (e.g. a frame's draw list); null stays null. */
|
||||
loadOptional(buffer: ArrayBuffer | null | undefined): JavaByteBufferLike | null {
|
||||
if (!buffer || buffer.byteLength === 0) return null;
|
||||
return this.load(new Uint8Array(buffer));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy `bytes` into a fresh Java direct buffer (position 0, limit
|
||||
* `bytes.length`) for a one-off crossing such as an atlas registration or a
|
||||
* file write. Unlike a JavaDirectBuffer nothing is pinned between calls: the
|
||||
* Java buffer is an ordinary object, freed once the JS side drops it.
|
||||
*/
|
||||
export function copyToJavaByteBuffer(
|
||||
bytes: Uint8Array,
|
||||
backend: DirectBufferBackend = nativeScriptBackend,
|
||||
): JavaByteBufferLike {
|
||||
// A zero-capacity direct buffer may have no address to view.
|
||||
const javaBuffer = backend.allocateDirect(Math.max(bytes.length, 1));
|
||||
backend.viewOf(javaBuffer).set(bytes);
|
||||
javaBuffer.limit(bytes.length);
|
||||
return javaBuffer;
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { copyToJavaByteBuffer } from './java-direct-buffer'
|
||||
|
||||
declare const com: any
|
||||
|
||||
/** Development-only checks of NativeScript metadata and bulk ByteBuffer crossings. */
|
||||
@@ -10,11 +12,11 @@ export function runKotlinProtocolSmokeTest(): void {
|
||||
const compositor = new native.SurfaceCompositor()
|
||||
compositor.configureScreen(3, 2)
|
||||
compositor.configureSurface('test', 0, 0, 3, 2, 0, 0)
|
||||
const pixels = new Uint8Array([0, 16, 255, 32, 48, 64])
|
||||
compositor.submitSurface('test', new native.AndroidByteReader(pixels.buffer), 0, 0, 3, 2, '1')
|
||||
const pixels = copyToJavaByteBuffer(new Uint8Array([0, 16, 255, 32, 48, 64]))
|
||||
compositor.submitSurface('test', new native.AndroidByteReader(pixels), 0, 0, 3, 2, '1')
|
||||
assert(hex(compositor.composite().gray) === '0010ff203040', 'composition/ByteBuffer')
|
||||
assert(hex(native.BmpUtil.pack4bppFromGray8(compositor.composite().gray, 3, 2)) === '01f02340', 'packing')
|
||||
const id = native.ImageAtlas.ensure('kotlin-smoke', 3, 2, new native.AndroidByteReader(pixels.buffer))
|
||||
const id = native.ImageAtlas.ensure('kotlin-smoke', 3, 2, new native.AndroidByteReader(pixels))
|
||||
assert(id > 0 && native.ImageAtlas.get(id).width === 3, 'image atlas/reused buffer')
|
||||
console.log('FACECLAW_KOTLIN_PROTOCOL_PASS Android')
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import { dimFactor256, type FaceclawCommunicatorBridge, type SurfaceOptions } from "./faceclaw-communicator";
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
@@ -49,6 +50,11 @@ function nonNegativeNumber(value: number): number {
|
||||
*/
|
||||
export class PreviewDisplayTarget implements DisplayTarget {
|
||||
private readonly compositor: any;
|
||||
// Reused Java-side buffers: passing a JS ArrayBuffer to Java leaks it
|
||||
// (java-direct-buffer.ts).
|
||||
private readonly framePixelsBuffer = new JavaDirectBuffer(640 * 480);
|
||||
private readonly frameDrawsBuffer = new JavaDirectBuffer();
|
||||
private readonly shellSceneBuffer = new JavaDirectBuffer();
|
||||
|
||||
constructor() {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
@@ -104,7 +110,7 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
||||
}
|
||||
|
||||
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
|
||||
this.compositor.submitShellScene(new Uint8Array(bytes).buffer, paintMs, frameId);
|
||||
this.compositor.submitShellScene(this.shellSceneBuffer.load(bytes), paintMs, frameId);
|
||||
}
|
||||
|
||||
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
|
||||
@@ -124,11 +130,11 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
||||
frameId = 0,
|
||||
glyphs: ArrayBuffer | null = null,
|
||||
): Promise<void> {
|
||||
// Copy for the same reason the bridge does: the source may be a view on a
|
||||
// larger buffer, and Java receives the backing ArrayBuffer.
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
// Copied into a reused Java direct buffer, sized to exactly this view's
|
||||
// bytes; passing the ArrayBuffer itself would leak it
|
||||
// (java-direct-buffer.ts).
|
||||
this.compositor.submitSurfaceFrame(
|
||||
snapshot.buffer,
|
||||
this.framePixelsBuffer.load(pixels8bpp),
|
||||
surfaceId,
|
||||
Math.round(rect.x),
|
||||
Math.round(rect.y),
|
||||
@@ -137,7 +143,7 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
||||
fingerprint,
|
||||
Math.round(nonNegativeNumber(paintMs)),
|
||||
Math.round(nonNegativeNumber(frameId)),
|
||||
glyphs,
|
||||
this.frameDrawsBuffer.loadOptional(glyphs),
|
||||
);
|
||||
// The Java side finishes the frame: the composite is the end of the line,
|
||||
// and worker isolates submit to the same object without coming through here.
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
import { copyToJavaByteBuffer } from './java-direct-buffer'
|
||||
|
||||
declare const com: any
|
||||
|
||||
// Byte payloads cross as fresh Java buffers: passing a JS ArrayBuffer to Java
|
||||
// leaks it (java-direct-buffer.ts).
|
||||
|
||||
/** Android bridge to the process-wide shared Kotlin atlases. */
|
||||
export function textureAtlasAvailable(): boolean { return !!global.isAndroid }
|
||||
export function textureFontId(key: string): number { return com.faceclaw.app.GlyphAtlas.fontId(key) }
|
||||
export function registerTextureGlyphs(buffer: ArrayBuffer, aa: boolean): void {
|
||||
const reader = new com.faceclaw.app.AndroidByteReader(buffer)
|
||||
const reader = new com.faceclaw.app.AndroidByteReader(copyToJavaByteBuffer(new Uint8Array(buffer)))
|
||||
if (aa) com.faceclaw.app.GlyphAtlas.registerAa(reader)
|
||||
else com.faceclaw.app.GlyphAtlas.register(reader)
|
||||
}
|
||||
export function registerFirmwareGlyphs(buffer: ArrayBuffer): void {
|
||||
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(buffer))
|
||||
com.faceclaw.app.FwGlyphAtlas.register(new com.faceclaw.app.AndroidByteReader(copyToJavaByteBuffer(new Uint8Array(buffer))))
|
||||
}
|
||||
export function textureImageId(key: string, width: number, height: number, pixels: Uint8Array): number {
|
||||
const buffer = pixels.byteOffset === 0 && pixels.byteLength === pixels.buffer.byteLength ? pixels.buffer : pixels.slice().buffer
|
||||
return com.faceclaw.app.ImageAtlas.ensure(key, width, height, new com.faceclaw.app.AndroidByteReader(buffer))
|
||||
return com.faceclaw.app.ImageAtlas.ensure(key, width, height, new com.faceclaw.app.AndroidByteReader(copyToJavaByteBuffer(pixels)))
|
||||
}
|
||||
@@ -1,12 +1,17 @@
|
||||
import type { WorkerAppReply } from "../ui/shell/worker-window";
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
// Reused Java-side buffer for this isolate's tone payloads: passing a JS
|
||||
// ArrayBuffer to Java leaks it (java-direct-buffer.ts).
|
||||
const buzzerBuffer = new JavaDirectBuffer();
|
||||
|
||||
/** Workers use the main-thread BLE session on iOS and the native queue on Android. */
|
||||
export function playWorkerBuzzerSequence(payload: Uint8Array): void {
|
||||
if (global.isIOS) {
|
||||
global.postMessage({ type: "buzzer-sequence", payload: Array.from(payload) } satisfies WorkerAppReply);
|
||||
} else {
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive()?.playBuzzerSequence(new Uint8Array(payload).buffer);
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive()?.playBuzzerSequence(buzzerBuffer.load(payload));
|
||||
}
|
||||
}
|
||||
@@ -52,11 +52,19 @@ export class ShellModalLayer implements Layer {
|
||||
this.stack.setBaseSize(interior);
|
||||
// Flattening bakes the inner stack's planes (glyphs included) so the blit
|
||||
// below transplants the finished modal content into this layer's plane.
|
||||
const inner = flattenPlanes(this.stack.paint(), interior);
|
||||
// The top plane's retained presentations (a menu's selection highlight
|
||||
// and scroll strip) stay unbaked: they are display lists the glasses
|
||||
// animate, and baking one freezes it on its first frame. Lower planes'
|
||||
// are baked so the top plane's raster still covers them.
|
||||
const planes = this.stack.paint();
|
||||
const top = planes[planes.length - 1]!;
|
||||
const inner = flattenPlanes(planes.map((plane) => plane === top ? { ...plane, image: plane.image.withDrawsBaked(false) } : plane), interior);
|
||||
const rect = modalRect();
|
||||
const innerX = rect.x + MODAL_PADDING, innerY = rect.y + MODAL_PADDING;
|
||||
image.fillRect(rect.x, rect.y, rect.width, rect.height, SHELL_OPAQUE_BLACK);
|
||||
image.drawRect(rect.x, rect.y, rect.width, rect.height, 110);
|
||||
image.bitBlt(inner, rect.x + MODAL_PADDING, rect.y + MODAL_PADDING, { transparentZero: true });
|
||||
image.bitBlt(inner, innerX, innerY, { transparentZero: true });
|
||||
top.image.copyPresentationsInto(image, innerX + top.x, innerY + top.y);
|
||||
return image;
|
||||
}
|
||||
|
||||
|
||||
@@ -92,10 +92,10 @@ export class WindowMenu {
|
||||
|
||||
/**
|
||||
* Open the menu, or the shell's system menu when this window has no
|
||||
* entries. `items` overrides the window's own entries for an app that
|
||||
* reuses this host for another list (a per-row action menu).
|
||||
* entries. `items` (and optionally `title`) override the window's own for
|
||||
* an app that reuses this host for another list (a per-row action menu).
|
||||
*/
|
||||
open(items: MenuItem[] = this.options.items()): void {
|
||||
open(items: MenuItem[] = this.options.items(), title: string = this.options.title()): void {
|
||||
if (this.stack) return;
|
||||
if (!items.length) {
|
||||
this.options.post({ type: "open-system-menu", windowId: this.options.windowId });
|
||||
@@ -106,7 +106,7 @@ export class WindowMenu {
|
||||
handleInput: () => {},
|
||||
};
|
||||
const stack = new LayerStack(base, { ...noopLayerActions }, this.options.size, this.options.isFocused);
|
||||
stack.push(new WindowMenuLayer(this.options.title(), items));
|
||||
stack.push(new WindowMenuLayer(title, items));
|
||||
this.stack = stack;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user