From c9fe92dc1c3410009c6e906056723875d29c69da Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sat, 4 Jul 2026 12:29:50 -0700 Subject: [PATCH] Massively improve framerate --- .../faceclaw/app/FaceclawBleCommunicator.java | 27 +++++----- .../com/faceclaw/app/PreviewBitmapUtil.java | 34 +++++++++++++ .../app/g2protocol/BleImageOptimizer.java | 6 ++- app/g2/dashboard-controller.ts | 18 ++----- app/native/faceclaw-communicator.ts | 31 ++---------- app/native/gray-image-preview.ts | 49 +++++-------------- 6 files changed, 73 insertions(+), 92 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 8d6e2fd6..f5ff5e45 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -4,7 +4,6 @@ import android.annotation.SuppressLint; import android.bluetooth.BluetoothGatt; import android.bluetooth.BluetoothGattCharacteristic; import android.content.Context; -import android.graphics.Bitmap; import android.os.Handler; import android.os.Looper; import android.os.PowerManager; @@ -226,8 +225,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } + /** + * image8bpp arrives as a ByteBuffer because NativeScript marshals a JS + * ArrayBuffer to one without the per-element bridge copy that a byte[] + * parameter would need (~150ms for a full frame). + */ public void submitDashboardImage( - byte[] image8bpp, + java.nio.ByteBuffer image8bpp, int width, int height, String fingerprint, @@ -238,7 +242,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { // Convert the full-screen 8bpp grayscale buffer into the BMP wire format here so // that all framing concerns live on the Java side. FrameTimings.getInstance().spanStart(frameId, "bmp-convert"); - byte[] bmp = image8bpp == null ? new byte[0] : BmpUtil.build4bppBmp(image8bpp, width, height); + byte[] gray; + if (image8bpp == null) { + gray = new byte[0]; + } else { + gray = new byte[image8bpp.remaining()]; + image8bpp.get(gray); + } + byte[] bmp = gray.length == 0 ? new byte[0] : BmpUtil.build4bppBmp(gray, width, height); FrameTimings.getInstance().spanEnd(frameId, "bmp-convert"); int supersededFrameId; synchronized (desiredTilesLock) { @@ -279,16 +290,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { interruptibleSleep.interrupt(); } - public Bitmap createPreviewBitmap(int[] colors, int width, int height) { - if (colors == null) { - throw new IllegalArgumentException("colors are required"); - } - if (width <= 0 || height <= 0 || colors.length < width * height) { - throw new IllegalArgumentException("invalid preview bitmap dimensions"); - } - return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); - } - public boolean sendShutdown(int exitMode) { int magic; long startedAtMs = SystemClock.elapsedRealtime(); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java new file mode 100644 index 00000000..fd1de064 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java @@ -0,0 +1,34 @@ +package com.faceclaw.app; + +import android.graphics.Bitmap; + +import java.nio.ByteBuffer; + +/** + * Builds the phone-UI preview bitmap from an 8bpp grayscale frame. The + * grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS + * ArrayBuffer to one without copying element-by-element); doing the + * gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the + * JS/Java bridge, where it used to cost ~150ms per preview. + */ +public final class PreviewBitmapUtil { + private PreviewBitmapUtil() {} + + public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) { + if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { + throw new IllegalArgumentException("invalid gray preview buffer"); + } + int[] lut = new int[256]; + for (int g = 0; g < 256; g++) { + int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))); + lut[g] = 0xff000000 | (v << 16) | (v << 8) | v; + } + byte[] bytes = new byte[width * height]; + gray.get(bytes); + int[] colors = new int[width * height]; + for (int i = 0; i < colors.length; i++) { + colors[i] = lut[bytes[i] & 0xff]; + } + return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java index 77f7a54b..76d7a874 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleImageOptimizer.java @@ -111,7 +111,11 @@ public final class BleImageOptimizer { } private static byte[] deflate(byte[] data) { - Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION); + // Level 6: BEST_COMPRESSION cost 18-109ms per frame on the BLE worker, + // but BEST_SPEED inflated typical payloads from ~2.7KB past the 3800-byte + // fragment boundary, adding a whole extra ack round trip (~350ms). The + // default level keeps payloads under one fragment at about half the CPU. + Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION); deflater.setInput(data); deflater.finish(); ByteArrayOutputStream out = new ByteArrayOutputStream(Math.max(64, data.length / 3)); diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index d81a5c6e..686d0875 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -9,7 +9,7 @@ import { mediaControllerBridge } from "../native/media-controller"; import { nightscoutBridge } from "../native/nightscout-bridge"; import { onAndroidNotificationPosted } from "../native/notification-icons"; import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict"; -import { grayImageToPreviewSource, grayImageToPreviewSourceWithBitmapFactory } from "../native/gray-image-preview"; +import { grayImageToPreviewSource } from "../native/gray-image-preview"; import { voiceControlBridge } from "../native/voice-control"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; import { @@ -145,7 +145,6 @@ class DashboardController { private queuedFrameId = 0; private lastForegroundNotificationUpdateAtMs = 0; private lastConnectedPreviewUpdateAtMs = 0; - private connectedPreviewUpdateSeq = 0; constructor() { setDashboardActions({ @@ -608,7 +607,7 @@ class DashboardController { await this.communicator.setG2ScreenOn(false); } if (updatePreviewAfterTransmit) { - await this.updateConnectedDisplayPreviewFromImage(image); + this.updateConnectedDisplayPreviewFromImage(image); } } else { frameTimings.finishFrame(frameId, "discarded: no communicator, preview only"); @@ -829,7 +828,7 @@ class DashboardController { this.setDisplayPreview(grayImageToPreviewSource(image)); } - private async updateConnectedDisplayPreviewFromImage(image: GrayImage): Promise { + private updateConnectedDisplayPreviewFromImage(image: GrayImage): void { if (!this.communicator) return; const now = Date.now(); if ( @@ -839,16 +838,7 @@ class DashboardController { return; } this.lastConnectedPreviewUpdateAtMs = now; - const previewUpdateId = ++this.connectedPreviewUpdateSeq; - console.log(`[preview#${previewUpdateId}] create start`); - const preview = await grayImageToPreviewSourceWithBitmapFactory( - image, - (createColors, width, height) => - this.communicator!.createPreviewBitmapWithColorFactory(createColors, width, height), - ); - console.log(`[preview#${previewUpdateId}] create done; setDisplayPreview start`); - this.setDisplayPreview(preview); - console.log(`[preview#${previewUpdateId}] setDisplayPreview done`); + this.updateDisplayPreviewFromImage(image); } private formatError(error: unknown): string { diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index edbb7a5c..6f3855ca 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -52,15 +52,6 @@ export type RawInputEvent = frameId: number; }; -function toJavaByteArray(bytes: Uint8Array): number[] { - const out = Array.create("byte", bytes.length) as number[]; - for (let i = 0; i < bytes.length; i++) { - const value = bytes[i]!; - out[i] = value > 127 ? value - 256 : value; - } - return out; -} - function nonNegativeNumber(value: number): number { const numeric = Number(value); return Number.isFinite(numeric) ? Math.max(0, numeric) : 0; @@ -300,11 +291,13 @@ export class FaceclawCommunicatorBridge { } async submitDashboardImage(image8bpp: Uint8Array, width: number, height: number, fingerprint: string, paintMs = -1, frameId = 0): Promise { + // Snapshot because the Java call is deferred; the buffer is passed as an + // ArrayBuffer, which NativeScript marshals to a ByteBuffer without the + // ~150ms per-element copy a byte[] parameter would need. const snapshot = new Uint8Array(image8bpp); await this.enqueueJavaCall(() => { - const javaBytes = frameTimings.span(frameId, "js-to-java-copy", () => toJavaByteArray(snapshot)); this.communicator.submitDashboardImage( - javaBytes, + snapshot.buffer, Math.round(width), Math.round(height), fingerprint, @@ -314,22 +307,6 @@ export class FaceclawCommunicatorBridge { }); } - async createPreviewBitmap(colors: number[], width: number, height: number): Promise { - return this.enqueueJavaCall(() => - this.communicator.createPreviewBitmap(colors, width, height), - ); - } - - async createPreviewBitmapWithColorFactory( - createColors: () => number[], - width: number, - height: number, - ): Promise { - return this.enqueueJavaCall(() => - this.communicator.createPreviewBitmap(createColors(), width, height), - ); - } - async disconnect(): Promise { await this.enqueueJavaCall(() => this.communicator.disconnect()); } diff --git a/app/native/gray-image-preview.ts b/app/native/gray-image-preview.ts index fe5229ac..06eb95f0 100644 --- a/app/native/gray-image-preview.ts +++ b/app/native/gray-image-preview.ts @@ -2,51 +2,26 @@ import { ImageSource } from "@nativescript/core"; import { type GrayImage } from "../graphics/image"; -const PREVIEW_BRIGHTEN_GAMMA = 0.7; -const PREVIEW_COLOR_LUT = buildPreviewColorLookup(); +declare const com: any; +const PREVIEW_BRIGHTEN_GAMMA = 0.7; + +/** + * Build the phone-UI preview for a frame. The pixel buffer is passed to Java + * as an ArrayBuffer (marshalled to a ByteBuffer) and expanded to ARGB there; + * doing the per-pixel work in JS or copying element-by-element across the + * bridge costs ~150ms per frame. + */ export function grayImageToPreviewSource(image: GrayImage): ImageSource | null { if (!global.isAndroid) { return null; } - const colors = grayImageToPreviewColors(image); - - const bitmap = android.graphics.Bitmap.createBitmap( - colors, + const bitmap = com.faceclaw.app.PreviewBitmapUtil.fromGray( + image.pixels.buffer, image.width, image.height, - android.graphics.Bitmap.Config.ARGB_8888, + PREVIEW_BRIGHTEN_GAMMA, ); return new ImageSource(bitmap); } - -export async function grayImageToPreviewSourceWithBitmapFactory( - image: GrayImage, - createBitmap: (createColors: () => number[], width: number, height: number) => Promise, -): Promise { - if (!global.isAndroid) { - return null; - } - - const bitmap = await createBitmap(() => grayImageToPreviewColors(image), image.width, image.height); - return new ImageSource(bitmap); -} - -function grayImageToPreviewColors(image: GrayImage): number[] { - const colors = Array.create("int", image.width * image.height) as number[]; - for (let i = 0; i < image.pixels.length; i++) { - colors[i] = PREVIEW_COLOR_LUT[image.pixels[i] ?? 0]!; - } - return colors; -} - -function buildPreviewColorLookup(): number[] { - const colors: number[] = []; - for (let gray = 0; gray <= 255; gray++) { - const normalized = gray / 255; - const preview = Math.max(0, Math.min(255, Math.round(255 * Math.pow(normalized, PREVIEW_BRIGHTEN_GAMMA)))); - colors[gray] = (255 << 24) | (preview << 16) | (preview << 8) | preview; - } - return colors; -}