Massively improve framerate

This commit is contained in:
James Babcock committed 2026-07-04 12:29:50 -07:00
1 parent 630b511d32
commit c9fe92dc1c
6 files changed
+73 -92

No files matched your search

+4 -14
View File
@@ -9,7 +9,7 @@ import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout-bridge";
import { onAndroidNotificationPosted } from "../native/notification-icons";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { grayImageToPreviewSource, grayImageToPreviewSourceWithBitmapFactory } from "../native/gray-image-preview";
import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { voiceControlBridge } from "../native/voice-control";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import {
@@ -145,7 +145,6 @@ class DashboardController {
private queuedFrameId = 0;
private lastForegroundNotificationUpdateAtMs = 0;
private lastConnectedPreviewUpdateAtMs = 0;
private connectedPreviewUpdateSeq = 0;
constructor() {
setDashboardActions({
@@ -608,7 +607,7 @@ class DashboardController {
await this.communicator.setG2ScreenOn(false);
}
if (updatePreviewAfterTransmit) {
await this.updateConnectedDisplayPreviewFromImage(image);
this.updateConnectedDisplayPreviewFromImage(image);
}
} else {
frameTimings.finishFrame(frameId, "discarded: no communicator, preview only");
@@ -829,7 +828,7 @@ class DashboardController {
this.setDisplayPreview(grayImageToPreviewSource(image));
}
private async updateConnectedDisplayPreviewFromImage(image: GrayImage): Promise<void> {
private updateConnectedDisplayPreviewFromImage(image: GrayImage): void {
if (!this.communicator) return;
const now = Date.now();
if (
@@ -839,16 +838,7 @@ class DashboardController {
return;
}
this.lastConnectedPreviewUpdateAtMs = now;
const previewUpdateId = ++this.connectedPreviewUpdateSeq;
console.log(`[preview#${previewUpdateId}] create start`);
const preview = await grayImageToPreviewSourceWithBitmapFactory(
image,
(createColors, width, height) =>
this.communicator!.createPreviewBitmapWithColorFactory(createColors, width, height),
);
console.log(`[preview#${previewUpdateId}] create done; setDisplayPreview start`);
this.setDisplayPreview(preview);
console.log(`[preview#${previewUpdateId}] setDisplayPreview done`);
this.updateDisplayPreviewFromImage(image);
}
private formatError(error: unknown): string {
+4 -27
View File
@@ -52,15 +52,6 @@ export type RawInputEvent =
frameId: number;
};
function toJavaByteArray(bytes: Uint8Array): number[] {
const out = Array.create("byte", bytes.length) as number[];
for (let i = 0; i < bytes.length; i++) {
const value = bytes[i]!;
out[i] = value > 127 ? value - 256 : value;
}
return out;
}
function nonNegativeNumber(value: number): number {
const numeric = Number(value);
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
@@ -300,11 +291,13 @@ export class FaceclawCommunicatorBridge {
}
async submitDashboardImage(image8bpp: Uint8Array, width: number, height: number, fingerprint: string, paintMs = -1, frameId = 0): Promise<void> {
// Snapshot because the Java call is deferred; the buffer is passed as an
// ArrayBuffer, which NativeScript marshals to a ByteBuffer without the
// ~150ms per-element copy a byte[] parameter would need.
const snapshot = new Uint8Array(image8bpp);
await this.enqueueJavaCall(() => {
const javaBytes = frameTimings.span(frameId, "js-to-java-copy", () => toJavaByteArray(snapshot));
this.communicator.submitDashboardImage(
javaBytes,
snapshot.buffer,
Math.round(width),
Math.round(height),
fingerprint,
@@ -314,22 +307,6 @@ export class FaceclawCommunicatorBridge {
});
}
async createPreviewBitmap(colors: number[], width: number, height: number): Promise<any> {
return this.enqueueJavaCall(() =>
this.communicator.createPreviewBitmap(colors, width, height),
);
}
async createPreviewBitmapWithColorFactory(
createColors: () => number[],
width: number,
height: number,
): Promise<any> {
return this.enqueueJavaCall(() =>
this.communicator.createPreviewBitmap(createColors(), width, height),
);
}
async disconnect(): Promise<void> {
await this.enqueueJavaCall(() => this.communicator.disconnect());
}
+12 -37
View File
@@ -2,51 +2,26 @@ import { ImageSource } from "@nativescript/core";
import { type GrayImage } from "../graphics/image";
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
const PREVIEW_COLOR_LUT = buildPreviewColorLookup();
declare const com: any;
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
/**
* Build the phone-UI preview for a frame. The pixel buffer is passed to Java
* as an ArrayBuffer (marshalled to a ByteBuffer) and expanded to ARGB there;
* doing the per-pixel work in JS or copying element-by-element across the
* bridge costs ~150ms per frame.
*/
export function grayImageToPreviewSource(image: GrayImage): ImageSource | null {
if (!global.isAndroid) {
return null;
}
const colors = grayImageToPreviewColors(image);
const bitmap = android.graphics.Bitmap.createBitmap(
colors,
const bitmap = com.faceclaw.app.PreviewBitmapUtil.fromGray(
image.pixels.buffer,
image.width,
image.height,
android.graphics.Bitmap.Config.ARGB_8888,
PREVIEW_BRIGHTEN_GAMMA,
);
return new ImageSource(bitmap);
}
export async function grayImageToPreviewSourceWithBitmapFactory(
image: GrayImage,
createBitmap: (createColors: () => number[], width: number, height: number) => Promise<any>,
): Promise<ImageSource | null> {
if (!global.isAndroid) {
return null;
}
const bitmap = await createBitmap(() => grayImageToPreviewColors(image), image.width, image.height);
return new ImageSource(bitmap);
}
function grayImageToPreviewColors(image: GrayImage): number[] {
const colors = Array.create("int", image.width * image.height) as number[];
for (let i = 0; i < image.pixels.length; i++) {
colors[i] = PREVIEW_COLOR_LUT[image.pixels[i] ?? 0]!;
}
return colors;
}
function buildPreviewColorLookup(): number[] {
const colors: number[] = [];
for (let gray = 0; gray <= 255; gray++) {
const normalized = gray / 255;
const preview = Math.max(0, Math.min(255, Math.round(255 * Math.pow(normalized, PREVIEW_BRIGHTEN_GAMMA))));
colors[gray] = (255 << 24) | (preview << 16) | (preview << 8) | preview;
}
return colors;
}