forked from Blendman974/faceclaw
Massively improve framerate
This commit is contained in:
1 parent
630b511d32
commit
c9fe92dc1c
6 files changed
+73
-92
No files matched your search
@@ -4,7 +4,6 @@ import android.annotation.SuppressLint;
|
||||
import android.bluetooth.BluetoothGatt;
|
||||
import android.bluetooth.BluetoothGattCharacteristic;
|
||||
import android.content.Context;
|
||||
import android.graphics.Bitmap;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.os.PowerManager;
|
||||
@@ -226,8 +225,13 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* image8bpp arrives as a ByteBuffer because NativeScript marshals a JS
|
||||
* ArrayBuffer to one without the per-element bridge copy that a byte[]
|
||||
* parameter would need (~150ms for a full frame).
|
||||
*/
|
||||
public void submitDashboardImage(
|
||||
byte[] image8bpp,
|
||||
java.nio.ByteBuffer image8bpp,
|
||||
int width,
|
||||
int height,
|
||||
String fingerprint,
|
||||
@@ -238,7 +242,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
// Convert the full-screen 8bpp grayscale buffer into the BMP wire format here so
|
||||
// that all framing concerns live on the Java side.
|
||||
FrameTimings.getInstance().spanStart(frameId, "bmp-convert");
|
||||
byte[] bmp = image8bpp == null ? new byte[0] : BmpUtil.build4bppBmp(image8bpp, width, height);
|
||||
byte[] gray;
|
||||
if (image8bpp == null) {
|
||||
gray = new byte[0];
|
||||
} else {
|
||||
gray = new byte[image8bpp.remaining()];
|
||||
image8bpp.get(gray);
|
||||
}
|
||||
byte[] bmp = gray.length == 0 ? new byte[0] : BmpUtil.build4bppBmp(gray, width, height);
|
||||
FrameTimings.getInstance().spanEnd(frameId, "bmp-convert");
|
||||
int supersededFrameId;
|
||||
synchronized (desiredTilesLock) {
|
||||
@@ -279,16 +290,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
interruptibleSleep.interrupt();
|
||||
}
|
||||
|
||||
public Bitmap createPreviewBitmap(int[] colors, int width, int height) {
|
||||
if (colors == null) {
|
||||
throw new IllegalArgumentException("colors are required");
|
||||
}
|
||||
if (width <= 0 || height <= 0 || colors.length < width * height) {
|
||||
throw new IllegalArgumentException("invalid preview bitmap dimensions");
|
||||
}
|
||||
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
|
||||
}
|
||||
|
||||
public boolean sendShutdown(int exitMode) {
|
||||
int magic;
|
||||
long startedAtMs = SystemClock.elapsedRealtime();
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Builds the phone-UI preview bitmap from an 8bpp grayscale frame. The
|
||||
* grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS
|
||||
* ArrayBuffer to one without copying element-by-element); doing the
|
||||
* gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the
|
||||
* JS/Java bridge, where it used to cost ~150ms per preview.
|
||||
*/
|
||||
public final class PreviewBitmapUtil {
|
||||
private PreviewBitmapUtil() {}
|
||||
|
||||
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) {
|
||||
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
|
||||
throw new IllegalArgumentException("invalid gray preview buffer");
|
||||
}
|
||||
int[] lut = new int[256];
|
||||
for (int g = 0; g < 256; g++) {
|
||||
int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma))));
|
||||
lut[g] = 0xff000000 | (v << 16) | (v << 8) | v;
|
||||
}
|
||||
byte[] bytes = new byte[width * height];
|
||||
gray.get(bytes);
|
||||
int[] colors = new int[width * height];
|
||||
for (int i = 0; i < colors.length; i++) {
|
||||
colors[i] = lut[bytes[i] & 0xff];
|
||||
}
|
||||
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -111,7 +111,11 @@ public final class BleImageOptimizer {
|
||||
}
|
||||
|
||||
private static byte[] deflate(byte[] data) {
|
||||
Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION);
|
||||
// Level 6: BEST_COMPRESSION cost 18-109ms per frame on the BLE worker,
|
||||
// but BEST_SPEED inflated typical payloads from ~2.7KB past the 3800-byte
|
||||
// fragment boundary, adding a whole extra ack round trip (~350ms). The
|
||||
// default level keeps payloads under one fragment at about half the CPU.
|
||||
Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION);
|
||||
deflater.setInput(data);
|
||||
deflater.finish();
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream(Math.max(64, data.length / 3));
|
||||
|
||||
@@ -9,7 +9,7 @@ import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { nightscoutBridge } from "../native/nightscout-bridge";
|
||||
import { onAndroidNotificationPosted } from "../native/notification-icons";
|
||||
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
|
||||
import { grayImageToPreviewSource, grayImageToPreviewSourceWithBitmapFactory } from "../native/gray-image-preview";
|
||||
import { grayImageToPreviewSource } from "../native/gray-image-preview";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import {
|
||||
@@ -145,7 +145,6 @@ class DashboardController {
|
||||
private queuedFrameId = 0;
|
||||
private lastForegroundNotificationUpdateAtMs = 0;
|
||||
private lastConnectedPreviewUpdateAtMs = 0;
|
||||
private connectedPreviewUpdateSeq = 0;
|
||||
|
||||
constructor() {
|
||||
setDashboardActions({
|
||||
@@ -608,7 +607,7 @@ class DashboardController {
|
||||
await this.communicator.setG2ScreenOn(false);
|
||||
}
|
||||
if (updatePreviewAfterTransmit) {
|
||||
await this.updateConnectedDisplayPreviewFromImage(image);
|
||||
this.updateConnectedDisplayPreviewFromImage(image);
|
||||
}
|
||||
} else {
|
||||
frameTimings.finishFrame(frameId, "discarded: no communicator, preview only");
|
||||
@@ -829,7 +828,7 @@ class DashboardController {
|
||||
this.setDisplayPreview(grayImageToPreviewSource(image));
|
||||
}
|
||||
|
||||
private async updateConnectedDisplayPreviewFromImage(image: GrayImage): Promise<void> {
|
||||
private updateConnectedDisplayPreviewFromImage(image: GrayImage): void {
|
||||
if (!this.communicator) return;
|
||||
const now = Date.now();
|
||||
if (
|
||||
@@ -839,16 +838,7 @@ class DashboardController {
|
||||
return;
|
||||
}
|
||||
this.lastConnectedPreviewUpdateAtMs = now;
|
||||
const previewUpdateId = ++this.connectedPreviewUpdateSeq;
|
||||
console.log(`[preview#${previewUpdateId}] create start`);
|
||||
const preview = await grayImageToPreviewSourceWithBitmapFactory(
|
||||
image,
|
||||
(createColors, width, height) =>
|
||||
this.communicator!.createPreviewBitmapWithColorFactory(createColors, width, height),
|
||||
);
|
||||
console.log(`[preview#${previewUpdateId}] create done; setDisplayPreview start`);
|
||||
this.setDisplayPreview(preview);
|
||||
console.log(`[preview#${previewUpdateId}] setDisplayPreview done`);
|
||||
this.updateDisplayPreviewFromImage(image);
|
||||
}
|
||||
|
||||
private formatError(error: unknown): string {
|
||||
|
||||
@@ -52,15 +52,6 @@ export type RawInputEvent =
|
||||
frameId: number;
|
||||
};
|
||||
|
||||
function toJavaByteArray(bytes: Uint8Array): number[] {
|
||||
const out = Array.create("byte", bytes.length) as number[];
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const value = bytes[i]!;
|
||||
out[i] = value > 127 ? value - 256 : value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function nonNegativeNumber(value: number): number {
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
|
||||
@@ -300,11 +291,13 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
|
||||
async submitDashboardImage(image8bpp: Uint8Array, width: number, height: number, fingerprint: string, paintMs = -1, frameId = 0): Promise<void> {
|
||||
// Snapshot because the Java call is deferred; the buffer is passed as an
|
||||
// ArrayBuffer, which NativeScript marshals to a ByteBuffer without the
|
||||
// ~150ms per-element copy a byte[] parameter would need.
|
||||
const snapshot = new Uint8Array(image8bpp);
|
||||
await this.enqueueJavaCall(() => {
|
||||
const javaBytes = frameTimings.span(frameId, "js-to-java-copy", () => toJavaByteArray(snapshot));
|
||||
this.communicator.submitDashboardImage(
|
||||
javaBytes,
|
||||
snapshot.buffer,
|
||||
Math.round(width),
|
||||
Math.round(height),
|
||||
fingerprint,
|
||||
@@ -314,22 +307,6 @@ export class FaceclawCommunicatorBridge {
|
||||
});
|
||||
}
|
||||
|
||||
async createPreviewBitmap(colors: number[], width: number, height: number): Promise<any> {
|
||||
return this.enqueueJavaCall(() =>
|
||||
this.communicator.createPreviewBitmap(colors, width, height),
|
||||
);
|
||||
}
|
||||
|
||||
async createPreviewBitmapWithColorFactory(
|
||||
createColors: () => number[],
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<any> {
|
||||
return this.enqueueJavaCall(() =>
|
||||
this.communicator.createPreviewBitmap(createColors(), width, height),
|
||||
);
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
await this.enqueueJavaCall(() => this.communicator.disconnect());
|
||||
}
|
||||
|
||||
@@ -2,51 +2,26 @@ import { ImageSource } from "@nativescript/core";
|
||||
|
||||
import { type GrayImage } from "../graphics/image";
|
||||
|
||||
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
|
||||
const PREVIEW_COLOR_LUT = buildPreviewColorLookup();
|
||||
declare const com: any;
|
||||
|
||||
const PREVIEW_BRIGHTEN_GAMMA = 0.7;
|
||||
|
||||
/**
|
||||
* Build the phone-UI preview for a frame. The pixel buffer is passed to Java
|
||||
* as an ArrayBuffer (marshalled to a ByteBuffer) and expanded to ARGB there;
|
||||
* doing the per-pixel work in JS or copying element-by-element across the
|
||||
* bridge costs ~150ms per frame.
|
||||
*/
|
||||
export function grayImageToPreviewSource(image: GrayImage): ImageSource | null {
|
||||
if (!global.isAndroid) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const colors = grayImageToPreviewColors(image);
|
||||
|
||||
const bitmap = android.graphics.Bitmap.createBitmap(
|
||||
colors,
|
||||
const bitmap = com.faceclaw.app.PreviewBitmapUtil.fromGray(
|
||||
image.pixels.buffer,
|
||||
image.width,
|
||||
image.height,
|
||||
android.graphics.Bitmap.Config.ARGB_8888,
|
||||
PREVIEW_BRIGHTEN_GAMMA,
|
||||
);
|
||||
return new ImageSource(bitmap);
|
||||
}
|
||||
|
||||
export async function grayImageToPreviewSourceWithBitmapFactory(
|
||||
image: GrayImage,
|
||||
createBitmap: (createColors: () => number[], width: number, height: number) => Promise<any>,
|
||||
): Promise<ImageSource | null> {
|
||||
if (!global.isAndroid) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bitmap = await createBitmap(() => grayImageToPreviewColors(image), image.width, image.height);
|
||||
return new ImageSource(bitmap);
|
||||
}
|
||||
|
||||
function grayImageToPreviewColors(image: GrayImage): number[] {
|
||||
const colors = Array.create("int", image.width * image.height) as number[];
|
||||
for (let i = 0; i < image.pixels.length; i++) {
|
||||
colors[i] = PREVIEW_COLOR_LUT[image.pixels[i] ?? 0]!;
|
||||
}
|
||||
return colors;
|
||||
}
|
||||
|
||||
function buildPreviewColorLookup(): number[] {
|
||||
const colors: number[] = [];
|
||||
for (let gray = 0; gray <= 255; gray++) {
|
||||
const normalized = gray / 255;
|
||||
const preview = Math.max(0, Math.min(255, Math.round(255 * Math.pow(normalized, PREVIEW_BRIGHTEN_GAMMA))));
|
||||
colors[gray] = (255 << 24) | (preview << 16) | (preview << 8) | preview;
|
||||
}
|
||||
return colors;
|
||||
}
|
||||
Reference in new issue
Block a user