App-switchign UI: Sidebar, top bar, launcher

This commit is contained in:
James Babcock committed 2026-07-06 20:06:29 -07:00
1 parent c83448873e
commit 1133f952e9
22 files changed
+2031 -337

No files matched your search

@@ -114,6 +114,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private byte[] desiredBmp = new byte[0];
private int desiredPaintMs;
private int desiredFrameId;
// Highest compositor sequence stored as the desired frame; composites that
// lost a store race to a newer one are discarded (their content is already
// included in the newer composite).
private long lastStoredCompositeSeq;
private final SurfaceCompositor compositor = new SurfaceCompositor();
private final ArrayDeque<OutboundMessage> pendingMessages = new ArrayDeque<>();
private final ArrayDeque<OutboundMessage> inFlightMessages = new ArrayDeque<>();
@@ -145,6 +151,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
running = true;
userDisconnectRequested = false;
shutdownRequested = false;
activeInstance = this;
workerThread = new Thread(this, "FaceclawBleCommunicator");
workerThread.start();
}
@@ -185,6 +192,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
public void close() {
if (activeInstance == this) {
activeInstance = null;
}
disconnect();
}
@@ -244,39 +254,105 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
// The most recently started communicator; lets app worker threads submit
// surface frames without holding a cross-isolate reference to the bridge
// object (JS wrappers do not cross isolates, but the Java instance does).
private static volatile FaceclawBleCommunicator activeInstance;
public static FaceclawBleCommunicator getActive() {
return activeInstance;
}
/** Set the compositor's output frame size. Call before configuring surfaces. */
public void configureCompositorScreen(int width, int height) {
compositor.configureScreen(width, height);
}
/** Show or hide a compositor surface; takes effect at the next composite. */
public void setSurfaceVisible(String id, boolean visible) {
compositor.setSurfaceVisible(id, visible);
}
/**
* image8bpp arrives as a ByteBuffer because NativeScript marshals a JS
* Blank (screen off) or unblank the composited output, immediately
* submitting the resulting frame. Retained surface state is untouched, so
* unblanking restores the previous screen content without repaints.
*/
public void setScreenBlanked(boolean blanked) {
compositor.setBlanked(blanked);
SurfaceCompositor.Composite composite = compositor.composite();
byte[] bmp = BmpUtil.build4bppBmp(composite.gray, composite.width, composite.height);
storeDesiredComposite(composite, bmp, 0, 0);
}
/**
* Create or reconfigure a compositor surface. transparency is one of the
* SurfaceCompositor.TRANSPARENCY_* constants. Geometry changes take effect
* when the next frame is submitted.
*/
public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) {
compositor.configureSurface(id, x, y, width, height, zOrder, transparency);
}
public void removeSurface(String id) {
compositor.removeSurface(id);
}
/**
* Apply an update to one compositor surface and submit the recomposited
* screen as the desired frame. The update covers the rect (rectX, rectY,
* rectWidth, rectHeight) in surface-local coordinates; contentFingerprint
* identifies the surface's full content after the update.
*
* pixels8bpp arrives as a ByteBuffer because NativeScript marshals a JS
* ArrayBuffer to one without the per-element bridge copy that a byte[]
* parameter would need (~150ms for a full frame).
*/
public void submitDashboardImage(
java.nio.ByteBuffer image8bpp,
int width,
int height,
String fingerprint,
public void submitSurfaceFrame(
java.nio.ByteBuffer pixels8bpp,
String surfaceId,
int rectX,
int rectY,
int rectWidth,
int rectHeight,
String contentFingerprint,
int paintMs,
int frameId
) {
Log.i(TAG, "Received an updated frame ");
// Convert the full-screen 8bpp grayscale buffer into the BMP wire format here so
Log.i(TAG, "Received an updated frame for surface " + surfaceId);
FrameTimings.getInstance().spanStart(frameId, "composite");
SurfaceCompositor.Composite composite = compositor.applyAndComposite(
surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint);
FrameTimings.getInstance().spanEnd(frameId, "composite");
// Convert the composited 8bpp buffer into the BMP wire format here so
// that all framing concerns live on the Java side.
FrameTimings.getInstance().spanStart(frameId, "bmp-convert");
byte[] gray;
if (image8bpp == null) {
gray = new byte[0];
} else {
gray = new byte[image8bpp.remaining()];
image8bpp.get(gray);
}
byte[] bmp = gray.length == 0 ? new byte[0] : BmpUtil.build4bppBmp(gray, width, height);
byte[] bmp = BmpUtil.build4bppBmp(composite.gray, composite.width, composite.height);
FrameTimings.getInstance().spanEnd(frameId, "bmp-convert");
int supersededFrameId;
storeDesiredComposite(composite, bmp, paintMs, frameId);
}
/** Store a composite as the desired frame unless a newer one won the race. */
private void storeDesiredComposite(SurfaceCompositor.Composite composite, byte[] bmp, int paintMs, int frameId) {
int supersededFrameId = 0;
boolean stale = false;
synchronized (desiredTilesLock) {
supersededFrameId = desiredFrameId;
desiredBmp = bmp;
desiredFingerprint = fingerprint == null ? "" : fingerprint;
desiredPaintMs = paintMs;
desiredFrameId = frameId;
if (composite.seq <= lastStoredCompositeSeq) {
// A concurrent submission composited after us and stored first;
// its composite already includes this surface update.
stale = true;
} else {
lastStoredCompositeSeq = composite.seq;
supersededFrameId = desiredFrameId;
desiredBmp = bmp;
desiredFingerprint = composite.fingerprint;
desiredPaintMs = paintMs;
desiredFrameId = frameId;
}
}
if (stale) {
finishFrame(frameId, "discarded: composite superseded before store");
return;
}
if (supersededFrameId != 0 && supersededFrameId != frameId) {
finishFrame(supersededFrameId, "discarded: superseded by frame#" + frameId + " before send");
@@ -0,0 +1,121 @@
package com.faceclaw.app;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.util.concurrent.CopyOnWriteArrayList;
/**
* App settings store shared by every JS isolate (main thread and app
* workers). Values live in a dedicated SharedPreferences file, distinct from
* NativeScript's ApplicationSettings file, so this store owns its keys
* outright (the old TS-side settings were deliberately abandoned, not
* migrated).
*
* Change notifications: each isolate registers one listener from its own
* thread. The registering thread's Looper is captured, and notifications are
* posted through it so the JS callback always runs on the isolate's own
* thread (calling into an isolate from a foreign thread is not allowed).
* NativeScript worker threads run a message loop, so both the main thread
* and workers have a Looper; a listener registered from a Looper-less thread
* is accepted but never notified (it can still read fresh values on demand).
*/
public final class FaceclawSettings {
private static final String TAG = "FaceclawSettings";
private static final String PREFS_NAME = "faceclaw_settings";
private static volatile FaceclawSettings instance;
private final SharedPreferences prefs;
private final CopyOnWriteArrayList<ListenerEntry> listeners = new CopyOnWriteArrayList<>();
private static final class ListenerEntry {
final FaceclawSettingsListener listener;
final Handler handler;
ListenerEntry(FaceclawSettingsListener listener, Handler handler) {
this.listener = listener;
this.handler = handler;
}
}
private FaceclawSettings(Context context) {
this.prefs = context.getApplicationContext()
.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE);
}
/** Initialize (idempotent) and return the singleton. */
public static FaceclawSettings getInstance(Context context) {
if (instance == null) {
synchronized (FaceclawSettings.class) {
if (instance == null) {
instance = new FaceclawSettings(context);
}
}
}
return instance;
}
/** Return the singleton; the main isolate must have initialized it first. */
public static FaceclawSettings getInstance() {
FaceclawSettings result = instance;
if (result == null) {
throw new IllegalStateException("FaceclawSettings not initialized; call getInstance(context) first");
}
return result;
}
public String getString(String key, String defaultValue) {
return prefs.getString(key, defaultValue);
}
public void setString(String key, String value) {
prefs.edit().putString(key, value).apply();
notifyChanged(key);
}
public boolean getBoolean(String key, boolean defaultValue) {
return prefs.getBoolean(key, defaultValue);
}
public void setBoolean(String key, boolean value) {
prefs.edit().putBoolean(key, value).apply();
notifyChanged(key);
}
/**
* Register a change listener. Must be called from the thread whose
* isolate owns the listener; that thread's Looper is captured for
* dispatch.
*/
public void registerListener(FaceclawSettingsListener listener) {
Looper looper = Looper.myLooper();
if (looper == null) {
Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified");
}
listeners.add(new ListenerEntry(listener, looper != null ? new Handler(looper) : null));
}
public void unregisterListener(FaceclawSettingsListener listener) {
for (ListenerEntry entry : listeners) {
if (entry.listener == listener) {
listeners.remove(entry);
}
}
}
private void notifyChanged(String key) {
for (ListenerEntry entry : listeners) {
if (entry.handler == null) continue;
entry.handler.post(() -> {
try {
entry.listener.onSettingChanged(key);
} catch (Exception e) {
Log.w(TAG, "settings listener failed for key " + key, e);
}
});
}
}
}
@@ -0,0 +1,6 @@
package com.faceclaw.app;
/** Change-notification callback for FaceclawSettings; implemented in JS. */
public interface FaceclawSettingsListener {
void onSettingChanged(String key);
}
@@ -0,0 +1,277 @@
package com.faceclaw.app;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Retained-surface compositor: the phone-side model of what is on the glasses
* screen.
*
* Each surface is an 8bpp grayscale buffer retained at its last-submitted
* contents, positioned on the screen with a z-order and a transparency mode.
* Sources on the TS side (today the dashboard; later the shell and per-app
* worker threads) submit updates to their surface, and the compositor
* recombines the retained surfaces into a full-screen frame that feeds the
* existing dedupe/compress/transmit pipeline. Because every surface is
* retained, a full screen frame can be regenerated at any time (frame drops,
* reconnects, previews, screenshots) without asking sources to repaint.
*
* Surface format contract (mirrored by the TS side):
* - Pixels are 8bpp grayscale, row-major, one byte per pixel. The wire
* pipeline quantizes to 4bpp downstream, so the low nibble is not visible.
* - TRANSPARENCY_COLOR_KEY surfaces treat pixel value 0 as fully transparent
* and value 1 as black. Quantization collapses 1 into the same 4bpp level
* as 0, so reserving 0 costs no visible shade; painters of color-key
* surfaces must clamp intentional black to 1.
* - An update may cover any rect of its surface; the rest is retained. The
* update buffer holds exactly rectWidth*rectHeight bytes, row-major.
* - Surfaces composite in ascending z-order onto a black (0) background.
* - Surface geometry changes take effect when the next frame composites;
* they do not trigger a recomposite by themselves.
*
* Thread safety: all methods are safe to call from any thread. Apply and
* composite happen atomically under an internal lock, and each composite
* carries a monotonic sequence number so callers can detect when a composite
* was superseded by a concurrent one before being acted on.
*/
public final class SurfaceCompositor {
public static final int TRANSPARENCY_OPAQUE = 0;
public static final int TRANSPARENCY_COLOR_KEY = 1;
/** One composited full-screen frame plus the metadata the pipeline needs. */
public static final class Composite {
/** Full-screen 8bpp grayscale pixels, screenWidth*screenHeight bytes. */
public final byte[] gray;
public final int width;
public final int height;
/**
* Stable identifier of screen content, combining every surface's
* geometry and content fingerprint; equal fingerprints mean equal
* composited pixels.
*/
public final String fingerprint;
/** Monotonic: a Composite with a higher seq contains strictly newer state. */
public final long seq;
Composite(byte[] gray, int width, int height, String fingerprint, long seq) {
this.gray = gray;
this.width = width;
this.height = height;
this.fingerprint = fingerprint;
this.seq = seq;
}
}
private static final class Surface {
final String id;
int x;
int y;
int width;
int height;
int zOrder;
int transparency;
boolean visible = true;
byte[] pixels;
String fingerprint = "";
Surface(String id) {
this.id = id;
}
}
private final Object lock = new Object();
private int screenWidth;
private int screenHeight;
private boolean blanked;
private final Map<String, Surface> surfaces = new HashMap<>();
private long nextCompositeSeq = 1;
/** Set the output frame size. Must be called before any surface work. */
public void configureScreen(int width, int height) {
if (width <= 0 || height <= 0) {
throw new IllegalArgumentException("bad screen size " + width + "x" + height);
}
synchronized (lock) {
this.screenWidth = width;
this.screenHeight = height;
}
}
/**
* Create a surface or update an existing one's geometry. A resize discards
* the surface's retained pixels (reset to 0).
*/
public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) {
if (id == null || id.isEmpty()) {
throw new IllegalArgumentException("surface id must be non-empty");
}
if (width <= 0 || height <= 0) {
throw new IllegalArgumentException("bad surface size " + width + "x" + height + " for " + id);
}
if (transparency != TRANSPARENCY_OPAQUE && transparency != TRANSPARENCY_COLOR_KEY) {
throw new IllegalArgumentException("bad transparency mode " + transparency + " for " + id);
}
synchronized (lock) {
requireScreenConfiguredLocked();
Surface surface = surfaces.get(id);
if (surface == null) {
surface = new Surface(id);
surfaces.put(id, surface);
}
if (surface.pixels == null || surface.width != width || surface.height != height) {
surface.pixels = new byte[width * height];
surface.fingerprint = "";
}
surface.x = x;
surface.y = y;
surface.width = width;
surface.height = height;
surface.zOrder = zOrder;
surface.transparency = transparency;
}
}
public void removeSurface(String id) {
synchronized (lock) {
surfaces.remove(id);
}
}
/**
* Hidden surfaces keep their retained pixels and accept updates, but are
* excluded from the composite and its fingerprint (used for background
* windows). Takes effect when the next frame composites.
*/
public void setSurfaceVisible(String id, boolean visible) {
synchronized (lock) {
Surface surface = surfaces.get(id);
if (surface == null) {
throw new IllegalArgumentException("unknown surface " + id);
}
surface.visible = visible;
}
}
/**
* While blanked (screen off), composites are all-zero regardless of
* surface content; retained state is untouched, so unblanking restores
* the screen without asking sources to repaint.
*/
public void setBlanked(boolean blanked) {
synchronized (lock) {
this.blanked = blanked;
}
}
/**
* Apply an update to one surface and composite the whole screen, as one
* atomic step. The update covers the rect (rectX, rectY, rectWidth,
* rectHeight) in surface-local coordinates; pixels must hold exactly
* rectWidth*rectHeight bytes. contentFingerprint identifies the surface's
* full content after this update.
*/
public Composite applyAndComposite(
String surfaceId,
ByteBuffer pixels,
int rectX,
int rectY,
int rectWidth,
int rectHeight,
String contentFingerprint
) {
synchronized (lock) {
requireScreenConfiguredLocked();
Surface surface = surfaces.get(surfaceId);
if (surface == null) {
throw new IllegalArgumentException("unknown surface " + surfaceId);
}
if (rectX < 0 || rectY < 0 || rectWidth <= 0 || rectHeight <= 0
|| rectX + rectWidth > surface.width || rectY + rectHeight > surface.height) {
throw new IllegalArgumentException("update rect " + rectWidth + "x" + rectHeight
+ "+" + rectX + "+" + rectY + " outside surface " + surfaceId
+ " (" + surface.width + "x" + surface.height + ")");
}
int expectedBytes = rectWidth * rectHeight;
if (pixels == null || pixels.remaining() != expectedBytes) {
throw new IllegalArgumentException("update buffer for " + surfaceId + " has "
+ (pixels == null ? 0 : pixels.remaining()) + " bytes, expected " + expectedBytes);
}
for (int row = 0; row < rectHeight; row++) {
int dstOffset = (rectY + row) * surface.width + rectX;
pixels.get(surface.pixels, dstOffset, rectWidth);
}
surface.fingerprint = contentFingerprint == null ? "" : contentFingerprint;
return compositeLocked();
}
}
/** Composite the current retained state without applying an update. */
public Composite composite() {
synchronized (lock) {
requireScreenConfiguredLocked();
return compositeLocked();
}
}
private Composite compositeLocked() {
byte[] gray = new byte[screenWidth * screenHeight];
if (blanked) {
return new Composite(gray, screenWidth, screenHeight,
"blanked:" + screenWidth + "x" + screenHeight, nextCompositeSeq++);
}
List<Surface> ordered = new ArrayList<>(surfaces.values());
ordered.sort(Comparator
.comparingInt((Surface s) -> s.zOrder)
.thenComparing(s -> s.id));
StringBuilder fingerprint = new StringBuilder();
fingerprint.append(screenWidth).append('x').append(screenHeight);
for (Surface surface : ordered) {
if (!surface.visible) continue;
blendLocked(gray, surface);
fingerprint.append('|').append(surface.id)
.append('@').append(surface.x).append(',').append(surface.y)
.append('+').append(surface.width).append('x').append(surface.height)
.append('#').append(surface.zOrder)
.append(':').append(surface.transparency)
.append(':').append(surface.fingerprint);
}
return new Composite(gray, screenWidth, screenHeight, fingerprint.toString(), nextCompositeSeq++);
}
private void blendLocked(byte[] gray, Surface surface) {
// Clip the surface rect to the screen.
int srcX = Math.max(0, -surface.x);
int srcY = Math.max(0, -surface.y);
int dstX = Math.max(0, surface.x);
int dstY = Math.max(0, surface.y);
int copyWidth = Math.min(surface.width - srcX, screenWidth - dstX);
int copyHeight = Math.min(surface.height - srcY, screenHeight - dstY);
if (copyWidth <= 0 || copyHeight <= 0) {
return;
}
for (int row = 0; row < copyHeight; row++) {
int srcOffset = (srcY + row) * surface.width + srcX;
int dstOffset = (dstY + row) * screenWidth + dstX;
if (surface.transparency == TRANSPARENCY_OPAQUE) {
System.arraycopy(surface.pixels, srcOffset, gray, dstOffset, copyWidth);
} else {
for (int col = 0; col < copyWidth; col++) {
byte value = surface.pixels[srcOffset + col];
if (value != 0) {
gray[dstOffset + col] = value;
}
}
}
}
}
private void requireScreenConfiguredLocked() {
if (screenWidth <= 0 || screenHeight <= 0) {
throw new IllegalStateException("configureScreen must be called first");
}
}
}