From 1133f952e9f6dcb01b112e1e342196d4693d5fb2 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Mon, 6 Jul 2026 20:06:29 -0700 Subject: [PATCH] App-switchign UI: Sidebar, top bar, launcher --- .../faceclaw/app/FaceclawBleCommunicator.java | 120 ++++- .../com/faceclaw/app/FaceclawSettings.java | 121 +++++ .../app/FaceclawSettingsListener.java | 6 + .../com/faceclaw/app/SurfaceCompositor.java | 277 +++++++++++ app/g2/dashboard-controller.ts | 385 ++++++++++++--- app/graphics/image.ts | 13 +- app/native/faceclaw-communicator.ts | 91 +++- app/native/settings-store.ts | 67 +++ app/ui/apps/stopwatch.ts | 72 --- app/ui/apps/voice-input.ts | 5 +- app/ui/dashboard-settings.ts | 35 +- app/ui/dashboard.ts | 160 +------ app/ui/dashboard/root-menu.ts | 14 +- app/ui/dashboard/settings-menus.ts | 5 +- app/ui/layers.ts | 9 +- app/ui/menu.ts | 6 +- app/ui/shell/chrome-layer.ts | 114 +++++ app/ui/shell/geometry.ts | 21 + app/ui/shell/launcher-app.ts | 92 ++++ app/ui/shell/shell.ts | 449 ++++++++++++++++++ app/ui/shell/worker-window.ts | 105 ++++ app/workers/stopwatch-app.worker.ts | 201 ++++++++ 22 files changed, 2031 insertions(+), 337 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsListener.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java create mode 100644 app/native/settings-store.ts delete mode 100644 app/ui/apps/stopwatch.ts create mode 100644 app/ui/shell/chrome-layer.ts create mode 100644 app/ui/shell/geometry.ts create mode 100644 app/ui/shell/launcher-app.ts create mode 100644 app/ui/shell/shell.ts create mode 100644 app/ui/shell/worker-window.ts create mode 100644 app/workers/stopwatch-app.worker.ts diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index b67ff1ab..0b399ca7 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -114,6 +114,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private byte[] desiredBmp = new byte[0]; private int desiredPaintMs; private int desiredFrameId; + // Highest compositor sequence stored as the desired frame; composites that + // lost a store race to a newer one are discarded (their content is already + // included in the newer composite). + private long lastStoredCompositeSeq; + + private final SurfaceCompositor compositor = new SurfaceCompositor(); private final ArrayDeque pendingMessages = new ArrayDeque<>(); private final ArrayDeque inFlightMessages = new ArrayDeque<>(); @@ -145,6 +151,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { running = true; userDisconnectRequested = false; shutdownRequested = false; + activeInstance = this; workerThread = new Thread(this, "FaceclawBleCommunicator"); workerThread.start(); } @@ -185,6 +192,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } public void close() { + if (activeInstance == this) { + activeInstance = null; + } disconnect(); } @@ -244,39 +254,105 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } + // The most recently started communicator; lets app worker threads submit + // surface frames without holding a cross-isolate reference to the bridge + // object (JS wrappers do not cross isolates, but the Java instance does). + private static volatile FaceclawBleCommunicator activeInstance; + + public static FaceclawBleCommunicator getActive() { + return activeInstance; + } + + /** Set the compositor's output frame size. Call before configuring surfaces. */ + public void configureCompositorScreen(int width, int height) { + compositor.configureScreen(width, height); + } + + /** Show or hide a compositor surface; takes effect at the next composite. */ + public void setSurfaceVisible(String id, boolean visible) { + compositor.setSurfaceVisible(id, visible); + } + /** - * image8bpp arrives as a ByteBuffer because NativeScript marshals a JS + * Blank (screen off) or unblank the composited output, immediately + * submitting the resulting frame. Retained surface state is untouched, so + * unblanking restores the previous screen content without repaints. + */ + public void setScreenBlanked(boolean blanked) { + compositor.setBlanked(blanked); + SurfaceCompositor.Composite composite = compositor.composite(); + byte[] bmp = BmpUtil.build4bppBmp(composite.gray, composite.width, composite.height); + storeDesiredComposite(composite, bmp, 0, 0); + } + + /** + * Create or reconfigure a compositor surface. transparency is one of the + * SurfaceCompositor.TRANSPARENCY_* constants. Geometry changes take effect + * when the next frame is submitted. + */ + public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) { + compositor.configureSurface(id, x, y, width, height, zOrder, transparency); + } + + public void removeSurface(String id) { + compositor.removeSurface(id); + } + + /** + * Apply an update to one compositor surface and submit the recomposited + * screen as the desired frame. The update covers the rect (rectX, rectY, + * rectWidth, rectHeight) in surface-local coordinates; contentFingerprint + * identifies the surface's full content after the update. + * + * pixels8bpp arrives as a ByteBuffer because NativeScript marshals a JS * ArrayBuffer to one without the per-element bridge copy that a byte[] * parameter would need (~150ms for a full frame). */ - public void submitDashboardImage( - java.nio.ByteBuffer image8bpp, - int width, - int height, - String fingerprint, + public void submitSurfaceFrame( + java.nio.ByteBuffer pixels8bpp, + String surfaceId, + int rectX, + int rectY, + int rectWidth, + int rectHeight, + String contentFingerprint, int paintMs, int frameId ) { - Log.i(TAG, "Received an updated frame "); - // Convert the full-screen 8bpp grayscale buffer into the BMP wire format here so + Log.i(TAG, "Received an updated frame for surface " + surfaceId); + FrameTimings.getInstance().spanStart(frameId, "composite"); + SurfaceCompositor.Composite composite = compositor.applyAndComposite( + surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint); + FrameTimings.getInstance().spanEnd(frameId, "composite"); + // Convert the composited 8bpp buffer into the BMP wire format here so // that all framing concerns live on the Java side. FrameTimings.getInstance().spanStart(frameId, "bmp-convert"); - byte[] gray; - if (image8bpp == null) { - gray = new byte[0]; - } else { - gray = new byte[image8bpp.remaining()]; - image8bpp.get(gray); - } - byte[] bmp = gray.length == 0 ? new byte[0] : BmpUtil.build4bppBmp(gray, width, height); + byte[] bmp = BmpUtil.build4bppBmp(composite.gray, composite.width, composite.height); FrameTimings.getInstance().spanEnd(frameId, "bmp-convert"); - int supersededFrameId; + storeDesiredComposite(composite, bmp, paintMs, frameId); + } + + /** Store a composite as the desired frame unless a newer one won the race. */ + private void storeDesiredComposite(SurfaceCompositor.Composite composite, byte[] bmp, int paintMs, int frameId) { + int supersededFrameId = 0; + boolean stale = false; synchronized (desiredTilesLock) { - supersededFrameId = desiredFrameId; - desiredBmp = bmp; - desiredFingerprint = fingerprint == null ? "" : fingerprint; - desiredPaintMs = paintMs; - desiredFrameId = frameId; + if (composite.seq <= lastStoredCompositeSeq) { + // A concurrent submission composited after us and stored first; + // its composite already includes this surface update. + stale = true; + } else { + lastStoredCompositeSeq = composite.seq; + supersededFrameId = desiredFrameId; + desiredBmp = bmp; + desiredFingerprint = composite.fingerprint; + desiredPaintMs = paintMs; + desiredFrameId = frameId; + } + } + if (stale) { + finishFrame(frameId, "discarded: composite superseded before store"); + return; } if (supersededFrameId != 0 && supersededFrameId != frameId) { finishFrame(supersededFrameId, "discarded: superseded by frame#" + frameId + " before send"); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java new file mode 100644 index 00000000..bec8c494 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.java @@ -0,0 +1,121 @@ +package com.faceclaw.app; + +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * App settings store shared by every JS isolate (main thread and app + * workers). Values live in a dedicated SharedPreferences file, distinct from + * NativeScript's ApplicationSettings file, so this store owns its keys + * outright (the old TS-side settings were deliberately abandoned, not + * migrated). + * + * Change notifications: each isolate registers one listener from its own + * thread. The registering thread's Looper is captured, and notifications are + * posted through it so the JS callback always runs on the isolate's own + * thread (calling into an isolate from a foreign thread is not allowed). + * NativeScript worker threads run a message loop, so both the main thread + * and workers have a Looper; a listener registered from a Looper-less thread + * is accepted but never notified (it can still read fresh values on demand). + */ +public final class FaceclawSettings { + private static final String TAG = "FaceclawSettings"; + private static final String PREFS_NAME = "faceclaw_settings"; + private static volatile FaceclawSettings instance; + + private final SharedPreferences prefs; + private final CopyOnWriteArrayList listeners = new CopyOnWriteArrayList<>(); + + private static final class ListenerEntry { + final FaceclawSettingsListener listener; + final Handler handler; + + ListenerEntry(FaceclawSettingsListener listener, Handler handler) { + this.listener = listener; + this.handler = handler; + } + } + + private FaceclawSettings(Context context) { + this.prefs = context.getApplicationContext() + .getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE); + } + + /** Initialize (idempotent) and return the singleton. */ + public static FaceclawSettings getInstance(Context context) { + if (instance == null) { + synchronized (FaceclawSettings.class) { + if (instance == null) { + instance = new FaceclawSettings(context); + } + } + } + return instance; + } + + /** Return the singleton; the main isolate must have initialized it first. */ + public static FaceclawSettings getInstance() { + FaceclawSettings result = instance; + if (result == null) { + throw new IllegalStateException("FaceclawSettings not initialized; call getInstance(context) first"); + } + return result; + } + + public String getString(String key, String defaultValue) { + return prefs.getString(key, defaultValue); + } + + public void setString(String key, String value) { + prefs.edit().putString(key, value).apply(); + notifyChanged(key); + } + + public boolean getBoolean(String key, boolean defaultValue) { + return prefs.getBoolean(key, defaultValue); + } + + public void setBoolean(String key, boolean value) { + prefs.edit().putBoolean(key, value).apply(); + notifyChanged(key); + } + + /** + * Register a change listener. Must be called from the thread whose + * isolate owns the listener; that thread's Looper is captured for + * dispatch. + */ + public void registerListener(FaceclawSettingsListener listener) { + Looper looper = Looper.myLooper(); + if (looper == null) { + Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified"); + } + listeners.add(new ListenerEntry(listener, looper != null ? new Handler(looper) : null)); + } + + public void unregisterListener(FaceclawSettingsListener listener) { + for (ListenerEntry entry : listeners) { + if (entry.listener == listener) { + listeners.remove(entry); + } + } + } + + private void notifyChanged(String key) { + for (ListenerEntry entry : listeners) { + if (entry.handler == null) continue; + entry.handler.post(() -> { + try { + entry.listener.onSettingChanged(key); + } catch (Exception e) { + Log.w(TAG, "settings listener failed for key " + key, e); + } + }); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsListener.java new file mode 100644 index 00000000..a34bd7ab --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettingsListener.java @@ -0,0 +1,6 @@ +package com.faceclaw.app; + +/** Change-notification callback for FaceclawSettings; implemented in JS. */ +public interface FaceclawSettingsListener { + void onSettingChanged(String key); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java new file mode 100644 index 00000000..a02baa3d --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/SurfaceCompositor.java @@ -0,0 +1,277 @@ +package com.faceclaw.app; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Retained-surface compositor: the phone-side model of what is on the glasses + * screen. + * + * Each surface is an 8bpp grayscale buffer retained at its last-submitted + * contents, positioned on the screen with a z-order and a transparency mode. + * Sources on the TS side (today the dashboard; later the shell and per-app + * worker threads) submit updates to their surface, and the compositor + * recombines the retained surfaces into a full-screen frame that feeds the + * existing dedupe/compress/transmit pipeline. Because every surface is + * retained, a full screen frame can be regenerated at any time (frame drops, + * reconnects, previews, screenshots) without asking sources to repaint. + * + * Surface format contract (mirrored by the TS side): + * - Pixels are 8bpp grayscale, row-major, one byte per pixel. The wire + * pipeline quantizes to 4bpp downstream, so the low nibble is not visible. + * - TRANSPARENCY_COLOR_KEY surfaces treat pixel value 0 as fully transparent + * and value 1 as black. Quantization collapses 1 into the same 4bpp level + * as 0, so reserving 0 costs no visible shade; painters of color-key + * surfaces must clamp intentional black to 1. + * - An update may cover any rect of its surface; the rest is retained. The + * update buffer holds exactly rectWidth*rectHeight bytes, row-major. + * - Surfaces composite in ascending z-order onto a black (0) background. + * - Surface geometry changes take effect when the next frame composites; + * they do not trigger a recomposite by themselves. + * + * Thread safety: all methods are safe to call from any thread. Apply and + * composite happen atomically under an internal lock, and each composite + * carries a monotonic sequence number so callers can detect when a composite + * was superseded by a concurrent one before being acted on. + */ +public final class SurfaceCompositor { + public static final int TRANSPARENCY_OPAQUE = 0; + public static final int TRANSPARENCY_COLOR_KEY = 1; + + /** One composited full-screen frame plus the metadata the pipeline needs. */ + public static final class Composite { + /** Full-screen 8bpp grayscale pixels, screenWidth*screenHeight bytes. */ + public final byte[] gray; + public final int width; + public final int height; + /** + * Stable identifier of screen content, combining every surface's + * geometry and content fingerprint; equal fingerprints mean equal + * composited pixels. + */ + public final String fingerprint; + /** Monotonic: a Composite with a higher seq contains strictly newer state. */ + public final long seq; + + Composite(byte[] gray, int width, int height, String fingerprint, long seq) { + this.gray = gray; + this.width = width; + this.height = height; + this.fingerprint = fingerprint; + this.seq = seq; + } + } + + private static final class Surface { + final String id; + int x; + int y; + int width; + int height; + int zOrder; + int transparency; + boolean visible = true; + byte[] pixels; + String fingerprint = ""; + + Surface(String id) { + this.id = id; + } + } + + private final Object lock = new Object(); + private int screenWidth; + private int screenHeight; + private boolean blanked; + private final Map surfaces = new HashMap<>(); + private long nextCompositeSeq = 1; + + /** Set the output frame size. Must be called before any surface work. */ + public void configureScreen(int width, int height) { + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("bad screen size " + width + "x" + height); + } + synchronized (lock) { + this.screenWidth = width; + this.screenHeight = height; + } + } + + /** + * Create a surface or update an existing one's geometry. A resize discards + * the surface's retained pixels (reset to 0). + */ + public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("surface id must be non-empty"); + } + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("bad surface size " + width + "x" + height + " for " + id); + } + if (transparency != TRANSPARENCY_OPAQUE && transparency != TRANSPARENCY_COLOR_KEY) { + throw new IllegalArgumentException("bad transparency mode " + transparency + " for " + id); + } + synchronized (lock) { + requireScreenConfiguredLocked(); + Surface surface = surfaces.get(id); + if (surface == null) { + surface = new Surface(id); + surfaces.put(id, surface); + } + if (surface.pixels == null || surface.width != width || surface.height != height) { + surface.pixels = new byte[width * height]; + surface.fingerprint = ""; + } + surface.x = x; + surface.y = y; + surface.width = width; + surface.height = height; + surface.zOrder = zOrder; + surface.transparency = transparency; + } + } + + public void removeSurface(String id) { + synchronized (lock) { + surfaces.remove(id); + } + } + + /** + * Hidden surfaces keep their retained pixels and accept updates, but are + * excluded from the composite and its fingerprint (used for background + * windows). Takes effect when the next frame composites. + */ + public void setSurfaceVisible(String id, boolean visible) { + synchronized (lock) { + Surface surface = surfaces.get(id); + if (surface == null) { + throw new IllegalArgumentException("unknown surface " + id); + } + surface.visible = visible; + } + } + + /** + * While blanked (screen off), composites are all-zero regardless of + * surface content; retained state is untouched, so unblanking restores + * the screen without asking sources to repaint. + */ + public void setBlanked(boolean blanked) { + synchronized (lock) { + this.blanked = blanked; + } + } + + /** + * Apply an update to one surface and composite the whole screen, as one + * atomic step. The update covers the rect (rectX, rectY, rectWidth, + * rectHeight) in surface-local coordinates; pixels must hold exactly + * rectWidth*rectHeight bytes. contentFingerprint identifies the surface's + * full content after this update. + */ + public Composite applyAndComposite( + String surfaceId, + ByteBuffer pixels, + int rectX, + int rectY, + int rectWidth, + int rectHeight, + String contentFingerprint + ) { + synchronized (lock) { + requireScreenConfiguredLocked(); + Surface surface = surfaces.get(surfaceId); + if (surface == null) { + throw new IllegalArgumentException("unknown surface " + surfaceId); + } + if (rectX < 0 || rectY < 0 || rectWidth <= 0 || rectHeight <= 0 + || rectX + rectWidth > surface.width || rectY + rectHeight > surface.height) { + throw new IllegalArgumentException("update rect " + rectWidth + "x" + rectHeight + + "+" + rectX + "+" + rectY + " outside surface " + surfaceId + + " (" + surface.width + "x" + surface.height + ")"); + } + int expectedBytes = rectWidth * rectHeight; + if (pixels == null || pixels.remaining() != expectedBytes) { + throw new IllegalArgumentException("update buffer for " + surfaceId + " has " + + (pixels == null ? 0 : pixels.remaining()) + " bytes, expected " + expectedBytes); + } + for (int row = 0; row < rectHeight; row++) { + int dstOffset = (rectY + row) * surface.width + rectX; + pixels.get(surface.pixels, dstOffset, rectWidth); + } + surface.fingerprint = contentFingerprint == null ? "" : contentFingerprint; + return compositeLocked(); + } + } + + /** Composite the current retained state without applying an update. */ + public Composite composite() { + synchronized (lock) { + requireScreenConfiguredLocked(); + return compositeLocked(); + } + } + + private Composite compositeLocked() { + byte[] gray = new byte[screenWidth * screenHeight]; + if (blanked) { + return new Composite(gray, screenWidth, screenHeight, + "blanked:" + screenWidth + "x" + screenHeight, nextCompositeSeq++); + } + List ordered = new ArrayList<>(surfaces.values()); + ordered.sort(Comparator + .comparingInt((Surface s) -> s.zOrder) + .thenComparing(s -> s.id)); + StringBuilder fingerprint = new StringBuilder(); + fingerprint.append(screenWidth).append('x').append(screenHeight); + for (Surface surface : ordered) { + if (!surface.visible) continue; + blendLocked(gray, surface); + fingerprint.append('|').append(surface.id) + .append('@').append(surface.x).append(',').append(surface.y) + .append('+').append(surface.width).append('x').append(surface.height) + .append('#').append(surface.zOrder) + .append(':').append(surface.transparency) + .append(':').append(surface.fingerprint); + } + return new Composite(gray, screenWidth, screenHeight, fingerprint.toString(), nextCompositeSeq++); + } + + private void blendLocked(byte[] gray, Surface surface) { + // Clip the surface rect to the screen. + int srcX = Math.max(0, -surface.x); + int srcY = Math.max(0, -surface.y); + int dstX = Math.max(0, surface.x); + int dstY = Math.max(0, surface.y); + int copyWidth = Math.min(surface.width - srcX, screenWidth - dstX); + int copyHeight = Math.min(surface.height - srcY, screenHeight - dstY); + if (copyWidth <= 0 || copyHeight <= 0) { + return; + } + for (int row = 0; row < copyHeight; row++) { + int srcOffset = (srcY + row) * surface.width + srcX; + int dstOffset = (dstY + row) * screenWidth + dstX; + if (surface.transparency == TRANSPARENCY_OPAQUE) { + System.arraycopy(surface.pixels, srcOffset, gray, dstOffset, copyWidth); + } else { + for (int col = 0; col < copyWidth; col++) { + byte value = surface.pixels[srcOffset + col]; + if (value != 0) { + gray[dstOffset + col] = value; + } + } + } + } + } + + private void requireScreenConfiguredLocked() { + if (screenWidth <= 0 || screenHeight <= 0) { + throw new IllegalStateException("configureScreen must be called first"); + } + } +} diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 98335ba6..9278c2e6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -15,18 +15,21 @@ import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { voiceControlBridge } from "../native/voice-control"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; import { - applyDashboardScreenTimeout, closeDashboardTextSettingEditor, - dashboardState, drawDashboard, - noteDashboardPhoneTextInput, + logToDashboard, openAndroidNotificationFromSleep, openTelepromptDocument, - receiveInput, + receiveDashboardWindowInput, resetDashboardSleepTimerAndWake, setDashboardBatteryLevels, setDashboardActions, } from "../ui/dashboard"; +import { inputEventToString, rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; +import { makeLetterWindowIcon } from "../ui/shell/chrome-layer"; +import { WorkerAppHost } from "../ui/shell/worker-window"; +import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app"; +import { APP_VIEWPORT } from "../ui/shell/geometry"; import { elevenLabsApiKeySetting, nightscoutApiTokenSetting, @@ -35,6 +38,8 @@ import { onAnySettingChanged, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, + screenTimeoutSetting, + screenTimeoutSettingToMs, systemCardNameSetting, voiceProviderSetting, type ConfigSettingString, @@ -64,9 +69,13 @@ type DashboardListener = (snapshot: DashboardSnapshot) => void; type LogLevel = "debug"|"info"|"warn"|"error"; const CONTAINER_NAME = "dashboard"; +// Compositor surfaces: the dashboard window in the app viewport, and the +// shell chrome (sidebar + top bar + overlays) above it with color-key +// transparency. +const DASHBOARD_WINDOW_SURFACE_ID = "window:dashboard"; +const SHELL_SURFACE_ID = "shell"; const DASHBOARD_INTERVAL_MS = 60_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; -const STOPWATCH_RENDER_INTERVAL_MS = 100; const TRANSCRIBE_RENDER_INTERVAL_MS = 250; const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000; const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000; @@ -125,6 +134,16 @@ function isDashboardContainerName(name: string): boolean { return normalized === CONTAINER_NAME || normalized.includes(CONTAINER_NAME); } +/** Top-left viewport-sized crop of a full-lens dashboard paint. */ +function cropToViewport(image: GrayImage): GrayImage { + if (image.width === APP_VIEWPORT.width && image.height === APP_VIEWPORT.height) { + return image; + } + const cropped = new GrayImage(APP_VIEWPORT.width, APP_VIEWPORT.height, 0); + cropped.bitBlt(image, 0, 0, { width: APP_VIEWPORT.width, height: APP_VIEWPORT.height }); + return cropped; +} + class DashboardController { private phase: ConnectionPhase = "disconnected"; private status = "Disconnected."; @@ -140,7 +159,6 @@ class DashboardController { private communicator: FaceclawCommunicatorBridge | null = null; private dashboardTimer: ReturnType | null = null; private screenTimeoutTimer: ReturnType | null = null; - private stopwatchRenderTimer: ReturnType | null = null; private transcribeRenderTimer: ReturnType | null = null; private offState: (() => void) | null = null; private offLog: (() => void) | null = null; @@ -160,6 +178,14 @@ class DashboardController { private renderQueued = false; private queuedRenderReason: "initial" | "interval" = "interval"; private queuedFrameId = 0; + private shellRenderInProgress = false; + private shellRenderQueued = false; + // Last painted surface images, for compositing the phone-side preview. + private lastShellImage: GrayImage | null = null; + private lastWindowImage: GrayImage | null = null; + // One shared worker hosts every stopwatch window; spawned on first launch. + private stopwatchHost: WorkerAppHost | null = null; + private nextWindowSerial = 1; private lastForegroundNotificationUpdateAtMs = 0; private lastConnectedPreviewUpdateAtMs = 0; // Consumed by the next renderDashboard: repaint with data sources not @@ -167,20 +193,81 @@ class DashboardController { private nextRenderWantsFreshData = false; constructor() { + const sharedActions = { + disconnect: () => this.disconnect(), + startTextSettingEdit: (setting: ConfigSettingString) => this.startTextSettingEdit(setting), + endTextSettingEdit: () => this.endTextSettingEdit(), + setTranscribeRenderActive: (active: boolean) => this.setTranscribeRenderActive(active), + startVoiceCapture: () => this.startVoiceCapture(), + stopVoiceCapture: () => this.stopVoiceCapture(), + playBuzzerNote: (note: number, oct: number, beat: number) => this.playBuzzerNote(note, oct, beat), + }; setDashboardActions({ + ...sharedActions, requestRender: () => { void this.requestRender("interval").catch((error) => { this.appendLog(`requested render failed: ${this.formatError(error)}`); }); }, - disconnect: () => this.disconnect(), - startTextSettingEdit: (setting) => this.startTextSettingEdit(setting), - endTextSettingEdit: () => this.endTextSettingEdit(), - setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active), - setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active), - startVoiceCapture: () => this.startVoiceCapture(), - stopVoiceCapture: () => this.stopVoiceCapture(), - playBuzzerNote: (note, oct, beat) => this.playBuzzerNote(note, oct, beat), + }); + shell.configure({ + actions: { + ...sharedActions, + // Shell overlays (voice dialog, long-press menu) live on the shell + // surface, so their repaints go through the shell render path. + requestRender: () => this.requestShellRender(), + }, + getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()), + requestShellRender: () => this.requestShellRender(), + onScreenStateChanged: (on) => { + const communicator = this.communicator; + if (!communicator) return; + void (async () => { + // Blanking is a compositor-level flag so worker-window surfaces go + // dark too; retained state survives for instant wake. + await communicator.setScreenBlanked(!on); + await communicator.setG2ScreenOn(on); + })().catch((error) => { + this.appendLog(`screen state change failed: ${this.formatError(error)}`); + }); + if (on) this.requestShellRender(); + }, + }); + // The launcher is pinned first in the sidebar and is the boot foreground. + shell.registerWindow( + createLauncherWindow({ + actions: { + ...sharedActions, + requestRender: () => shell.foregroundWindow()?.requestRender(), + }, + apps: [{ appId: "stopwatch", label: "Stopwatch" }], + launchApp: (appId) => this.launchApp(appId), + submitFrame: (image, paintMs, frameId) => + this.submitWindowFrame(LAUNCHER_SURFACE_ID, image, paintMs, frameId), + setSurfaceVisible: (visible) => { + this.setWindowSurfaceVisible(LAUNCHER_SURFACE_ID, visible); + }, + }), + ); + shell.registerWindow({ + appId: "dashboard", + windowId: "dashboard", + title: "Dashboard", + surfaceId: DASHBOARD_WINDOW_SURFACE_ID, + closeable: false, + drawIcon: makeLetterWindowIcon("D"), + handleInput: async (event, frameId) => { + await receiveDashboardWindowInput(event); + await this.requestRender("interval", frameId); + }, + requestRender: () => { + void this.requestRender("interval").catch((error) => { + this.appendLog(`window render failed: ${this.formatError(error)}`); + }); + }, + setForeground: (foreground) => { + this.setWindowSurfaceVisible(DASHBOARD_WINDOW_SURFACE_ID, foreground); + }, }); this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => { void this.handleAndroidNotificationPosted(notificationKey).catch((error) => { @@ -292,7 +379,7 @@ class DashboardController { } setActiveTextSettingValue(value: string): void { - noteDashboardPhoneTextInput(); + shell.noteUserActivity(); const setting = this.activeTextSetting; if (!setting) return; if (setting.get() === value) return; @@ -377,7 +464,12 @@ class DashboardController { headset: state.battery, headsetCharging: state.chargingStatus > 0, }); + shell.setBatteryLevels({ + headset: state.battery, + headsetCharging: state.chargingStatus > 0, + }); if ((this.phase === "connected" || this.phase === "charging") && this.communicator) { + this.requestShellRender(); void this.requestRender("interval").catch((error) => { const message = this.formatError(error); this.appendLog(`battery update failed: ${message}`); @@ -440,9 +532,30 @@ class DashboardController { await mediaControllerBridge.start(); await nightscoutBridge.start(); + // Register the compositor surfaces: the dashboard window in the app + // viewport, and the shell chrome above it. + await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); + await communicator.configureSurface(SHELL_SURFACE_ID, { + x: 0, + y: 0, + width: G2_LENS_WIDTH, + height: G2_LENS_HEIGHT, + zOrder: 1, + transparency: "color-key", + }); + // Window surfaces for every live window (including any launched while + // disconnected); only the foreground window's surface is composited. + const foregroundWindowId = shell.foregroundWindow()?.windowId; + for (const window of shell.getWindows()) { + await this.configureWindowSurface(window.surfaceId, window.windowId === foregroundWindowId); + } await communicator.start(); + shell.foregroundWindow()?.requestRender(); + this.requestShellRender(); await this.requestRender("initial"); this.dashboardTimer = setInterval(() => { + // Also refreshes the top-bar clock on the shell surface. + this.requestShellRender(); void this.requestRender("interval").catch((error) => { const message = this.formatError(error); this.setStatus(`Dashboard update failed: ${message}`); @@ -451,9 +564,9 @@ class DashboardController { }, DASHBOARD_INTERVAL_MS); this.screenTimeoutTimer = setInterval(() => { if (this.phase !== "connected" || !this.communicator) return; - if (!applyDashboardScreenTimeout()) return; - this.setStopwatchRenderActive(false); + if (!shell.applyScreenTimeout()) return; this.endTextSettingEdit(); + this.requestShellRender(); void this.requestRender("interval").catch((error) => { const message = this.formatError(error); this.appendLog(`screen timeout render failed: ${message}`); @@ -570,6 +683,7 @@ class DashboardController { openTelepromptDocument(text); this.appendLog(`teleprompt document received (${text.length} chars)`); if (this.phase === "connected" && this.communicator) { + this.requestShellRender(); await this.requestRender("interval"); return; } @@ -729,7 +843,12 @@ class DashboardController { if (paintUsedStaleData) { frameTimings.logFrame(frameId, "painted with stale data; will schedule a fresh-data repaint"); } - const fingerprint = frameTimings.span(frameId, "fingerprint", () => image.fingerprint()); + // The dashboard still paints the full lens; its window surface only shows + // the viewport-sized top-left crop (content near the right/bottom edges + // clips until the legacy layouts are resized). + const cropped = frameTimings.span(frameId, "crop", () => cropToViewport(image)); + const fingerprint = frameTimings.span(frameId, "fingerprint", () => cropped.fingerprint()); + this.lastWindowImage = cropped; const updatePreviewAfterTransmit = this.phase === "connected"; if (!updatePreviewAfterTransmit) { this.updateDisplayPreviewFromImage(image); @@ -743,23 +862,30 @@ class DashboardController { return; } if (this.communicator) { - if (dashboardState.screenOn) { + if (shell.isScreenOn()) { await this.communicator.setG2ScreenOn(true); } - console.log("submitDashboardImage"); - const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); + console.log("submitSurfaceFrame"); + const buffer = frameTimings.span(frameId, "to8bpp", () => cropped.to8bppBuffer()); // The Java side owns the frame from here: it finishes it on last-packet // ack, dedup, supersede, or timeout. - await this.communicator.submitDashboardImage(buffer, image.width, image.height, fingerprint, paintMs, frameId); + await this.communicator.submitSurfaceFrame( + DASHBOARD_WINDOW_SURFACE_ID, + buffer, + { x: 0, y: 0, width: cropped.width, height: cropped.height }, + fingerprint, + paintMs, + frameId, + ); const outcome = await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); if (outcome === null) { frameTimings.logFrame(frameId, "transmit backpressure wait timed out; render loop continuing"); } - if (!dashboardState.screenOn) { + if (!shell.isScreenOn()) { await this.communicator.setG2ScreenOn(false); } if (updatePreviewAfterTransmit) { - this.updateConnectedDisplayPreviewFromImage(image); + this.updateConnectedCompositePreview(); } } else { frameTimings.finishFrame(frameId, "discarded: no communicator, preview only"); @@ -783,11 +909,16 @@ class DashboardController { const frameId = event.frameId > 0 ? event.frameId : frameTimings.startFrame(`input:${event.kind} (untracked source)`); frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventName(event.eventType)}`); - let renderRequested = false; + let frameOwned = false; try { + const inputEvent = rawInputEventToInputEvent(event); + logToDashboard(inputEventToString(inputEvent)); frameTimings.spanStart(frameId, "handle-input"); + let outcome: ShellInputOutcome; try { - await receiveInput(event); + // The shell consumes its reserved gestures and forwards the rest to + // the focused window; the outcome says which surfaces changed. + outcome = await shell.receiveInput(inputEvent, frameId); } finally { frameTimings.spanEnd(frameId, "handle-input"); } @@ -807,12 +938,7 @@ class DashboardController { if (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) { this.lastInput = eventName(event.eventType); } - renderRequested = true; - await this.requestRender("interval", frameId); - return; - } - - if (event.kind === "text-click" && isDashboardContainerName(event.containerName)) { + } else if (event.kind === "text-click" && isDashboardContainerName(event.containerName)) { if ( event.eventType === OsEventTypeList.SCROLL_TOP_EVENT || event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT @@ -820,17 +946,164 @@ class DashboardController { this.lastSys = `TEXT/${eventName(event.eventType)}`; this.lastInput = `TEXT_${eventName(event.eventType)}`; this.appendLog(`text-event ${this.lastSys}`); - renderRequested = true; - await this.requestRender("interval", frameId); } } + + if (outcome.shell) { + this.requestShellRender(); + } + if (outcome.window) { + // The window adapter owned frameId (render or explicit finish). + frameOwned = true; + } else if (outcome.shell) { + frameOwned = true; + frameTimings.finishFrame(frameId, "input consumed by shell; chrome render scheduled"); + } } finally { - if (!renderRequested) { + if (!frameOwned) { frameTimings.finishFrame(frameId, "discarded: input did not trigger a render"); } } } + /** + * Launch an app from the launcher: allocate a window (in the app's shared + * worker, spawned on first launch), configure its surface, and foreground + * it. Launching an already-running app opens another window of it. + */ + private async launchApp(appId: string): Promise { + if (appId !== "stopwatch") { + this.appendLog(`unknown app: ${appId}`); + return; + } + if (!this.stopwatchHost) { + this.stopwatchHost = new WorkerAppHost("stopwatch", new Worker("../workers/stopwatch-app.worker")); + } + const serial = this.nextWindowSerial++; + const windowId = `stopwatch:${serial}`; + const surfaceId = `window:${windowId}`; + const window = this.stopwatchHost.openWindow({ + windowId, + surfaceId, + title: `Stopwatch ${serial}`, + iconLetter: "S", + viewport: { width: APP_VIEWPORT.width, height: APP_VIEWPORT.height }, + setSurfaceVisible: (visible) => this.setWindowSurfaceVisible(surfaceId, visible), + removeSurface: () => this.removeWindowSurface(surfaceId), + }); + shell.registerWindow(window); + // Configure the surface before foregrounding so the worker's first frame + // has somewhere to land. + await this.configureWindowSurface(surfaceId, false); + shell.focusWindow(windowId); + this.requestShellRender(); + this.appendLog(`launched ${windowId}`); + } + + /** Create/refresh a window surface on the compositor, if connected. */ + private async configureWindowSurface(surfaceId: string, visible: boolean): Promise { + const communicator = this.communicator; + if (!communicator) return; + await communicator.configureSurface(surfaceId, { + x: APP_VIEWPORT.x, + y: APP_VIEWPORT.y, + width: APP_VIEWPORT.width, + height: APP_VIEWPORT.height, + zOrder: 0, + transparency: "opaque", + }); + await communicator.setSurfaceVisible(surfaceId, visible); + } + + private removeWindowSurface(surfaceId: string): void { + const communicator = this.communicator; + if (!communicator) return; + void communicator.removeSurface(surfaceId).catch((error) => { + this.appendLog(`surface removal failed: ${this.formatError(error)}`); + }); + } + + /** Submit a painted frame for an in-process window (e.g. the launcher). */ + private async submitWindowFrame(surfaceId: string, image: GrayImage, paintMs: number, frameId: number): Promise { + const communicator = this.communicator; + if (!communicator || this.phase === "charging") { + frameTimings.finishFrame(frameId, "discarded: window frame with no active connection"); + return; + } + const fingerprint = image.fingerprint(); + const buffer = image.to8bppBuffer(); + await communicator.submitSurfaceFrame( + surfaceId, + buffer, + { x: 0, y: 0, width: image.width, height: image.height }, + fingerprint, + paintMs, + frameId, + ); + } + + /** Flip a window surface's compositor visibility; fire-and-forget. */ + private setWindowSurfaceVisible(surfaceId: string, visible: boolean): void { + const communicator = this.communicator; + if (!communicator) return; + void communicator.setSurfaceVisible(surfaceId, visible).catch((error) => { + this.appendLog(`surface visibility change failed: ${this.formatError(error)}`); + }); + } + + /** + * Re-render and resubmit the shell surface (sidebar, top bar, shell + * overlays). Coalesces like requestRender: one render in flight, at most + * one queued. + */ + requestShellRender(): void { + if (this.shellRenderInProgress) { + this.shellRenderQueued = true; + return; + } + this.shellRenderInProgress = true; + void (async () => { + try { + do { + this.shellRenderQueued = false; + await this.renderShell(); + } while (this.shellRenderQueued); + } catch (error) { + this.appendLog(`shell render failed: ${this.formatError(error)}`); + } finally { + this.shellRenderInProgress = false; + } + })(); + } + + private async renderShell(): Promise { + const frameId = frameTimings.startFrame("render:shell"); + beginRenderPass(true); + const paintStartedAtMs = Date.now(); + const image = frameTimings.span(frameId, "paint", () => + frameTimings.runWithFrame(frameId, () => shell.paintSurface()), + ); + const paintMs = Date.now() - paintStartedAtMs; + endRenderPass(); + this.lastShellImage = image; + if (!this.communicator || this.phase === "charging") { + frameTimings.finishFrame(frameId, "discarded: shell render with no active connection"); + return; + } + const fingerprint = frameTimings.span(frameId, "fingerprint", () => image.fingerprint()); + const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); + await this.communicator.submitSurfaceFrame( + SHELL_SURFACE_ID, + buffer, + { x: 0, y: 0, width: image.width, height: image.height }, + fingerprint, + paintMs, + frameId, + ); + await this.communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); + this.updateConnectedCompositePreview(); + } + private async handleWakeWord(keyword: string): Promise { const normalized = keyword.trim(); if (normalized.length === 0) return; @@ -838,6 +1111,7 @@ class DashboardController { const changed = resetDashboardSleepTimerAndWake(); if (this.phase === "connected" && this.communicator) { if (changed) { + this.requestShellRender(); await this.requestRender("initial"); } return; @@ -859,6 +1133,7 @@ class DashboardController { this.appendLog("android notification woke dashboard"); if (this.phase === "connected" && this.communicator) { + this.requestShellRender(); await this.requestRender("initial"); return; } @@ -873,22 +1148,6 @@ class DashboardController { await this.communicator.playBuzzerNote(note, oct, beat); } - private setStopwatchRenderActive(active: boolean): void { - if (!active || this.phase !== "connected" || !this.communicator) { - if (this.stopwatchRenderTimer) { - clearInterval(this.stopwatchRenderTimer); - this.stopwatchRenderTimer = null; - } - return; - } - if (this.stopwatchRenderTimer) return; - this.stopwatchRenderTimer = setInterval(() => { - void this.requestRender("interval").catch((error) => { - this.appendLog(`stopwatch render failed: ${this.formatError(error)}`); - }); - }, STOPWATCH_RENDER_INTERVAL_MS); - } - private setTranscribeRenderActive(active: boolean): void { if (!active || this.phase !== "connected" || !this.communicator) { if (this.transcribeRenderTimer) { @@ -914,10 +1173,6 @@ class DashboardController { clearInterval(this.screenTimeoutTimer); this.screenTimeoutTimer = null; } - if (this.stopwatchRenderTimer) { - clearInterval(this.stopwatchRenderTimer); - this.stopwatchRenderTimer = null; - } if (this.transcribeRenderTimer) { clearInterval(this.transcribeRenderTimer); this.transcribeRenderTimer = null; @@ -966,7 +1221,23 @@ class DashboardController { this.setDisplayPreview(grayImageToPreviewSource(image)); } - private updateConnectedDisplayPreviewFromImage(image: GrayImage): void { + /** + * Phone-side preview of what is actually on the glasses: the window surface + * in its viewport with the shell chrome composited over it, mirroring the + * Java compositor. + */ + private buildCompositePreviewImage(): GrayImage { + const preview = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + if (this.lastWindowImage) { + preview.bitBlt(this.lastWindowImage, APP_VIEWPORT.x, APP_VIEWPORT.y); + } + if (this.lastShellImage) { + preview.bitBlt(this.lastShellImage, 0, 0, { transparentZero: true }); + } + return preview; + } + + private updateConnectedCompositePreview(): void { if (!this.communicator) return; const now = Date.now(); if ( @@ -976,7 +1247,7 @@ class DashboardController { return; } this.lastConnectedPreviewUpdateAtMs = now; - this.updateDisplayPreviewFromImage(image); + this.updateDisplayPreviewFromImage(this.buildCompositePreviewImage()); } private formatError(error: unknown): string { diff --git a/app/graphics/image.ts b/app/graphics/image.ts index 90258595..85f557d2 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -139,7 +139,14 @@ export class GrayImage { source: GrayImage, dx: number, dy: number, - opts: { sx?: number; sy?: number; width?: number; height?: number } = {}, + opts: { + sx?: number; + sy?: number; + width?: number; + height?: number; + /** Skip source pixels with value 0 (color-key transparency). */ + transparentZero?: boolean; + } = {}, ): void { const srcX = Math.max(0, (opts.sx ?? 0) | 0); const srcY = Math.max(0, (opts.sy ?? 0) | 0); @@ -156,7 +163,9 @@ export class GrayImage { const sx = srcX + col; const tx = destX + col; if (sx < 0 || sx >= source.width || tx < 0 || tx >= this.width) continue; - this.pixels[ty * this.width + tx] = source.pixels[sy * source.width + sx] ?? 0; + const value = source.pixels[sy * source.width + sx] ?? 0; + if (opts.transparentZero && value === 0) continue; + this.pixels[ty * this.width + tx] = value; } } } diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index c7af133b..6244f141 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -33,6 +33,22 @@ export type FirmwareInfo = { capabilities: string; }; +/** + * Compositor surface configuration. Position/size are in screen pixels; + * surfaces composite in ascending zOrder onto a black background. A + * "color-key" surface treats pixel value 0 as transparent and 1 as black + * (painters must clamp intentional black to 1); "opaque" surfaces cover + * their whole rect. + */ +export type SurfaceOptions = { + x: number; + y: number; + width: number; + height: number; + zOrder: number; + transparency: "opaque" | "color-key"; +}; + export type RawInputEvent = | { kind: "list-click"; @@ -315,16 +331,81 @@ export class FaceclawCommunicatorBridge { await this.enqueueJavaCall(() => this.communicator.setFirmwareDebugFlags(Boolean(enabled))); } - async submitDashboardImage(image8bpp: Uint8Array, width: number, height: number, fingerprint: string, paintMs = -1, frameId = 0): Promise { + /** Set the compositor's output frame size. Call before configuring surfaces. */ + async configureCompositorScreen(width: number, height: number): Promise { + await this.enqueueJavaCall(() => { + this.communicator.configureCompositorScreen(Math.round(width), Math.round(height)); + }); + } + + /** + * Create or reconfigure a compositor surface. Geometry changes take effect + * when the next frame is submitted. + */ + async configureSurface(id: string, options: SurfaceOptions): Promise { + const transparency = options.transparency === "color-key" ? 1 : 0; + await this.enqueueJavaCall(() => { + this.communicator.configureSurface( + id, + Math.round(options.x), + Math.round(options.y), + Math.round(options.width), + Math.round(options.height), + Math.round(options.zOrder), + transparency, + ); + }); + } + + async removeSurface(id: string): Promise { + await this.enqueueJavaCall(() => { + this.communicator.removeSurface(id); + }); + } + + /** Show or hide a compositor surface; takes effect at the next composite. */ + async setSurfaceVisible(id: string, visible: boolean): Promise { + await this.enqueueJavaCall(() => { + this.communicator.setSurfaceVisible(id, Boolean(visible)); + }); + } + + /** + * Blank (screen off) or unblank the composited output; retained surface + * state survives, so unblanking restores the screen without repaints. + */ + async setScreenBlanked(blanked: boolean): Promise { + await this.enqueueJavaCall(() => { + this.communicator.setScreenBlanked(Boolean(blanked)); + }); + } + + /** + * Apply an update covering rect (in surface-local coordinates) to a surface + * and submit the recomposited screen to the glasses. pixels8bpp holds + * rect.width*rect.height bytes, row-major; fingerprint identifies the + * surface's full content after the update. + */ + async submitSurfaceFrame( + surfaceId: string, + pixels8bpp: Uint8Array, + rect: { x: number; y: number; width: number; height: number }, + fingerprint: string, + paintMs = -1, + frameId = 0, + ): Promise { // Snapshot because the Java call is deferred; the buffer is passed as an // ArrayBuffer, which NativeScript marshals to a ByteBuffer without the // ~150ms per-element copy a byte[] parameter would need. - const snapshot = new Uint8Array(image8bpp); + const snapshot = new Uint8Array(pixels8bpp); await this.enqueueJavaCall(() => { - this.communicator.submitDashboardImage( + this.communicator.submitSurfaceFrame( snapshot.buffer, - Math.round(width), - Math.round(height), + surfaceId, + Math.round(rect.x), + Math.round(rect.y), + Math.round(rect.width), + Math.round(rect.height), fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), diff --git a/app/native/settings-store.ts b/app/native/settings-store.ts new file mode 100644 index 00000000..e738f410 --- /dev/null +++ b/app/native/settings-store.ts @@ -0,0 +1,67 @@ +/** + * TS wrapper for the Java FaceclawSettings singleton: settings storage shared + * by every isolate (main thread and app workers), with cross-isolate change + * notifications. Each isolate that subscribes registers one Java listener; + * Java dispatches through the registering thread's Looper so callbacks run on + * this isolate's own thread. + */ +import { Utils } from "@nativescript/core"; + +declare const com: any; + +let javaInstance: any = null; +// The Java-side listener proxy must stay referenced or it gets GC'd. +let retainedListenerProxy: any = null; +const changeListeners = new Set<(key: string) => void>(); + +function getJava(): any { + if (javaInstance === null) { + const context = Utils.android?.getApplicationContext?.(); + javaInstance = context + ? com.faceclaw.app.FaceclawSettings.getInstance(context) + : com.faceclaw.app.FaceclawSettings.getInstance(); + } + return javaInstance; +} + +export function getStringSetting(key: string, defaultValue: string): string { + return String(getJava().getString(key, defaultValue)); +} + +export function setStringSetting(key: string, value: string): void { + getJava().setString(key, value); +} + +export function getBooleanSetting(key: string, defaultValue: boolean): boolean { + return Boolean(getJava().getBoolean(key, defaultValue)); +} + +export function setBooleanSetting(key: string, value: boolean): void { + getJava().setBoolean(key, value); +} + +/** + * Subscribe to setting changes from any isolate (including this one). The + * callback runs on this isolate's own thread, one message-loop tick after the + * change. + */ +export function onSettingsStoreChanged(listener: (key: string) => void): () => void { + if (retainedListenerProxy === null) { + retainedListenerProxy = new com.faceclaw.app.FaceclawSettingsListener({ + onSettingChanged: (key: string) => { + for (const registered of Array.from(changeListeners)) { + try { + registered(String(key)); + } catch (error) { + console.warn("settings change listener failed", error); + } + } + }, + }); + getJava().registerListener(retainedListenerProxy); + } + changeListeners.add(listener); + return () => { + changeListeners.delete(listener); + }; +} diff --git a/app/ui/apps/stopwatch.ts b/app/ui/apps/stopwatch.ts deleted file mode 100644 index 988603bf..00000000 --- a/app/ui/apps/stopwatch.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { getDefaultSmallFont, getFont } from "../../graphics/bdffont"; -import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; -import { Layer, type DashboardInputEvent, type LayerContext } from "../layers"; - -const stopwatchFont = getFont("terminus32"); - -export class StopwatchLayer implements Layer { - private readonly startedAtMs = Date.now(); - private pausedAtMs: number | null = null; - private pausedDurationMs = 0; - - paint(ctx: LayerContext): GrayImage { - const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - const elapsedMs = this.elapsedMs(); - const timeLabel = formatStopwatchElapsed(elapsedMs); - const stateLabel = this.pausedAtMs === null ? "Running" : "Paused"; - const smallFont = getDefaultSmallFont(); - - image.drawText(smallFont, 24, 24, "Stopwatch", 180); - - const quadrantWidth = G2_LENS_WIDTH / 2; - const quadrantHeight = G2_LENS_HEIGHT / 2; - const timeX = Math.max(0, Math.round((quadrantWidth - stopwatchFont.measureText(timeLabel)) / 2)); - const timeY = Math.max(0, Math.round((quadrantHeight - stopwatchFont.lineHeight) / 2)); - image.drawText(stopwatchFont, timeX, timeY, timeLabel, 245); - image.drawText(smallFont, 90, 142, stateLabel, 150); - image.drawText(smallFont, 90, 160, "Click: pause / resume", 150); - image.drawText(smallFont, 90, 178, "Double-click: back", 150); - return image; - } - - handleInput(event: DashboardInputEvent, ctx: LayerContext): void { - if (event.type === "double-click") { - void ctx.actions.setStopwatchRenderActive(false); - ctx.stack.pop(); - return; - } - if (event.type === "click") { - if (this.pausedAtMs === null) { - this.pausedAtMs = Date.now(); - void ctx.actions.setStopwatchRenderActive(false); - } else { - this.pausedDurationMs += Date.now() - this.pausedAtMs; - this.pausedAtMs = null; - void ctx.actions.setStopwatchRenderActive(true); - } - } - } - - private elapsedMs(): number { - const now = this.pausedAtMs ?? Date.now(); - return Math.max(0, now - this.startedAtMs - this.pausedDurationMs); - } -} - -function pad2(value: number): string { - return String(value).padStart(2, "0"); -} - -function formatStopwatchElapsed(elapsedMs: number): string { - const totalTenths = Math.floor(elapsedMs / 100); - const tenths = totalTenths % 10; - const totalSeconds = Math.floor(totalTenths / 10); - const seconds = totalSeconds % 60; - const totalMinutes = Math.floor(totalSeconds / 60); - const minutes = totalMinutes % 60; - const hours = Math.floor(totalMinutes / 60); - if (hours > 0) { - return `${hours}:${pad2(minutes)}:${pad2(seconds)}.${tenths}`; - } - return `${pad2(minutes)}:${pad2(seconds)}.${tenths}`; -} diff --git a/app/ui/apps/voice-input.ts b/app/ui/apps/voice-input.ts index db6e15de..51496038 100644 --- a/app/ui/apps/voice-input.ts +++ b/app/ui/apps/voice-input.ts @@ -54,8 +54,9 @@ export class VoiceInputLayer implements Layer { const font = getDefaultSmallFont(); const image = paintBelow(); - // Solid dialog box over the underlying UI. - image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 0, 10); + // Solid dialog box over the underlying UI. Fill 1, not 0: identical after + // 4bpp quantization, but 0 is transparent on the color-key shell surface. + image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10); image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10); const left = DIALOG_X + 16; diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 4b32a020..6f718a9e 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -1,4 +1,10 @@ -import { ApplicationSettings } from "@nativescript/core"; +import { + getBooleanSetting, + getStringSetting, + onSettingsStoreChanged, + setBooleanSetting, + setStringSetting, +} from "~/native/settings-store"; import { getDefaultSmallFont } from "~/graphics/bdffont"; import { wrapText } from "~/graphics/textwrap"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from "./menu"; @@ -34,9 +40,11 @@ type ConfigSettingOptions = { formatValue?: (value: TValue) => string; }; -// Fired after any setting's set(); lets phone-side UI that depends on -// settings toggled from the glasses (e.g. the raw-screenshot button) update -// without waiting for an unrelated snapshot emit. +// Fired after any setting changes, in any isolate (storage lives in the Java +// FaceclawSettings store and broadcasts to every isolate). Lets phone-side UI +// that depends on settings toggled from the glasses (e.g. the raw-screenshot +// button) update without waiting for an unrelated snapshot emit. Delivery is +// asynchronous: one message-loop tick after the set(). const settingChangeListeners = new Set<() => void>(); export function onAnySettingChanged(listener: () => void): () => void { @@ -46,11 +54,11 @@ export function onAnySettingChanged(listener: () => void): () => void { }; } -function notifySettingChanged(): void { +onSettingsStoreChanged(() => { for (const listener of Array.from(settingChangeListeners)) { listener(); } -} +}); export abstract class ConfigSetting { readonly id: TId; @@ -82,12 +90,11 @@ export class ConfigSettingBoolean extends ConfigSet } get(): boolean { - return ApplicationSettings.getBoolean(this.storageKey, this.defaultValue); + return getBooleanSetting(this.storageKey, this.defaultValue); } set(value: boolean): boolean { - ApplicationSettings.setBoolean(this.storageKey, value); - notifySettingChanged(); + setBooleanSetting(this.storageKey, value); return value; } @@ -120,13 +127,12 @@ export class ConfigSettingEnum extends ConfigSett } get(): string { - return this.normalizer(ApplicationSettings.getString(this.storageKey, this.defaultValue)); + return this.normalizer(getStringSetting(this.storageKey, this.defaultValue)); } set(value: string): string { const normalized = this.normalizer(value); - ApplicationSettings.setString(this.storageKey, normalized); - notifySettingChanged(); + setStringSetting(this.storageKey, normalized); return normalized; } } diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts index 3cba75ed..697a7f63 100644 --- a/app/ui/dashboard.ts +++ b/app/ui/dashboard.ts @@ -1,16 +1,14 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; -import { type RawInputEvent } from "../native/faceclaw-communicator"; import { mediaControllerBridge } from "../native/media-controller"; import { nightscoutBridge } from "../native/nightscout-bridge"; -import { EventSourceType, OsEventTypeList } from "../g2/events"; import { getDashboardPlugin, type DashboardPluginCardBounds, type DashboardPluginState } from "./dashboard-plugins"; -import { EditTextSettingLayer, isNightscoutSettingsConfigured, screenTimeoutSettingToMs, screenTimeoutSetting, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings"; +import { EditTextSettingLayer, isNightscoutSettingsConfigured, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings"; import { Layer, LayerActions, LayerStack, type DashboardInputEvent, type LayerContext } from "./layers"; import { SingleNotificationLayer } from "./notifications"; import { TelepromptLayer } from "./apps/teleprompt"; -import { VoiceInputLayer } from "./apps/voice-input"; import { drawSystemCard } from "./dashboard/system-card"; import { createRootMenuLayer } from "./dashboard/root-menu"; +import { inputEventToString, shell } from "./shell/shell"; type DashboardCardId = "system" | DashboardSlotId; export type DashboardBatteryLevels = { @@ -21,16 +19,12 @@ export type DashboardBatteryLevels = { type DashboardState = { logLines: string[]; - screenOn: boolean; - lastInputAtMs: number; telepromptDocumentText: string | null; battery: DashboardBatteryLevels; }; export let dashboardState: DashboardState = { logLines: [] as string[], - screenOn: true, - lastInputAtMs: Date.now(), telepromptDocumentText: null as string | null, battery: { headset: null, @@ -44,7 +38,6 @@ const dashboardActions: LayerActions = { disconnect: () => {}, startTextSettingEdit: () => {}, endTextSettingEdit: () => {}, - setStopwatchRenderActive: () => {}, setTranscribeRenderActive: () => {}, startVoiceCapture: () => {}, stopVoiceCapture: () => {}, @@ -52,114 +45,15 @@ const dashboardActions: LayerActions = { }; export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 }; -// The push-to-talk dialog is a singleton on the layer stack; tracked here so -// long-press-release (a global event) can reach it and so a second long-press -// doesn't stack duplicates. -let activeVoiceInputLayer: VoiceInputLayer | null = null; - -function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent { - if (event.kind === "sys-event") { - if (event.eventType === OsEventTypeList.CLICK_EVENT) { - return { - type: "click", - source: eventSourceToString(event.eventSource), - }; - } else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) { - return { - type: "double-click", - source: eventSourceToString(event.eventSource), - }; - } else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { - return { type: "scroll-down" }; - } else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) { - return { type: "scroll-up" }; - } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) { - // CFW-forwarded long-press (replaces the firmware's force-quit dialog). - // The CFW gates this to the ring, so eventSource may be 0 (unknown); - // eventSourceToString falls back to "ring". - return { type: "long-press", source: eventSourceToString(event.eventSource) }; - } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) { - return { type: "long-press-release", source: eventSourceToString(event.eventSource) }; - } - } else if (event.kind === "text-click") { - if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { - return { type: "scroll-down" }; - } else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) { - return { type: "scroll-up" }; - } - } - return { - type: "unknown", - kind: event.kind, - eventSource: event.eventSource, - eventType: event.eventType, - }; -} - -function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" { - if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) { - return "ring"; - } else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) { - return "left-arm"; - } else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) { - return "right-arm"; - } - return "ring"; -} - -export async function receiveInput(event: RawInputEvent): Promise { - const inputEvent = rawInputEventToInputEvent(event); - dashboardState.lastInputAtMs = Date.now(); - dashboardState.logLines.push(eventToString(inputEvent)); - if (!dashboardState.screenOn) { - if (inputEvent.type === "double-click") { - dashboardState.screenOn = true; - if (dashboardLayers.isAtBase()) { - dashboardLayers.push(createRootMenuLayer()); - } - } - return; - } - // Push-to-talk is intercepted globally so it works over any UI: long-press - // opens the voice dialog and starts the mic; releasing it stops the mic. - if (inputEvent.type === "long-press") { - if (!activeVoiceInputLayer) { - const layer = new VoiceInputLayer(dashboardActions, () => { - if (activeVoiceInputLayer === layer) activeVoiceInputLayer = null; - }); - activeVoiceInputLayer = layer; - dashboardLayers.push(layer); - layer.startCapture(); - } - return; - } - if (inputEvent.type === "long-press-release") { - activeVoiceInputLayer?.endCapture(); - return; - } +/** + * Input handler for the dashboard window. Global gestures (wake, sleep, + * long-press/push-to-talk) are consumed by the shell before this is called. + */ +export async function receiveDashboardWindowInput(inputEvent: DashboardInputEvent): Promise { + dashboardState.logLines.push(inputEventToString(inputEvent)); await dashboardLayers.handleInput(inputEvent); } -function eventToString(event: DashboardInputEvent): string { - switch (event.type) { - case "click": - return `Click from ${event.source}`; - case "double-click": - return `Double click from ${event.source}`; - case "scroll-up": - return `Scroll up`; - case "scroll-down": - return `Scroll down`; - case "long-press": - return `Long press from ${event.source}`; - case "long-press-release": - return `Long press release from ${event.source}`; - default: - case "unknown": - return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`; - } -} - export function logToDashboard(message: string): void { dashboardState.logLines.push(message); } @@ -177,40 +71,26 @@ export function closeDashboardTextSettingEditor(): boolean { } export function drawDashboard(): GrayImage { - if (!dashboardState.screenOn) { + if (!shell.isScreenOn()) { return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); } return dashboardLayers.paint(); } -export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean { - const timeoutMs = screenTimeoutSettingToMs(screenTimeoutSetting.get()); - if (timeoutMs === null || !dashboardState.screenOn) return false; - if (nowMs - dashboardState.lastInputAtMs < timeoutMs) return false; - dashboardState.screenOn = false; - return true; -} - -export function noteDashboardPhoneTextInput(nowMs = Date.now()): void { - dashboardState.lastInputAtMs = nowMs; -} - export function openTelepromptDocument(text?: string): void { if (text !== undefined) { dashboardState.telepromptDocumentText = text; } - dashboardState.lastInputAtMs = Date.now(); - dashboardState.screenOn = true; + shell.wake("window"); dashboardLayers.clearToBase(); dashboardLayers.push(createRootMenuLayer()); dashboardLayers.push(new TelepromptLayer(dashboardState.telepromptDocumentText)); } export function openAndroidNotificationFromSleep(notificationKey: string, nowMs = Date.now()): boolean { - if (!notificationKey || dashboardState.screenOn) return false; + if (!notificationKey || shell.isScreenOn()) return false; - dashboardState.lastInputAtMs = nowMs; - dashboardState.screenOn = true; + shell.wake("window", nowMs); dashboardLayers.clearToBase(); dashboardLayers.push(createRootMenuLayer()); dashboardLayers.push(new SingleNotificationLayer(notificationKey, { @@ -221,15 +101,12 @@ export function openAndroidNotificationFromSleep(notificationKey: string, nowMs } function closeNewNotificationTrigger(ctx: LayerContext): void { - dashboardState.screenOn = false; + shell.sleep(); ctx.stack.clearToBase(); } export function resetDashboardSleepTimerAndWake(nowMs = Date.now()): boolean { - dashboardState.lastInputAtMs = nowMs; - if (dashboardState.screenOn) return false; - - dashboardState.screenOn = true; + if (!shell.wake("sidebar", nowMs)) return false; dashboardLayers.clearToBase(); dashboardLayers.push(createRootMenuLayer()); return true; @@ -254,10 +131,6 @@ export function getPluginIdForSlot(slot: DashboardSlotId): DashboardPluginId { class DashboardLayer implements Layer { paint(): GrayImage { - if (!dashboardState.screenOn) { - return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); - } - const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); const pluginState = getPluginState(); drawSystemCard(image, getCardBounds("system")); @@ -283,9 +156,8 @@ class DashboardLayer implements Layer { } if (event.type === "double-click") { - dashboardState.screenOn = false; - ctx.stack.clearToBase(); - void dashboardActions.endTextSettingEdit(); + // Backing out of the window's root returns focus to the shell sidebar. + shell.yieldFocusToSidebar(); } } } diff --git a/app/ui/dashboard/root-menu.ts b/app/ui/dashboard/root-menu.ts index a3ea3fd5..8af97456 100644 --- a/app/ui/dashboard/root-menu.ts +++ b/app/ui/dashboard/root-menu.ts @@ -8,21 +8,20 @@ import { NotificationsListLayer } from "../notifications"; import { createNightscoutSettingsMenuLayer, createSettingsMenuLayer } from "./settings-menus"; import { TelepromptLayer } from "../apps/teleprompt"; import { TerminalAppLayer } from "../apps/terminal"; -import { StopwatchLayer } from "../apps/stopwatch"; import { TranscribeLayer } from "../apps/transcribe"; import { ScreenTestLayer } from "../apps/screen-test"; import { BuzzerDemoLayer } from "../apps/buzzer-demo"; import { AboutPage } from "./about-page"; import { GrayImage } from "~/graphics/image"; +import { shell } from "../shell/shell"; const rootMenuFont = getDefaultSmallFont(); class RootMenuLayer extends MenuLayer { async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { if (event.type === "double-click") { - dashboardState.screenOn = false; - ctx.stack.clearToBase(); - void ctx.actions.endTextSettingEdit(); + // Backing out of the window's root returns focus to the shell sidebar. + shell.yieldFocusToSidebar(); return; } await super.handleInput(event, ctx); @@ -86,13 +85,6 @@ function createAppsMenuLayer(): MenuLayer { ctx.stack.push(new TelepromptLayer(dashboardState.telepromptDocumentText)); }, }, - { - label: "Stopwatch", - onSelect: (ctx) => { - ctx.stack.push(new StopwatchLayer()); - void ctx.actions.setStopwatchRenderActive(true); - }, - }, { label: "Transcribe", onSelect: (ctx) => { diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 52b1055f..a803715a 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -1,4 +1,5 @@ -import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard"; +import { TOP_LEFT_MENU_LAYOUT } from "../dashboard"; +import { shell } from "../shell/shell"; import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings"; import { MenuLayer } from "../menu"; @@ -59,7 +60,7 @@ function createDisplaySettingsMenuLayer(): MenuLayer { [ enumSettingMenuItem(screenTimeoutSetting, { onChange: (ctx, newValue, oldValue) => { - dashboardState.lastInputAtMs = Date.now(); + shell.noteUserActivity(); }, }), ], diff --git a/app/ui/layers.ts b/app/ui/layers.ts index d431700c..6069328b 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -18,7 +18,6 @@ export type LayerActions = { disconnect: () => Promise | void; startTextSettingEdit: (setting: ConfigSettingString) => Promise | void; endTextSettingEdit: () => Promise | void; - setStopwatchRenderActive: (active: boolean) => Promise | void; setTranscribeRenderActive: (active: boolean) => Promise | void; /** Start voice capture with the provider chosen in settings. */ startVoiceCapture: () => Promise | void; @@ -53,13 +52,17 @@ export interface Layer { export class LayerStack { private readonly layers: Layer[]; private readonly ctx: LayerContext; + private readonly baseWidth: number; + private readonly baseHeight: number; - constructor(baseLayer: Layer, actions: LayerActions) { + constructor(baseLayer: Layer, actions: LayerActions, baseSize?: { width: number; height: number }) { this.layers = [baseLayer]; this.ctx = { stack: this, actions, }; + this.baseWidth = baseSize?.width ?? G2_LENS_WIDTH; + this.baseHeight = baseSize?.height ?? G2_LENS_HEIGHT; } push(layer: Layer): void { @@ -112,7 +115,7 @@ export class LayerStack { return cachedBelow; } if (index <= 0) { - cachedBelow = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + cachedBelow = new GrayImage(this.baseWidth, this.baseHeight, 0); } else if (layer.paintOverBase) { cachedBelow = this.paintLayer(0); } else { diff --git a/app/ui/menu.ts b/app/ui/menu.ts index 17d84089..3109e3d9 100644 --- a/app/ui/menu.ts +++ b/app/ui/menu.ts @@ -61,7 +61,7 @@ export function drawToggleMenuItem( const switchX = x + width - switchWidth - 2; const switchY = y + MENU_TOGGLE_SWITCH_Y_OFFSET; image.drawText(font, x, y + 3, label, 200); - const offFill = selected ? 0 : 18; + const offFill = selected ? 1 : 18; image.fillRoundedRect(switchX, switchY, switchWidth, switchHeight, enabled ? 70 : offFill, 8); image.drawRoundedRect(switchX, switchY, switchWidth, switchHeight, enabled ? 130 : 55, 8); const knobSize = 12; @@ -113,7 +113,9 @@ export class MenuLayer implements Layer { this.clampScrollToSelection(visibleRowCount); const image = paintBelow(); - image.fillRoundedRect(x, y, width, height, 0); + // Fill 1, not 0: identical after 4bpp quantization, but 0 is the + // transparent color key when a menu paints on the shell surface. + image.fillRoundedRect(x, y, width, height, 1); image.drawRoundedRect(x, y, width, height, 72); if (this.title) { image.drawText(font, x + 12, y + 8, this.title, 220); diff --git a/app/ui/shell/chrome-layer.ts b/app/ui/shell/chrome-layer.ts new file mode 100644 index 00000000..dc0b7dcf --- /dev/null +++ b/app/ui/shell/chrome-layer.ts @@ -0,0 +1,114 @@ +import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; +import { getDefaultMediumFont } from "../../graphics/bdffont"; +import { drawBattery } from "../../graphics/battery"; +import { readActiveNotificationIcons } from "../../native/notification-icons"; +import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness"; +import { Layer } from "../layers"; +import { SHELL_OPAQUE_BLACK, SIDEBAR_WIDTH, TOP_BAR_HEIGHT } from "./geometry"; + +const ICON_SIZE = 32; +const ICON_MARGIN_X = ((SIDEBAR_WIDTH - ICON_SIZE) / 2) | 0; +const ICON_SPACING = 8; +const NOTIFICATION_ICON_SIZE = 24; +const BORDER_VALUE = 40; + +export type ShellChromeWindow = { + windowId: string; + title: string; + attention: boolean; + drawIcon: (image: GrayImage, x: number, y: number, size: number) => void; +}; + +export type ShellChromeState = { + windows: ShellChromeWindow[]; + selectedIndex: number; + focus: "sidebar" | "window"; + battery: { headset: number | null; headsetCharging: boolean | null }; +}; + +/** Reusable placeholder window icon: rounded outline with a single letter. */ +export function makeLetterWindowIcon(letter: string): ShellChromeWindow["drawIcon"] { + return (image, x, y, size) => { + const font = getDefaultMediumFont(); + image.drawRoundedRect(x, y, size, size, 120, 6); + const textX = x + Math.max(0, ((size - font.measureText(letter)) / 2) | 0); + const textY = y + Math.max(0, ((size - font.lineHeight) / 2) | 0); + image.drawText(font, textX, textY, letter, 210); + }; +} + +/** + * Base layer of the shell surface: the window sidebar on the left and the + * status top bar. Everything not explicitly painted stays 0 (transparent on + * the color-key shell surface), so the app viewport shows through. + */ +export class ShellChromeLayer implements Layer { + constructor(private readonly getState: () => ShellChromeState) {} + + paint(): GrayImage { + const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + const state = this.getState(); + this.drawSidebar(image, state); + this.drawTopBar(image, state); + return image; + } + + handleInput(): void { + // Shell input is handled by the shell state machine before it reaches the + // layer stack; the chrome itself never consumes events. + } + + private drawSidebar(image: GrayImage, state: ShellChromeState): void { + image.fillRect(0, 0, SIDEBAR_WIDTH, G2_LENS_HEIGHT, SHELL_OPAQUE_BLACK); + image.drawLine(SIDEBAR_WIDTH - 1, 0, SIDEBAR_WIDTH - 1, G2_LENS_HEIGHT - 1, BORDER_VALUE); + for (let index = 0; index < state.windows.length; index++) { + const window = state.windows[index]!; + const y = TOP_BAR_HEIGHT + 6 + index * (ICON_SIZE + ICON_SPACING); + if (y + ICON_SIZE > G2_LENS_HEIGHT) break; + if (index === state.selectedIndex) { + const highlight = state.focus === "sidebar" ? 200 : 70; + image.drawRoundedRect(0, y - 2, SIDEBAR_WIDTH - 3, ICON_SIZE + 4, highlight, 6); + } + window.drawIcon(image, ICON_MARGIN_X, y, ICON_SIZE); + if (window.attention) { + image.fillRoundedRect(ICON_MARGIN_X + ICON_SIZE - 7, y - 1, 8, 8, 255, 4); + } + } + } + + private drawTopBar(image: GrayImage, state: ShellChromeState): void { + const font = getDefaultMediumFont(); + image.fillRect(SIDEBAR_WIDTH, 0, G2_LENS_WIDTH - SIDEBAR_WIDTH, TOP_BAR_HEIGHT, SHELL_OPAQUE_BLACK); + image.drawLine(SIDEBAR_WIDTH, TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, TOP_BAR_HEIGHT - 1, BORDER_VALUE); + + const now = new Date(); + const clock = `${now.getHours()}:${String(now.getMinutes()).padStart(2, "0")}`; + const clockX = SIDEBAR_WIDTH + 10; + const textY = Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0); + image.drawText(font, clockX, textY, clock, 210); + + let batteryLeft = G2_LENS_WIDTH; + if (state.battery.headset !== null) { + const battery = drawBattery(state.battery.headset, Boolean(state.battery.headsetCharging)); + batteryLeft = G2_LENS_WIDTH - battery.width - 8; + image.bitBlt(battery, batteryLeft, Math.max(0, ((TOP_BAR_HEIGHT - battery.height) / 2) | 0), { + transparentZero: true, + }); + } + + const iconsX = clockX + font.measureText(clock) + 16; + const maxIcons = Math.max(0, ((batteryLeft - 8 - iconsX) / (NOTIFICATION_ICON_SIZE + 4)) | 0); + if (maxIcons > 0) { + const { icons, stale } = readActiveNotificationIcons(maxIcons, renderPassAllowsStaleData()); + if (stale) { + noteStaleDataUsed(); + } + const iconY = ((TOP_BAR_HEIGHT - NOTIFICATION_ICON_SIZE) / 2) | 0; + for (let index = 0; index < icons.length; index++) { + image.bitBlt(icons[index]!, iconsX + index * (NOTIFICATION_ICON_SIZE + 4), iconY, { + transparentZero: true, + }); + } + } + } +} diff --git a/app/ui/shell/geometry.ts b/app/ui/shell/geometry.ts new file mode 100644 index 00000000..8313041e --- /dev/null +++ b/app/ui/shell/geometry.ts @@ -0,0 +1,21 @@ +import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../../graphics/image"; + +/** Top bar: 24px notification icons plus a little padding. */ +export const TOP_BAR_HEIGHT = 28; +/** Left sidebar: 32px window icons plus a little padding. */ +export const SIDEBAR_WIDTH = 36; + +/** + * On the color-key shell surface, pixel value 0 is transparent; 1 is the + * darkest opaque shade (identical to 0 after 4bpp quantization). Shell + * painting must use this for intentional black. + */ +export const SHELL_OPAQUE_BLACK = 1; + +/** Screen region belonging to the foreground app window (compile-time fixed). */ +export const APP_VIEWPORT = { + x: SIDEBAR_WIDTH, + y: TOP_BAR_HEIGHT, + width: G2_LENS_WIDTH - SIDEBAR_WIDTH, + height: G2_LENS_HEIGHT - TOP_BAR_HEIGHT, +} as const; diff --git a/app/ui/shell/launcher-app.ts b/app/ui/shell/launcher-app.ts new file mode 100644 index 00000000..e84d298d --- /dev/null +++ b/app/ui/shell/launcher-app.ts @@ -0,0 +1,92 @@ +import { GrayImage } from "../../graphics/image"; +import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack } from "../layers"; +import { MenuLayer } from "../menu"; +import { makeLetterWindowIcon } from "./chrome-layer"; +import { APP_VIEWPORT } from "./geometry"; +import { shell, type ShellWindow } from "./shell"; + +export type LauncherAppEntry = { + appId: string; + label: string; +}; + +export type LauncherOptions = { + actions: LayerActions; + apps: LauncherAppEntry[]; + launchApp: (appId: string) => Promise | void; + /** Submit a painted viewport-sized frame to this window's surface. */ + submitFrame: (image: GrayImage, paintMs: number, frameId: number) => Promise; + /** Flip the launcher's compositor surface visibility on foreground changes. */ + setSurfaceVisible: (visible: boolean) => void; +}; + +export const LAUNCHER_WINDOW_ID = "launcher"; +export const LAUNCHER_SURFACE_ID = "window:launcher"; + +/** The launcher's root menu; backing out of it yields to the sidebar. */ +class LauncherMenuLayer extends MenuLayer { + async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { + if (event.type === "double-click") { + shell.yieldFocusToSidebar(); + return; + } + await super.handleInput(event, ctx); + } +} + +/** + * The launcher: a pinned, uncloseable in-process window listing launchable + * apps. Selecting an app asks the controller to launch it (new window, + * possibly in an existing app worker) and foregrounds the new window. + */ +export function createLauncherWindow(options: LauncherOptions): ShellWindow { + const baseLayer: Layer = new LauncherMenuLayer( + "Apps", + options.apps.map((app) => ({ + label: app.label, + onSelect: async () => { + await options.launchApp(app.appId); + }, + })), + { + x: 8, + y: 8, + width: 272, + minHeight: 0, + maxHeight: APP_VIEWPORT.height - 16, + }, + ); + const stack = new LayerStack(baseLayer, options.actions, { + width: APP_VIEWPORT.width, + height: APP_VIEWPORT.height, + }); + + async function render(frameId: number): Promise { + const paintStartedAtMs = Date.now(); + const image = stack.paint(); + await options.submitFrame(image, Date.now() - paintStartedAtMs, frameId); + } + + return { + appId: "launcher", + windowId: LAUNCHER_WINDOW_ID, + title: "Apps", + surfaceId: LAUNCHER_SURFACE_ID, + closeable: false, + drawIcon: makeLetterWindowIcon("A"), + handleInput: async (event, frameId) => { + await stack.handleInput(event); + // Java-side dedup finishes no-change frames, so an unconditional + // resubmit keeps frame ownership simple. + await render(frameId); + }, + requestRender: () => { + void render(0).catch((error) => { + console.error(`launcher render failed: ${error}`); + }); + }, + setForeground: (foreground) => { + options.setSurfaceVisible(foreground); + }, + }; +} diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts new file mode 100644 index 00000000..e97750bf --- /dev/null +++ b/app/ui/shell/shell.ts @@ -0,0 +1,449 @@ +import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; +import { EventSourceType, OsEventTypeList } from "../../g2/events"; +import type { RawInputEvent } from "../../native/faceclaw-communicator"; +import { DashboardInputEvent, LayerActions, LayerStack } from "../layers"; +import { MenuLayer, type MenuItem } from "../menu"; +import { VoiceInputLayer } from "../apps/voice-input"; +import { ShellChromeLayer, type ShellChromeState, type ShellChromeWindow } from "./chrome-layer"; +import { SIDEBAR_WIDTH, TOP_BAR_HEIGHT } from "./geometry"; + +/** + * The shell: owns the window registry, focus, screen on/off, and the shell + * surface (sidebar + top bar + shell overlays such as the long-press menu and + * the push-to-talk dialog). Runs on the main thread; windows will later live + * in worker threads, today the only window is the in-process dashboard. + * + * Input flow: every event enters via receiveInput. The shell consumes its + * reserved gestures (long-press, and everything while the sidebar or a shell + * overlay has focus) and forwards the rest to the focused window. Windows + * never see events the shell consumed, and the shell keeps working when a + * window's handler hangs (the long-press menu is the escape hatch). + */ + +export type ShellWindow = { + appId: string; + windowId: string; + title: string; + /** Compositor surface this window renders to; configured at connect / launch. */ + surfaceId: string; + /** Whether the shell's long-press menu offers Close Window (launcher and dashboard are pinned). */ + closeable: boolean; + /** App-side cleanup when the shell closes the window (worker notification, surface removal). */ + close?: () => void; + drawIcon: ShellChromeWindow["drawIcon"]; + /** + * Handle an input event the shell forwarded. Ownership of frameId (latency + * tracking) passes to the window: it must eventually reach a frame submit + * or a finishFrame call. + */ + handleInput: (event: DashboardInputEvent, frameId: number) => Promise | void; + /** Repaint and resubmit this window's surface. */ + requestRender: () => void; + /** Foreground state changed: this window's surface is (not) the visible one. */ + setForeground?: (foreground: boolean) => void; + /** Screen turned on/off; hidden or screen-off windows should stop painting. */ + setScreenOn?: (on: boolean) => void; +}; + +export type ShellConfig = { + /** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */ + actions: LayerActions; + getScreenTimeoutMs: () => number | null; + requestShellRender: () => void; + /** Screen on/off changed: the controller blanks/unblanks the compositor. */ + onScreenStateChanged: (on: boolean) => void; +}; + +/** Which surfaces need re-rendering after an input event. */ +export type ShellInputOutcome = { shell: boolean; window: boolean }; + +type FocusKind = "sidebar" | "window"; + +const noopActions: LayerActions = { + requestRender: () => {}, + disconnect: () => {}, + startTextSettingEdit: () => {}, + endTextSettingEdit: () => {}, + setTranscribeRenderActive: () => {}, + startVoiceCapture: () => {}, + stopVoiceCapture: () => {}, + playBuzzerNote: () => {}, +}; + +/** Long-press overlay menu; closing it returns focus to the sidebar. */ +class ShellOverlayMenuLayer extends MenuLayer { + constructor(items: MenuItem[], private readonly onClosed: () => void) { + super(null, items, { + x: SIDEBAR_WIDTH + 8, + y: TOP_BAR_HEIGHT + 8, + width: 272, + minHeight: 0, + }); + } + + onRemoved(): void { + this.onClosed(); + } +} + +class Shell { + private windows: ShellWindow[] = []; + private selectedIndex = 0; + private focus: FocusKind = "sidebar"; + private screenOn = true; + private lastInputAtMs = Date.now(); + private battery: ShellChromeState["battery"] = { headset: null, headsetCharging: null }; + private attention = new Map(); + private activeVoiceLayer: VoiceInputLayer | null = null; + private readonly actions: LayerActions = { ...noopActions }; + private config: ShellConfig = { + actions: noopActions, + getScreenTimeoutMs: () => null, + requestShellRender: () => {}, + onScreenStateChanged: () => {}, + }; + private readonly stack = new LayerStack( + new ShellChromeLayer(() => this.chromeState()), + this.actions, + ); + + configure(config: ShellConfig): void { + this.config = config; + this.stack.setActions(config.actions); + } + + /** Add a window (or replace one with the same windowId, keeping its slot). */ + registerWindow(window: ShellWindow): void { + const existing = this.windows.findIndex((w) => w.windowId === window.windowId); + if (existing >= 0) { + this.windows[existing] = window; + } else { + this.windows.push(window); + } + } + + removeWindow(windowId: string): void { + const index = this.windows.findIndex((w) => w.windowId === windowId); + if (index < 0) return; + const wasSelected = index === this.selectedIndex; + this.windows.splice(index, 1); + this.attention.delete(windowId); + if (this.selectedIndex > index) { + this.selectedIndex--; + } else if (this.selectedIndex >= this.windows.length) { + this.selectedIndex = Math.max(0, this.windows.length - 1); + } + if (this.focus === "window" && (wasSelected || !this.windows.length)) { + this.focus = "sidebar"; + } + if (wasSelected) { + // Hand the foreground to whatever is now selected. + const next = this.windows[this.selectedIndex]; + next?.setForeground?.(true); + next?.requestRender(); + } + this.config.requestShellRender(); + } + + /** Close the foreground window via the shell (long-press menu action). */ + closeForegroundWindow(): void { + const window = this.foregroundWindow(); + if (!window || !window.closeable) return; + try { + window.close?.(); + } catch (error) { + console.warn(`window ${window.windowId} close failed`, error); + } + this.removeWindow(window.windowId); + } + + getWindows(): readonly ShellWindow[] { + return this.windows; + } + + setWindowAttention(windowId: string, attention: boolean): void { + if (Boolean(this.attention.get(windowId)) === attention) return; + this.attention.set(windowId, attention); + this.config.requestShellRender(); + } + + setBatteryLevels(levels: Partial): void { + this.battery = { ...this.battery, ...levels }; + } + + isScreenOn(): boolean { + return this.screenOn; + } + + noteUserActivity(nowMs = Date.now()): void { + this.lastInputAtMs = nowMs; + } + + /** Turn the screen on (if off) and set focus. Returns whether it was off. */ + wake(focus: FocusKind, nowMs = Date.now()): boolean { + this.lastInputAtMs = nowMs; + this.focus = focus; + if (this.screenOn) return false; + this.screenOn = true; + this.config.onScreenStateChanged(true); + for (const window of this.windows) { + window.setScreenOn?.(true); + } + // Refresh the foreground window; the compositor restored its retained + // frame, but its content may be stale (e.g. a running stopwatch). + this.foregroundWindow()?.requestRender(); + return true; + } + + /** Turn the screen off, closing any shell overlays. Sidebar selection is kept. */ + sleep(): void { + if (!this.screenOn) return; + this.screenOn = false; + this.stack.clearToBase(); + for (const window of this.windows) { + window.setScreenOn?.(false); + } + this.config.onScreenStateChanged(false); + } + + /** Foreground and focus a window by id (e.g. a wake path opening content in it). */ + focusWindow(windowId: string): void { + const index = this.windows.findIndex((w) => w.windowId === windowId); + if (index < 0) return; + this.setSelectedIndex(index); + this.focus = "window"; + } + + /** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */ + applyScreenTimeout(nowMs = Date.now()): boolean { + const timeoutMs = this.config.getScreenTimeoutMs(); + if (timeoutMs === null || !this.screenOn) return false; + if (nowMs - this.lastInputAtMs < timeoutMs) return false; + this.sleep(); + return true; + } + + /** Called by a window when the user backs out of its root (double-tap). */ + yieldFocusToSidebar(): void { + if (this.focus === "sidebar") return; + this.focus = "sidebar"; + this.config.requestShellRender(); + } + + /** Paint the shell surface: transparent chrome, or all-transparent when asleep. */ + paintSurface(): GrayImage { + if (!this.screenOn) { + return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + } + return this.stack.paint(); + } + + async receiveInput(event: DashboardInputEvent, frameId = 0): Promise { + this.lastInputAtMs = Date.now(); + + if (!this.screenOn) { + if (event.type === "double-click") { + this.wake("sidebar"); + return { shell: true, window: false }; + } + return { shell: false, window: false }; + } + + // Long-press is reserved by the shell: PTT from the sidebar, the overlay + // menu from inside a window. Windows never see it. + if (event.type === "long-press") { + if (!this.activeVoiceLayer && this.stack.isAtBase()) { + if (this.focus === "sidebar") { + this.openVoiceDialog(); + } else { + this.openOverlayMenu(); + } + } + return { shell: true, window: false }; + } + if (event.type === "long-press-release") { + this.activeVoiceLayer?.endCapture(); + return { shell: true, window: false }; + } + + if (!this.stack.isAtBase()) { + await this.stack.handleInput(event); + return { shell: true, window: false }; + } + + if (this.focus === "sidebar") { + return this.handleSidebarInput(event); + } + + const window = this.foregroundWindow(); + if (window) { + // The window owns frameId from here (render or explicit finish). + await window.handleInput(event, frameId); + return { shell: false, window: true }; + } + return { shell: false, window: false }; + } + + foregroundWindow(): ShellWindow | undefined { + return this.windows[this.selectedIndex] ?? this.windows[0]; + } + + private handleSidebarInput(event: DashboardInputEvent): ShellInputOutcome { + switch (event.type) { + case "double-click": + this.sleep(); + return { shell: true, window: false }; + case "scroll-up": + this.moveSelection(-1); + return { shell: true, window: false }; + case "scroll-down": + this.moveSelection(1); + return { shell: true, window: false }; + case "click": + if (this.windows.length) { + this.focus = "window"; + } + return { shell: true, window: false }; + default: + return { shell: false, window: false }; + } + } + + private moveSelection(delta: number): void { + if (!this.windows.length) return; + const count = this.windows.length; + this.setSelectedIndex((this.selectedIndex + delta + count) % count); + } + + /** Change selection; the selected window is the foreground window. */ + private setSelectedIndex(index: number): void { + if (index === this.selectedIndex) return; + const previous = this.windows[this.selectedIndex]; + this.selectedIndex = index; + const next = this.windows[index]; + previous?.setForeground?.(false); + next?.setForeground?.(true); + next?.requestRender(); + } + + private openVoiceDialog(): void { + const layer = new VoiceInputLayer(this.config.actions, () => { + if (this.activeVoiceLayer === layer) { + this.activeVoiceLayer = null; + } + }); + this.activeVoiceLayer = layer; + this.stack.push(layer); + layer.startCapture(); + } + + private openOverlayMenu(): void { + const foreground = this.foregroundWindow(); + const items: MenuItem[] = [ + { + label: "Voice input", + onSelect: () => { + this.openVoiceDialog(); + }, + }, + ]; + if (foreground?.closeable) { + items.push({ + label: "Close window", + onSelect: (ctx) => { + // Pop the menu first (its onRemoved returns focus to the sidebar), + // then close the window the menu was opened over. + ctx.stack.pop(); + this.closeForegroundWindow(); + }, + }); + } + this.stack.push(new ShellOverlayMenuLayer(items, () => this.yieldFocusToSidebar())); + } + + private chromeState(): ShellChromeState { + return { + windows: this.windows.map((window) => ({ + windowId: window.windowId, + title: window.title, + attention: Boolean(this.attention.get(window.windowId)), + drawIcon: window.drawIcon, + })), + selectedIndex: this.selectedIndex, + focus: this.focus, + battery: this.battery, + }; + } +} + +export const shell = new Shell(); + +export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent { + if (event.kind === "sys-event") { + if (event.eventType === OsEventTypeList.CLICK_EVENT) { + return { + type: "click", + source: eventSourceToString(event.eventSource), + }; + } else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) { + return { + type: "double-click", + source: eventSourceToString(event.eventSource), + }; + } else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { + return { type: "scroll-down" }; + } else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) { + return { type: "scroll-up" }; + } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) { + // CFW-forwarded long-press (replaces the firmware's force-quit dialog). + // The CFW gates this to the ring, so eventSource may be 0 (unknown); + // eventSourceToString falls back to "ring". + return { type: "long-press", source: eventSourceToString(event.eventSource) }; + } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) { + return { type: "long-press-release", source: eventSourceToString(event.eventSource) }; + } + } else if (event.kind === "text-click") { + if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { + return { type: "scroll-down" }; + } else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) { + return { type: "scroll-up" }; + } + } + return { + type: "unknown", + kind: event.kind, + eventSource: event.eventSource, + eventType: event.eventType, + }; +} + +function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" { + if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) { + return "ring"; + } else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) { + return "left-arm"; + } else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) { + return "right-arm"; + } + return "ring"; +} + +export function inputEventToString(event: DashboardInputEvent): string { + switch (event.type) { + case "click": + return `Click from ${event.source}`; + case "double-click": + return `Double click from ${event.source}`; + case "scroll-up": + return `Scroll up`; + case "scroll-down": + return `Scroll down`; + case "long-press": + return `Long press from ${event.source}`; + case "long-press-release": + return `Long press release from ${event.source}`; + default: + case "unknown": + return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`; + } +} diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts new file mode 100644 index 00000000..e41fa0c0 --- /dev/null +++ b/app/ui/shell/worker-window.ts @@ -0,0 +1,105 @@ +import { makeLetterWindowIcon } from "./chrome-layer"; +import { shell, type ShellWindow } from "./shell"; + +/** + * Messages between the shell (main thread) and an app worker. One worker + * hosts one app, which may have several windows; messages are routed by + * windowId. Everything crossing this boundary is small JSON; pixels go + * worker→Java directly. + */ +export type WorkerAppMessage = + | { type: "open-window"; windowId: string; surfaceId: string; viewport: { width: number; height: number } } + | { type: "close-window"; windowId: string } + | { type: "input"; windowId: string; event: unknown; frameId: number } + | { type: "render"; windowId: string } + | { type: "foreground"; windowId: string; foreground: boolean } + | { type: "screen"; on: boolean }; + +export type WorkerAppReply = { type: "yield-focus"; windowId: string }; + +export type WorkerWindowOptions = { + windowId: string; + title: string; + iconLetter: string; + surfaceId: string; + viewport: { width: number; height: number }; + /** Flip the window's compositor surface visibility. */ + setSurfaceVisible: (visible: boolean) => void; + /** Remove the window's compositor surface (on close). */ + removeSurface: () => void; +}; + +/** + * Owns the Worker for one app and adapts its windows to the shell's window + * interface: forwards input and lifecycle over postMessage, relays + * yield-focus back to the shell, and tells the worker when windows open and + * close. The worker submits frames straight to the Java compositor, so no + * pixels cross this boundary. + */ +export class WorkerAppHost { + private readonly openWindows = new Set(); + + constructor( + private readonly appId: string, + private readonly worker: Worker, + ) { + worker.onmessage = (event: MessageEvent) => { + const message = event.data as WorkerAppReply | undefined; + if (message?.type === "yield-focus") { + // Only the focused window's yield is meaningful. + if (shell.foregroundWindow()?.windowId === message.windowId) { + shell.yieldFocusToSidebar(); + } + } + }; + worker.onerror = (error) => { + console.error(`worker app ${appId} error: ${JSON.stringify(error)}`); + }; + } + + windowCount(): number { + return this.openWindows.size; + } + + openWindow(options: WorkerWindowOptions): ShellWindow { + this.openWindows.add(options.windowId); + this.post({ + type: "open-window", + windowId: options.windowId, + surfaceId: options.surfaceId, + viewport: options.viewport, + }); + return { + appId: this.appId, + windowId: options.windowId, + title: options.title, + surfaceId: options.surfaceId, + closeable: true, + close: () => { + this.openWindows.delete(options.windowId); + this.post({ type: "close-window", windowId: options.windowId }); + options.removeSurface(); + }, + drawIcon: makeLetterWindowIcon(options.iconLetter), + handleInput: (event, frameId) => { + this.post({ type: "input", windowId: options.windowId, event, frameId }); + }, + requestRender: () => { + this.post({ type: "render", windowId: options.windowId }); + }, + setForeground: (foreground) => { + options.setSurfaceVisible(foreground); + this.post({ type: "foreground", windowId: options.windowId, foreground }); + }, + setScreenOn: (on) => { + // Screen state is per-app, but sending per-window keeps the protocol + // uniform; the worker treats it globally. + this.post({ type: "screen", on }); + }, + }; + } + + private post(message: WorkerAppMessage): void { + this.worker.postMessage(message); + } +} diff --git a/app/workers/stopwatch-app.worker.ts b/app/workers/stopwatch-app.worker.ts new file mode 100644 index 00000000..c5c9c14c --- /dev/null +++ b/app/workers/stopwatch-app.worker.ts @@ -0,0 +1,201 @@ +/** + * Stopwatch app, hosted in its own worker thread. One worker serves every + * stopwatch window; per-window state is keyed by windowId. The shell (main + * thread) sends open/close/input/lifecycle messages over postMessage; frames + * are painted here and submitted directly to the Java compositor from this + * worker's own thread — no pixels cross the isolate boundary. + */ +import "@nativescript/core/globals"; +import { GrayImage } from "../graphics/image"; +import { getDefaultSmallFont, getFont } from "../graphics/bdffont"; +import * as frameTimings from "../native/frame-timings"; +import type { DashboardInputEvent } from "../ui/layers"; +import type { WorkerAppMessage } from "../ui/shell/worker-window"; + +declare const global: any; +declare const com: any; + +const RENDER_INTERVAL_MS = 100; + +const stopwatchFont = getFont("terminus32"); +const smallFont = getDefaultSmallFont(); + +type StopwatchWindow = { + windowId: string; + surfaceId: string; + viewportWidth: number; + viewportHeight: number; + foreground: boolean; + // Accumulated time survives pausing, backgrounding, and screen-off; only + // rendering pauses. + accumulatedMs: number; + runningSinceMs: number | null; + renderTimer: ReturnType | null; + lastSubmittedFingerprint: string; +}; + +const windows = new Map(); +let screenOn = true; + +global.onmessage = (event: { data: WorkerAppMessage }) => { + const message = event.data; + switch (message.type) { + case "open-window": + windows.set(message.windowId, { + windowId: message.windowId, + surfaceId: message.surfaceId, + viewportWidth: message.viewport.width, + viewportHeight: message.viewport.height, + foreground: false, + accumulatedMs: 0, + runningSinceMs: null, + renderTimer: null, + lastSubmittedFingerprint: "", + }); + break; + case "close-window": { + const window = windows.get(message.windowId); + if (window?.renderTimer) clearInterval(window.renderTimer); + windows.delete(message.windowId); + break; + } + case "input": { + const window = windows.get(message.windowId); + if (!window) { + frameTimings.finishFrame(message.frameId, "discarded: unknown stopwatch window"); + break; + } + handleInput(window, message.event as DashboardInputEvent, message.frameId); + break; + } + case "render": { + const window = windows.get(message.windowId); + if (window) renderAndSubmit(window, 0); + break; + } + case "foreground": { + const window = windows.get(message.windowId); + if (!window) break; + window.foreground = message.foreground; + updateRenderTimer(window); + if (window.foreground) renderAndSubmit(window, 0); + break; + } + case "screen": + screenOn = message.on; + for (const window of windows.values()) { + updateRenderTimer(window); + } + break; + } +}; + +function handleInput(window: StopwatchWindow, event: DashboardInputEvent, frameId: number): void { + if (event.type === "double-click") { + // Back out of the app root: return focus to the shell sidebar. State keeps + // running; the frame is finished here since no repaint happens. + frameTimings.finishFrame(frameId, "discarded: stopwatch yielded focus"); + global.postMessage({ type: "yield-focus", windowId: window.windowId }); + return; + } + if (event.type === "click") { + if (window.runningSinceMs === null) { + window.runningSinceMs = Date.now(); + } else { + window.accumulatedMs += Date.now() - window.runningSinceMs; + window.runningSinceMs = null; + } + updateRenderTimer(window); + renderAndSubmit(window, frameId); + return; + } + frameTimings.finishFrame(frameId, "discarded: stopwatch ignored input"); +} + +function updateRenderTimer(window: StopwatchWindow): void { + const shouldRun = window.runningSinceMs !== null && window.foreground && screenOn; + if (shouldRun && window.renderTimer === null) { + window.renderTimer = setInterval(() => renderAndSubmit(window, 0), RENDER_INTERVAL_MS); + } else if (!shouldRun && window.renderTimer !== null) { + clearInterval(window.renderTimer); + window.renderTimer = null; + } +} + +function elapsedMs(window: StopwatchWindow): number { + const runningMs = window.runningSinceMs === null ? 0 : Date.now() - window.runningSinceMs; + return window.accumulatedMs + runningMs; +} + +function paint(window: StopwatchWindow): GrayImage { + const image = new GrayImage(window.viewportWidth, window.viewportHeight, 0); + const elapsed = elapsedMs(window); + const timeLabel = formatStopwatchElapsed(elapsed); + const stateLabel = window.runningSinceMs === null ? (elapsed > 0 ? "Paused" : "Ready") : "Running"; + + image.drawText(smallFont, 24, 16, "Stopwatch", 180); + const timeX = Math.max(0, Math.round((window.viewportWidth - stopwatchFont.measureText(timeLabel)) / 2)); + const timeY = Math.max(0, Math.round((window.viewportHeight / 2 - stopwatchFont.lineHeight) / 2) + 20); + image.drawText(stopwatchFont, timeX, timeY, timeLabel, 245); + + const hintsY = window.viewportHeight - 66; + image.drawText(smallFont, 90, hintsY, stateLabel, 150); + image.drawText(smallFont, 90, hintsY + 18, "Click: start / pause", 150); + image.drawText(smallFont, 90, hintsY + 36, "Double-click: back to sidebar", 150); + return image; +} + +function renderAndSubmit(window: StopwatchWindow, inputFrameId: number): void { + const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`); + try { + const paintStartedAtMs = Date.now(); + const image = frameTimings.span(frameId, "paint", () => + frameTimings.runWithFrame(frameId, () => paint(window)), + ); + const paintMs = Date.now() - paintStartedAtMs; + const fingerprint = image.fingerprint(); + if (fingerprint === window.lastSubmittedFingerprint) { + frameTimings.finishFrame(frameId, "discarded: stopwatch content unchanged"); + return; + } + const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + if (!communicator) { + frameTimings.finishFrame(frameId, "discarded: no active communicator"); + return; + } + const buffer = image.to8bppBuffer(); + communicator.submitSurfaceFrame( + buffer.buffer, + window.surfaceId, + 0, + 0, + image.width, + image.height, + fingerprint, + paintMs, + frameId, + ); + window.lastSubmittedFingerprint = fingerprint; + } catch (error) { + frameTimings.finishFrame(frameId, `discarded: stopwatch render failed`); + console.error(`stopwatch worker render failed: ${error}`); + } +} + +function pad2(value: number): string { + return String(value).padStart(2, "0"); +} + +function formatStopwatchElapsed(totalMs: number): string { + const totalTenths = Math.floor(totalMs / 100); + const tenths = totalTenths % 10; + const totalSeconds = Math.floor(totalTenths / 10); + const seconds = totalSeconds % 60; + const totalMinutes = Math.floor(totalSeconds / 60); + const minutes = totalMinutes % 60; + const hours = Math.floor(totalMinutes / 60); + if (hours > 0) { + return `${hours}:${pad2(minutes)}:${pad2(seconds)}.${tenths}`; + } + return `${pad2(minutes)}:${pad2(seconds)}.${tenths}`; +}