From 602f66f15b7caa39444089ab03dce590f29c0ac3 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 30 Aug 2026 19:15:10 -0700 Subject: [PATCH] Make preview-only mode work --- .../app/FaceclawPreviewCompositor.java | 185 +++++++++++++++ CHANGELOG | 5 +- app/apps/blocks/blocks-app.worker.ts | 5 +- app/apps/freecell/freecell-app.worker.ts | 5 +- .../minesweeper/minesweeper-app.worker.ts | 5 +- app/apps/navigate/navigate-app.worker.ts | 5 +- app/apps/pinball/pinball-app.worker.ts | 5 +- app/apps/roam/roam-app.worker.ts | 5 +- app/apps/terminal/terminal-app.worker.ts | 5 +- app/apps/timer/timer-app.worker.ts | 5 +- app/g2/dashboard-controller.ts | 223 ++++++++++++++---- app/g2/glasses-display-state.ts | 6 +- app/native/active-display.ts | 19 ++ app/native/preview-display.ts | 169 +++++++++++++ app/phone-ui/main-view-model.ts | 23 +- 15 files changed, 607 insertions(+), 63 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java create mode 100644 app/native/active-display.ts create mode 100644 app/native/preview-display.ts diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java new file mode 100644 index 00000000..06d803c4 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java @@ -0,0 +1,185 @@ +package com.faceclaw.app; + +import android.content.Context; + +/** + * Headless display target for preview-only mode (no glasses paired). Hosts + * the same SurfaceCompositor the BLE communicator uses, so the TS side can + * run its full surface/render pipeline unchanged; composited frames simply + * have nowhere to go, and the phone-mirror preview, screenshots, and GIF + * recordings read the retained composite back exactly as they do on a live + * connection. + * + * The method signatures mirror the display subset of FaceclawBleCommunicator + * (the TS DisplayTarget interface is typed against that subset). + */ +public final class FaceclawPreviewCompositor { + private final Context appContext; + private final SurfaceCompositor compositor = new SurfaceCompositor(); + private final android.os.Handler mainHandler = new android.os.Handler(android.os.Looper.getMainLooper()); + + // Notified (on the main looper, where the phone UI's JS runs) after each + // applied surface frame, standing in for the BLE path's per-frame metrics + // callback so the phone mirror refreshes promptly instead of waiting for + // its safety-net poll. + private volatile Runnable frameListener; + + // Active animated-GIF screen recording, or null when idle. Frames are + // pushed by recordScreenFrame(), which the TS side calls at each + // phone-preview flush. + private volatile GifScreenRecorder screenRecorder; + + // Worker-app isolates locate their frame target through a static, like + // FaceclawBleCommunicator.getActive(); Java statics are shared across + // isolates, unlike anything on the JS side. + private static volatile FaceclawPreviewCompositor activeInstance; + + public static FaceclawPreviewCompositor getActive() { + return activeInstance; + } + + public FaceclawPreviewCompositor(Context context) { + this.appContext = context.getApplicationContext(); + } + + /** Publish this compositor as the workers' frame target. */ + public void makeActive() { + activeInstance = this; + } + + /** Withdraw from workers (a real connection is taking over). */ + public void release() { + if (activeInstance == this) { + activeInstance = null; + } + frameListener = null; + } + + public void setFrameListener(Runnable listener) { + this.frameListener = listener; + } + + /** Set the compositor's output frame size. Call before configuring surfaces. */ + public void configureCompositorScreen(int width, int height) { + compositor.configureScreen(width, height); + } + + public void configureSurface(String id, int x, int y, int width, int height, int zOrder, int transparency) { + compositor.configureSurface(id, x, y, width, height, zOrder, transparency); + } + + public void removeSurface(String id) { + compositor.removeSurface(id); + } + + public void setSurfaceVisible(String id, boolean visible) { + compositor.setSurfaceVisible(id, visible); + } + + /** Blank (screen off) or unblank the output; retained surface state survives. */ + public void setScreenBlanked(boolean blanked) { + compositor.setBlanked(blanked); + } + + /** + * Apply an update to one compositor surface. Unlike the connected path + * there is no desired-frame store or transmit pipeline downstream; the + * preview/screenshot calls recomposite the retained state on demand, so + * applying the update is the whole job and the frame finishes here. + */ + public void submitSurfaceFrame( + java.nio.ByteBuffer pixels8bpp, + String surfaceId, + int rectX, + int rectY, + int rectWidth, + int rectHeight, + String contentFingerprint, + int paintMs, + int frameId, + java.nio.ByteBuffer glyphs + ) { + compositor.applyAndComposite( + surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs); + FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)"); + Runnable listener = frameListener; + if (listener != null) { + mainHandler.post(listener); + } + } + + /** + * The current composited screen as a phone-UI preview bitmap, or null + * before any surface has been configured. + */ + public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return null; + } + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green); + } + + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ + public String saveCompositePngScreenshot() throws java.io.IOException { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return ""; + } + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); + } + + /** + * Save the current composite cropped to the given screen rect (the region + * the shell says is actually occupied). The rect is clamped to the screen; + * a degenerate rect falls back to the full screen. + */ + public String saveCompositePngScreenshot(int cropX, int cropY, int cropWidth, int cropHeight) + throws java.io.IOException { + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return ""; + } + int x = Math.max(0, cropX); + int y = Math.max(0, cropY); + int width = Math.min(composite.width - x, cropWidth - (x - cropX)); + int height = Math.min(composite.height - y, cropHeight - (y - cropY)); + if (width <= 0 || height <= 0 || (x == 0 && y == 0 && width == composite.width && height == composite.height)) { + return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height); + } + byte[] cropped = new byte[width * height]; + for (int row = 0; row < height; row++) { + System.arraycopy(composite.gray, (y + row) * composite.width + x, cropped, row * width, width); + } + return ScreenshotUtil.savePngScreenshot(appContext, cropped, width, height); + } + + /** Begin collecting composite frames for an animated-GIF screen recording. */ + public void startScreenRecording() { + screenRecorder = new GifScreenRecorder(); + } + + /** Capture the current composite into the active recording; no-op when idle. */ + public void recordScreenFrame() { + GifScreenRecorder recorder = screenRecorder; + if (recorder == null) { + return; + } + SurfaceCompositor.Composite composite = compositor.previewComposite(); + if (composite == null) { + return; + } + recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()); + } + + /** Finish the recording and save it as an animated GIF; returns the path or "". */ + public String stopScreenRecording() throws java.io.IOException { + GifScreenRecorder recorder = screenRecorder; + screenRecorder = null; + if (recorder == null) { + return ""; + } + return recorder.save(appContext); + } +} diff --git a/CHANGELOG b/CHANGELOG index c6743e0b..28e8893f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ # Chagelog -## 0.6.1 - Future +## 0.6.2 - Future +* Preview Only mode works + +## 0.6.1 - 2026-08-30 * Settings import/export scripts work on release builds * Fixed an issue where the onboarding flow failed to detect the existing glasses firmware version * Update launcher, files, and game UIs to work better with watch input diff --git a/app/apps/blocks/blocks-app.worker.ts b/app/apps/blocks/blocks-app.worker.ts index 15db7d81..4c9b270e 100644 --- a/app/apps/blocks/blocks-app.worker.ts +++ b/app/apps/blocks/blocks-app.worker.ts @@ -15,6 +15,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects"; import { defaultWindowMenuItems, WindowMenu } from "../../ui/window-menu"; import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-window"; @@ -666,9 +667,9 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: blocks content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/freecell/freecell-app.worker.ts b/app/apps/freecell/freecell-app.worker.ts index 2294232c..2529c48d 100644 --- a/app/apps/freecell/freecell-app.worker.ts +++ b/app/apps/freecell/freecell-app.worker.ts @@ -20,6 +20,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects"; import { defaultWindowMenuItems, WindowMenu } from "../../ui/window-menu"; import type { MenuItem } from "../../ui/menu"; @@ -775,9 +776,9 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: freecell content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/minesweeper/minesweeper-app.worker.ts b/app/apps/minesweeper/minesweeper-app.worker.ts index f42cea93..edc91513 100644 --- a/app/apps/minesweeper/minesweeper-app.worker.ts +++ b/app/apps/minesweeper/minesweeper-app.worker.ts @@ -20,6 +20,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects"; import { defaultWindowMenuItems, WindowMenu } from "../../ui/window-menu"; import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-window"; @@ -739,9 +740,9 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void frameTimings.finishFrame(frameId, "discarded: minesweeper content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/navigate/navigate-app.worker.ts b/app/apps/navigate/navigate-app.worker.ts index 8dbaff7c..96404d6b 100644 --- a/app/apps/navigate/navigate-app.worker.ts +++ b/app/apps/navigate/navigate-app.worker.ts @@ -13,6 +13,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { defaultWindowMenuItems, WindowMenu } from "../../ui/window-menu"; import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-window"; import type { ToolSpec, ToolResult } from "../../assistant/tool-registry"; @@ -878,9 +879,9 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: navigate content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/pinball/pinball-app.worker.ts b/app/apps/pinball/pinball-app.worker.ts index a35eb533..65bf6348 100644 --- a/app/apps/pinball/pinball-app.worker.ts +++ b/app/apps/pinball/pinball-app.worker.ts @@ -26,6 +26,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { getStringSetting, setStringSetting } from "../../native/settings-store"; import { buildSoundSequencePayload, type Step } from "../../ui/sound-effects"; import { defaultWindowMenuItems, WindowMenu } from "../../ui/window-menu"; @@ -1070,9 +1071,9 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: pinball content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/roam/roam-app.worker.ts b/app/apps/roam/roam-app.worker.ts index 88384a42..d78ea93a 100644 --- a/app/apps/roam/roam-app.worker.ts +++ b/app/apps/roam/roam-app.worker.ts @@ -14,6 +14,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { truncateText, wrapText } from "../../graphics/textwrap"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { onSettingsStoreChanged } from "../../native/settings-store"; import type { MenuItem } from "../../ui/menu"; import { defaultWindowMenuItems, WindowMenu } from "../../ui/window-menu"; @@ -664,9 +665,9 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: roam content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/terminal/terminal-app.worker.ts b/app/apps/terminal/terminal-app.worker.ts index 941d9381..961f267c 100644 --- a/app/apps/terminal/terminal-app.worker.ts +++ b/app/apps/terminal/terminal-app.worker.ts @@ -38,6 +38,7 @@ import { getDefaultSmallFont, getTerminalFontConfig } from "../../graphics/ui-fo import { truncateText } from "../../graphics/textwrap"; import { TERMINAL_ICON_GLYPHS } from "../../graphics/icons"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures"; import { G2MirrorClient, type G2MirrorClientOptions, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client"; import { onSettingsStoreChanged } from "../../native/settings-store"; @@ -1595,9 +1596,9 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: terminal content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/apps/timer/timer-app.worker.ts b/app/apps/timer/timer-app.worker.ts index b12887ce..1fdd1a92 100644 --- a/app/apps/timer/timer-app.worker.ts +++ b/app/apps/timer/timer-app.worker.ts @@ -10,6 +10,7 @@ import { prepareFrameDraws } from "../../graphics/glyph-wire"; import { getFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import * as frameTimings from "../../native/frame-timings"; +import { getActiveDisplay } from "../../native/active-display"; import { cancelTimerNotification, fireTimerNotification, @@ -664,9 +665,9 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void { frameTimings.finishFrame(frameId, "discarded: timer content unchanged"); return; } - const communicator = com.faceclaw.app.FaceclawBleCommunicator.getActive(); + const communicator = getActiveDisplay(); if (!communicator) { - frameTimings.finishFrame(frameId, "discarded: no active communicator"); + frameTimings.finishFrame(frameId, "discarded: no active display"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 64ae67e6..f2b74f65 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -69,6 +69,8 @@ import { closeRunningPackage, launchInstalledPackage } from "../apps/evenhub/man import { wearerVerificationOptions } from "../apps/microphones/speakers"; import { micSession } from "../apps/microphones/mic-session"; import { glassesDisplayLabel } from "./glasses-display-state"; +import { PreviewDisplayTarget, type DisplayTarget } from "../native/preview-display"; +import { isPreviewOnlyMode } from "../phone-ui/onboarding-state"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting"; @@ -100,6 +102,12 @@ export type DashboardSnapshot = { firmwareWarningVisible: boolean; screenRecordingActive: boolean; batteryOptimizationWarningVisible: boolean; + /** + * True while the headless preview display is standing in for a glasses + * connection (preview-only mode): the mirror is live and interactive, but + * nothing is paired, so "Disconnected" would be the wrong label. + */ + previewMode: boolean; }; type DashboardListener = (snapshot: DashboardSnapshot) => void; @@ -208,6 +216,9 @@ class DashboardController { private welcomeSoundArmed = false; private communicator: FaceclawCommunicatorBridge | null = null; + // Headless stand-in for the compositor when no glasses are paired + // (preview-only mode); a real connection always outranks it. See `display`. + private previewTarget: PreviewDisplayTarget | null = null; private shellRefreshTimer: ReturnType | null = null; private previewTimer: ReturnType | null = null; private screenTimeoutTimer: ReturnType | null = null; @@ -483,16 +494,28 @@ class DashboardController { } const communicator = this.communicator; - if (!communicator) return; - // Blanking is a compositor-level flag so worker-window surfaces go dark - // too; retained state survives while the EvenHub page is absent. - void (async () => { - await communicator.setScreenBlanked(true); - await communicator.setG2ScreenOn(false); - })().catch((error) => { - this.appendLog(`screen sleep failed: ${this.formatError(error)}`); - }); - this.scheduleEvenHubSuspend(); + if (communicator) { + // Blanking is a compositor-level flag so worker-window surfaces go dark + // too; retained state survives while the EvenHub page is absent. + void (async () => { + await communicator.setScreenBlanked(true); + await communicator.setG2ScreenOn(false); + })().catch((error) => { + this.appendLog(`screen sleep failed: ${this.formatError(error)}`); + }); + this.scheduleEvenHubSuspend(); + return; + } + const preview = this.previewTarget; + if (!preview) return; + // Preview mode: the mirror goes dark like the lens would; retained + // surfaces survive for the next wake. + void preview + .setScreenBlanked(true) + .then(() => this.schedulePreviewUpdate()) + .catch((error) => { + this.appendLog(`preview screen sleep failed: ${this.formatError(error)}`); + }); } private syncLockScreenSettingIfChanged(): void { @@ -599,7 +622,17 @@ class DashboardController { private ensureEvenHubSessionActive(frameId = 0): Promise { if (this.evenHubResumePromise) return this.evenHubResumePromise; const communicator = this.communicator; - if (!communicator || this.phase === "charging" || this.phase === "disconnected") { + if (!communicator) { + // Preview mode has no plugin session to restore; waking is just + // unblanking the headless compositor so the mirror lights back up. + const preview = this.previewTarget; + if (!preview) return Promise.resolve(false); + return preview.setScreenBlanked(false).then(() => { + this.schedulePreviewUpdate(); + return true; + }); + } + if (this.phase === "charging" || this.phase === "disconnected") { return Promise.resolve(false); } @@ -792,8 +825,8 @@ class DashboardController { /** Save the occupied part of the composited screen as a 4-bit grayscale PNG. */ saveScreenshot(): string { - const path = this.communicator?.saveScreenshot(shell.screenshotCropRect()) ?? ""; - this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: not connected"); + const path = this.display?.saveScreenshot(shell.screenshotCropRect()) ?? ""; + this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: no display"); return path; } @@ -804,15 +837,15 @@ class DashboardController { */ startScreenRecording(): void { if (this.screenRecordingActive) return; - const communicator = this.communicator; - if (!communicator) { - this.appendLog("screen recording skipped: not connected"); + const display = this.display; + if (!display) { + this.appendLog("screen recording skipped: no display"); return; } - communicator.startScreenRecording(); + display.startScreenRecording(); // Capture the starting frame immediately rather than waiting for the // next preview flush. - communicator.recordScreenFrame(); + display.recordScreenFrame(); this.screenRecordingActive = true; this.appendLog("screen recording started"); this.emit(); @@ -824,7 +857,7 @@ class DashboardController { this.screenRecordingActive = false; let path = ""; try { - path = this.communicator?.stopScreenRecording() ?? ""; + path = this.display?.stopScreenRecording() ?? ""; } finally { this.emit(); } @@ -866,6 +899,7 @@ class DashboardController { firmwareWarningVisible: this.firmwareWarningMessage.length > 0, screenRecordingActive: this.screenRecordingActive, batteryOptimizationWarningVisible: this.batteryOptimizationWarningVisible, + previewMode: this.isPreviewDisplayActive(), }; } @@ -883,6 +917,11 @@ class DashboardController { if (this.silentMode && this.phase === "connected") { return "Connected (Silent mode enabled)"; } + // The preview compositor keeps the mirror live (including as a black + // frame while the simulated screen is off), so never cover it. + if (this.isPreviewDisplayActive()) { + return ""; + } const connectionFailing = this.phase === "disconnected" || this.phase === "connecting"; if ( connectionFailing && @@ -902,6 +941,7 @@ class DashboardController { screenOn: shell.isScreenOn(), battery: this.lastHeadsetBattery, foregroundTitle: shell.getForegroundApp()?.title ?? null, + previewMode: this.isPreviewDisplayActive(), }); } @@ -974,6 +1014,95 @@ class DashboardController { this.emit(); } + /** + * Where composited frames go and where the phone mirror, screenshots, and + * recordings come from: the live glasses connection when there is one, + * otherwise the headless preview compositor (preview-only mode), otherwise + * nothing (frames are discarded). + */ + private get display(): DisplayTarget | null { + return this.communicator ?? this.previewTarget; + } + + private isPreviewDisplayActive(): boolean { + return this.communicator === null && this.previewTarget !== null; + } + + /** + * Bring up the headless preview display for preview-only mode (onboarding's + * "Preview Only" path): the same shell/app/compositor pipeline as a live + * session minus the BLE transport, so the phone mirror and its touch/watch + * controls work with no glasses paired. Idempotent; called from the main + * page alongside autoConnect. A real connection outranks it — connect() + * tears it down before creating the communicator. + */ + async ensurePreviewDisplay(): Promise { + if (!isPreviewOnlyMode()) return; + if (this.communicator || this.previewTarget || this.connectRunning) return; + const target = new PreviewDisplayTarget(); + // Published before the awaits below so a second call (or connect()) sees + // it; the target's calls all complete in microtasks, so the screen size + // is in place before any real render can reach the compositor. + this.previewTarget = target; + try { + await target.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); + await target.configureSurface(SHELL_SURFACE_ID, { + x: 0, + y: 0, + width: G2_LENS_WIDTH, + height: G2_LENS_HEIGHT, + zOrder: 1, + transparency: "color-key", + }); + const foregroundWindowId = shell.foregroundWindow()?.windowId; + for (const window of shell.getWindows()) { + await this.configureWindowSurface( + window.surfaceId, + window.windowId === foregroundWindowId, + window.heightMode, + target, + ); + } + } catch (error) { + if (this.previewTarget === target) this.previewTarget = null; + this.appendLog(`preview display setup failed: ${this.formatError(error)}`); + return; + } + // Workers find the compositor through its Java static; the per-frame + // callback keeps the mirror current (the connected path gets the same + // from Java's frame metrics). + target.activate(() => this.schedulePreviewUpdate()); + this.setStatus("Preview mode (no glasses paired)."); + this.appendLog("Preview-only display active; frames render to the phone mirror only."); + shell.foregroundWindow()?.requestRender(); + this.requestShellRender(); + // The same recurring upkeep a live session gets: top-bar clock refresh, + // the preview safety-net poll, and the screen timeout (a timed-out + // preview screen wakes on double-tap, like the glasses would). + this.shellRefreshTimer = setInterval(() => { + this.requestShellRender(); + this.updateCompositePreview(); + }, SHELL_REFRESH_INTERVAL_MS); + this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS); + this.screenTimeoutTimer = setInterval(() => { + if (!this.isPreviewDisplayActive()) return; + if (!shell.applyScreenTimeout()) return; + this.endTextSettingEdit(); + this.requestShellRender(); + }, SCREEN_TIMEOUT_CHECK_MS); + this.emit(); + } + + /** Drop the headless preview display (a real connection is taking over). */ + private teardownPreviewDisplay(): void { + const target = this.previewTarget; + if (!target) return; + this.previewTarget = null; + target.release(); + this.clearDashboardTimer(); + this.appendLog("Preview-only display released."); + } + async connect(): Promise { if (this.phase !== "disconnected") return; // Connecting is the explicit way out of the manual-disconnected state. @@ -987,6 +1116,9 @@ class DashboardController { this.appendLog(`error: ${message}`); throw new Error(message); } + // Glasses are configured, so the real pipeline owns the display from here + // (also frees the preview timers for connect() to replace). + this.teardownPreviewDisplay(); this.lastInput = "waiting..."; this.lastSys = "none yet"; this.welcomeSoundArmed = isWelcomeSoundPending(); @@ -1503,7 +1635,7 @@ class DashboardController { * Coordinates are fractions of the mirror (0..1). */ async handleMirrorTouch(kind: MirrorTouchKind, nx: number, ny: number): Promise { - if (this.phase !== "connected" && this.phase !== "charging") return; + if (this.phase !== "connected" && this.phase !== "charging" && !this.isPreviewDisplayActive()) return; if (this.glassesLocked) { // Same as the ring: only a double-tap reaches a locked display. if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch"); @@ -1979,35 +2111,36 @@ class DashboardController { } } - /** Create/refresh a window surface on the compositor, if connected. */ + /** Create/refresh a window surface on the compositor, if a display target exists. */ private async configureWindowSurface( surfaceId: string, visible: boolean, heightMode: WindowHeightMode = "min", + // ensurePreviewDisplay passes its not-yet-published target explicitly. + target: DisplayTarget | null = this.display, ): Promise { - const communicator = this.communicator; - if (!communicator) return; - await communicator.configureSurface(surfaceId, { + if (!target) return; + await target.configureSurface(surfaceId, { ...appViewportRect(heightMode), zOrder: 0, transparency: "opaque", }); - await communicator.setSurfaceVisible(surfaceId, visible); + await target.setSurfaceVisible(surfaceId, visible); } private removeWindowSurface(surfaceId: string): void { - const communicator = this.communicator; - if (!communicator) return; - void communicator.removeSurface(surfaceId).catch((error) => { + const display = this.display; + if (!display) return; + void display.removeSurface(surfaceId).catch((error) => { this.appendLog(`surface removal failed: ${this.formatError(error)}`); }); } /** Submit a painted frame for an in-process window (e.g. the launcher). */ private async submitWindowFrame(surfaceId: string, planes: Plane[], paintMs: number, frameId: number): Promise { - const communicator = this.communicator; - if (!communicator || this.phase === "charging") { - frameTimings.finishFrame(frameId, "discarded: window frame with no active connection"); + const display = this.display; + if (!display || this.phase === "charging") { + frameTimings.finishFrame(frameId, "discarded: window frame with no display target"); return; } const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); @@ -2015,7 +2148,7 @@ class DashboardController { const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)); await frameTimings.spanAsync(frameId, "submit", () => - communicator.submitSurfaceFrame( + display.submitSurfaceFrame( surfaceId, buffer, { x: 0, y: 0, width: image.width, height: image.height }, @@ -2029,9 +2162,9 @@ class DashboardController { /** Flip a window surface's compositor visibility; fire-and-forget. */ private setWindowSurfaceVisible(surfaceId: string, visible: boolean): void { - const communicator = this.communicator; - if (!communicator) return; - void communicator.setSurfaceVisible(surfaceId, visible).catch((error) => { + const display = this.display; + if (!display) return; + void display.setSurfaceVisible(surfaceId, visible).catch((error) => { this.appendLog(`surface visibility change failed: ${this.formatError(error)}`); }); } @@ -2091,9 +2224,9 @@ class DashboardController { this.nextShellRenderWantsFreshData = true; this.requestShellRender(frameId); } - const communicator = this.communicator; - if (!communicator || this.phase === "charging") { - frameTimings.finishFrame(frameId, "discarded: shell render with no active connection"); + const display = this.display; + if (!display || this.phase === "charging") { + frameTimings.finishFrame(frameId, "discarded: shell render with no display target"); return; } const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); @@ -2104,7 +2237,7 @@ class DashboardController { // behind another surface's submission, which is otherwise an unexplained // jump between the paint spans and the composite. await frameTimings.spanAsync(frameId, "submit", () => - communicator.submitSurfaceFrame( + display.submitSurfaceFrame( SHELL_SURFACE_ID, buffer, { x: 0, y: 0, width: image.width, height: image.height }, @@ -2117,8 +2250,9 @@ class DashboardController { // Backpressure: the next shell render waits for this one to reach the // glasses. Timing out here means the loop was blocked for the full timeout // and any input arriving meanwhile had its chrome repaint delayed, so say - // so in the frame rather than leaving a silent stall. - const outcome = await communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); + // so in the frame rather than leaving a silent stall. (The preview target + // resolves immediately; nothing transmits.) + const outcome = await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); if (outcome === null) { frameTimings.logFrame( frameId, @@ -2249,7 +2383,8 @@ class DashboardController { } private updateCompositePreview(): void { - if (!this.communicator) return; + const display = this.display; + if (!display) return; // The floor comes first (and stamps even when backgrounded below): with // it after the background check, per-frame refresh requests from a // backgrounded app never advance the timestamp and every one of them @@ -2273,9 +2408,9 @@ class DashboardController { } if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) { this.lastRecordCaptureAtMs = now; - this.communicator.recordScreenFrame(); + display.recordScreenFrame(); } - const preview = this.communicator.getCompositePreview(previewColorSetting.get() === "green"); + const preview = display.getCompositePreview(previewColorSetting.get() === "green"); if (preview) { this.setDisplayPreview(preview); } diff --git a/app/g2/glasses-display-state.ts b/app/g2/glasses-display-state.ts index 88f951ae..ebda5fa2 100644 --- a/app/g2/glasses-display-state.ts +++ b/app/g2/glasses-display-state.ts @@ -4,6 +4,8 @@ export type GlassesDisplayState = { screenOn: boolean; battery: number | null; foregroundTitle: string | null; + /** The headless preview display is standing in for a connection (preview-only mode). */ + previewMode: boolean; }; /** Compact status shown where the foreground-window title normally lives. */ @@ -13,7 +15,9 @@ export function glassesDisplayLabel(state: GlassesDisplayState): string { if (state.phase === "charging") { return chargingDisplayLabel(state.battery); } - if (state.phase !== "connected") { + // Preview mode has a live (simulated) screen despite the disconnected + // phase, so fall through to the screen-state/foreground labels. + if (state.phase !== "connected" && !state.previewMode) { return "Glasses disconnected"; } if (state.silentMode) { diff --git a/app/native/active-display.ts b/app/native/active-display.ts new file mode 100644 index 00000000..a82db467 --- /dev/null +++ b/app/native/active-display.ts @@ -0,0 +1,19 @@ +declare const com: any; + +/** + * The Java object a worker isolate submits surface frames to: the live BLE + * communicator, or the preview-only compositor standing in when no glasses + * are paired (both expose the same submitSurfaceFrame signature). Located + * through Java statics because those are shared across isolates, unlike any + * JS-side state. + * + * Frame submission only. Glasses-hardware calls (e.g. playBuzzerSequence) + * should keep using FaceclawBleCommunicator.getActive() directly, so they + * skip cleanly when nothing is connected. + */ +export function getActiveDisplay(): any { + return ( + com.faceclaw.app.FaceclawBleCommunicator.getActive() ?? + com.faceclaw.app.FaceclawPreviewCompositor.getActive() + ); +} diff --git a/app/native/preview-display.ts b/app/native/preview-display.ts new file mode 100644 index 00000000..c0fa7f4e --- /dev/null +++ b/app/native/preview-display.ts @@ -0,0 +1,169 @@ +import { ImageSource, Utils } from "@nativescript/core"; +import type { FaceclawCommunicatorBridge, SurfaceOptions } from "./faceclaw-communicator"; + +declare const com: any; +declare const java: any; + +const PREVIEW_BRIGHTEN_GAMMA = 0.7; + +/** + * The display subset of the communicator bridge: everything the dashboard + * controller needs to composite frames and read them back for the phone + * mirror, screenshots, and recordings. FaceclawCommunicatorBridge satisfies + * it by construction (Pick), and PreviewDisplayTarget implements it against + * a headless compositor with no BLE transport behind it. + */ +export type DisplayTarget = Pick< + FaceclawCommunicatorBridge, + | "configureCompositorScreen" + | "configureSurface" + | "removeSurface" + | "setSurfaceVisible" + | "setScreenBlanked" + | "submitSurfaceFrame" + | "waitForFrameFinished" + | "getCompositePreview" + | "saveScreenshot" + | "startScreenRecording" + | "recordScreenFrame" + | "stopScreenRecording" +>; + +function nonNegativeNumber(value: number): number { + const numeric = Number(value); + return Number.isFinite(numeric) ? Math.max(0, numeric) : 0; +} + +/** + * Headless display target for preview-only mode (onboarding's "Preview Only" + * path): hosts the same Java SurfaceCompositor a live connection uses, so + * the whole shell/app render pipeline works with no glasses paired. Frames + * end at the compositor; the phone mirror polls them back as usual. + * + * Unlike the communicator bridge, calls go straight into Java: they are all + * quick in-memory operations, so there is no call queue and no transmit + * backpressure to wait on. + */ +export class PreviewDisplayTarget implements DisplayTarget { + private readonly compositor: any; + + constructor() { + const context = Utils.android.getApplicationContext(); + if (!context) throw new Error("Android application context unavailable"); + this.compositor = new com.faceclaw.app.FaceclawPreviewCompositor(context); + } + + /** + * Publish the compositor as the worker isolates' frame target (they locate + * it through the Java static, like the BLE communicator) and receive a + * main-thread callback per applied frame — the preview path's stand-in for + * the connected path's frame-metrics callback. + */ + activate(onFrameComposited: () => void): void { + this.compositor.setFrameListener( + new java.lang.Runnable({ run: () => onFrameComposited() }), + ); + this.compositor.makeActive(); + } + + /** Withdraw from workers and drop the frame callback. */ + release(): void { + this.compositor.release(); + } + + async configureCompositorScreen(width: number, height: number): Promise { + this.compositor.configureCompositorScreen(Math.round(width), Math.round(height)); + } + + async configureSurface(id: string, options: SurfaceOptions): Promise { + const transparency = options.transparency === "color-key" ? 1 : 0; + this.compositor.configureSurface( + id, + Math.round(options.x), + Math.round(options.y), + Math.round(options.width), + Math.round(options.height), + Math.round(options.zOrder), + transparency, + ); + } + + async removeSurface(id: string): Promise { + this.compositor.removeSurface(id); + } + + async setSurfaceVisible(id: string, visible: boolean): Promise { + this.compositor.setSurfaceVisible(id, Boolean(visible)); + } + + async setScreenBlanked(blanked: boolean): Promise { + this.compositor.setScreenBlanked(Boolean(blanked)); + } + + async submitSurfaceFrame( + surfaceId: string, + pixels8bpp: Uint8Array, + rect: { x: number; y: number; width: number; height: number }, + fingerprint: string, + paintMs = -1, + frameId = 0, + glyphs: ArrayBuffer | null = null, + ): Promise { + // Copy for the same reason the bridge does: the source may be a view on a + // larger buffer, and Java receives the backing ArrayBuffer. + const snapshot = new Uint8Array(pixels8bpp); + this.compositor.submitSurfaceFrame( + snapshot.buffer, + surfaceId, + Math.round(rect.x), + Math.round(rect.y), + Math.round(rect.width), + Math.round(rect.height), + fingerprint, + Math.round(nonNegativeNumber(paintMs)), + Math.round(nonNegativeNumber(frameId)), + glyphs, + ); + // The Java side finishes the frame: the composite is the end of the line, + // and worker isolates submit to the same object without coming through here. + } + + /** Composites are synchronous and nothing transmits, so there is never backpressure. */ + waitForFrameFinished(): Promise { + return Promise.resolve("composited"); + } + + getCompositePreview(green = false): ImageSource | null { + if (!global.isAndroid) return null; + const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); + return bitmap ? new ImageSource(bitmap) : null; + } + + saveScreenshot(crop?: { x: number; y: number; width: number; height: number }): string { + if (!global.isAndroid) return ""; + if (!crop) return String(this.compositor.saveCompositePngScreenshot()); + return String( + this.compositor.saveCompositePngScreenshot( + Math.round(crop.x), + Math.round(crop.y), + Math.round(crop.width), + Math.round(crop.height), + ), + ); + } + + startScreenRecording(): void { + if (!global.isAndroid) return; + this.compositor.startScreenRecording(); + } + + recordScreenFrame(): void { + if (!global.isAndroid) return; + this.compositor.recordScreenFrame(); + } + + stopScreenRecording(): string { + if (!global.isAndroid) return ""; + return String(this.compositor.stopScreenRecording()); + } +} diff --git a/app/phone-ui/main-view-model.ts b/app/phone-ui/main-view-model.ts index 56fb5a63..ea4e4835 100644 --- a/app/phone-ui/main-view-model.ts +++ b/app/phone-ui/main-view-model.ts @@ -62,6 +62,7 @@ export class MainViewModel extends Observable { private _screenRecordingActive = false; private _batteryOptimizationWarningVisible = false; private _warningsModalVisible = false; + private _previewMode = false; private _phase: "disconnected" | "connecting" | "connected" | "charging" | "disconnecting" = "disconnected"; // A new view model is built on every navigation to the main page; these @@ -84,6 +85,7 @@ export class MainViewModel extends Observable { if (this.unsubscribers.length > 0) return; this.unsubscribers.push(dashboardController.subscribe((snapshot) => { this.status = snapshot.status; + this.previewMode = snapshot.previewMode; this.displayPreview = snapshot.displayPreview; this.displayPreviewMessage = snapshot.displayPreviewMessage; this.phase = snapshot.phase; @@ -148,7 +150,22 @@ export class MainViewModel extends Observable { // phase still "connecting"; keep that distinction visible. return this._status.startsWith("Reconnecting") ? "Reconnecting" : "Connecting"; default: - return this._status.startsWith("Failed") ? "Failed" : "Disconnected"; + if (this._status.startsWith("Failed")) return "Failed"; + // The headless preview display is live; "Disconnected" would suggest + // the interactive mirror below it is broken. + return this._previewMode ? "Preview" : "Disconnected"; + } + } + + get previewMode(): boolean { + return this._previewMode; + } + + set previewMode(value: boolean) { + if (this._previewMode !== value) { + this._previewMode = value; + this.notifyPropertyChange("previewMode", value); + this.notifyPropertyChange("connectionStatusLabel", this.connectionStatusLabel); } } @@ -667,6 +684,10 @@ export class MainViewModel extends Observable { */ async autoConnect(): Promise { if (this.phase !== "disconnected") return; + // Preview-only users get the headless preview display instead of a + // connection; a no-op in every other state (including while suppressed: + // suppression is about not re-dialing glasses, and there are none). + await dashboardController.ensurePreviewDisplay(); if (isAutoReconnectSuppressed()) return; const addresses = loadDeviceAddresses(); if (!isValidMacAddress(addresses.right) || !isValidMacAddress(addresses.left)) return;