Make preview-only mode work

This commit is contained in:
James Babcock committed 2026-08-30 19:17:30 -07:00
1 parent 2d45b65098
commit 602f66f15b
15 files changed
+607 -63

No files matched your search

+179 -44
View File
@@ -69,6 +69,8 @@ import { closeRunningPackage, launchInstalledPackage } from "../apps/evenhub/man
import { wearerVerificationOptions } from "../apps/microphones/speakers";
import { micSession } from "../apps/microphones/mic-session";
import { glassesDisplayLabel } from "./glasses-display-state";
import { PreviewDisplayTarget, type DisplayTarget } from "../native/preview-display";
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
@@ -100,6 +102,12 @@ export type DashboardSnapshot = {
firmwareWarningVisible: boolean;
screenRecordingActive: boolean;
batteryOptimizationWarningVisible: boolean;
/**
* True while the headless preview display is standing in for a glasses
* connection (preview-only mode): the mirror is live and interactive, but
* nothing is paired, so "Disconnected" would be the wrong label.
*/
previewMode: boolean;
};
type DashboardListener = (snapshot: DashboardSnapshot) => void;
@@ -208,6 +216,9 @@ class DashboardController {
private welcomeSoundArmed = false;
private communicator: FaceclawCommunicatorBridge | null = null;
// Headless stand-in for the compositor when no glasses are paired
// (preview-only mode); a real connection always outranks it. See `display`.
private previewTarget: PreviewDisplayTarget | null = null;
private shellRefreshTimer: ReturnType<typeof setInterval> | null = null;
private previewTimer: ReturnType<typeof setInterval> | null = null;
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
@@ -483,16 +494,28 @@ class DashboardController {
}
const communicator = this.communicator;
if (!communicator) return;
// Blanking is a compositor-level flag so worker-window surfaces go dark
// too; retained state survives while the EvenHub page is absent.
void (async () => {
await communicator.setScreenBlanked(true);
await communicator.setG2ScreenOn(false);
})().catch((error) => {
this.appendLog(`screen sleep failed: ${this.formatError(error)}`);
});
this.scheduleEvenHubSuspend();
if (communicator) {
// Blanking is a compositor-level flag so worker-window surfaces go dark
// too; retained state survives while the EvenHub page is absent.
void (async () => {
await communicator.setScreenBlanked(true);
await communicator.setG2ScreenOn(false);
})().catch((error) => {
this.appendLog(`screen sleep failed: ${this.formatError(error)}`);
});
this.scheduleEvenHubSuspend();
return;
}
const preview = this.previewTarget;
if (!preview) return;
// Preview mode: the mirror goes dark like the lens would; retained
// surfaces survive for the next wake.
void preview
.setScreenBlanked(true)
.then(() => this.schedulePreviewUpdate())
.catch((error) => {
this.appendLog(`preview screen sleep failed: ${this.formatError(error)}`);
});
}
private syncLockScreenSettingIfChanged(): void {
@@ -599,7 +622,17 @@ class DashboardController {
private ensureEvenHubSessionActive(frameId = 0): Promise<boolean> {
if (this.evenHubResumePromise) return this.evenHubResumePromise;
const communicator = this.communicator;
if (!communicator || this.phase === "charging" || this.phase === "disconnected") {
if (!communicator) {
// Preview mode has no plugin session to restore; waking is just
// unblanking the headless compositor so the mirror lights back up.
const preview = this.previewTarget;
if (!preview) return Promise.resolve(false);
return preview.setScreenBlanked(false).then(() => {
this.schedulePreviewUpdate();
return true;
});
}
if (this.phase === "charging" || this.phase === "disconnected") {
return Promise.resolve(false);
}
@@ -792,8 +825,8 @@ class DashboardController {
/** Save the occupied part of the composited screen as a 4-bit grayscale PNG. */
saveScreenshot(): string {
const path = this.communicator?.saveScreenshot(shell.screenshotCropRect()) ?? "";
this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: not connected");
const path = this.display?.saveScreenshot(shell.screenshotCropRect()) ?? "";
this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: no display");
return path;
}
@@ -804,15 +837,15 @@ class DashboardController {
*/
startScreenRecording(): void {
if (this.screenRecordingActive) return;
const communicator = this.communicator;
if (!communicator) {
this.appendLog("screen recording skipped: not connected");
const display = this.display;
if (!display) {
this.appendLog("screen recording skipped: no display");
return;
}
communicator.startScreenRecording();
display.startScreenRecording();
// Capture the starting frame immediately rather than waiting for the
// next preview flush.
communicator.recordScreenFrame();
display.recordScreenFrame();
this.screenRecordingActive = true;
this.appendLog("screen recording started");
this.emit();
@@ -824,7 +857,7 @@ class DashboardController {
this.screenRecordingActive = false;
let path = "";
try {
path = this.communicator?.stopScreenRecording() ?? "";
path = this.display?.stopScreenRecording() ?? "";
} finally {
this.emit();
}
@@ -866,6 +899,7 @@ class DashboardController {
firmwareWarningVisible: this.firmwareWarningMessage.length > 0,
screenRecordingActive: this.screenRecordingActive,
batteryOptimizationWarningVisible: this.batteryOptimizationWarningVisible,
previewMode: this.isPreviewDisplayActive(),
};
}
@@ -883,6 +917,11 @@ class DashboardController {
if (this.silentMode && this.phase === "connected") {
return "Connected (Silent mode enabled)";
}
// The preview compositor keeps the mirror live (including as a black
// frame while the simulated screen is off), so never cover it.
if (this.isPreviewDisplayActive()) {
return "";
}
const connectionFailing = this.phase === "disconnected" || this.phase === "connecting";
if (
connectionFailing &&
@@ -902,6 +941,7 @@ class DashboardController {
screenOn: shell.isScreenOn(),
battery: this.lastHeadsetBattery,
foregroundTitle: shell.getForegroundApp()?.title ?? null,
previewMode: this.isPreviewDisplayActive(),
});
}
@@ -974,6 +1014,95 @@ class DashboardController {
this.emit();
}
/**
* Where composited frames go and where the phone mirror, screenshots, and
* recordings come from: the live glasses connection when there is one,
* otherwise the headless preview compositor (preview-only mode), otherwise
* nothing (frames are discarded).
*/
private get display(): DisplayTarget | null {
return this.communicator ?? this.previewTarget;
}
private isPreviewDisplayActive(): boolean {
return this.communicator === null && this.previewTarget !== null;
}
/**
* Bring up the headless preview display for preview-only mode (onboarding's
* "Preview Only" path): the same shell/app/compositor pipeline as a live
* session minus the BLE transport, so the phone mirror and its touch/watch
* controls work with no glasses paired. Idempotent; called from the main
* page alongside autoConnect. A real connection outranks it — connect()
* tears it down before creating the communicator.
*/
async ensurePreviewDisplay(): Promise<void> {
if (!isPreviewOnlyMode()) return;
if (this.communicator || this.previewTarget || this.connectRunning) return;
const target = new PreviewDisplayTarget();
// Published before the awaits below so a second call (or connect()) sees
// it; the target's calls all complete in microtasks, so the screen size
// is in place before any real render can reach the compositor.
this.previewTarget = target;
try {
await target.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT);
await target.configureSurface(SHELL_SURFACE_ID, {
x: 0,
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
zOrder: 1,
transparency: "color-key",
});
const foregroundWindowId = shell.foregroundWindow()?.windowId;
for (const window of shell.getWindows()) {
await this.configureWindowSurface(
window.surfaceId,
window.windowId === foregroundWindowId,
window.heightMode,
target,
);
}
} catch (error) {
if (this.previewTarget === target) this.previewTarget = null;
this.appendLog(`preview display setup failed: ${this.formatError(error)}`);
return;
}
// Workers find the compositor through its Java static; the per-frame
// callback keeps the mirror current (the connected path gets the same
// from Java's frame metrics).
target.activate(() => this.schedulePreviewUpdate());
this.setStatus("Preview mode (no glasses paired).");
this.appendLog("Preview-only display active; frames render to the phone mirror only.");
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
// The same recurring upkeep a live session gets: top-bar clock refresh,
// the preview safety-net poll, and the screen timeout (a timed-out
// preview screen wakes on double-tap, like the glasses would).
this.shellRefreshTimer = setInterval(() => {
this.requestShellRender();
this.updateCompositePreview();
}, SHELL_REFRESH_INTERVAL_MS);
this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS);
this.screenTimeoutTimer = setInterval(() => {
if (!this.isPreviewDisplayActive()) return;
if (!shell.applyScreenTimeout()) return;
this.endTextSettingEdit();
this.requestShellRender();
}, SCREEN_TIMEOUT_CHECK_MS);
this.emit();
}
/** Drop the headless preview display (a real connection is taking over). */
private teardownPreviewDisplay(): void {
const target = this.previewTarget;
if (!target) return;
this.previewTarget = null;
target.release();
this.clearDashboardTimer();
this.appendLog("Preview-only display released.");
}
async connect(): Promise<void> {
if (this.phase !== "disconnected") return;
// Connecting is the explicit way out of the manual-disconnected state.
@@ -987,6 +1116,9 @@ class DashboardController {
this.appendLog(`error: ${message}`);
throw new Error(message);
}
// Glasses are configured, so the real pipeline owns the display from here
// (also frees the preview timers for connect() to replace).
this.teardownPreviewDisplay();
this.lastInput = "waiting...";
this.lastSys = "none yet";
this.welcomeSoundArmed = isWelcomeSoundPending();
@@ -1503,7 +1635,7 @@ class DashboardController {
* Coordinates are fractions of the mirror (0..1).
*/
async handleMirrorTouch(kind: MirrorTouchKind, nx: number, ny: number): Promise<void> {
if (this.phase !== "connected" && this.phase !== "charging") return;
if (this.phase !== "connected" && this.phase !== "charging" && !this.isPreviewDisplayActive()) return;
if (this.glassesLocked) {
// Same as the ring: only a double-tap reaches a locked display.
if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch");
@@ -1979,35 +2111,36 @@ class DashboardController {
}
}
/** Create/refresh a window surface on the compositor, if connected. */
/** Create/refresh a window surface on the compositor, if a display target exists. */
private async configureWindowSurface(
surfaceId: string,
visible: boolean,
heightMode: WindowHeightMode = "min",
// ensurePreviewDisplay passes its not-yet-published target explicitly.
target: DisplayTarget | null = this.display,
): Promise<void> {
const communicator = this.communicator;
if (!communicator) return;
await communicator.configureSurface(surfaceId, {
if (!target) return;
await target.configureSurface(surfaceId, {
...appViewportRect(heightMode),
zOrder: 0,
transparency: "opaque",
});
await communicator.setSurfaceVisible(surfaceId, visible);
await target.setSurfaceVisible(surfaceId, visible);
}
private removeWindowSurface(surfaceId: string): void {
const communicator = this.communicator;
if (!communicator) return;
void communicator.removeSurface(surfaceId).catch((error) => {
const display = this.display;
if (!display) return;
void display.removeSurface(surfaceId).catch((error) => {
this.appendLog(`surface removal failed: ${this.formatError(error)}`);
});
}
/** Submit a painted frame for an in-process window (e.g. the launcher). */
private async submitWindowFrame(surfaceId: string, planes: Plane[], paintMs: number, frameId: number): Promise<void> {
const communicator = this.communicator;
if (!communicator || this.phase === "charging") {
frameTimings.finishFrame(frameId, "discarded: window frame with no active connection");
const display = this.display;
if (!display || this.phase === "charging") {
frameTimings.finishFrame(frameId, "discarded: window frame with no display target");
return;
}
const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes));
@@ -2015,7 +2148,7 @@ class DashboardController {
const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer());
const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws));
await frameTimings.spanAsync(frameId, "submit", () =>
communicator.submitSurfaceFrame(
display.submitSurfaceFrame(
surfaceId,
buffer,
{ x: 0, y: 0, width: image.width, height: image.height },
@@ -2029,9 +2162,9 @@ class DashboardController {
/** Flip a window surface's compositor visibility; fire-and-forget. */
private setWindowSurfaceVisible(surfaceId: string, visible: boolean): void {
const communicator = this.communicator;
if (!communicator) return;
void communicator.setSurfaceVisible(surfaceId, visible).catch((error) => {
const display = this.display;
if (!display) return;
void display.setSurfaceVisible(surfaceId, visible).catch((error) => {
this.appendLog(`surface visibility change failed: ${this.formatError(error)}`);
});
}
@@ -2091,9 +2224,9 @@ class DashboardController {
this.nextShellRenderWantsFreshData = true;
this.requestShellRender(frameId);
}
const communicator = this.communicator;
if (!communicator || this.phase === "charging") {
frameTimings.finishFrame(frameId, "discarded: shell render with no active connection");
const display = this.display;
if (!display || this.phase === "charging") {
frameTimings.finishFrame(frameId, "discarded: shell render with no display target");
return;
}
const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes));
@@ -2104,7 +2237,7 @@ class DashboardController {
// behind another surface's submission, which is otherwise an unexplained
// jump between the paint spans and the composite.
await frameTimings.spanAsync(frameId, "submit", () =>
communicator.submitSurfaceFrame(
display.submitSurfaceFrame(
SHELL_SURFACE_ID,
buffer,
{ x: 0, y: 0, width: image.width, height: image.height },
@@ -2117,8 +2250,9 @@ class DashboardController {
// Backpressure: the next shell render waits for this one to reach the
// glasses. Timing out here means the loop was blocked for the full timeout
// and any input arriving meanwhile had its chrome repaint delayed, so say
// so in the frame rather than leaving a silent stall.
const outcome = await communicator.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
// so in the frame rather than leaving a silent stall. (The preview target
// resolves immediately; nothing transmits.)
const outcome = await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS);
if (outcome === null) {
frameTimings.logFrame(
frameId,
@@ -2249,7 +2383,8 @@ class DashboardController {
}
private updateCompositePreview(): void {
if (!this.communicator) return;
const display = this.display;
if (!display) return;
// The floor comes first (and stamps even when backgrounded below): with
// it after the background check, per-frame refresh requests from a
// backgrounded app never advance the timestamp and every one of them
@@ -2273,9 +2408,9 @@ class DashboardController {
}
if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) {
this.lastRecordCaptureAtMs = now;
this.communicator.recordScreenFrame();
display.recordScreenFrame();
}
const preview = this.communicator.getCompositePreview(previewColorSetting.get() === "green");
const preview = display.getCompositePreview(previewColorSetting.get() === "green");
if (preview) {
this.setDisplayPreview(preview);
}
+5 -1
View File
@@ -4,6 +4,8 @@ export type GlassesDisplayState = {
screenOn: boolean;
battery: number | null;
foregroundTitle: string | null;
/** The headless preview display is standing in for a connection (preview-only mode). */
previewMode: boolean;
};
/** Compact status shown where the foreground-window title normally lives. */
@@ -13,7 +15,9 @@ export function glassesDisplayLabel(state: GlassesDisplayState): string {
if (state.phase === "charging") {
return chargingDisplayLabel(state.battery);
}
if (state.phase !== "connected") {
// Preview mode has a live (simulated) screen despite the disconnected
// phase, so fall through to the screen-state/foreground labels.
if (state.phase !== "connected" && !state.previewMode) {
return "Glasses disconnected";
}
if (state.silentMode) {