First iteration of glanceboard

This commit is contained in:
James Babcock committed 2026-09-11 10:47:19 -07:00
1 parent a3660fac61
commit 8a340fc4ad
36 files changed
+2874 -357

No files matched your search

+71 -5
View File
@@ -43,6 +43,8 @@ const MIRROR_TOUCH_GESTURES: Record<Exclude<MirrorTouchKind, "tap">, WearRemoteI
"swipe-right": "swipe-right",
};
import { findSoundEffect, playSoundEffect } from "../ui/sound-effects";
import { GlanceHost } from "./glance-host";
import { glanceEventForGesture, type GlanceEvent } from "./glance-state";
import { isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
@@ -52,6 +54,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire";
import { getDefaultMediumFont } from "../graphics/ui-fonts";
import { wrapText } from "../graphics/textwrap";
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
import { type InputEvent } from "../ui/gestures";
import { registerSystemTools } from "../assistant/system-tools";
import { updateGlassesPresence } from "./glasses-presence";
import { timerEngine } from "../apps/timer/timer-engine";
@@ -252,6 +255,26 @@ class DashboardController {
private screenTimeoutTimer: ReturnType<typeof setInterval> | null = null;
private evenHubSuspendTimer: ReturnType<typeof setTimeout> | null = null;
private evenHubSessionSuspended = false;
/**
* The Glanceboard: the alternate sleep-time display, on its own surface
* above the shell. Fed sleep-time input from handleInputEvent; it uses the
* same wake barrier as the shell and hands the compositor back to the
* screen-off path when it hides.
*/
private readonly glance = new GlanceHost({
getDisplay: () => this.display,
getProvider: () => ALL_APPS.find((app) => app.glanceboard)?.glanceboard ?? null,
canShow: () => (this.phase === "connected" || this.isPreviewDisplayActive()) && !this.glassesLocked,
ensureSessionActive: (frameId) => this.ensureEvenHubSessionActive(frameId),
onHiddenWhileAsleep: () => {
if (!shell.isScreenOn()) this.handleScreenStateChanged(false);
},
onVisibilityChanged: () => {
this.schedulePreviewUpdate();
this.emit();
},
appendLog: (message) => this.appendLog(message),
});
private evenHubResumePromise: Promise<boolean> | null = null;
/** Faceclaw's firmware at the required revision reported in; its extensions can be used. */
private customFirmwareConfirmed = false;
@@ -359,6 +382,8 @@ class DashboardController {
this.emit();
},
onScreenStateChanged: (on) => {
// Any wake of the regular UI replaces a showing Glanceboard.
if (on) this.glance.dismiss();
this.handleScreenStateChanged(on);
if (on) this.requestShellRender();
this.emit();
@@ -788,6 +813,7 @@ class DashboardController {
if (
!suspendEvenHubWhenScreenOffSetting.get() ||
shell.isScreenOn() ||
this.glance.isVisible() ||
this.phase !== "connected" ||
!this.communicator ||
this.evenHubSessionSuspended
@@ -801,9 +827,11 @@ class DashboardController {
if (
!suspendEvenHubWhenScreenOffSetting.get() ||
shell.isScreenOn() ||
this.glance.isVisible() ||
this.phase !== "connected" ||
this.communicator !== communicator
) {
// A showing Glanceboard re-arms this when it hides.
return;
}
@@ -829,6 +857,7 @@ class DashboardController {
if (
!suspendEvenHubWhenScreenOffSetting.get() ||
shell.isScreenOn() ||
this.glance.isVisible() ||
this.phase !== "connected" ||
this.communicator !== communicator
) {
@@ -1250,6 +1279,7 @@ class DashboardController {
const target = this.previewTarget;
if (!target) return;
this.previewTarget = null;
this.glance.reset();
target.release();
this.clearDashboardTimer();
this.appendLog("Preview-only display released.");
@@ -1294,6 +1324,7 @@ class DashboardController {
this.customFirmwareConfirmed = false;
this.faceclawWakeLeaseState = null;
this.lockSurfaceConfigured = false;
this.glance.reset();
this.evenHubResumePromise = null;
this.connectRunning = true;
@@ -1549,6 +1580,7 @@ class DashboardController {
}
this.communicator = null;
this.lockSurfaceConfigured = false;
this.glance.reset();
this.customFirmwareConfirmed = false;
this.faceclawWakeLeaseState = null;
this.evenHubResumePromise = null;
@@ -1683,6 +1715,7 @@ class DashboardController {
const communicator = this.communicator;
this.communicator = null;
this.lockSurfaceConfigured = false;
this.glance.reset();
this.evenHubSessionSuspended = false;
this.evenHubResumePromise = null;
@@ -1756,14 +1789,14 @@ class DashboardController {
* finger-down/finger-up (the watch) hold for as long as the user does.
*/
async injectSyntheticRingInput(kind: WearRemoteInputKind, origin: SyntheticInputOrigin = "ring"): Promise<void> {
// Match the ring while the display is dark: only a double-click wakes it,
// and that wake is handled by Shell.receiveInput. This check also protects
// against a watch acting on a stale state snapshot.
// Match the ring while the display is dark: a double-click wakes it
// (handled by Shell.receiveInput) and a tap or hold shows the
// Glanceboard; scrolls and the menu gesture mean nothing. This check also
// protects against a watch acting on a stale state snapshot.
if (
origin === "watch" &&
!shell.isScreenOn() &&
kind !== "double-click" &&
kind !== "long-press-release"
(kind === "scroll-up" || kind === "scroll-down" || kind === "short-then-long-press")
) {
this.appendLog(`${kind} (watch scheme) ignored while display is off`);
return;
@@ -1794,7 +1827,12 @@ class DashboardController {
return;
}
if (!shell.isScreenOn()) {
// Asleep, the mirror offers the sleep gestures: double-tap wakes, a
// tap shows the Glanceboard for its timeout. The mirror's hold has no
// real release (the synthetic long-press releases at once, which would
// only flash the board), so it counts as a tap here too.
if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch");
else if (kind === "tap" || kind === "long-press") await this.injectSyntheticRingInput("click", "watch");
return;
}
if (kind !== "tap") {
@@ -2056,6 +2094,20 @@ class DashboardController {
}
return;
}
if (!shell.isScreenOn()) {
// Sleep-time gestures belong to the Glanceboard: a tap or a hold
// shows it (the shell would ignore them), and a double-tap while it
// is up dismisses it and falls through to the normal wake below.
const glanceEvent = this.glanceEventFor(inputEvent, event);
if (glanceEvent?.type === "dismiss") {
this.glance.dismiss();
} else if (glanceEvent) {
frameTimings.annotateFrame(frameId, `glanceboard ${glanceEvent.type}`);
await this.glance.handleEvent(glanceEvent, frameId);
frameOwned = true;
return;
}
}
const wakewordShouldWake =
event.kind === "even-ai" &&
event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP &&
@@ -2129,6 +2181,20 @@ class DashboardController {
}
}
/**
* The Glanceboard event a sleep-time input means, or null when the shell
* should see the input as usual. A display-wake carries the head-tilt
* gesture (HEAD_UP_EVENT) as a press; the double-tap display-wake stays the
* regular UI's wake.
*/
private glanceEventFor(inputEvent: InputEvent, event: RawInputEvent): GlanceEvent | null {
if (!this.glance.isEnabled()) return null;
if (inputEvent.type === "display-wake") {
return event.eventType === OsEventTypeList.HEAD_UP_EVENT ? { type: "press" } : null;
}
return glanceEventForGesture(inputEvent.type, this.glance.isVisible());
}
/** Launch or focus an in-process singleton app (notifications, debug tests). */
private async launchInProcessApp(
windowId: string,
+7
View File
@@ -71,6 +71,12 @@ export const OsEventTypeList = {
RING_LONG_PRESS_RELEASE_EVENT: 10,
/** CFW extension: the 2.2.9 tap-then-hold Menu gesture. */
SHORT_THEN_LONG_PRESS_EVENT: 11,
/**
* CFW extension (head-tilt wake branch): the IMU head-up gesture, forwarded
* while the display sleeps. Reaches the phone as a display-wake carrying
* this eventType; the glance host treats it as a press.
*/
HEAD_UP_EVENT: 12,
} as const;
export const OsEventTypeName: Record<number, string> = {
@@ -86,4 +92,5 @@ export const OsEventTypeName: Record<number, string> = {
9: "RING_LONG_PRESS_EVENT",
10: "RING_LONG_PRESS_RELEASE_EVENT",
11: "SHORT_THEN_LONG_PRESS_EVENT",
12: "HEAD_UP_EVENT",
};
+262
View File
@@ -0,0 +1,262 @@
/**
* Shows the Glanceboard while the shell sleeps. The board is not a window:
* it lives on its own full-screen opaque compositor surface above the shell
* (below only the lock screen), so it never composites with the sidebar,
* top bar, or the sleeping app's retained frame. The host owns that surface,
* the visibility state machine (glance-state.ts), its auto-hide timer, and
* the board's render loop; the controller feeds it sleep-time input and the
* wake barrier, and takes the compositor back to blank when the board hides.
*/
import { GrayImage, G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image";
import { prepareFrameDraws } from "../graphics/glyph-wire";
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
import * as frameTimings from "../native/frame-timings";
import { type DisplayTarget } from "../native/preview-display";
import { minWindowTop } from "../ui/shell/geometry";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition";
import { GLANCE_HIDDEN, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state";
export const GLANCE_SURFACE_ID = "glance";
/** Above the shell (1) and every window (0); below the lock screen (1000). */
export const GLANCE_SURFACE_Z_ORDER = 900;
export type GlanceHostOptions = {
getDisplay: () => DisplayTarget | null;
getProvider: () => GlanceboardProvider | null;
/** Whether a board may show right now (connected or preview, not locked). */
canShow: () => boolean;
/**
* The controller's wake barrier: restore the EvenHub session if it was
* suspended and unblank the compositor. Resolves once the frame is visible.
*/
ensureSessionActive: (frameId: number) => Promise<boolean>;
/**
* The board hid on its own (timer or release) with the shell still
* asleep: the controller re-blanks and re-arms the screen-off power save.
*/
onHiddenWhileAsleep: () => void;
/** Visibility changed either way (phone mirror refresh). */
onVisibilityChanged: () => void;
appendLog: (message: string) => void;
};
export class GlanceHost {
private state: GlanceState = GLANCE_HIDDEN;
private board: GlanceBoardInstance | null = null;
/** Whether the surface is currently visible on the compositor. */
private shown = false;
private hideTimer: ReturnType<typeof setTimeout> | null = null;
private configuredFor: DisplayTarget | null = null;
/** Serializes show/hide against each other and against in-flight renders. */
private queue: Promise<void> = Promise.resolve();
private rendering = false;
private renderPending = false;
private nextRenderWantsFreshData = false;
constructor(private readonly options: GlanceHostOptions) {}
isVisible(): boolean {
return this.state.visible;
}
/** Whether sleep-time gestures should reach the board at all. */
isEnabled(): boolean {
return this.options.getProvider()?.isEnabled() ?? false;
}
/**
* Apply a glance event from sleep-time input. The frame is owned from here:
* a show submits it with the board's first frame, anything else finishes it.
*/
async handleEvent(event: GlanceEvent, frameId: number): Promise<void> {
const before = this.state;
this.state = reduceGlance(before, event, Date.now());
this.armHideTimer();
if (this.state.visible === before.visible) {
frameTimings.finishFrame(frameId, this.state.visible ? "glanceboard stays up" : "glance event while hidden");
return;
}
if (this.state.visible) {
await this.enqueue(() => this.show(frameId));
} else {
frameTimings.finishFrame(frameId, "glanceboard hidden");
await this.enqueue(() => this.hide(true));
}
}
/**
* Hide because the regular UI is waking (double-tap, a notification, ...):
* the shell's own wake path handles the compositor from here, so no re-blank.
*/
dismiss(): void {
if (!this.state.visible && !this.shown && !this.board) return;
this.state = GLANCE_HIDDEN;
this.armHideTimer();
void this.enqueue(() => this.hide(false));
}
/** The display target went away (disconnect, preview teardown): forget everything. */
reset(): void {
this.state = GLANCE_HIDDEN;
this.armHideTimer();
this.board?.stop();
this.board = null;
this.shown = false;
this.configuredFor = null;
this.renderPending = false;
}
private enqueue(operation: () => Promise<void>): Promise<void> {
const run = this.queue.then(operation, operation).catch((error) => {
this.options.appendLog(`glanceboard operation failed: ${String(error)}`);
});
this.queue = run;
return run;
}
private armHideTimer(): void {
if (this.hideTimer !== null) clearTimeout(this.hideTimer);
this.hideTimer = null;
const hideAtMs = this.state.hideAtMs;
if (!this.state.visible || hideAtMs === null) return;
this.hideTimer = setTimeout(() => {
this.hideTimer = null;
void this.handleEvent({ type: "timeout" }, frameTimings.startFrame("glanceboard timeout"));
}, Math.max(0, hideAtMs - Date.now()));
}
private async configureSurface(display: DisplayTarget): Promise<void> {
await display.configureSurface(GLANCE_SURFACE_ID, {
x: 0,
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
zOrder: GLANCE_SURFACE_Z_ORDER,
transparency: "opaque",
});
await display.setSurfaceVisible(GLANCE_SURFACE_ID, false);
this.configuredFor = display;
}
private async show(frameId: number): Promise<void> {
if (!this.state.visible) {
// Hidden again before this ran (a quick release); nothing to do.
frameTimings.finishFrame(frameId, "glanceboard show superseded");
return;
}
const display = this.options.getDisplay();
const provider = this.options.getProvider();
if (!display || !provider || !this.options.canShow()) {
this.state = GLANCE_HIDDEN;
this.armHideTimer();
frameTimings.finishFrame(frameId, "glanceboard unavailable");
return;
}
if (this.configuredFor !== display) await this.configureSurface(display);
if (!this.board) {
this.board = provider.createBoard(() => this.requestRender());
this.board.start();
}
// Submit the first frame and reveal the surface while the compositor is
// still blanked, so the unblank below composites the board on top of the
// sleeping UI in one step rather than flashing the app first.
await this.render(frameId);
await display.setSurfaceVisible(GLANCE_SURFACE_ID, true);
this.shown = true;
const ready = await this.options.ensureSessionActive(frameId);
if (!ready) this.options.appendLog("glanceboard: wake barrier did not complete");
this.options.onVisibilityChanged();
}
private async hide(stillAsleep: boolean): Promise<void> {
const wasShown = this.shown;
this.shown = false;
const display = this.options.getDisplay();
if (wasShown && display && this.configuredFor === display) {
// Blank before revealing what is under the board: hiding the surface
// first recomposites the sleeping UI's retained frames and transmits
// them, a visible flash of the main UI before the screen goes dark.
// Blanked, the surface change composites to the same black frame. (The
// controller's screen-off path below blanks again, harmlessly.)
if (stillAsleep) {
await display.setScreenBlanked(true);
}
await display.setSurfaceVisible(GLANCE_SURFACE_ID, false);
}
this.board?.stop();
this.board = null;
this.renderPending = false;
if (wasShown && stillAsleep) this.options.onHiddenWhileAsleep();
if (wasShown) this.options.onVisibilityChanged();
}
private requestRender(): void {
if (!this.board) return;
if (this.rendering) {
this.renderPending = true;
return;
}
void this.enqueue(() => this.render(frameTimings.startFrame("render:glanceboard")));
}
/** Paint the board into a full-screen frame and submit it to the surface. */
private async render(frameId: number): Promise<void> {
const board = this.board;
const display = this.options.getDisplay();
if (!board || !display || this.configuredFor !== display) {
frameTimings.finishFrame(frameId, "discarded: glanceboard not showing");
return;
}
this.rendering = true;
try {
frameTimings.annotateFrame(frameId, "glanceboard");
const wantFreshData = this.nextRenderWantsFreshData;
this.nextRenderWantsFreshData = false;
beginRenderPass(!wantFreshData);
const paintStartedAtMs = Date.now();
const frame = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => {
const boardImage = board.paint();
const full = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
// Centred horizontally in the panel, in the standard window band
// vertically (Display > Vertical position).
const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0);
const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop()));
boardImage.composeInto(full, x, y);
return full;
}),
);
const paintUsedStaleData = endRenderPass();
const paintMs = Date.now() - paintStartedAtMs;
const planes: Plane[] = [{ image: frame, x: 0, y: 0 }];
const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes));
const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes));
const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer());
const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws));
await frameTimings.spanAsync(frameId, "submit", () =>
display.submitSurfaceFrame(
GLANCE_SURFACE_ID,
buffer,
{ x: 0, y: 0, width: image.width, height: image.height },
fingerprint,
paintMs,
frameId,
preparedDraws,
),
);
if (paintUsedStaleData) {
// Same contract as the windows: a paint that used cached data (the
// notification icons) gets one follow-up with fresh data.
this.nextRenderWantsFreshData = true;
this.renderPending = true;
}
} finally {
this.rendering = false;
if (this.renderPending && this.board) {
this.renderPending = false;
this.requestRender();
}
}
}
}
+81
View File
@@ -0,0 +1,81 @@
/**
* The Glanceboard's visibility rules, as a pure reducer so they can be tested
* without the display plumbing. The board is an alternate sleep-time display:
* while the shell is asleep a press (single tap, or a head-tilt wake) shows it
* for a short while, a hold (long-press) shows it until the press is released,
* and a double-tap dismisses it in favour of the regular UI. GlanceHost feeds
* these events in and applies the resulting visibility to the compositor.
*
* No NativeScript imports: this module is compiled and run under plain node
* by tests/glance-state.test.cjs.
*/
/** How long a press keeps the board up; each further press restarts it. */
export const GLANCE_TIMEOUT_MS = 3000;
export type GlanceState = {
visible: boolean;
/** A long-press is holding the board up; the timer does not run meanwhile. */
holding: boolean;
/** When the board hides on its own (epoch ms), or null while held / hidden. */
hideAtMs: number | null;
};
export type GlanceEvent =
/** Single tap or head-tilt wake: show, and (re)start the auto-hide timer. */
| { type: "press" }
/** Long-press: show and keep showing until the release. */
| { type: "hold" }
/** The long-press ended. */
| { type: "release" }
/** The auto-hide timer fired. */
| { type: "timeout" }
/** Double-tap (or any other wake of the regular UI): hide now. */
| { type: "dismiss" };
export const GLANCE_HIDDEN: GlanceState = Object.freeze({ visible: false, holding: false, hideAtMs: null });
export function reduceGlance(state: GlanceState, event: GlanceEvent, nowMs: number): GlanceState {
switch (event.type) {
case "press":
// A tap during a hold changes nothing: the hold decides when it ends.
if (state.holding) return state;
return { visible: true, holding: false, hideAtMs: nowMs + GLANCE_TIMEOUT_MS };
case "hold":
return { visible: true, holding: true, hideAtMs: null };
case "release":
// Releases also follow presses the board was never holding for (a
// tap-then-hold, a release delivered late); those mean nothing here.
return state.holding ? GLANCE_HIDDEN : state;
case "timeout":
if (!state.visible || state.holding || state.hideAtMs === null || nowMs < state.hideAtMs) return state;
return GLANCE_HIDDEN;
case "dismiss":
return GLANCE_HIDDEN;
}
}
/**
* Which glance event, if any, a shell input gesture means while the screen is
* off. A double-tap only concerns the board while it is showing (it then
* dismisses it before waking the regular UI); everything else the shell
* already handles or ignores while asleep.
*/
export function glanceEventForGesture(
gesture: "click" | "double-click" | "long-press" | "long-press-release" | "head-tilt" | string,
boardVisible: boolean,
): GlanceEvent | null {
switch (gesture) {
case "click":
case "head-tilt":
return { type: "press" };
case "long-press":
return { type: "hold" };
case "long-press-release":
return { type: "release" };
case "double-click":
return boardVisible ? { type: "dismiss" } : null;
default:
return null;
}
}