Dim the screen underneath when a context menu is open

This commit is contained in:
James Babcock committed 2026-09-03 01:25:10 -07:00
1 parent 83c3732ba7
commit 9234458dd6
13 files changed
+211 -14

No files matched your search

@@ -808,6 +808,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
compositor.removeSurface(id);
}
/**
* Dim every surface below zOrder belowZOrder to factor256/256 (see
* SurfaceCompositor.setUnderlayDim). Takes effect with the next submitted
* frame: the shell always submits its own surface right after changing
* this, so no recomposite happens here.
*/
public void setUnderlayDim(int belowZOrder, int factor256) {
compositor.setUnderlayDim(belowZOrder, factor256);
}
/**
* Apply an update to one compositor surface and submit the recomposited
* screen as the desired frame. The update covers the rect (rectX, rectY,
@@ -72,6 +72,11 @@ public final class FaceclawPreviewCompositor {
compositor.removeSurface(id);
}
/** Dim every surface below zOrder belowZOrder to factor256/256 (see SurfaceCompositor). */
public void setUnderlayDim(int belowZOrder, int factor256) {
compositor.setUnderlayDim(belowZOrder, factor256);
}
public void setSurfaceVisible(String id, boolean visible) {
compositor.setSurfaceVisible(id, visible);
}
@@ -161,6 +161,9 @@ public final class SurfaceCompositor {
private int screenWidth;
private int screenHeight;
private boolean blanked;
/** See setUnderlayDim: surfaces with zOrder below this are dimmed by underlayDim/256. */
private int underlayDimBelowZOrder = Integer.MIN_VALUE;
private int underlayDim = 256;
private final Map<String, Surface> surfaces = new HashMap<>();
private long nextCompositeSeq = 1;
@@ -216,6 +219,33 @@ public final class SurfaceCompositor {
}
}
/**
* Dim every surface whose zOrder is below belowZOrder to factor256/256 of
* its brightness (256 = no dimming): how a shell overlay that dims what it
* covers (the TS Layer.dimUnderneath) reaches the window surfaces beneath
* the shell surface, which the shell's own layer stack cannot paint.
* Visible pixels stay at least 1 (the color-key black); glyph and
* firmware-text draws keep their cached-draw form with a dimmed value;
* image draws leave the composite's draw list (their pixels, already
* baked into the surface, dim as raster). Takes effect when the next
* frame composites.
*/
public void setUnderlayDim(int belowZOrder, int factor256) {
synchronized (lock) {
underlayDimBelowZOrder = belowZOrder;
underlayDim = Math.max(0, Math.min(256, factor256));
}
}
/** The dim factor (256 = none) that applies to a surface. */
private int dimForLocked(Surface surface) {
return surface.zOrder < underlayDimBelowZOrder ? underlayDim : 256;
}
private static int dimValue(int value, int dim) {
return Math.max(1, (value * dim + 128) >> 8);
}
/**
* Hidden surfaces keep their retained pixels and accept updates, but are
* excluded from the composite and its fingerprint (used for background
@@ -410,9 +440,12 @@ public final class SurfaceCompositor {
for (Surface surface : ordered) {
if (!surface.visible) continue;
blendLocked(gray, surface);
int dim = dimForLocked(surface);
for (ScreenDraw draw : surface.draws) {
if (dim < 256 && draw.kind == ScreenDraw.KIND_IMAGE) continue;
int value = dim < 256 ? dimValue(draw.value, dim) : draw.value;
draws.add(new ScreenDraw(draw.kind, draw.fontId, draw.encoding, draw.imageId,
draw.x + surface.x, draw.y + surface.y, draw.value,
draw.x + surface.x, draw.y + surface.y, value,
draw.fwCps, draw.fwDx, draw.fwInk));
}
fingerprint.append('|').append(surface.id)
@@ -421,6 +454,9 @@ public final class SurfaceCompositor {
.append('#').append(surface.zOrder)
.append(':').append(surface.transparency)
.append(':').append(surface.fingerprint);
if (dim < 256) {
fingerprint.append(":dim").append(dim);
}
}
return new Composite(gray, screenWidth, screenHeight, fingerprint.toString(), nextCompositeSeq++,
draws.toArray(new ScreenDraw[0]));
@@ -437,10 +473,22 @@ public final class SurfaceCompositor {
if (copyWidth <= 0 || copyHeight <= 0) {
return;
}
int dim = dimForLocked(surface);
for (int row = 0; row < copyHeight; row++) {
int srcOffset = (srcY + row) * surface.width + srcX;
int dstOffset = (dstY + row) * screenWidth + dstX;
if (surface.transparency == TRANSPARENCY_OPAQUE) {
if (dim < 256) {
// Dimmed: opaque or color-keyed, 0 stays 0 (black / transparent)
// and everything visible scales, never below 1.
for (int col = 0; col < copyWidth; col++) {
int value = surface.pixels[srcOffset + col] & 0xff;
if (value != 0) {
gray[dstOffset + col] = (byte) dimValue(value, dim);
} else if (surface.transparency == TRANSPARENCY_OPAQUE) {
gray[dstOffset + col] = 0;
}
}
} else if (surface.transparency == TRANSPARENCY_OPAQUE) {
System.arraycopy(surface.pixels, srcOffset, gray, dstOffset, copyWidth);
} else {
for (int col = 0; col < copyWidth; col++) {
+1
View File
@@ -5,6 +5,7 @@
* System and app context menus show gesture help for switching to the other context menu
* Move some items from app context menus to the system context menu
* Long-press now opens the system context menu and tap-then-hold the app context menu (games keep long-press as a move while playing)
* Content beneath an open context menu is dimmed
## 0.6.4 - 2026-09-02
* Context menus are centered and include window title
+22 -2
View File
@@ -122,6 +122,8 @@ type DashboardListener = (snapshot: DashboardSnapshot) => void;
// The shell chrome (sidebar + top bar + overlays) composites above all app
// window surfaces with color-key transparency.
const SHELL_SURFACE_ID = "shell";
/** The shell chrome composites above every window surface (zOrder 0). */
const SHELL_SURFACE_Z_ORDER = 1;
const LOCK_SCREEN_SURFACE_ID = "lock-screen";
const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glasses.";
// Top-bar clock refresh; the phone-side preview polls the Java composite so
@@ -265,6 +267,8 @@ class DashboardController {
/** Input frame that requested the next shell render; see requestShellRender. */
private pendingShellRenderCauseFrameId = 0;
private nextShellRenderWantsFreshData = false;
/** The under-shell dim last sent to the display (see Shell.underlayDim); 1 = none. */
private appliedUnderlayDim = 1;
// One shared worker per app hosts all its windows; spawned on first launch.
private readonly appHosts = new Map<string, WorkerAppHost>();
// The window hosting the on-glasses text-setting editor (the Settings app
@@ -1084,9 +1088,13 @@ class DashboardController {
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
zOrder: 1,
zOrder: SHELL_SURFACE_Z_ORDER,
transparency: "color-key",
});
// A fresh (or reused) compositor starts undimmed; the shell render
// loop re-applies the current dim from here.
await target.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1);
this.appliedUnderlayDim = 1;
const foregroundWindowId = shell.foregroundWindow()?.windowId;
for (const window of shell.getWindows()) {
await this.configureWindowSurface(
@@ -1364,9 +1372,13 @@ class DashboardController {
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
zOrder: 1,
zOrder: SHELL_SURFACE_Z_ORDER,
transparency: "color-key",
});
// A fresh (or reused) compositor starts undimmed; the shell render
// loop re-applies the current dim from here.
await communicator.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1);
this.appliedUnderlayDim = 1;
await this.configureLockSurface(communicator);
const foregroundWindowId = shell.foregroundWindow()?.windowId;
for (const window of shell.getWindows()) {
@@ -2292,6 +2304,14 @@ class DashboardController {
frameTimings.finishFrame(frameId, "discarded: shell render with no display target");
return;
}
// A shell overlay that dims what it covers (a context menu) must dim the
// window surfaces too, which live below the shell surface in the
// compositor: forward the factor before this frame composites.
const underlayDim = shell.underlayDim();
if (underlayDim !== this.appliedUnderlayDim) {
this.appliedUnderlayDim = underlayDim;
await display.setUnderlayDim(SHELL_SURFACE_Z_ORDER, underlayDim);
}
const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes));
const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes));
const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer());
+36
View File
@@ -520,6 +520,37 @@ export class GrayImage {
}
}
/**
* A copy of this image with its brightness scaled by `factor` (0..1):
* raster and glyph / firmware-text draws alike, keeping the draws deferred
* so they stay on the cached-glyph wire path. Transparent (0) pixels stay
* transparent and anything visible stays at least 1 (black), so the
* color-key convention survives. Image draws (icons) are baked into the
* raster first: their sources are immutable, content-addressed cache
* entries, and a dimmed variant of each would only pollute that cache.
*/
dimmed(factor: number): GrayImage {
const scale = Math.max(0, Math.min(1, factor));
const copy = new GrayImage(this.width, this.height, 0);
copy.pixels.set(this.pixels);
for (const placed of this.drawList) {
if (placed.kind === "image") {
copy.bitBlt(placed.source, placed.x, placed.y, { transparentZero: true });
}
}
const pixels = copy.pixels;
for (let i = 0; i < pixels.length; i++) {
const value = pixels[i]!;
if (value !== 0) pixels[i] = dimValue(value, scale);
}
for (const placed of this.drawList) {
if (placed.kind !== "image") {
copy.drawList.push({ ...placed, value: dimValue(placed.value, scale) });
}
}
return copy;
}
/**
* This image with its deferred draws rasterized into the pixel buffer.
* Returns this image unchanged when there are none, otherwise a baked copy
@@ -612,6 +643,11 @@ function rasterizeGlyph(
}
}
/** Scale a visible (non-zero) brightness, never below 1 (the color-key black). */
function dimValue(value: number, scale: number): number {
return Math.max(1, Math.round(value * scale));
}
function clampByte(value: number): number {
return Math.max(0, Math.min(255, Math.round(value)));
}
+5
View File
@@ -30,6 +30,11 @@ export function singlePlane(image: GrayImage): Plane[] {
return [{ image, x: 0, y: 0 }];
}
/** The planes with their brightness scaled by `factor` (see GrayImage.dimmed). */
export function dimPlanes(planes: readonly Plane[], factor: number): Plane[] {
return planes.map((plane) => ({ ...plane, image: plane.image.dimmed(factor) }));
}
/**
* Flatten planes into one image: for each plane in order, blend its raster
* (0 = transparent) and then its glyphs on top. A later plane's raster covers
+17
View File
@@ -120,6 +120,11 @@ function nonNegativeNumber(value: number): number {
return Number.isFinite(numeric) ? Math.max(0, numeric) : 0;
}
/** A 0..1 brightness factor as the compositor's 0..256 fixed-point form. */
export function dimFactor256(factor: number): number {
return Math.round(Math.max(0, Math.min(1, factor)) * 256);
}
export class FaceclawCommunicatorBridge {
private readonly communicator: any;
private readonly listenerProxy: any;
@@ -524,6 +529,18 @@ export class FaceclawCommunicatorBridge {
});
}
/**
* Dim every compositor surface whose zOrder is below `belowZOrder` to
* `factor` (0..1; 1 = no dimming) of its brightness; takes effect at the
* next composite. How a shell overlay's Layer.dimUnderneath reaches the
* window surfaces beneath the shell surface.
*/
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
await this.enqueueJavaCall(() => {
this.communicator.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
});
}
/** Show or hide a compositor surface; takes effect at the next composite. */
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
await this.enqueueJavaCall(() => {
+6 -1
View File
@@ -1,5 +1,5 @@
import { ImageSource, Utils } from "@nativescript/core";
import type { FaceclawCommunicatorBridge, SurfaceOptions } from "./faceclaw-communicator";
import { dimFactor256, type FaceclawCommunicatorBridge, type SurfaceOptions } from "./faceclaw-communicator";
declare const com: any;
declare const java: any;
@@ -19,6 +19,7 @@ export type DisplayTarget = Pick<
| "configureSurface"
| "removeSurface"
| "setSurfaceVisible"
| "setUnderlayDim"
| "setScreenBlanked"
| "submitSurfaceFrame"
| "waitForFrameFinished"
@@ -96,6 +97,10 @@ export class PreviewDisplayTarget implements DisplayTarget {
this.compositor.setSurfaceVisible(id, Boolean(visible));
}
async setUnderlayDim(belowZOrder: number, factor: number): Promise<void> {
this.compositor.setUnderlayDim(Math.round(belowZOrder), dimFactor256(factor));
}
async setScreenBlanked(blanked: boolean): Promise<void> {
this.compositor.setScreenBlanked(Boolean(blanked));
}
+34 -6
View File
@@ -1,5 +1,5 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { type Plane } from "../graphics/plane";
import { dimPlanes, type Plane } from "../graphics/plane";
import { spanCurrent } from "../native/frame-timings";
import { type ConfigSettingBoolean, type ConfigSettingString } from "./dashboard-settings";
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
@@ -80,6 +80,15 @@ export interface LayerContext {
export interface Layer {
readonly paintOverBase?: boolean;
/**
* Dim everything painted beneath this layer: a 0..1 brightness factor
* applied to the pixels and glyphs of every plane below it in the stack
* (only meaningful for layers that call paintBelow). false, the default,
* leaves the planes below as painted. For the shell's stack the factor also
* reaches the window surfaces beneath the shell surface (see
* LayerStack.baseDim).
*/
readonly dimUnderneath?: false | number;
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage;
handleInput(event: InputEvent, ctx: LayerContext): Promise<void> | void;
/**
@@ -110,6 +119,7 @@ export class LayerStack {
private baseWidth: number;
private baseHeight: number;
private readonly focusedFn: () => boolean;
private lastBaseDim: number | false = 1;
constructor(
baseLayer: Layer,
@@ -203,7 +213,19 @@ export class LayerStack {
* PaintBelow). All planes share the stack's base size at offset (0, 0).
*/
paint(): Plane[] {
return this.paintLayer(this.layers.length - 1);
this.lastBaseDim = false;
return this.paintLayer(this.layers.length - 1, 1);
}
/**
* The brightness factor the last paint applied to the base plane: 1 when
* nothing dims it, a smaller factor under a layer with dimUnderneath, and
* false when the base was not painted at all (a stacked layer replaced it).
* A stack whose base is transparent over something else (the shell chrome
* over the window surfaces) forwards this to what lies beneath.
*/
baseDim(): number | false {
return this.lastBaseDim;
}
async handleInput(event: InputEvent): Promise<void> {
@@ -235,7 +257,8 @@ export class LayerStack {
}
private paintLayer(index: number): Plane[] {
/** Paint layer `index`; `dimSoFar` is the factor the layers above apply to it. */
private paintLayer(index: number, dimSoFar: number): Plane[] {
const layer = this.layers[index]!;
let canvas: GrayImage | null = null;
let belowRequested = false;
@@ -249,10 +272,15 @@ export class LayerStack {
}),
);
const ownPlane: Plane = { image, x: 0, y: 0 };
if (index <= 0 || !belowRequested) {
if (index <= 0) {
this.lastBaseDim = dimSoFar;
return [ownPlane];
}
const below = layer.paintOverBase ? this.paintLayer(0) : this.paintLayer(index - 1);
return [...below, ownPlane];
if (!belowRequested) {
return [ownPlane];
}
const dim = layer.dimUnderneath || 1;
const below = this.paintLayer(layer.paintOverBase ? 0 : index - 1, dimSoFar * dim);
return [...(dim < 1 ? dimPlanes(below, dim) : below), ownPlane];
}
}
+10 -1
View File
@@ -31,9 +31,11 @@ export type MenuLayout = {
maxHeight?: number;
/**
* Dim hint line pinned to the bottom edge inside the box, below the items
* (gesture help such as "●- system menu"). Takes a line off the item area.
* (gesture help such as "●— system menu"). Takes a line off the item area.
*/
footer?: string;
/** Brightness factor for everything beneath the menu (see Layer.dimUnderneath). Default: none. */
dimUnderneath?: number;
/**
* Paint as a standalone page: the layers below stay in the stack for back
* navigation but are not composited underneath. Default: false — the menu
@@ -182,10 +184,17 @@ export function drawRightValueMenuItem(
image.drawText(font, valueX, y + LIST_ROW_TEXT_INSET, value, 220);
}
/** How far context menus (window and system) dim the content beneath them. */
export const CONTEXT_MENU_DIM = 0.25;
export class MenuLayer implements Layer {
private selectedIndex = 0;
private scrollRow = 0;
get dimUnderneath(): false | number {
return this.layout.dimUnderneath ?? false;
}
constructor(
private readonly title: string | null,
private readonly items: MenuItem[],
+13 -1
View File
@@ -15,7 +15,7 @@ import {
makeInputEvent,
} from "../gestures";
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
@@ -206,6 +206,7 @@ class ShellOverlayMenuLayer extends MenuLayer {
width,
minHeight: 150,
footer,
dimUnderneath: CONTEXT_MENU_DIM,
});
}
@@ -603,6 +604,17 @@ class Shell {
return this.stack.paint();
}
/**
* The brightness factor a shell overlay (a context menu) currently applies
* to what lies beneath the shell surface, i.e. the window surfaces: 1 when
* nothing dims them. Read after paintSurface; the controller forwards it to
* the compositor, which cannot see the shell's layer stack.
*/
underlayDim(): number {
const dim = this.stack.baseDim();
return this.screenOn && dim !== false ? dim : 1;
}
async receiveInput(event: InputEvent, frameId = 0): Promise<ShellInputOutcome> {
const previous = this.lastInput;
this.lastInput = event;
+2 -1
View File
@@ -2,7 +2,7 @@ import { GrayImage } from "../graphics/image";
import { singlePlane, type Plane } from "../graphics/plane";
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
import { type Layer, LayerStack, noopLayerActions } from "./layers";
import { MenuLayer, type MenuItem, type MenuLayout } from "./menu";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
import type { WorkerAppReply } from "./shell/worker-window";
/**
@@ -23,6 +23,7 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
width: 272,
minHeight: 150,
footer: gestureHints([[GESTURE_LONG_PRESS, "system menu"]]),
dimUnderneath: CONTEXT_MENU_DIM,
};
export class WindowMenuLayer extends MenuLayer {