Apps submit mutli-plane images with glyphs, rather than mere images

This commit is contained in:
James Babcock committed 2026-08-16 21:01:20 -07:00
1 parent df85fb4072
commit fdf683761e
22 files changed
+348 -101

No files matched your search

+3 -3
View File
@@ -1,4 +1,4 @@
import { GrayImage } from "../graphics/image";
import { type Plane } from "../graphics/plane";
import { type IconName } from "../graphics/icons";
import { type LayerActions } from "../ui/layers";
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
@@ -60,8 +60,8 @@ export type AppContext = {
) => Promise<void>;
/** Get this app's shared worker host, spawning the worker on first use. */
ensureWorkerHost: (createWorker: () => Worker) => WorkerAppHost;
/** Submit a painted frame for a window surface (windows created outside launchInProcessApp). */
submitWindowFrame: (surfaceId: string, image: GrayImage, paintMs: number, frameId: number) => Promise<void>;
/** Submit a window surface's painted frame, as planes (windows created outside launchInProcessApp). */
submitWindowFrame: (surfaceId: string, planes: Plane[], paintMs: number, frameId: number) => Promise<void>;
setWindowSurfaceVisible: (surfaceId: string, visible: boolean) => void;
requestShellRender: () => void;
appendLog: (message: string) => void;
+6 -4
View File
@@ -9,6 +9,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import * as frameTimings from "../../native/frame-timings";
import type { DashboardInputEvent } from "../../ui/layers";
@@ -527,11 +528,11 @@ function updateTickTimer(window: BlocksWindow): void {
}
}
function paint(window: BlocksWindow): GrayImage {
function paint(window: BlocksWindow): Plane[] {
if (window.menu?.isOpen()) {
return window.menu.paint();
}
return paintContent(window);
return singlePlane(paintContent(window));
}
function paintContent(window: BlocksWindow): GrayImage {
@@ -632,11 +633,11 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(window)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === window.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: blocks content unchanged");
return;
@@ -646,6 +647,7 @@ function renderAndSubmit(window: BlocksWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+19 -6
View File
@@ -13,6 +13,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import * as frameTimings from "../../native/frame-timings";
import type { DashboardInputEvent } from "../../ui/layers";
@@ -598,11 +599,20 @@ function layoutFor(window: FreecellWindow): Layout {
};
}
function paint(window: FreecellWindow): GrayImage {
function paint(window: FreecellWindow): Plane[] {
if (window.menu?.isOpen()) {
return window.menu.paint();
}
return paintContent(window);
const planes = singlePlane(paintContent(window));
if (window.phase === "won") {
// The win box goes on its own plane: card labels are deferred glyphs that
// render above their own image's raster, so an overlay drawn into the
// board image could not cover them.
const overlay = new GrayImage(window.viewportWidth, window.viewportHeight, 0);
paintWinOverlay(overlay, window);
planes.push({ image: overlay, x: 0, y: 0 });
}
return planes;
}
function paintContent(window: FreecellWindow): GrayImage {
@@ -635,7 +645,7 @@ function paintContent(window: FreecellWindow): GrayImage {
paintCascade(image, window, layout, i);
}
paintCursor(image, window, layout);
if (window.phase === "won") paintWinOverlay(image, window);
// The win overlay is painted onto its own plane by paint(), not here.
return image;
}
@@ -702,7 +712,9 @@ function paintWinOverlay(image: GrayImage, window: FreecellWindow): void {
const height = 120;
const x = Math.round((window.viewportWidth - width) / 2);
const y = Math.round((window.viewportHeight - height) / 2);
image.fillRoundedRect(x, y, width, height, 0, 8);
// Fill 1, not 0: identical after 4bpp quantization, but 0 is transparent
// when this overlay composites as its own plane over the board.
image.fillRoundedRect(x, y, width, height, 1, 8);
image.drawRoundedRect(x, y, width, height, 200, 8);
drawCenteredIn(image, largeFont, x, width, y + 14, "You win!", 255);
drawCenteredIn(image, smallFont, x, width, y + 56, `${window.moves} moves`, 170);
@@ -726,11 +738,11 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(window)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === window.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: freecell content unchanged");
return;
@@ -740,6 +752,7 @@ function renderAndSubmit(window: FreecellWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+1 -1
View File
@@ -37,7 +37,7 @@ const launcherApp: AppDefinition = {
],
launchApp: (appId) => ctx.launchApp(appId),
uninstallApp: (appId) => ctx.uninstallApp(appId),
submitFrame: (image, paintMs, frameId) => ctx.submitWindowFrame(LAUNCHER_SURFACE_ID, image, paintMs, frameId),
submitFrame: (planes, paintMs, frameId) => ctx.submitWindowFrame(LAUNCHER_SURFACE_ID, planes, paintMs, frameId),
setSurfaceVisible: (visible) => ctx.setWindowSurfaceVisible(LAUNCHER_SURFACE_ID, visible),
}),
);
+3 -2
View File
@@ -1,6 +1,7 @@
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { truncateText } from "../../graphics/textwrap";
import { GrayImage } from "../../graphics/image";
import { type Plane } from "../../graphics/plane";
import { renderIcon, type IconName } from "../../graphics/icons";
import { clamp } from "../../util/numeric-util";
import {
@@ -37,8 +38,8 @@ export type LauncherOptions = {
apps: () => LauncherAppEntry[];
launchApp: (appId: string) => Promise<void> | void;
uninstallApp: (appId: string) => Promise<void> | void;
/** Submit a painted viewport-sized frame to this window's surface. */
submitFrame: (image: GrayImage, paintMs: number, frameId: number) => Promise<void>;
/** Submit a painted viewport-sized frame (as planes) to this window's surface. */
submitFrame: (planes: Plane[], paintMs: number, frameId: number) => Promise<void>;
/** Flip the launcher's compositor surface visibility on foreground changes. */
setSurfaceVisible: (visible: boolean) => void;
};
@@ -13,6 +13,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import * as frameTimings from "../../native/frame-timings";
import type { DashboardInputEvent } from "../../ui/layers";
@@ -552,11 +553,11 @@ function elapsedMs(window: MinesweeperWindow): number {
return window.elapsedMs + runningMs;
}
function paint(window: MinesweeperWindow): GrayImage {
function paint(window: MinesweeperWindow): Plane[] {
if (window.menu?.isOpen()) {
return window.menu.paint();
}
return paintContent(window);
return singlePlane(paintContent(window));
}
function paintContent(window: MinesweeperWindow): GrayImage {
@@ -696,11 +697,11 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(window)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === window.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: minesweeper content unchanged");
return;
@@ -710,6 +711,7 @@ function renderAndSubmit(window: MinesweeperWindow, inputFrameId: number): void
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+6 -4
View File
@@ -8,6 +8,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import * as frameTimings from "../../native/frame-timings";
import type { DashboardInputEvent } from "../../ui/layers";
@@ -667,9 +668,9 @@ function handleInput(win: NavWindow, event: DashboardInputEvent, frameId: number
// ---------------------------------------------------------------------------
// Painting
function paint(win: NavWindow): GrayImage {
function paint(win: NavWindow): Plane[] {
if (win.menu?.isOpen()) return win.menu.paint();
return paintContent(win);
return singlePlane(paintContent(win));
}
function paintContent(win: NavWindow): GrayImage {
@@ -858,11 +859,11 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${win.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(win)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === win.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: navigate content unchanged");
return;
@@ -872,6 +873,7 @@ function renderAndSubmit(win: NavWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+6 -4
View File
@@ -19,6 +19,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import * as frameTimings from "../../native/frame-timings";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
@@ -907,11 +908,11 @@ function drawFlipper(image: GrayImage, flipper: Flipper): void {
}
}
function paint(window: PinballWindow): GrayImage {
function paint(window: PinballWindow): Plane[] {
if (window.menu?.isOpen()) {
return window.menu.paint();
}
return paintContent(window);
return singlePlane(paintContent(window));
}
function paintContent(window: PinballWindow): GrayImage {
@@ -1025,11 +1026,11 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(window)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === window.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: pinball content unchanged");
return;
@@ -1039,6 +1040,7 @@ function renderAndSubmit(window: PinballWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+6 -4
View File
@@ -7,6 +7,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont } from "../../graphics/bdffont";
import { truncateText, wrapText } from "../../graphics/textwrap";
import * as frameTimings from "../../native/frame-timings";
@@ -513,9 +514,9 @@ async function handleRoamTool(name: string, args: any): Promise<ToolResult> {
// ---------------------------------------------------------------------------
// Painting
function paint(win: RoamWindow): GrayImage {
function paint(win: RoamWindow): Plane[] {
if (win.menu?.isOpen()) return win.menu.paint();
return paintContent(win);
return singlePlane(paintContent(win));
}
function paintContent(win: RoamWindow): GrayImage {
@@ -578,11 +579,11 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${win.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(win)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === win.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: roam content unchanged");
return;
@@ -592,6 +593,7 @@ function renderAndSubmit(win: RoamWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+2 -2
View File
@@ -1,4 +1,4 @@
import { GrayImage } from "../../graphics/image";
import { type Plane } from "../../graphics/plane";
import { LayerActions } from "../../ui/layers";
import { EditTextSettingLayer } from "../../ui/dashboard-settings";
import { createSettingsPanelLayer } from "../../ui/dashboard/settings-menus";
@@ -10,7 +10,7 @@ export const SETTINGS_SURFACE_ID = "window:settings";
export type SettingsAppOptions = {
actions: LayerActions;
submitFrame: (image: GrayImage, paintMs: number, frameId: number) => Promise<void>;
submitFrame: (planes: Plane[], paintMs: number, frameId: number) => Promise<void>;
setSurfaceVisible: (visible: boolean) => void;
removeSurface: () => void;
onClosed: () => void;
+20 -11
View File
@@ -32,6 +32,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getFont } from "../../graphics/bdffont";
import { TERMINAL_ICON_GLYPHS } from "../../graphics/icons";
import * as frameTimings from "../../native/frame-timings";
@@ -1367,11 +1368,11 @@ async function launchAndOpenView(control: ControlConnection, preset: string): Pr
return socket;
}
function paint(window: TerminalWindow): GrayImage {
function paint(window: TerminalWindow): Plane[] {
if (window.menu?.isOpen()) {
return window.menu.paint();
}
return paintContent(window);
return singlePlane(paintContent(window));
}
function paintContent(window: TerminalWindow): GrayImage {
@@ -1486,7 +1487,17 @@ function paintView(window: ViewWindow): GrayImage {
image.fillRect(window.emulator.cursorCol() * CELL_WIDTH, cursorScreenRow * CELL_HEIGHT, CELL_WIDTH, CELL_HEIGHT, 70);
}
// Stale content stays visible across a disconnect, so flag it: a status
// line over the bottom row whenever the session isn't actually attached.
// Text is deferred glyphs (always on top of raster), so the covered rows
// must be suppressed rather than painted over.
const statusBannerY = window.client.state().phase !== "attached"
? window.viewportHeight - CELL_HEIGHT
: null;
for (let row = 0; row < rows; row++) {
const rowY = row * CELL_HEIGHT;
if (statusBannerY !== null && rowY + CELL_HEIGHT > statusBannerY) continue;
const absolute = top + row;
let text = "";
if (absolute >= historyNext) {
@@ -1495,18 +1506,15 @@ function paintView(window: ViewWindow): GrayImage {
} else if (absolute >= window.archiveStart) {
text = window.archive[absolute - window.archiveStart] ?? "";
}
if (text.length) image.drawText(terminalFont, 0, row * CELL_HEIGHT, text, 200);
if (text.length) image.drawText(terminalFont, 0, rowY, text, 200);
}
if (!following) {
drawScrollIndicator(image, top, window.archiveStart, bottomTop);
}
// Stale content stays visible across a disconnect, so flag it: a status
// line over the bottom row whenever the session isn't actually attached.
if (window.client.state().phase !== "attached") {
const y = window.viewportHeight - CELL_HEIGHT;
image.fillRect(0, y, window.viewportWidth, CELL_HEIGHT, 0);
image.drawText(terminalFont, 0, y, window.status, 170);
if (statusBannerY !== null) {
image.fillRect(0, statusBannerY, window.viewportWidth, CELL_HEIGHT, 0);
image.drawText(terminalFont, 0, statusBannerY, window.status, 170);
}
return image;
}
@@ -1539,11 +1547,11 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(window)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === window.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: terminal content unchanged");
return;
@@ -1553,6 +1561,7 @@ function renderAndSubmit(window: TerminalWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+6 -4
View File
@@ -5,6 +5,7 @@
*/
import "@nativescript/core/globals";
import { GrayImage } from "../../graphics/image";
import { flattenPlanes, planesFingerprint, singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import * as frameTimings from "../../native/frame-timings";
import {
@@ -498,11 +499,11 @@ function updateRenderTimer(window: TimerWindow): void {
}
}
function paint(window: TimerWindow): GrayImage {
function paint(window: TimerWindow): Plane[] {
if (window.menu?.isOpen()) {
return window.menu.paint();
}
return paintContent(window);
return singlePlane(paintContent(window));
}
function paintContent(window: TimerWindow): GrayImage {
@@ -645,11 +646,11 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void {
const frameId = inputFrameId > 0 ? inputFrameId : frameTimings.startFrame(`render:${window.windowId}`);
try {
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => paint(window)),
);
const paintMs = Date.now() - paintStartedAtMs;
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
if (fingerprint === window.lastSubmittedFingerprint) {
frameTimings.finishFrame(frameId, "discarded: timer content unchanged");
return;
@@ -659,6 +660,7 @@ function renderAndSubmit(window: TimerWindow, inputFrameId: number): void {
frameTimings.finishFrame(frameId, "discarded: no active communicator");
return;
}
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
communicator.submitSurfaceFrame(
buffer.buffer,
+10 -7
View File
@@ -16,6 +16,7 @@ import { isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboa
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { flattenPlanes, planesFingerprint, type Plane } from "../graphics/plane";
import { getDefaultMediumFont } from "../graphics/bdffont";
import { wrapText } from "../graphics/textwrap";
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
@@ -1384,7 +1385,7 @@ class DashboardController {
...this.sharedActions,
requestRender: () => {}, // rebound by createInProcessWindow
},
submitFrame: (image, paintMs, frameId) => this.submitWindowFrame(surfaceId, image, paintMs, frameId),
submitFrame: (planes, paintMs, frameId) => this.submitWindowFrame(surfaceId, planes, paintMs, frameId),
setSurfaceVisible: (visible) => this.setWindowSurfaceVisible(surfaceId, visible),
removeSurface: () => this.removeWindowSurface(surfaceId),
onClosed: () => {
@@ -1440,8 +1441,8 @@ class DashboardController {
uninstallApp: (appId) => this.uninstallApp(appId),
launchInProcessApp: (windowId, surfaceId, create) => this.launchInProcessApp(windowId, surfaceId, create),
ensureWorkerHost: (createWorker) => this.ensureWorkerHost(app.appId, createWorker),
submitWindowFrame: (surfaceId, image, paintMs, frameId) =>
this.submitWindowFrame(surfaceId, image, paintMs, frameId),
submitWindowFrame: (surfaceId, planes, paintMs, frameId) =>
this.submitWindowFrame(surfaceId, planes, paintMs, frameId),
setWindowSurfaceVisible: (surfaceId, visible) => this.setWindowSurfaceVisible(surfaceId, visible),
requestShellRender: () => this.requestShellRender(),
appendLog: (message) => this.appendLog(message),
@@ -1554,13 +1555,14 @@ class DashboardController {
}
/** Submit a painted frame for an in-process window (e.g. the launcher). */
private async submitWindowFrame(surfaceId: string, image: GrayImage, paintMs: number, frameId: number): Promise<void> {
private async submitWindowFrame(surfaceId: string, planes: Plane[], paintMs: number, frameId: number): Promise<void> {
const communicator = this.communicator;
if (!communicator || this.phase === "charging") {
frameTimings.finishFrame(frameId, "discarded: window frame with no active connection");
return;
}
const fingerprint = image.fingerprint();
const fingerprint = planesFingerprint(planes);
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = image.to8bppBuffer();
await communicator.submitSurfaceFrame(
surfaceId,
@@ -1612,7 +1614,7 @@ class DashboardController {
this.nextShellRenderWantsFreshData = false;
beginRenderPass(!wantFreshData);
const paintStartedAtMs = Date.now();
const image = frameTimings.span(frameId, "paint", () =>
const planes = frameTimings.span(frameId, "paint", () =>
frameTimings.runWithFrame(frameId, () => shell.paintSurface()),
);
const paintMs = Date.now() - paintStartedAtMs;
@@ -1627,7 +1629,8 @@ class DashboardController {
frameTimings.finishFrame(frameId, "discarded: shell render with no active connection");
return;
}
const fingerprint = frameTimings.span(frameId, "fingerprint", () => image.fingerprint());
const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes));
const image = frameTimings.span(frameId, "flatten", () => flattenPlanes(planes));
const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer());
await this.communicator.submitSurfaceFrame(
SHELL_SURFACE_ID,
+8
View File
@@ -11,7 +11,15 @@ export type Glyph = {
bitmapRows: number[];
};
let nextFontFingerprintId = 1;
export class BdfFont {
/**
* Process-local identity for content fingerprints (fonts are cached
* singletons, so two draws with the same id + encoding are the same glyph
* bitmap). Only meaningful within one JS context; never persist it.
*/
readonly fingerprintId = nextFontFingerprintId++;
readonly ascent: number;
readonly descent: number;
readonly lineHeight: number;
+120 -18
View File
@@ -21,10 +21,31 @@ export function imageFromAsciiArt(lines: readonly string[], value = 255): GrayIm
return image;
}
/**
* One deferred glyph draw. Text drawn through drawGlyph/drawText/
* drawTextWrapped is retained here rather than baked into the pixel buffer,
* so the compositor can later ship glyphs as display-list references instead
* of pixels. (x, y) and value are the drawGlyph arguments (y is the top of
* the line; the baseline is y + font.ascent).
*/
export type PlacedGlyph = {
font: BdfFont;
glyph: Glyph;
x: number;
y: number;
value: number;
};
export class GrayImage {
readonly width: number;
readonly height: number;
readonly pixels: Uint8Array;
/**
* Deferred glyph draws, always rendered on top of the raster pixels
* regardless of draw order. An overlay that must cover text below it needs
* its own plane (see plane.ts) or must suppress the covered draws.
*/
private readonly glyphList: PlacedGlyph[] = [];
constructor(width: number, height: number, fill = 0) {
this.width = width;
@@ -33,14 +54,24 @@ export class GrayImage {
this.clear(fill);
}
get glyphs(): readonly PlacedGlyph[] {
return this.glyphList;
}
hasGlyphs(): boolean {
return this.glyphList.length > 0;
}
clone(): GrayImage {
const clone = new GrayImage(this.width, this.height, 0);
clone.pixels.set(this.pixels);
clone.glyphList.push(...this.glyphList);
return clone;
}
clear(value = 0): void {
this.pixels.fill(clampByte(value));
this.glyphList.length = 0;
}
setPixel(x: number, y: number, value: number): void {
@@ -107,7 +138,6 @@ export class GrayImage {
drawText(font: BdfFont, x: number, y: number, text: string, value: number): void {
let cursorX = x;
const fill = clampByte(value);
for (const char of text) {
if (char === "\n") {
cursorX = x;
@@ -116,7 +146,7 @@ export class GrayImage {
}
const glyph = font.getGlyph(char.codePointAt(0) ?? 32);
if (!glyph) continue;
this.drawGlyph(font, glyph, cursorX, y, fill);
this.drawGlyph(font, glyph, cursorX, y, value);
cursorX += glyph.dwidthX;
}
}
@@ -168,6 +198,23 @@ export class GrayImage {
this.pixels[ty * this.width + tx] = value;
}
}
// Carry the source's deferred glyphs along, translated into destination
// space. Membership in the copied rect is judged by the glyph's anchor
// (fine for the whole-image blits this is used for; a sub-rect blit can
// clip a glyph's overhang imperfectly).
if (source.glyphList.length) {
const offsetX = destX - srcX;
const offsetY = destY - srcY;
for (const placed of source.glyphList) {
if (
placed.x >= srcX && placed.x < srcX + copyWidth &&
placed.y >= srcY && placed.y < srcY + copyHeight
) {
this.glyphList.push({ ...placed, x: placed.x + offsetX, y: placed.y + offsetY });
}
}
}
}
fillRoundedRect(
@@ -251,31 +298,80 @@ export class GrayImage {
hash ^= this.pixels[i]!;
hash = Math.imul(hash, 16777619);
}
for (const placed of this.glyphList) {
hash = mixInt(hash, placed.font.fingerprintId);
hash = mixInt(hash, placed.glyph.encoding);
hash = mixInt(hash, placed.x);
hash = mixInt(hash, placed.y);
hash = mixInt(hash, placed.value);
}
return `fnv:${(hash >>> 0).toString(16)}`;
}
/**
* Snapshot of the full image as a single 8-bit-per-pixel grayscale buffer,
* row-major and top-to-bottom. The Java side turns this into whatever wire
* format the firmware currently expects (today: a 4bpp BMP), so all framing
* concerns stay on one side of the bridge.
* row-major and top-to-bottom, with any deferred glyphs baked in. The Java
* side turns this into whatever wire format the firmware currently expects
* (today: a 4bpp BMP), so all framing concerns stay on one side of the
* bridge.
*/
to8bppBuffer(): Uint8Array {
return Uint8Array.from(this.pixels);
if (!this.glyphList.length) {
return Uint8Array.from(this.pixels);
}
// withGlyphsBaked's buffer is a private temporary, safe to hand out.
return this.withGlyphsBaked().pixels;
}
private drawGlyph(font: BdfFont, glyph: Glyph, x: number, y: number, value: number): void {
const baselineY = y + font.ascent;
const top = baselineY - (glyph.bbxHeight + glyph.bbxY);
const left = x + glyph.bbxX;
const rowBitWidth = ((glyph.bbxWidth + 7) >> 3) << 3;
for (let row = 0; row < glyph.bbxHeight; row++) {
const bits = glyph.bitmapRows[row] ?? 0;
for (let col = 0; col < glyph.bbxWidth; col++) {
const shift = rowBitWidth - 1 - col;
if (((bits >> shift) & 1) !== 0) {
this.setPixelUnchecked(left + col, top + row, value);
}
/**
* Record a glyph draw. Deferred: the glyph joins the image's glyph list
* (rendered above all raster pixels) instead of being baked into the
* buffer.
*/
drawGlyph(font: BdfFont, glyph: Glyph, x: number, y: number, value: number): void {
this.glyphList.push({ font, glyph, x, y, value: clampByte(value) });
}
/**
* This image with its glyph list rasterized into the pixel buffer. Returns
* this image unchanged when there are no glyphs, otherwise a baked copy
* with an empty glyph list.
*/
withGlyphsBaked(): GrayImage {
if (!this.glyphList.length) return this;
const baked = new GrayImage(this.width, this.height, 0);
baked.pixels.set(this.pixels);
this.bakeGlyphsInto(baked, 0, 0);
return baked;
}
/** Rasterize this image's glyph list into target, translated by (dx, dy). */
bakeGlyphsInto(target: GrayImage, dx: number, dy: number): void {
for (const placed of this.glyphList) {
rasterizeGlyph(target, placed.font, placed.glyph, placed.x + dx, placed.y + dy, placed.value);
}
}
}
/** Bake one BDF glyph into an image's pixel buffer (y is the top of the line). */
function rasterizeGlyph(
target: GrayImage,
font: BdfFont,
glyph: Glyph,
x: number,
y: number,
value: number,
): void {
const baselineY = y + font.ascent;
const top = baselineY - (glyph.bbxHeight + glyph.bbxY);
const left = x + glyph.bbxX;
const rowBitWidth = ((glyph.bbxWidth + 7) >> 3) << 3;
for (let row = 0; row < glyph.bbxHeight; row++) {
const bits = glyph.bitmapRows[row] ?? 0;
for (let col = 0; col < glyph.bbxWidth; col++) {
const shift = rowBitWidth - 1 - col;
if (((bits >> shift) & 1) !== 0) {
target.setPixel(left + col, top + row, value);
}
}
}
@@ -285,6 +381,12 @@ function clampByte(value: number): number {
return Math.max(0, Math.min(255, Math.round(value)));
}
/** Fold a 32-bit int into an FNV-style running hash. */
function mixInt(hash: number, value: number): number {
hash ^= value | 0;
return Math.imul(hash, 16777619);
}
function roundedRectRowSpan(
row: number,
x: number,
+74
View File
@@ -0,0 +1,74 @@
import { GrayImage } from "./image";
/**
* The unit apps (and the shell) submit to the compositor: instead of one
* flattened image, a frame is an ordered array of planes, each an image plus
* placement metadata. Planes composite in array order, first at the bottom,
* with the shell's transparency convention: an 8-bit value of 0 is
* transparent, 1 is black (the two quantize to the same 4bpp shade, so
* reserving 0 costs nothing visible).
*
* This representation exists to carry structure the wire pipeline will later
* exploit: a plane's glyph list can become on-glasses display-list references
* (texture cache), and the offset can diverge per lens for stereo depth. For
* now the compositor boundary just flattens (flattenPlanes) and feeds the
* existing incremental-update and screen-preview pipeline.
*/
export type Plane = {
image: GrayImage;
/**
* Offset of the plane's top-left within the submitted frame. Applied
* identically to both lenses for now; a per-lens divergence (stereo depth
* for floating overlays) is the planned extension.
*/
x: number;
y: number;
};
/** Wrap a single full-frame image as a one-plane submission. */
export function singlePlane(image: GrayImage): Plane[] {
return [{ image, x: 0, y: 0 }];
}
/**
* Flatten planes into one image: for each plane in order, blend its raster
* (0 = transparent) and then its glyphs on top. A later plane's raster covers
* an earlier plane's glyphs — this is what lets an overlay plane occlude text
* beneath it, which raster drawn into the *same* image cannot do (glyphs
* always render above their own image's raster).
*
* The output size defaults to the planes' joint extent (in practice the base
* plane's size, since overlays share its dimensions).
*/
export function flattenPlanes(
planes: readonly Plane[],
size?: { width: number; height: number },
): GrayImage {
const width = size?.width ?? Math.max(1, ...planes.map((plane) => plane.x + plane.image.width));
const height = size?.height ?? Math.max(1, ...planes.map((plane) => plane.y + plane.image.height));
const only = planes.length === 1 ? planes[0]! : undefined;
if (only && only.x === 0 && only.y === 0 && only.image.width === width && only.image.height === height) {
return only.image.withGlyphsBaked();
}
const target = new GrayImage(width, height, 0);
for (const plane of planes) {
// Baking first (rather than bitBlt's glyph carry-over) is what enforces
// the plane ordering: the next plane's raster must be able to cover this
// plane's glyphs.
target.bitBlt(plane.image.withGlyphsBaked(), plane.x, plane.y, { transparentZero: true });
}
return target;
}
/**
* Stable identifier for a plane stack's composited content: equal
* fingerprints mean an equal flattened frame. Feeds the same dedupe paths
* that single-image fingerprints did.
*/
export function planesFingerprint(planes: readonly Plane[]): string {
return planes
.map((plane) => `${plane.x},${plane.y}+${plane.image.width}x${plane.image.height}:${plane.image.fingerprint()}`)
.join("|");
}
+4 -3
View File
@@ -17,10 +17,11 @@ export function grayImageToPreviewSource(image: GrayImage): ImageSource | null {
return null;
}
const baked = image.withGlyphsBaked();
const bitmap = com.faceclaw.app.PreviewBitmapUtil.fromGray(
image.pixels.buffer,
image.width,
image.height,
baked.pixels.buffer,
baked.width,
baked.height,
PREVIEW_BRIGHTEN_GAMMA,
);
return new ImageSource(bitmap);
+33 -15
View File
@@ -1,5 +1,5 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { BdfFont } from "../graphics/bdffont";
import { type Plane } from "../graphics/plane";
import { spanCurrent } from "../native/frame-timings";
import { type ConfigSettingString } from "./dashboard-settings";
@@ -58,6 +58,18 @@ export const noopLayerActions: LayerActions = {
playBuzzerSequence: () => {},
};
/**
* Hands a layer its paint canvas: a fresh transparent (0-filled) image the
* size of the stack. Calling it also declares that the layers below should
* stay visible beneath this layer — each layer becomes its own plane, so a
* layer that never calls paintBelow fully replaces what is under it (its
* plane is submitted alone), while one that does gets composited above the
* planes below. Repeat calls return the same canvas.
*
* Note the plane model's occlusion rule: raster painted onto the canvas
* covers lower planes entirely (including their text), but glyphs drawn into
* the *same* image always render above that image's own raster.
*/
export type PaintBelow = () => GrayImage;
function notifyRemoved(layer: Layer | undefined): void {
@@ -153,7 +165,12 @@ export class LayerStack {
Object.assign(this.ctx.actions, actions);
}
paint(): GrayImage {
/**
* Paint the stack as planes, bottom to top: the top layer's plane last,
* preceded by the planes of the layers it asked to remain visible (see
* PaintBelow). All planes share the stack's base size at offset (0, 0).
*/
paint(): Plane[] {
return this.paintLayer(this.layers.length - 1);
}
@@ -161,23 +178,24 @@ export class LayerStack {
await this.layers[this.layers.length - 1]!.handleInput(event, this.ctx);
}
private paintLayer(index: number): GrayImage {
private paintLayer(index: number): Plane[] {
const layer = this.layers[index]!;
let cachedBelow: GrayImage | null = null;
return spanCurrent(`paint[${index}]:${layer.constructor.name}`, () =>
let canvas: GrayImage | null = null;
let belowRequested = false;
const image = spanCurrent(`paint[${index}]:${layer.constructor.name}`, () =>
layer.paint(this.ctx, () => {
if (cachedBelow) {
return cachedBelow;
belowRequested = true;
if (!canvas) {
canvas = new GrayImage(this.baseWidth, this.baseHeight, 0);
}
if (index <= 0) {
cachedBelow = new GrayImage(this.baseWidth, this.baseHeight, 0);
} else if (layer.paintOverBase) {
cachedBelow = this.paintLayer(0);
} else {
cachedBelow = this.paintLayer(index - 1);
}
return cachedBelow;
return canvas;
}),
);
const ownPlane: Plane = { image, x: 0, y: 0 };
if (index <= 0 || !belowRequested) {
return [ownPlane];
}
const below = layer.paintOverBase ? this.paintLayer(0) : this.paintLayer(index - 1);
return [...below, ownPlane];
}
}
+5 -4
View File
@@ -1,4 +1,5 @@
import { GrayImage } from "../../graphics/image";
import { type Plane } from "../../graphics/plane";
import * as frameTimings from "../../native/frame-timings";
import { beginRenderPass, endRenderPass } from "../../util/render-freshness";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, PaintBelow } from "../layers";
@@ -37,7 +38,7 @@ export type InProcessWindowOptions = {
/** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions;
baseLayer: Layer;
submitFrame: (image: GrayImage, paintMs: number, frameId: number) => Promise<void>;
submitFrame: (planes: Plane[], paintMs: number, frameId: number) => Promise<void>;
setSurfaceVisible: (visible: boolean) => void;
removeSurface?: () => void;
onClosed?: () => void;
@@ -52,7 +53,7 @@ export type InProcessWindow = {
/** The controller-provided plumbing common to every in-process app window. */
export type InProcessAppOptions = {
actions: LayerActions;
submitFrame: (image: GrayImage, paintMs: number, frameId: number) => Promise<void>;
submitFrame: (planes: Plane[], paintMs: number, frameId: number) => Promise<void>;
setSurfaceVisible: (visible: boolean) => void;
removeSurface: () => void;
onClosed: () => void;
@@ -89,9 +90,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
nextRenderWantsFreshData = false;
beginRenderPass(!wantFreshData);
const paintStartedAtMs = Date.now();
const image = stack.paint();
const planes = stack.paint();
const paintUsedStaleData = endRenderPass();
await options.submitFrame(image, Date.now() - paintStartedAtMs, frameId);
await options.submitFrame(planes, Date.now() - paintStartedAtMs, frameId);
if (paintUsedStaleData) {
nextRenderWantsFreshData = true;
requestRender();
+4 -1
View File
@@ -1,4 +1,5 @@
import { GrayImage } from "../../graphics/image";
import { flattenPlanes } from "../../graphics/plane";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, PaintBelow } from "../layers";
import { appViewportRect, appViewportSize, SHELL_OPAQUE_BLACK } from "./geometry";
@@ -43,7 +44,9 @@ export class ShellModalLayer implements Layer {
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
const image = paintBelow();
const inner = this.stack.paint();
// Flattening bakes the inner stack's planes (glyphs included) so the blit
// below transplants the finished modal content into this layer's plane.
const inner = flattenPlanes(this.stack.paint(), MODAL_INTERIOR);
const rect = modalRect();
image.fillRoundedRect(rect.x, rect.y, rect.width, rect.height, SHELL_OPAQUE_BLACK, 8);
image.drawRoundedRect(rect.x, rect.y, rect.width, rect.height, 110, 8);
+3 -2
View File
@@ -1,4 +1,5 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont } from "../../graphics/bdffont";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
@@ -470,9 +471,9 @@ class Shell {
}
/** Paint the shell surface: transparent chrome, or all-transparent when asleep. */
paintSurface(): GrayImage {
paintSurface(): Plane[] {
if (!this.screenOn) {
return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
return singlePlane(new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0));
}
return this.stack.paint();
}
+3 -2
View File
@@ -1,4 +1,5 @@
import { GrayImage } from "../graphics/image";
import { singlePlane, type Plane } from "../graphics/plane";
import {
noopLayerActions,
LayerStack,
@@ -61,8 +62,8 @@ export class WindowMenu {
}
/** Paint the window content with the menu over it (content alone if closed). */
paint(): GrayImage {
return this.stack ? this.stack.paint() : this.options.paintBase();
paint(): Plane[] {
return this.stack ? this.stack.paint() : singlePlane(this.options.paintBase());
}
/** Route an input event to the menu; the menu closes by popping itself. */