Redesign handling of long-press gesture and context menus

This commit is contained in:
James Babcock committed 2026-07-22 22:45:14 -07:00
1 parent 3452945322
commit afc05d6a19
10 files changed
+455 -56

No files matched your search

+49
View File
@@ -1,6 +1,9 @@
import { GrayImage } from "../../graphics/image";
import * as frameTimings from "../../native/frame-timings";
import { beginRenderPass, endRenderPass } from "../../util/render-freshness";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, PaintBelow } from "../layers";
import { type MenuItem } from "../menu";
import { WindowMenuLayer } from "../window-menu";
import { windowIcon } from "./chrome-layer";
import { type IconName } from "../../graphics/icons";
import { APP_VIEWPORT } from "./geometry";
@@ -21,6 +24,12 @@ export type InProcessWindowOptions = {
/** Lucide icon name for the sidebar indicator; falls back to iconLetter. */
icon?: IconName;
closeable: boolean;
/**
* App-specific entries for the window's long-press menu, listed ahead of
* the default Voice input / Close window entries. Called at open time, so
* the items can reflect current app state.
*/
menuItems?: () => MenuItem[];
/** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions;
baseLayer: Layer;
@@ -62,8 +71,15 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// to keep the paint fast; when they do, schedule one follow-up render that
// requires fresh data (same contract as the dashboard render loop).
let nextRenderWantsFreshData = false;
let closed = false;
async function render(frameId: number): Promise<void> {
if (closed) {
// The surface is gone (e.g. Close window picked from this window's own
// menu); dropping the frame beats submitting to a removed surface.
frameTimings.finishFrame(frameId, "discarded: window closed");
return;
}
const wantFreshData = nextRenderWantsFreshData;
nextRenderWantsFreshData = false;
beginRenderPass(!wantFreshData);
@@ -77,6 +93,31 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
}
}
// The window's long-press menu: app-specific items, then the defaults every
// window shares. In-process apps run on the main thread, so the default
// items act on the shell directly (workers post messages instead).
const openWindowMenu = () => {
if (stack.topMatches((layer) => layer instanceof WindowMenuLayer)) return;
const items: MenuItem[] = [...(options.menuItems?.() ?? [])];
items.push({
label: "Voice input",
onSelect: (ctx) => {
ctx.stack.pop();
shell.startVoiceInput();
},
});
if (options.closeable) {
items.push({
label: "Close window",
onSelect: (ctx) => {
ctx.stack.pop();
shell.closeWindow(options.windowId);
},
});
}
stack.push(new WindowMenuLayer(items));
};
const window: ShellWindow = {
appId: options.appId,
windowId: options.windowId,
@@ -84,6 +125,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
surfaceId: `window:${options.windowId}`,
closeable: options.closeable,
close: () => {
closed = true;
// Fire onRemoved for any pushed layers so they release resources (e.g. a
// demo that enabled a hardware stream) even when closed from within.
stack.clearToBase();
@@ -92,6 +134,13 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
},
drawIcon: windowIcon(options.icon, options.iconLetter),
handleInput: async (event, frameId) => {
// The default long-press response: the window menu. Handled here (not
// per-layer) so it works over submenus and app content alike.
if (event.type === "long-press") {
openWindowMenu();
await render(frameId);
return;
}
await stack.handleInput(event);
await render(frameId);
},
+8 -1
View File
@@ -19,6 +19,8 @@ export type SettingsAppOptions = {
export type SettingsAppWindow = {
window: ShellWindow;
requestRender: () => void;
/** Select a section in the left column by label (e.g. "Terminal"). */
focusSection: (label: string) => void;
/** Whether the glasses-side text-setting editor is the top layer. */
isTextEditorOnTop: () => boolean;
/** Pop the text-setting editor if it is on top; returns whether it was. */
@@ -32,6 +34,7 @@ export type SettingsAppWindow = {
* controller, which lives on the main thread.
*/
export function createSettingsAppWindow(options: SettingsAppOptions): SettingsAppWindow {
const panel = createSettingsPanelLayer();
const { window, stack, requestRender } = createInProcessWindow({
appId: "settings",
windowId: SETTINGS_WINDOW_ID,
@@ -42,7 +45,7 @@ export function createSettingsAppWindow(options: SettingsAppOptions): SettingsAp
actions: options.actions,
// Not wrapped in YieldAtRootLayer: the panel routes double-click itself
// (right column -> left column, then left column -> sidebar).
baseLayer: createSettingsPanelLayer(),
baseLayer: panel,
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
@@ -51,6 +54,10 @@ export function createSettingsAppWindow(options: SettingsAppOptions): SettingsAp
return {
window,
requestRender,
focusSection: (label) => {
panel.focusSection(label);
requestRender();
},
isTextEditorOnTop: () => stack.topMatches((layer) => layer instanceof EditTextSettingLayer),
closeTextEditor: () => stack.popIfTop((layer) => layer instanceof EditTextSettingLayer),
};
+104 -52
View File
@@ -1,7 +1,7 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, LayerActions, LayerStack } from "../layers";
import { DashboardInputEvent, LayerActions, LayerStack, noopLayerActions } from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer } from "../apps/voice-input";
import { SingleNotificationLayer } from "../notifications";
@@ -17,15 +17,17 @@ import { SIDEBAR_WIDTH, TOP_BAR_HEIGHT } from "./geometry";
/**
* The shell: owns the window registry, focus, screen on/off, and the shell
* surface (sidebar + top bar + shell overlays such as the long-press menu and
* the push-to-talk dialog). Runs on the main thread; windows will later live
* in worker threads, today the only window is the in-process dashboard.
* surface (sidebar + top bar + shell overlays such as the escape menu and
* the voice dialog). Runs on the main thread; windows are hosted in-process
* or in per-app worker threads.
*
* Input flow: every event enters via receiveInput. The shell consumes its
* reserved gestures (long-press, and everything while the sidebar or a shell
* overlay has focus) and forwards the rest to the focused window. Windows
* never see events the shell consumed, and the shell keeps working when a
* window's handler hangs (the long-press menu is the escape hatch).
* Input flow: every event enters via receiveInput. The shell consumes
* everything while the sidebar or a shell overlay has focus and forwards the
* rest to the focused window. Long-press goes to the foreground window (from
* the sidebar it focuses the window first); by convention apps answer it with
* a window menu that ends in Voice input / Close window. Holding the press
* past the escape threshold opens the shell's own escape menu, so the shell
* keeps working when a window's handler hangs.
*/
export type ShellWindow = {
@@ -34,7 +36,7 @@ export type ShellWindow = {
title: string;
/** Compositor surface this window renders to; configured at connect / launch. */
surfaceId: string;
/** Whether the shell's long-press menu offers Close Window (launcher and dashboard are pinned). */
/** Whether close menu entries (app menu default, shell escape menu) apply (the launcher is pinned). */
closeable: boolean;
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
close?: () => void;
@@ -72,19 +74,16 @@ export type ShellInputOutcome = { shell: boolean; window: boolean };
type FocusKind = "sidebar" | "window";
const noopActions: LayerActions = {
requestRender: () => {},
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
startVoiceCapture: () => {},
stopVoiceCapture: () => {},
startContinuousVoiceCapture: () => {},
stopContinuousVoiceCapture: () => {},
playBuzzerSequence: () => {},
};
const noopActions: LayerActions = noopLayerActions;
/** Long-press overlay menu; closing it returns focus to the sidebar. */
/**
* Hold a long-press this much past the long-press event (which the firmware
* itself only fires after a shorter hold) and the shell opens its own escape
* menu — the recovery path when an app ignores or mishandles the gesture.
*/
const LONG_PRESS_ESCAPE_MENU_MS = 4000;
/** Shell-surface overlay menu (the escape menu); closing it returns focus to the sidebar. */
class ShellOverlayMenuLayer extends MenuLayer {
constructor(items: MenuItem[], private readonly onClosed: () => void) {
super(null, items, {
@@ -112,6 +111,7 @@ class Shell {
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
actions: noopActions,
@@ -186,9 +186,9 @@ class Shell {
this.config.requestShellRender();
}
/** Close the foreground window via the shell (long-press menu action). */
closeForegroundWindow(): void {
const window = this.foregroundWindow();
/** Close a window by id, if it is closeable (menu actions route here). */
closeWindow(windowId: string): void {
const window = this.windows.find((w) => w.windowId === windowId);
if (!window || !window.closeable) return;
try {
window.close?.();
@@ -198,6 +198,12 @@ class Shell {
this.removeWindow(window.windowId);
}
/** Close the foreground window via the shell (escape menu action). */
closeForegroundWindow(): void {
const window = this.foregroundWindow();
if (window) this.closeWindow(window.windowId);
}
getWindows(): readonly ShellWindow[] {
return this.windows;
}
@@ -253,6 +259,7 @@ class Shell {
/** Turn the screen off, closing any shell overlays. Sidebar selection is kept. */
sleep(): void {
if (!this.screenOn) return;
this.cancelEscapeMenuTimer();
this.screenOn = false;
this.stack.clearToBase();
for (const window of this.windows) {
@@ -350,6 +357,12 @@ class Shell {
this.lastInputAtMs = Date.now();
// Anything but the long-press itself means the press ended (or the event
// stream moved on), so the escape countdown stops.
if (event.type !== "long-press") {
this.cancelEscapeMenuTimer();
}
if (!this.screenOn) {
if (event.type === "double-click") {
this.wake("sidebar");
@@ -358,21 +371,37 @@ class Shell {
return { shell: false, window: false };
}
// Long-press is reserved by the shell: PTT from the sidebar, the overlay
// menu from inside a window. Windows never see it.
// Long-press goes to the foreground window (from the sidebar it focuses
// the window first); apps conventionally answer it with their window
// menu. The escape timer runs regardless of what the app does with it:
// holding the press long enough opens the shell's own menu.
if (event.type === "long-press") {
if (!this.activeVoiceLayer && this.stack.isAtBase()) {
if (this.focus === "sidebar") {
this.openVoiceDialog();
} else {
this.openOverlayMenu();
}
this.startEscapeMenuTimer();
if (this.activeVoiceLayer || !this.stack.isAtBase()) {
return { shell: true, window: false };
}
return { shell: true, window: false };
const window = this.foregroundWindow();
if (!window) {
return { shell: true, window: false };
}
if (this.focus === "sidebar") {
this.focus = "window";
}
// The window owns frameId from here (render or explicit finish).
await window.handleInput(event, frameId);
return { shell: true, window: true };
}
if (event.type === "long-press-release") {
this.activeVoiceLayer?.endCapture();
return { shell: true, window: false };
if (this.activeVoiceLayer || !this.stack.isAtBase() || this.focus !== "window") {
return { shell: true, window: false };
}
const window = this.foregroundWindow();
if (!window) {
return { shell: true, window: false };
}
await window.handleInput(event, frameId);
return { shell: false, window: true };
}
if (!this.stack.isAtBase()) {
@@ -476,24 +505,46 @@ class Shell {
this.foregroundWindow()?.receiveTextInput?.(text);
}
private openOverlayMenu(): void {
/**
* Open the voice dialog aimed at the foreground window. Called when the
* user picks Voice input from a window's menu (in-process directly, worker
* apps via a start-voice-input message). The menu click already ended the
* press, so the dialog finishes on click instead of long-press-release.
*/
startVoiceInput(): void {
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
// The transcript is aimed at the window whose menu requested it.
this.focus = "window";
this.openVoiceDialog(true);
this.config.requestShellRender();
}
private startEscapeMenuTimer(): void {
this.cancelEscapeMenuTimer();
this.escapeMenuTimer = setTimeout(() => {
this.escapeMenuTimer = null;
this.openEscapeMenu();
}, LONG_PRESS_ESCAPE_MENU_MS);
}
private cancelEscapeMenuTimer(): void {
if (this.escapeMenuTimer !== null) {
clearTimeout(this.escapeMenuTimer);
this.escapeMenuTimer = null;
}
}
/**
* The held-long-press safeguard menu. Shell-owned and shell-drawn (never
* the app's), so an unresponsive app can always be closed. It opens over
* whatever the app did with the long-press, including its own menu.
*/
private openEscapeMenu(): void {
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
const foreground = this.foregroundWindow();
if (!foreground?.closeable) return;
const items: MenuItem[] = [
{
label: "Voice input",
onSelect: (ctx) => {
// Pop the menu first (its onRemoved yields focus to the sidebar),
// then take focus back: the transcript is aimed at the window the
// menu was opened over. The button is no longer held here, so the
// dialog finishes on click instead of long-press-release.
ctx.stack.pop();
this.focus = "window";
this.openVoiceDialog(true);
},
},
];
if (foreground?.closeable) {
items.push({
label: "Close window",
onSelect: (ctx) => {
// Pop the menu first (its onRemoved returns focus to the sidebar),
@@ -501,9 +552,10 @@ class Shell {
ctx.stack.pop();
this.closeForegroundWindow();
},
});
}
},
];
this.stack.push(new ShellOverlayMenuLayer(items, () => this.yieldFocusToSidebar()));
this.config.requestShellRender();
}
private chromeState(): ShellChromeState {
+33
View File
@@ -36,6 +36,21 @@ export type WorkerAppReply =
}
| { type: "set-title"; windowId: string; title: string }
| { type: "set-attention"; windowId: string; attention: boolean }
| {
/** Window-menu pick: open the shell's voice dialog aimed at this window. */
type: "start-voice-input";
windowId: string;
}
| {
/** Window-menu pick: close this window (the shell owns the close path). */
type: "close-window-request";
windowId: string;
}
| {
/** Open or focus the Settings app, optionally jumping to a section. */
type: "open-settings";
section?: string;
}
| {
/**
* Set or clear the app's top-bar tray icon. Pixels ride the JSON
@@ -66,6 +81,8 @@ export type WorkerAppHostOptions = {
setSurfaceVisible: (surfaceId: string, visible: boolean) => void;
removeSurface: (surfaceId: string) => void;
requestShellRender: () => void;
/** Open or focus the Settings app, optionally selecting a section. */
openSettings: (section?: string) => void;
};
/**
@@ -107,6 +124,22 @@ export class WorkerAppHost {
case "set-attention":
shell.setWindowAttention(message.windowId, message.attention);
break;
case "start-voice-input":
// Menus only open on the focused window, but re-check foreground:
// the reply crosses a thread boundary and focus may have moved.
if (shell.foregroundWindow()?.windowId === message.windowId) {
shell.startVoiceInput();
}
break;
case "close-window-request":
if (this.openWindows.has(message.windowId)) {
shell.closeWindow(message.windowId);
this.options.requestShellRender();
}
break;
case "open-settings":
this.options.openSettings(message.section);
break;
case "set-tray-icon": {
let icon: GrayImage | null = null;
if (message.icon) {