Files
faceclaw/app/ui/shell/shell.ts
T
James Babcock a02b849200 Borderless modes have different padding inside the builtin apps
Most of the builtin apps had some padding between the window edge and
their content, because they expect to be displayed with a line border on
their edge. We added a customization option to remove the border, so,
that padding is not always necessary. Make the layouts dynamically
adjust, to omit the padding when it isn't needed.
2026-11-11 15:19:29 -08:00

2222 lines
89 KiB
TypeScript

import { encodeShellScene } from "../../graphics/shell-scene";
import { GrayImage } from "../../graphics/image";
import { singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { EvenAIStatus, EventSourceType, OsEventTypeList, WatchGestureType } from "../../g2/events";
import { acceptInput } from "../input-monitor";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import {
directionalFallback,
type DirectionalInputEvent,
GESTURE_SHORT_THEN_LONG_PRESS,
gestureHints,
InputEvent,
type InputEventPayload,
type InputSource,
isDirectionalInput,
isWatchInput,
makeInputEvent,
} from "../gestures";
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { KeyboardInputLayer, type KeyboardInputSession, type KeyboardInputVisibility } from "./keyboard-input";
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
anthropicApiKeySetting,
assistantHermesKeySetting,
assistantHermesUrlSetting,
assistantBackendSetting,
assistantBridgeHostSetting,
assistantBridgePortSetting,
assistantBridgeTokenSetting,
assistantModelSetting,
assistantOpenAiCompatibleKeySetting,
assistantOpenAiCompatibleModelSetting,
assistantOpenAiCompatibleUrlSetting,
assistantSkipConfirmationSetting,
veniceWebSearchSetting,
batteryIndicatorSettingsKey,
brightnessSetting,
newNotificationDisplaySetting,
newNotificationDisplayTimeoutMs,
onAnySettingChanged,
openAiApiKeySetting,
shouldPopUpNewNotification,
statusBarPositionSetting,
statusBarVisibilitySetting,
timeFormatSetting,
wakeWordActionSetting,
wakeToLastAppSetting,
windowBorderSetting,
} from "../dashboard-settings";
import { onAmbientCardsChanged } from "./ambient-cards";
import {
sameStatusItem,
ShellChromeLayer,
sidebarContentSpan,
type ShellChromeState,
type ShellChromeWindow,
type StatusItem,
} from "./chrome-layer";
import { ShellModalLayer } from "./modal-layer";
import { NotificationModalQueue } from "./notification-modal-queue";
import { NotificationPopupTimeout } from "./notification-popup-timeout";
import { ALL_NOTIFICATIONS, readActiveNotifications } from "../../native/notification-icons";
import { ToolDebugMenuLayer } from "./tool-debug-layer";
import type { InProcessWindow } from "./in-process-window";
import { BrightnessPickerLayer } from "./brightness-picker-layer";
import { toolRegistry } from "../../assistant/tool-registry";
import {
appViewportRect,
minWindowTop,
sidebarWidth,
switcherPosition,
switcherRowHeight,
uiDepth,
windowBandHeight,
windowFramed,
windowTop,
type WindowHeightMode,
screenHeight,
screenWidth,
} from "./geometry";
import type { AppSwitcherPosition } from "../dashboard-settings";
import type { Viewport } from "../viewport";
/**
* The shell: owns the window registry, focus, screen on/off, and the shell
* surface (sidebar + top bar + shell overlays such as the escape menu and
* the voice dialog). Runs on the main thread; windows are hosted in-process
* or in per-app worker threads.
*
* Input flow: every event enters via receiveInput. The shell consumes
* everything while the sidebar or a shell overlay has focus and forwards the
* rest to the focused window. Long-press opens the shell-owned system menu
* (Focus app switcher, Voice input, Brightness, Close window, Debug) without reaching the
* app, so the shell keeps working when a window's handler hangs; a window
* that claims long-press for a move of its own gets it forwarded instead, and
* holding the press past the escape threshold still opens the system menu.
* The 2.2.9 tap-then-hold gesture goes to the foreground window (from the
* sidebar it focuses the window first); by convention apps answer it with
* their own context menu, or ask for the system menu when they have none.
*/
export type ShellWindow = {
appId: string;
windowId: string;
title: string;
/** Compositor surface this window renders to; configured at connect / launch. */
surfaceId: string;
/** Whether the system menu offers Close window (the launcher is pinned). */
closeable: boolean;
/**
* True when tap-then-hold currently opens the window's own context menu
* (with at least one entry). While false, the system menu shows no
* app-menu hint and a tap-then-hold over the open system menu leaves it
* open instead of switching menus.
*/
hasAppMenu?: () => boolean;
/**
* True while the window gives long-press a meaning of its own (a game
* move). The shell then forwards long-presses to it instead of opening the
* system menu; holding the press past the escape threshold still opens it.
*/
claimsLongPress?: () => boolean;
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/**
* True while the window shows something the wearer watches without
* touching anything (a playing video): the idle screen timeout waits, as
* it does for voice capture, while this window is in the foreground.
*/
keepsScreenOn?: () => boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
*/
acceptsDirectional?: boolean;
/**
* Window height: the standard 288px band ("min") or the full screen
* ("max", terminal views). Decides the surface rect and where the shell
* draws this window's top bar.
*/
heightMode: WindowHeightMode;
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
close?: () => void;
drawIcon: ShellChromeWindow["drawIcon"];
/**
* Handle an input event the shell forwarded. Ownership of frameId (latency
* tracking) passes to the window: it must eventually reach a frame submit
* or a finishFrame call.
*/
handleInput: (event: InputEvent, frameId: number) => Promise<void> | void;
/** Repaint and resubmit this window's surface. */
requestRender: () => void;
/**
* Re-measure against the current display mode and app switcher (viewport
* size / band) and repaint. Windows without it are closed and relaunched
* by the controller instead on a display mode change, and left at the size
* they opened with when the bottom switcher changes height.
*/
relayout?: () => void;
/**
* The viewport the window is laid out at (it changes on relayout), so the
* controller can tell a window whose size, or whose edges, fell behind the
* geometry from one whose surface merely moved.
*/
layoutSize?: () => Viewport;
/**
* A touch from the phone's mirror at (x, y) in app-viewport coordinates.
* True if the window acted on it; otherwise the controller sends a select.
*/
hitTest?: (x: number, y: number) => Promise<boolean> | boolean;
/**
* Deliver a text string to the window (e.g. finalized voice input). Optional:
* only windows that consume typed text (the terminal) implement it.
*/
receiveTextInput?: (text: string, options?: { submit?: boolean }) => void;
/** Foreground state changed: this window's surface is (not) the visible one. */
setForeground?: (foreground: boolean) => void;
/**
* Input focus moved into this window. `lastInput` is the most recent input
* event the shell received — for a focus that a click or swipe caused, the
* event that caused it. A programmatic focus (a worker's focus-window
* request, a wake path) can deliver an older event: check timestampMs
* before treating it as current.
*/
onFocus?: (lastInput: InputEvent | null) => void;
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
setScreenOn?: (on: boolean) => void;
/**
* Input focus arrived at or left this window: it is (no longer) where
* ordinary input goes. Unlike setForeground / onFocus, this also tracks
* shell overlays (the system menu, a notification modal, the voice dialog)
* and screen-off, so a game can pause on any of them. Sent on change only.
*/
setInputFocus?: (focused: boolean) => void;
};
export type ShellConfig = {
/** Hosts without microphone support hide and reject voice entry points. */
voiceInputEnabled?: boolean;
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
actions: LayerActions;
getScreenTimeoutMs: () => number | null;
requestShellRender: () => void;
/**
* Asked before the voice dialog opens; resolving false swallows the open.
* Preview-only mode uses it to turn the tap into a mic-permission prompt
* when RECORD_AUDIO isn't granted yet (the phone mic is the source there).
*/
prepareVoiceCapture?: () => Promise<boolean>;
/**
* The keyboard dialog opened (with the session the phone types into) or
* closed (null) by any path; the phone UI shows/hides its typing panel.
*/
onKeyboardInputChanged?: (session: KeyboardInputSession | null) => void;
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
onScreenStateChanged: (on: boolean) => void;
/** Window registered/removed or foreground changed (persists the open-app list). */
onWindowsChanged?: () => void;
/**
* The bottom switcher's layout changed its height (geometry.ts
* setSwitcherBarHeight): windows have to move, or resize, to suit. Called
* after the shell paint that found it.
*/
onSwitcherBarHeightChanged?: () => void;
/**
* The Notifications app's window, opened in the background (neither
* foregrounded nor focused) unless it is open already, for the switcher
* to show a notification's detail view in; `opened` says this call opened
* it. Hosts that leave it out offer no notifications to select.
*/
openNotificationsWindow?: () => { window: InProcessWindow; opened: boolean } | null;
};
/**
* What makes an app's tray icon (see Shell.setTrayIcon) a widget the app
* switcher's selection can move onto, as it can onto a notification's icon
* (bottom switcher only; see Shell.moveSelection).
*/
export type TrayWidget = {
/**
* The window to show while the selection is on the widget, opened in the
* background (neither foregrounded nor focused) unless it is open
* already; `opened` says this call opened it, and the shell closes it
* again once the selection moves on. Null when it can't be opened.
*/
openWindow: () => { window: ShellWindow; opened: boolean } | null;
};
/**
* A new-notification popup: the shell modal, and the notification view inside
* it (null for an app notification, which brings its own layer).
*/
type NotificationPopup = { modal: ShellModalLayer; content: SingleNotificationLayer | null };
/**
* The app switcher's selection while it is on a status bar item rather than
* a window's icon (bottom switcher only; see Shell.moveSelection): a
* notification's icon or an app widget. The item's window is the
* foreground window meanwhile: a notification's is the Notifications app's,
* with the notification's detail view on top of its stack; a widget's is
* whatever window it gives (TrayWidget.openWindow).
*/
type StatusSelection = {
item: StatusItem;
/** Where the item sat among the selectable ones, for once it has gone. */
index: number;
host: ShellWindow;
/** A notification's detail view, and the host's stack it is on. */
detail: { stack: LayerStack; layer: SingleNotificationLayer } | null;
/**
* The host window was opened for the selection: it has no switcher icon
* of its own, and closes once the selection moves off it.
*/
transient: boolean;
/** The window selected before, to go back to when the screen sleeps. */
returnWindowId: string | null;
};
/** Which surfaces need re-rendering after an input event. */
export type ShellInputOutcome = { shell: boolean; window: boolean };
/**
* Assistant overlay activity, for mirrors outside the glasses (the watch).
* "streaming" carries the reply so far (replace semantics); "done" the final
* reply; "error" the message; "closed" fires when the overlay leaves the
* stack by any path.
*/
export type AssistantActivityEvent = {
phase: "thinking" | "streaming" | "done" | "error" | "closed";
text: string;
};
export type FocusKind = "sidebar" | "window";
/**
* A notification an app posts itself, rather than one Android posted. It
* opens in the same modal (and waits in the same queue) as Android
* notifications, but is not listed in the Notifications app.
*/
export type AppNotification = {
/** Unique per notification; must not collide with Android notification keys. */
key: string;
/** The modal's content; `close` takes the modal down. */
createLayer: (close: () => void) => Layer;
};
const noopActions: LayerActions = noopLayerActions;
/**
* For a window that claims long-press: hold the press this much past the
* long-press event (which the firmware itself only fires after a shorter
* hold) and the shell opens the system menu anyway — the recovery path when
* the app ignores or mishandles the gesture.
*/
const LONG_PRESS_ESCAPE_MENU_MS = 4000;
/** Settings-store key for the keyboard dialog's last-chosen KeyboardInputVisibility. */
const KEYBOARD_INPUT_VISIBILITY_KEY = "keyboardInput.visibility";
/** Shell-surface overlay menu (the system menu); closing it returns focus to the sidebar. */
class ShellOverlayMenuLayer extends MenuLayer {
/**
* Set before popping to keep focus on the window: an entry that acts on
* the window (Voice input aims its transcript at it) must not first hand
* focus to the sidebar.
*/
keepWindowFocus = false;
constructor(items: MenuItem[], footer: string | undefined, private readonly onClosed: () => void) {
// Aligned to the min-height window band (like the sidebar), wherever the
// vertical position setting currently puts it; centered over the
// application area, i.e. the part of the screen beside a side strip
// (the whole width with a bottom row, or over an immersive window, where
// the strip overlays).
const width = 272;
const area = appViewportRect("min");
super("System", items, {
x: area.x + (((area.width - width) / 2) | 0),
y: area.y + 8,
width,
minHeight: 150,
squareCorners: true,
footer,
dimUnderneath: CONTEXT_MENU_DIM,
depth: 4,
});
}
onRemoved(): void {
if (!this.keepWindowFocus) this.onClosed();
}
}
/** How long a show_alert popup stays before auto-dismissing. */
const ALERT_DISMISS_MS = 6000;
const ALERT_X = 40;
/** Alert width: the screen less ALERT_X on each side. */
function alertW(): number {
return screenWidth() - 2 * ALERT_X;
}
const ALERT_Y = 96;
const ALERT_H = 96;
/**
* A brief text popup on the shell surface (the assistant's show_alert tool and
* other short notices). Auto-dismisses after a few seconds; a click or
* double-click dismisses it early.
*/
class ShellAlertLayer implements Layer {
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(private readonly text: string, private readonly onDismiss: () => void) {
this.timer = setTimeout(() => {
this.timer = null;
this.onDismiss();
}, ALERT_DISMISS_MS);
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const image = paintBelow();
const font = getDefaultSmallFont();
// Positioned within the min-height window band, like the other shell overlays.
const alertY = minWindowTop() + ALERT_Y;
image.fillRect(ALERT_X, alertY, alertW(), ALERT_H, 1);
image.drawRect(ALERT_X, alertY, alertW(), ALERT_H, 90);
image.drawText(font, ALERT_X + 16, alertY + 12, "Assistant", 200);
image.drawTextWrapped({
font,
x: ALERT_X + 16,
y: alertY + 18 + font.lineHeight + 4,
width: alertW() - 32,
text: this.text,
value: 235,
});
return image;
}
handleInput(event: InputEvent, _ctx: LayerContext): void {
if (event.type === "click" || event.type === "double-click") {
this.clearTimer();
this.onDismiss();
}
}
onRemoved(): void {
this.clearTimer();
}
private clearTimer(): void {
if (this.timer !== null) {
clearTimeout(this.timer);
this.timer = null;
}
}
}
/**
* What a watch swipe means while the app switcher has focus, as the ring
* gesture it stands for. Swipes are spatial: along the strip they move the
* selection, toward the app area they enter the selected window, and away
* from it there is nowhere further to go (ignored). A popup's row of icons
* lies over the window, so either way across it enters the window.
*/
const SWITCHER_SWIPES: Record<
AppSwitcherPosition,
Partial<Record<DirectionalInputEvent["type"], "scroll-up" | "scroll-down" | "click">>
> = {
left: { "swipe-up": "scroll-up", "swipe-down": "scroll-down", "swipe-right": "click" },
right: { "swipe-up": "scroll-up", "swipe-down": "scroll-down", "swipe-left": "click" },
bottom: { "swipe-left": "scroll-up", "swipe-right": "scroll-down", "swipe-up": "click" },
popup: { "swipe-left": "scroll-up", "swipe-right": "scroll-down", "swipe-up": "click", "swipe-down": "click" },
};
/**
* Every setting the shell's paint depends on that nothing else repaints it
* for (the top bar's contents and placement, the window frame, the scene's
* stereo depth), as one comparable string.
*/
function chromeSettingsKey(): string {
return `${batteryIndicatorSettingsKey()}|${timeFormatSetting.get()}|${uiDepth()}|${statusBarPositionSetting.get()}`
+ `|${statusBarVisibilitySetting.get()}|${windowBorderSetting.get()}`;
}
class Shell {
private windows: ShellWindow[] = [];
private selectedIndex = 0;
/** Window ids in most-recently-visible-first order; closing the visible window returns to the next entry. */
private mruWindowIds: string[] = [];
private focus: FocusKind = "sidebar";
/** The window last told it holds input focus (see syncInputFocus). */
private inputFocusedWindowId: string | null = null;
private screenOn = true;
private lastInputAtMs = Date.now();
/** The most recent input event received, for windows gaining focus (see ShellWindow.onFocus). */
private lastInput: InputEvent | null = null;
private battery: ShellChromeState["battery"] = {
headset: null, headsetCharging: null, ring: null, ringCharging: null, watch: null, watchCharging: null,
case: null, caseCharging: null,
};
private readonly batteryListeners = new Set<() => void>();
private attention = new Map<string, boolean>();
/** Set while the switcher's selection is on a status bar item (see StatusSelection). */
private selectedStatus: StatusSelection | null = null;
// App-provided top-bar tray icons, keyed by owner id; drawn between the
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, { icon: GrayImage; widget?: TrayWidget }>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
// Messages sent from keyboard dialogs, newest first; in memory only.
private readonly keyboardInputHistory: string[] = [];
// App-posted notifications that are open or queued, by key.
private readonly appNotifications = new Map<string, AppNotification>();
private readonly notificationModals = new NotificationModalQueue<NotificationPopup>({
open: (notificationKey) => this.pushNotificationModal(notificationKey),
isAvailable: (notificationKey) => {
if (this.appNotifications.has(notificationKey)) return true;
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
return !!notification && shouldPopUpNewNotification(notification.packageName);
},
sleep: () => this.sleep(),
});
private conversations: AssistantConversations | null = null;
private get assistantSession(): AssistantSession | null {
return this.conversations?.current().session ?? null;
}
getAssistantConversations(): AssistantConversations {
if (!this.conversations) {
this.conversations = new AssistantConversations(
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
() => assistantModelSetting.get(),
(value) => setStringSetting("assistant.conversations", value),
getStringSetting("assistant.conversations", ""),
);
}
return this.conversations;
}
async prepareChatVoiceCapture(): Promise<boolean> {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
}
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>();
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
actions: noopActions,
getScreenTimeoutMs: () => null,
requestShellRender: () => {},
onScreenStateChanged: () => {},
};
private readonly chrome = new ShellChromeLayer(() => this.chromeState(), () => this.switcherBarHeightChanged());
private readonly stack = new LayerStack(this.chrome, this.actions);
private switcherBarChangePending = false;
// Top-bar settings we mirror into the chrome; a change to any of them
// repaints the shell surface so the top bar reflects it immediately.
private chromeSettingsSubscribed = false;
private lastChromeSettingsKey: string | null = null;
configure(config: ShellConfig): void {
// Nearly every state change ends in a shell render request, so it is
// the natural point to tell windows about input-focus changes too.
this.config = {
...config,
requestShellRender: () => {
this.syncInputFocus();
config.requestShellRender();
},
};
this.stack.setActions(config.actions);
this.subscribeToChromeSettings();
this.subscribeToAmbientCards();
}
// Ambient (encounter) cards live in the chrome paint; a posted or expired
// card repaints the shell surface so it appears and disappears on time.
private ambientCardsSubscribed = false;
private subscribeToAmbientCards(): void {
if (this.ambientCardsSubscribed) return;
this.ambientCardsSubscribed = true;
onAmbientCardsChanged(() => {
if (this.screenOn) {
this.config.requestShellRender();
}
});
}
private subscribeToChromeSettings(): void {
if (this.chromeSettingsSubscribed) return;
this.chromeSettingsSubscribed = true;
this.lastChromeSettingsKey = chromeSettingsKey();
onAnySettingChanged(() => {
const key = chromeSettingsKey();
if (key === this.lastChromeSettingsKey) return;
this.lastChromeSettingsKey = key;
this.config.requestShellRender();
});
}
/**
* The chrome found, mid-paint, that the bottom switcher's height changed.
* The controller moves or resizes windows once the paint is over: a
* window's relayout repaints it, which mustn't happen inside the shell's
* own paint.
*/
private switcherBarHeightChanged(): void {
if (this.switcherBarChangePending) return;
this.switcherBarChangePending = true;
setTimeout(() => {
this.switcherBarChangePending = false;
this.config.onSwitcherBarHeightChanged?.();
}, 0);
}
/** Add a window (or replace one with the same windowId, keeping its slot). */
registerWindow(window: ShellWindow): void {
const existing = this.windows.findIndex((w) => w.windowId === window.windowId);
if (existing >= 0) {
this.windows[existing] = window;
if (existing === this.selectedIndex) this.noteWindowVisible(window.windowId);
} else {
this.windows.push(window);
if (this.windows.length - 1 === this.selectedIndex) this.noteWindowVisible(window.windowId);
}
this.config.onWindowsChanged?.();
}
/** Move a window to the front of the most-recently-visible order. */
private noteWindowVisible(windowId: string): void {
this.mruWindowIds = this.mruWindowIds.filter((id) => id !== windowId);
this.mruWindowIds.unshift(windowId);
}
/** Index of the most recently visible window still in the registry, or -1. */
private mostRecentWindowIndex(): number {
for (const id of this.mruWindowIds) {
const index = this.windows.findIndex((w) => w.windowId === id);
if (index >= 0) return index;
}
return -1;
}
removeWindow(windowId: string): void {
const index = this.windows.findIndex((w) => w.windowId === windowId);
if (index < 0) return;
// Closing the window under a selected status bar item (from the system
// menu) takes the selection with it.
if (this.selectedStatus?.host.windowId === windowId) this.selectedStatus = null;
const wasSelected = index === this.selectedIndex;
this.windows.splice(index, 1);
this.attention.delete(windowId);
this.mruWindowIds = this.mruWindowIds.filter((id) => id !== windowId);
if (wasSelected) {
// Return to the most recently visible remaining window.
const mruIndex = this.mostRecentWindowIndex();
this.selectedIndex =
mruIndex >= 0 ? mruIndex : Math.min(index, Math.max(0, this.windows.length - 1));
} else if (this.selectedIndex > index) {
this.selectedIndex--;
}
if (this.focus === "window" && (wasSelected || !this.windows.length)) {
this.focus = "sidebar";
}
if (wasSelected) {
// Hand the foreground to whatever is now selected.
const next = this.windows[this.selectedIndex];
if (next) {
this.noteWindowVisible(next.windowId);
next.setForeground?.(true);
next.requestRender();
}
}
this.config.onWindowsChanged?.();
this.config.requestShellRender();
}
/** Close a window by id, if it is closeable (menu actions route here). */
closeWindow(windowId: string): void {
const window = this.windows.find((w) => w.windowId === windowId);
if (!window || !window.closeable) return;
try {
window.close?.();
} catch (error) {
console.warn(`window ${window.windowId} close failed`, error);
}
this.removeWindow(window.windowId);
}
/** Close the foreground window via the shell (escape menu action). */
closeForegroundWindow(): void {
const window = this.foregroundWindow();
if (window) this.closeWindow(window.windowId);
}
getWindows(): readonly ShellWindow[] {
return this.windows;
}
setWindowAttention(windowId: string, attention: boolean): void {
if (Boolean(this.attention.get(windowId)) === attention) return;
this.attention.set(windowId, attention);
this.config.requestShellRender();
}
setBatteryLevels(levels: Partial<ShellChromeState["battery"]>): void {
const next = { ...this.battery, ...levels };
const keys = Object.keys(next) as (keyof ShellChromeState["battery"])[];
if (keys.every((key) => next[key] === this.battery[key])) return;
this.battery = next;
for (const listener of Array.from(this.batteryListeners)) listener();
}
/** Called after any battery level or charging state changes. Returns an unsubscribe function. */
onBatteryLevelsChanged(listener: () => void): () => void {
this.batteryListeners.add(listener);
return () => {
this.batteryListeners.delete(listener);
};
}
/**
* Set or clear an app's top-bar tray icon (a small grayscale image, drawn
* between the notification icons and the battery indicators). Small and
* infrequently updated by design; not a framebuffer. With `widget`, the
* switcher's selection can move onto it. A widget that goes, or stops
* being one, while the selection is on it leaves the selection on its
* window, as an ordinary window.
*/
setTrayIcon(ownerId: string, icon: GrayImage | null, widget?: TrayWidget): void {
if (icon) {
this.trayIcons.set(ownerId, { icon, widget });
} else if (!this.trayIcons.delete(ownerId)) {
return;
}
if ((!icon || !widget) && sameStatusItem(this.selectedStatus?.item, { kind: "widget", key: ownerId })) {
this.endStatusSelection(true);
}
this.config.requestShellRender();
}
isScreenOn(): boolean {
return this.screenOn;
}
/** Current G2, R1, and Wear OS watch battery levels (top bar, Glanceboard, assistant). */
getBatteryLevels(): ShellChromeState["battery"] {
return this.battery;
}
/**
* The foreground app, or null when only the launcher is showing. The launcher
* is a pinned window but counts as "no app" for the assistant's context.
*/
getForegroundApp(): { appId: string; title: string } | null {
const window = this.foregroundWindow();
if (!window || window.appId === "launcher") return null;
return { appId: window.appId, title: window.title };
}
noteUserActivity(nowMs = Date.now()): void {
this.lastInputAtMs = nowMs;
}
/** Where input currently goes: the sidebar strip or the foreground window. */
getFocus(): FocusKind {
return this.focus;
}
/** True while a shell overlay (menu, dialog, voice input) is above the chrome. */
hasOverlay(): boolean {
return !this.stack.isAtBase();
}
/** The window whose sidebar icon is at screen (x, y), for mirror touches. */
windowAtSidebarPoint(x: number, y: number): ShellWindow | null {
const windows = this.switcherWindows();
const index = this.chrome.windowIndexAt(x, y, windows.length);
return index === null ? null : windows[index] ?? null;
}
/**
* A touch from the phone's mirror on a selectable status bar item's icon
* (a notification's or an app widget's): select it and go into its
* window, as a touch on a window's icon focuses that window. False when
* there is no such icon at (x, y).
*/
focusStatusItemAt(x: number, y: number): boolean {
const items = this.selectableStatusItems();
const touched = this.chrome.statusItemAt(x, y);
const index = items.findIndex((item) => sameStatusItem(item, touched));
if (index < 0) return false;
this.selectStatusItem(items[index]!, index);
const selected = this.selectedStatus;
const host = selected && sameStatusItem(selected.item, touched) ? selected.host : null;
if (!host) return false;
const alreadyFocused = this.isFocusTarget(host);
this.focus = "window";
if (!alreadyFocused) host.onFocus?.(this.lastInput);
host.requestRender();
this.syncInputFocus();
return true;
}
/** Whether input focus currently targets this window (regardless of screen state). */
private isFocusTarget(window: ShellWindow | undefined): boolean {
return !!window && this.focus === "window" && this.foregroundWindow() === window;
}
/**
* The window ordinary input reaches right now: the foreground window with
* focus in-window, the screen on, and no shell overlay (system menu,
* notification modal, voice dialog, alert) above it.
*/
private inputTargetWindow(): ShellWindow | undefined {
if (!this.screenOn || this.focus !== "window" || !this.stack.isAtBase() || this.activeVoiceLayer) {
return undefined;
}
return this.foregroundWindow();
}
/**
* Tell windows when they gain or lose input focus (ShellWindow.setInputFocus).
* Called from every shell render request and the input/wake/sleep paths,
* which between them follow every focus-affecting state change; it diffs
* against the last notification, so calling it often is cheap.
*/
private syncInputFocus(): void {
const target = this.inputTargetWindow();
const targetId = target?.windowId ?? null;
if (targetId === this.inputFocusedWindowId) return;
const previous = this.windows.find((w) => w.windowId === this.inputFocusedWindowId);
this.inputFocusedWindowId = targetId;
previous?.setInputFocus?.(false);
target?.setInputFocus?.(true);
}
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
wake(focus: FocusKind, nowMs = Date.now()): boolean {
this.lastInputAtMs = nowMs;
const gaining = focus === "window" ? this.foregroundWindow() : undefined;
const alreadyFocused = this.isFocusTarget(gaining);
this.focus = focus;
if (gaining && !alreadyFocused) gaining.onFocus?.(this.lastInput);
if (this.screenOn) {
this.syncInputFocus();
return false;
}
this.screenOn = true;
this.config.onScreenStateChanged(true);
for (const window of this.windows) {
window.setScreenOn?.(true);
}
// Refresh the foreground window; the compositor restored its retained
// frame, but its content may be stale (e.g. a running stopwatch).
this.foregroundWindow()?.requestRender();
this.syncInputFocus();
return true;
}
/**
* Turn the screen off, closing any shell overlays. Sidebar selection is
* kept, except that one on a status bar item goes back to its window.
*/
sleep(): void {
if (!this.screenOn) return;
this.returnFromStatusItem();
this.cancelEscapeMenuTimer();
this.screenOn = false;
this.stack.clearToBase();
for (const window of this.windows) {
window.setScreenOn?.(false);
}
this.syncInputFocus();
this.config.onScreenStateChanged(false);
}
/** Foreground and focus a window by id (e.g. a wake path opening content in it). */
focusWindow(windowId: string): void {
const index = this.windows.findIndex((w) => w.windowId === windowId);
if (index < 0) return;
const target = this.windows[index];
const alreadyFocused = this.isFocusTarget(target);
this.setSelectedIndex(index);
// Asked for by id, a status bar item's window stays, whether or not it
// was opened for the selection.
this.endStatusSelection(target === this.selectedStatus?.host);
this.focus = "window";
if (!alreadyFocused) target?.onFocus?.(this.lastInput);
this.syncInputFocus();
}
/** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */
applyScreenTimeout(nowMs = Date.now()): boolean {
const timeoutMs = this.config.getScreenTimeoutMs();
if (timeoutMs === null || !this.screenOn) return false;
// An open voice dialog suspends the timeout: a long dictation or refine
// has no button presses, but the screen must stay on for it. Sliding
// lastInputAtMs forward also restarts the full timeout when it closes.
// The keyboard dialog likewise: typing happens on the phone, not the
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
// A foreground window that is playing video holds it off the same way.
const foreground = this.foregroundWindow();
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
this.lastInputAtMs = nowMs;
return false;
}
if (nowMs - this.lastInputAtMs < timeoutMs) return false;
this.sleep();
return true;
}
/**
* Show a new notification in a shell modal over the app viewport, or queue
* it behind the one already open (see NotificationModalQueue). If
* notifications woke the screen, closing the last modal goes back to sleep
* (matching the old sleep-popup behavior).
*/
openNotificationModal(notificationKey: string, wokeScreen: boolean): void {
if (!this.screenOn) return;
this.notificationModals.post(notificationKey, wokeScreen);
this.config.requestShellRender();
}
/**
* A notification went away on the phone: cancelled by its source (e.g. a
* placeholder replaced moments later by the real notification under another
* key), or dismissed. A popup queued for it is dropped, and one showing it
* closes so the next one opens, as if the user had closed it -- unless it
* is asking to confirm turning off its source, which outlives the
* notification it dismissed.
*/
notificationRemoved(notificationKey: string): void {
const popup = this.notificationModals.withdrawn(notificationKey);
if (popup && !popup.content?.outlivesNotification()) {
this.closeNotificationModal(popup, notificationKey);
}
}
/**
* Show an app's own notification the way new Android notifications are
* shown: in the notification modal, waking the screen for it if it was off.
*/
postAppNotification(notification: AppNotification): void {
this.appNotifications.set(notification.key, notification);
const wokeScreen = this.screenOn ? false : this.wake("sidebar");
this.openNotificationModal(notification.key, wokeScreen);
}
private pushNotificationModal(notificationKey: string): NotificationPopup {
const appNotification = this.appNotifications.get(notificationKey);
const close = () => this.closeNotificationModal(popup, notificationKey);
const content = appNotification
? null
: new SingleNotificationLayer(notificationKey, { origin: "new-notification-modal", onClose: close });
// A phone notification's popup closes once left untouched for the
// "Display new notifications" time, as if closed by hand: the notification
// stays on the phone and in the status bar. An app's own notification
// (e.g. a Nightscout alert) is nowhere else to be found, so it stays up.
const timeout = new NotificationPopupTimeout(
appNotification ? null : newNotificationDisplayTimeoutMs(newNotificationDisplaySetting.get()),
() => {
// Something over the popup (a voice dialog, the system menu) is in
// use; count that as interaction rather than closing underneath it.
if (this.stack.topMatches((layer) => layer === popup.modal)) close();
else timeout.restart();
},
);
const popup: NotificationPopup = {
content,
modal: new ShellModalLayer(
appNotification ? appNotification.createLayer(close) : content!,
this.config.actions,
{
removed: () => {
timeout.stop();
this.appNotifications.delete(notificationKey);
this.notificationModals.removed(popup, this.screenOn);
},
input: () => timeout.restart(),
},
),
};
this.stack.push(popup.modal);
timeout.restart();
return popup;
}
private closeNotificationModal(popup: NotificationPopup, notificationKey: string): void {
this.appNotifications.delete(notificationKey);
// The queue first: it opens the next notification or puts the screen back
// to sleep, and then ignores the removal below. remove() rather than a
// pop, so a menu opened over the modal doesn't leave it stuck.
this.notificationModals.closed(popup);
this.stack.remove(popup.modal);
this.config.requestShellRender();
}
/** Called by a window when the user backs out of its root (double-tap). */
yieldFocusToSidebar(): void {
if (this.focus === "sidebar") return;
this.focus = "sidebar";
// Repaint the window so its selection highlight dims to the unfocused
// style this frame.
this.foregroundWindow()?.requestRender();
this.config.requestShellRender();
}
/**
* Paint the shell surface: transparent chrome, or all-transparent when
* asleep. The scene also carries the whole display's stereo depth (none
* under a Full Screen or immersive window).
*/
paintScene(): Uint8Array {
return this.screenOn
? encodeShellScene(this.stack.paintUndimmed(), uiDepth(this.foregroundWindow()?.appId))
: encodeShellScene([]);
}
paintSurface(): Plane[] {
if (!this.screenOn) {
return singlePlane(new GrayImage(screenWidth(), screenHeight(), 0));
}
return this.stack.paint();
}
/**
* The brightness factor a shell overlay (a context menu) currently applies
* to what lies beneath the shell surface, i.e. the window surfaces: 1 when
* nothing dims them. Read after paintSurface; the controller forwards it to
* the compositor, which cannot see the shell's layer stack.
*/
underlayDim(): number {
const dim = this.stack.baseDim();
return this.screenOn && dim !== false ? dim : 1;
}
async receiveInput(event: InputEvent, frameId = 0): Promise<ShellInputOutcome> {
if (!acceptInput(event)) return { shell: false, window: false };
try {
return await this.routeInput(event, frameId);
} finally {
// Whatever the input did (opened an overlay, moved focus, slept the
// screen), windows learn about the resulting input-focus change.
this.syncInputFocus();
}
}
private async routeInput(event: InputEvent, frameId: number): Promise<ShellInputOutcome> {
// Touch-down supplements gestures. It must not wake the screen, operate
// menus, or cancel the hold-to-escape timer. Apps can opt into it later.
if (event.type === "ring-press") {
const window = this.foregroundWindow();
if (this.screenOn && this.focus === "window" && this.stack.isAtBase() && window) {
await window.handleInput(event, frameId);
return { shell: false, window: true };
}
return { shell: false, window: false };
}
const previous = this.lastInput;
this.lastInput = event;
// A visible window may paint a source-dependent indicator (see
// lastInputEvent); when the source class flips (watch <-> ring/arms),
// repaint it so the indicator follows the device now in use.
if (this.screenOn && previous && isWatchInput(previous) !== isWatchInput(event)) {
this.foregroundWindow()?.requestRender();
}
// The stock lifecycle has already interpreted the physical double tap as
// "wake". Keep that directionality if delivery is delayed or duplicated.
if (event.type === "display-wake") {
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake(this.userWakeFocus());
return { shell: wokeScreen, window: false };
}
// The wakeword is handled before the screen-off short-circuit so its
// configured action can work from a dark screen. With the CFW the stock
// Even AI app never launches, so the firmware does not power the display
// for us either -- actions that need it wake the screen themselves.
if (event.type === "wakeword") {
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
const action = wakeWordActionSetting.get();
if (action === "off") {
return { shell: false, window: false };
}
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake("sidebar");
if (action === "voice-input" && !this.activeVoiceLayer && !this.activeKeyboardLayer) {
if (this.assistantLayer) {
// The assistant overlay is up; a wakeword continues that conversation.
this.startAssistantFollowUp(true);
} else {
// Wakeword defaults the highlight to Send to Assistant.
this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" });
}
}
return { shell: wokeScreen || action === "voice-input", window: false };
}
this.lastInputAtMs = Date.now();
// Anything but the long-press itself means the press ended (or the event
// stream moved on), so the escape countdown stops.
if (event.type !== "long-press") {
this.cancelEscapeMenuTimer();
}
if (!this.screenOn) {
if (event.type === "double-click") {
this.wake(this.userWakeFocus());
return { shell: true, window: false };
}
return { shell: false, window: false };
}
// Long-press is the shell's own gesture: it opens the system menu
// directly, never reaching the app — over the app's own context menu
// too (the window closes that on system-menu-opened), while an already
// open system menu just stays. Its later generic release is consumed
// while the shell overlay is active and therefore cannot leak into the
// app. The exception is a window that claims long-press for a move of
// its own: it gets the press forwarded, with the escape timer running
// so that holding the press long enough still opens the system menu.
if (event.type === "long-press") {
if (this.activeVoiceLayer || !this.stack.isAtBase()) {
return { shell: true, window: false };
}
const window = this.foregroundWindow();
if (!window) {
return { shell: true, window: false };
}
if (!window.holdToTalk && !window.claimsLongPress?.()) {
this.openEscapeMenu();
return { shell: true, window: false };
}
if (!window.holdToTalk) this.startEscapeMenuTimer();
if (this.focus === "sidebar") {
this.focus = "window";
window.onFocus?.(this.lastInput);
}
// The window owns frameId from here (render or explicit finish).
await window.handleInput(event, frameId);
return { shell: true, window: true };
}
// The 2.2.9 tap-then-hold gesture is the app's context-menu gesture: it
// goes to the foreground window (from the sidebar it focuses the window
// first), which answers with its own menu or asks for the system menu
// when it has none.
if (event.type === "short-then-long-press") {
const window = this.foregroundWindow();
if (window?.holdToTalk) {
this.openEscapeMenu();
return { shell: true, window: false };
}
// Over the open system menu it switches to the app's context menu:
// close the system menu and deliver the gesture to the window as
// usual. A window without a menu of its own would only ask for the
// system menu back, so for it the open menu stays.
if (
!this.activeVoiceLayer &&
window?.hasAppMenu?.() &&
this.stack.popIfTop((layer) => layer instanceof ShellOverlayMenuLayer)
) {
this.config.requestShellRender();
}
if (this.activeVoiceLayer || !this.stack.isAtBase()) {
return { shell: true, window: false };
}
if (!window) {
return { shell: true, window: false };
}
if (this.focus === "sidebar") {
this.focus = "window";
window.onFocus?.(this.lastInput);
}
// The window owns frameId from here (render or explicit finish).
await window.handleInput(event, frameId);
return { shell: true, window: true };
}
if (event.type === "long-press-release") {
this.activeVoiceLayer?.endCapture();
if (this.activeVoiceLayer || !this.stack.isAtBase() || this.focus !== "window") {
return { shell: true, window: false };
}
const window = this.foregroundWindow();
if (!window) {
return { shell: true, window: false };
}
await window.handleInput(event, frameId);
return { shell: false, window: true };
}
if (!this.stack.isAtBase()) {
await this.stack.handleInput(event);
return { shell: true, window: false };
}
if (this.focus === "sidebar") {
return this.handleSidebarInput(event);
}
const window = this.foregroundWindow();
if (window) {
// The window owns frameId from here (render or explicit finish).
const delivered = isDirectionalInput(event) && !window.acceptsDirectional ? directionalFallback(event) : event;
await window.handleInput(delivered, frameId);
return { shell: false, window: true };
}
return { shell: false, window: false };
}
/** Where a user's double-tap wake lands: the last app, or the switcher (setting). */
userWakeFocus(): FocusKind {
return wakeToLastAppSetting.get() && this.foregroundWindow() ? "window" : "sidebar";
}
foregroundWindow(): ShellWindow | undefined {
return this.windows[this.selectedIndex] ?? this.windows[0];
}
/**
* Screen rect actually occupied by content, for cropping screenshots: the
* foreground window's band (full height for a max-height window, the
* vertical-position-dependent 288px band otherwise) plus the switcher: a
* side strip's icon columns (minus the outer dead strip the one-column
* variant leaves), or the row under it, and the window frame's sides. A
* popup switcher lies within the band.
*/
screenshotCropRect(): { x: number; y: number; width: number; height: number } {
const appId = this.foregroundWindow()?.appId;
const heightMode = this.foregroundWindow()?.heightMode ?? "min";
const viewport = appViewportRect(heightMode, appId);
let left = viewport.x, right = viewport.x + viewport.width;
if (sidebarWidth(appId) > 0) {
const strip = sidebarContentSpan(this.windows.length);
left = Math.min(left, strip.left);
right = Math.max(right, strip.right);
}
if (windowFramed(appId)) {
left -= 1;
right += 1;
}
const y = windowTop(heightMode, appId);
const bottom = y + windowBandHeight(heightMode, appId) + switcherRowHeight(appId);
return { x: left, y, width: right - left, height: bottom - y };
}
/**
* Compact description of where input is currently going, for the frame
* timing export: "which app was drawing" is usually the first thing you need
* to interpret an input-to-display latency, and it is not recoverable from
* the input event itself.
*/
describeInputTarget(): string {
const foreground = this.foregroundWindow()?.windowId ?? "none";
if (!this.screenOn) return `fg=${foreground} target=screen-off`;
if (this.activeVoiceLayer) return `fg=${foreground} target=voice`;
if (this.activeKeyboardLayer?.shownOnGlasses) return `fg=${foreground} target=keyboard`;
if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`;
return `fg=${foreground} target=${this.focus}`;
}
/**
* The most recent input event received, from any source. Windows whose
* appearance depends on the input device in use (e.g. the launcher's
* defocused selection preview) can read it at paint time; the shell
* repaints the foreground window whenever the source class changes, so
* such paints stay current even while the window is not focused.
*/
lastInputEvent(): InputEvent | null {
return this.lastInput;
}
/** Whether the most recent input came from the watch (see lastInputEvent). */
lastInputWasWatch(): boolean {
return this.lastInput !== null && isWatchInput(this.lastInput);
}
/** Whether a window is the current input target (foreground + focus in-window). */
isWindowFocused(windowId: string): boolean {
return this.screenOn && this.focus === "window" && this.foregroundWindow()?.windowId === windowId;
}
/**
* Whether a window's content is on screen: it's the foreground window and the
* screen is on. Focus-independent — the app viewport stays visible while the
* sidebar is focused (the sidebar is just the left strip).
*/
isWindowVisible(windowId: string): boolean {
return this.screenOn && this.foregroundWindow()?.windowId === windowId;
}
private handleSidebarInput(event: InputEvent): ShellInputOutcome {
const type = isDirectionalInput(event) ? SWITCHER_SWIPES[switcherPosition()][event.type] : event.type;
switch (type) {
case "double-click":
// A double-tap at the root (the app switcher selected) turns the
// display off — from the ring and the watch scheme alike; watch-scheme
// gestures wake it again (or a ring double-tap does).
this.sleep();
return { shell: true, window: false };
case "scroll-up":
this.moveSelection(-1);
return { shell: true, window: false };
case "scroll-down":
this.moveSelection(1);
return { shell: true, window: false };
case "click":
if (this.windows.length) {
this.focus = "window";
this.foregroundWindow()?.onFocus?.(this.lastInput);
// Repaint the window now so its selection highlight reflects focus
// this frame, not one frame late.
this.foregroundWindow()?.requestRender();
}
return { shell: true, window: false };
default:
return { shell: false, window: false };
}
}
/**
* Step the switcher's selection through the windows and then, with a
* bottom switcher, on through the status bar's selectable items whose
* icons are showing (notifications, then app widgets), wrapping round at
* either end.
*/
private moveSelection(delta: number): void {
const windows = this.switcherWindows();
const items = this.selectableStatusItems();
const count = windows.length + items.length;
if (!count) return;
let current = this.selectedIndex;
const selected = this.selectedStatus;
if (selected) {
// Once its item has gone, the selection stands between the
// neighbours it left.
const at = items.findIndex((item) => sameStatusItem(item, selected.item));
current = windows.length + (at >= 0 ? at : Math.min(selected.index, items.length) - (delta > 0 ? 1 : 0));
}
const next = (((current + delta) % count) + count) % count;
if (next < windows.length) {
this.selectSwitcherWindow(windows[next]!);
} else {
this.selectStatusItem(items[next - windows.length]!, next - windows.length);
}
}
/**
* Windows with an icon in the switcher: all of them, except a window
* opened just to show a selected status bar item.
*/
private switcherWindows(): ShellWindow[] {
const hidden = this.selectedStatus?.transient ? this.selectedStatus.host : null;
return hidden ? this.windows.filter((window) => window !== hidden) : this.windows;
}
/**
* The status bar items the switcher's selection can move onto, in order:
* with a bottom switcher (the selection reaches them past the last
* window), those whose icons are showing that have a window to show,
* which notifications have where the host gives the Notifications window,
* and app widgets always do.
*/
private selectableStatusItems(): StatusItem[] {
if (switcherPosition() !== "bottom") return [];
return this.chrome.statusItems().filter((item) => item.kind === "notification"
? !!this.config.openNotificationsWindow
: !!this.trayIcons.get(item.key)?.widget);
}
/** Move the switcher's selection onto a window, off a status bar item if it was on one. */
private selectSwitcherWindow(window: ShellWindow): void {
// Foreground first, so a status bar item's window closing with its
// selection can't hand the foreground elsewhere first.
this.setSelectedIndex(this.windows.indexOf(window));
this.endStatusSelection(window === this.selectedStatus?.host);
}
/**
* Move the switcher's selection onto a status bar item: its window
* (opened in the background first, if need be) comes to the foreground,
* a notification's showing its detail view. The window the selection was
* on before is left as endStatusSelection leaves it, unless it is this
* one too.
*/
private selectStatusItem(item: StatusItem, index: number): void {
const previous = this.selectedStatus;
if (previous && sameStatusItem(previous.item, item)) {
previous.index = index;
return;
}
const returnWindowId = previous ? previous.returnWindowId : (this.foregroundWindow()?.windowId ?? null);
const opened = this.openStatusItemWindow(item);
const hostIndex = opened ? this.windows.indexOf(opened.window) : -1;
if (!opened || hostIndex < 0) return;
const host = opened.window;
const sameHost = previous?.host === host;
if (sameHost && previous.detail) previous.detail.stack.popThrough(previous.detail.layer);
const detail = item.kind === "notification" && opened.stack
? { stack: opened.stack, layer: this.notificationDetailLayer(item.key) } : null;
if (detail) detail.stack.push(detail.layer);
this.selectedStatus = {
item, index, host, detail, returnWindowId,
transient: sameHost ? previous.transient : opened.opened,
};
if (hostIndex === this.selectedIndex) {
host.requestRender();
} else {
this.setSelectedIndex(hostIndex);
}
if (previous && !sameHost) this.releaseStatusWindow(previous, false);
}
/**
* The window a status bar item shows in, opened in the background unless
* it is open already (`opened` says this call opened it), and for a
* notification, the stack its detail view goes on.
*/
private openStatusItemWindow(item: StatusItem): { window: ShellWindow; opened: boolean; stack?: LayerStack } | null {
if (item.kind === "widget") return this.trayIcons.get(item.key)?.widget?.openWindow() ?? null;
const opened = this.config.openNotificationsWindow?.();
return opened ? { window: opened.window.window, opened: opened.opened, stack: opened.window.stack } : null;
}
/**
* Take the switcher's selection off its status bar item, leaving its
* window as releaseStatusWindow does.
*/
private endStatusSelection(keepWindow = false): void {
const selected = this.selectedStatus;
if (!selected) return;
this.selectedStatus = null;
this.releaseStatusWindow(selected, keepWindow);
}
/**
* A status bar item's window, once the selection has left the item: it
* drops a notification's detail view (and anything opened over it), and
* closes if it was opened for the selection, unless `keepWindow` (it is
* the window being selected instead).
*/
private releaseStatusWindow(selection: StatusSelection, keepWindow: boolean): void {
if (selection.detail) selection.detail.stack.popThrough(selection.detail.layer);
if (selection.transient && !keepWindow) {
this.closeWindow(selection.host.windowId);
return;
}
selection.host.requestRender();
// A window opened for the selection is now one with an icon of its own.
if (selection.transient) this.config.onWindowsChanged?.();
}
/** End a status bar item selection by going back to the window selected before it. */
private returnFromStatusItem(): void {
const selected = this.selectedStatus;
if (!selected) return;
const back = this.windows.find((window) => window.windowId === selected.returnWindowId);
if (back) this.setSelectedIndex(this.windows.indexOf(back));
this.endStatusSelection(back === selected.host);
}
private notificationDetailLayer(key: string): SingleNotificationLayer {
const layer: SingleNotificationLayer = new SingleNotificationLayer(key, {
origin: "notification-tray",
onClose: (_ctx, gone) => this.notificationDetailClosed(layer, gone),
});
return layer;
}
/**
* A selected notification's detail view asked to close. Back (or a
* double-tap) hands focus back to the switcher, its selection still on
* the notification. Once the notification is gone (dismissed, or removed
* by an action), the selection also moves on to the status bar item that
* took its place, or back to the windows when there are none left; that
* waits for the current paint to finish, since a vanished notification
* closes its view from inside its paint.
*/
private notificationDetailClosed(layer: SingleNotificationLayer, gone: boolean): void {
if (!gone) {
if (this.selectedStatus?.detail?.layer === layer) this.yieldFocusToSidebar();
return;
}
setTimeout(() => {
const selected = this.selectedStatus;
if (selected?.detail?.layer !== layer) return;
this.focus = "sidebar";
const items = this.selectableStatusItems().filter((item) => !sameStatusItem(item, selected.item));
const windows = this.switcherWindows();
if (items.length) {
const index = Math.min(selected.index, items.length - 1);
this.selectStatusItem(items[index]!, index);
} else if (windows.length) {
this.selectSwitcherWindow(windows[windows.length - 1]!);
}
this.config.requestShellRender();
}, 0);
}
/** Change selection; the selected window is the foreground window. */
private setSelectedIndex(index: number): void {
if (index === this.selectedIndex) return;
const previous = this.windows[this.selectedIndex];
this.selectedIndex = index;
const next = this.windows[index];
if (next) this.noteWindowVisible(next.windowId);
previous?.setForeground?.(false);
next?.setForeground?.(true);
next?.requestRender();
this.config.onWindowsChanged?.();
}
// True while a voice-dialog open waits on prepareVoiceCapture (e.g. the
// preview-mode permission prompt); a second tap must not queue another open.
private voiceDialogPending = false;
private openVoiceDialog(options: {
finishOnClick?: boolean;
handsFree?: boolean;
defaultTarget: "assistant" | "app";
}): void {
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
this.voiceDialogPending = true;
void (async () => {
let ready = true;
try {
ready = (await this.config.prepareVoiceCapture?.()) ?? true;
} catch {
ready = false;
} finally {
this.voiceDialogPending = false;
}
// Re-checked after the await: another path may have opened a dialog
// (or torn down the base state) while a permission prompt was up.
if (!ready || this.activeVoiceLayer || this.activeKeyboardLayer) return;
this.openVoiceDialogNow(options);
this.config.requestShellRender();
})();
}
private openVoiceDialogNow(options: {
finishOnClick?: boolean;
handsFree?: boolean;
defaultTarget: "assistant" | "app";
}): void {
const targets = this.buildVoiceSendTargets();
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
if (defaultIndex < 0) defaultIndex = 0;
// Skip the menu only for a hands-free (wakeword) capture aimed at the
// assistant, when the user has opted into it.
const autoSend =
Boolean(options.handsFree) &&
options.defaultTarget === "assistant" &&
targets[defaultIndex]?.id === "assistant" &&
assistantSkipConfirmationSetting.get();
const layer = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeVoiceLayer === layer) {
this.activeVoiceLayer = null;
voiceActivity.setActive(false);
// The idle countdown restarts in full once voice input ends.
this.noteUserActivity();
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === layer);
},
sendTargets: targets,
defaultTargetIndex: defaultIndex,
finishOnClick: options.finishOnClick ?? false,
handsFree: options.handsFree ?? false,
autoSend,
});
this.activeVoiceLayer = layer;
voiceActivity.setActive(true);
this.stack.push(layer);
layer.startCapture();
}
/**
* The send destinations offered by the voice dialog: the assistant (when an
* API key is configured) and/or typing into the foreground window (when it
* accepts text). Order fixes the menu row order.
*/
private buildVoiceSendTargets(): VoiceSendTarget[] {
const targets: VoiceSendTarget[] = [];
if (this.isAssistantAvailable()) {
targets.push({
id: "assistant",
label: "Send to Assistant",
onSend: (text) => this.sendToAssistant(text),
});
}
if (this.foregroundWindow()?.receiveTextInput) {
targets.push({
id: "app",
label: "Type Into App",
onSend: (text) => this.sendTextToForegroundWindow(text),
});
}
// Guarantee at least one destination so the dialog is never a dead end.
if (targets.length === 0) {
targets.push({
id: "app",
label: "Type Into App",
onSend: (text) => this.sendTextToForegroundWindow(text),
});
}
return targets;
}
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
sendTextToForegroundWindow(text: string, options?: { submit?: boolean }): void {
this.foregroundWindow()?.receiveTextInput?.(text, options);
}
/**
* Open the voice dialog aimed at the foreground window. Called when the
* user picks Voice input from the system menu (or an app asks via a
* start-voice-input message). The menu click already ended the press, so
* the dialog finishes on click instead of long-press-release. Returns
* false when the dialog can't open now (voice input disabled, screen off,
* another dialog or overlay up); true means it is opening, though a
* microphone-permission prompt can still stop it.
*/
startVoiceInput(): boolean {
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return false;
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return false;
// The transcript is aimed at the window whose menu requested it; the menu
// entry point defaults the highlight to Type Into App.
this.focus = "window";
this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" });
this.config.requestShellRender();
return true;
}
/**
* Open the keyboard dialog: the voice dialog's typed twin, driven by the
* phone's keyboard button (beside the mic button, whose wakeword this
* mirrors: it wakes a dark screen too). Over the assistant overlay the text
* continues that conversation; otherwise it goes to the usual destinations
* with Send to Assistant highlighted. Returns the session the phone types
* into (the already-open one if there is one), or null while a voice
* dialog is up. The phone can keep the dialog off the glasses (see
* KeyboardInputVisibility); that choice persists across dialogs.
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
const storedVisibility = getStringSetting(KEYBOARD_INPUT_VISIBILITY_KEY, "both");
const visibility: KeyboardInputVisibility =
storedVisibility === "phone" || storedVisibility === "glasses" ? storedVisibility : "both";
if (visibility !== "phone" && !this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
let targets: VoiceSendTarget[];
let defaultIndex = 0;
if (assistantLayer && assistantSession) {
targets = [
{
id: "assistant",
label: "Send",
// Phone-only, the overlay can close while the dialog is open; the
// text then starts a fresh overlay on the same conversation.
onSend: (text) => {
if (this.assistantLayer === assistantLayer) {
this.runAssistantTurn(assistantSession, assistantLayer, text);
} else {
this.sendToAssistant(text);
}
},
},
];
} else {
targets = this.buildVoiceSendTargets();
defaultIndex = Math.max(0, targets.findIndex((target) => target.id === "assistant"));
}
const layer = new KeyboardInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeKeyboardLayer === layer) {
this.activeKeyboardLayer = null;
this.config.onKeyboardInputChanged?.(null);
// The idle countdown restarts in full once keyboard input ends.
this.noteUserActivity();
}
},
dismiss: () => {
// Wherever it sits: a menu opened over the dialog stays open.
this.stack.remove(layer);
// A send or discard from the phone side arrives outside the input
// path, so nothing else asks for the repaint.
this.config.requestShellRender();
},
setShownOnGlasses: (shown) => {
if (shown) {
if (!this.screenOn) this.wake("sidebar");
this.stack.push(layer);
} else {
this.stack.remove(layer);
}
this.config.requestShellRender();
},
visibility,
onVisibilityChanged: (value) => setStringSetting(KEYBOARD_INPUT_VISIBILITY_KEY, value),
history: this.keyboardInputHistory,
sendTargets: targets,
defaultTargetIndex: defaultIndex,
});
this.activeKeyboardLayer = layer;
if (layer.shownOnGlasses) this.stack.push(layer);
this.config.onKeyboardInputChanged?.(layer);
this.config.requestShellRender();
return layer;
}
isAssistantAvailable(): boolean {
const current = this.getAssistantConversations().current();
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
}
/** What to set up when isAssistantAvailable() is false. */
assistantSetupHint(): string {
if (assistantBackendSetting.get() === "external") return "Configure the agent bridge host and token in Settings.";
const model = this.getAssistantConversations().current().model;
if (model === "hermes") return "Set the Hermes URL in Settings > Assistant.";
if (model === "openai-compatible") return "Set the compatible API URL and model ID in Settings > Assistant.";
return global.isIOS ? "Configure an assistant model in Settings." : "Set an API key or download the on-phone model in Settings.";
}
/** Observe assistant turns (see AssistantActivityEvent). */
onAssistantActivity(listener: (event: AssistantActivityEvent) => void): () => void {
this.assistantActivityListeners.add(listener);
return () => {
this.assistantActivityListeners.delete(listener);
};
}
/** Observe showAlert popups (mirrored to the watch). */
onAlertShown(listener: (text: string) => void): () => void {
this.alertListeners.add(listener);
return () => {
this.alertListeners.delete(listener);
};
}
private emitAssistantActivity(event: AssistantActivityEvent): void {
for (const listener of Array.from(this.assistantActivityListeners)) {
try {
listener(event);
} catch (error) {
console.warn("assistant activity listener failed", error);
}
}
}
/**
* Dismiss the assistant overlay (cancelling any in-flight turn), as Done
* would. Returns whether an overlay was actually open to close.
*/
closeAssistant(): boolean {
return this.closeAssistantLayer();
}
private ensureAssistantSession(): AssistantSession | null {
return this.getAssistantConversations().ensureSession();
}
private resolveAssistantConfiguration(
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
): AssistantBackendConfig | null {
if (assistantBackendSetting.get() === "external") {
const host = assistantBridgeHostSetting.get().trim();
const token = assistantBridgeTokenSetting.get();
if (!host || !token) return null;
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
return { kind: "external", bridge: { host, port, token } };
}
const llm = resolveAssistantModel(model, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
hermesUrl: assistantHermesUrlSetting.get(),
hermesKey: assistantHermesKeySetting.get(),
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
veniceWebSearch: veniceWebSearchSetting.get(),
});
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null;
}
private buildAssistantContext(): AssistantContext {
const foreground = this.getForegroundApp();
return {
foregroundApp: foreground?.appId ?? null,
foregroundTitle: foreground?.title ?? null,
screenOn: this.screenOn,
localTime: formatAssistantTime(new Date()),
headsetBattery: this.battery.headset,
};
}
/**
* Start (or continue) an assistant conversation from a finalized utterance.
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay.
*/
sendToAssistant(text: string, showOverlay = true): void {
text = text.trim();
if (!text) return;
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert(this.assistantSetupHint());
return;
}
if (session.isTurnActive()) {
this.showAlert("The assistant is still working on the previous request");
return;
}
if (!this.screenOn) this.wake("sidebar");
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
this.runAssistantTurn(session, null, text);
return;
}
let layer = this.assistantLayer;
if (!layer) {
const created = new AssistantLayer(this.config.actions, {
onFollowUp: () => this.startAssistantFollowUp(),
onCancel: () => this.assistantSession?.cancel(),
onClose: () => this.closeAssistantLayer(),
onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the overlay reference; keep shared history.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" });
},
});
layer = created;
this.assistantLayer = created;
this.stack.push(created);
}
this.runAssistantTurn(session, layer, text);
this.config.requestShellRender();
}
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
if (session.isTurnActive()) return;
layer?.startTurn();
let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar;
layer?.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
},
onToolActivity: (label) => layer?.onToolActivity(label),
onTurnDone: () => {
layer?.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar });
},
onError: (message) => {
layer?.onError(message);
this.emitAssistantActivity({ phase: "error", text: message });
},
});
}
/**
* Record another utterance in the current assistant conversation. handsFree
* (from a wakeword over the overlay) starts the mic immediately; otherwise
* (the Follow-up menu button) a click ends the utterance.
*/
private startAssistantFollowUp(handsFree = false): void {
const layer = this.assistantLayer;
const session = this.assistantSession;
if (!layer || !session || this.activeVoiceLayer || this.activeKeyboardLayer) return;
const voice = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeVoiceLayer === voice) {
this.activeVoiceLayer = null;
voiceActivity.setActive(false);
this.noteUserActivity();
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === voice);
},
sendTargets: [
{ id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) },
],
finishOnClick: !handsFree,
handsFree,
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
});
this.activeVoiceLayer = voice;
voiceActivity.setActive(true);
this.stack.push(voice);
voice.startCapture();
this.config.requestShellRender();
}
private closeAssistantLayer(): boolean {
// Popping fires the layer's onRemoved, which cancels the turn and clears
// this.assistantLayer. popThrough also closes anything stacked above the
// overlay (a follow-up voice dialog, an alert), so a close command works
// no matter what the conversation is showing.
const layer = this.assistantLayer;
const closed = layer ? this.stack.popThrough(layer) : false;
this.noteUserActivity();
this.config.requestShellRender();
return closed;
}
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
showAlert(text: string): void {
if (!this.screenOn) this.wake("sidebar");
for (const listener of Array.from(this.alertListeners)) {
try {
listener(text);
} catch (error) {
console.warn("alert listener failed", error);
}
}
const layer = new ShellAlertLayer(text, () => {
this.stack.popIfTop((top) => top === layer);
this.config.requestShellRender();
});
this.stack.push(layer);
this.config.requestShellRender();
}
private startEscapeMenuTimer(): void {
this.cancelEscapeMenuTimer();
this.escapeMenuTimer = setTimeout(() => {
this.escapeMenuTimer = null;
this.openEscapeMenu();
}, LONG_PRESS_ESCAPE_MENU_MS);
}
private cancelEscapeMenuTimer(): void {
if (this.escapeMenuTimer !== null) {
clearTimeout(this.escapeMenuTimer);
this.escapeMenuTimer = null;
}
}
/**
* A window's answer to tap-then-hold when it has no context menu of its
* own: open the system menu in its place, so both gestures land on the
* same menu. Ignored unless the window is still the foreground one.
*/
openSystemMenu(windowId: string): void {
if (this.foregroundWindow()?.windowId !== windowId) return;
this.openEscapeMenu();
}
/**
* The system/escape menu: the entries every window shares (Focus app
* switcher, Voice input, Brightness, Close window) plus Debug. Shell-owned and
* shell-drawn (never the app's), so an unresponsive app can always be
* closed. It opens for long-press (over the app's own menu too), after an
* extended hold in a window that claims long-press, and on a window's
* request when tap-then-hold finds it has no menu of its own.
*/
private openEscapeMenu(): void {
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
const foreground = this.foregroundWindow();
if (!foreground) return;
const items: MenuItem[] = [];
if (foreground.closeable) {
items.push({
label: "Close window",
onSelect: (ctx) => {
// Pop the menu first (its onRemoved returns focus to the sidebar),
// then close the window the menu was opened over.
ctx.stack.pop();
this.closeForegroundWindow();
},
});
}
// Close window sits first but the menu opens on Focus app switcher, so a
// reflexive tap never closes the window.
const initialSelection = items.length;
items.push(
{
// Defocus the app (hand focus to the sidebar) without closing it: the
// reliable way out of an app that consumes double-click. Closing the
// menu is what yields focus, so popping is the whole action.
label: "Focus app switcher",
onSelect: (ctx) => {
ctx.stack.pop();
},
},
...(this.config.voiceInputEnabled === false ? [] : [{
label: "Voice input",
onSelect: (ctx) => {
// The transcript is aimed at the foreground window, so keep focus
// there through the pop instead of yielding to the sidebar.
layer.keepWindowFocus = true;
ctx.stack.pop();
this.startVoiceInput();
},
}]),
);
if (brightnessSetting.get() !== "auto") {
items.push({
label: "Brightness",
onSelect: (ctx) => {
ctx.stack.pop();
ctx.stack.push(new BrightnessPickerLayer(() => this.yieldFocusToSidebar()));
this.config.requestShellRender();
},
});
}
items.push({
label: "Debug",
onSelect: (ctx) => {
ctx.stack.pop();
this.openToolDebugDialog();
},
});
// Gesture help: tap-then-hold switches to the app's own context menu,
// when the app has one (otherwise it opens this very menu, not worth a
// hint).
const footer = foreground.hasAppMenu?.()
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
: undefined;
const layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
layer.selectItem(initialSelection);
this.stack.push(layer);
// Tell the window the system menu opened over it: an app with its own
// context menu up closes it, so the two context menus never stack.
void foreground.handleInput(makeInputEvent({ type: "system-menu-opened" }), 0);
this.config.requestShellRender();
}
/**
* Escape menu > Debug: list the assistant tools registered for the
* foreground window's app (live or gated). System-wide "always" tools are
* constant and omitted.
*/
private openToolDebugDialog(): void {
const foreground = this.foregroundWindow();
const appId = foreground?.appId ?? null;
const entries = appId
? toolRegistry
.listToolsForDebug()
.filter(
(entry) =>
entry.spec.name.startsWith(`app.${appId}.`) || entry.windowId === foreground!.windowId,
)
: [];
this.stack.push(new ToolDebugMenuLayer(appId, entries, () => this.yieldFocusToSidebar()));
this.config.requestShellRender();
}
private chromeState(): ShellChromeState {
return {
windows: this.switcherWindows().map((window) => ({
windowId: window.windowId,
title: window.title,
attention: Boolean(this.attention.get(window.windowId)),
drawIcon: window.drawIcon,
})),
selectedIndex: this.selectedStatus ? -1 : this.selectedIndex,
selectedStatusItem: this.selectedStatus?.item ?? null,
focus: this.focus,
foregroundHeightMode: this.foregroundWindow()?.heightMode ?? "min",
foregroundAppId: this.foregroundWindow()?.appId,
battery: this.battery,
trayIcons: Array.from(this.trayIcons.keys())
.sort()
.map((ownerId) => ({ ownerId, icon: this.trayIcons.get(ownerId)!.icon })),
};
}
}
export const shell = new Shell();
const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
/** Human-readable local time for the assistant's per-turn context. */
function formatAssistantTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`;
return `${day}, ${hours}:${minutes} ${meridiem}`;
}
export function rawInputEventToInputEvent(event: RawInputEvent): InputEvent {
return { ...makeInputEvent(rawInputEventToPayload(event)),
...(event.ringInput ? { ringInput: { ...event.ringInput } } : {}) };
}
function rawInputEventToPayload(event: RawInputEvent): InputEventPayload {
if (event.kind === "sys-event") {
if (event.eventType === OsEventTypeList.RING_PRESS_EVENT &&
(event.eventSource === EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL ||
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING)) {
// The firmware emits this dedicated ID only for full raw source 4.
// Its stock sender leaves the source unspecified for extension 14.
return { type: "ring-press", source: "ring" };
} else if (event.eventType === OsEventTypeList.RING_PRESS_EVENT &&
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH) {
// Synthetic: the watch pad's (or the phone's watch pad's) touch-down.
return { type: "ring-press", source: "watch" };
} else if (event.eventType === OsEventTypeList.CLICK_EVENT) {
return {
type: "click",
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) {
return {
type: "double-click",
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return scrollEvent("scroll-down", event.eventSource);
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return scrollEvent("scroll-up", event.eventSource);
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) {
// CFW-forwarded long-press (replaces the firmware's force-quit dialog).
// Current CFW supplies the physical source for ring and temple presses;
// eventSourceToString's ring fallback keeps older CFW builds compatible.
return { type: "long-press", source: eventSourceToString(event.eventSource) };
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
} else if (event.eventType === OsEventTypeList.SHORT_THEN_LONG_PRESS_EVENT) {
return { type: "short-then-long-press", source: eventSourceToString(event.eventSource) };
}
} else if (event.kind === "watch-gesture") {
// Synthetic, from the Wear OS remote (app/g2/wear-remote.ts); the glasses
// firmware never produces these.
if (event.eventType === WatchGestureType.SWIPE_LEFT) {
return { type: "swipe-left", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_RIGHT) {
return { type: "swipe-right", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_UP) {
return { type: "swipe-up", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_DOWN) {
return { type: "swipe-down", source: "watch" };
}
} else if (event.kind === "even-ai") {
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
// wakeword interests us -- ENTER means the user manually opened the stock
// assistant, and EXIT is it tearing down.
if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) {
return { type: "wakeword" };
}
} else if (event.kind === "display-wake") {
// Outside an EvenHub page the firmware consumes the physical double tap
// itself and reports only that the stock display lifecycle woke.
return { type: "display-wake" };
} else if (event.kind === "text-click") {
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return scrollEvent("scroll-down", event.eventSource);
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return scrollEvent("scroll-up", event.eventSource);
}
}
return {
type: "unknown",
kind: event.kind,
eventSource: event.eventSource,
eventType: event.eventType,
};
}
/** Preserve explicit sources; stock scroll notifications usually omit them. */
function scrollEvent(type: "scroll-up" | "scroll-down", eventSource: number): InputEventPayload {
return eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH ||
eventSource === EventSourceType.TOUCH_EVENT_FROM_RING ||
eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L ||
eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R
? { type, source: eventSourceToString(eventSource) } : { type };
}
function eventSourceToString(eventSource: number): InputSource {
if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
return "ring";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) {
return "left-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) {
return "right-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH) {
return "watch";
}
return "ring";
}
export function inputEventToString(event: InputEvent): string {
switch (event.type) {
case "ring-press":
return `Press from ${event.source}`;
case "click":
return `Click from ${event.source}`;
case "double-click":
return `Double click from ${event.source}`;
case "scroll-up":
return `Scroll up`;
case "scroll-down":
return `Scroll down`;
case "long-press":
return `Long press from ${event.source}`;
case "long-press-release":
return `Long press release from ${event.source}`;
case "short-then-long-press":
return `Short then long press from ${event.source}`;
case "swipe-left":
return `Swipe left from ${event.source}`;
case "swipe-right":
return `Swipe right from ${event.source}`;
case "swipe-up":
return `Swipe up from ${event.source}`;
case "swipe-down":
return `Swipe down from ${event.source}`;
case "display-wake":
return `Display wake`;
case "wakeword":
return `Wakeword`;
case "system-menu-opened":
return `System menu opened`;
default:
case "unknown":
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
}
}