import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/bdffont"; import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events"; import type { RawInputEvent } from "../../native/faceclaw-communicator"; import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers"; import { MenuLayer, type MenuItem } from "../menu"; import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input"; import { AssistantLayer } from "../apps/assistant"; import { AssistantSession } from "../../assistant/session"; import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models"; import type { AssistantContext } from "../../assistant/types"; import { SingleNotificationLayer } from "../notifications"; import { anthropicApiKeySetting, assistantModelSetting, assistantSkipConfirmationSetting, batteryDisplayModeSetting, onAnySettingChanged, openAiApiKeySetting, timeFormatSetting, wakeWordActionSetting, } from "../dashboard-settings"; import { ShellChromeLayer, type ShellChromeState, type ShellChromeWindow } from "./chrome-layer"; import { ShellModalLayer } from "./modal-layer"; import { ToolDebugMenuLayer } from "./tool-debug-layer"; import { toolRegistry } from "../../assistant/tool-registry"; import { SIDEBAR_WIDTH, TOP_BAR_HEIGHT } from "./geometry"; /** * The shell: owns the window registry, focus, screen on/off, and the shell * surface (sidebar + top bar + shell overlays such as the escape menu and * the voice dialog). Runs on the main thread; windows are hosted in-process * or in per-app worker threads. * * Input flow: every event enters via receiveInput. The shell consumes * everything while the sidebar or a shell overlay has focus and forwards the * rest to the focused window. Long-press goes to the foreground window (from * the sidebar it focuses the window first); by convention apps answer it with * a window menu that ends in Voice input / Close window. Holding the press * past the escape threshold opens the shell's own escape menu, so the shell * keeps working when a window's handler hangs. */ export type ShellWindow = { appId: string; windowId: string; title: string; /** Compositor surface this window renders to; configured at connect / launch. */ surfaceId: string; /** Whether close menu entries (app menu default, shell escape menu) apply (the launcher is pinned). */ closeable: boolean; /** App-side cleanup when the shell closes the window (worker notification, surface removal). */ close?: () => void; drawIcon: ShellChromeWindow["drawIcon"]; /** * Handle an input event the shell forwarded. Ownership of frameId (latency * tracking) passes to the window: it must eventually reach a frame submit * or a finishFrame call. */ handleInput: (event: DashboardInputEvent, frameId: number) => Promise | void; /** Repaint and resubmit this window's surface. */ requestRender: () => void; /** * Deliver a text string to the window (e.g. finalized voice input). Optional: * only windows that consume typed text (the terminal) implement it. */ receiveTextInput?: (text: string) => void; /** Foreground state changed: this window's surface is (not) the visible one. */ setForeground?: (foreground: boolean) => void; /** Screen turned on/off; hidden or screen-off windows should stop painting. */ setScreenOn?: (on: boolean) => void; }; export type ShellConfig = { /** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */ actions: LayerActions; getScreenTimeoutMs: () => number | null; requestShellRender: () => void; /** Screen on/off changed: the controller blanks/unblanks the compositor. */ onScreenStateChanged: (on: boolean) => void; }; /** Which surfaces need re-rendering after an input event. */ export type ShellInputOutcome = { shell: boolean; window: boolean }; type FocusKind = "sidebar" | "window"; const noopActions: LayerActions = noopLayerActions; /** * Hold a long-press this much past the long-press event (which the firmware * itself only fires after a shorter hold) and the shell opens its own escape * menu — the recovery path when an app ignores or mishandles the gesture. */ const LONG_PRESS_ESCAPE_MENU_MS = 4000; /** Shell-surface overlay menu (the escape menu); closing it returns focus to the sidebar. */ class ShellOverlayMenuLayer extends MenuLayer { constructor(items: MenuItem[], private readonly onClosed: () => void) { super(null, items, { x: SIDEBAR_WIDTH + 8, y: TOP_BAR_HEIGHT + 8, width: 272, minHeight: 0, }); } onRemoved(): void { this.onClosed(); } } /** How long a show_alert popup stays before auto-dismissing. */ const ALERT_DISMISS_MS = 6000; const ALERT_X = 40; const ALERT_W = G2_LENS_WIDTH - 80; const ALERT_Y = 96; const ALERT_H = 96; /** * A brief text popup on the shell surface (the assistant's show_alert tool and * other short notices). Auto-dismisses after a few seconds; a click or * double-click dismisses it early. */ class ShellAlertLayer implements Layer { private timer: ReturnType | null = null; constructor(private readonly text: string, private readonly onDismiss: () => void) { this.timer = setTimeout(() => { this.timer = null; this.onDismiss(); }, ALERT_DISMISS_MS); } paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage { const image = paintBelow(); const font = getDefaultSmallFont(); image.fillRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 1, 10); image.drawRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 90, 10); image.drawText(font, ALERT_X + 16, ALERT_Y + 12, "Assistant", 200); image.drawTextWrapped({ font, x: ALERT_X + 16, y: ALERT_Y + 34, width: ALERT_W - 32, text: this.text, value: 235, }); return image; } handleInput(event: DashboardInputEvent, _ctx: LayerContext): void { if (event.type === "click" || event.type === "double-click") { this.clearTimer(); this.onDismiss(); } } onRemoved(): void { this.clearTimer(); } private clearTimer(): void { if (this.timer !== null) { clearTimeout(this.timer); this.timer = null; } } } class Shell { private windows: ShellWindow[] = []; private selectedIndex = 0; /** Window ids in most-recently-visible-first order; closing the visible window returns to the next entry. */ private mruWindowIds: string[] = []; private focus: FocusKind = "sidebar"; private screenOn = true; private lastInputAtMs = Date.now(); private battery: ShellChromeState["battery"] = { headset: null, headsetCharging: null }; private attention = new Map(); // App-provided top-bar tray icons, keyed by owner id; drawn between the // notification icons and the battery indicators. private readonly trayIcons = new Map(); private activeVoiceLayer: VoiceInputLayer | null = null; private assistantSession: AssistantSession | null = null; private assistantLayer: AssistantLayer | null = null; private escapeMenuTimer: ReturnType | null = null; private readonly actions: LayerActions = { ...noopActions }; private config: ShellConfig = { actions: noopActions, getScreenTimeoutMs: () => null, requestShellRender: () => {}, onScreenStateChanged: () => {}, }; private readonly stack = new LayerStack( new ShellChromeLayer(() => this.chromeState()), this.actions, ); // Top-bar settings we mirror into the chrome; a change to either repaints // the shell surface so the top bar reflects it immediately. private topBarSettingsSubscribed = false; private lastBatteryDisplayMode: string | null = null; private lastTimeFormat: string | null = null; configure(config: ShellConfig): void { this.config = config; this.stack.setActions(config.actions); this.subscribeToTopBarSettings(); } private subscribeToTopBarSettings(): void { if (this.topBarSettingsSubscribed) return; this.topBarSettingsSubscribed = true; this.lastBatteryDisplayMode = batteryDisplayModeSetting.get(); this.lastTimeFormat = timeFormatSetting.get(); onAnySettingChanged(() => { const batteryMode = batteryDisplayModeSetting.get(); const timeFormat = timeFormatSetting.get(); if (batteryMode === this.lastBatteryDisplayMode && timeFormat === this.lastTimeFormat) { return; } this.lastBatteryDisplayMode = batteryMode; this.lastTimeFormat = timeFormat; this.config.requestShellRender(); }); } /** Add a window (or replace one with the same windowId, keeping its slot). */ registerWindow(window: ShellWindow): void { const existing = this.windows.findIndex((w) => w.windowId === window.windowId); if (existing >= 0) { this.windows[existing] = window; if (existing === this.selectedIndex) this.noteWindowVisible(window.windowId); } else { this.windows.push(window); if (this.windows.length - 1 === this.selectedIndex) this.noteWindowVisible(window.windowId); } } /** Move a window to the front of the most-recently-visible order. */ private noteWindowVisible(windowId: string): void { this.mruWindowIds = this.mruWindowIds.filter((id) => id !== windowId); this.mruWindowIds.unshift(windowId); } /** Index of the most recently visible window still in the registry, or -1. */ private mostRecentWindowIndex(): number { for (const id of this.mruWindowIds) { const index = this.windows.findIndex((w) => w.windowId === id); if (index >= 0) return index; } return -1; } removeWindow(windowId: string): void { const index = this.windows.findIndex((w) => w.windowId === windowId); if (index < 0) return; const wasSelected = index === this.selectedIndex; this.windows.splice(index, 1); this.attention.delete(windowId); this.mruWindowIds = this.mruWindowIds.filter((id) => id !== windowId); if (wasSelected) { // Return to the most recently visible remaining window. const mruIndex = this.mostRecentWindowIndex(); this.selectedIndex = mruIndex >= 0 ? mruIndex : Math.min(index, Math.max(0, this.windows.length - 1)); } else if (this.selectedIndex > index) { this.selectedIndex--; } if (this.focus === "window" && (wasSelected || !this.windows.length)) { this.focus = "sidebar"; } if (wasSelected) { // Hand the foreground to whatever is now selected. const next = this.windows[this.selectedIndex]; if (next) { this.noteWindowVisible(next.windowId); next.setForeground?.(true); next.requestRender(); } } this.config.requestShellRender(); } /** Close a window by id, if it is closeable (menu actions route here). */ closeWindow(windowId: string): void { const window = this.windows.find((w) => w.windowId === windowId); if (!window || !window.closeable) return; try { window.close?.(); } catch (error) { console.warn(`window ${window.windowId} close failed`, error); } this.removeWindow(window.windowId); } /** Close the foreground window via the shell (escape menu action). */ closeForegroundWindow(): void { const window = this.foregroundWindow(); if (window) this.closeWindow(window.windowId); } getWindows(): readonly ShellWindow[] { return this.windows; } setWindowAttention(windowId: string, attention: boolean): void { if (Boolean(this.attention.get(windowId)) === attention) return; this.attention.set(windowId, attention); this.config.requestShellRender(); } setBatteryLevels(levels: Partial): void { this.battery = { ...this.battery, ...levels }; } /** * Set or clear an app's top-bar tray icon (a small grayscale image, drawn * between the notification icons and the battery indicators). Small and * infrequently updated by design; not a framebuffer. */ setTrayIcon(ownerId: string, icon: GrayImage | null): void { if (icon) { this.trayIcons.set(ownerId, icon); } else if (!this.trayIcons.delete(ownerId)) { return; } this.config.requestShellRender(); } isScreenOn(): boolean { return this.screenOn; } /** Current headset battery levels (for the assistant's get_state tool). */ getBatteryLevels(): ShellChromeState["battery"] { return this.battery; } /** * The foreground app, or null when only the launcher is showing. The launcher * is a pinned window but counts as "no app" for the assistant's context. */ getForegroundApp(): { appId: string; title: string } | null { const window = this.foregroundWindow(); if (!window || window.appId === "launcher") return null; return { appId: window.appId, title: window.title }; } noteUserActivity(nowMs = Date.now()): void { this.lastInputAtMs = nowMs; } /** Turn the screen on (if off) and set focus. Returns whether it was off. */ wake(focus: FocusKind, nowMs = Date.now()): boolean { this.lastInputAtMs = nowMs; this.focus = focus; if (this.screenOn) return false; this.screenOn = true; this.config.onScreenStateChanged(true); for (const window of this.windows) { window.setScreenOn?.(true); } // Refresh the foreground window; the compositor restored its retained // frame, but its content may be stale (e.g. a running stopwatch). this.foregroundWindow()?.requestRender(); return true; } /** Turn the screen off, closing any shell overlays. Sidebar selection is kept. */ sleep(): void { if (!this.screenOn) return; this.cancelEscapeMenuTimer(); this.screenOn = false; this.stack.clearToBase(); for (const window of this.windows) { window.setScreenOn?.(false); } this.config.onScreenStateChanged(false); } /** Foreground and focus a window by id (e.g. a wake path opening content in it). */ focusWindow(windowId: string): void { const index = this.windows.findIndex((w) => w.windowId === windowId); if (index < 0) return; this.setSelectedIndex(index); this.focus = "window"; } /** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */ applyScreenTimeout(nowMs = Date.now()): boolean { const timeoutMs = this.config.getScreenTimeoutMs(); if (timeoutMs === null || !this.screenOn) return false; // An open voice dialog suspends the timeout: a long dictation or refine // has no button presses, but the screen must stay on for it. Sliding // lastInputAtMs forward also restarts the full timeout when it closes. // An in-flight assistant turn suspends it for the same reason (a tool loop // can run for a while with no input); once the turn ends and the // Follow-up/Done menu is showing, the normal idle timeout resumes. if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) { this.lastInputAtMs = nowMs; return false; } if (nowMs - this.lastInputAtMs < timeoutMs) return false; this.sleep(); return true; } /** * Show a new notification in a shell modal over the app viewport. If the * notification woke the screen, closing the modal goes back to sleep * (matching the old sleep-popup behavior). */ openNotificationModal(notificationKey: string, wokeScreen: boolean): void { if (!this.screenOn) return; const modal: ShellModalLayer = new ShellModalLayer( new SingleNotificationLayer(notificationKey, { origin: "new-notification-modal", closeModal: () => this.closeNotificationModal(modal, wokeScreen), }), this.config.actions, ); this.stack.push(modal); this.config.requestShellRender(); } private closeNotificationModal(modal: ShellModalLayer, wokeScreen: boolean): void { this.stack.popIfTop((layer) => layer === modal); if (wokeScreen) { this.sleep(); } this.config.requestShellRender(); } /** Called by a window when the user backs out of its root (double-tap). */ yieldFocusToSidebar(): void { if (this.focus === "sidebar") return; this.focus = "sidebar"; // Repaint the window so its selection highlight dims to the unfocused // style this frame. this.foregroundWindow()?.requestRender(); this.config.requestShellRender(); } /** Paint the shell surface: transparent chrome, or all-transparent when asleep. */ paintSurface(): GrayImage { if (!this.screenOn) { return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); } return this.stack.paint(); } async receiveInput(event: DashboardInputEvent, frameId = 0): Promise { // The stock lifecycle has already interpreted the physical double tap as // "wake". Keep that directionality if delivery is delayed or duplicated. if (event.type === "display-wake") { this.lastInputAtMs = Date.now(); const wokeScreen = !this.screenOn && this.wake("sidebar"); return { shell: wokeScreen, window: false }; } // The wakeword is handled before the screen-off short-circuit so its // configured action can work from a dark screen. With the CFW the stock // Even AI app never launches, so the firmware does not power the display // for us either -- actions that need it wake the screen themselves. if (event.type === "wakeword") { const action = wakeWordActionSetting.get(); if (action === "off") { return { shell: false, window: false }; } this.lastInputAtMs = Date.now(); const wokeScreen = !this.screenOn && this.wake("sidebar"); if (action === "voice-input" && !this.activeVoiceLayer) { if (this.assistantLayer) { // The assistant overlay is up; a wakeword continues that conversation. this.startAssistantFollowUp(true); } else { // Wakeword defaults the highlight to Send to Assistant. this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" }); } } return { shell: wokeScreen || action === "voice-input", window: false }; } this.lastInputAtMs = Date.now(); // Anything but the long-press itself means the press ended (or the event // stream moved on), so the escape countdown stops. if (event.type !== "long-press") { this.cancelEscapeMenuTimer(); } if (!this.screenOn) { if (event.type === "double-click") { this.wake("sidebar"); return { shell: true, window: false }; } return { shell: false, window: false }; } // Long-press goes to the foreground window (from the sidebar it focuses // the window first); apps conventionally answer it with their window // menu. The escape timer runs regardless of what the app does with it: // holding the press long enough opens the shell's own menu. if (event.type === "long-press") { this.startEscapeMenuTimer(); if (this.activeVoiceLayer || !this.stack.isAtBase()) { return { shell: true, window: false }; } const window = this.foregroundWindow(); if (!window) { return { shell: true, window: false }; } if (this.focus === "sidebar") { this.focus = "window"; } // The window owns frameId from here (render or explicit finish). await window.handleInput(event, frameId); return { shell: true, window: true }; } if (event.type === "long-press-release") { this.activeVoiceLayer?.endCapture(); if (this.activeVoiceLayer || !this.stack.isAtBase() || this.focus !== "window") { return { shell: true, window: false }; } const window = this.foregroundWindow(); if (!window) { return { shell: true, window: false }; } await window.handleInput(event, frameId); return { shell: false, window: true }; } if (!this.stack.isAtBase()) { await this.stack.handleInput(event); return { shell: true, window: false }; } if (this.focus === "sidebar") { return this.handleSidebarInput(event); } const window = this.foregroundWindow(); if (window) { // The window owns frameId from here (render or explicit finish). await window.handleInput(event, frameId); return { shell: false, window: true }; } return { shell: false, window: false }; } foregroundWindow(): ShellWindow | undefined { return this.windows[this.selectedIndex] ?? this.windows[0]; } /** Whether a window is the current input target (foreground + focus in-window). */ isWindowFocused(windowId: string): boolean { return this.screenOn && this.focus === "window" && this.foregroundWindow()?.windowId === windowId; } /** * Whether a window's content is on screen: it's the foreground window and the * screen is on. Focus-independent — the app viewport stays visible while the * sidebar is focused (the sidebar is just the left strip). */ isWindowVisible(windowId: string): boolean { return this.screenOn && this.foregroundWindow()?.windowId === windowId; } private handleSidebarInput(event: DashboardInputEvent): ShellInputOutcome { switch (event.type) { case "double-click": this.sleep(); return { shell: true, window: false }; case "scroll-up": this.moveSelection(-1); return { shell: true, window: false }; case "scroll-down": this.moveSelection(1); return { shell: true, window: false }; case "click": if (this.windows.length) { this.focus = "window"; // Repaint the window now so its selection highlight reflects focus // this frame, not one frame late. this.foregroundWindow()?.requestRender(); } return { shell: true, window: false }; default: return { shell: false, window: false }; } } private moveSelection(delta: number): void { if (!this.windows.length) return; const count = this.windows.length; this.setSelectedIndex((this.selectedIndex + delta + count) % count); } /** Change selection; the selected window is the foreground window. */ private setSelectedIndex(index: number): void { if (index === this.selectedIndex) return; const previous = this.windows[this.selectedIndex]; this.selectedIndex = index; const next = this.windows[index]; if (next) this.noteWindowVisible(next.windowId); previous?.setForeground?.(false); next?.setForeground?.(true); next?.requestRender(); } private openVoiceDialog(options: { finishOnClick?: boolean; handsFree?: boolean; defaultTarget: "assistant" | "app"; }): void { const targets = this.buildVoiceSendTargets(); let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget); if (defaultIndex < 0) defaultIndex = 0; // Skip the menu only for a hands-free (wakeword) capture aimed at the // assistant, when the user has opted into it. const autoSend = Boolean(options.handsFree) && options.defaultTarget === "assistant" && targets[defaultIndex]?.id === "assistant" && assistantSkipConfirmationSetting.get(); const layer = new VoiceInputLayer({ actions: this.config.actions, onClosed: () => { if (this.activeVoiceLayer === layer) { this.activeVoiceLayer = null; // The idle countdown restarts in full once voice input ends. this.noteUserActivity(); } }, dismiss: () => { this.stack.popIfTop((top) => top === layer); }, sendTargets: targets, defaultTargetIndex: defaultIndex, finishOnClick: options.finishOnClick ?? false, handsFree: options.handsFree ?? false, autoSend, }); this.activeVoiceLayer = layer; this.stack.push(layer); layer.startCapture(); } /** * The send destinations offered by the voice dialog: the assistant (when an * API key is configured) and/or typing into the foreground window (when it * accepts text). Order fixes the menu row order. */ private buildVoiceSendTargets(): VoiceSendTarget[] { const targets: VoiceSendTarget[] = []; if (this.isAssistantAvailable()) { targets.push({ id: "assistant", label: "Send to Assistant", onSend: (text) => this.sendToAssistant(text), }); } if (this.foregroundWindow()?.receiveTextInput) { targets.push({ id: "app", label: "Type Into App", onSend: (text) => this.sendTextToForegroundWindow(text), }); } // Guarantee at least one destination so the dialog is never a dead end. if (targets.length === 0) { targets.push({ id: "app", label: "Type Into App", onSend: (text) => this.sendTextToForegroundWindow(text), }); } return targets; } /** Deliver a text string to the foreground window (e.g. finalized voice input). */ sendTextToForegroundWindow(text: string): void { this.foregroundWindow()?.receiveTextInput?.(text); } /** * Open the voice dialog aimed at the foreground window. Called when the * user picks Voice input from a window's menu (in-process directly, worker * apps via a start-voice-input message). The menu click already ended the * press, so the dialog finishes on click instead of long-press-release. */ startVoiceInput(): void { if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return; // The transcript is aimed at the window whose menu requested it; the menu // entry point defaults the highlight to Type Into App. this.focus = "window"; this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" }); this.config.requestShellRender(); } private isAssistantAvailable(): boolean { return this.resolveAssistantConfiguration() !== null; } private ensureAssistantSession(): AssistantSession | null { const llm = this.resolveAssistantConfiguration(); if (!llm) return null; if ( !this.assistantSession || this.assistantSession.isExpired() || !this.assistantSession.matchesConfiguration(llm) ) { this.assistantSession?.cancel(); this.assistantSession = new AssistantSession(llm); } return this.assistantSession; } private resolveAssistantConfiguration(): ResolvedAssistantModel | null { return resolveAssistantModel(assistantModelSetting.get(), { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(), }); } private buildAssistantContext(): AssistantContext { const foreground = this.getForegroundApp(); return { foregroundApp: foreground?.appId ?? null, foregroundTitle: foreground?.title ?? null, screenOn: this.screenOn, localTime: formatAssistantTime(new Date()), headsetBattery: this.battery.headset, }; } /** * Start (or continue) an assistant conversation from a finalized utterance. * Opens the assistant overlay if it isn't already up; a follow-up reuses the * existing session and overlay. */ sendToAssistant(text: string): void { const session = this.ensureAssistantSession(); if (!session) { this.showAlert("Set an API key for the selected assistant model in Settings."); return; } if (!this.screenOn) this.wake("sidebar"); let layer = this.assistantLayer; if (!layer) { const created = new AssistantLayer(this.config.actions, { onFollowUp: () => this.startAssistantFollowUp(), onCancel: () => this.assistantSession?.cancel(), onClose: () => this.closeAssistantLayer(), onRemoved: () => { // Removed by any path (Done, or the screen sleeping mid-conversation): // stop the turn and drop the reference so a later query starts clean. this.assistantSession?.cancel(); if (this.assistantLayer === created) this.assistantLayer = null; }, }); layer = created; this.assistantLayer = created; this.stack.push(created); } this.runAssistantTurn(session, layer, text); this.config.requestShellRender(); } private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void { layer.startTurn(); session.sendUtterance(text, this.buildAssistantContext(), { onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar), onToolActivity: (label) => layer.onToolActivity(label), onTurnDone: () => layer.onTurnDone(), onError: (message) => layer.onError(message), }); } /** * Record another utterance in the current assistant conversation. handsFree * (from a wakeword over the overlay) starts the mic immediately; otherwise * (the Follow-up menu button) a click ends the utterance. */ private startAssistantFollowUp(handsFree = false): void { const layer = this.assistantLayer; const session = this.assistantSession; if (!layer || !session || this.activeVoiceLayer) return; const voice = new VoiceInputLayer({ actions: this.config.actions, onClosed: () => { if (this.activeVoiceLayer === voice) { this.activeVoiceLayer = null; this.noteUserActivity(); } }, dismiss: () => { this.stack.popIfTop((top) => top === voice); }, sendTargets: [ { id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) }, ], finishOnClick: !handsFree, handsFree, autoSend: handsFree && assistantSkipConfirmationSetting.get(), }); this.activeVoiceLayer = voice; this.stack.push(voice); voice.startCapture(); this.config.requestShellRender(); } private closeAssistantLayer(): void { // Popping fires the layer's onRemoved, which cancels the turn and clears // this.assistantLayer. const layer = this.assistantLayer; if (layer) this.stack.popIfTop((top) => top === layer); this.noteUserActivity(); this.config.requestShellRender(); } /** Show a brief text popup on the lenses (assistant show_alert / notices). */ showAlert(text: string): void { if (!this.screenOn) this.wake("sidebar"); const layer = new ShellAlertLayer(text, () => { this.stack.popIfTop((top) => top === layer); this.config.requestShellRender(); }); this.stack.push(layer); this.config.requestShellRender(); } private startEscapeMenuTimer(): void { this.cancelEscapeMenuTimer(); this.escapeMenuTimer = setTimeout(() => { this.escapeMenuTimer = null; this.openEscapeMenu(); }, LONG_PRESS_ESCAPE_MENU_MS); } private cancelEscapeMenuTimer(): void { if (this.escapeMenuTimer !== null) { clearTimeout(this.escapeMenuTimer); this.escapeMenuTimer = null; } } /** * The held-long-press safeguard menu. Shell-owned and shell-drawn (never * the app's), so an unresponsive app can always be closed. It opens over * whatever the app did with the long-press, including its own menu. */ private openEscapeMenu(): void { if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return; const foreground = this.foregroundWindow(); if (!foreground) return; const items: MenuItem[] = []; if (foreground.closeable) { items.push({ label: "Close window", onSelect: (ctx) => { // Pop the menu first (its onRemoved returns focus to the sidebar), // then close the window the menu was opened over. ctx.stack.pop(); this.closeForegroundWindow(); }, }); } items.push({ label: "Debug", onSelect: (ctx) => { ctx.stack.pop(); this.openToolDebugDialog(); }, }); this.stack.push(new ShellOverlayMenuLayer(items, () => this.yieldFocusToSidebar())); this.config.requestShellRender(); } /** * Escape menu > Debug: list the assistant tools registered for the * foreground window's app (live or gated). System-wide "always" tools are * constant and omitted. */ private openToolDebugDialog(): void { const foreground = this.foregroundWindow(); const appId = foreground?.appId ?? null; const entries = appId ? toolRegistry .listToolsForDebug() .filter( (entry) => entry.spec.name.startsWith(`app.${appId}.`) || entry.windowId === foreground!.windowId, ) : []; this.stack.push(new ToolDebugMenuLayer(appId, entries, () => this.yieldFocusToSidebar())); this.config.requestShellRender(); } private chromeState(): ShellChromeState { return { windows: this.windows.map((window) => ({ windowId: window.windowId, title: window.title, attention: Boolean(this.attention.get(window.windowId)), drawIcon: window.drawIcon, })), selectedIndex: this.selectedIndex, focus: this.focus, battery: this.battery, trayIcons: Array.from(this.trayIcons.keys()) .sort() .map((key) => this.trayIcons.get(key)!), }; } } export const shell = new Shell(); const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; /** Human-readable local time for the assistant's per-turn context. */ function formatAssistantTime(date: Date): string { let hours = date.getHours(); const minutes = date.getMinutes().toString().padStart(2, "0"); const meridiem = hours >= 12 ? "PM" : "AM"; hours = hours % 12; if (hours === 0) hours = 12; const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`; return `${day}, ${hours}:${minutes} ${meridiem}`; } export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent { if (event.kind === "sys-event") { if (event.eventType === OsEventTypeList.CLICK_EVENT) { return { type: "click", source: eventSourceToString(event.eventSource), }; } else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) { return { type: "double-click", source: eventSourceToString(event.eventSource), }; } else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { return { type: "scroll-down" }; } else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) { return { type: "scroll-up" }; } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) { // CFW-forwarded long-press (replaces the firmware's force-quit dialog). // The CFW gates this to the ring, so eventSource may be 0 (unknown); // eventSourceToString falls back to "ring". return { type: "long-press", source: eventSourceToString(event.eventSource) }; } else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) { return { type: "long-press-release", source: eventSourceToString(event.eventSource) }; } } else if (event.kind === "even-ai") { // sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the // wakeword interests us -- ENTER means the user manually opened the stock // assistant, and EXIT is it tearing down. if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) { return { type: "wakeword" }; } } else if (event.kind === "display-wake") { // Outside an EvenHub page the firmware consumes the physical double tap // itself and reports only that the stock display lifecycle woke. return { type: "display-wake" }; } else if (event.kind === "text-click") { if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) { return { type: "scroll-down" }; } else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) { return { type: "scroll-up" }; } } return { type: "unknown", kind: event.kind, eventSource: event.eventSource, eventType: event.eventType, }; } function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" { if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) { return "ring"; } else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) { return "left-arm"; } else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) { return "right-arm"; } return "ring"; } export function inputEventToString(event: DashboardInputEvent): string { switch (event.type) { case "click": return `Click from ${event.source}`; case "double-click": return `Double click from ${event.source}`; case "scroll-up": return `Scroll up`; case "scroll-down": return `Scroll down`; case "long-press": return `Long press from ${event.source}`; case "long-press-release": return `Long press release from ${event.source}`; case "display-wake": return `Display wake`; case "wakeword": return `Wakeword`; default: case "unknown": return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`; } }