2254 lines
90 KiB
TypeScript
2254 lines
90 KiB
TypeScript
import { encodeShellScene } from "../../graphics/shell-scene";
|
|
import { GrayImage } from "../../graphics/image";
|
|
import { singlePlane, type Plane } from "../../graphics/plane";
|
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
|
import { EvenAIStatus, EventSourceType, OsEventTypeList, WatchGestureType } from "../../g2/events";
|
|
import { acceptInput } from "../input-monitor";
|
|
import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
|
import {
|
|
directionalFallback,
|
|
type DirectionalInputEvent,
|
|
GESTURE_SHORT_THEN_LONG_PRESS,
|
|
gestureHints,
|
|
InputEvent,
|
|
type InputEventPayload,
|
|
type InputSource,
|
|
isDirectionalInput,
|
|
isWatchInput,
|
|
makeInputEvent,
|
|
} from "../gestures";
|
|
import type { KeyEvent } from "../../g2/keyboard/keys";
|
|
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
|
|
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
|
|
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
|
|
import { KeyboardInputLayer, type KeyboardInputSession, type KeyboardInputVisibility } from "./keyboard-input";
|
|
import { voiceActivity } from "./voice-activity";
|
|
import { AssistantLayer } from "./assistant";
|
|
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
|
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
|
|
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
|
|
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
|
import type { AssistantContext } from "../../assistant/types";
|
|
import { SingleNotificationLayer } from "../notifications";
|
|
import {
|
|
anthropicApiKeySetting,
|
|
assistantHermesKeySetting,
|
|
assistantHermesUrlSetting,
|
|
assistantBackendSetting,
|
|
assistantBridgeHostSetting,
|
|
assistantBridgePortSetting,
|
|
assistantBridgeTokenSetting,
|
|
assistantModelSetting,
|
|
assistantOpenAiCompatibleKeySetting,
|
|
assistantOpenAiCompatibleModelSetting,
|
|
assistantOpenAiCompatibleUrlSetting,
|
|
assistantSkipConfirmationSetting,
|
|
veniceWebSearchSetting,
|
|
batteryIndicatorSettingsKey,
|
|
brightnessSetting,
|
|
newNotificationDisplaySetting,
|
|
newNotificationDisplayTimeoutMs,
|
|
onAnySettingChanged,
|
|
openAiApiKeySetting,
|
|
shouldPopUpNewNotification,
|
|
statusBarPositionSetting,
|
|
statusBarVisibilitySetting,
|
|
timeFormatSetting,
|
|
wakeWordActionSetting,
|
|
wakeToLastAppSetting,
|
|
windowBorderSetting,
|
|
} from "../dashboard-settings";
|
|
import { onAmbientCardsChanged } from "./ambient-cards";
|
|
import {
|
|
sameStatusItem,
|
|
ShellChromeLayer,
|
|
sidebarContentSpan,
|
|
type ShellChromeState,
|
|
type ShellChromeWindow,
|
|
type StatusItem,
|
|
} from "./chrome-layer";
|
|
import { ShellModalLayer } from "./modal-layer";
|
|
import { NotificationModalQueue } from "./notification-modal-queue";
|
|
import { NotificationPopupTimeout } from "./notification-popup-timeout";
|
|
import { ALL_NOTIFICATIONS, readActiveNotifications } from "../../native/notification-icons";
|
|
import { ToolDebugMenuLayer } from "./tool-debug-layer";
|
|
import type { InProcessWindow } from "./in-process-window";
|
|
import { BrightnessPickerLayer } from "./brightness-picker-layer";
|
|
import { toolRegistry } from "../../assistant/tool-registry";
|
|
import {
|
|
appViewportRect,
|
|
minWindowTop,
|
|
sidebarWidth,
|
|
switcherPosition,
|
|
switcherRowHeight,
|
|
uiDepth,
|
|
windowBandHeight,
|
|
windowFramed,
|
|
windowTop,
|
|
type WindowHeightMode,
|
|
screenHeight,
|
|
screenWidth,
|
|
} from "./geometry";
|
|
import type { AppSwitcherPosition } from "../dashboard-settings";
|
|
import type { Viewport } from "../viewport";
|
|
|
|
/**
|
|
* The shell: owns the window registry, focus, screen on/off, and the shell
|
|
* surface (sidebar + top bar + shell overlays such as the escape menu and
|
|
* the voice dialog). Runs on the main thread; windows are hosted in-process
|
|
* or in per-app worker threads.
|
|
*
|
|
* Input flow: every event enters via receiveInput. The shell consumes
|
|
* everything while the sidebar or a shell overlay has focus and forwards the
|
|
* rest to the focused window. Long-press opens the shell-owned system menu
|
|
* (Focus app switcher, Voice input, Brightness, Close window, Debug) without reaching the
|
|
* app, so the shell keeps working when a window's handler hangs; a window
|
|
* that claims long-press for a move of its own gets it forwarded instead, and
|
|
* holding the press past the escape threshold still opens the system menu.
|
|
* The 2.2.9 tap-then-hold gesture goes to the foreground window (from the
|
|
* sidebar it focuses the window first); by convention apps answer it with
|
|
* their own context menu, or ask for the system menu when they have none.
|
|
*/
|
|
|
|
export type ShellWindow = {
|
|
appId: string;
|
|
windowId: string;
|
|
title: string;
|
|
/** Compositor surface this window renders to; configured at connect / launch. */
|
|
surfaceId: string;
|
|
/** Whether the system menu offers Close window (the launcher is pinned). */
|
|
closeable: boolean;
|
|
/**
|
|
* True when tap-then-hold currently opens the window's own context menu
|
|
* (with at least one entry). While false, the system menu shows no
|
|
* app-menu hint and a tap-then-hold over the open system menu leaves it
|
|
* open instead of switching menus.
|
|
*/
|
|
hasAppMenu?: () => boolean;
|
|
/**
|
|
* True while the window gives long-press a meaning of its own (a game
|
|
* move). The shell then forwards long-presses to it instead of opening the
|
|
* system menu; holding the press past the escape threshold still opens it.
|
|
*/
|
|
claimsLongPress?: () => boolean;
|
|
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
|
|
holdToTalk?: boolean;
|
|
/** A window owns microphone capture outside the shell voice dialog. */
|
|
isVoiceCapturing?: () => boolean;
|
|
/**
|
|
* True while the window shows something the wearer watches without
|
|
* touching anything (a playing video): the idle screen timeout waits, as
|
|
* it does for voice capture, while this window is in the foreground.
|
|
*/
|
|
keepsScreenOn?: () => boolean;
|
|
/**
|
|
* True when the window gives swipe-left / swipe-right (watch directional
|
|
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
|
*/
|
|
acceptsDirectional?: boolean;
|
|
/**
|
|
* Window height: the standard 288px band ("min") or the full screen
|
|
* ("max", terminal views). Decides the surface rect and where the shell
|
|
* draws this window's top bar.
|
|
*/
|
|
heightMode: WindowHeightMode;
|
|
/**
|
|
* False for a window that must not bring its app back when Faceclaw
|
|
* restarts (the controller's open-apps restore relaunches apps, not
|
|
* windows): one that shows something a plain launch of its app would not,
|
|
* where the relaunch would do something else in its place.
|
|
*/
|
|
restoresApp?: boolean;
|
|
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
|
|
close?: () => void;
|
|
drawIcon: ShellChromeWindow["drawIcon"];
|
|
/**
|
|
* Handle an input event the shell forwarded. Ownership of frameId (latency
|
|
* tracking) passes to the window: it must eventually reach a frame submit
|
|
* or a finishFrame call.
|
|
*/
|
|
handleInput: (event: InputEvent, frameId: number) => Promise<void> | void;
|
|
/** Repaint and resubmit this window's surface. */
|
|
requestRender: () => void;
|
|
/**
|
|
* Re-measure against the current display mode and app switcher (viewport
|
|
* size / band) and repaint. Windows without it are closed and relaunched
|
|
* by the controller instead on a display mode change, and left at the size
|
|
* they opened with when the bottom switcher changes height.
|
|
*/
|
|
relayout?: () => void;
|
|
/**
|
|
* The viewport the window is laid out at (it changes on relayout), so the
|
|
* controller can tell a window whose size, or whose edges, fell behind the
|
|
* geometry from one whose surface merely moved.
|
|
*/
|
|
layoutSize?: () => Viewport;
|
|
/**
|
|
* A touch from the phone's mirror at (x, y) in app-viewport coordinates.
|
|
* True if the window acted on it; otherwise the controller sends a select.
|
|
*/
|
|
hitTest?: (x: number, y: number) => Promise<boolean> | boolean;
|
|
/**
|
|
* Deliver a text string to the window (e.g. finalized voice input). Optional:
|
|
* only windows that consume typed text (the terminal) implement it.
|
|
*/
|
|
receiveTextInput?: (text: string, options?: { submit?: boolean }) => void;
|
|
/**
|
|
* Take a key from a hardware keyboard (app/g2/keyboard) as a key. Optional:
|
|
* only windows that give keys a meaning of their own (the terminal)
|
|
* implement it. They get the keys no keybinding claimed while they have
|
|
* input focus, presses and releases alike; the rest of the time, and for
|
|
* every other window, the navigation keys act as gestures (see
|
|
* app/g2/keyboard/bindings.ts).
|
|
*/
|
|
handleKeyEvent?: (event: KeyEvent) => void;
|
|
/** Foreground state changed: this window's surface is (not) the visible one. */
|
|
setForeground?: (foreground: boolean) => void;
|
|
/**
|
|
* Input focus moved into this window. `lastInput` is the most recent input
|
|
* event the shell received — for a focus that a click or swipe caused, the
|
|
* event that caused it. A programmatic focus (a worker's focus-window
|
|
* request, a wake path) can deliver an older event: check timestampMs
|
|
* before treating it as current.
|
|
*/
|
|
onFocus?: (lastInput: InputEvent | null) => void;
|
|
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
|
|
setScreenOn?: (on: boolean) => void;
|
|
/**
|
|
* Input focus arrived at or left this window: it is (no longer) where
|
|
* ordinary input goes. Unlike setForeground / onFocus, this also tracks
|
|
* shell overlays (the system menu, a notification modal, the voice dialog)
|
|
* and screen-off, so a game can pause on any of them. Sent on change only.
|
|
*/
|
|
setInputFocus?: (focused: boolean) => void;
|
|
};
|
|
|
|
export type ShellConfig = {
|
|
/** Hosts without microphone support hide and reject voice entry points. */
|
|
voiceInputEnabled?: boolean;
|
|
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
|
|
actions: LayerActions;
|
|
getScreenTimeoutMs: () => number | null;
|
|
requestShellRender: () => void;
|
|
/**
|
|
* Asked before the voice dialog opens; resolving false swallows the open.
|
|
* Preview-only mode uses it to turn the tap into a mic-permission prompt
|
|
* when RECORD_AUDIO isn't granted yet (the phone mic is the source there).
|
|
*/
|
|
prepareVoiceCapture?: () => Promise<boolean>;
|
|
/**
|
|
* The keyboard dialog opened (with the session the phone types into) or
|
|
* closed (null) by any path; the phone UI shows/hides its typing panel.
|
|
*/
|
|
onKeyboardInputChanged?: (session: KeyboardInputSession | null) => void;
|
|
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
|
|
onScreenStateChanged: (on: boolean) => void;
|
|
/** Window registered/removed or foreground changed (persists the open-app list). */
|
|
onWindowsChanged?: () => void;
|
|
/**
|
|
* The bottom switcher's layout changed its height (geometry.ts
|
|
* setSwitcherBarHeight): windows have to move, or resize, to suit. Called
|
|
* after the shell paint that found it.
|
|
*/
|
|
onSwitcherBarHeightChanged?: () => void;
|
|
/**
|
|
* The Notifications app's window, opened in the background (neither
|
|
* foregrounded nor focused) unless it is open already, for the switcher
|
|
* to show a notification's detail view in; `opened` says this call opened
|
|
* it. Hosts that leave it out offer no notifications to select.
|
|
*/
|
|
openNotificationsWindow?: () => { window: InProcessWindow; opened: boolean } | null;
|
|
};
|
|
|
|
/**
|
|
* What makes an app's tray icon (see Shell.setTrayIcon) a widget the app
|
|
* switcher's selection can move onto, as it can onto a notification's icon
|
|
* (bottom switcher only; see Shell.moveSelection).
|
|
*/
|
|
export type TrayWidget = {
|
|
/**
|
|
* The window to show while the selection is on the widget, opened in the
|
|
* background (neither foregrounded nor focused) unless it is open
|
|
* already; `opened` says this call opened it, and the shell closes it
|
|
* again once the selection moves on. Null when it can't be opened.
|
|
*/
|
|
openWindow: () => { window: ShellWindow; opened: boolean } | null;
|
|
};
|
|
|
|
/**
|
|
* A new-notification popup: the shell modal, and the notification view inside
|
|
* it (null for an app notification, which brings its own layer).
|
|
*/
|
|
type NotificationPopup = { modal: ShellModalLayer; content: SingleNotificationLayer | null };
|
|
|
|
/**
|
|
* The app switcher's selection while it is on a status bar item rather than
|
|
* a window's icon (bottom switcher only; see Shell.moveSelection): a
|
|
* notification's icon or an app widget. The item's window is the
|
|
* foreground window meanwhile: a notification's is the Notifications app's,
|
|
* with the notification's detail view on top of its stack; a widget's is
|
|
* whatever window it gives (TrayWidget.openWindow).
|
|
*/
|
|
type StatusSelection = {
|
|
item: StatusItem;
|
|
/** Where the item sat among the selectable ones, for once it has gone. */
|
|
index: number;
|
|
host: ShellWindow;
|
|
/** A notification's detail view, and the host's stack it is on. */
|
|
detail: { stack: LayerStack; layer: SingleNotificationLayer } | null;
|
|
/**
|
|
* The host window was opened for the selection: it has no switcher icon
|
|
* of its own, and closes once the selection moves off it.
|
|
*/
|
|
transient: boolean;
|
|
/** The window selected before, to go back to when the screen sleeps. */
|
|
returnWindowId: string | null;
|
|
};
|
|
|
|
/** Which surfaces need re-rendering after an input event. */
|
|
export type ShellInputOutcome = { shell: boolean; window: boolean };
|
|
|
|
/**
|
|
* Assistant overlay activity, for mirrors outside the glasses (the watch).
|
|
* "streaming" carries the reply so far (replace semantics); "done" the final
|
|
* reply; "error" the message; "closed" fires when the overlay leaves the
|
|
* stack by any path.
|
|
*/
|
|
export type AssistantActivityEvent = {
|
|
phase: "thinking" | "streaming" | "done" | "error" | "closed";
|
|
text: string;
|
|
};
|
|
|
|
export type FocusKind = "sidebar" | "window";
|
|
|
|
/**
|
|
* A notification an app posts itself, rather than one Android posted. It
|
|
* opens in the same modal (and waits in the same queue) as Android
|
|
* notifications, but is not listed in the Notifications app.
|
|
*/
|
|
export type AppNotification = {
|
|
/** Unique per notification; must not collide with Android notification keys. */
|
|
key: string;
|
|
/** The modal's content; `close` takes the modal down. */
|
|
createLayer: (close: () => void) => Layer;
|
|
};
|
|
|
|
const noopActions: LayerActions = noopLayerActions;
|
|
|
|
/**
|
|
* For a window that claims long-press: hold the press this much past the
|
|
* long-press event (which the firmware itself only fires after a shorter
|
|
* hold) and the shell opens the system menu anyway — the recovery path when
|
|
* the app ignores or mishandles the gesture.
|
|
*/
|
|
const LONG_PRESS_ESCAPE_MENU_MS = 4000;
|
|
|
|
/** Settings-store key for the keyboard dialog's last-chosen KeyboardInputVisibility. */
|
|
const KEYBOARD_INPUT_VISIBILITY_KEY = "keyboardInput.visibility";
|
|
|
|
/** Shell-surface overlay menu (the system menu); closing it returns focus to the sidebar. */
|
|
class ShellOverlayMenuLayer extends MenuLayer {
|
|
/**
|
|
* Set before popping to keep focus on the window: an entry that acts on
|
|
* the window (Voice input aims its transcript at it) must not first hand
|
|
* focus to the sidebar.
|
|
*/
|
|
keepWindowFocus = false;
|
|
|
|
constructor(items: MenuItem[], footer: string | undefined, private readonly onClosed: () => void) {
|
|
// Aligned to the min-height window band (like the sidebar), wherever the
|
|
// vertical position setting currently puts it; centered over the
|
|
// application area, i.e. the part of the screen beside a side strip
|
|
// (the whole width with a bottom row, or over an immersive window, where
|
|
// the strip overlays).
|
|
const width = 272;
|
|
const area = appViewportRect("min");
|
|
super("System", items, {
|
|
x: area.x + (((area.width - width) / 2) | 0),
|
|
y: area.y + 8,
|
|
width,
|
|
minHeight: 150,
|
|
squareCorners: true,
|
|
footer,
|
|
dimUnderneath: CONTEXT_MENU_DIM,
|
|
depth: 4,
|
|
});
|
|
}
|
|
|
|
onRemoved(): void {
|
|
if (!this.keepWindowFocus) this.onClosed();
|
|
}
|
|
}
|
|
|
|
/** How long a show_alert popup stays before auto-dismissing. */
|
|
const ALERT_DISMISS_MS = 6000;
|
|
const ALERT_X = 40;
|
|
/** Alert width: the screen less ALERT_X on each side. */
|
|
function alertW(): number {
|
|
return screenWidth() - 2 * ALERT_X;
|
|
}
|
|
const ALERT_Y = 96;
|
|
const ALERT_H = 96;
|
|
|
|
/**
|
|
* A brief text popup on the shell surface (the assistant's show_alert tool and
|
|
* other short notices). Auto-dismisses after a few seconds; a click or
|
|
* double-click dismisses it early.
|
|
*/
|
|
class ShellAlertLayer implements Layer {
|
|
private timer: ReturnType<typeof setTimeout> | null = null;
|
|
|
|
constructor(private readonly text: string, private readonly onDismiss: () => void) {
|
|
this.timer = setTimeout(() => {
|
|
this.timer = null;
|
|
this.onDismiss();
|
|
}, ALERT_DISMISS_MS);
|
|
}
|
|
|
|
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
|
const image = paintBelow();
|
|
const font = getDefaultSmallFont();
|
|
// Positioned within the min-height window band, like the other shell overlays.
|
|
const alertY = minWindowTop() + ALERT_Y;
|
|
image.fillRect(ALERT_X, alertY, alertW(), ALERT_H, 1);
|
|
image.drawRect(ALERT_X, alertY, alertW(), ALERT_H, 90);
|
|
image.drawText(font, ALERT_X + 16, alertY + 12, "Assistant", 200);
|
|
image.drawTextWrapped({
|
|
font,
|
|
x: ALERT_X + 16,
|
|
y: alertY + 18 + font.lineHeight + 4,
|
|
width: alertW() - 32,
|
|
text: this.text,
|
|
value: 235,
|
|
});
|
|
return image;
|
|
}
|
|
|
|
handleInput(event: InputEvent, _ctx: LayerContext): void {
|
|
if (event.type === "click" || event.type === "double-click") {
|
|
this.clearTimer();
|
|
this.onDismiss();
|
|
}
|
|
}
|
|
|
|
onRemoved(): void {
|
|
this.clearTimer();
|
|
}
|
|
|
|
private clearTimer(): void {
|
|
if (this.timer !== null) {
|
|
clearTimeout(this.timer);
|
|
this.timer = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* What a watch swipe means while the app switcher has focus, as the ring
|
|
* gesture it stands for. Swipes are spatial: along the strip they move the
|
|
* selection, toward the app area they enter the selected window, and away
|
|
* from it there is nowhere further to go (ignored). A popup's row of icons
|
|
* lies over the window, so either way across it enters the window.
|
|
*/
|
|
const SWITCHER_SWIPES: Record<
|
|
AppSwitcherPosition,
|
|
Partial<Record<DirectionalInputEvent["type"], "scroll-up" | "scroll-down" | "click">>
|
|
> = {
|
|
left: { "swipe-up": "scroll-up", "swipe-down": "scroll-down", "swipe-right": "click" },
|
|
right: { "swipe-up": "scroll-up", "swipe-down": "scroll-down", "swipe-left": "click" },
|
|
bottom: { "swipe-left": "scroll-up", "swipe-right": "scroll-down", "swipe-up": "click" },
|
|
popup: { "swipe-left": "scroll-up", "swipe-right": "scroll-down", "swipe-up": "click", "swipe-down": "click" },
|
|
};
|
|
|
|
/**
|
|
* Every setting the shell's paint depends on that nothing else repaints it
|
|
* for (the top bar's contents and placement, the window frame, the scene's
|
|
* stereo depth), as one comparable string.
|
|
*/
|
|
function chromeSettingsKey(): string {
|
|
return `${batteryIndicatorSettingsKey()}|${timeFormatSetting.get()}|${uiDepth()}|${statusBarPositionSetting.get()}`
|
|
+ `|${statusBarVisibilitySetting.get()}|${windowBorderSetting.get()}`;
|
|
}
|
|
|
|
class Shell {
|
|
private windows: ShellWindow[] = [];
|
|
private selectedIndex = 0;
|
|
/** Window ids in most-recently-visible-first order; closing the visible window returns to the next entry. */
|
|
private mruWindowIds: string[] = [];
|
|
private focus: FocusKind = "sidebar";
|
|
/** The window last told it holds input focus (see syncInputFocus). */
|
|
private inputFocusedWindowId: string | null = null;
|
|
private screenOn = true;
|
|
private lastInputAtMs = Date.now();
|
|
/** The most recent input event received, for windows gaining focus (see ShellWindow.onFocus). */
|
|
private lastInput: InputEvent | null = null;
|
|
private battery: ShellChromeState["battery"] = {
|
|
headset: null, headsetCharging: null, ring: null, ringCharging: null, watch: null, watchCharging: null,
|
|
case: null, caseCharging: null,
|
|
};
|
|
private readonly batteryListeners = new Set<() => void>();
|
|
private attention = new Map<string, boolean>();
|
|
/** Set while the switcher's selection is on a status bar item (see StatusSelection). */
|
|
private selectedStatus: StatusSelection | null = null;
|
|
// App-provided top-bar tray icons, keyed by owner id; drawn between the
|
|
// notification icons and the battery indicators.
|
|
private readonly trayIcons = new Map<string, { icon: GrayImage; widget?: TrayWidget }>();
|
|
private activeVoiceLayer: VoiceInputLayer | null = null;
|
|
private activeKeyboardLayer: KeyboardInputLayer | null = null;
|
|
// Messages sent from keyboard dialogs, newest first; in memory only.
|
|
private readonly keyboardInputHistory: string[] = [];
|
|
// App-posted notifications that are open or queued, by key.
|
|
private readonly appNotifications = new Map<string, AppNotification>();
|
|
private readonly notificationModals = new NotificationModalQueue<NotificationPopup>({
|
|
open: (notificationKey) => this.pushNotificationModal(notificationKey),
|
|
isAvailable: (notificationKey) => {
|
|
if (this.appNotifications.has(notificationKey)) return true;
|
|
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
|
|
return !!notification && shouldPopUpNewNotification(notification.packageName);
|
|
},
|
|
sleep: () => this.sleep(),
|
|
});
|
|
private conversations: AssistantConversations | null = null;
|
|
private get assistantSession(): AssistantSession | null {
|
|
return this.conversations?.current().session ?? null;
|
|
}
|
|
|
|
getAssistantConversations(): AssistantConversations {
|
|
if (!this.conversations) {
|
|
this.conversations = new AssistantConversations(
|
|
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
|
|
() => assistantModelSetting.get(),
|
|
(value) => setStringSetting("assistant.conversations", value),
|
|
getStringSetting("assistant.conversations", ""),
|
|
);
|
|
}
|
|
return this.conversations;
|
|
}
|
|
|
|
async prepareChatVoiceCapture(): Promise<boolean> {
|
|
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
|
|
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
|
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
|
|
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
|
|
}
|
|
private assistantLayer: AssistantLayer | null = null;
|
|
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
|
|
private readonly alertListeners = new Set<(text: string) => void>();
|
|
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
|
|
private readonly actions: LayerActions = { ...noopActions };
|
|
private config: ShellConfig = {
|
|
actions: noopActions,
|
|
getScreenTimeoutMs: () => null,
|
|
requestShellRender: () => {},
|
|
onScreenStateChanged: () => {},
|
|
};
|
|
private readonly chrome = new ShellChromeLayer(() => this.chromeState(), () => this.switcherBarHeightChanged());
|
|
private readonly stack = new LayerStack(this.chrome, this.actions);
|
|
private switcherBarChangePending = false;
|
|
|
|
// Top-bar settings we mirror into the chrome; a change to any of them
|
|
// repaints the shell surface so the top bar reflects it immediately.
|
|
private chromeSettingsSubscribed = false;
|
|
private lastChromeSettingsKey: string | null = null;
|
|
|
|
configure(config: ShellConfig): void {
|
|
// Nearly every state change ends in a shell render request, so it is
|
|
// the natural point to tell windows about input-focus changes too.
|
|
this.config = {
|
|
...config,
|
|
requestShellRender: () => {
|
|
this.syncInputFocus();
|
|
config.requestShellRender();
|
|
},
|
|
};
|
|
this.stack.setActions(config.actions);
|
|
this.subscribeToChromeSettings();
|
|
this.subscribeToAmbientCards();
|
|
}
|
|
|
|
// Ambient (encounter) cards live in the chrome paint; a posted or expired
|
|
// card repaints the shell surface so it appears and disappears on time.
|
|
private ambientCardsSubscribed = false;
|
|
|
|
private subscribeToAmbientCards(): void {
|
|
if (this.ambientCardsSubscribed) return;
|
|
this.ambientCardsSubscribed = true;
|
|
onAmbientCardsChanged(() => {
|
|
if (this.screenOn) {
|
|
this.config.requestShellRender();
|
|
}
|
|
});
|
|
}
|
|
|
|
private subscribeToChromeSettings(): void {
|
|
if (this.chromeSettingsSubscribed) return;
|
|
this.chromeSettingsSubscribed = true;
|
|
this.lastChromeSettingsKey = chromeSettingsKey();
|
|
onAnySettingChanged(() => {
|
|
const key = chromeSettingsKey();
|
|
if (key === this.lastChromeSettingsKey) return;
|
|
this.lastChromeSettingsKey = key;
|
|
this.config.requestShellRender();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* The chrome found, mid-paint, that the bottom switcher's height changed.
|
|
* The controller moves or resizes windows once the paint is over: a
|
|
* window's relayout repaints it, which mustn't happen inside the shell's
|
|
* own paint.
|
|
*/
|
|
private switcherBarHeightChanged(): void {
|
|
if (this.switcherBarChangePending) return;
|
|
this.switcherBarChangePending = true;
|
|
setTimeout(() => {
|
|
this.switcherBarChangePending = false;
|
|
this.config.onSwitcherBarHeightChanged?.();
|
|
}, 0);
|
|
}
|
|
|
|
/** Add a window (or replace one with the same windowId, keeping its slot). */
|
|
registerWindow(window: ShellWindow): void {
|
|
const existing = this.windows.findIndex((w) => w.windowId === window.windowId);
|
|
if (existing >= 0) {
|
|
this.windows[existing] = window;
|
|
if (existing === this.selectedIndex) this.noteWindowVisible(window.windowId);
|
|
} else {
|
|
this.windows.push(window);
|
|
if (this.windows.length - 1 === this.selectedIndex) this.noteWindowVisible(window.windowId);
|
|
}
|
|
this.config.onWindowsChanged?.();
|
|
}
|
|
|
|
/** Move a window to the front of the most-recently-visible order. */
|
|
private noteWindowVisible(windowId: string): void {
|
|
this.mruWindowIds = this.mruWindowIds.filter((id) => id !== windowId);
|
|
this.mruWindowIds.unshift(windowId);
|
|
}
|
|
|
|
/** Index of the most recently visible window still in the registry, or -1. */
|
|
private mostRecentWindowIndex(): number {
|
|
for (const id of this.mruWindowIds) {
|
|
const index = this.windows.findIndex((w) => w.windowId === id);
|
|
if (index >= 0) return index;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
removeWindow(windowId: string): void {
|
|
const index = this.windows.findIndex((w) => w.windowId === windowId);
|
|
if (index < 0) return;
|
|
// Closing the window under a selected status bar item (from the system
|
|
// menu) takes the selection with it.
|
|
if (this.selectedStatus?.host.windowId === windowId) this.selectedStatus = null;
|
|
const wasSelected = index === this.selectedIndex;
|
|
this.windows.splice(index, 1);
|
|
this.attention.delete(windowId);
|
|
this.mruWindowIds = this.mruWindowIds.filter((id) => id !== windowId);
|
|
if (wasSelected) {
|
|
// Return to the most recently visible remaining window.
|
|
const mruIndex = this.mostRecentWindowIndex();
|
|
this.selectedIndex =
|
|
mruIndex >= 0 ? mruIndex : Math.min(index, Math.max(0, this.windows.length - 1));
|
|
} else if (this.selectedIndex > index) {
|
|
this.selectedIndex--;
|
|
}
|
|
if (this.focus === "window" && (wasSelected || !this.windows.length)) {
|
|
this.focus = "sidebar";
|
|
}
|
|
if (wasSelected) {
|
|
// Hand the foreground to whatever is now selected.
|
|
const next = this.windows[this.selectedIndex];
|
|
if (next) {
|
|
this.noteWindowVisible(next.windowId);
|
|
next.setForeground?.(true);
|
|
next.requestRender();
|
|
}
|
|
}
|
|
this.config.onWindowsChanged?.();
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
/** Close a window by id, if it is closeable (menu actions route here). */
|
|
closeWindow(windowId: string): void {
|
|
const window = this.windows.find((w) => w.windowId === windowId);
|
|
if (!window || !window.closeable) return;
|
|
try {
|
|
window.close?.();
|
|
} catch (error) {
|
|
console.warn(`window ${window.windowId} close failed`, error);
|
|
}
|
|
this.removeWindow(window.windowId);
|
|
}
|
|
|
|
/** Close the foreground window via the shell (escape menu action). */
|
|
closeForegroundWindow(): void {
|
|
const window = this.foregroundWindow();
|
|
if (window) this.closeWindow(window.windowId);
|
|
}
|
|
|
|
getWindows(): readonly ShellWindow[] {
|
|
return this.windows;
|
|
}
|
|
|
|
setWindowAttention(windowId: string, attention: boolean): void {
|
|
if (Boolean(this.attention.get(windowId)) === attention) return;
|
|
this.attention.set(windowId, attention);
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
setBatteryLevels(levels: Partial<ShellChromeState["battery"]>): void {
|
|
const next = { ...this.battery, ...levels };
|
|
const keys = Object.keys(next) as (keyof ShellChromeState["battery"])[];
|
|
if (keys.every((key) => next[key] === this.battery[key])) return;
|
|
this.battery = next;
|
|
for (const listener of Array.from(this.batteryListeners)) listener();
|
|
}
|
|
|
|
/** Called after any battery level or charging state changes. Returns an unsubscribe function. */
|
|
onBatteryLevelsChanged(listener: () => void): () => void {
|
|
this.batteryListeners.add(listener);
|
|
return () => {
|
|
this.batteryListeners.delete(listener);
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Set or clear an app's top-bar tray icon (a small grayscale image, drawn
|
|
* between the notification icons and the battery indicators). Small and
|
|
* infrequently updated by design; not a framebuffer. With `widget`, the
|
|
* switcher's selection can move onto it. A widget that goes, or stops
|
|
* being one, while the selection is on it leaves the selection on its
|
|
* window, as an ordinary window.
|
|
*/
|
|
setTrayIcon(ownerId: string, icon: GrayImage | null, widget?: TrayWidget): void {
|
|
if (icon) {
|
|
this.trayIcons.set(ownerId, { icon, widget });
|
|
} else if (!this.trayIcons.delete(ownerId)) {
|
|
return;
|
|
}
|
|
if ((!icon || !widget) && sameStatusItem(this.selectedStatus?.item, { kind: "widget", key: ownerId })) {
|
|
this.endStatusSelection(true);
|
|
}
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
isScreenOn(): boolean {
|
|
return this.screenOn;
|
|
}
|
|
|
|
/** Current G2, R1, and Wear OS watch battery levels (top bar, Glanceboard, assistant). */
|
|
getBatteryLevels(): ShellChromeState["battery"] {
|
|
return this.battery;
|
|
}
|
|
|
|
/**
|
|
* The foreground app, or null when only the launcher is showing. The launcher
|
|
* is a pinned window but counts as "no app" for the assistant's context.
|
|
*/
|
|
getForegroundApp(): { appId: string; title: string } | null {
|
|
const window = this.foregroundWindow();
|
|
if (!window || window.appId === "launcher") return null;
|
|
return { appId: window.appId, title: window.title };
|
|
}
|
|
|
|
noteUserActivity(nowMs = Date.now()): void {
|
|
this.lastInputAtMs = nowMs;
|
|
}
|
|
|
|
/** Where input currently goes: the sidebar strip or the foreground window. */
|
|
getFocus(): FocusKind {
|
|
return this.focus;
|
|
}
|
|
|
|
/** True while a shell overlay (menu, dialog, voice input) is above the chrome. */
|
|
hasOverlay(): boolean {
|
|
return !this.stack.isAtBase();
|
|
}
|
|
|
|
/** The window whose sidebar icon is at screen (x, y), for mirror touches. */
|
|
windowAtSidebarPoint(x: number, y: number): ShellWindow | null {
|
|
const windows = this.switcherWindows();
|
|
const index = this.chrome.windowIndexAt(x, y, windows.length);
|
|
return index === null ? null : windows[index] ?? null;
|
|
}
|
|
|
|
/**
|
|
* A touch from the phone's mirror on a selectable status bar item's icon
|
|
* (a notification's or an app widget's): select it and go into its
|
|
* window, as a touch on a window's icon focuses that window. False when
|
|
* there is no such icon at (x, y).
|
|
*/
|
|
focusStatusItemAt(x: number, y: number): boolean {
|
|
const items = this.selectableStatusItems();
|
|
const touched = this.chrome.statusItemAt(x, y);
|
|
const index = items.findIndex((item) => sameStatusItem(item, touched));
|
|
if (index < 0) return false;
|
|
this.selectStatusItem(items[index]!, index);
|
|
const selected = this.selectedStatus;
|
|
const host = selected && sameStatusItem(selected.item, touched) ? selected.host : null;
|
|
if (!host) return false;
|
|
const alreadyFocused = this.isFocusTarget(host);
|
|
this.focus = "window";
|
|
if (!alreadyFocused) host.onFocus?.(this.lastInput);
|
|
host.requestRender();
|
|
this.syncInputFocus();
|
|
return true;
|
|
}
|
|
|
|
/** Whether input focus currently targets this window (regardless of screen state). */
|
|
private isFocusTarget(window: ShellWindow | undefined): boolean {
|
|
return !!window && this.focus === "window" && this.foregroundWindow() === window;
|
|
}
|
|
|
|
/**
|
|
* The window ordinary input reaches right now: the foreground window with
|
|
* focus in-window, the screen on, and no shell overlay (system menu,
|
|
* notification modal, voice dialog, alert) above it.
|
|
*/
|
|
private inputTargetWindow(): ShellWindow | undefined {
|
|
if (!this.screenOn || this.focus !== "window" || !this.stack.isAtBase() || this.activeVoiceLayer) {
|
|
return undefined;
|
|
}
|
|
return this.foregroundWindow();
|
|
}
|
|
|
|
/**
|
|
* Tell windows when they gain or lose input focus (ShellWindow.setInputFocus).
|
|
* Called from every shell render request and the input/wake/sleep paths,
|
|
* which between them follow every focus-affecting state change; it diffs
|
|
* against the last notification, so calling it often is cheap.
|
|
*/
|
|
private syncInputFocus(): void {
|
|
const target = this.inputTargetWindow();
|
|
const targetId = target?.windowId ?? null;
|
|
if (targetId === this.inputFocusedWindowId) return;
|
|
const previous = this.windows.find((w) => w.windowId === this.inputFocusedWindowId);
|
|
this.inputFocusedWindowId = targetId;
|
|
previous?.setInputFocus?.(false);
|
|
target?.setInputFocus?.(true);
|
|
}
|
|
|
|
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
|
|
wake(focus: FocusKind, nowMs = Date.now()): boolean {
|
|
this.lastInputAtMs = nowMs;
|
|
const gaining = focus === "window" ? this.foregroundWindow() : undefined;
|
|
const alreadyFocused = this.isFocusTarget(gaining);
|
|
this.focus = focus;
|
|
if (gaining && !alreadyFocused) gaining.onFocus?.(this.lastInput);
|
|
if (this.screenOn) {
|
|
this.syncInputFocus();
|
|
return false;
|
|
}
|
|
this.screenOn = true;
|
|
this.config.onScreenStateChanged(true);
|
|
for (const window of this.windows) {
|
|
window.setScreenOn?.(true);
|
|
}
|
|
// Refresh the foreground window; the compositor restored its retained
|
|
// frame, but its content may be stale (e.g. a running stopwatch).
|
|
this.foregroundWindow()?.requestRender();
|
|
this.syncInputFocus();
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Turn the screen off, closing any shell overlays. Sidebar selection is
|
|
* kept, except that one on a status bar item goes back to its window.
|
|
*/
|
|
sleep(): void {
|
|
if (!this.screenOn) return;
|
|
this.returnFromStatusItem();
|
|
this.cancelEscapeMenuTimer();
|
|
this.screenOn = false;
|
|
this.stack.clearToBase();
|
|
for (const window of this.windows) {
|
|
window.setScreenOn?.(false);
|
|
}
|
|
this.syncInputFocus();
|
|
this.config.onScreenStateChanged(false);
|
|
}
|
|
|
|
/** Foreground and focus a window by id (e.g. a wake path opening content in it). */
|
|
focusWindow(windowId: string): void {
|
|
const index = this.windows.findIndex((w) => w.windowId === windowId);
|
|
if (index < 0) return;
|
|
const target = this.windows[index];
|
|
const alreadyFocused = this.isFocusTarget(target);
|
|
this.setSelectedIndex(index);
|
|
// Asked for by id, a status bar item's window stays, whether or not it
|
|
// was opened for the selection.
|
|
this.endStatusSelection(target === this.selectedStatus?.host);
|
|
this.focus = "window";
|
|
if (!alreadyFocused) target?.onFocus?.(this.lastInput);
|
|
this.syncInputFocus();
|
|
}
|
|
|
|
/** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */
|
|
applyScreenTimeout(nowMs = Date.now()): boolean {
|
|
const timeoutMs = this.config.getScreenTimeoutMs();
|
|
if (timeoutMs === null || !this.screenOn) return false;
|
|
// An open voice dialog suspends the timeout: a long dictation or refine
|
|
// has no button presses, but the screen must stay on for it. Sliding
|
|
// lastInputAtMs forward also restarts the full timeout when it closes.
|
|
// The keyboard dialog likewise: typing happens on the phone, not the
|
|
// ring. An in-flight assistant turn suspends it for the same reason (a
|
|
// tool loop can run for a while with no input); once the turn ends and
|
|
// the Done/Follow-up menu is showing, the normal idle timeout resumes.
|
|
// A foreground window that is playing video holds it off the same way.
|
|
const foreground = this.foregroundWindow();
|
|
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
|
|
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
|
|
this.lastInputAtMs = nowMs;
|
|
return false;
|
|
}
|
|
if (nowMs - this.lastInputAtMs < timeoutMs) return false;
|
|
this.sleep();
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Show a new notification in a shell modal over the app viewport, or queue
|
|
* it behind the one already open (see NotificationModalQueue). If
|
|
* notifications woke the screen, closing the last modal goes back to sleep
|
|
* (matching the old sleep-popup behavior).
|
|
*/
|
|
openNotificationModal(notificationKey: string, wokeScreen: boolean): void {
|
|
if (!this.screenOn) return;
|
|
this.notificationModals.post(notificationKey, wokeScreen);
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
/**
|
|
* A notification went away on the phone: cancelled by its source (e.g. a
|
|
* placeholder replaced moments later by the real notification under another
|
|
* key), or dismissed. A popup queued for it is dropped, and one showing it
|
|
* closes so the next one opens, as if the user had closed it -- unless it
|
|
* is asking to confirm turning off its source, which outlives the
|
|
* notification it dismissed.
|
|
*/
|
|
notificationRemoved(notificationKey: string): void {
|
|
const popup = this.notificationModals.withdrawn(notificationKey);
|
|
if (popup && !popup.content?.outlivesNotification()) {
|
|
this.closeNotificationModal(popup, notificationKey);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Show an app's own notification the way new Android notifications are
|
|
* shown: in the notification modal, waking the screen for it if it was off.
|
|
*/
|
|
postAppNotification(notification: AppNotification): void {
|
|
this.appNotifications.set(notification.key, notification);
|
|
const wokeScreen = this.screenOn ? false : this.wake("sidebar");
|
|
this.openNotificationModal(notification.key, wokeScreen);
|
|
}
|
|
|
|
private pushNotificationModal(notificationKey: string): NotificationPopup {
|
|
const appNotification = this.appNotifications.get(notificationKey);
|
|
const close = () => this.closeNotificationModal(popup, notificationKey);
|
|
const content = appNotification
|
|
? null
|
|
: new SingleNotificationLayer(notificationKey, { origin: "new-notification-modal", onClose: close });
|
|
// A phone notification's popup closes once left untouched for the
|
|
// "Display new notifications" time, as if closed by hand: the notification
|
|
// stays on the phone and in the status bar. An app's own notification
|
|
// (e.g. a Nightscout alert) is nowhere else to be found, so it stays up.
|
|
const timeout = new NotificationPopupTimeout(
|
|
appNotification ? null : newNotificationDisplayTimeoutMs(newNotificationDisplaySetting.get()),
|
|
() => {
|
|
// Something over the popup (a voice dialog, the system menu) is in
|
|
// use; count that as interaction rather than closing underneath it.
|
|
if (this.stack.topMatches((layer) => layer === popup.modal)) close();
|
|
else timeout.restart();
|
|
},
|
|
);
|
|
const popup: NotificationPopup = {
|
|
content,
|
|
modal: new ShellModalLayer(
|
|
appNotification ? appNotification.createLayer(close) : content!,
|
|
this.config.actions,
|
|
{
|
|
removed: () => {
|
|
timeout.stop();
|
|
this.appNotifications.delete(notificationKey);
|
|
this.notificationModals.removed(popup, this.screenOn);
|
|
},
|
|
input: () => timeout.restart(),
|
|
},
|
|
),
|
|
};
|
|
this.stack.push(popup.modal);
|
|
timeout.restart();
|
|
return popup;
|
|
}
|
|
|
|
private closeNotificationModal(popup: NotificationPopup, notificationKey: string): void {
|
|
this.appNotifications.delete(notificationKey);
|
|
// The queue first: it opens the next notification or puts the screen back
|
|
// to sleep, and then ignores the removal below. remove() rather than a
|
|
// pop, so a menu opened over the modal doesn't leave it stuck.
|
|
this.notificationModals.closed(popup);
|
|
this.stack.remove(popup.modal);
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
/** Called by a window when the user backs out of its root (double-tap). */
|
|
yieldFocusToSidebar(): void {
|
|
if (this.focus === "sidebar") return;
|
|
this.focus = "sidebar";
|
|
// Repaint the window so its selection highlight dims to the unfocused
|
|
// style this frame.
|
|
this.foregroundWindow()?.requestRender();
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
/**
|
|
* Paint the shell surface: transparent chrome, or all-transparent when
|
|
* asleep. The scene also carries the whole display's stereo depth (none
|
|
* under a Full Screen or immersive window).
|
|
*/
|
|
paintScene(): Uint8Array {
|
|
return this.screenOn
|
|
? encodeShellScene(this.stack.paintUndimmed(), uiDepth(this.foregroundWindow()?.appId))
|
|
: encodeShellScene([]);
|
|
}
|
|
|
|
paintSurface(): Plane[] {
|
|
if (!this.screenOn) {
|
|
return singlePlane(new GrayImage(screenWidth(), screenHeight(), 0));
|
|
}
|
|
return this.stack.paint();
|
|
}
|
|
|
|
/**
|
|
* The brightness factor a shell overlay (a context menu) currently applies
|
|
* to what lies beneath the shell surface, i.e. the window surfaces: 1 when
|
|
* nothing dims them. Read after paintSurface; the controller forwards it to
|
|
* the compositor, which cannot see the shell's layer stack.
|
|
*/
|
|
underlayDim(): number {
|
|
const dim = this.stack.baseDim();
|
|
return this.screenOn && dim !== false ? dim : 1;
|
|
}
|
|
|
|
async receiveInput(event: InputEvent, frameId = 0): Promise<ShellInputOutcome> {
|
|
if (!acceptInput(event)) return { shell: false, window: false };
|
|
try {
|
|
return await this.routeInput(event, frameId);
|
|
} finally {
|
|
// Whatever the input did (opened an overlay, moved focus, slept the
|
|
// screen), windows learn about the resulting input-focus change.
|
|
this.syncInputFocus();
|
|
}
|
|
}
|
|
|
|
private async routeInput(event: InputEvent, frameId: number): Promise<ShellInputOutcome> {
|
|
// Touch-down supplements gestures. It must not wake the screen, operate
|
|
// menus, or cancel the hold-to-escape timer. Apps can opt into it later.
|
|
if (event.type === "ring-press") {
|
|
const window = this.foregroundWindow();
|
|
if (this.screenOn && this.focus === "window" && this.stack.isAtBase() && window) {
|
|
await window.handleInput(event, frameId);
|
|
return { shell: false, window: true };
|
|
}
|
|
return { shell: false, window: false };
|
|
}
|
|
const previous = this.lastInput;
|
|
this.lastInput = event;
|
|
// A visible window may paint a source-dependent indicator (see
|
|
// lastInputEvent); when the source class flips (watch <-> ring/arms),
|
|
// repaint it so the indicator follows the device now in use.
|
|
if (this.screenOn && previous && isWatchInput(previous) !== isWatchInput(event)) {
|
|
this.foregroundWindow()?.requestRender();
|
|
}
|
|
// The stock lifecycle has already interpreted the physical double tap as
|
|
// "wake". Keep that directionality if delivery is delayed or duplicated.
|
|
if (event.type === "display-wake") {
|
|
this.lastInputAtMs = Date.now();
|
|
const wokeScreen = !this.screenOn && this.wake(this.userWakeFocus());
|
|
return { shell: wokeScreen, window: false };
|
|
}
|
|
|
|
// The wakeword is handled before the screen-off short-circuit so its
|
|
// configured action can work from a dark screen. With the CFW the stock
|
|
// Even AI app never launches, so the firmware does not power the display
|
|
// for us either -- actions that need it wake the screen themselves.
|
|
if (event.type === "wakeword") {
|
|
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
|
|
const action = wakeWordActionSetting.get();
|
|
if (action === "off") {
|
|
return { shell: false, window: false };
|
|
}
|
|
|
|
this.lastInputAtMs = Date.now();
|
|
const wokeScreen = !this.screenOn && this.wake("sidebar");
|
|
if (action === "voice-input" && !this.activeVoiceLayer && !this.activeKeyboardLayer) {
|
|
if (this.assistantLayer) {
|
|
// The assistant overlay is up; a wakeword continues that conversation.
|
|
this.startAssistantFollowUp(true);
|
|
} else {
|
|
// Wakeword defaults the highlight to Send to Assistant.
|
|
this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" });
|
|
}
|
|
}
|
|
return { shell: wokeScreen || action === "voice-input", window: false };
|
|
}
|
|
|
|
this.lastInputAtMs = Date.now();
|
|
|
|
// Anything but the long-press itself means the press ended (or the event
|
|
// stream moved on), so the escape countdown stops.
|
|
if (event.type !== "long-press") {
|
|
this.cancelEscapeMenuTimer();
|
|
}
|
|
|
|
if (!this.screenOn) {
|
|
if (event.type === "double-click") {
|
|
this.wake(this.userWakeFocus());
|
|
return { shell: true, window: false };
|
|
}
|
|
return { shell: false, window: false };
|
|
}
|
|
|
|
// Long-press is the shell's own gesture: it opens the system menu
|
|
// directly, never reaching the app — over the app's own context menu
|
|
// too (the window closes that on system-menu-opened), while an already
|
|
// open system menu just stays. Its later generic release is consumed
|
|
// while the shell overlay is active and therefore cannot leak into the
|
|
// app. The exception is a window that claims long-press for a move of
|
|
// its own: it gets the press forwarded, with the escape timer running
|
|
// so that holding the press long enough still opens the system menu.
|
|
if (event.type === "long-press") {
|
|
if (this.activeVoiceLayer || !this.stack.isAtBase()) {
|
|
return { shell: true, window: false };
|
|
}
|
|
const window = this.foregroundWindow();
|
|
if (!window) {
|
|
return { shell: true, window: false };
|
|
}
|
|
if (!window.holdToTalk && !window.claimsLongPress?.()) {
|
|
this.openEscapeMenu();
|
|
return { shell: true, window: false };
|
|
}
|
|
if (!window.holdToTalk) this.startEscapeMenuTimer();
|
|
if (this.focus === "sidebar") {
|
|
this.focus = "window";
|
|
window.onFocus?.(this.lastInput);
|
|
}
|
|
// The window owns frameId from here (render or explicit finish).
|
|
await window.handleInput(event, frameId);
|
|
return { shell: true, window: true };
|
|
}
|
|
|
|
// The 2.2.9 tap-then-hold gesture is the app's context-menu gesture: it
|
|
// goes to the foreground window (from the sidebar it focuses the window
|
|
// first), which answers with its own menu or asks for the system menu
|
|
// when it has none.
|
|
if (event.type === "short-then-long-press") {
|
|
const window = this.foregroundWindow();
|
|
if (window?.holdToTalk) {
|
|
this.openEscapeMenu();
|
|
return { shell: true, window: false };
|
|
}
|
|
// Over the open system menu it switches to the app's context menu:
|
|
// close the system menu and deliver the gesture to the window as
|
|
// usual. A window without a menu of its own would only ask for the
|
|
// system menu back, so for it the open menu stays.
|
|
if (
|
|
!this.activeVoiceLayer &&
|
|
window?.hasAppMenu?.() &&
|
|
this.stack.popIfTop((layer) => layer instanceof ShellOverlayMenuLayer)
|
|
) {
|
|
this.config.requestShellRender();
|
|
}
|
|
if (this.activeVoiceLayer || !this.stack.isAtBase()) {
|
|
return { shell: true, window: false };
|
|
}
|
|
if (!window) {
|
|
return { shell: true, window: false };
|
|
}
|
|
if (this.focus === "sidebar") {
|
|
this.focus = "window";
|
|
window.onFocus?.(this.lastInput);
|
|
}
|
|
// The window owns frameId from here (render or explicit finish).
|
|
await window.handleInput(event, frameId);
|
|
return { shell: true, window: true };
|
|
}
|
|
if (event.type === "long-press-release") {
|
|
this.activeVoiceLayer?.endCapture();
|
|
if (this.activeVoiceLayer || !this.stack.isAtBase() || this.focus !== "window") {
|
|
return { shell: true, window: false };
|
|
}
|
|
const window = this.foregroundWindow();
|
|
if (!window) {
|
|
return { shell: true, window: false };
|
|
}
|
|
await window.handleInput(event, frameId);
|
|
return { shell: false, window: true };
|
|
}
|
|
|
|
if (!this.stack.isAtBase()) {
|
|
await this.stack.handleInput(event);
|
|
return { shell: true, window: false };
|
|
}
|
|
|
|
if (this.focus === "sidebar") {
|
|
return this.handleSidebarInput(event);
|
|
}
|
|
|
|
const window = this.foregroundWindow();
|
|
if (window) {
|
|
// The window owns frameId from here (render or explicit finish).
|
|
const delivered = isDirectionalInput(event) && !window.acceptsDirectional ? directionalFallback(event) : event;
|
|
await window.handleInput(delivered, frameId);
|
|
return { shell: false, window: true };
|
|
}
|
|
return { shell: false, window: false };
|
|
}
|
|
|
|
/** Where a user's double-tap wake lands: the last app, or the switcher (setting). */
|
|
userWakeFocus(): FocusKind {
|
|
return wakeToLastAppSetting.get() && this.foregroundWindow() ? "window" : "sidebar";
|
|
}
|
|
|
|
foregroundWindow(): ShellWindow | undefined {
|
|
return this.windows[this.selectedIndex] ?? this.windows[0];
|
|
}
|
|
|
|
/**
|
|
* Screen rect actually occupied by content, for cropping screenshots: the
|
|
* foreground window's band (full height for a max-height window, the
|
|
* vertical-position-dependent 288px band otherwise) plus the switcher: a
|
|
* side strip's icon columns (minus the outer dead strip the one-column
|
|
* variant leaves), or the row under it, and the window frame's sides. A
|
|
* popup switcher lies within the band.
|
|
*/
|
|
screenshotCropRect(): { x: number; y: number; width: number; height: number } {
|
|
const appId = this.foregroundWindow()?.appId;
|
|
const heightMode = this.foregroundWindow()?.heightMode ?? "min";
|
|
const viewport = appViewportRect(heightMode, appId);
|
|
let left = viewport.x, right = viewport.x + viewport.width;
|
|
if (sidebarWidth(appId) > 0) {
|
|
const strip = sidebarContentSpan(this.windows.length);
|
|
left = Math.min(left, strip.left);
|
|
right = Math.max(right, strip.right);
|
|
}
|
|
if (windowFramed(appId)) {
|
|
left -= 1;
|
|
right += 1;
|
|
}
|
|
const y = windowTop(heightMode, appId);
|
|
const bottom = y + windowBandHeight(heightMode, appId) + switcherRowHeight(appId);
|
|
return { x: left, y, width: right - left, height: bottom - y };
|
|
}
|
|
|
|
/**
|
|
* Compact description of where input is currently going, for the frame
|
|
* timing export: "which app was drawing" is usually the first thing you need
|
|
* to interpret an input-to-display latency, and it is not recoverable from
|
|
* the input event itself.
|
|
*/
|
|
describeInputTarget(): string {
|
|
const foreground = this.foregroundWindow()?.windowId ?? "none";
|
|
if (!this.screenOn) return `fg=${foreground} target=screen-off`;
|
|
if (this.activeVoiceLayer) return `fg=${foreground} target=voice`;
|
|
if (this.activeKeyboardLayer?.shownOnGlasses) return `fg=${foreground} target=keyboard`;
|
|
if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`;
|
|
return `fg=${foreground} target=${this.focus}`;
|
|
}
|
|
|
|
/**
|
|
* The most recent input event received, from any source. Windows whose
|
|
* appearance depends on the input device in use (e.g. the launcher's
|
|
* defocused selection preview) can read it at paint time; the shell
|
|
* repaints the foreground window whenever the source class changes, so
|
|
* such paints stay current even while the window is not focused.
|
|
*/
|
|
lastInputEvent(): InputEvent | null {
|
|
return this.lastInput;
|
|
}
|
|
|
|
/** Whether the most recent input came from the watch (see lastInputEvent). */
|
|
lastInputWasWatch(): boolean {
|
|
return this.lastInput !== null && isWatchInput(this.lastInput);
|
|
}
|
|
|
|
/** Whether a window is the current input target (foreground + focus in-window). */
|
|
isWindowFocused(windowId: string): boolean {
|
|
return this.screenOn && this.focus === "window" && this.foregroundWindow()?.windowId === windowId;
|
|
}
|
|
|
|
/**
|
|
* Whether a window's content is on screen: it's the foreground window and the
|
|
* screen is on. Focus-independent — the app viewport stays visible while the
|
|
* sidebar is focused (the sidebar is just the left strip).
|
|
*/
|
|
isWindowVisible(windowId: string): boolean {
|
|
return this.screenOn && this.foregroundWindow()?.windowId === windowId;
|
|
}
|
|
|
|
private handleSidebarInput(event: InputEvent): ShellInputOutcome {
|
|
const type = isDirectionalInput(event) ? SWITCHER_SWIPES[switcherPosition()][event.type] : event.type;
|
|
switch (type) {
|
|
case "double-click":
|
|
// A double-tap at the root (the app switcher selected) turns the
|
|
// display off — from the ring and the watch scheme alike; watch-scheme
|
|
// gestures wake it again (or a ring double-tap does).
|
|
this.sleep();
|
|
return { shell: true, window: false };
|
|
case "scroll-up":
|
|
this.moveSelection(-1);
|
|
return { shell: true, window: false };
|
|
case "scroll-down":
|
|
this.moveSelection(1);
|
|
return { shell: true, window: false };
|
|
case "click":
|
|
if (this.windows.length) {
|
|
this.focus = "window";
|
|
this.foregroundWindow()?.onFocus?.(this.lastInput);
|
|
// Repaint the window now so its selection highlight reflects focus
|
|
// this frame, not one frame late.
|
|
this.foregroundWindow()?.requestRender();
|
|
}
|
|
return { shell: true, window: false };
|
|
default:
|
|
return { shell: false, window: false };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Step the switcher's selection through the windows and then, with a
|
|
* bottom switcher, on through the status bar's selectable items whose
|
|
* icons are showing (notifications, then app widgets), wrapping round at
|
|
* either end.
|
|
*/
|
|
private moveSelection(delta: number): void {
|
|
const windows = this.switcherWindows();
|
|
const items = this.selectableStatusItems();
|
|
const count = windows.length + items.length;
|
|
if (!count) return;
|
|
let current = this.selectedIndex;
|
|
const selected = this.selectedStatus;
|
|
if (selected) {
|
|
// Once its item has gone, the selection stands between the
|
|
// neighbours it left.
|
|
const at = items.findIndex((item) => sameStatusItem(item, selected.item));
|
|
current = windows.length + (at >= 0 ? at : Math.min(selected.index, items.length) - (delta > 0 ? 1 : 0));
|
|
}
|
|
const next = (((current + delta) % count) + count) % count;
|
|
if (next < windows.length) {
|
|
this.selectSwitcherWindow(windows[next]!);
|
|
} else {
|
|
this.selectStatusItem(items[next - windows.length]!, next - windows.length);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Windows with an icon in the switcher: all of them, except a window
|
|
* opened just to show a selected status bar item.
|
|
*/
|
|
private switcherWindows(): ShellWindow[] {
|
|
const hidden = this.selectedStatus?.transient ? this.selectedStatus.host : null;
|
|
return hidden ? this.windows.filter((window) => window !== hidden) : this.windows;
|
|
}
|
|
|
|
/**
|
|
* The status bar items the switcher's selection can move onto, in order:
|
|
* with a bottom switcher (the selection reaches them past the last
|
|
* window), those whose icons are showing that have a window to show,
|
|
* which notifications have where the host gives the Notifications window,
|
|
* and app widgets always do.
|
|
*/
|
|
private selectableStatusItems(): StatusItem[] {
|
|
if (switcherPosition() !== "bottom") return [];
|
|
return this.chrome.statusItems().filter((item) => item.kind === "notification"
|
|
? !!this.config.openNotificationsWindow
|
|
: !!this.trayIcons.get(item.key)?.widget);
|
|
}
|
|
|
|
/** Move the switcher's selection onto a window, off a status bar item if it was on one. */
|
|
private selectSwitcherWindow(window: ShellWindow): void {
|
|
// Foreground first, so a status bar item's window closing with its
|
|
// selection can't hand the foreground elsewhere first.
|
|
this.setSelectedIndex(this.windows.indexOf(window));
|
|
this.endStatusSelection(window === this.selectedStatus?.host);
|
|
}
|
|
|
|
/**
|
|
* Move the switcher's selection onto a status bar item: its window
|
|
* (opened in the background first, if need be) comes to the foreground,
|
|
* a notification's showing its detail view. The window the selection was
|
|
* on before is left as endStatusSelection leaves it, unless it is this
|
|
* one too.
|
|
*/
|
|
private selectStatusItem(item: StatusItem, index: number): void {
|
|
const previous = this.selectedStatus;
|
|
if (previous && sameStatusItem(previous.item, item)) {
|
|
previous.index = index;
|
|
return;
|
|
}
|
|
const returnWindowId = previous ? previous.returnWindowId : (this.foregroundWindow()?.windowId ?? null);
|
|
const opened = this.openStatusItemWindow(item);
|
|
const hostIndex = opened ? this.windows.indexOf(opened.window) : -1;
|
|
if (!opened || hostIndex < 0) return;
|
|
const host = opened.window;
|
|
const sameHost = previous?.host === host;
|
|
if (sameHost && previous.detail) previous.detail.stack.popThrough(previous.detail.layer);
|
|
const detail = item.kind === "notification" && opened.stack
|
|
? { stack: opened.stack, layer: this.notificationDetailLayer(item.key) } : null;
|
|
if (detail) detail.stack.push(detail.layer);
|
|
this.selectedStatus = {
|
|
item, index, host, detail, returnWindowId,
|
|
transient: sameHost ? previous.transient : opened.opened,
|
|
};
|
|
if (hostIndex === this.selectedIndex) {
|
|
host.requestRender();
|
|
} else {
|
|
this.setSelectedIndex(hostIndex);
|
|
}
|
|
if (previous && !sameHost) this.releaseStatusWindow(previous, false);
|
|
}
|
|
|
|
/**
|
|
* The window a status bar item shows in, opened in the background unless
|
|
* it is open already (`opened` says this call opened it), and for a
|
|
* notification, the stack its detail view goes on.
|
|
*/
|
|
private openStatusItemWindow(item: StatusItem): { window: ShellWindow; opened: boolean; stack?: LayerStack } | null {
|
|
if (item.kind === "widget") return this.trayIcons.get(item.key)?.widget?.openWindow() ?? null;
|
|
const opened = this.config.openNotificationsWindow?.();
|
|
return opened ? { window: opened.window.window, opened: opened.opened, stack: opened.window.stack } : null;
|
|
}
|
|
|
|
/**
|
|
* Take the switcher's selection off its status bar item, leaving its
|
|
* window as releaseStatusWindow does.
|
|
*/
|
|
private endStatusSelection(keepWindow = false): void {
|
|
const selected = this.selectedStatus;
|
|
if (!selected) return;
|
|
this.selectedStatus = null;
|
|
this.releaseStatusWindow(selected, keepWindow);
|
|
}
|
|
|
|
/**
|
|
* A status bar item's window, once the selection has left the item: it
|
|
* drops a notification's detail view (and anything opened over it), and
|
|
* closes if it was opened for the selection, unless `keepWindow` (it is
|
|
* the window being selected instead).
|
|
*/
|
|
private releaseStatusWindow(selection: StatusSelection, keepWindow: boolean): void {
|
|
if (selection.detail) selection.detail.stack.popThrough(selection.detail.layer);
|
|
if (selection.transient && !keepWindow) {
|
|
this.closeWindow(selection.host.windowId);
|
|
return;
|
|
}
|
|
selection.host.requestRender();
|
|
// A window opened for the selection is now one with an icon of its own.
|
|
if (selection.transient) this.config.onWindowsChanged?.();
|
|
}
|
|
|
|
/** End a status bar item selection by going back to the window selected before it. */
|
|
private returnFromStatusItem(): void {
|
|
const selected = this.selectedStatus;
|
|
if (!selected) return;
|
|
const back = this.windows.find((window) => window.windowId === selected.returnWindowId);
|
|
if (back) this.setSelectedIndex(this.windows.indexOf(back));
|
|
this.endStatusSelection(back === selected.host);
|
|
}
|
|
|
|
private notificationDetailLayer(key: string): SingleNotificationLayer {
|
|
const layer: SingleNotificationLayer = new SingleNotificationLayer(key, {
|
|
origin: "notification-tray",
|
|
onClose: (_ctx, gone) => this.notificationDetailClosed(layer, gone),
|
|
});
|
|
return layer;
|
|
}
|
|
|
|
/**
|
|
* A selected notification's detail view asked to close. Back (or a
|
|
* double-tap) hands focus back to the switcher, its selection still on
|
|
* the notification. Once the notification is gone (dismissed, or removed
|
|
* by an action), the selection also moves on to the status bar item that
|
|
* took its place, or back to the windows when there are none left; that
|
|
* waits for the current paint to finish, since a vanished notification
|
|
* closes its view from inside its paint.
|
|
*/
|
|
private notificationDetailClosed(layer: SingleNotificationLayer, gone: boolean): void {
|
|
if (!gone) {
|
|
if (this.selectedStatus?.detail?.layer === layer) this.yieldFocusToSidebar();
|
|
return;
|
|
}
|
|
setTimeout(() => {
|
|
const selected = this.selectedStatus;
|
|
if (selected?.detail?.layer !== layer) return;
|
|
this.focus = "sidebar";
|
|
const items = this.selectableStatusItems().filter((item) => !sameStatusItem(item, selected.item));
|
|
const windows = this.switcherWindows();
|
|
if (items.length) {
|
|
const index = Math.min(selected.index, items.length - 1);
|
|
this.selectStatusItem(items[index]!, index);
|
|
} else if (windows.length) {
|
|
this.selectSwitcherWindow(windows[windows.length - 1]!);
|
|
}
|
|
this.config.requestShellRender();
|
|
}, 0);
|
|
}
|
|
|
|
/** Change selection; the selected window is the foreground window. */
|
|
private setSelectedIndex(index: number): void {
|
|
if (index === this.selectedIndex) return;
|
|
const previous = this.windows[this.selectedIndex];
|
|
this.selectedIndex = index;
|
|
const next = this.windows[index];
|
|
if (next) this.noteWindowVisible(next.windowId);
|
|
previous?.setForeground?.(false);
|
|
next?.setForeground?.(true);
|
|
next?.requestRender();
|
|
this.config.onWindowsChanged?.();
|
|
}
|
|
|
|
// True while a voice-dialog open waits on prepareVoiceCapture (e.g. the
|
|
// preview-mode permission prompt); a second tap must not queue another open.
|
|
private voiceDialogPending = false;
|
|
|
|
private openVoiceDialog(options: {
|
|
finishOnClick?: boolean;
|
|
handsFree?: boolean;
|
|
defaultTarget: "assistant" | "app";
|
|
}): void {
|
|
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
|
|
this.voiceDialogPending = true;
|
|
void (async () => {
|
|
let ready = true;
|
|
try {
|
|
ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
|
} catch {
|
|
ready = false;
|
|
} finally {
|
|
this.voiceDialogPending = false;
|
|
}
|
|
// Re-checked after the await: another path may have opened a dialog
|
|
// (or torn down the base state) while a permission prompt was up.
|
|
if (!ready || this.activeVoiceLayer || this.activeKeyboardLayer) return;
|
|
this.openVoiceDialogNow(options);
|
|
this.config.requestShellRender();
|
|
})();
|
|
}
|
|
|
|
private openVoiceDialogNow(options: {
|
|
finishOnClick?: boolean;
|
|
handsFree?: boolean;
|
|
defaultTarget: "assistant" | "app";
|
|
}): void {
|
|
const targets = this.buildVoiceSendTargets();
|
|
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
|
|
if (defaultIndex < 0) defaultIndex = 0;
|
|
// Skip the menu only for a hands-free (wakeword) capture aimed at the
|
|
// assistant, when the user has opted into it.
|
|
const autoSend =
|
|
Boolean(options.handsFree) &&
|
|
options.defaultTarget === "assistant" &&
|
|
targets[defaultIndex]?.id === "assistant" &&
|
|
assistantSkipConfirmationSetting.get();
|
|
|
|
const layer = new VoiceInputLayer({
|
|
actions: this.config.actions,
|
|
onClosed: () => {
|
|
if (this.activeVoiceLayer === layer) {
|
|
this.activeVoiceLayer = null;
|
|
voiceActivity.setActive(false);
|
|
// The idle countdown restarts in full once voice input ends.
|
|
this.noteUserActivity();
|
|
}
|
|
},
|
|
dismiss: () => {
|
|
this.stack.popIfTop((top) => top === layer);
|
|
},
|
|
sendTargets: targets,
|
|
defaultTargetIndex: defaultIndex,
|
|
finishOnClick: options.finishOnClick ?? false,
|
|
handsFree: options.handsFree ?? false,
|
|
autoSend,
|
|
});
|
|
this.activeVoiceLayer = layer;
|
|
voiceActivity.setActive(true);
|
|
this.stack.push(layer);
|
|
layer.startCapture();
|
|
}
|
|
|
|
/**
|
|
* The send destinations offered by the voice dialog: the assistant (when an
|
|
* API key is configured) and/or typing into the foreground window (when it
|
|
* accepts text). Order fixes the menu row order.
|
|
*/
|
|
private buildVoiceSendTargets(): VoiceSendTarget[] {
|
|
const targets: VoiceSendTarget[] = [];
|
|
if (this.isAssistantAvailable()) {
|
|
targets.push({
|
|
id: "assistant",
|
|
label: "Send to Assistant",
|
|
onSend: (text) => this.sendToAssistant(text),
|
|
});
|
|
}
|
|
if (this.foregroundWindow()?.receiveTextInput) {
|
|
targets.push({
|
|
id: "app",
|
|
label: "Type Into App",
|
|
onSend: (text) => this.sendTextToForegroundWindow(text),
|
|
});
|
|
}
|
|
// Guarantee at least one destination so the dialog is never a dead end.
|
|
if (targets.length === 0) {
|
|
targets.push({
|
|
id: "app",
|
|
label: "Type Into App",
|
|
onSend: (text) => this.sendTextToForegroundWindow(text),
|
|
});
|
|
}
|
|
return targets;
|
|
}
|
|
|
|
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
|
|
sendTextToForegroundWindow(text: string, options?: { submit?: boolean }): void {
|
|
this.foregroundWindow()?.receiveTextInput?.(text, options);
|
|
}
|
|
|
|
/**
|
|
* Offer a hardware-keyboard key to the window that has input focus. True
|
|
* when that window takes keys (ShellWindow.handleKeyEvent) and got this
|
|
* one; false when the key is the caller's to interpret: the screen is off,
|
|
* the sidebar or a shell overlay has focus, or the window has no use for
|
|
* keys. Typing counts as activity for the screen timeout.
|
|
*/
|
|
receiveKeyEvent(event: KeyEvent): boolean {
|
|
const window = this.inputTargetWindow();
|
|
if (!window?.handleKeyEvent) return false;
|
|
if (event.phase === "down") this.lastInputAtMs = Date.now();
|
|
window.handleKeyEvent(event);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Open the voice dialog aimed at the foreground window. Called when the
|
|
* user picks Voice input from the system menu (or an app asks via a
|
|
* start-voice-input message). The menu click already ended the press, so
|
|
* the dialog finishes on click instead of long-press-release. Returns
|
|
* false when the dialog can't open now (voice input disabled, screen off,
|
|
* another dialog or overlay up); true means it is opening, though a
|
|
* microphone-permission prompt can still stop it.
|
|
*/
|
|
startVoiceInput(): boolean {
|
|
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return false;
|
|
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return false;
|
|
// The transcript is aimed at the window whose menu requested it; the menu
|
|
// entry point defaults the highlight to Type Into App.
|
|
this.focus = "window";
|
|
this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" });
|
|
this.config.requestShellRender();
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Open the keyboard dialog: the voice dialog's typed twin, driven by the
|
|
* phone's keyboard button (beside the mic button, whose wakeword this
|
|
* mirrors: it wakes a dark screen too). Over the assistant overlay the text
|
|
* continues that conversation; otherwise it goes to the usual destinations
|
|
* with Send to Assistant highlighted. Returns the session the phone types
|
|
* into (the already-open one if there is one), or null while a voice
|
|
* dialog is up. The phone can keep the dialog off the glasses (see
|
|
* KeyboardInputVisibility); that choice persists across dialogs.
|
|
*/
|
|
startKeyboardInput(): KeyboardInputSession | null {
|
|
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
|
|
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
|
|
const storedVisibility = getStringSetting(KEYBOARD_INPUT_VISIBILITY_KEY, "both");
|
|
const visibility: KeyboardInputVisibility =
|
|
storedVisibility === "phone" || storedVisibility === "glasses" ? storedVisibility : "both";
|
|
if (visibility !== "phone" && !this.screenOn) this.wake("sidebar");
|
|
const assistantLayer = this.assistantLayer;
|
|
const assistantSession = this.assistantSession;
|
|
let targets: VoiceSendTarget[];
|
|
let defaultIndex = 0;
|
|
if (assistantLayer && assistantSession) {
|
|
targets = [
|
|
{
|
|
id: "assistant",
|
|
label: "Send",
|
|
// Phone-only, the overlay can close while the dialog is open; the
|
|
// text then starts a fresh overlay on the same conversation.
|
|
onSend: (text) => {
|
|
if (this.assistantLayer === assistantLayer) {
|
|
this.runAssistantTurn(assistantSession, assistantLayer, text);
|
|
} else {
|
|
this.sendToAssistant(text);
|
|
}
|
|
},
|
|
},
|
|
];
|
|
} else {
|
|
targets = this.buildVoiceSendTargets();
|
|
defaultIndex = Math.max(0, targets.findIndex((target) => target.id === "assistant"));
|
|
}
|
|
const layer = new KeyboardInputLayer({
|
|
actions: this.config.actions,
|
|
onClosed: () => {
|
|
if (this.activeKeyboardLayer === layer) {
|
|
this.activeKeyboardLayer = null;
|
|
this.config.onKeyboardInputChanged?.(null);
|
|
// The idle countdown restarts in full once keyboard input ends.
|
|
this.noteUserActivity();
|
|
}
|
|
},
|
|
dismiss: () => {
|
|
// Wherever it sits: a menu opened over the dialog stays open.
|
|
this.stack.remove(layer);
|
|
// A send or discard from the phone side arrives outside the input
|
|
// path, so nothing else asks for the repaint.
|
|
this.config.requestShellRender();
|
|
},
|
|
setShownOnGlasses: (shown) => {
|
|
if (shown) {
|
|
if (!this.screenOn) this.wake("sidebar");
|
|
this.stack.push(layer);
|
|
} else {
|
|
this.stack.remove(layer);
|
|
}
|
|
this.config.requestShellRender();
|
|
},
|
|
visibility,
|
|
onVisibilityChanged: (value) => setStringSetting(KEYBOARD_INPUT_VISIBILITY_KEY, value),
|
|
history: this.keyboardInputHistory,
|
|
sendTargets: targets,
|
|
defaultTargetIndex: defaultIndex,
|
|
});
|
|
this.activeKeyboardLayer = layer;
|
|
if (layer.shownOnGlasses) this.stack.push(layer);
|
|
this.config.onKeyboardInputChanged?.(layer);
|
|
this.config.requestShellRender();
|
|
return layer;
|
|
}
|
|
|
|
isAssistantAvailable(): boolean {
|
|
const current = this.getAssistantConversations().current();
|
|
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
|
|
}
|
|
|
|
/** What to set up when isAssistantAvailable() is false. */
|
|
assistantSetupHint(): string {
|
|
if (assistantBackendSetting.get() === "external") return "Configure the agent bridge host and token in Settings.";
|
|
const model = this.getAssistantConversations().current().model;
|
|
if (model === "hermes") return "Set the Hermes URL in Settings > Assistant.";
|
|
if (model === "openai-compatible") return "Set the compatible API URL and model ID in Settings > Assistant.";
|
|
return global.isIOS ? "Configure an assistant model in Settings." : "Set an API key or download the on-phone model in Settings.";
|
|
}
|
|
|
|
/** Observe assistant turns (see AssistantActivityEvent). */
|
|
onAssistantActivity(listener: (event: AssistantActivityEvent) => void): () => void {
|
|
this.assistantActivityListeners.add(listener);
|
|
return () => {
|
|
this.assistantActivityListeners.delete(listener);
|
|
};
|
|
}
|
|
|
|
/** Observe showAlert popups (mirrored to the watch). */
|
|
onAlertShown(listener: (text: string) => void): () => void {
|
|
this.alertListeners.add(listener);
|
|
return () => {
|
|
this.alertListeners.delete(listener);
|
|
};
|
|
}
|
|
|
|
private emitAssistantActivity(event: AssistantActivityEvent): void {
|
|
for (const listener of Array.from(this.assistantActivityListeners)) {
|
|
try {
|
|
listener(event);
|
|
} catch (error) {
|
|
console.warn("assistant activity listener failed", error);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Dismiss the assistant overlay (cancelling any in-flight turn), as Done
|
|
* would. Returns whether an overlay was actually open to close.
|
|
*/
|
|
closeAssistant(): boolean {
|
|
return this.closeAssistantLayer();
|
|
}
|
|
|
|
private ensureAssistantSession(): AssistantSession | null {
|
|
return this.getAssistantConversations().ensureSession();
|
|
}
|
|
|
|
private resolveAssistantConfiguration(
|
|
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
|
|
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
|
|
): AssistantBackendConfig | null {
|
|
if (assistantBackendSetting.get() === "external") {
|
|
const host = assistantBridgeHostSetting.get().trim();
|
|
const token = assistantBridgeTokenSetting.get();
|
|
if (!host || !token) return null;
|
|
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
|
return { kind: "external", bridge: { host, port, token } };
|
|
}
|
|
const llm = resolveAssistantModel(model, {
|
|
anthropic: anthropicApiKeySetting.get(),
|
|
openai: openAiApiKeySetting.get(),
|
|
hermesUrl: assistantHermesUrlSetting.get(),
|
|
hermesKey: assistantHermesKeySetting.get(),
|
|
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
|
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
|
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
|
veniceWebSearch: veniceWebSearchSetting.get(),
|
|
});
|
|
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
|
|
return llm ? { kind: "direct", llm } : null;
|
|
}
|
|
|
|
private buildAssistantContext(): AssistantContext {
|
|
const foreground = this.getForegroundApp();
|
|
return {
|
|
foregroundApp: foreground?.appId ?? null,
|
|
foregroundTitle: foreground?.title ?? null,
|
|
screenOn: this.screenOn,
|
|
localTime: formatAssistantTime(new Date()),
|
|
headsetBattery: this.battery.headset,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Start (or continue) an assistant conversation from a finalized utterance.
|
|
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
|
* existing session and overlay.
|
|
*/
|
|
sendToAssistant(text: string, showOverlay = true): void {
|
|
text = text.trim();
|
|
if (!text) return;
|
|
const session = this.ensureAssistantSession();
|
|
if (!session) {
|
|
this.showAlert(this.assistantSetupHint());
|
|
return;
|
|
}
|
|
if (session.isTurnActive()) {
|
|
this.showAlert("The assistant is still working on the previous request");
|
|
return;
|
|
}
|
|
if (!this.screenOn) this.wake("sidebar");
|
|
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
|
|
this.runAssistantTurn(session, null, text);
|
|
return;
|
|
}
|
|
let layer = this.assistantLayer;
|
|
if (!layer) {
|
|
const created = new AssistantLayer(this.config.actions, {
|
|
onFollowUp: () => this.startAssistantFollowUp(),
|
|
onCancel: () => this.assistantSession?.cancel(),
|
|
onClose: () => this.closeAssistantLayer(),
|
|
onRemoved: () => {
|
|
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
|
// stop the turn and drop the overlay reference; keep shared history.
|
|
this.assistantSession?.cancel();
|
|
if (this.assistantLayer === created) this.assistantLayer = null;
|
|
this.emitAssistantActivity({ phase: "closed", text: "" });
|
|
},
|
|
});
|
|
layer = created;
|
|
this.assistantLayer = created;
|
|
this.stack.push(created);
|
|
}
|
|
this.runAssistantTurn(session, layer, text);
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
|
|
if (session.isTurnActive()) return;
|
|
layer?.startTurn();
|
|
let replySoFar = "";
|
|
this.emitAssistantActivity({ phase: "thinking", text: "" });
|
|
session.sendUtterance(text, this.buildAssistantContext(), {
|
|
onTextDelta: (delta, textSoFar) => {
|
|
replySoFar = textSoFar;
|
|
layer?.onTextDelta(delta, textSoFar);
|
|
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
|
|
},
|
|
onToolActivity: (label) => layer?.onToolActivity(label),
|
|
onTurnDone: () => {
|
|
layer?.onTurnDone();
|
|
this.emitAssistantActivity({ phase: "done", text: replySoFar });
|
|
},
|
|
onError: (message) => {
|
|
layer?.onError(message);
|
|
this.emitAssistantActivity({ phase: "error", text: message });
|
|
},
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Record another utterance in the current assistant conversation. handsFree
|
|
* (from a wakeword over the overlay) starts the mic immediately; otherwise
|
|
* (the Follow-up menu button) a click ends the utterance.
|
|
*/
|
|
private startAssistantFollowUp(handsFree = false): void {
|
|
const layer = this.assistantLayer;
|
|
const session = this.assistantSession;
|
|
if (!layer || !session || this.activeVoiceLayer || this.activeKeyboardLayer) return;
|
|
const voice = new VoiceInputLayer({
|
|
actions: this.config.actions,
|
|
onClosed: () => {
|
|
if (this.activeVoiceLayer === voice) {
|
|
this.activeVoiceLayer = null;
|
|
voiceActivity.setActive(false);
|
|
this.noteUserActivity();
|
|
}
|
|
},
|
|
dismiss: () => {
|
|
this.stack.popIfTop((top) => top === voice);
|
|
},
|
|
sendTargets: [
|
|
{ id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) },
|
|
],
|
|
finishOnClick: !handsFree,
|
|
handsFree,
|
|
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
|
|
});
|
|
this.activeVoiceLayer = voice;
|
|
voiceActivity.setActive(true);
|
|
this.stack.push(voice);
|
|
voice.startCapture();
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
private closeAssistantLayer(): boolean {
|
|
// Popping fires the layer's onRemoved, which cancels the turn and clears
|
|
// this.assistantLayer. popThrough also closes anything stacked above the
|
|
// overlay (a follow-up voice dialog, an alert), so a close command works
|
|
// no matter what the conversation is showing.
|
|
const layer = this.assistantLayer;
|
|
const closed = layer ? this.stack.popThrough(layer) : false;
|
|
this.noteUserActivity();
|
|
this.config.requestShellRender();
|
|
return closed;
|
|
}
|
|
|
|
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
|
|
showAlert(text: string): void {
|
|
if (!this.screenOn) this.wake("sidebar");
|
|
for (const listener of Array.from(this.alertListeners)) {
|
|
try {
|
|
listener(text);
|
|
} catch (error) {
|
|
console.warn("alert listener failed", error);
|
|
}
|
|
}
|
|
const layer = new ShellAlertLayer(text, () => {
|
|
this.stack.popIfTop((top) => top === layer);
|
|
this.config.requestShellRender();
|
|
});
|
|
this.stack.push(layer);
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
private startEscapeMenuTimer(): void {
|
|
this.cancelEscapeMenuTimer();
|
|
this.escapeMenuTimer = setTimeout(() => {
|
|
this.escapeMenuTimer = null;
|
|
this.openEscapeMenu();
|
|
}, LONG_PRESS_ESCAPE_MENU_MS);
|
|
}
|
|
|
|
private cancelEscapeMenuTimer(): void {
|
|
if (this.escapeMenuTimer !== null) {
|
|
clearTimeout(this.escapeMenuTimer);
|
|
this.escapeMenuTimer = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A window's answer to tap-then-hold when it has no context menu of its
|
|
* own: open the system menu in its place, so both gestures land on the
|
|
* same menu. Ignored unless the window is still the foreground one.
|
|
*/
|
|
openSystemMenu(windowId: string): void {
|
|
if (this.foregroundWindow()?.windowId !== windowId) return;
|
|
this.openEscapeMenu();
|
|
}
|
|
|
|
/**
|
|
* The system/escape menu: the entries every window shares (Focus app
|
|
* switcher, Voice input, Brightness, Close window) plus Debug. Shell-owned and
|
|
* shell-drawn (never the app's), so an unresponsive app can always be
|
|
* closed. It opens for long-press (over the app's own menu too), after an
|
|
* extended hold in a window that claims long-press, and on a window's
|
|
* request when tap-then-hold finds it has no menu of its own.
|
|
*/
|
|
private openEscapeMenu(): void {
|
|
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
|
const foreground = this.foregroundWindow();
|
|
if (!foreground) return;
|
|
const items: MenuItem[] = [];
|
|
if (foreground.closeable) {
|
|
items.push({
|
|
label: "Close window",
|
|
onSelect: (ctx) => {
|
|
// Pop the menu first (its onRemoved returns focus to the sidebar),
|
|
// then close the window the menu was opened over.
|
|
ctx.stack.pop();
|
|
this.closeForegroundWindow();
|
|
},
|
|
});
|
|
}
|
|
// Close window sits first but the menu opens on Focus app switcher, so a
|
|
// reflexive tap never closes the window.
|
|
const initialSelection = items.length;
|
|
items.push(
|
|
{
|
|
// Defocus the app (hand focus to the sidebar) without closing it: the
|
|
// reliable way out of an app that consumes double-click. Closing the
|
|
// menu is what yields focus, so popping is the whole action.
|
|
label: "Focus app switcher",
|
|
onSelect: (ctx) => {
|
|
ctx.stack.pop();
|
|
},
|
|
},
|
|
...(this.config.voiceInputEnabled === false ? [] : [{
|
|
label: "Voice input",
|
|
onSelect: (ctx) => {
|
|
// The transcript is aimed at the foreground window, so keep focus
|
|
// there through the pop instead of yielding to the sidebar.
|
|
layer.keepWindowFocus = true;
|
|
ctx.stack.pop();
|
|
this.startVoiceInput();
|
|
},
|
|
}]),
|
|
);
|
|
if (brightnessSetting.get() !== "auto") {
|
|
items.push({
|
|
label: "Brightness",
|
|
onSelect: (ctx) => {
|
|
ctx.stack.pop();
|
|
ctx.stack.push(new BrightnessPickerLayer(() => this.yieldFocusToSidebar()));
|
|
this.config.requestShellRender();
|
|
},
|
|
});
|
|
}
|
|
items.push({
|
|
label: "Debug",
|
|
onSelect: (ctx) => {
|
|
ctx.stack.pop();
|
|
this.openToolDebugDialog();
|
|
},
|
|
});
|
|
// Gesture help: tap-then-hold switches to the app's own context menu,
|
|
// when the app has one (otherwise it opens this very menu, not worth a
|
|
// hint).
|
|
const footer = foreground.hasAppMenu?.()
|
|
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
|
|
: undefined;
|
|
const layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
|
|
layer.selectItem(initialSelection);
|
|
this.stack.push(layer);
|
|
// Tell the window the system menu opened over it: an app with its own
|
|
// context menu up closes it, so the two context menus never stack.
|
|
void foreground.handleInput(makeInputEvent({ type: "system-menu-opened" }), 0);
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
/**
|
|
* Escape menu > Debug: list the assistant tools registered for the
|
|
* foreground window's app (live or gated). System-wide "always" tools are
|
|
* constant and omitted.
|
|
*/
|
|
private openToolDebugDialog(): void {
|
|
const foreground = this.foregroundWindow();
|
|
const appId = foreground?.appId ?? null;
|
|
const entries = appId
|
|
? toolRegistry
|
|
.listToolsForDebug()
|
|
.filter(
|
|
(entry) =>
|
|
entry.spec.name.startsWith(`app.${appId}.`) || entry.windowId === foreground!.windowId,
|
|
)
|
|
: [];
|
|
this.stack.push(new ToolDebugMenuLayer(appId, entries, () => this.yieldFocusToSidebar()));
|
|
this.config.requestShellRender();
|
|
}
|
|
|
|
private chromeState(): ShellChromeState {
|
|
return {
|
|
windows: this.switcherWindows().map((window) => ({
|
|
windowId: window.windowId,
|
|
title: window.title,
|
|
attention: Boolean(this.attention.get(window.windowId)),
|
|
drawIcon: window.drawIcon,
|
|
})),
|
|
selectedIndex: this.selectedStatus ? -1 : this.selectedIndex,
|
|
selectedStatusItem: this.selectedStatus?.item ?? null,
|
|
focus: this.focus,
|
|
foregroundHeightMode: this.foregroundWindow()?.heightMode ?? "min",
|
|
foregroundAppId: this.foregroundWindow()?.appId,
|
|
battery: this.battery,
|
|
trayIcons: Array.from(this.trayIcons.keys())
|
|
.sort()
|
|
.map((ownerId) => ({ ownerId, icon: this.trayIcons.get(ownerId)!.icon })),
|
|
};
|
|
}
|
|
}
|
|
|
|
export const shell = new Shell();
|
|
|
|
const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
|
const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
|
|
|
/** Human-readable local time for the assistant's per-turn context. */
|
|
function formatAssistantTime(date: Date): string {
|
|
let hours = date.getHours();
|
|
const minutes = date.getMinutes().toString().padStart(2, "0");
|
|
const meridiem = hours >= 12 ? "PM" : "AM";
|
|
hours = hours % 12;
|
|
if (hours === 0) hours = 12;
|
|
const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`;
|
|
return `${day}, ${hours}:${minutes} ${meridiem}`;
|
|
}
|
|
|
|
export function rawInputEventToInputEvent(event: RawInputEvent): InputEvent {
|
|
return { ...makeInputEvent(rawInputEventToPayload(event)),
|
|
...(event.ringInput ? { ringInput: { ...event.ringInput } } : {}) };
|
|
}
|
|
|
|
function rawInputEventToPayload(event: RawInputEvent): InputEventPayload {
|
|
if (event.kind === "sys-event") {
|
|
if (event.eventType === OsEventTypeList.RING_PRESS_EVENT &&
|
|
(event.eventSource === EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL ||
|
|
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING)) {
|
|
// The firmware emits this dedicated ID only for full raw source 4.
|
|
// Its stock sender leaves the source unspecified for extension 14.
|
|
return { type: "ring-press", source: "ring" };
|
|
} else if (event.eventType === OsEventTypeList.RING_PRESS_EVENT &&
|
|
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH) {
|
|
// Synthetic: the watch pad's (or the phone's watch pad's) touch-down.
|
|
return { type: "ring-press", source: "watch" };
|
|
} else if (event.eventType === OsEventTypeList.CLICK_EVENT) {
|
|
return {
|
|
type: "click",
|
|
source: eventSourceToString(event.eventSource),
|
|
};
|
|
} else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) {
|
|
return {
|
|
type: "double-click",
|
|
source: eventSourceToString(event.eventSource),
|
|
};
|
|
} else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
|
|
return scrollEvent("scroll-down", event.eventSource);
|
|
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
|
|
return scrollEvent("scroll-up", event.eventSource);
|
|
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) {
|
|
// CFW-forwarded long-press (replaces the firmware's force-quit dialog).
|
|
// Current CFW supplies the physical source for ring and temple presses;
|
|
// eventSourceToString's ring fallback keeps older CFW builds compatible.
|
|
return { type: "long-press", source: eventSourceToString(event.eventSource) };
|
|
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
|
|
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
|
|
} else if (event.eventType === OsEventTypeList.SHORT_THEN_LONG_PRESS_EVENT) {
|
|
return { type: "short-then-long-press", source: eventSourceToString(event.eventSource) };
|
|
}
|
|
} else if (event.kind === "watch-gesture") {
|
|
// Synthetic, from the Wear OS remote (app/g2/wear-remote.ts); the glasses
|
|
// firmware never produces these.
|
|
if (event.eventType === WatchGestureType.SWIPE_LEFT) {
|
|
return { type: "swipe-left", source: "watch" };
|
|
} else if (event.eventType === WatchGestureType.SWIPE_RIGHT) {
|
|
return { type: "swipe-right", source: "watch" };
|
|
} else if (event.eventType === WatchGestureType.SWIPE_UP) {
|
|
return { type: "swipe-up", source: "watch" };
|
|
} else if (event.eventType === WatchGestureType.SWIPE_DOWN) {
|
|
return { type: "swipe-down", source: "watch" };
|
|
}
|
|
} else if (event.kind === "even-ai") {
|
|
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
|
|
// wakeword interests us -- ENTER means the user manually opened the stock
|
|
// assistant, and EXIT is it tearing down.
|
|
if (event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP) {
|
|
return { type: "wakeword" };
|
|
}
|
|
} else if (event.kind === "display-wake") {
|
|
// Outside an EvenHub page the firmware consumes the physical double tap
|
|
// itself and reports only that the stock display lifecycle woke.
|
|
return { type: "display-wake" };
|
|
} else if (event.kind === "text-click") {
|
|
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
|
|
return scrollEvent("scroll-down", event.eventSource);
|
|
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
|
|
return scrollEvent("scroll-up", event.eventSource);
|
|
}
|
|
}
|
|
return {
|
|
type: "unknown",
|
|
kind: event.kind,
|
|
eventSource: event.eventSource,
|
|
eventType: event.eventType,
|
|
};
|
|
}
|
|
|
|
/** Preserve explicit sources; stock scroll notifications usually omit them. */
|
|
function scrollEvent(type: "scroll-up" | "scroll-down", eventSource: number): InputEventPayload {
|
|
return eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH ||
|
|
eventSource === EventSourceType.TOUCH_EVENT_FROM_RING ||
|
|
eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L ||
|
|
eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R
|
|
? { type, source: eventSourceToString(eventSource) } : { type };
|
|
}
|
|
|
|
function eventSourceToString(eventSource: number): InputSource {
|
|
if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
|
|
return "ring";
|
|
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) {
|
|
return "left-arm";
|
|
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) {
|
|
return "right-arm";
|
|
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH) {
|
|
return "watch";
|
|
}
|
|
return "ring";
|
|
}
|
|
|
|
export function inputEventToString(event: InputEvent): string {
|
|
switch (event.type) {
|
|
case "ring-press":
|
|
return `Press from ${event.source}`;
|
|
case "click":
|
|
return `Click from ${event.source}`;
|
|
case "double-click":
|
|
return `Double click from ${event.source}`;
|
|
case "scroll-up":
|
|
return `Scroll up`;
|
|
case "scroll-down":
|
|
return `Scroll down`;
|
|
case "long-press":
|
|
return `Long press from ${event.source}`;
|
|
case "long-press-release":
|
|
return `Long press release from ${event.source}`;
|
|
case "short-then-long-press":
|
|
return `Short then long press from ${event.source}`;
|
|
case "swipe-left":
|
|
return `Swipe left from ${event.source}`;
|
|
case "swipe-right":
|
|
return `Swipe right from ${event.source}`;
|
|
case "swipe-up":
|
|
return `Swipe up from ${event.source}`;
|
|
case "swipe-down":
|
|
return `Swipe down from ${event.source}`;
|
|
case "display-wake":
|
|
return `Display wake`;
|
|
case "wakeword":
|
|
return `Wakeword`;
|
|
case "system-menu-opened":
|
|
return `System menu opened`;
|
|
default:
|
|
case "unknown":
|
|
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
|
|
}
|
|
}
|