Add Wear OS companion app and remote controls

Add a Wearable Data Layer bridge for syncing dashboard state and commands.

Introduce a standalone watch app with assistant, app launcher, touchpad remote, settings, and connectivity UI.
This commit is contained in:
Kalani Helekunihi committed 2026-08-23 15:02:35 -05:00
1 parent cc4b69fb80
commit 8fa04d1b46
85 files changed
+5813 -102

No files matched your search

+67
View File
@@ -223,6 +223,36 @@ export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
description: "Whether the top-bar clock shows 24-hour or 12-hour time.",
});
/**
* How much of the 640x480 panel the UI uses. "576x288" is the stock band
* (sidebar + a 288px-tall window at the vertical position); "576x480" keeps
* the sidebar and gives every window the full height; "640x480" is the whole
* panel, with the sidebar an overlay that shows only while it has focus.
*/
export const DISPLAY_MODE_VALUES = ["576x288", "576x480", "640x480"] as const;
export type DisplayModeSetting = (typeof DISPLAY_MODE_VALUES)[number];
const DISPLAY_MODE_LABELS: Record<DisplayModeSetting, string> = {
"576x288": "Band · 576×288",
"576x480": "Tall · 576×480",
"640x480": "Full panel · 640×480",
};
export function displayModeLabel(value: DisplayModeSetting): string {
return DISPLAY_MODE_LABELS[value] ?? value;
}
export const displayModeSetting = new ConfigSettingEnum<DisplayModeSetting>({
id: "display-mode",
label: "Display mode",
storageKey: "display.mode",
defaultValue: "576x288",
values: DISPLAY_MODE_VALUES,
formatValue: displayModeLabel,
description:
"Band: the stock 576×288 window beside the sidebar. Tall: the sidebar plus full-height windows. Full panel: the whole 640×480 display; the sidebar overlays the app only while you are in it. Open apps reopen in the new size.",
});
export const brightnessSetting = new ConfigSettingEnum<BrightnessSetting>({
id: "brightness",
label: "Brightness",
@@ -252,6 +282,43 @@ export const lockScreenEnabledSetting = new ConfigSettingBoolean({
"Lock the glasses after they are taken off while the phone is locked. Unlocking the phone unlocks the glasses.",
});
// Wear OS watch remote (app/g2/wear-remote.ts, wear/). All three are read on
// every watch message, so a change applies immediately.
export const watchRemoteEnabledSetting = new ConfigSettingBoolean({
id: "watch-remote-enabled",
label: "Watch remote control",
storageKey: "watch.remoteEnabled",
defaultValue: true,
description:
"Accept input from the Faceclaw Wear OS app: spatial swipes, taps, holds, crown, voice queries and app commands. The ring's scheme is unaffected. Turn off to ignore the watch.",
});
export const watchCanUnlockSetting = new ConfigSettingBoolean({
id: "watch-can-unlock",
label: "Watch can unlock glasses",
storageKey: "watch.canUnlock",
defaultValue: true,
description:
"Let the watch unlock the glasses' lock screen (which otherwise waits for the phone to be unlocked). Your watch is on your wrist; turn this off if you would rather it stay a phone-only unlock.",
});
export const watchWakesDisplaySetting = new ConfigSettingBoolean({
id: "watch-wakes-display",
label: "Watch input wakes display",
storageKey: "watch.wakesDisplay",
defaultValue: true,
description:
"Any gesture from the watch turns the glasses display on (the ring only wakes it with a double-tap). Off: watch input is ignored while the display is off, as ring input is.",
});
export const watchMirrorAssistantSetting = new ConfigSettingBoolean({
id: "watch-mirror-assistant",
label: "Mirror assistant to watch",
storageKey: "watch.mirrorAssistant",
defaultValue: true,
description: "Stream assistant replies and on-glasses alerts to the watch so they can be read from the wrist.",
});
export type VerticalPosition = "top" | "upper" | "middle" | "lower" | "bottom";
const VERTICAL_POSITION_LABELS: Record<VerticalPosition, string> = {
+39
View File
@@ -27,6 +27,7 @@ import {
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
brightnessSetting,
displayModeSetting,
elevenLabsApiKeySetting,
mapboxApiKeySetting,
openAiApiKeySetting,
@@ -48,7 +49,12 @@ import {
voiceProviderSetting,
screenTimeoutSetting,
wakeWordActionSetting,
watchCanUnlockSetting,
watchMirrorAssistantSetting,
watchRemoteEnabledSetting,
watchWakesDisplaySetting,
} from "../dashboard-settings";
import { wearBridge } from "../../native/wear-bridge";
import { SettingsPanelLayer, type SettingsSection } from "./settings-panel";
import { terminalFontPickerMenuItem, uiFontPickerMenuItem } from "../font-picker";
@@ -74,6 +80,8 @@ function settingsSections(): SettingsSection[] {
// Where min-height windows (and the sidebar) sit vertically on the
// screen; the dashboard controller repositions surfaces on change.
enumSettingMenuItem(verticalPositionSetting),
// Band / tall / full-panel; the dashboard controller reflows windows.
enumSettingMenuItem(displayModeSetting),
// Controls the top-bar battery indicators (icon vs percentage).
enumSettingMenuItem(batteryDisplayModeSetting),
// Controls the top-bar clock (24-hour vs 12-hour).
@@ -133,6 +141,18 @@ function settingsSections(): SettingsSection[] {
textSettingMenuItem(roamApiTokenSetting),
],
},
{
label: "Watch",
// Wear OS remote (wear/); the status line above the items says whether
// a watch running the companion app is currently reachable.
items: [
toggleSettingMenuItem(watchRemoteEnabledSetting),
toggleSettingMenuItem(watchWakesDisplaySetting),
toggleSettingMenuItem(watchCanUnlockSetting),
toggleSettingMenuItem(watchMirrorAssistantSetting),
],
renderDetail: renderWatchStatus,
},
{
label: "Developer",
items: [
@@ -260,6 +280,25 @@ function readBundledDoc(fileName: string): string {
}
}
function renderWatchStatus(args: { image: GrayImage; x: number; y: number; width: number }): number {
const { image, x, y, width } = args;
const font = getDefaultSmallFont();
let status: string;
if (!wearBridge.isAvailable()) {
status = "Google Play services is unavailable on this phone, so no watch can connect.";
} else {
const connection = wearBridge.getWatchConnection();
status = connection.reachable
? `Connected to ${connection.watchName || "a watch"}.`
: "No watch connected. Install the Faceclaw watch app (wear/ in the source tree) on a Wear OS watch paired with this phone.";
}
const lines = wrapText(font, status, width);
for (let i = 0; i < lines.length; i++) {
image.drawText(font, x, y + i * font.lineHeight, lines[i]!, 170);
}
return lines.length * font.lineHeight + 10;
}
function renderAbout(args: { image: GrayImage; x: number; y: number; width: number }): number {
const { image, x, y, width } = args;
const font = getDefaultSmallFont();
+13
View File
@@ -41,6 +41,9 @@ const NO_MENU = new MenuLayer(null, []);
* column, out to the sidebar). Third-level menus open as centered modals.
*/
export class SettingsPanelLayer implements Layer {
// Watch swipes map onto the two columns: right goes into a section's
// items, left comes back out (and out to the sidebar from the left column).
readonly acceptsDirectional = true;
private leftIndex = 0;
private rightIndex = 0;
private focus: "left" | "right" = "left";
@@ -185,14 +188,17 @@ export class SettingsPanelLayer implements Layer {
if (this.focus === "left") {
switch (event.type) {
case "scroll-up":
case "swipe-up":
this.leftIndex = (this.leftIndex + this.sections.length - 1) % this.sections.length;
this.resetRight();
return;
case "scroll-down":
case "swipe-down":
this.leftIndex = (this.leftIndex + 1) % this.sections.length;
this.resetRight();
return;
case "click":
case "swipe-right":
// Only enter sections that have something interactive on the right.
if (this.section().items.length) {
this.focus = "right";
@@ -200,6 +206,7 @@ export class SettingsPanelLayer implements Layer {
}
return;
case "double-click":
case "swipe-left":
shell.yieldFocusToSidebar();
return;
default:
@@ -210,18 +217,22 @@ export class SettingsPanelLayer implements Layer {
const items = this.section().items;
switch (event.type) {
case "scroll-up":
case "swipe-up":
if (items.length) this.rightIndex = (this.rightIndex + items.length - 1) % items.length;
return;
case "scroll-down":
case "swipe-down":
if (items.length) this.rightIndex = (this.rightIndex + 1) % items.length;
return;
case "click":
case "swipe-right":
if (items.length) {
const item = items[clamp(this.rightIndex, 0, items.length - 1)]!;
if (!isMenuItemDisabled(item)) await item.onSelect(ctx, NO_MENU);
}
return;
case "double-click":
case "swipe-left":
this.focus = "left";
return;
default:
@@ -242,6 +253,8 @@ export class SettingsPanelLayer implements Layer {
* layers (modal menus, the text editor) stack above it as before.
*/
export class SettingsDescriptionOverlayLayer implements Layer {
readonly acceptsDirectional = true;
constructor(private readonly panel: SettingsPanelLayer) {}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
+88 -7
View File
@@ -3,13 +3,35 @@ import { type Plane } from "../graphics/plane";
import { spanCurrent } from "../native/frame-timings";
import { type ConfigSettingString } from "./dashboard-settings";
/**
* Where an input came from. The ring and the glasses' arms are the stock
* sources; "watch" is the Wear OS remote, which components may treat with a
* richer scheme (see isWatchInput) — the ring's semantics are never changed
* by it.
*/
export type InputSource = "ring" | "left-arm" | "right-arm" | "watch";
export type DashboardInputEvent =
| { type: "click"; source: "ring" | "left-arm" | "right-arm" }
| { type: "double-click"; source: "ring" | "left-arm" | "right-arm" }
| { type: "scroll-up" }
| { type: "scroll-down" }
| { type: "long-press"; source: "ring" | "left-arm" | "right-arm" }
| { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" }
| { type: "click"; source: InputSource }
| { type: "double-click"; source: InputSource }
/** Ring scroll (or a watch crown turn, then tagged source "watch"). */
| { type: "scroll-up"; source?: InputSource }
| { type: "scroll-down"; source?: InputSource }
| { type: "long-press"; source: InputSource }
| { type: "long-press-release"; source: InputSource }
/**
* Spatial (four-way) input, which only a watch can produce: the ring's
* scroll is a one-dimensional cursor, these are directions. Components with
* a spatial meaning for them opt in with Layer.acceptsDirectional /
* ShellWindow.acceptsDirectional and must then handle all four; for the
* rest, directionalFallback turns up/down into scroll, right into "select"
* (click) and left into "back" (double-click), so a swipe always does
* something sensible.
*/
| { type: "swipe-up"; source: "watch" }
| { type: "swipe-down"; source: "watch" }
| { type: "swipe-left"; source: "watch" }
| { type: "swipe-right"; source: "watch" }
/**
* The stock display lifecycle woke while no EvenHub page was running. This
* is wake-only, unlike a normal double-click (which turns an on screen off).
@@ -24,6 +46,44 @@ export type DashboardInputEvent =
| { type: "wakeword" }
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
/** True for input from the Wear OS remote (any type that carries a source). */
export function isWatchInput(event: DashboardInputEvent): boolean {
return "source" in event && event.source === "watch";
}
export type DirectionalInputEvent = Extract<
DashboardInputEvent,
{ type: "swipe-up" | "swipe-down" | "swipe-left" | "swipe-right" }
>;
export function isDirectionalInput(event: DashboardInputEvent): event is DirectionalInputEvent {
return (
event.type === "swipe-up" ||
event.type === "swipe-down" ||
event.type === "swipe-left" ||
event.type === "swipe-right"
);
}
/**
* The ring-vocabulary equivalent of a directional swipe: up/down scroll,
* right selects, left backs out.
*/
export function directionalFallback(event: DashboardInputEvent): DashboardInputEvent {
switch (event.type) {
case "swipe-up":
return { type: "scroll-up" };
case "swipe-down":
return { type: "scroll-down" };
case "swipe-right":
return { type: "click", source: "ring" };
case "swipe-left":
return { type: "double-click", source: "ring" };
default:
return event;
}
}
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
@@ -96,6 +156,17 @@ export interface Layer {
readonly paintOverBase?: boolean;
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage;
handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> | void;
/**
* True when handleInput gives swipe-left / swipe-right a meaning of its own.
* Otherwise the stack hands the layer directionalFallback(event) instead.
*/
readonly acceptsDirectional?: boolean;
/**
* A touch at (x, y) in this layer's own canvas coordinates, from the phone's
* mirror view. Return true if the layer acted on it (selected / opened what
* is there); false sends a plain select (click) instead.
*/
hitTest?(x: number, y: number, ctx: LayerContext): Promise<boolean> | boolean;
/** Called when the layer leaves the stack by any path (pop or clearToBase). */
onRemoved?(): void;
/**
@@ -195,7 +266,17 @@ export class LayerStack {
}
async handleInput(event: DashboardInputEvent): Promise<void> {
await this.layers[this.layers.length - 1]!.handleInput(event, this.ctx);
const top = this.layers[this.layers.length - 1]!;
const delivered = isDirectionalInput(event) && !top.acceptsDirectional ? directionalFallback(event) : event;
await top.handleInput(delivered, this.ctx);
}
/** Pass a mirror touch to the base layer when nothing is stacked on it. */
async hitTest(x: number, y: number): Promise<boolean> {
if (this.layers.length !== 1) return false;
const base = this.layers[0]!;
if (!base.hitTest) return false;
return await base.hitTest(x, y, this.ctx);
}
/** Hand text to the top layer; false if it doesn't take text input. */
+36 -4
View File
@@ -13,6 +13,7 @@ import {
minWindowTop,
SHELL_OPAQUE_BLACK,
SIDEBAR_WIDTH,
sidebarWidth,
TOP_BAR_HEIGHT,
windowTop,
type WindowHeightMode,
@@ -166,7 +167,11 @@ export class ShellChromeLayer implements Layer {
paint(): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const state = this.getState();
this.drawSidebar(image, state);
// Full-panel mode: the strip is an overlay, present only while the user
// is in it (the window underneath keeps its full width the rest of the time).
if (sidebarWidth() > 0 || state.focus === "sidebar") {
this.drawSidebar(image, state);
}
this.drawTopBar(image, state);
return image;
}
@@ -176,6 +181,30 @@ export class ShellChromeLayer implements Layer {
// layer stack; the chrome itself never consumes events.
}
/**
* Which window's sidebar icon is under (x, y) on screen, if any — the same
* slot geometry drawSidebar uses, so a touch on the phone's mirror lands on
* the icon the mirror showed.
*/
windowIndexAt(x: number, y: number, windowCount: number): number | null {
if (x < 0 || x >= SIDEBAR_WIDTH || windowCount === 0) return null;
const variant = sidebarVariant(windowCount);
const rows = rowsPerColumn(variant);
const listTop = minWindowTop() + TOP_BAR_HEIGHT + LIST_MARGIN;
const itemStride = variant.iconSize + ICON_SPACING;
const lastVisible = Math.min(windowCount, this.scrollRow + rows * variant.columns);
for (let index = this.scrollRow; index < lastVisible; index++) {
const position = index - this.scrollRow;
const column = variant.columns - 1 - ((position / rows) | 0);
const slotY = listTop + (position % rows) * itemStride;
const left = columnLeft(variant, column);
if (x >= left && x < left + variant.columnWidth && y >= slotY - 2 && y < slotY + variant.iconSize + 2) {
return index;
}
}
return null;
}
private drawSidebar(image: GrayImage, state: ShellChromeState): void {
// The sidebar always occupies the min-height window band at the preferred
// vertical position: it never moves when the foreground window's height
@@ -266,13 +295,16 @@ export class ShellChromeLayer implements Layer {
// its height mode puts that (screen top for max height). It moves when
// the foreground switches to a window of a different height.
const barTop = windowTop(state.foregroundHeightMode);
image.fillRect(SIDEBAR_WIDTH, barTop, G2_LENS_WIDTH - SIDEBAR_WIDTH, TOP_BAR_HEIGHT, SHELL_OPAQUE_BLACK);
image.drawLine(SIDEBAR_WIDTH, barTop + TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, barTop + TOP_BAR_HEIGHT - 1, BORDER_VALUE);
// The bar spans the app viewport; with the sidebar overlaid (full-panel
// mode, sidebar focused) it still starts past the strip.
const barLeft = sidebarWidth() > 0 || state.focus === "sidebar" ? SIDEBAR_WIDTH : 0;
image.fillRect(barLeft, barTop, G2_LENS_WIDTH - barLeft, TOP_BAR_HEIGHT, SHELL_OPAQUE_BLACK);
image.drawLine(barLeft, barTop + TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, barTop + TOP_BAR_HEIGHT - 1, BORDER_VALUE);
const now = new Date();
const clock = `${WEEKDAYS[now.getDay()]} ${now.getDate()} ${MONTHS[now.getMonth()]} ` +
formatClockTime(now);
const clockX = SIDEBAR_WIDTH + 10;
const clockX = barLeft + 10;
const textY = barTop + Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
image.drawText(font, clockX, textY, clock, 210);
+32 -8
View File
@@ -1,15 +1,39 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../../graphics/image";
import { verticalPositionSetting } from "../dashboard-settings";
import { displayModeSetting, type DisplayModeSetting, verticalPositionSetting } from "../dashboard-settings";
/** Top bar: 24px notification icons plus a little padding. */
export const TOP_BAR_HEIGHT = 28;
/**
* Sidebar width. Leaves the app viewport exactly 576px wide — the surface
* width EvenHub apps expect. Icon column layout within the strip (one wide
* column vs. two narrow ones) is the chrome layer's business.
* Width of the sidebar strip as painted. Leaves the app viewport exactly
* 576px wide — the surface width EvenHub apps expect — except in the
* full-panel display mode, where the strip overlays the app (see
* sidebarWidth). Icon column layout within the strip (one wide column vs.
* two narrow ones) is the chrome layer's business.
*/
export const SIDEBAR_WIDTH = 64;
/** The Display > Display mode setting (see dashboard-settings.ts). */
export function displayMode(): DisplayModeSetting {
return displayModeSetting.get();
}
/**
* How much of the screen's width the sidebar takes away from app windows:
* the strip, or nothing in the full-panel mode (where the chrome paints the
* strip over the window while the sidebar has focus).
*/
export function sidebarWidth(): number {
return displayMode() === "640x480" ? 0 : SIDEBAR_WIDTH;
}
/**
* The height an app's requested mode actually gets: the band mode honours
* it, the taller modes give every window the whole screen.
*/
export function effectiveHeightMode(mode: WindowHeightMode): WindowHeightMode {
return displayMode() === "576x288" ? mode : "max";
}
/**
* X of the display's true horizontal centre, in app-viewport coordinates. The
* sidebar pushes the viewport to the right, so UI the wearer physically aims
@@ -17,7 +41,7 @@ export const SIDEBAR_WIDTH = 64;
* than use the viewport's own centre.
*/
export function screenCenterInViewportX(): number {
return Math.round(G2_LENS_WIDTH / 2) - SIDEBAR_WIDTH;
return Math.round(G2_LENS_WIDTH / 2) - sidebarWidth();
}
/**
@@ -43,7 +67,7 @@ export const MIN_WINDOW_MAX_TOP = G2_LENS_HEIGHT - MIN_WINDOW_HEIGHT;
/** Total height (top bar + content) of a window's band in the given mode. */
export function windowBandHeight(mode: WindowHeightMode): number {
switch (mode) {
switch (effectiveHeightMode(mode)) {
case "max":
return G2_LENS_HEIGHT;
case "medium":
@@ -89,7 +113,7 @@ export function windowTop(mode: WindowHeightMode): number {
/** App-content viewport size for a height mode (independent of vertical position). */
export function appViewportSize(mode: WindowHeightMode): { width: number; height: number } {
return {
width: G2_LENS_WIDTH - SIDEBAR_WIDTH,
width: G2_LENS_WIDTH - sidebarWidth(),
height: windowBandHeight(mode) - TOP_BAR_HEIGHT,
};
}
@@ -102,7 +126,7 @@ export function appViewportRect(mode: WindowHeightMode): {
height: number;
} {
return {
x: SIDEBAR_WIDTH,
x: sidebarWidth(),
y: windowTop(mode) + TOP_BAR_HEIGHT,
...appViewportSize(mode),
};
+22
View File
@@ -168,6 +168,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
title: options.title,
surfaceId: `window:${options.windowId}`,
closeable: options.closeable,
// The window's own LayerStack decides per layer whether a swipe is
// directional or falls back to click / double-click.
acceptsDirectional: true,
heightMode,
close: () => {
closed = true;
@@ -195,6 +198,16 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
await render(frameId);
},
requestRender,
relayout: () => {
stack.setBaseSize(appViewportSize(heightMode));
options.reconfigureSurface?.(heightMode);
requestRender();
},
hitTest: async (x, y) => {
const handled = await stack.hitTest(x, y);
if (handled) requestRender();
return handled;
},
receiveTextInput: options.receiveTextInput,
setForeground: (foreground) => {
options.setSurfaceVisible(foreground);
@@ -223,6 +236,15 @@ export class YieldAtRootLayer implements Layer {
return this.inner.paintOverBase;
}
/** Directional swipes reach the inner layer only if it understands them. */
get acceptsDirectional(): boolean | undefined {
return this.inner.acceptsDirectional;
}
hitTest(x: number, y: number, ctx: LayerContext): Promise<boolean> | boolean {
return this.inner.hitTest ? this.inner.hitTest(x, y, ctx) : false;
}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
return this.inner.paint(ctx, paintBelow);
}
+160 -17
View File
@@ -1,9 +1,20 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import { EvenAIStatus, EventSourceType, OsEventTypeList, WatchGestureType } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import {
DashboardInputEvent,
directionalFallback,
type InputSource,
isDirectionalInput,
isWatchInput,
Layer,
LayerActions,
LayerContext,
LayerStack,
noopLayerActions,
} from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { voiceActivity } from "./voice-activity";
@@ -62,6 +73,11 @@ export type ShellWindow = {
surfaceId: string;
/** Whether close menu entries (app menu default, shell escape menu) apply (the launcher is pinned). */
closeable: boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
*/
acceptsDirectional?: boolean;
/**
* Window height: the standard 288px band ("min") or the full screen
* ("max", terminal views). Decides the surface rect and where the shell
@@ -79,6 +95,17 @@ export type ShellWindow = {
handleInput: (event: DashboardInputEvent, frameId: number) => Promise<void> | void;
/** Repaint and resubmit this window's surface. */
requestRender: () => void;
/**
* Re-measure against the current display mode (viewport size / band) and
* repaint. Windows without it (workers, whose canvas is fixed at open) are
* closed and relaunched by the controller instead.
*/
relayout?: () => void;
/**
* A touch from the phone's mirror at (x, y) in app-viewport coordinates.
* True if the window acted on it; otherwise the controller sends a select.
*/
hitTest?: (x: number, y: number) => Promise<boolean> | boolean;
/**
* Deliver a text string to the window (e.g. finalized voice input). Optional:
* only windows that consume typed text (the terminal) implement it.
@@ -104,7 +131,18 @@ export type ShellConfig = {
/** Which surfaces need re-rendering after an input event. */
export type ShellInputOutcome = { shell: boolean; window: boolean };
type FocusKind = "sidebar" | "window";
/**
* Assistant overlay activity, for mirrors outside the glasses (the watch).
* "streaming" carries the reply so far (replace semantics); "done" the final
* reply; "error" the message; "closed" fires when the overlay leaves the
* stack by any path.
*/
export type AssistantActivityEvent = {
phase: "thinking" | "streaming" | "done" | "error" | "closed";
text: string;
};
export type FocusKind = "sidebar" | "window";
const noopActions: LayerActions = noopLayerActions;
@@ -209,6 +247,8 @@ class Shell {
private activeVoiceLayer: VoiceInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>();
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
@@ -217,10 +257,8 @@ class Shell {
requestShellRender: () => {},
onScreenStateChanged: () => {},
};
private readonly stack = new LayerStack(
new ShellChromeLayer(() => this.chromeState()),
this.actions,
);
private readonly chrome = new ShellChromeLayer(() => this.chromeState());
private readonly stack = new LayerStack(this.chrome, this.actions);
// Top-bar settings we mirror into the chrome; a change to either repaints
// the shell surface so the top bar reflects it immediately.
@@ -379,6 +417,22 @@ class Shell {
this.lastInputAtMs = nowMs;
}
/** Where input currently goes: the sidebar strip or the foreground window. */
getFocus(): FocusKind {
return this.focus;
}
/** True while a shell overlay (menu, dialog, voice input) is above the chrome. */
hasOverlay(): boolean {
return !this.stack.isAtBase();
}
/** The window whose sidebar icon is at screen (x, y), for mirror touches. */
windowAtSidebarPoint(x: number, y: number): ShellWindow | null {
const index = this.chrome.windowIndexAt(x, y, this.windows.length);
return index === null ? null : this.windows[index] ?? null;
}
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
wake(focus: FocusKind, nowMs = Date.now()): boolean {
this.lastInputAtMs = nowMs;
@@ -572,7 +626,8 @@ class Shell {
const window = this.foregroundWindow();
if (window) {
// The window owns frameId from here (render or explicit finish).
await window.handleInput(event, frameId);
const delivered = isDirectionalInput(event) && !window.acceptsDirectional ? directionalFallback(event) : event;
await window.handleInput(delivered, frameId);
return { shell: false, window: true };
}
return { shell: false, window: false };
@@ -630,15 +685,25 @@ class Shell {
private handleSidebarInput(event: DashboardInputEvent): ShellInputOutcome {
switch (event.type) {
case "double-click":
// The ring's double-tap at the root turns the display off. For the
// watch, "back" at the root is simply the end of the road: it wakes
// the display on any gesture, so sleeping here would be a trap; the
// watch's Status screen turns the display off on purpose.
if (isWatchInput(event)) return { shell: false, window: false };
this.sleep();
return { shell: true, window: false };
case "scroll-up":
case "swipe-up":
this.moveSelection(-1);
return { shell: true, window: false };
case "scroll-down":
case "swipe-down":
this.moveSelection(1);
return { shell: true, window: false };
case "click":
case "swipe-right":
// Right: into the selected window (spatially, the window is to the
// sidebar's right). Left has nowhere further to go and is ignored.
if (this.windows.length) {
this.focus = "window";
// Repaint the window now so its selection highlight reflects focus
@@ -763,10 +828,41 @@ class Shell {
this.config.requestShellRender();
}
private isAssistantAvailable(): boolean {
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
}
/** Observe assistant turns (see AssistantActivityEvent). */
onAssistantActivity(listener: (event: AssistantActivityEvent) => void): () => void {
this.assistantActivityListeners.add(listener);
return () => {
this.assistantActivityListeners.delete(listener);
};
}
/** Observe showAlert popups (mirrored to the watch). */
onAlertShown(listener: (text: string) => void): () => void {
this.alertListeners.add(listener);
return () => {
this.alertListeners.delete(listener);
};
}
private emitAssistantActivity(event: AssistantActivityEvent): void {
for (const listener of Array.from(this.assistantActivityListeners)) {
try {
listener(event);
} catch (error) {
console.warn("assistant activity listener failed", error);
}
}
}
/** Dismiss the assistant overlay (cancelling any in-flight turn), as Done would. */
closeAssistant(): void {
this.closeAssistantLayer();
}
private ensureAssistantSession(): AssistantSession | null {
const config = this.resolveAssistantConfiguration();
if (!config) return null;
@@ -834,6 +930,7 @@ class Shell {
// stop the turn and drop the reference so a later query starts clean.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" });
},
});
layer = created;
@@ -846,11 +943,23 @@ class Shell {
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar),
onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar;
layer.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
},
onToolActivity: (label) => layer.onToolActivity(label),
onTurnDone: () => layer.onTurnDone(),
onError: (message) => layer.onError(message),
onTurnDone: () => {
layer.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar });
},
onError: (message) => {
layer.onError(message);
this.emitAssistantActivity({ phase: "error", text: message });
},
});
}
@@ -901,6 +1010,13 @@ class Shell {
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
showAlert(text: string): void {
if (!this.screenOn) this.wake("sidebar");
for (const listener of Array.from(this.alertListeners)) {
try {
listener(text);
} catch (error) {
console.warn("alert listener failed", error);
}
}
const layer = new ShellAlertLayer(text, () => {
this.stack.popIfTop((top) => top === layer);
this.config.requestShellRender();
@@ -1024,9 +1140,9 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
return scrollEvent("scroll-down", event.eventSource);
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
return scrollEvent("scroll-up", event.eventSource);
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) {
// CFW-forwarded long-press (replaces the firmware's force-quit dialog).
// The CFW gates this to the ring, so eventSource may be 0 (unknown);
@@ -1035,6 +1151,18 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
}
} else if (event.kind === "watch-gesture") {
// Synthetic, from the Wear OS remote (app/g2/wear-remote.ts); the glasses
// firmware never produces these.
if (event.eventType === WatchGestureType.SWIPE_LEFT) {
return { type: "swipe-left", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_RIGHT) {
return { type: "swipe-right", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_UP) {
return { type: "swipe-up", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_DOWN) {
return { type: "swipe-down", source: "watch" };
}
} else if (event.kind === "even-ai") {
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
// wakeword interests us -- ENTER means the user manually opened the stock
@@ -1048,9 +1176,9 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
return { type: "display-wake" };
} else if (event.kind === "text-click") {
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
return scrollEvent("scroll-down", event.eventSource);
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
return scrollEvent("scroll-up", event.eventSource);
}
}
return {
@@ -1061,13 +1189,20 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
};
}
function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" {
/** Scroll events only carry a source when it is the watch (the stock ones never needed one). */
function scrollEvent(type: "scroll-up" | "scroll-down", eventSource: number): DashboardInputEvent {
return eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH ? { type, source: "watch" } : { type };
}
function eventSourceToString(eventSource: number): InputSource {
if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
return "ring";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) {
return "left-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) {
return "right-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH) {
return "watch";
}
return "ring";
}
@@ -1086,6 +1221,14 @@ export function inputEventToString(event: DashboardInputEvent): string {
return `Long press from ${event.source}`;
case "long-press-release":
return `Long press release from ${event.source}`;
case "swipe-left":
return `Swipe left from ${event.source}`;
case "swipe-right":
return `Swipe right from ${event.source}`;
case "swipe-up":
return `Swipe up from ${event.source}`;
case "swipe-down":
return `Swipe down from ${event.source}`;
case "display-wake":
return `Display wake`;
case "wakeword":