Add Wear OS companion app and remote controls

Add a Wearable Data Layer bridge for syncing dashboard state and commands.

Introduce a standalone watch app with assistant, app launcher, touchpad remote, settings, and connectivity UI.
This commit is contained in:
Kalani Helekunihi committed 2026-08-23 15:02:35 -05:00
1 parent cc4b69fb80
commit 8fa04d1b46
85 files changed
+5813 -102

No files matched your search

+75
View File
@@ -55,3 +55,78 @@ Button.-primary {
border-width: 1;
border-radius: 8;
}
/* The phone's controller for the glasses (phone-ui/remote-controls.xml). */
.remote-controls {
margin: 0 8 0 8;
}
.display-controls {
margin-bottom: 6;
}
.mode-button {
font-size: 13;
padding: 6 10;
margin-right: 6;
}
.brightness-icon {
font-size: 16;
margin: 0 4 0 4;
color: #444444;
}
.auto-toggle Label {
font-size: 13;
margin-right: 2;
}
.touchpad {
height: 230;
background-color: #1f1f1f;
border-radius: 14;
border-width: 1;
border-color: #3a3a3a;
color: #f0f0f0;
margin-top: 2;
}
.pad-focus {
font-size: 20;
color: #ffffff;
}
.pad-dpad-button {
width: 46;
height: 46;
margin: 6;
padding: 0;
font-size: 16;
border-radius: 23;
background-color: #333333;
color: #e8e8e8;
}
.pad-dpad-down {
margin-bottom: 30;
}
.pad-legend {
font-size: 11;
color: #8a8a8a;
margin-bottom: 8;
}
.action-row {
margin-top: 6;
}
.action-button {
margin: 2;
font-size: 14;
}
.ring-row {
margin-top: 4;
}
.ring-label {
font-size: 12;
color: #666666;
margin: 0 6 0 4;
}
.ring-button {
width: 44;
height: 36;
margin: 1;
padding: 0;
font-size: 13;
}
+10
View File
@@ -4,6 +4,7 @@
* makes the content area exactly the 576x288 surface EvenHub apps expect.
*/
import { GrayImage } from "../../graphics/image";
import { EVENHUB_SCREEN_WIDTH } from "./compositor";
import { type DashboardInputEvent, type Layer, type LayerContext } from "../../ui/layers";
import {
createInProcessWindow,
@@ -20,6 +21,15 @@ class EvenHubAppLayer implements Layer {
paint(ctx: LayerContext): GrayImage {
const size = ctx.stack.getBaseSize();
// EvenHub apps lay out for the stock 576-wide surface. In the full-panel
// display mode the window is 640 wide; paint at stock width and centre
// it rather than leaving all the slack on the right.
if (size.width > EVENHUB_SCREEN_WIDTH) {
const content = this.session.paint({ width: EVENHUB_SCREEN_WIDTH, height: size.height }, ctx.stack.isFocused());
const image = new GrayImage(size.width, size.height, 0);
content.composeInto(image, Math.floor((size.width - EVENHUB_SCREEN_WIDTH) / 2), 0);
return image;
}
return this.session.paint(size, ctx.stack.isFocused());
}
+8 -3
View File
@@ -20,7 +20,7 @@
import { ApplicationSettings } from "@nativescript/core";
import { GrayImage } from "../../graphics/image";
import { EvenHubFont } from "../../graphics/evenhub-font";
import { type DashboardInputEvent, type Layer } from "../../ui/layers";
import { type DashboardInputEvent, type InputSource, type Layer } from "../../ui/layers";
import {
anthropicApiKeySetting,
elevenLabsApiKeySetting,
@@ -1346,12 +1346,17 @@ function luminance(r: number, g: number, b: number): number {
return Math.round(0.2126 * r + 0.7152 * g + 0.0722 * b);
}
/** ring/left-arm/right-arm -> PB EventSourceType. */
function gestureSource(source: "ring" | "left-arm" | "right-arm"): number {
/**
* ring/left-arm/right-arm -> PB EventSourceType. The watch is reported to
* EvenHub apps as the ring: they only know stock hardware, and the watch's
* ring-vocabulary gestures mean the same thing.
*/
function gestureSource(source: InputSource): number {
switch (source) {
case "right-arm":
return 1;
case "ring":
case "watch":
return 2;
case "left-arm":
return 3;
+115 -15
View File
@@ -5,7 +5,7 @@ import { GrayImage } from "../../graphics/image";
import { type Plane } from "../../graphics/plane";
import { renderIcon, type IconName } from "../../graphics/icons";
import { clamp } from "../../util/numeric-util";
import { DashboardInputEvent, Layer, LayerActions, LayerContext } from "../../ui/layers";
import { DashboardInputEvent, isWatchInput, Layer, LayerActions, LayerContext } from "../../ui/layers";
import { drawSelectionHighlight, MenuLayer, scrollToKeepSelectionVisible, type MenuItem } from "../../ui/menu";
import { iconGridMinRowHeight } from "../../ui/metrics";
import { WINDOW_MENU_LAYOUT } from "../../ui/window-menu";
@@ -64,12 +64,19 @@ type LauncherGridEntry =
* picks the item); a tap launches an app or opens a folder (the same grid,
* restricted to that folder's apps). Double-click backs out one level (item →
* row, folder → top grid), and from the top level yields to the sidebar.
* That is the ring's scheme; the watch (see handleWatchInput) skips row mode
* and moves one cell at a time in four directions.
*
* The folder grouping lives in the settings store (see launcher-folders.ts)
* and is re-read every paint, so assistant folder tools take effect without
* the layer holding any copy of the state.
*/
class LauncherGridLayer implements Layer {
// Watch swipes are spatial: up/down move between rows, left/right between
// columns. From row mode, right goes into the row (left backs out, like
// double-click); from the first column, left returns to row mode, which
// keeps "left" pointing toward the sidebar all the way out.
readonly acceptsDirectional = true;
private mode: LauncherMode = "row";
private selectedRow = 0;
private selectedCol = 1;
@@ -307,6 +314,10 @@ class LauncherGridLayer implements Layer {
}
async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
if (isWatchInput(event)) {
await this.handleWatchInput(event, ctx);
return;
}
const entries = this.entries();
const rows = this.rowCount(entries.length);
switch (event.type) {
@@ -341,20 +352,7 @@ class LauncherGridLayer implements Layer {
this.itemsInRow(entries.length, this.selectedRow) - 1,
);
} else {
const entry = entries[this.selectedRow * COLS + this.selectedCol];
if (!entry) return;
if (entry.kind === "folder") {
this.currentFolder = entry.name;
this.mode = "row";
this.selectedRow = 0;
this.scrollRow = 0;
} else {
// Return to the top grid before launching, so the launcher never
// shows (even briefly) stale folder contents when re-entered.
if (this.currentFolder !== null) this.exitFolder();
this.mode = "row";
await this.options.launchApp(entry.appId);
}
await this.openSelected(entries);
}
return;
}
@@ -371,6 +369,108 @@ class LauncherGridLayer implements Layer {
return;
}
}
/**
* The watch's scheme: there is no row mode to enter first. The selection is
* always one cell; up/down/left/right (and the crown) move it spatially,
* select opens it, back leaves the folder or the grid, and stepping left
* past the first column leaves for the sidebar, which is where "left" goes.
* Row mode is restored on the way out so a ring user finds the grid as
* they left it.
*/
private async handleWatchInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
const entries = this.entries();
const rows = this.rowCount(entries.length);
if (this.mode === "row") {
this.mode = "item";
this.selectedCol = clamp(this.selectedCol, 0, this.itemsInRow(entries.length, this.selectedRow) - 1);
}
const leave = () => {
this.mode = "row";
shell.yieldFocusToSidebar();
};
switch (event.type) {
case "swipe-up":
case "swipe-down":
case "scroll-up":
case "scroll-down": {
const delta = event.type === "swipe-down" || event.type === "scroll-down" ? 1 : -1;
this.selectedRow = clamp(this.selectedRow + delta, 0, rows - 1);
// Keep the column; a shorter last row clamps it.
this.selectedCol = clamp(this.selectedCol, 0, this.itemsInRow(entries.length, this.selectedRow) - 1);
return;
}
case "swipe-right":
this.selectedCol = clamp(this.selectedCol + 1, 0, this.itemsInRow(entries.length, this.selectedRow) - 1);
return;
case "swipe-left":
if (this.selectedCol > 0) {
this.selectedCol--;
} else if (this.currentFolder !== null) {
this.exitFolder();
this.mode = "item";
} else {
leave();
}
return;
case "click":
await this.openSelected(entries);
return;
case "double-click":
if (this.currentFolder !== null) {
this.exitFolder();
this.mode = "item";
} else {
leave();
}
return;
default:
return;
}
}
/**
* A touch on the phone's mirror: the cell under it becomes the selection
* and opens (the same as a watch select on it). Uses the geometry paint
* lays the grid out with, so it lands on what the mirror showed.
*/
async hitTest(x: number, y: number, ctx: LayerContext): Promise<boolean> {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const entries = this.entries();
const gridTop = this.currentFolder !== null ? GRID_TOP + font.lineHeight + 4 : GRID_TOP;
const gridBottom = height - 4;
if (y < gridTop || y >= gridBottom || x < 0 || x >= width) return false;
const rowH = iconGridMinRowHeight(font, ICON_SIZE, LABEL_GAP);
const colW = width / COLS;
const row = this.scrollRow + Math.floor((y - gridTop) / rowH);
const col = Math.floor(x / colW);
if (row < 0 || row >= this.rowCount(entries.length)) return false;
if (col < 0 || col >= this.itemsInRow(entries.length, row)) return false;
this.mode = "item";
this.selectedRow = row;
this.selectedCol = col;
await this.openSelected(entries);
return true;
}
/** Open the selected cell: enter a folder, or launch the app. */
private async openSelected(entries: readonly LauncherGridEntry[]): Promise<void> {
const entry = entries[this.selectedRow * COLS + this.selectedCol];
if (!entry) return;
if (entry.kind === "folder") {
this.currentFolder = entry.name;
this.mode = "row";
this.selectedRow = 0;
this.scrollRow = 0;
} else {
// Return to the top grid before launching, so the launcher never
// shows (even briefly) stale folder contents when re-entered.
if (this.currentFolder !== null) this.exitFolder();
this.mode = "row";
await this.options.launchApp(entry.appId);
}
}
}
/**
+25 -1
View File
@@ -17,7 +17,14 @@ import {
} from "../../native/media-controller";
import { mediaBrowserBridge, type MediaBrowserApp } from "../../native/media-browser";
import { MediaBrowseLayer } from "./media-browse";
import { Layer, type DashboardInputEvent, type LayerContext, type PaintBelow } from "../../ui/layers";
import {
directionalFallback,
isDirectionalInput,
Layer,
type DashboardInputEvent,
type LayerContext,
type PaintBelow,
} from "../../ui/layers";
import {
createInProcessWindow,
YieldAtRootLayer,
@@ -59,6 +66,8 @@ type FocusColumn = "actions" | "playlist";
* player's queue when it exposes one (scroll to a track, click to jump).
*/
class MusicAppLayer implements Layer {
// Watch swipes skip tracks: right = next, left = previous.
readonly acceptsDirectional = true;
private focusColumn: FocusColumn = "actions";
private selectedActionIndex = 0;
private selectedQueueIndex = 0;
@@ -215,6 +224,10 @@ class MusicAppLayer implements Layer {
return;
}
const media = mediaControllerBridge.snapshot();
if ((!media.accessEnabled || !media.available) && isDirectionalInput(event)) {
// Nothing to skip between yet: the setup prompts take select / back.
return this.handleInput(directionalFallback(event), ctx);
}
if (!media.accessEnabled) {
if (event.type === "click") {
mediaControllerBridge.openNotificationAccessSettings();
@@ -233,6 +246,16 @@ class MusicAppLayer implements Layer {
const actions = this.buildActions(media, queue);
this.reconcileSelection(actions, queue);
switch (event.type) {
case "swipe-right":
if (media.canSkipNext) await mediaControllerBridge.skipNext();
return;
case "swipe-left":
if (media.canSkipPrevious) await mediaControllerBridge.skipPrevious();
return;
case "swipe-up":
return this.handleInput({ type: "scroll-up" }, ctx);
case "swipe-down":
return this.handleInput({ type: "scroll-down" }, ctx);
case "scroll-up":
if (this.focusColumn === "playlist") {
this.selectedQueueIndex = Math.max(0, this.selectedQueueIndex - 1);
@@ -437,6 +460,7 @@ class VolumeModalLayer implements Layer {
/** Let back leave the playlist column before the root wrapper yields to the shell. */
class MusicRootLayer implements Layer {
readonly acceptsDirectional = true;
private readonly yieldAtRoot: YieldAtRootLayer;
constructor(private readonly music: MusicAppLayer) {
+264 -24
View File
@@ -1,5 +1,5 @@
import { Application, ImageSource } from "@nativescript/core";
import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName, WatchGestureType, WatchGestureTypeName } from "./events";
import { loadDeviceAddresses } from "./device-addresses";
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
import { FaceclawCommunicatorBridge, type RawInputEvent } from "../native/faceclaw-communicator";
@@ -12,6 +12,29 @@ import { openEvenAppSettings, readEvenAppNotificationState } from "../native/eve
import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { firmwareIncompatibilityMessage } from "./firmware-compat";
import { resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy";
import { WearRemote, type WearRemoteInputKind } from "./wear-remote";
/** Who a synthetic (non-firmware) input stands for. */
type SyntheticInputOrigin = "ring" | "watch";
/** Gestures the phone's mirror view reports (see handleMirrorTouch). */
export type MirrorTouchKind =
| "tap"
| "double-tap"
| "long-press"
| "swipe-up"
| "swipe-down"
| "swipe-left"
| "swipe-right";
const MIRROR_TOUCH_GESTURES: Record<Exclude<MirrorTouchKind, "tap">, WearRemoteInputKind> = {
"double-tap": "double-click",
"long-press": "long-press",
"swipe-up": "swipe-up",
"swipe-down": "swipe-down",
"swipe-left": "swipe-left",
"swipe-right": "swipe-right",
};
import { findSoundEffect, playSoundEffect } from "../ui/sound-effects";
import { isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
@@ -33,9 +56,9 @@ import { ALL_APPS } from "../apps/all-apps";
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry";
import { appViewportRect, SIDEBAR_WIDTH, sidebarWidth, type WindowHeightMode } from "../ui/shell/geometry";
import { type LayerActions } from "../ui/layers";
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, displayModeSetting, watchWakesDisplaySetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -84,13 +107,20 @@ const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glas
// Top-bar clock refresh; the phone-side preview polls the Java composite so
// it reflects every app (including worker apps the TS side never renders).
const SHELL_REFRESH_INTERVAL_MS = 60_000;
// Safety-net poll only; the preview is event-driven (every composited frame
// schedules a refresh), so this just covers anything that slips through.
const PREVIEW_INTERVAL_MS = 1_000;
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
const EVENHUB_SCREEN_OFF_SUSPEND_DELAY_MS = 5_000;
const EVENHUB_WAKE_READY_TIMEOUT_MS = 4_500;
const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000;
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000;
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000;
// Preview refresh floor. Refreshes are scheduled per composited frame with a
// trailing update, so the mirror tracks the glasses within this bound instead
// of the old 1s poll (which lagged up to two polls behind).
const CONNECTED_PREVIEW_MIN_UPDATE_MS = 150;
// The GIF recorder keeps its old cadence; per-frame captures would balloon it.
const RECORDING_MIN_CAPTURE_MS = 1_000;
// Below this, a disconnect is more likely a flat battery than a BLE problem.
const LOW_BATTERY_PERCENT = 5;
const EVEN_APP_DETECTED_MESSAGE =
@@ -138,6 +168,9 @@ function eventLabel(kind: string, eventType: number): string {
if (kind === "even-ai") {
return EvenAIStatusName[eventType] ?? `EVEN_AI_UNKNOWN_${eventType}`;
}
if (kind === "watch-gesture") {
return WatchGestureTypeName[eventType] ?? `WATCH_UNKNOWN_${eventType}`;
}
return eventName(eventType);
}
@@ -182,6 +215,8 @@ class DashboardController {
private glassesWorn: boolean | null = null;
private phoneLocked = false;
private glassesLocked = false;
// Wear OS watch remote; constructed last so it sees a fully wired shell.
private wearRemote: WearRemote | null = null;
private lockSurfaceConfigured = false;
private lastLockScreenEnabled = lockScreenEnabledSetting.get();
private offState: (() => void) | null = null;
@@ -265,10 +300,14 @@ class DashboardController {
},
getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()),
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.persistOpenApps(),
onWindowsChanged: () => {
this.persistOpenApps();
this.wearRemote?.schedulePublish();
},
onScreenStateChanged: (on) => {
this.handleScreenStateChanged(on);
if (on) this.requestShellRender();
this.wearRemote?.schedulePublish();
},
});
// Boot hooks register windows that exist from startup (the launcher,
@@ -290,6 +329,7 @@ class DashboardController {
this.pushBrightness();
this.syncEvenHubScreenOffSetting();
this.applyVerticalPositionIfChanged();
this.applyDisplayModeIfChanged();
this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged();
});
@@ -297,6 +337,24 @@ class DashboardController {
// connection stays up (with re-dial) so proactive tool calls work
// outside voice turns.
this.syncAssistantBridge();
// The watch drives the same synthetic-input path as the phone UI's test
// buttons, plus app/window/lock commands; it mirrors the state below.
this.wearRemote = new WearRemote({
apps: LAUNCHABLE_APPS,
injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"),
launchApp: (appId) => this.launchApp(appId),
connect: () => this.connect(),
disconnect: () => this.disconnect(),
setGlassesLocked: (locked, reason) => this.setGlassesLocked(locked, reason),
requestShellRender: () => this.requestShellRender(),
getState: () => ({
phase: this.phase,
status: this.status,
glassesLocked: this.glassesLocked,
glassesWorn: this.glassesWorn,
}),
appendLog: (line) => this.appendLog(line),
});
}
// Bridge settings changes re-dial the connection; unrelated setting changes
@@ -339,6 +397,48 @@ class DashboardController {
// don't trigger a full reposition.
private lastVerticalPosition = verticalPositionSetting.get();
private lastDisplayMode = displayModeSetting.get();
/**
* Display mode changed (Settings > Display, or the phone page's picker):
* every window's viewport size changes. In-process windows re-measure in
* place; worker windows get their canvas once at open, so they are closed
* and launched again at the new size.
*/
private applyDisplayModeIfChanged(): void {
const mode = displayModeSetting.get();
if (mode === this.lastDisplayMode) return;
this.lastDisplayMode = mode;
this.appendLog(`display mode: ${mode}`);
const foregroundWindowId = shell.foregroundWindow()?.windowId;
void (async () => {
const relaunch: string[] = [];
for (const window of Array.from(shell.getWindows())) {
if (window.relayout) {
window.relayout();
await this.configureWindowSurface(
window.surfaceId,
window.windowId === foregroundWindowId,
window.heightMode,
);
} else if (window.closeable) {
relaunch.push(window.appId);
shell.closeWindow(window.windowId);
}
}
for (const appId of relaunch) {
await this.launchApp(appId);
}
if (foregroundWindowId && shell.getWindows().some((w) => w.windowId === foregroundWindowId)) {
shell.focusWindow(foregroundWindowId);
}
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
})().catch((error) => {
this.appendLog(`display mode change failed: ${this.formatError(error)}`);
});
}
private applyVerticalPositionIfChanged(): void {
const position = verticalPositionSetting.get();
if (position === this.lastVerticalPosition) return;
@@ -400,6 +500,7 @@ class DashboardController {
private handleWearState(wearing: boolean): void {
this.glassesWorn = wearing;
this.appendLog(wearing ? "glasses wear state: ON_HEAD" : "glasses wear state: OFF_HEAD");
this.wearRemote?.schedulePublish();
if (!wearing && this.phoneLocked && lockScreenEnabledSetting.get()) {
this.setGlassesLocked(true, "glasses removed while phone locked");
}
@@ -420,6 +521,7 @@ class DashboardController {
if (locked === this.glassesLocked) return;
this.glassesLocked = locked;
this.appendLog(`glasses ${locked ? "locked" : "unlocked"}: ${reason}`);
this.wearRemote?.schedulePublish();
void this.syncLockSurface().catch((error) => {
this.appendLog(`lock screen update failed: ${this.formatError(error)}`);
});
@@ -947,6 +1049,7 @@ class DashboardController {
// Repaint the top bar (battery indicators live in the shell chrome).
this.requestShellRender();
}
this.wearRemote?.schedulePublish();
});
this.offEvenAppConflict = communicator.onEvenAppConflict((message) => {
this.refreshEvenAppStatus();
@@ -959,6 +1062,10 @@ class DashboardController {
this.emit();
});
this.offFrameMetrics = communicator.onFrameMetrics(() => {
// Every composited frame refreshes the phone-side mirror (bounded by
// CONNECTED_PREVIEW_MIN_UPDATE_MS) so it tracks the glasses instead
// of trailing the 1s safety-net poll.
this.schedulePreviewUpdate();
if (this.phase === "connected") {
this.setStatus("Connected.");
// A rendered frame means the session is warmed up (fixedLayoutCreated),
@@ -1255,20 +1362,94 @@ class DashboardController {
}
}
async injectSyntheticRingInput(
kind: "click" | "double-click" | "scroll-up" | "scroll-down" | "long-press" | "wakeword",
): Promise<void> {
/**
* Feed a ring gesture from somewhere other than the ring (the phone UI's
* test buttons, the watch). "long-press" is a complete short hold;
* "long-press-start" / "long-press-release" let a source with a real
* finger-down/finger-up (the watch) hold for as long as the user does.
*/
async injectSyntheticRingInput(kind: WearRemoteInputKind, origin: SyntheticInputOrigin = "ring"): Promise<void> {
// Watch-scheme input (the Wear app, the phone's pad/d-pad, mirror touches)
// wakes a dark display and then lands normally, resuming the previous
// focus. The ring's own scheme keeps its double-tap-only wake.
if (
origin === "watch" &&
!shell.isScreenOn() &&
!this.glassesLocked &&
(this.phase === "connected" || this.phase === "charging") &&
watchWakesDisplaySetting.get() &&
kind !== "double-click" &&
kind !== "wakeword" &&
kind !== "long-press-release"
) {
shell.wake(shell.getFocus());
this.requestShellRender();
this.appendLog(`${kind} (watch scheme) woke the display`);
}
if (kind === "long-press") {
// Hardware delivers a press event and then a release when the finger
// lifts; emulate a short hold so the escape-menu countdown never fires.
await this.handleInputEvent(this.buildSyntheticRingInput("long-press"));
await this.handleInputEvent(this.buildSyntheticRingInput("long-press-release"));
await this.handleInputEvent(this.buildSyntheticRingInput("long-press", origin));
await this.handleInputEvent(this.buildSyntheticRingInput("long-press-release", origin));
return;
}
const event = this.buildSyntheticRingInput(kind);
const event = this.buildSyntheticRingInput(kind === "long-press-start" ? "long-press" : kind, origin);
await this.handleInputEvent(event);
}
/**
* A touch on the phone's mirror of the glasses display. Gestures other than
* a tap are the watch scheme (swipes navigate, double-tap is back, a hold
* is the menu). A tap lands on what the mirror shows: a sidebar icon
* switches to that window, a launcher cell opens, anything else selects.
* Coordinates are fractions of the mirror (0..1).
*/
async handleMirrorTouch(kind: MirrorTouchKind, nx: number, ny: number): Promise<void> {
if (this.phase !== "connected" && this.phase !== "charging") return;
if (this.glassesLocked) {
// Same as the ring: only a double-tap reaches a locked display.
if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch");
return;
}
if (!shell.isScreenOn()) {
shell.wake(shell.getFocus());
this.requestShellRender();
}
if (kind !== "tap") {
await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch");
return;
}
const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH);
const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT);
const stripShown = sidebarWidth() > 0 || shell.getFocus() === "sidebar";
this.appendLog(`mirror tap at ${x},${y}`);
if (!shell.hasOverlay() && stripShown && x < SIDEBAR_WIDTH) {
const target = shell.windowAtSidebarPoint(x, y);
if (target) {
this.appendLog(`mirror tap: sidebar -> ${target.title}`);
shell.focusWindow(target.windowId);
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
}
return;
}
const window = shell.foregroundWindow();
if (window && !shell.hasOverlay()) {
const rect = appViewportRect(window.heightMode);
const inside = x >= rect.x && x < rect.x + rect.width && y >= rect.y && y < rect.y + rect.height;
if (shell.getFocus() !== "window") {
shell.focusWindow(window.windowId);
this.requestShellRender();
}
if (inside && window.hitTest && (await window.hitTest(x - rect.x, y - rect.y))) {
this.appendLog(`mirror tap: ${window.title} at ${x - rect.x},${y - rect.y}`);
this.requestShellRender();
return;
}
}
await this.injectSyntheticRingInput("click", "watch");
}
/** A document arrived via Android's Share intent: open it as a new window. */
async openSharedTextDocument(text: string): Promise<void> {
this.appendLog(`shared text document received (${text.length} chars)`);
@@ -1410,7 +1591,8 @@ class DashboardController {
if (this.glassesLocked) {
const ringDoubleTap =
inputEvent.type === "double-click" &&
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING;
(event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING ||
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH);
const suspendedDisplayWake = inputEvent.type === "display-wake";
if (ringDoubleTap || suspendedDisplayWake) {
if (shell.isScreenOn()) {
@@ -1463,6 +1645,9 @@ class DashboardController {
outcome = { ...outcome, shell: true };
}
if (event.kind === "watch-gesture") {
this.appendLog(`watch-gesture ${eventLabel(event.kind, event.eventType)}`);
}
if (event.kind === "sys-event") {
this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`;
this.appendLog(`sys-event ${this.lastSys}`);
@@ -1473,7 +1658,10 @@ class DashboardController {
) {
this.appendLog("display state invalidated by firmware exit event");
}
if (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
if (
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING ||
event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH
) {
this.lastInput = eventName(event.eventType);
}
}
@@ -1813,7 +2001,7 @@ class DashboardController {
`still unsent after ${FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS}ms; shell render loop released`,
);
}
this.updateCompositePreview();
this.schedulePreviewUpdate();
}
private async handleWakeWord(keyword: string): Promise<void> {
@@ -1878,6 +2066,10 @@ class DashboardController {
clearInterval(this.previewTimer);
this.previewTimer = null;
}
if (this.previewTrailingTimer) {
clearTimeout(this.previewTrailingTimer);
this.previewTrailingTimer = null;
}
if (this.screenTimeoutTimer) {
clearInterval(this.screenTimeoutTimer);
this.screenTimeoutTimer = null;
@@ -1928,6 +2120,27 @@ class DashboardController {
* foreground, including worker apps the TS side never renders). Throttled
* to avoid rebuilding the bitmap faster than the phone UI needs it.
*/
private previewTrailingTimer: ReturnType<typeof setTimeout> | null = null;
private lastRecordCaptureAtMs = 0;
/**
* Refresh the preview now if the floor allows, otherwise once the floor
* expires — so the last frame of a burst always reaches the mirror instead
* of waiting for the safety-net poll.
*/
private schedulePreviewUpdate(): void {
if (this.previewTrailingTimer) return;
const wait = this.lastConnectedPreviewUpdateAtMs + CONNECTED_PREVIEW_MIN_UPDATE_MS - Date.now();
if (wait <= 0) {
this.updateCompositePreview();
return;
}
this.previewTrailingTimer = setTimeout(() => {
this.previewTrailingTimer = null;
this.updateCompositePreview();
}, wait);
}
private updateCompositePreview(): void {
if (!this.communicator) return;
// The connected foreground service intentionally keeps this controller
@@ -1947,9 +2160,8 @@ class DashboardController {
return;
}
this.lastConnectedPreviewUpdateAtMs = now;
// The recorder captures at the same cadence the phone-side preview is
// refreshed, so the GIF matches what the phone display showed.
if (this.screenRecordingActive) {
if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) {
this.lastRecordCaptureAtMs = now;
this.communicator.recordScreenFrame();
}
const preview = this.communicator.getCompositePreview();
@@ -1973,10 +2185,37 @@ class DashboardController {
| "scroll-down"
| "long-press"
| "long-press-release"
| "wakeword",
| "wakeword"
| "swipe-left"
| "swipe-right"
| "swipe-up"
| "swipe-down",
origin: SyntheticInputOrigin = "ring",
): RawInputEvent {
const frameId = frameTimings.startFrame(`input:synthetic:${kind}`);
// The phone UI's test buttons stand in for the ring; the watch is its own
// source so the UI can give it a richer scheme (see InputSource).
const eventSource =
origin === "watch" ? EventSourceType.TOUCH_EVENT_FROM_WATCH : EventSourceType.TOUCH_EVENT_FROM_RING;
switch (kind) {
case "swipe-left":
case "swipe-right":
case "swipe-up":
case "swipe-down":
// Watch-only directional input; see WatchGestureType.
return {
kind: "watch-gesture",
containerName: "",
eventType: {
"swipe-left": WatchGestureType.SWIPE_LEFT,
"swipe-right": WatchGestureType.SWIPE_RIGHT,
"swipe-up": WatchGestureType.SWIPE_UP,
"swipe-down": WatchGestureType.SWIPE_DOWN,
}[kind],
eventSource: 0,
systemExitReasonCode: 0,
frameId,
};
case "wakeword":
// Same event the glasses report for the spoken wakeword (sid 0x07).
return {
@@ -1992,7 +2231,7 @@ class DashboardController {
kind: "sys-event",
containerName: "",
eventType: OsEventTypeList.RING_LONG_PRESS_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
eventSource,
systemExitReasonCode: 0,
frameId,
};
@@ -2001,7 +2240,7 @@ class DashboardController {
kind: "sys-event",
containerName: "",
eventType: OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
eventSource,
systemExitReasonCode: 0,
frameId,
};
@@ -2010,7 +2249,7 @@ class DashboardController {
kind: "sys-event",
containerName: "",
eventType: OsEventTypeList.CLICK_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
eventSource,
systemExitReasonCode: 0,
frameId,
};
@@ -2019,7 +2258,7 @@ class DashboardController {
kind: "sys-event",
containerName: "",
eventType: OsEventTypeList.DOUBLE_CLICK_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
eventSource,
systemExitReasonCode: 0,
frameId,
};
@@ -2028,7 +2267,7 @@ class DashboardController {
kind: "text-click",
containerName: "",
eventType: OsEventTypeList.SCROLL_TOP_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
eventSource,
systemExitReasonCode: 0,
frameId,
};
@@ -2038,7 +2277,7 @@ class DashboardController {
kind: "text-click",
containerName: "",
eventType: OsEventTypeList.SCROLL_BOTTOM_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FROM_RING,
eventSource,
systemExitReasonCode: 0,
frameId,
};
@@ -2050,6 +2289,7 @@ class DashboardController {
for (const listener of this.listeners) {
listener(snapshot);
}
this.wearRemote?.schedulePublish();
}
}
+25
View File
@@ -3,6 +3,12 @@ export const EventSourceType = {
TOUCH_EVENT_FROM_GLASSES_R: 1,
TOUCH_EVENT_FROM_RING: 2,
TOUCH_EVENT_FROM_GLASSES_L: 3,
/**
* Synthetic: the Wear OS remote (app/g2/wear-remote.ts). Not a firmware
* value. Lets the UI give watch input its own, richer scheme (direct
* spatial selection) while ring input keeps the stock one.
*/
TOUCH_EVENT_FROM_WATCH: 4,
} as const;
export const EventSourceTypeName: Record<number, string> = {
@@ -10,6 +16,7 @@ export const EventSourceTypeName: Record<number, string> = {
1: "TOUCH_EVENT_FROM_GLASSES_R",
2: "TOUCH_EVENT_FROM_RING",
3: "TOUCH_EVENT_FROM_GLASSES_L",
4: "TOUCH_EVENT_FROM_WATCH",
};
/**
@@ -32,6 +39,24 @@ export const EvenAIStatusName: Record<number, string> = {
3: "EVEN_AI_EXIT",
};
/**
* Synthetic gestures from the Wear OS remote (RawInputEvent kind
* "watch-gesture"). Not a firmware enum: the glasses never send these.
*/
export const WatchGestureType = {
SWIPE_LEFT: 0,
SWIPE_RIGHT: 1,
SWIPE_UP: 2,
SWIPE_DOWN: 3,
} as const;
export const WatchGestureTypeName: Record<number, string> = {
0: "SWIPE_LEFT",
1: "SWIPE_RIGHT",
2: "SWIPE_UP",
3: "SWIPE_DOWN",
};
export const OsEventTypeList = {
CLICK_EVENT: 0,
SCROLL_TOP_EVENT: 1,
+490
View File
@@ -0,0 +1,490 @@
/**
* The Wear OS watch as an input device and status mirror for the glasses.
*
* Inbound, the watch sends ring-style gestures (tap, double-tap, hold,
* swipe/crown scrolls, "Hey Even") which feed the exact synthetic-ring path
* the phone UI's test buttons use, plus higher-level commands the ring has no
* way to express: launch an app, switch or close a window, wake/sleep/lock
* the display, send a spoken or typed query straight to the assistant, or
* type text into the foreground app.
*
* Outbound, the dashboard state (connection, battery, screen, lock, open
* windows, launchable apps) is mirrored into a Data Layer item so the watch
* UI can show it and offer the right actions; assistant replies and alerts
* stream to the watch as events so a query made from the wrist can be read
* from the wrist.
*
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the
* message shapes are documented in wear/PROTOCOL.md.
*/
import { type AppDefinition } from "../apps/app-definition";
import { getInstalledEvenHubApps, installedEvenHubAppId } from "../apps/evenhub/installed-apps";
import { WEAR_PATHS, wearBridge, type WearMessage } from "../native/wear-bridge";
import {
DISPLAY_MODE_VALUES,
displayModeSetting,
onAnySettingChanged,
watchCanUnlockSetting,
watchMirrorAssistantSetting,
watchRemoteEnabledSetting,
} from "../ui/dashboard-settings";
import { shell, type AssistantActivityEvent } from "../ui/shell/shell";
import { voiceActivity } from "../ui/shell/voice-activity";
import { FACECLAW_VERSION } from "../version";
/** Protocol revision carried in the state item; bump on incompatible changes. */
export const WEAR_PROTOCOL_VERSION = 1;
export type WearRemoteInputKind =
| "click"
| "double-click"
| "scroll-up"
| "scroll-down"
| "long-press"
| "long-press-start"
| "long-press-release"
| "wakeword"
| "swipe-left"
| "swipe-right"
| "swipe-up"
| "swipe-down";
const INPUT_KINDS: readonly WearRemoteInputKind[] = [
"click",
"double-click",
"scroll-up",
"scroll-down",
"long-press",
"long-press-start",
"long-press-release",
"wakeword",
"swipe-left",
"swipe-right",
"swipe-up",
"swipe-down",
];
const COMMANDS = [
"launch-app",
"focus-window",
"close-window",
"sidebar",
"wake",
"sleep",
"lock",
"unlock",
"connect",
"disconnect",
"close-assistant",
"display-mode",
] as const;
type WearCommand = (typeof COMMANDS)[number];
const MAX_SCROLL_STEPS = 12;
const MAX_TEXT_LENGTH = 2000;
const STATE_PUBLISH_DEBOUNCE_MS = 150;
const STATE_REFRESH_INTERVAL_MS = 30_000;
const ASSISTANT_STREAM_MIN_INTERVAL_MS = 250;
/** What the dashboard controller lends the watch remote. */
export type WearRemoteHost = {
/** The launcher-grid apps; installed EvenHub packages are added dynamically. */
apps: readonly AppDefinition[];
injectInput: (kind: WearRemoteInputKind) => Promise<void>;
launchApp: (appId: string) => Promise<void>;
connect: () => Promise<void>;
disconnect: () => Promise<void>;
setGlassesLocked: (locked: boolean, reason: string) => void;
requestShellRender: () => void;
getState: () => {
phase: string;
status: string;
glassesLocked: boolean;
glassesWorn: boolean | null;
};
appendLog: (line: string) => void;
};
export class WearRemote {
private readonly active: boolean;
private publishTimer: ReturnType<typeof setTimeout> | null = null;
private watchReachable = false;
private watchName = "";
// Assistant text streams as deltas many times a second; the watch gets a
// snapshot every ASSISTANT_STREAM_MIN_INTERVAL_MS plus every phase change.
private assistantStreamTimer: ReturnType<typeof setTimeout> | null = null;
private assistantPendingText: string | null = null;
private lastAssistantSentAtMs = 0;
constructor(private readonly host: WearRemoteHost) {
this.active = wearBridge.isAvailable();
if (!this.active) {
host.appendLog("watch: Google Play services unavailable; watch remote disabled");
return;
}
wearBridge.onMessage((message) => {
void this.handleMessage(message).catch((error) => {
host.appendLog(`watch: ${message.path} failed: ${formatError(error)}`);
});
});
wearBridge.onWatchConnection((connection) => {
const changed = connection.reachable !== this.watchReachable || connection.watchName !== this.watchName;
this.watchReachable = connection.reachable;
this.watchName = connection.watchName;
if (changed) {
host.appendLog(
connection.reachable ? `watch: ${connection.watchName || "watch"} reachable` : "watch: no watch reachable",
);
}
// A watch that just came into range gets the current state at once.
if (connection.reachable) this.publishNow(true);
});
shell.onAssistantActivity((event) => this.forwardAssistantActivity(event));
shell.onAlertShown((text) => {
if (watchMirrorAssistantSetting.get()) {
wearBridge.sendEvent({ type: "alert", text });
}
});
voiceActivity.subscribe(() => this.schedulePublish());
onAnySettingChanged(() => this.schedulePublish());
// Catch-all refresh for state the controller does not announce (cheap:
// identical states are dropped on the Java side).
setInterval(() => this.schedulePublish(), STATE_REFRESH_INTERVAL_MS);
this.schedulePublish();
}
/** Whether a watch running the Faceclaw watch app is reachable right now. */
isWatchReachable(): boolean {
return this.watchReachable;
}
getWatchName(): string {
return this.watchName;
}
/** Mirror the dashboard state to the watch soon; coalesces bursts. */
schedulePublish(): void {
if (!this.active || this.publishTimer) return;
this.publishTimer = setTimeout(() => {
this.publishTimer = null;
this.publishNow(false);
}, STATE_PUBLISH_DEBOUNCE_MS);
}
private publishNow(force: boolean): void {
if (!this.active) return;
try {
wearBridge.publishState(this.buildState(), force);
} catch (error) {
this.host.appendLog(`watch: state publish failed: ${formatError(error)}`);
}
}
private buildState(): Record<string, unknown> {
const state = this.host.getState();
const battery = shell.getBatteryLevels();
const foregroundId = shell.foregroundWindow()?.windowId ?? null;
const foreground = shell.getForegroundApp();
return {
protocol: WEAR_PROTOCOL_VERSION,
version: FACECLAW_VERSION,
phase: state.phase,
status: state.status,
connected: state.phase === "connected" || state.phase === "charging",
screenOn: shell.isScreenOn(),
locked: state.glassesLocked,
worn: state.glassesWorn,
listening: voiceActivity.isActive(),
battery: battery.headset,
charging: battery.headsetCharging === true,
foreground: foreground ? { appId: foreground.appId, title: foreground.title } : null,
windows: shell.getWindows().map((window) => ({
windowId: window.windowId,
title: window.title,
appId: window.appId,
focused: window.windowId === foregroundId,
closeable: window.closeable,
acceptsText: typeof window.receiveTextInput === "function",
})),
apps: this.listApps(),
displayMode: displayModeSetting.get(),
remoteEnabled: watchRemoteEnabledSetting.get(),
canUnlock: watchCanUnlockSetting.get(),
mirrorAssistant: watchMirrorAssistantSetting.get(),
assistantAvailable: shell.isAssistantAvailable(),
};
}
/** Launchable apps, matching the launcher grid and the apps.launch tool. */
private listApps(): Array<{ appId: string; title: string }> {
const apps = this.host.apps.map((app) => ({ appId: app.appId, title: app.title }));
for (const installed of getInstalledEvenHubApps()) {
apps.push({ appId: installedEvenHubAppId(installed.packageId), title: installed.name });
}
return apps;
}
private async handleMessage(message: WearMessage): Promise<void> {
const seq = readNumber(message.payload.seq, 0);
const ack = (ok: boolean, text = "") => wearBridge.sendAck(message.nodeId, seq, ok, text);
if (message.path === WEAR_PATHS.stateRequest) {
this.publishNow(true);
ack(true);
return;
}
if (!watchRemoteEnabledSetting.get()) {
ack(false, "Watch control is turned off in Faceclaw's Settings > Watch.");
return;
}
switch (message.path) {
case WEAR_PATHS.input:
await this.handleInput(message.payload, ack);
return;
case WEAR_PATHS.command:
await this.handleCommand(message.payload, ack);
return;
case WEAR_PATHS.assistant: {
const text = readText(message.payload.text);
if (!text) {
ack(false, "Nothing to send.");
return;
}
if (!shell.isAssistantAvailable()) {
ack(false, "Set up the assistant in Faceclaw's Settings first.");
return;
}
this.host.appendLog(`watch: assistant query (${text.length} chars)`);
shell.sendToAssistant(text);
ack(true);
return;
}
case WEAR_PATHS.text: {
const text = readText(message.payload.text);
if (!text) {
ack(false, "Nothing to type.");
return;
}
const window = shell.foregroundWindow();
if (!window?.receiveTextInput) {
ack(false, `${window ? window.title : "The current app"} does not accept typed text.`);
return;
}
if (!shell.isScreenOn()) shell.wake("window");
this.host.appendLog(`watch: typed ${text.length} chars into ${window.title}`);
shell.sendTextToForegroundWindow(text);
ack(true);
return;
}
default:
ack(false, `Unknown message: ${message.path}`);
}
}
private async handleInput(payload: Record<string, unknown>, ack: (ok: boolean, text?: string) => void): Promise<void> {
const gesture = String(payload.gesture ?? "");
if (!(INPUT_KINDS as readonly string[]).includes(gesture)) {
ack(false, `Unknown gesture: ${gesture}`);
return;
}
const kind = gesture as WearRemoteInputKind;
const state = this.host.getState();
if (state.phase !== "connected" && state.phase !== "charging") {
ack(false, "The glasses are not connected.");
return;
}
// The lock only yields to a double-click, exactly as the ring's does
// (handled inside the controller); a dark display wakes on any watch-
// scheme gesture (injectSyntheticRingInput, shared with the phone's own
// pad and d-pad).
const repeatable = kind === "scroll-up" || kind === "scroll-down" || kind === "swipe-up" || kind === "swipe-down";
const steps = repeatable ? Math.min(MAX_SCROLL_STEPS, Math.max(1, Math.round(readNumber(payload.steps, 1)))) : 1;
for (let i = 0; i < steps; i++) {
await this.host.injectInput(kind);
}
ack(true);
}
private async handleCommand(payload: Record<string, unknown>, ack: (ok: boolean, text?: string) => void): Promise<void> {
const command = String(payload.command ?? "");
if (!(COMMANDS as readonly string[]).includes(command)) {
ack(false, `Unknown command: ${command}`);
return;
}
const state = this.host.getState();
const connected = state.phase === "connected" || state.phase === "charging";
this.host.appendLog(`watch: ${command}`);
switch (command as WearCommand) {
case "connect":
if (connected) {
ack(true, "Already connected.");
return;
}
ack(true, "Connecting...");
await this.host.connect();
return;
case "disconnect":
if (!connected && state.phase === "disconnected") {
ack(true, "Already disconnected.");
return;
}
ack(true, "Disconnecting...");
await this.host.disconnect();
return;
default:
break;
}
if (!connected) {
ack(false, "The glasses are not connected.");
return;
}
if (state.glassesLocked && command !== "unlock" && command !== "lock") {
ack(false, "The glasses are locked.");
return;
}
switch (command as WearCommand) {
case "launch-app": {
const appId = String(payload.appId ?? "").trim();
const app = this.listApps().find((entry) => entry.appId === appId);
if (!app) {
ack(false, `Unknown app: ${appId}`);
return;
}
shell.wake("window");
await this.host.launchApp(appId);
this.schedulePublish();
ack(true, `Opened ${app.title}.`);
return;
}
case "focus-window": {
const windowId = String(payload.windowId ?? "").trim();
const window = shell.getWindows().find((entry) => entry.windowId === windowId);
if (!window) {
ack(false, "That window is no longer open.");
return;
}
shell.wake("window");
shell.focusWindow(windowId);
this.host.requestShellRender();
this.schedulePublish();
ack(true);
return;
}
case "close-window": {
const windowId = String(payload.windowId ?? "").trim();
const window = shell.getWindows().find((entry) => entry.windowId === windowId);
if (!window) {
ack(false, "That window is no longer open.");
return;
}
if (!window.closeable) {
ack(false, `${window.title} cannot be closed.`);
return;
}
shell.closeWindow(windowId);
this.host.requestShellRender();
this.schedulePublish();
ack(true, `Closed ${window.title}.`);
return;
}
case "sidebar":
shell.wake("sidebar");
shell.yieldFocusToSidebar();
this.host.requestShellRender();
ack(true);
return;
case "wake":
if (!shell.wake("sidebar")) {
ack(true, "The display is already on.");
return;
}
this.host.requestShellRender();
ack(true);
return;
case "sleep":
if (!shell.isScreenOn()) {
ack(true, "The display is already off.");
return;
}
shell.sleep();
ack(true);
return;
case "lock":
this.host.setGlassesLocked(true, "locked from the watch");
this.schedulePublish();
ack(true);
return;
case "unlock":
if (!watchCanUnlockSetting.get()) {
ack(false, "Unlocking from the watch is turned off in Settings > Watch.");
return;
}
this.host.setGlassesLocked(false, "unlocked from the watch");
this.schedulePublish();
ack(true);
return;
case "close-assistant":
shell.closeAssistant();
ack(true);
return;
case "display-mode": {
const value = String(payload.value ?? "");
if (!(DISPLAY_MODE_VALUES as readonly string[]).includes(value)) {
ack(false, `Unknown display mode: ${value}`);
return;
}
displayModeSetting.set(value as (typeof DISPLAY_MODE_VALUES)[number]);
this.schedulePublish();
ack(true, `Display mode: ${value}`);
return;
}
default:
ack(false, `Unhandled command: ${command}`);
}
}
private forwardAssistantActivity(event: AssistantActivityEvent): void {
if (!watchMirrorAssistantSetting.get()) return;
if (event.phase === "streaming") {
// Coalesce: the newest text wins, sent no more often than the interval.
this.assistantPendingText = event.text;
if (this.assistantStreamTimer) return;
const wait = Math.max(0, ASSISTANT_STREAM_MIN_INTERVAL_MS - (Date.now() - this.lastAssistantSentAtMs));
this.assistantStreamTimer = setTimeout(() => {
this.assistantStreamTimer = null;
if (this.assistantPendingText === null) return;
this.sendAssistantEvent("streaming", this.assistantPendingText);
this.assistantPendingText = null;
}, wait);
return;
}
// A phase change supersedes any queued stream snapshot.
if (this.assistantStreamTimer) {
clearTimeout(this.assistantStreamTimer);
this.assistantStreamTimer = null;
}
this.assistantPendingText = null;
this.sendAssistantEvent(event.phase, event.text);
this.schedulePublish();
}
private sendAssistantEvent(phase: AssistantActivityEvent["phase"], text: string): void {
this.lastAssistantSentAtMs = Date.now();
wearBridge.sendEvent({ type: "assistant", phase, text });
}
}
function readNumber(value: unknown, fallback: number): number {
const numeric = typeof value === "number" ? value : Number(value);
return Number.isFinite(numeric) ? numeric : fallback;
}
function readText(value: unknown): string {
const text = typeof value === "string" ? value : "";
return text.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "").trim().slice(0, MAX_TEXT_LENGTH);
}
function formatError(error: unknown): string {
return (error as Error)?.message ?? String(error);
}
+14
View File
@@ -533,6 +533,20 @@ export class GrayImage {
return baked;
}
/**
* Place this image into target at (dx, dy): raster via bitBlt, deferred
* draws carried over as deferred draws (translated), so firmware-text runs
* and cached glyphs stay on their cheap wire paths instead of being baked
* to raster. The composed-in draws render above target's earlier raster,
* matching the usual draw-above-own-raster rule.
*/
composeInto(target: GrayImage, dx: number, dy: number): void {
target.bitBlt(this, dx, dy);
for (const placed of this.drawList) {
target.drawList.push({ ...placed, x: placed.x + dx, y: placed.y + dy });
}
}
/** Rasterize this image's deferred draws into target, translated by (dx, dy). */
bakeDrawsInto(target: GrayImage, dx: number, dy: number): void {
for (const placed of this.drawList) {
+12
View File
@@ -89,6 +89,18 @@ export type RawInputEvent =
systemExitReasonCode: number;
frameId: number;
}
| {
/**
* Synthetic directional gesture from the Wear OS remote; `eventType` is
* a WatchGestureType (app/g2/events.ts). Never produced by Java.
*/
kind: "watch-gesture";
containerName: string;
eventType: number;
eventSource: number;
systemExitReasonCode: number;
frameId: number;
}
| {
/**
* Stock display-lifecycle wake observed after a ring or arm double tap
+178
View File
@@ -0,0 +1,178 @@
/**
* TS wrapper for the Java FaceclawWearBridge: the phone side of the Wear OS
* watch remote. Inbound watch messages arrive as (path, JSON) pairs on the
* main isolate's thread; outbound calls publish dashboard state and send
* messages to the watch. The semantic mapping (what a watch "click" does)
* lives in app/g2/wear-remote.ts; this file only crosses the Java boundary.
*
* Every call is a no-op when Google Play services is missing (the Wearable
* Data Layer is part of it), so the rest of the app never has to care.
*/
import { Utils } from "@nativescript/core";
declare const com: any;
/** Message paths shared with the watch app (wear/app/.../Protocol.kt). */
export const WEAR_PATHS = {
/** Watch -> phone: a ring-style gesture. */
input: "/faceclaw/input",
/** Watch -> phone: a shell command (launch app, wake, lock, ...). */
command: "/faceclaw/command",
/** Watch -> phone: a text query for the assistant. */
assistant: "/faceclaw/assistant",
/** Watch -> phone: text to type into the foreground app. */
text: "/faceclaw/text",
/** Watch -> phone: please re-publish the state item. */
stateRequest: "/faceclaw/state/request",
/** Phone -> watch: reply to any of the above. */
ack: "/faceclaw/ack",
/** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */
event: "/faceclaw/event",
/** Phone -> watch Data Layer item: the mirrored dashboard state. */
state: "/faceclaw/state",
} as const;
export type WearMessage = {
path: string;
/** Parsed JSON payload ({} when the watch sent none or it was malformed). */
payload: Record<string, unknown>;
nodeId: string;
};
export type WearWatchConnection = {
reachable: boolean;
watchName: string;
};
type MessageListener = (message: WearMessage) => void;
type ConnectionListener = (connection: WearWatchConnection) => void;
class WearBridge {
private java: any | null | undefined = undefined;
// The Java-side listener proxy must stay referenced or it gets GC'd.
private retainedListenerProxy: any = null;
private readonly messageListeners = new Set<MessageListener>();
private readonly connectionListeners = new Set<ConnectionListener>();
private lastConnection: WearWatchConnection = { reachable: false, watchName: "" };
private getJava(): any | null {
if (this.java !== undefined) return this.java;
if (!global.isAndroid) {
this.java = null;
return null;
}
try {
const context = Utils.android.getApplicationContext();
const bridge = context ? com.faceclaw.app.FaceclawWearBridge.getInstance(context) : null;
this.java = bridge && bridge.isAvailable() ? bridge : null;
} catch (error) {
console.warn("wear bridge unavailable", error);
this.java = null;
}
return this.java;
}
/** Whether the Wearable Data Layer exists on this phone (Play services). */
isAvailable(): boolean {
return this.getJava() !== null;
}
private ensureListener(): void {
if (this.retainedListenerProxy) return;
const java = this.getJava();
if (!java) return;
this.retainedListenerProxy = new com.faceclaw.app.FaceclawWearListener({
onMessage: (path: string, json: string, nodeId: string) => {
const message: WearMessage = {
path: String(path),
payload: parsePayload(String(json)),
nodeId: String(nodeId),
};
for (const listener of Array.from(this.messageListeners)) {
try {
listener(message);
} catch (error) {
console.warn("wear message listener failed", error);
}
}
},
onWatchConnection: (reachable: boolean, watchName: string) => {
this.lastConnection = { reachable: Boolean(reachable), watchName: String(watchName ?? "") };
for (const listener of Array.from(this.connectionListeners)) {
try {
listener(this.lastConnection);
} catch (error) {
console.warn("wear connection listener failed", error);
}
}
},
});
java.setListener(this.retainedListenerProxy);
}
/** Subscribe to watch messages. Registers the Java listener on first use. */
onMessage(listener: MessageListener): () => void {
this.messageListeners.add(listener);
this.ensureListener();
return () => {
this.messageListeners.delete(listener);
};
}
/** Subscribe to watch reachability; the current value is delivered immediately. */
onWatchConnection(listener: ConnectionListener): () => void {
this.connectionListeners.add(listener);
this.ensureListener();
listener(this.getWatchConnection());
return () => {
this.connectionListeners.delete(listener);
};
}
getWatchConnection(): WearWatchConnection {
const java = this.getJava();
if (!java) return { reachable: false, watchName: "" };
try {
return {
reachable: Boolean(java.isWatchReachable()),
watchName: String(java.getWatchName() ?? ""),
};
} catch {
return this.lastConnection;
}
}
/** Re-query reachable watches (e.g. when the phone UI wants a fresh answer). */
refreshWatchNodes(): void {
this.getJava()?.refreshWatchNodes();
}
/**
* Mirror the dashboard state to the watch. Unchanged states are dropped on
* the Java side unless `force` (a watch explicitly asked for a refresh).
*/
publishState(state: Record<string, unknown>, force = false): void {
this.getJava()?.publishState(JSON.stringify(state), force);
}
/** Send an event to every reachable watch. */
sendEvent(payload: Record<string, unknown>): void {
this.getJava()?.sendToWatch(WEAR_PATHS.event, JSON.stringify(payload));
}
/** Acknowledge a watch message; `seq` echoes the watch's sequence number. */
sendAck(nodeId: string, seq: number, ok: boolean, message = ""): void {
this.getJava()?.sendAck(nodeId, seq, ok, true, message);
}
}
function parsePayload(json: string): Record<string, unknown> {
try {
const parsed = JSON.parse(json);
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
} catch {
return {};
}
}
export const wearBridge = new WearBridge();
+8 -20
View File
@@ -1,4 +1,4 @@
<Page xmlns="http://schemas.nativescript.org/tns.xsd" navigatingTo="navigatingTo" loaded="loaded" unloaded="unloaded">
<Page xmlns="http://schemas.nativescript.org/tns.xsd" xmlns:rc="phone-ui" navigatingTo="navigatingTo" loaded="loaded" unloaded="unloaded">
<ActionBar title="Faceclaw">
<ActionItem text="{{ buttonLabel }}" tap="{{ onTap }}" isEnabled="{{ canRun }}" android.position="popup" />
<ActionItem text="Configure devices" tap="{{ onConfigureTap }}" isEnabled="{{ canRun }}" android.position="popup" />
@@ -11,7 +11,8 @@
<GridLayout>
<GridLayout rows="auto,*" visibility="{{ portraitLayoutVisibility }}">
<StackLayout row="0" visibility="{{ displayPreviewVisibility }}">
<Image src="{{ displayPreview }}" stretch="fill" height="{{ displayPreviewHeight }}" />
<Image src="{{ displayPreview }}" stretch="fill" height="{{ displayPreviewHeight }}"
tap="{{ onMirrorTap }}" doubleTap="{{ onMirrorDoubleTap }}" longPress="{{ onMirrorLongPress }}" swipe="{{ onMirrorSwipe }}" />
</StackLayout>
<GridLayout
row="0"
@@ -36,14 +37,7 @@
</StackLayout>
</StackLayout>
<StackLayout row="1">
<GridLayout rows="auto,auto,auto" columns="*,*,*" class="m-t-8">
<Button row="0" col="1" text="Swipe up" tap="{{ onSyntheticUpTap }}" />
<Button row="1" col="0" text="Double-click" tap="{{ onSyntheticLeftTap }}" />
<Button row="1" col="1" text="Long press" tap="{{ onSyntheticLongPressTap }}" />
<Button row="1" col="2" text="Click" tap="{{ onSyntheticRightTap }}" />
<Button row="0" col="2" text="Microphone" tap="{{ onSyntheticMicTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<rc:remote-controls />
<Button text="Stop recording" tap="{{ onStopRecordingTap }}" visibility="{{ stopRecordingButtonVisibility }}" />
</StackLayout>
<Label row="2" text="{{ status }}" class="m-t-16" textWrap="true" />
@@ -65,13 +59,14 @@
<Button text="Allow unrestricted background usage" tap="{{ onAllowBackgroundUsageTap }}" class="-primary" />
</StackLayout>
</StackLayout>
<GridLayout row="1" rows="auto,auto,*" columns="*,260">
<GridLayout row="1" rows="auto,auto,*" columns="*,360">
<StackLayout row="0" col="0" visibility="{{ displayPreviewVisibility }}" horizontalAlignment="left">
<Image
src="{{ displayPreview }}"
stretch="fill"
width="{{ landscapeDisplayPreviewWidth }}"
height="{{ landscapeDisplayPreviewHeight }}" />
height="{{ landscapeDisplayPreviewHeight }}"
tap="{{ onMirrorTap }}" doubleTap="{{ onMirrorDoubleTap }}" longPress="{{ onMirrorLongPress }}" swipe="{{ onMirrorSwipe }}" />
</StackLayout>
<GridLayout
row="0"
@@ -84,14 +79,7 @@
<Label text="{{ displayPreviewMessage }}" textWrap="true" horizontalAlignment="center" verticalAlignment="center" />
</GridLayout>
<StackLayout row="0" col="1" class="m-l-16">
<GridLayout rows="auto,auto,auto" columns="*,*,*">
<Button row="0" col="1" text="Swipe up" tap="{{ onSyntheticUpTap }}" />
<Button row="1" col="0" text="Double-click" tap="{{ onSyntheticLeftTap }}" />
<Button row="1" col="1" text="Long press" tap="{{ onSyntheticLongPressTap }}" />
<Button row="1" col="2" text="Click" tap="{{ onSyntheticRightTap }}" />
<Button row="0" col="2" text="Microphone" tap="{{ onSyntheticMicTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<rc:remote-controls />
<Button text="Stop recording" tap="{{ onStopRecordingTap }}" visibility="{{ stopRecordingButtonVisibility }}" />
</StackLayout>
<Label row="1" col="0" colSpan="2" text="{{ status }}" class="m-t-16" textWrap="true" />
+258 -2
View File
@@ -1,5 +1,28 @@
import { Application, Frame, ImageSource, Observable, Screen } from "@nativescript/core";
import { dashboardController } from "../g2/dashboard-controller";
import {
Application,
Dialogs,
Frame,
ImageSource,
Observable,
Screen,
SwipeDirection,
type GestureEventData,
type SwipeGestureEventData,
type TouchGestureEventData,
type View,
} from "@nativescript/core";
import { dashboardController, type MirrorTouchKind } from "../g2/dashboard-controller";
import { shell } from "../ui/shell/shell";
import {
brightnessSetting,
DISPLAY_MODE_VALUES,
displayModeLabel,
displayModeSetting,
onAnySettingChanged,
type BrightnessSetting,
type DisplayModeSetting,
} from "../ui/dashboard-settings";
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
import { isValidMacAddress, loadDeviceAddresses } from "../g2/device-addresses";
import { isAutoReconnectSuppressed } from "../g2/reconnect-policy";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image";
@@ -55,7 +78,10 @@ export class MainViewModel extends Observable {
this.firmwareWarningVisible = snapshot.firmwareWarningVisible;
this.screenRecordingActive = snapshot.screenRecordingActive;
this.batteryOptimizationWarningVisible = snapshot.batteryOptimizationWarningVisible;
this.refreshPadFocusLine();
});
// Brightness / display mode can change from the glasses' Settings app too.
onAnySettingChanged(() => this.refreshDisplayControls());
}
get status(): string {
@@ -601,6 +627,220 @@ export class MainViewModel extends Observable {
await dashboardController.injectSyntheticRingInput("wakeword");
}
// ---- the phone's own controller: touchpad, d-pad, mirror touch ----
//
// Everything here is the watch scheme (origin "watch"): spatial swipes,
// tap = select, double-tap / two fingers = back, hold = menu. The ring row
// above stays on the ring's own scheme.
private padPointers = 0;
private padTwoFingerDown = false;
/** What the next gesture lands on, as the watch pad shows it. */
get padFocusLine(): string {
if (this.phase !== "connected" && this.phase !== "charging") return "Glasses disconnected";
if (!shell.isScreenOn()) return "Display off";
const foreground = shell.getForegroundApp();
return foreground ? foreground.title : "Launcher";
}
private refreshPadFocusLine(): void {
this.notifyPropertyChange("padFocusLine", this.padFocusLine);
}
async onPadTap(): Promise<void> {
if (this.padTwoFingerDown) return;
await dashboardController.injectSyntheticRingInput("click", "watch");
this.refreshPadFocusLine();
}
async onPadDoubleTap(): Promise<void> {
await dashboardController.injectSyntheticRingInput("double-click", "watch");
this.refreshPadFocusLine();
}
async onPadLongPress(): Promise<void> {
await dashboardController.injectSyntheticRingInput("long-press", "watch");
this.refreshPadFocusLine();
}
async onPadSwipe(args: SwipeGestureEventData): Promise<void> {
await dashboardController.injectSyntheticRingInput(swipeKind(args.direction), "watch");
this.refreshPadFocusLine();
}
/** Two fingers down and up without moving: back (the watch's two-finger tap). */
async onPadTouch(args: TouchGestureEventData): Promise<void> {
const count = args.getPointerCount();
if (args.action === "down" || args.action === "move") {
this.padPointers = Math.max(this.padPointers, count);
if (count >= 2) this.padTwoFingerDown = true;
return;
}
if (args.action === "up" || args.action === "cancel") {
const twoFinger = this.padTwoFingerDown;
this.padPointers = 0;
if (twoFinger) {
// Let the single-tap recognizer's delayed tap see the flag first.
setTimeout(() => {
this.padTwoFingerDown = false;
}, 400);
if (args.action === "up") {
await dashboardController.injectSyntheticRingInput("double-click", "watch");
this.refreshPadFocusLine();
}
}
}
}
async onDpadUp(): Promise<void> {
await dashboardController.injectSyntheticRingInput("swipe-up", "watch");
this.refreshPadFocusLine();
}
async onDpadDown(): Promise<void> {
await dashboardController.injectSyntheticRingInput("swipe-down", "watch");
this.refreshPadFocusLine();
}
async onDpadLeft(): Promise<void> {
await dashboardController.injectSyntheticRingInput("swipe-left", "watch");
this.refreshPadFocusLine();
}
async onDpadRight(): Promise<void> {
await dashboardController.injectSyntheticRingInput("swipe-right", "watch");
this.refreshPadFocusLine();
}
async onDpadSelect(): Promise<void> {
await dashboardController.injectSyntheticRingInput("click", "watch");
this.refreshPadFocusLine();
}
async onBackTap(): Promise<void> {
await dashboardController.injectSyntheticRingInput("double-click", "watch");
this.refreshPadFocusLine();
}
async onMenuTap(): Promise<void> {
await dashboardController.injectSyntheticRingInput("long-press", "watch");
this.refreshPadFocusLine();
}
// ---- touching the mirror itself ----
get mirrorTouchEnabled(): boolean {
return getBooleanSetting(MIRROR_TOUCH_KEY, true);
}
onMirrorTouchChange(args: { value?: boolean; object?: { checked?: boolean } }): void {
const on = typeof args.value === "boolean" ? args.value : Boolean(args.object?.checked);
setBooleanSetting(MIRROR_TOUCH_KEY, on);
this.notifyPropertyChange("mirrorTouchEnabled", on);
}
private mirrorFraction(args: GestureEventData & { getX?: () => number; getY?: () => number }): { nx: number; ny: number } | null {
const view = args.object as View | undefined;
const size = view?.getActualSize?.();
if (!view || !size || !size.width || !size.height || !args.getX || !args.getY) return null;
// On Android the gesture's getX/getY are screen-absolute (in DIPs), not
// view-local; the view's own screen position makes them local.
const origin = view.getLocationOnScreen?.() ?? { x: 0, y: 0 };
const localX = args.getX() - origin.x;
const localY = args.getY() - origin.y;
return { nx: localX / size.width, ny: localY / size.height };
}
private async mirrorGesture(kind: MirrorTouchKind, args: GestureEventData): Promise<void> {
if (!this.mirrorTouchEnabled) return;
const at = this.mirrorFraction(args) ?? { nx: 0.5, ny: 0.5 };
await dashboardController.handleMirrorTouch(kind, at.nx, at.ny);
this.refreshPadFocusLine();
}
onMirrorTap(args: GestureEventData): Promise<void> {
return this.mirrorGesture("tap", args);
}
onMirrorDoubleTap(args: GestureEventData): Promise<void> {
return this.mirrorGesture("double-tap", args);
}
onMirrorLongPress(args: GestureEventData): Promise<void> {
return this.mirrorGesture("long-press", args);
}
onMirrorSwipe(args: SwipeGestureEventData): Promise<void> {
return this.mirrorGesture(swipeKind(args.direction), args);
}
// ---- display mode and brightness, beside the mirror ----
get displayModeLabel(): string {
return displayModeLabel(displayModeSetting.get()) + " ▾";
}
async onDisplayModeTap(): Promise<void> {
const current = displayModeSetting.get();
const options = DISPLAY_MODE_VALUES.map((value) => displayModeLabel(value) + (value === current ? " ✓" : ""));
const picked = await Dialogs.action({ title: "Display mode", cancelButtonText: "Cancel", actions: options });
const index = options.indexOf(picked);
if (index < 0) return;
const value = DISPLAY_MODE_VALUES[index] as DisplayModeSetting;
if (value !== current) displayModeSetting.set(value);
this.notifyPropertyChange("displayModeLabel", this.displayModeLabel);
}
get brightnessAuto(): boolean {
return brightnessSetting.get() === "auto";
}
get brightnessSliderEnabled(): boolean {
return !this.brightnessAuto;
}
/** The slider's position; while Auto, the last manual level (or 50). */
get brightnessPercent(): number {
const value = brightnessSetting.get();
if (value === "auto") return this.lastManualBrightness;
const numeric = parseInt(value, 10);
return Number.isFinite(numeric) ? numeric : 50;
}
private lastManualBrightness = 50;
onBrightnessChange(args: { value?: number; object?: { value?: number } }): void {
if (this.brightnessAuto) return;
const raw = typeof args.value === "number" ? args.value : Number(args.object?.value ?? NaN);
if (!Number.isFinite(raw)) return;
// The setting only has every tenth level; snap to the nearest.
const level = Math.min(100, Math.max(0, Math.round(raw / 10) * 10));
this.lastManualBrightness = level;
const value = String(level) as BrightnessSetting;
if (brightnessSetting.get() !== value) brightnessSetting.set(value);
}
onBrightnessAutoChange(args: { value?: boolean; object?: { checked?: boolean } }): void {
const on = typeof args.value === "boolean" ? args.value : Boolean(args.object?.checked);
if (on) {
if (brightnessSetting.get() !== "auto") {
this.lastManualBrightness = this.brightnessPercent;
brightnessSetting.set("auto");
}
} else if (brightnessSetting.get() === "auto") {
brightnessSetting.set(String(this.lastManualBrightness) as BrightnessSetting);
}
this.refreshDisplayControls();
}
private refreshDisplayControls(): void {
this.notifyPropertyChange("brightnessAuto", this.brightnessAuto);
this.notifyPropertyChange("brightnessSliderEnabled", this.brightnessSliderEnabled);
this.notifyPropertyChange("brightnessPercent", this.brightnessPercent);
this.notifyPropertyChange("displayModeLabel", this.displayModeLabel);
}
private appendLog(line: string): void {
const stamp = new Date().toISOString().slice(11, 19);
this.log = this.log ? `${this.log}\n[${stamp}] ${line}` : `[${stamp}] ${line}`;
@@ -621,3 +861,19 @@ export class MainViewModel extends Observable {
return `${sanitized.slice(0, 237)}...`;
}
}
const MIRROR_TOUCH_KEY = "phone.mirrorTouch";
/** NativeScript swipe direction -> the watch-scheme directional gesture. */
function swipeKind(direction: SwipeDirection): "swipe-up" | "swipe-down" | "swipe-left" | "swipe-right" {
switch (direction) {
case SwipeDirection.up:
return "swipe-up";
case SwipeDirection.down:
return "swipe-down";
case SwipeDirection.left:
return "swipe-left";
default:
return "swipe-right";
}
}
+47
View File
@@ -0,0 +1,47 @@
<!--
The phone's controller for the glasses: the same scheme as the watch pad
(tap = select, double-tap or two fingers = back, hold = menu, swipes
navigate), a d-pad for the same four directions, the display-mode picker
and brightness beside the mirror, and a compact ring simulator for the R1's
own scheme. Used by main-page.xml in both orientations; binds to the page's
MainViewModel.
-->
<StackLayout xmlns="http://schemas.nativescript.org/tns.xsd" class="remote-controls">
<GridLayout columns="auto,*,auto" class="display-controls">
<Button col="0" text="{{ displayModeLabel }}" tap="{{ onDisplayModeTap }}" class="mode-button" />
<GridLayout col="1" columns="auto,*" verticalAlignment="center" class="brightness">
<Label col="0" text="☀" class="brightness-icon" verticalAlignment="center" />
<Slider col="1" width="100%" minValue="0" maxValue="100" value="{{ brightnessPercent }}" valueChange="{{ onBrightnessChange }}" isEnabled="{{ brightnessSliderEnabled }}" />
</GridLayout>
<StackLayout col="2" orientation="horizontal" verticalAlignment="center" class="auto-toggle">
<Label text="Auto" verticalAlignment="center" />
<Switch checked="{{ brightnessAuto }}" checkedChange="{{ onBrightnessAutoChange }}" />
</StackLayout>
</GridLayout>
<GridLayout class="touchpad" tap="{{ onPadTap }}" doubleTap="{{ onPadDoubleTap }}" longPress="{{ onPadLongPress }}" swipe="{{ onPadSwipe }}" touch="{{ onPadTouch }}">
<Label text="{{ padFocusLine }}" class="pad-focus" horizontalAlignment="center" verticalAlignment="center" />
<Label text="● select ●● back ‒ menu" class="pad-legend" horizontalAlignment="center" verticalAlignment="bottom" />
<Button text="▲" tap="{{ onDpadUp }}" class="pad-dpad-button" horizontalAlignment="center" verticalAlignment="top" />
<Button text="▼" tap="{{ onDpadDown }}" class="pad-dpad-button pad-dpad-down" horizontalAlignment="center" verticalAlignment="bottom" />
<Button text="◀" tap="{{ onDpadLeft }}" class="pad-dpad-button" horizontalAlignment="left" verticalAlignment="center" />
<Button text="▶" tap="{{ onDpadRight }}" class="pad-dpad-button" horizontalAlignment="right" verticalAlignment="center" />
</GridLayout>
<GridLayout columns="*,*,*" class="action-row">
<Button col="0" text="Back" tap="{{ onBackTap }}" class="action-button" />
<Button col="1" text="Menu" tap="{{ onMenuTap }}" class="action-button" />
<Button col="2" text="Hey Even" tap="{{ onSyntheticMicTap }}" class="action-button" />
</GridLayout>
<GridLayout columns="auto,*,*,*,*,*,auto,auto" class="ring-row">
<Label col="0" text="Ring" class="ring-label" verticalAlignment="center" />
<Button col="1" text="▲" tap="{{ onSyntheticUpTap }}" class="ring-button" />
<Button col="2" text="▼" tap="{{ onSyntheticDownTap }}" class="ring-button" />
<Button col="3" text="●" tap="{{ onSyntheticRightTap }}" class="ring-button" />
<Button col="4" text="●●" tap="{{ onSyntheticLeftTap }}" class="ring-button" />
<Button col="5" text="‒" tap="{{ onSyntheticLongPressTap }}" class="ring-button" />
<Label col="6" text="Touch mirror" class="ring-label" verticalAlignment="center" />
<Switch col="7" checked="{{ mirrorTouchEnabled }}" checkedChange="{{ onMirrorTouchChange }}" />
</GridLayout>
</StackLayout>
+67
View File
@@ -223,6 +223,36 @@ export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
description: "Whether the top-bar clock shows 24-hour or 12-hour time.",
});
/**
* How much of the 640x480 panel the UI uses. "576x288" is the stock band
* (sidebar + a 288px-tall window at the vertical position); "576x480" keeps
* the sidebar and gives every window the full height; "640x480" is the whole
* panel, with the sidebar an overlay that shows only while it has focus.
*/
export const DISPLAY_MODE_VALUES = ["576x288", "576x480", "640x480"] as const;
export type DisplayModeSetting = (typeof DISPLAY_MODE_VALUES)[number];
const DISPLAY_MODE_LABELS: Record<DisplayModeSetting, string> = {
"576x288": "Band · 576×288",
"576x480": "Tall · 576×480",
"640x480": "Full panel · 640×480",
};
export function displayModeLabel(value: DisplayModeSetting): string {
return DISPLAY_MODE_LABELS[value] ?? value;
}
export const displayModeSetting = new ConfigSettingEnum<DisplayModeSetting>({
id: "display-mode",
label: "Display mode",
storageKey: "display.mode",
defaultValue: "576x288",
values: DISPLAY_MODE_VALUES,
formatValue: displayModeLabel,
description:
"Band: the stock 576×288 window beside the sidebar. Tall: the sidebar plus full-height windows. Full panel: the whole 640×480 display; the sidebar overlays the app only while you are in it. Open apps reopen in the new size.",
});
export const brightnessSetting = new ConfigSettingEnum<BrightnessSetting>({
id: "brightness",
label: "Brightness",
@@ -252,6 +282,43 @@ export const lockScreenEnabledSetting = new ConfigSettingBoolean({
"Lock the glasses after they are taken off while the phone is locked. Unlocking the phone unlocks the glasses.",
});
// Wear OS watch remote (app/g2/wear-remote.ts, wear/). All three are read on
// every watch message, so a change applies immediately.
export const watchRemoteEnabledSetting = new ConfigSettingBoolean({
id: "watch-remote-enabled",
label: "Watch remote control",
storageKey: "watch.remoteEnabled",
defaultValue: true,
description:
"Accept input from the Faceclaw Wear OS app: spatial swipes, taps, holds, crown, voice queries and app commands. The ring's scheme is unaffected. Turn off to ignore the watch.",
});
export const watchCanUnlockSetting = new ConfigSettingBoolean({
id: "watch-can-unlock",
label: "Watch can unlock glasses",
storageKey: "watch.canUnlock",
defaultValue: true,
description:
"Let the watch unlock the glasses' lock screen (which otherwise waits for the phone to be unlocked). Your watch is on your wrist; turn this off if you would rather it stay a phone-only unlock.",
});
export const watchWakesDisplaySetting = new ConfigSettingBoolean({
id: "watch-wakes-display",
label: "Watch input wakes display",
storageKey: "watch.wakesDisplay",
defaultValue: true,
description:
"Any gesture from the watch turns the glasses display on (the ring only wakes it with a double-tap). Off: watch input is ignored while the display is off, as ring input is.",
});
export const watchMirrorAssistantSetting = new ConfigSettingBoolean({
id: "watch-mirror-assistant",
label: "Mirror assistant to watch",
storageKey: "watch.mirrorAssistant",
defaultValue: true,
description: "Stream assistant replies and on-glasses alerts to the watch so they can be read from the wrist.",
});
export type VerticalPosition = "top" | "upper" | "middle" | "lower" | "bottom";
const VERTICAL_POSITION_LABELS: Record<VerticalPosition, string> = {
+39
View File
@@ -27,6 +27,7 @@ import {
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
brightnessSetting,
displayModeSetting,
elevenLabsApiKeySetting,
mapboxApiKeySetting,
openAiApiKeySetting,
@@ -48,7 +49,12 @@ import {
voiceProviderSetting,
screenTimeoutSetting,
wakeWordActionSetting,
watchCanUnlockSetting,
watchMirrorAssistantSetting,
watchRemoteEnabledSetting,
watchWakesDisplaySetting,
} from "../dashboard-settings";
import { wearBridge } from "../../native/wear-bridge";
import { SettingsPanelLayer, type SettingsSection } from "./settings-panel";
import { terminalFontPickerMenuItem, uiFontPickerMenuItem } from "../font-picker";
@@ -74,6 +80,8 @@ function settingsSections(): SettingsSection[] {
// Where min-height windows (and the sidebar) sit vertically on the
// screen; the dashboard controller repositions surfaces on change.
enumSettingMenuItem(verticalPositionSetting),
// Band / tall / full-panel; the dashboard controller reflows windows.
enumSettingMenuItem(displayModeSetting),
// Controls the top-bar battery indicators (icon vs percentage).
enumSettingMenuItem(batteryDisplayModeSetting),
// Controls the top-bar clock (24-hour vs 12-hour).
@@ -133,6 +141,18 @@ function settingsSections(): SettingsSection[] {
textSettingMenuItem(roamApiTokenSetting),
],
},
{
label: "Watch",
// Wear OS remote (wear/); the status line above the items says whether
// a watch running the companion app is currently reachable.
items: [
toggleSettingMenuItem(watchRemoteEnabledSetting),
toggleSettingMenuItem(watchWakesDisplaySetting),
toggleSettingMenuItem(watchCanUnlockSetting),
toggleSettingMenuItem(watchMirrorAssistantSetting),
],
renderDetail: renderWatchStatus,
},
{
label: "Developer",
items: [
@@ -260,6 +280,25 @@ function readBundledDoc(fileName: string): string {
}
}
function renderWatchStatus(args: { image: GrayImage; x: number; y: number; width: number }): number {
const { image, x, y, width } = args;
const font = getDefaultSmallFont();
let status: string;
if (!wearBridge.isAvailable()) {
status = "Google Play services is unavailable on this phone, so no watch can connect.";
} else {
const connection = wearBridge.getWatchConnection();
status = connection.reachable
? `Connected to ${connection.watchName || "a watch"}.`
: "No watch connected. Install the Faceclaw watch app (wear/ in the source tree) on a Wear OS watch paired with this phone.";
}
const lines = wrapText(font, status, width);
for (let i = 0; i < lines.length; i++) {
image.drawText(font, x, y + i * font.lineHeight, lines[i]!, 170);
}
return lines.length * font.lineHeight + 10;
}
function renderAbout(args: { image: GrayImage; x: number; y: number; width: number }): number {
const { image, x, y, width } = args;
const font = getDefaultSmallFont();
+13
View File
@@ -41,6 +41,9 @@ const NO_MENU = new MenuLayer(null, []);
* column, out to the sidebar). Third-level menus open as centered modals.
*/
export class SettingsPanelLayer implements Layer {
// Watch swipes map onto the two columns: right goes into a section's
// items, left comes back out (and out to the sidebar from the left column).
readonly acceptsDirectional = true;
private leftIndex = 0;
private rightIndex = 0;
private focus: "left" | "right" = "left";
@@ -185,14 +188,17 @@ export class SettingsPanelLayer implements Layer {
if (this.focus === "left") {
switch (event.type) {
case "scroll-up":
case "swipe-up":
this.leftIndex = (this.leftIndex + this.sections.length - 1) % this.sections.length;
this.resetRight();
return;
case "scroll-down":
case "swipe-down":
this.leftIndex = (this.leftIndex + 1) % this.sections.length;
this.resetRight();
return;
case "click":
case "swipe-right":
// Only enter sections that have something interactive on the right.
if (this.section().items.length) {
this.focus = "right";
@@ -200,6 +206,7 @@ export class SettingsPanelLayer implements Layer {
}
return;
case "double-click":
case "swipe-left":
shell.yieldFocusToSidebar();
return;
default:
@@ -210,18 +217,22 @@ export class SettingsPanelLayer implements Layer {
const items = this.section().items;
switch (event.type) {
case "scroll-up":
case "swipe-up":
if (items.length) this.rightIndex = (this.rightIndex + items.length - 1) % items.length;
return;
case "scroll-down":
case "swipe-down":
if (items.length) this.rightIndex = (this.rightIndex + 1) % items.length;
return;
case "click":
case "swipe-right":
if (items.length) {
const item = items[clamp(this.rightIndex, 0, items.length - 1)]!;
if (!isMenuItemDisabled(item)) await item.onSelect(ctx, NO_MENU);
}
return;
case "double-click":
case "swipe-left":
this.focus = "left";
return;
default:
@@ -242,6 +253,8 @@ export class SettingsPanelLayer implements Layer {
* layers (modal menus, the text editor) stack above it as before.
*/
export class SettingsDescriptionOverlayLayer implements Layer {
readonly acceptsDirectional = true;
constructor(private readonly panel: SettingsPanelLayer) {}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
+88 -7
View File
@@ -3,13 +3,35 @@ import { type Plane } from "../graphics/plane";
import { spanCurrent } from "../native/frame-timings";
import { type ConfigSettingString } from "./dashboard-settings";
/**
* Where an input came from. The ring and the glasses' arms are the stock
* sources; "watch" is the Wear OS remote, which components may treat with a
* richer scheme (see isWatchInput) — the ring's semantics are never changed
* by it.
*/
export type InputSource = "ring" | "left-arm" | "right-arm" | "watch";
export type DashboardInputEvent =
| { type: "click"; source: "ring" | "left-arm" | "right-arm" }
| { type: "double-click"; source: "ring" | "left-arm" | "right-arm" }
| { type: "scroll-up" }
| { type: "scroll-down" }
| { type: "long-press"; source: "ring" | "left-arm" | "right-arm" }
| { type: "long-press-release"; source: "ring" | "left-arm" | "right-arm" }
| { type: "click"; source: InputSource }
| { type: "double-click"; source: InputSource }
/** Ring scroll (or a watch crown turn, then tagged source "watch"). */
| { type: "scroll-up"; source?: InputSource }
| { type: "scroll-down"; source?: InputSource }
| { type: "long-press"; source: InputSource }
| { type: "long-press-release"; source: InputSource }
/**
* Spatial (four-way) input, which only a watch can produce: the ring's
* scroll is a one-dimensional cursor, these are directions. Components with
* a spatial meaning for them opt in with Layer.acceptsDirectional /
* ShellWindow.acceptsDirectional and must then handle all four; for the
* rest, directionalFallback turns up/down into scroll, right into "select"
* (click) and left into "back" (double-click), so a swipe always does
* something sensible.
*/
| { type: "swipe-up"; source: "watch" }
| { type: "swipe-down"; source: "watch" }
| { type: "swipe-left"; source: "watch" }
| { type: "swipe-right"; source: "watch" }
/**
* The stock display lifecycle woke while no EvenHub page was running. This
* is wake-only, unlike a normal double-click (which turns an on screen off).
@@ -24,6 +46,44 @@ export type DashboardInputEvent =
| { type: "wakeword" }
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
/** True for input from the Wear OS remote (any type that carries a source). */
export function isWatchInput(event: DashboardInputEvent): boolean {
return "source" in event && event.source === "watch";
}
export type DirectionalInputEvent = Extract<
DashboardInputEvent,
{ type: "swipe-up" | "swipe-down" | "swipe-left" | "swipe-right" }
>;
export function isDirectionalInput(event: DashboardInputEvent): event is DirectionalInputEvent {
return (
event.type === "swipe-up" ||
event.type === "swipe-down" ||
event.type === "swipe-left" ||
event.type === "swipe-right"
);
}
/**
* The ring-vocabulary equivalent of a directional swipe: up/down scroll,
* right selects, left backs out.
*/
export function directionalFallback(event: DashboardInputEvent): DashboardInputEvent {
switch (event.type) {
case "swipe-up":
return { type: "scroll-up" };
case "swipe-down":
return { type: "scroll-down" };
case "swipe-right":
return { type: "click", source: "ring" };
case "swipe-left":
return { type: "double-click", source: "ring" };
default:
return event;
}
}
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
@@ -96,6 +156,17 @@ export interface Layer {
readonly paintOverBase?: boolean;
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage;
handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> | void;
/**
* True when handleInput gives swipe-left / swipe-right a meaning of its own.
* Otherwise the stack hands the layer directionalFallback(event) instead.
*/
readonly acceptsDirectional?: boolean;
/**
* A touch at (x, y) in this layer's own canvas coordinates, from the phone's
* mirror view. Return true if the layer acted on it (selected / opened what
* is there); false sends a plain select (click) instead.
*/
hitTest?(x: number, y: number, ctx: LayerContext): Promise<boolean> | boolean;
/** Called when the layer leaves the stack by any path (pop or clearToBase). */
onRemoved?(): void;
/**
@@ -195,7 +266,17 @@ export class LayerStack {
}
async handleInput(event: DashboardInputEvent): Promise<void> {
await this.layers[this.layers.length - 1]!.handleInput(event, this.ctx);
const top = this.layers[this.layers.length - 1]!;
const delivered = isDirectionalInput(event) && !top.acceptsDirectional ? directionalFallback(event) : event;
await top.handleInput(delivered, this.ctx);
}
/** Pass a mirror touch to the base layer when nothing is stacked on it. */
async hitTest(x: number, y: number): Promise<boolean> {
if (this.layers.length !== 1) return false;
const base = this.layers[0]!;
if (!base.hitTest) return false;
return await base.hitTest(x, y, this.ctx);
}
/** Hand text to the top layer; false if it doesn't take text input. */
+36 -4
View File
@@ -13,6 +13,7 @@ import {
minWindowTop,
SHELL_OPAQUE_BLACK,
SIDEBAR_WIDTH,
sidebarWidth,
TOP_BAR_HEIGHT,
windowTop,
type WindowHeightMode,
@@ -166,7 +167,11 @@ export class ShellChromeLayer implements Layer {
paint(): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const state = this.getState();
this.drawSidebar(image, state);
// Full-panel mode: the strip is an overlay, present only while the user
// is in it (the window underneath keeps its full width the rest of the time).
if (sidebarWidth() > 0 || state.focus === "sidebar") {
this.drawSidebar(image, state);
}
this.drawTopBar(image, state);
return image;
}
@@ -176,6 +181,30 @@ export class ShellChromeLayer implements Layer {
// layer stack; the chrome itself never consumes events.
}
/**
* Which window's sidebar icon is under (x, y) on screen, if any — the same
* slot geometry drawSidebar uses, so a touch on the phone's mirror lands on
* the icon the mirror showed.
*/
windowIndexAt(x: number, y: number, windowCount: number): number | null {
if (x < 0 || x >= SIDEBAR_WIDTH || windowCount === 0) return null;
const variant = sidebarVariant(windowCount);
const rows = rowsPerColumn(variant);
const listTop = minWindowTop() + TOP_BAR_HEIGHT + LIST_MARGIN;
const itemStride = variant.iconSize + ICON_SPACING;
const lastVisible = Math.min(windowCount, this.scrollRow + rows * variant.columns);
for (let index = this.scrollRow; index < lastVisible; index++) {
const position = index - this.scrollRow;
const column = variant.columns - 1 - ((position / rows) | 0);
const slotY = listTop + (position % rows) * itemStride;
const left = columnLeft(variant, column);
if (x >= left && x < left + variant.columnWidth && y >= slotY - 2 && y < slotY + variant.iconSize + 2) {
return index;
}
}
return null;
}
private drawSidebar(image: GrayImage, state: ShellChromeState): void {
// The sidebar always occupies the min-height window band at the preferred
// vertical position: it never moves when the foreground window's height
@@ -266,13 +295,16 @@ export class ShellChromeLayer implements Layer {
// its height mode puts that (screen top for max height). It moves when
// the foreground switches to a window of a different height.
const barTop = windowTop(state.foregroundHeightMode);
image.fillRect(SIDEBAR_WIDTH, barTop, G2_LENS_WIDTH - SIDEBAR_WIDTH, TOP_BAR_HEIGHT, SHELL_OPAQUE_BLACK);
image.drawLine(SIDEBAR_WIDTH, barTop + TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, barTop + TOP_BAR_HEIGHT - 1, BORDER_VALUE);
// The bar spans the app viewport; with the sidebar overlaid (full-panel
// mode, sidebar focused) it still starts past the strip.
const barLeft = sidebarWidth() > 0 || state.focus === "sidebar" ? SIDEBAR_WIDTH : 0;
image.fillRect(barLeft, barTop, G2_LENS_WIDTH - barLeft, TOP_BAR_HEIGHT, SHELL_OPAQUE_BLACK);
image.drawLine(barLeft, barTop + TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, barTop + TOP_BAR_HEIGHT - 1, BORDER_VALUE);
const now = new Date();
const clock = `${WEEKDAYS[now.getDay()]} ${now.getDate()} ${MONTHS[now.getMonth()]} ` +
formatClockTime(now);
const clockX = SIDEBAR_WIDTH + 10;
const clockX = barLeft + 10;
const textY = barTop + Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
image.drawText(font, clockX, textY, clock, 210);
+32 -8
View File
@@ -1,15 +1,39 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../../graphics/image";
import { verticalPositionSetting } from "../dashboard-settings";
import { displayModeSetting, type DisplayModeSetting, verticalPositionSetting } from "../dashboard-settings";
/** Top bar: 24px notification icons plus a little padding. */
export const TOP_BAR_HEIGHT = 28;
/**
* Sidebar width. Leaves the app viewport exactly 576px wide — the surface
* width EvenHub apps expect. Icon column layout within the strip (one wide
* column vs. two narrow ones) is the chrome layer's business.
* Width of the sidebar strip as painted. Leaves the app viewport exactly
* 576px wide — the surface width EvenHub apps expect — except in the
* full-panel display mode, where the strip overlays the app (see
* sidebarWidth). Icon column layout within the strip (one wide column vs.
* two narrow ones) is the chrome layer's business.
*/
export const SIDEBAR_WIDTH = 64;
/** The Display > Display mode setting (see dashboard-settings.ts). */
export function displayMode(): DisplayModeSetting {
return displayModeSetting.get();
}
/**
* How much of the screen's width the sidebar takes away from app windows:
* the strip, or nothing in the full-panel mode (where the chrome paints the
* strip over the window while the sidebar has focus).
*/
export function sidebarWidth(): number {
return displayMode() === "640x480" ? 0 : SIDEBAR_WIDTH;
}
/**
* The height an app's requested mode actually gets: the band mode honours
* it, the taller modes give every window the whole screen.
*/
export function effectiveHeightMode(mode: WindowHeightMode): WindowHeightMode {
return displayMode() === "576x288" ? mode : "max";
}
/**
* X of the display's true horizontal centre, in app-viewport coordinates. The
* sidebar pushes the viewport to the right, so UI the wearer physically aims
@@ -17,7 +41,7 @@ export const SIDEBAR_WIDTH = 64;
* than use the viewport's own centre.
*/
export function screenCenterInViewportX(): number {
return Math.round(G2_LENS_WIDTH / 2) - SIDEBAR_WIDTH;
return Math.round(G2_LENS_WIDTH / 2) - sidebarWidth();
}
/**
@@ -43,7 +67,7 @@ export const MIN_WINDOW_MAX_TOP = G2_LENS_HEIGHT - MIN_WINDOW_HEIGHT;
/** Total height (top bar + content) of a window's band in the given mode. */
export function windowBandHeight(mode: WindowHeightMode): number {
switch (mode) {
switch (effectiveHeightMode(mode)) {
case "max":
return G2_LENS_HEIGHT;
case "medium":
@@ -89,7 +113,7 @@ export function windowTop(mode: WindowHeightMode): number {
/** App-content viewport size for a height mode (independent of vertical position). */
export function appViewportSize(mode: WindowHeightMode): { width: number; height: number } {
return {
width: G2_LENS_WIDTH - SIDEBAR_WIDTH,
width: G2_LENS_WIDTH - sidebarWidth(),
height: windowBandHeight(mode) - TOP_BAR_HEIGHT,
};
}
@@ -102,7 +126,7 @@ export function appViewportRect(mode: WindowHeightMode): {
height: number;
} {
return {
x: SIDEBAR_WIDTH,
x: sidebarWidth(),
y: windowTop(mode) + TOP_BAR_HEIGHT,
...appViewportSize(mode),
};
+22
View File
@@ -168,6 +168,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
title: options.title,
surfaceId: `window:${options.windowId}`,
closeable: options.closeable,
// The window's own LayerStack decides per layer whether a swipe is
// directional or falls back to click / double-click.
acceptsDirectional: true,
heightMode,
close: () => {
closed = true;
@@ -195,6 +198,16 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
await render(frameId);
},
requestRender,
relayout: () => {
stack.setBaseSize(appViewportSize(heightMode));
options.reconfigureSurface?.(heightMode);
requestRender();
},
hitTest: async (x, y) => {
const handled = await stack.hitTest(x, y);
if (handled) requestRender();
return handled;
},
receiveTextInput: options.receiveTextInput,
setForeground: (foreground) => {
options.setSurfaceVisible(foreground);
@@ -223,6 +236,15 @@ export class YieldAtRootLayer implements Layer {
return this.inner.paintOverBase;
}
/** Directional swipes reach the inner layer only if it understands them. */
get acceptsDirectional(): boolean | undefined {
return this.inner.acceptsDirectional;
}
hitTest(x: number, y: number, ctx: LayerContext): Promise<boolean> | boolean {
return this.inner.hitTest ? this.inner.hitTest(x, y, ctx) : false;
}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
return this.inner.paint(ctx, paintBelow);
}
+160 -17
View File
@@ -1,9 +1,20 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { singlePlane, type Plane } from "../../graphics/plane";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import { EvenAIStatus, EventSourceType, OsEventTypeList, WatchGestureType } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import {
DashboardInputEvent,
directionalFallback,
type InputSource,
isDirectionalInput,
isWatchInput,
Layer,
LayerActions,
LayerContext,
LayerStack,
noopLayerActions,
} from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { voiceActivity } from "./voice-activity";
@@ -62,6 +73,11 @@ export type ShellWindow = {
surfaceId: string;
/** Whether close menu entries (app menu default, shell escape menu) apply (the launcher is pinned). */
closeable: boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
*/
acceptsDirectional?: boolean;
/**
* Window height: the standard 288px band ("min") or the full screen
* ("max", terminal views). Decides the surface rect and where the shell
@@ -79,6 +95,17 @@ export type ShellWindow = {
handleInput: (event: DashboardInputEvent, frameId: number) => Promise<void> | void;
/** Repaint and resubmit this window's surface. */
requestRender: () => void;
/**
* Re-measure against the current display mode (viewport size / band) and
* repaint. Windows without it (workers, whose canvas is fixed at open) are
* closed and relaunched by the controller instead.
*/
relayout?: () => void;
/**
* A touch from the phone's mirror at (x, y) in app-viewport coordinates.
* True if the window acted on it; otherwise the controller sends a select.
*/
hitTest?: (x: number, y: number) => Promise<boolean> | boolean;
/**
* Deliver a text string to the window (e.g. finalized voice input). Optional:
* only windows that consume typed text (the terminal) implement it.
@@ -104,7 +131,18 @@ export type ShellConfig = {
/** Which surfaces need re-rendering after an input event. */
export type ShellInputOutcome = { shell: boolean; window: boolean };
type FocusKind = "sidebar" | "window";
/**
* Assistant overlay activity, for mirrors outside the glasses (the watch).
* "streaming" carries the reply so far (replace semantics); "done" the final
* reply; "error" the message; "closed" fires when the overlay leaves the
* stack by any path.
*/
export type AssistantActivityEvent = {
phase: "thinking" | "streaming" | "done" | "error" | "closed";
text: string;
};
export type FocusKind = "sidebar" | "window";
const noopActions: LayerActions = noopLayerActions;
@@ -209,6 +247,8 @@ class Shell {
private activeVoiceLayer: VoiceInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>();
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
@@ -217,10 +257,8 @@ class Shell {
requestShellRender: () => {},
onScreenStateChanged: () => {},
};
private readonly stack = new LayerStack(
new ShellChromeLayer(() => this.chromeState()),
this.actions,
);
private readonly chrome = new ShellChromeLayer(() => this.chromeState());
private readonly stack = new LayerStack(this.chrome, this.actions);
// Top-bar settings we mirror into the chrome; a change to either repaints
// the shell surface so the top bar reflects it immediately.
@@ -379,6 +417,22 @@ class Shell {
this.lastInputAtMs = nowMs;
}
/** Where input currently goes: the sidebar strip or the foreground window. */
getFocus(): FocusKind {
return this.focus;
}
/** True while a shell overlay (menu, dialog, voice input) is above the chrome. */
hasOverlay(): boolean {
return !this.stack.isAtBase();
}
/** The window whose sidebar icon is at screen (x, y), for mirror touches. */
windowAtSidebarPoint(x: number, y: number): ShellWindow | null {
const index = this.chrome.windowIndexAt(x, y, this.windows.length);
return index === null ? null : this.windows[index] ?? null;
}
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
wake(focus: FocusKind, nowMs = Date.now()): boolean {
this.lastInputAtMs = nowMs;
@@ -572,7 +626,8 @@ class Shell {
const window = this.foregroundWindow();
if (window) {
// The window owns frameId from here (render or explicit finish).
await window.handleInput(event, frameId);
const delivered = isDirectionalInput(event) && !window.acceptsDirectional ? directionalFallback(event) : event;
await window.handleInput(delivered, frameId);
return { shell: false, window: true };
}
return { shell: false, window: false };
@@ -630,15 +685,25 @@ class Shell {
private handleSidebarInput(event: DashboardInputEvent): ShellInputOutcome {
switch (event.type) {
case "double-click":
// The ring's double-tap at the root turns the display off. For the
// watch, "back" at the root is simply the end of the road: it wakes
// the display on any gesture, so sleeping here would be a trap; the
// watch's Status screen turns the display off on purpose.
if (isWatchInput(event)) return { shell: false, window: false };
this.sleep();
return { shell: true, window: false };
case "scroll-up":
case "swipe-up":
this.moveSelection(-1);
return { shell: true, window: false };
case "scroll-down":
case "swipe-down":
this.moveSelection(1);
return { shell: true, window: false };
case "click":
case "swipe-right":
// Right: into the selected window (spatially, the window is to the
// sidebar's right). Left has nowhere further to go and is ignored.
if (this.windows.length) {
this.focus = "window";
// Repaint the window now so its selection highlight reflects focus
@@ -763,10 +828,41 @@ class Shell {
this.config.requestShellRender();
}
private isAssistantAvailable(): boolean {
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
}
/** Observe assistant turns (see AssistantActivityEvent). */
onAssistantActivity(listener: (event: AssistantActivityEvent) => void): () => void {
this.assistantActivityListeners.add(listener);
return () => {
this.assistantActivityListeners.delete(listener);
};
}
/** Observe showAlert popups (mirrored to the watch). */
onAlertShown(listener: (text: string) => void): () => void {
this.alertListeners.add(listener);
return () => {
this.alertListeners.delete(listener);
};
}
private emitAssistantActivity(event: AssistantActivityEvent): void {
for (const listener of Array.from(this.assistantActivityListeners)) {
try {
listener(event);
} catch (error) {
console.warn("assistant activity listener failed", error);
}
}
}
/** Dismiss the assistant overlay (cancelling any in-flight turn), as Done would. */
closeAssistant(): void {
this.closeAssistantLayer();
}
private ensureAssistantSession(): AssistantSession | null {
const config = this.resolveAssistantConfiguration();
if (!config) return null;
@@ -834,6 +930,7 @@ class Shell {
// stop the turn and drop the reference so a later query starts clean.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" });
},
});
layer = created;
@@ -846,11 +943,23 @@ class Shell {
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar),
onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar;
layer.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
},
onToolActivity: (label) => layer.onToolActivity(label),
onTurnDone: () => layer.onTurnDone(),
onError: (message) => layer.onError(message),
onTurnDone: () => {
layer.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar });
},
onError: (message) => {
layer.onError(message);
this.emitAssistantActivity({ phase: "error", text: message });
},
});
}
@@ -901,6 +1010,13 @@ class Shell {
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
showAlert(text: string): void {
if (!this.screenOn) this.wake("sidebar");
for (const listener of Array.from(this.alertListeners)) {
try {
listener(text);
} catch (error) {
console.warn("alert listener failed", error);
}
}
const layer = new ShellAlertLayer(text, () => {
this.stack.popIfTop((top) => top === layer);
this.config.requestShellRender();
@@ -1024,9 +1140,9 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
return scrollEvent("scroll-down", event.eventSource);
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
return scrollEvent("scroll-up", event.eventSource);
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) {
// CFW-forwarded long-press (replaces the firmware's force-quit dialog).
// The CFW gates this to the ring, so eventSource may be 0 (unknown);
@@ -1035,6 +1151,18 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
}
} else if (event.kind === "watch-gesture") {
// Synthetic, from the Wear OS remote (app/g2/wear-remote.ts); the glasses
// firmware never produces these.
if (event.eventType === WatchGestureType.SWIPE_LEFT) {
return { type: "swipe-left", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_RIGHT) {
return { type: "swipe-right", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_UP) {
return { type: "swipe-up", source: "watch" };
} else if (event.eventType === WatchGestureType.SWIPE_DOWN) {
return { type: "swipe-down", source: "watch" };
}
} else if (event.kind === "even-ai") {
// sid 0x07 EvenAIDataPackage; eventType carries eEvenAIStatus. Only the
// wakeword interests us -- ENTER means the user manually opened the stock
@@ -1048,9 +1176,9 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
return { type: "display-wake" };
} else if (event.kind === "text-click") {
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
return scrollEvent("scroll-down", event.eventSource);
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
return scrollEvent("scroll-up", event.eventSource);
}
}
return {
@@ -1061,13 +1189,20 @@ export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputE
};
}
function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" {
/** Scroll events only carry a source when it is the watch (the stock ones never needed one). */
function scrollEvent(type: "scroll-up" | "scroll-down", eventSource: number): DashboardInputEvent {
return eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH ? { type, source: "watch" } : { type };
}
function eventSourceToString(eventSource: number): InputSource {
if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
return "ring";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) {
return "left-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) {
return "right-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH) {
return "watch";
}
return "ring";
}
@@ -1086,6 +1221,14 @@ export function inputEventToString(event: DashboardInputEvent): string {
return `Long press from ${event.source}`;
case "long-press-release":
return `Long press release from ${event.source}`;
case "swipe-left":
return `Swipe left from ${event.source}`;
case "swipe-right":
return `Swipe right from ${event.source}`;
case "swipe-up":
return `Swipe up from ${event.source}`;
case "swipe-down":
return `Swipe down from ${event.source}`;
case "display-wake":
return `Display wake`;
case "wakeword":