App-switchign UI: Sidebar, top bar, launcher

This commit is contained in:
James Babcock committed 2026-07-06 20:06:29 -07:00
1 parent c83448873e
commit 1133f952e9
22 files changed
+2031 -337

No files matched your search

-72
View File
@@ -1,72 +0,0 @@
import { getDefaultSmallFont, getFont } from "../../graphics/bdffont";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { Layer, type DashboardInputEvent, type LayerContext } from "../layers";
const stopwatchFont = getFont("terminus32");
export class StopwatchLayer implements Layer {
private readonly startedAtMs = Date.now();
private pausedAtMs: number | null = null;
private pausedDurationMs = 0;
paint(ctx: LayerContext): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const elapsedMs = this.elapsedMs();
const timeLabel = formatStopwatchElapsed(elapsedMs);
const stateLabel = this.pausedAtMs === null ? "Running" : "Paused";
const smallFont = getDefaultSmallFont();
image.drawText(smallFont, 24, 24, "Stopwatch", 180);
const quadrantWidth = G2_LENS_WIDTH / 2;
const quadrantHeight = G2_LENS_HEIGHT / 2;
const timeX = Math.max(0, Math.round((quadrantWidth - stopwatchFont.measureText(timeLabel)) / 2));
const timeY = Math.max(0, Math.round((quadrantHeight - stopwatchFont.lineHeight) / 2));
image.drawText(stopwatchFont, timeX, timeY, timeLabel, 245);
image.drawText(smallFont, 90, 142, stateLabel, 150);
image.drawText(smallFont, 90, 160, "Click: pause / resume", 150);
image.drawText(smallFont, 90, 178, "Double-click: back", 150);
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "double-click") {
void ctx.actions.setStopwatchRenderActive(false);
ctx.stack.pop();
return;
}
if (event.type === "click") {
if (this.pausedAtMs === null) {
this.pausedAtMs = Date.now();
void ctx.actions.setStopwatchRenderActive(false);
} else {
this.pausedDurationMs += Date.now() - this.pausedAtMs;
this.pausedAtMs = null;
void ctx.actions.setStopwatchRenderActive(true);
}
}
}
private elapsedMs(): number {
const now = this.pausedAtMs ?? Date.now();
return Math.max(0, now - this.startedAtMs - this.pausedDurationMs);
}
}
function pad2(value: number): string {
return String(value).padStart(2, "0");
}
function formatStopwatchElapsed(elapsedMs: number): string {
const totalTenths = Math.floor(elapsedMs / 100);
const tenths = totalTenths % 10;
const totalSeconds = Math.floor(totalTenths / 10);
const seconds = totalSeconds % 60;
const totalMinutes = Math.floor(totalSeconds / 60);
const minutes = totalMinutes % 60;
const hours = Math.floor(totalMinutes / 60);
if (hours > 0) {
return `${hours}:${pad2(minutes)}:${pad2(seconds)}.${tenths}`;
}
return `${pad2(minutes)}:${pad2(seconds)}.${tenths}`;
}
+3 -2
View File
@@ -54,8 +54,9 @@ export class VoiceInputLayer implements Layer {
const font = getDefaultSmallFont();
const image = paintBelow();
// Solid dialog box over the underlying UI.
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 0, 10);
// Solid dialog box over the underlying UI. Fill 1, not 0: identical after
// 4bpp quantization, but 0 is transparent on the color-key shell surface.
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
+20 -15
View File
@@ -1,4 +1,10 @@
import { ApplicationSettings } from "@nativescript/core";
import {
getBooleanSetting,
getStringSetting,
onSettingsStoreChanged,
setBooleanSetting,
setStringSetting,
} from "~/native/settings-store";
import { getDefaultSmallFont } from "~/graphics/bdffont";
import { wrapText } from "~/graphics/textwrap";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from "./menu";
@@ -34,9 +40,11 @@ type ConfigSettingOptions<TValue, TId extends string> = {
formatValue?: (value: TValue) => string;
};
// Fired after any setting's set(); lets phone-side UI that depends on
// settings toggled from the glasses (e.g. the raw-screenshot button) update
// without waiting for an unrelated snapshot emit.
// Fired after any setting changes, in any isolate (storage lives in the Java
// FaceclawSettings store and broadcasts to every isolate). Lets phone-side UI
// that depends on settings toggled from the glasses (e.g. the raw-screenshot
// button) update without waiting for an unrelated snapshot emit. Delivery is
// asynchronous: one message-loop tick after the set().
const settingChangeListeners = new Set<() => void>();
export function onAnySettingChanged(listener: () => void): () => void {
@@ -46,11 +54,11 @@ export function onAnySettingChanged(listener: () => void): () => void {
};
}
function notifySettingChanged(): void {
onSettingsStoreChanged(() => {
for (const listener of Array.from(settingChangeListeners)) {
listener();
}
}
});
export abstract class ConfigSetting<TValue, TId extends string = string> {
readonly id: TId;
@@ -82,12 +90,11 @@ export class ConfigSettingBoolean<TId extends string = string> extends ConfigSet
}
get(): boolean {
return ApplicationSettings.getBoolean(this.storageKey, this.defaultValue);
return getBooleanSetting(this.storageKey, this.defaultValue);
}
set(value: boolean): boolean {
ApplicationSettings.setBoolean(this.storageKey, value);
notifySettingChanged();
setBooleanSetting(this.storageKey, value);
return value;
}
@@ -120,13 +127,12 @@ export class ConfigSettingEnum<TValue extends string, TId extends string = strin
}
get(): TValue {
return this.normalizer(ApplicationSettings.getString(this.storageKey, this.defaultValue));
return this.normalizer(getStringSetting(this.storageKey, this.defaultValue));
}
set(value: TValue): TValue {
const normalized = this.normalizer(value);
ApplicationSettings.setString(this.storageKey, normalized);
notifySettingChanged();
setStringSetting(this.storageKey, normalized);
return normalized;
}
@@ -155,13 +161,12 @@ export class ConfigSettingString<TId extends string = string> extends ConfigSett
}
get(): string {
return this.normalizer(ApplicationSettings.getString(this.storageKey, this.defaultValue));
return this.normalizer(getStringSetting(this.storageKey, this.defaultValue));
}
set(value: string): string {
const normalized = this.normalizer(value);
ApplicationSettings.setString(this.storageKey, normalized);
notifySettingChanged();
setStringSetting(this.storageKey, normalized);
return normalized;
}
}
+16 -144
View File
@@ -1,16 +1,14 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { type RawInputEvent } from "../native/faceclaw-communicator";
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout-bridge";
import { EventSourceType, OsEventTypeList } from "../g2/events";
import { getDashboardPlugin, type DashboardPluginCardBounds, type DashboardPluginState } from "./dashboard-plugins";
import { EditTextSettingLayer, isNightscoutSettingsConfigured, screenTimeoutSettingToMs, screenTimeoutSetting, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings";
import { EditTextSettingLayer, isNightscoutSettingsConfigured, type DashboardSlotId, DashboardPluginId, bottomRightSlotSetting, bottomLeftSlotSetting, dashboardSlotIds } from "./dashboard-settings";
import { Layer, LayerActions, LayerStack, type DashboardInputEvent, type LayerContext } from "./layers";
import { SingleNotificationLayer } from "./notifications";
import { TelepromptLayer } from "./apps/teleprompt";
import { VoiceInputLayer } from "./apps/voice-input";
import { drawSystemCard } from "./dashboard/system-card";
import { createRootMenuLayer } from "./dashboard/root-menu";
import { inputEventToString, shell } from "./shell/shell";
type DashboardCardId = "system" | DashboardSlotId;
export type DashboardBatteryLevels = {
@@ -21,16 +19,12 @@ export type DashboardBatteryLevels = {
type DashboardState = {
logLines: string[];
screenOn: boolean;
lastInputAtMs: number;
telepromptDocumentText: string | null;
battery: DashboardBatteryLevels;
};
export let dashboardState: DashboardState = {
logLines: [] as string[],
screenOn: true,
lastInputAtMs: Date.now(),
telepromptDocumentText: null as string | null,
battery: {
headset: null,
@@ -44,7 +38,6 @@ const dashboardActions: LayerActions = {
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
setStopwatchRenderActive: () => {},
setTranscribeRenderActive: () => {},
startVoiceCapture: () => {},
stopVoiceCapture: () => {},
@@ -52,114 +45,15 @@ const dashboardActions: LayerActions = {
};
export const TOP_LEFT_MENU_LAYOUT = { x: 8, y: 8, width: 272 };
// The push-to-talk dialog is a singleton on the layer stack; tracked here so
// long-press-release (a global event) can reach it and so a second long-press
// doesn't stack duplicates.
let activeVoiceInputLayer: VoiceInputLayer | null = null;
function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
if (event.kind === "sys-event") {
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
return {
type: "click",
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) {
return {
type: "double-click",
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) {
// CFW-forwarded long-press (replaces the firmware's force-quit dialog).
// The CFW gates this to the ring, so eventSource may be 0 (unknown);
// eventSourceToString falls back to "ring".
return { type: "long-press", source: eventSourceToString(event.eventSource) };
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
}
} else if (event.kind === "text-click") {
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
}
}
return {
type: "unknown",
kind: event.kind,
eventSource: event.eventSource,
eventType: event.eventType,
};
}
function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" {
if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
return "ring";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) {
return "left-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) {
return "right-arm";
}
return "ring";
}
export async function receiveInput(event: RawInputEvent): Promise<void> {
const inputEvent = rawInputEventToInputEvent(event);
dashboardState.lastInputAtMs = Date.now();
dashboardState.logLines.push(eventToString(inputEvent));
if (!dashboardState.screenOn) {
if (inputEvent.type === "double-click") {
dashboardState.screenOn = true;
if (dashboardLayers.isAtBase()) {
dashboardLayers.push(createRootMenuLayer());
}
}
return;
}
// Push-to-talk is intercepted globally so it works over any UI: long-press
// opens the voice dialog and starts the mic; releasing it stops the mic.
if (inputEvent.type === "long-press") {
if (!activeVoiceInputLayer) {
const layer = new VoiceInputLayer(dashboardActions, () => {
if (activeVoiceInputLayer === layer) activeVoiceInputLayer = null;
});
activeVoiceInputLayer = layer;
dashboardLayers.push(layer);
layer.startCapture();
}
return;
}
if (inputEvent.type === "long-press-release") {
activeVoiceInputLayer?.endCapture();
return;
}
/**
* Input handler for the dashboard window. Global gestures (wake, sleep,
* long-press/push-to-talk) are consumed by the shell before this is called.
*/
export async function receiveDashboardWindowInput(inputEvent: DashboardInputEvent): Promise<void> {
dashboardState.logLines.push(inputEventToString(inputEvent));
await dashboardLayers.handleInput(inputEvent);
}
function eventToString(event: DashboardInputEvent): string {
switch (event.type) {
case "click":
return `Click from ${event.source}`;
case "double-click":
return `Double click from ${event.source}`;
case "scroll-up":
return `Scroll up`;
case "scroll-down":
return `Scroll down`;
case "long-press":
return `Long press from ${event.source}`;
case "long-press-release":
return `Long press release from ${event.source}`;
default:
case "unknown":
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
}
}
export function logToDashboard(message: string): void {
dashboardState.logLines.push(message);
}
@@ -177,40 +71,26 @@ export function closeDashboardTextSettingEditor(): boolean {
}
export function drawDashboard(): GrayImage {
if (!dashboardState.screenOn) {
if (!shell.isScreenOn()) {
return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
}
return dashboardLayers.paint();
}
export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean {
const timeoutMs = screenTimeoutSettingToMs(screenTimeoutSetting.get());
if (timeoutMs === null || !dashboardState.screenOn) return false;
if (nowMs - dashboardState.lastInputAtMs < timeoutMs) return false;
dashboardState.screenOn = false;
return true;
}
export function noteDashboardPhoneTextInput(nowMs = Date.now()): void {
dashboardState.lastInputAtMs = nowMs;
}
export function openTelepromptDocument(text?: string): void {
if (text !== undefined) {
dashboardState.telepromptDocumentText = text;
}
dashboardState.lastInputAtMs = Date.now();
dashboardState.screenOn = true;
shell.wake("window");
dashboardLayers.clearToBase();
dashboardLayers.push(createRootMenuLayer());
dashboardLayers.push(new TelepromptLayer(dashboardState.telepromptDocumentText));
}
export function openAndroidNotificationFromSleep(notificationKey: string, nowMs = Date.now()): boolean {
if (!notificationKey || dashboardState.screenOn) return false;
if (!notificationKey || shell.isScreenOn()) return false;
dashboardState.lastInputAtMs = nowMs;
dashboardState.screenOn = true;
shell.wake("window", nowMs);
dashboardLayers.clearToBase();
dashboardLayers.push(createRootMenuLayer());
dashboardLayers.push(new SingleNotificationLayer(notificationKey, {
@@ -221,15 +101,12 @@ export function openAndroidNotificationFromSleep(notificationKey: string, nowMs
}
function closeNewNotificationTrigger(ctx: LayerContext): void {
dashboardState.screenOn = false;
shell.sleep();
ctx.stack.clearToBase();
}
export function resetDashboardSleepTimerAndWake(nowMs = Date.now()): boolean {
dashboardState.lastInputAtMs = nowMs;
if (dashboardState.screenOn) return false;
dashboardState.screenOn = true;
if (!shell.wake("sidebar", nowMs)) return false;
dashboardLayers.clearToBase();
dashboardLayers.push(createRootMenuLayer());
return true;
@@ -254,10 +131,6 @@ export function getPluginIdForSlot(slot: DashboardSlotId): DashboardPluginId {
class DashboardLayer implements Layer {
paint(): GrayImage {
if (!dashboardState.screenOn) {
return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
}
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const pluginState = getPluginState();
drawSystemCard(image, getCardBounds("system"));
@@ -283,9 +156,8 @@ class DashboardLayer implements Layer {
}
if (event.type === "double-click") {
dashboardState.screenOn = false;
ctx.stack.clearToBase();
void dashboardActions.endTextSettingEdit();
// Backing out of the window's root returns focus to the shell sidebar.
shell.yieldFocusToSidebar();
}
}
}
+3 -11
View File
@@ -8,21 +8,20 @@ import { NotificationsListLayer } from "../notifications";
import { createNightscoutSettingsMenuLayer, createSettingsMenuLayer } from "./settings-menus";
import { TelepromptLayer } from "../apps/teleprompt";
import { TerminalAppLayer } from "../apps/terminal";
import { StopwatchLayer } from "../apps/stopwatch";
import { TranscribeLayer } from "../apps/transcribe";
import { ScreenTestLayer } from "../apps/screen-test";
import { BuzzerDemoLayer } from "../apps/buzzer-demo";
import { AboutPage } from "./about-page";
import { GrayImage } from "~/graphics/image";
import { shell } from "../shell/shell";
const rootMenuFont = getDefaultSmallFont();
class RootMenuLayer extends MenuLayer {
async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
dashboardState.screenOn = false;
ctx.stack.clearToBase();
void ctx.actions.endTextSettingEdit();
// Backing out of the window's root returns focus to the shell sidebar.
shell.yieldFocusToSidebar();
return;
}
await super.handleInput(event, ctx);
@@ -86,13 +85,6 @@ function createAppsMenuLayer(): MenuLayer {
ctx.stack.push(new TelepromptLayer(dashboardState.telepromptDocumentText));
},
},
{
label: "Stopwatch",
onSelect: (ctx) => {
ctx.stack.push(new StopwatchLayer());
void ctx.actions.setStopwatchRenderActive(true);
},
},
{
label: "Transcribe",
onSelect: (ctx) => {
+3 -2
View File
@@ -1,4 +1,5 @@
import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
import { TOP_LEFT_MENU_LAYOUT } from "../dashboard";
import { shell } from "../shell/shell";
import { batteryDisplayModeSetting, dashboardSlotIds, elevenLabsApiKeySetting, enumSettingMenuItem, firmwareDebugFlagsSetting, rawScreenshotsEnabledSetting, saveVoiceRecordingsSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceProviderSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { MenuLayer } from "../menu";
@@ -59,7 +60,7 @@ function createDisplaySettingsMenuLayer(): MenuLayer {
[
enumSettingMenuItem(screenTimeoutSetting, {
onChange: (ctx, newValue, oldValue) => {
dashboardState.lastInputAtMs = Date.now();
shell.noteUserActivity();
},
}),
],
+6 -3
View File
@@ -18,7 +18,6 @@ export type LayerActions = {
disconnect: () => Promise<void> | void;
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void;
setStopwatchRenderActive: (active: boolean) => Promise<void> | void;
setTranscribeRenderActive: (active: boolean) => Promise<void> | void;
/** Start voice capture with the provider chosen in settings. */
startVoiceCapture: () => Promise<void> | void;
@@ -53,13 +52,17 @@ export interface Layer {
export class LayerStack {
private readonly layers: Layer[];
private readonly ctx: LayerContext;
private readonly baseWidth: number;
private readonly baseHeight: number;
constructor(baseLayer: Layer, actions: LayerActions) {
constructor(baseLayer: Layer, actions: LayerActions, baseSize?: { width: number; height: number }) {
this.layers = [baseLayer];
this.ctx = {
stack: this,
actions,
};
this.baseWidth = baseSize?.width ?? G2_LENS_WIDTH;
this.baseHeight = baseSize?.height ?? G2_LENS_HEIGHT;
}
push(layer: Layer): void {
@@ -112,7 +115,7 @@ export class LayerStack {
return cachedBelow;
}
if (index <= 0) {
cachedBelow = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
cachedBelow = new GrayImage(this.baseWidth, this.baseHeight, 0);
} else if (layer.paintOverBase) {
cachedBelow = this.paintLayer(0);
} else {
+4 -2
View File
@@ -61,7 +61,7 @@ export function drawToggleMenuItem(
const switchX = x + width - switchWidth - 2;
const switchY = y + MENU_TOGGLE_SWITCH_Y_OFFSET;
image.drawText(font, x, y + 3, label, 200);
const offFill = selected ? 0 : 18;
const offFill = selected ? 1 : 18;
image.fillRoundedRect(switchX, switchY, switchWidth, switchHeight, enabled ? 70 : offFill, 8);
image.drawRoundedRect(switchX, switchY, switchWidth, switchHeight, enabled ? 130 : 55, 8);
const knobSize = 12;
@@ -113,7 +113,9 @@ export class MenuLayer implements Layer {
this.clampScrollToSelection(visibleRowCount);
const image = paintBelow();
image.fillRoundedRect(x, y, width, height, 0);
// Fill 1, not 0: identical after 4bpp quantization, but 0 is the
// transparent color key when a menu paints on the shell surface.
image.fillRoundedRect(x, y, width, height, 1);
image.drawRoundedRect(x, y, width, height, 72);
if (this.title) {
image.drawText(font, x + 12, y + 8, this.title, 220);
+114
View File
@@ -0,0 +1,114 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultMediumFont } from "../../graphics/bdffont";
import { drawBattery } from "../../graphics/battery";
import { readActiveNotificationIcons } from "../../native/notification-icons";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
import { Layer } from "../layers";
import { SHELL_OPAQUE_BLACK, SIDEBAR_WIDTH, TOP_BAR_HEIGHT } from "./geometry";
const ICON_SIZE = 32;
const ICON_MARGIN_X = ((SIDEBAR_WIDTH - ICON_SIZE) / 2) | 0;
const ICON_SPACING = 8;
const NOTIFICATION_ICON_SIZE = 24;
const BORDER_VALUE = 40;
export type ShellChromeWindow = {
windowId: string;
title: string;
attention: boolean;
drawIcon: (image: GrayImage, x: number, y: number, size: number) => void;
};
export type ShellChromeState = {
windows: ShellChromeWindow[];
selectedIndex: number;
focus: "sidebar" | "window";
battery: { headset: number | null; headsetCharging: boolean | null };
};
/** Reusable placeholder window icon: rounded outline with a single letter. */
export function makeLetterWindowIcon(letter: string): ShellChromeWindow["drawIcon"] {
return (image, x, y, size) => {
const font = getDefaultMediumFont();
image.drawRoundedRect(x, y, size, size, 120, 6);
const textX = x + Math.max(0, ((size - font.measureText(letter)) / 2) | 0);
const textY = y + Math.max(0, ((size - font.lineHeight) / 2) | 0);
image.drawText(font, textX, textY, letter, 210);
};
}
/**
* Base layer of the shell surface: the window sidebar on the left and the
* status top bar. Everything not explicitly painted stays 0 (transparent on
* the color-key shell surface), so the app viewport shows through.
*/
export class ShellChromeLayer implements Layer {
constructor(private readonly getState: () => ShellChromeState) {}
paint(): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
const state = this.getState();
this.drawSidebar(image, state);
this.drawTopBar(image, state);
return image;
}
handleInput(): void {
// Shell input is handled by the shell state machine before it reaches the
// layer stack; the chrome itself never consumes events.
}
private drawSidebar(image: GrayImage, state: ShellChromeState): void {
image.fillRect(0, 0, SIDEBAR_WIDTH, G2_LENS_HEIGHT, SHELL_OPAQUE_BLACK);
image.drawLine(SIDEBAR_WIDTH - 1, 0, SIDEBAR_WIDTH - 1, G2_LENS_HEIGHT - 1, BORDER_VALUE);
for (let index = 0; index < state.windows.length; index++) {
const window = state.windows[index]!;
const y = TOP_BAR_HEIGHT + 6 + index * (ICON_SIZE + ICON_SPACING);
if (y + ICON_SIZE > G2_LENS_HEIGHT) break;
if (index === state.selectedIndex) {
const highlight = state.focus === "sidebar" ? 200 : 70;
image.drawRoundedRect(0, y - 2, SIDEBAR_WIDTH - 3, ICON_SIZE + 4, highlight, 6);
}
window.drawIcon(image, ICON_MARGIN_X, y, ICON_SIZE);
if (window.attention) {
image.fillRoundedRect(ICON_MARGIN_X + ICON_SIZE - 7, y - 1, 8, 8, 255, 4);
}
}
}
private drawTopBar(image: GrayImage, state: ShellChromeState): void {
const font = getDefaultMediumFont();
image.fillRect(SIDEBAR_WIDTH, 0, G2_LENS_WIDTH - SIDEBAR_WIDTH, TOP_BAR_HEIGHT, SHELL_OPAQUE_BLACK);
image.drawLine(SIDEBAR_WIDTH, TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, TOP_BAR_HEIGHT - 1, BORDER_VALUE);
const now = new Date();
const clock = `${now.getHours()}:${String(now.getMinutes()).padStart(2, "0")}`;
const clockX = SIDEBAR_WIDTH + 10;
const textY = Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
image.drawText(font, clockX, textY, clock, 210);
let batteryLeft = G2_LENS_WIDTH;
if (state.battery.headset !== null) {
const battery = drawBattery(state.battery.headset, Boolean(state.battery.headsetCharging));
batteryLeft = G2_LENS_WIDTH - battery.width - 8;
image.bitBlt(battery, batteryLeft, Math.max(0, ((TOP_BAR_HEIGHT - battery.height) / 2) | 0), {
transparentZero: true,
});
}
const iconsX = clockX + font.measureText(clock) + 16;
const maxIcons = Math.max(0, ((batteryLeft - 8 - iconsX) / (NOTIFICATION_ICON_SIZE + 4)) | 0);
if (maxIcons > 0) {
const { icons, stale } = readActiveNotificationIcons(maxIcons, renderPassAllowsStaleData());
if (stale) {
noteStaleDataUsed();
}
const iconY = ((TOP_BAR_HEIGHT - NOTIFICATION_ICON_SIZE) / 2) | 0;
for (let index = 0; index < icons.length; index++) {
image.bitBlt(icons[index]!, iconsX + index * (NOTIFICATION_ICON_SIZE + 4), iconY, {
transparentZero: true,
});
}
}
}
}
+21
View File
@@ -0,0 +1,21 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../../graphics/image";
/** Top bar: 24px notification icons plus a little padding. */
export const TOP_BAR_HEIGHT = 28;
/** Left sidebar: 32px window icons plus a little padding. */
export const SIDEBAR_WIDTH = 36;
/**
* On the color-key shell surface, pixel value 0 is transparent; 1 is the
* darkest opaque shade (identical to 0 after 4bpp quantization). Shell
* painting must use this for intentional black.
*/
export const SHELL_OPAQUE_BLACK = 1;
/** Screen region belonging to the foreground app window (compile-time fixed). */
export const APP_VIEWPORT = {
x: SIDEBAR_WIDTH,
y: TOP_BAR_HEIGHT,
width: G2_LENS_WIDTH - SIDEBAR_WIDTH,
height: G2_LENS_HEIGHT - TOP_BAR_HEIGHT,
} as const;
+92
View File
@@ -0,0 +1,92 @@
import { GrayImage } from "../../graphics/image";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack } from "../layers";
import { MenuLayer } from "../menu";
import { makeLetterWindowIcon } from "./chrome-layer";
import { APP_VIEWPORT } from "./geometry";
import { shell, type ShellWindow } from "./shell";
export type LauncherAppEntry = {
appId: string;
label: string;
};
export type LauncherOptions = {
actions: LayerActions;
apps: LauncherAppEntry[];
launchApp: (appId: string) => Promise<void> | void;
/** Submit a painted viewport-sized frame to this window's surface. */
submitFrame: (image: GrayImage, paintMs: number, frameId: number) => Promise<void>;
/** Flip the launcher's compositor surface visibility on foreground changes. */
setSurfaceVisible: (visible: boolean) => void;
};
export const LAUNCHER_WINDOW_ID = "launcher";
export const LAUNCHER_SURFACE_ID = "window:launcher";
/** The launcher's root menu; backing out of it yields to the sidebar. */
class LauncherMenuLayer extends MenuLayer {
async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
shell.yieldFocusToSidebar();
return;
}
await super.handleInput(event, ctx);
}
}
/**
* The launcher: a pinned, uncloseable in-process window listing launchable
* apps. Selecting an app asks the controller to launch it (new window,
* possibly in an existing app worker) and foregrounds the new window.
*/
export function createLauncherWindow(options: LauncherOptions): ShellWindow {
const baseLayer: Layer = new LauncherMenuLayer(
"Apps",
options.apps.map((app) => ({
label: app.label,
onSelect: async () => {
await options.launchApp(app.appId);
},
})),
{
x: 8,
y: 8,
width: 272,
minHeight: 0,
maxHeight: APP_VIEWPORT.height - 16,
},
);
const stack = new LayerStack(baseLayer, options.actions, {
width: APP_VIEWPORT.width,
height: APP_VIEWPORT.height,
});
async function render(frameId: number): Promise<void> {
const paintStartedAtMs = Date.now();
const image = stack.paint();
await options.submitFrame(image, Date.now() - paintStartedAtMs, frameId);
}
return {
appId: "launcher",
windowId: LAUNCHER_WINDOW_ID,
title: "Apps",
surfaceId: LAUNCHER_SURFACE_ID,
closeable: false,
drawIcon: makeLetterWindowIcon("A"),
handleInput: async (event, frameId) => {
await stack.handleInput(event);
// Java-side dedup finishes no-change frames, so an unconditional
// resubmit keeps frame ownership simple.
await render(frameId);
},
requestRender: () => {
void render(0).catch((error) => {
console.error(`launcher render failed: ${error}`);
});
},
setForeground: (foreground) => {
options.setSurfaceVisible(foreground);
},
};
}
+449
View File
@@ -0,0 +1,449 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { EventSourceType, OsEventTypeList } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, LayerActions, LayerStack } from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer } from "../apps/voice-input";
import { ShellChromeLayer, type ShellChromeState, type ShellChromeWindow } from "./chrome-layer";
import { SIDEBAR_WIDTH, TOP_BAR_HEIGHT } from "./geometry";
/**
* The shell: owns the window registry, focus, screen on/off, and the shell
* surface (sidebar + top bar + shell overlays such as the long-press menu and
* the push-to-talk dialog). Runs on the main thread; windows will later live
* in worker threads, today the only window is the in-process dashboard.
*
* Input flow: every event enters via receiveInput. The shell consumes its
* reserved gestures (long-press, and everything while the sidebar or a shell
* overlay has focus) and forwards the rest to the focused window. Windows
* never see events the shell consumed, and the shell keeps working when a
* window's handler hangs (the long-press menu is the escape hatch).
*/
export type ShellWindow = {
appId: string;
windowId: string;
title: string;
/** Compositor surface this window renders to; configured at connect / launch. */
surfaceId: string;
/** Whether the shell's long-press menu offers Close Window (launcher and dashboard are pinned). */
closeable: boolean;
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
close?: () => void;
drawIcon: ShellChromeWindow["drawIcon"];
/**
* Handle an input event the shell forwarded. Ownership of frameId (latency
* tracking) passes to the window: it must eventually reach a frame submit
* or a finishFrame call.
*/
handleInput: (event: DashboardInputEvent, frameId: number) => Promise<void> | void;
/** Repaint and resubmit this window's surface. */
requestRender: () => void;
/** Foreground state changed: this window's surface is (not) the visible one. */
setForeground?: (foreground: boolean) => void;
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
setScreenOn?: (on: boolean) => void;
};
export type ShellConfig = {
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
actions: LayerActions;
getScreenTimeoutMs: () => number | null;
requestShellRender: () => void;
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
onScreenStateChanged: (on: boolean) => void;
};
/** Which surfaces need re-rendering after an input event. */
export type ShellInputOutcome = { shell: boolean; window: boolean };
type FocusKind = "sidebar" | "window";
const noopActions: LayerActions = {
requestRender: () => {},
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
setTranscribeRenderActive: () => {},
startVoiceCapture: () => {},
stopVoiceCapture: () => {},
playBuzzerNote: () => {},
};
/** Long-press overlay menu; closing it returns focus to the sidebar. */
class ShellOverlayMenuLayer extends MenuLayer {
constructor(items: MenuItem[], private readonly onClosed: () => void) {
super(null, items, {
x: SIDEBAR_WIDTH + 8,
y: TOP_BAR_HEIGHT + 8,
width: 272,
minHeight: 0,
});
}
onRemoved(): void {
this.onClosed();
}
}
class Shell {
private windows: ShellWindow[] = [];
private selectedIndex = 0;
private focus: FocusKind = "sidebar";
private screenOn = true;
private lastInputAtMs = Date.now();
private battery: ShellChromeState["battery"] = { headset: null, headsetCharging: null };
private attention = new Map<string, boolean>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
actions: noopActions,
getScreenTimeoutMs: () => null,
requestShellRender: () => {},
onScreenStateChanged: () => {},
};
private readonly stack = new LayerStack(
new ShellChromeLayer(() => this.chromeState()),
this.actions,
);
configure(config: ShellConfig): void {
this.config = config;
this.stack.setActions(config.actions);
}
/** Add a window (or replace one with the same windowId, keeping its slot). */
registerWindow(window: ShellWindow): void {
const existing = this.windows.findIndex((w) => w.windowId === window.windowId);
if (existing >= 0) {
this.windows[existing] = window;
} else {
this.windows.push(window);
}
}
removeWindow(windowId: string): void {
const index = this.windows.findIndex((w) => w.windowId === windowId);
if (index < 0) return;
const wasSelected = index === this.selectedIndex;
this.windows.splice(index, 1);
this.attention.delete(windowId);
if (this.selectedIndex > index) {
this.selectedIndex--;
} else if (this.selectedIndex >= this.windows.length) {
this.selectedIndex = Math.max(0, this.windows.length - 1);
}
if (this.focus === "window" && (wasSelected || !this.windows.length)) {
this.focus = "sidebar";
}
if (wasSelected) {
// Hand the foreground to whatever is now selected.
const next = this.windows[this.selectedIndex];
next?.setForeground?.(true);
next?.requestRender();
}
this.config.requestShellRender();
}
/** Close the foreground window via the shell (long-press menu action). */
closeForegroundWindow(): void {
const window = this.foregroundWindow();
if (!window || !window.closeable) return;
try {
window.close?.();
} catch (error) {
console.warn(`window ${window.windowId} close failed`, error);
}
this.removeWindow(window.windowId);
}
getWindows(): readonly ShellWindow[] {
return this.windows;
}
setWindowAttention(windowId: string, attention: boolean): void {
if (Boolean(this.attention.get(windowId)) === attention) return;
this.attention.set(windowId, attention);
this.config.requestShellRender();
}
setBatteryLevels(levels: Partial<ShellChromeState["battery"]>): void {
this.battery = { ...this.battery, ...levels };
}
isScreenOn(): boolean {
return this.screenOn;
}
noteUserActivity(nowMs = Date.now()): void {
this.lastInputAtMs = nowMs;
}
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
wake(focus: FocusKind, nowMs = Date.now()): boolean {
this.lastInputAtMs = nowMs;
this.focus = focus;
if (this.screenOn) return false;
this.screenOn = true;
this.config.onScreenStateChanged(true);
for (const window of this.windows) {
window.setScreenOn?.(true);
}
// Refresh the foreground window; the compositor restored its retained
// frame, but its content may be stale (e.g. a running stopwatch).
this.foregroundWindow()?.requestRender();
return true;
}
/** Turn the screen off, closing any shell overlays. Sidebar selection is kept. */
sleep(): void {
if (!this.screenOn) return;
this.screenOn = false;
this.stack.clearToBase();
for (const window of this.windows) {
window.setScreenOn?.(false);
}
this.config.onScreenStateChanged(false);
}
/** Foreground and focus a window by id (e.g. a wake path opening content in it). */
focusWindow(windowId: string): void {
const index = this.windows.findIndex((w) => w.windowId === windowId);
if (index < 0) return;
this.setSelectedIndex(index);
this.focus = "window";
}
/** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */
applyScreenTimeout(nowMs = Date.now()): boolean {
const timeoutMs = this.config.getScreenTimeoutMs();
if (timeoutMs === null || !this.screenOn) return false;
if (nowMs - this.lastInputAtMs < timeoutMs) return false;
this.sleep();
return true;
}
/** Called by a window when the user backs out of its root (double-tap). */
yieldFocusToSidebar(): void {
if (this.focus === "sidebar") return;
this.focus = "sidebar";
this.config.requestShellRender();
}
/** Paint the shell surface: transparent chrome, or all-transparent when asleep. */
paintSurface(): GrayImage {
if (!this.screenOn) {
return new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
}
return this.stack.paint();
}
async receiveInput(event: DashboardInputEvent, frameId = 0): Promise<ShellInputOutcome> {
this.lastInputAtMs = Date.now();
if (!this.screenOn) {
if (event.type === "double-click") {
this.wake("sidebar");
return { shell: true, window: false };
}
return { shell: false, window: false };
}
// Long-press is reserved by the shell: PTT from the sidebar, the overlay
// menu from inside a window. Windows never see it.
if (event.type === "long-press") {
if (!this.activeVoiceLayer && this.stack.isAtBase()) {
if (this.focus === "sidebar") {
this.openVoiceDialog();
} else {
this.openOverlayMenu();
}
}
return { shell: true, window: false };
}
if (event.type === "long-press-release") {
this.activeVoiceLayer?.endCapture();
return { shell: true, window: false };
}
if (!this.stack.isAtBase()) {
await this.stack.handleInput(event);
return { shell: true, window: false };
}
if (this.focus === "sidebar") {
return this.handleSidebarInput(event);
}
const window = this.foregroundWindow();
if (window) {
// The window owns frameId from here (render or explicit finish).
await window.handleInput(event, frameId);
return { shell: false, window: true };
}
return { shell: false, window: false };
}
foregroundWindow(): ShellWindow | undefined {
return this.windows[this.selectedIndex] ?? this.windows[0];
}
private handleSidebarInput(event: DashboardInputEvent): ShellInputOutcome {
switch (event.type) {
case "double-click":
this.sleep();
return { shell: true, window: false };
case "scroll-up":
this.moveSelection(-1);
return { shell: true, window: false };
case "scroll-down":
this.moveSelection(1);
return { shell: true, window: false };
case "click":
if (this.windows.length) {
this.focus = "window";
}
return { shell: true, window: false };
default:
return { shell: false, window: false };
}
}
private moveSelection(delta: number): void {
if (!this.windows.length) return;
const count = this.windows.length;
this.setSelectedIndex((this.selectedIndex + delta + count) % count);
}
/** Change selection; the selected window is the foreground window. */
private setSelectedIndex(index: number): void {
if (index === this.selectedIndex) return;
const previous = this.windows[this.selectedIndex];
this.selectedIndex = index;
const next = this.windows[index];
previous?.setForeground?.(false);
next?.setForeground?.(true);
next?.requestRender();
}
private openVoiceDialog(): void {
const layer = new VoiceInputLayer(this.config.actions, () => {
if (this.activeVoiceLayer === layer) {
this.activeVoiceLayer = null;
}
});
this.activeVoiceLayer = layer;
this.stack.push(layer);
layer.startCapture();
}
private openOverlayMenu(): void {
const foreground = this.foregroundWindow();
const items: MenuItem[] = [
{
label: "Voice input",
onSelect: () => {
this.openVoiceDialog();
},
},
];
if (foreground?.closeable) {
items.push({
label: "Close window",
onSelect: (ctx) => {
// Pop the menu first (its onRemoved returns focus to the sidebar),
// then close the window the menu was opened over.
ctx.stack.pop();
this.closeForegroundWindow();
},
});
}
this.stack.push(new ShellOverlayMenuLayer(items, () => this.yieldFocusToSidebar()));
}
private chromeState(): ShellChromeState {
return {
windows: this.windows.map((window) => ({
windowId: window.windowId,
title: window.title,
attention: Boolean(this.attention.get(window.windowId)),
drawIcon: window.drawIcon,
})),
selectedIndex: this.selectedIndex,
focus: this.focus,
battery: this.battery,
};
}
}
export const shell = new Shell();
export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
if (event.kind === "sys-event") {
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
return {
type: "click",
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.DOUBLE_CLICK_EVENT) {
return {
type: "double-click",
source: eventSourceToString(event.eventSource),
};
} else if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_EVENT) {
// CFW-forwarded long-press (replaces the firmware's force-quit dialog).
// The CFW gates this to the ring, so eventSource may be 0 (unknown);
// eventSourceToString falls back to "ring".
return { type: "long-press", source: eventSourceToString(event.eventSource) };
} else if (event.eventType === OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT) {
return { type: "long-press-release", source: eventSourceToString(event.eventSource) };
}
} else if (event.kind === "text-click") {
if (event.eventType === OsEventTypeList.SCROLL_BOTTOM_EVENT) {
return { type: "scroll-down" };
} else if (event.eventType === OsEventTypeList.SCROLL_TOP_EVENT) {
return { type: "scroll-up" };
}
}
return {
type: "unknown",
kind: event.kind,
eventSource: event.eventSource,
eventType: event.eventType,
};
}
function eventSourceToString(eventSource: number): "ring" | "left-arm" | "right-arm" {
if (eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) {
return "ring";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_L) {
return "left-arm";
} else if (eventSource === EventSourceType.TOUCH_EVENT_FROM_GLASSES_R) {
return "right-arm";
}
return "ring";
}
export function inputEventToString(event: DashboardInputEvent): string {
switch (event.type) {
case "click":
return `Click from ${event.source}`;
case "double-click":
return `Double click from ${event.source}`;
case "scroll-up":
return `Scroll up`;
case "scroll-down":
return `Scroll down`;
case "long-press":
return `Long press from ${event.source}`;
case "long-press-release":
return `Long press release from ${event.source}`;
default:
case "unknown":
return `Unknown event: ${event.kind} ${event.eventSource} ${event.eventType}`;
}
}
+105
View File
@@ -0,0 +1,105 @@
import { makeLetterWindowIcon } from "./chrome-layer";
import { shell, type ShellWindow } from "./shell";
/**
* Messages between the shell (main thread) and an app worker. One worker
* hosts one app, which may have several windows; messages are routed by
* windowId. Everything crossing this boundary is small JSON; pixels go
* worker→Java directly.
*/
export type WorkerAppMessage =
| { type: "open-window"; windowId: string; surfaceId: string; viewport: { width: number; height: number } }
| { type: "close-window"; windowId: string }
| { type: "input"; windowId: string; event: unknown; frameId: number }
| { type: "render"; windowId: string }
| { type: "foreground"; windowId: string; foreground: boolean }
| { type: "screen"; on: boolean };
export type WorkerAppReply = { type: "yield-focus"; windowId: string };
export type WorkerWindowOptions = {
windowId: string;
title: string;
iconLetter: string;
surfaceId: string;
viewport: { width: number; height: number };
/** Flip the window's compositor surface visibility. */
setSurfaceVisible: (visible: boolean) => void;
/** Remove the window's compositor surface (on close). */
removeSurface: () => void;
};
/**
* Owns the Worker for one app and adapts its windows to the shell's window
* interface: forwards input and lifecycle over postMessage, relays
* yield-focus back to the shell, and tells the worker when windows open and
* close. The worker submits frames straight to the Java compositor, so no
* pixels cross this boundary.
*/
export class WorkerAppHost {
private readonly openWindows = new Set<string>();
constructor(
private readonly appId: string,
private readonly worker: Worker,
) {
worker.onmessage = (event: MessageEvent) => {
const message = event.data as WorkerAppReply | undefined;
if (message?.type === "yield-focus") {
// Only the focused window's yield is meaningful.
if (shell.foregroundWindow()?.windowId === message.windowId) {
shell.yieldFocusToSidebar();
}
}
};
worker.onerror = (error) => {
console.error(`worker app ${appId} error: ${JSON.stringify(error)}`);
};
}
windowCount(): number {
return this.openWindows.size;
}
openWindow(options: WorkerWindowOptions): ShellWindow {
this.openWindows.add(options.windowId);
this.post({
type: "open-window",
windowId: options.windowId,
surfaceId: options.surfaceId,
viewport: options.viewport,
});
return {
appId: this.appId,
windowId: options.windowId,
title: options.title,
surfaceId: options.surfaceId,
closeable: true,
close: () => {
this.openWindows.delete(options.windowId);
this.post({ type: "close-window", windowId: options.windowId });
options.removeSurface();
},
drawIcon: makeLetterWindowIcon(options.iconLetter),
handleInput: (event, frameId) => {
this.post({ type: "input", windowId: options.windowId, event, frameId });
},
requestRender: () => {
this.post({ type: "render", windowId: options.windowId });
},
setForeground: (foreground) => {
options.setSurfaceVisible(foreground);
this.post({ type: "foreground", windowId: options.windowId, foreground });
},
setScreenOn: (on) => {
// Screen state is per-app, but sending per-window keeps the protocol
// uniform; the worker treats it globally.
this.post({ type: "screen", on });
},
};
}
private post(message: WorkerAppMessage): void {
this.worker.postMessage(message);
}
}