Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-11-01 11:00:38 -08:00
commit fcea0b369a
218 files changed
+14438 -2259

No files matched your search

+84 -7
View File
@@ -16,8 +16,10 @@ import {
assistantModelProvider,
type AssistantModel,
} from "~/assistant/models";
import { isLocalModelReady } from "../native/llama";
import { isLocalModelReady } from "../native/local-llm";
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
import { ANIMATION_SPEEDS, animationSpeedLabel, scaleAnimationDuration, type AnimationSpeed } from "./animation-speed";
import { MENU_ANIMATION_KEY } from "./menu-animation-pref";
@@ -36,6 +38,7 @@ export type BatteryDisplayMode = "icon" | "percentage" | "stacked" | "stacked-pe
export type BatteryIndicatorVisibility = "always" | "low" | "never";
export type TimeFormat = "24h" | "12h";
export type TemperatureUnit = "fahrenheit" | "celsius";
export type DistanceUnit = "miles" | "kilometers";
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "3m" | "never";
// Auto runs Faceclaw's light curve. Numeric values use the calibrated firmware
// scale, clamped to 2–100; 2 is also the endpoint for screen fades.
@@ -321,14 +324,17 @@ export function batteryIndicatorSettingsKey(): string {
].join("|");
}
/** Exported so the Navigate worker can repaint when it changes. */
export const TIME_FORMAT_SETTING_KEY = "display.timeFormat";
export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
id: "timeFormat",
label: "Time format",
storageKey: "display.timeFormat",
storageKey: TIME_FORMAT_SETTING_KEY,
defaultValue: "24h",
values: ["24h", "12h"],
formatValue: timeFormatLabel,
description: "Whether the top-bar clock shows 24-hour or 12-hour time.",
description: "Whether the top-bar clock and other times of day, such as Navigate's arrival time, show 24-hour or 12-hour time.",
});
export const temperatureUnitSetting = new ConfigSettingEnum<TemperatureUnit>({
@@ -341,6 +347,19 @@ export const temperatureUnitSetting = new ConfigSettingEnum<TemperatureUnit>({
description: "Whether the Weather app, its top-bar indicator, and the Glanceboard's system card show temperatures in Fahrenheit or Celsius.",
});
/** Exported so the Navigate worker can repaint when it changes. */
export const DISTANCE_UNIT_SETTING_KEY = "display.distanceUnit";
export const distanceUnitSetting = new ConfigSettingEnum<DistanceUnit>({
id: "distanceUnit",
label: "Distance unit",
storageKey: DISTANCE_UNIT_SETTING_KEY,
defaultValue: "miles",
values: ["miles", "kilometers"],
formatValue: (value) => value === "kilometers" ? "Kilometers" : "Miles",
description: "Whether the Weather app shows wind speed in mph or km/h, and the Navigate app shows distances in miles and feet or kilometers and meters.",
});
/**
* Where the app switcher (the open-window icons) sits: a strip along the
* left or right edge of the display, a row under the window, or a popup box
@@ -919,7 +938,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
export type AssistantBackendKind = "direct" | "external";
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
direct: global.isIOS ? "Cloud API" : "On-phone",
direct: "On-phone",
external: "My own agent (bridge)",
};
@@ -931,7 +950,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
values: global.isIOS ? ["direct"] : ["direct", "external"],
formatValue: (value) => assistantBackendLabels[value] ?? value,
description:
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys) or your configured OpenAI-compatible API, or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantHermesUrlSetting = new ConfigSettingString({
@@ -997,6 +1016,48 @@ export const assistantAllowProactiveSetting = new ConfigSettingBoolean({
"Let the external agent use glasses tools outside of a conversation, e.g. showing an alert when a long-running job finishes. Rate-limited; only tools marked proactive-safe are allowed.",
});
export const assistantOpenAiCompatibleUrlSetting = new ConfigSettingString({
id: "assistant-openai-compatible-url",
label: "Compatible API URL",
storageKey: "assistant.openAiCompatibleUrl",
defaultValue: "",
editorTitle: "OpenAI-compatible API root",
glassesEditTitle: "Edit compatible API URL",
formatValue: (value) => value || "(not set)",
validate: (value) => (!value.trim() || normalizeOpenAiCompatibleUrl(value) ? null : "Enter an http:// or https:// API root."),
description: "Explicit API root for the OpenAI-compatible Chat Completions assistant. Venice: https://api.venice.ai/api/v1. Open WebUI: https://your-host/api. The provider's path is not guessed.",
});
export const assistantOpenAiCompatibleKeySetting = new ConfigSettingString({
id: "assistant-openai-compatible-key",
label: "Compatible API key",
storageKey: "assistant.openAiCompatibleKey",
defaultValue: "",
editorTitle: "OpenAI-compatible API key (optional)",
glassesEditTitle: "Edit compatible API key",
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(none)"),
description: "Bearer key sent only to the compatible assistant URL above. Optional for private servers that do not require authentication.",
});
export const assistantOpenAiCompatibleModelSetting = new ConfigSettingString({
id: "assistant-openai-compatible-model",
label: "Compatible model ID",
storageKey: "assistant.openAiCompatibleModel",
defaultValue: "",
editorTitle: "OpenAI-compatible model ID",
glassesEditTitle: "Edit compatible model ID",
formatValue: (value) => value || "(not set)",
description: "Exact model ID sent to the compatible provider. For glasses tools, choose a model that supports function calling.",
});
export const veniceWebSearchSetting = new ConfigSettingBoolean({
id: "venice-web-search",
label: "Venice web search",
storageKey: "assistant.veniceWebSearch",
defaultValue: false,
description: "Allow supported Venice models to search the web for current information. Shown only when OpenAI-compatible is the assistant model and its URL is the official Venice API. Search queries are processed by Venice's search service and may incur additional charges.",
});
export const elevenLabsApiKeySetting = new ConfigSettingString({
id: "elevenlabs-api-key",
label: "ElevenLabs key",
@@ -1101,11 +1162,12 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
if (provider === "local") return !isLocalModelReady();
if (provider === "hermes") return assistantHermesUrlSetting.get().trim().length === 0;
// Hermes and OpenAI-compatible stay selectable before they're configured:
// their rows only appear in Settings once one is the selected model.
return false;
},
description:
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
global.isIOS ? "Model used by the assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model; it never picks OpenAI-compatible or Hermes agent. Choosing either of those shows its settings below." : "Model used by the assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model; it never picks OpenAI-compatible or Hermes agent. Choosing either of those shows its settings below.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
@@ -1119,6 +1181,21 @@ export const mapboxApiKeySetting = new ConfigSettingString({
description: "Mapbox public token (pk. prefix), used by the Navigate app for maps, geocoding, and directions.",
});
/** Edited from the Music app's Spotify menu; a sign-in is tied to the client ID it was made with. */
export const spotifyClientIdSetting = new ConfigSettingString({
id: "spotify-client-id",
label: "Client ID",
storageKey: SPOTIFY_CLIENT_ID_KEY,
defaultValue: "",
editorTitle: "Spotify client ID (from your app in the Spotify Developer Dashboard)",
glassesEditTitle: "Edit Spotify client ID",
normalize: (value) => (value ?? "").trim(),
validate: (value) => (!value || isValidSpotifyClientId(value) ? null
: "A Spotify client ID is 32 letters and digits, shown on your app's page in the Spotify Developer Dashboard."),
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
description: "The client ID of your own Spotify developer app, which Faceclaw signs in with.",
});
/**
* Staging buffer for the Terminal app's "Add connection" flow: the worker
* asks the shell to open the phone text editor on this setting, the user
+69 -17
View File
@@ -12,7 +12,7 @@ import {
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
} from "../../native/llama";
} from "../../native/local-llm";
import {
ASR_MODELS,
asrModelState,
@@ -42,13 +42,18 @@ import {
assistantBridgePortSetting,
assistantBridgeTokenSetting,
assistantModelSetting,
assistantOpenAiCompatibleKeySetting,
assistantOpenAiCompatibleModelSetting,
assistantOpenAiCompatibleUrlSetting,
assistantSkipConfirmationSetting,
veniceWebSearchSetting,
batteryDisplayModeSetting,
brightnessSetting,
autoBrightnessMinSetting,
autoBrightnessMaxSetting,
autoBrightnessCurveSetting,
dataCollectionSetting,
distanceUnitSetting,
glassesBatteryVisibilitySetting,
phoneBatteryVisibilitySetting,
ringBatteryVisibilitySetting,
@@ -93,6 +98,8 @@ import {
import { wearBridge } from "../../native/wear-bridge";
import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from "./settings-panel";
import { uiFontPickerMenuItem } from "../font-picker";
import { compatibleModelDiscoveryMenuItem } from "../openai-compatible-model-picker";
import { isVeniceApiBaseUrl } from "../../native/openai-chat";
/** The Settings app's master-detail panel (sections on the left, contents on the right). */
export function createSettingsPanelLayer(): SettingsPanelLayer {
@@ -141,22 +148,42 @@ function settingsSections(): SettingsSection[] {
},
{
label: "Assistant",
items: [
// iOS offers cloud APIs; Android also supports local models and the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
textSettingMenuItem(assistantHermesUrlSetting),
textSettingMenuItem(assistantHermesKeySetting),
...(!global.isIOS ? [localModelMenuItem()] : []),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no local LLM or agent bridge yet.
...(!global.isIOS ? [
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
textSettingMenuItem(assistantBridgeTokenSetting),
toggleSettingMenuItem(assistantAllowProactiveSetting),
] : []),
],
// Re-read on every paint: provider-specific rows follow the selected
// model (and, for Venice, the compatible URL).
get items() {
const selectedModel = assistantModelSetting.get();
return [
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
...(selectedModel === "hermes" ? [
textSettingMenuItem(assistantHermesUrlSetting),
textSettingMenuItem(assistantHermesKeySetting),
] : []),
...(selectedModel === "openai-compatible" ? [
textSettingMenuItem(assistantOpenAiCompatibleUrlSetting),
textSettingMenuItem(assistantOpenAiCompatibleKeySetting),
compatibleModelDiscoveryMenuItem({
getBaseUrl: () => assistantOpenAiCompatibleUrlSetting.get(),
getApiKey: () => assistantOpenAiCompatibleKeySetting.get(),
getModel: () => assistantOpenAiCompatibleModelSetting.get(),
setModel: (model) => assistantOpenAiCompatibleModelSetting.set(model),
}, assistantOpenAiCompatibleModelSetting),
textSettingMenuItem(assistantOpenAiCompatibleModelSetting),
...(isVeniceApiBaseUrl(assistantOpenAiCompatibleUrlSetting.get())
? [toggleSettingMenuItem(veniceWebSearchSetting)] : []),
] : []),
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no agent bridge yet.
...(!global.isIOS ? [
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
textSettingMenuItem(assistantBridgeTokenSetting),
toggleSettingMenuItem(assistantAllowProactiveSetting),
] : []),
];
},
},
{
label: "API Keys",
@@ -292,6 +319,8 @@ function customizationRows(): () => MenuItem[] {
enumSettingMenuItem(timeFormatSetting),
// Fahrenheit or Celsius, wherever weather temperatures show.
enumSettingMenuItem(temperatureUnitSetting),
// Miles or kilometers: Weather's wind speed and Navigate's distances.
enumSettingMenuItem(distanceUnitSetting),
];
const bottomRows = [font, position, depth, statusBar, ...rest];
const popupRows = [font, position, depth, statusBar, statusBarVisibility, windowBorder, ...rest];
@@ -396,6 +425,29 @@ function localModelStatusText(): string {
return "not downloaded";
}
/**
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
* installs and manages; the row only reports whether it can be used.
*/
function systemModelMenuItem(): MenuItem {
const statusText = (): string => {
const state = localModelState();
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
};
return {
label: "On-phone model",
description:
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
"Faster and more private than the cloud models, but simpler, with a short memory. " +
"Used automatically when no API key is set.",
onSelect: () => {},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
},
};
}
/** Download/cancel/delete management for the on-phone assistant model. */
function localModelMenuItem(): MenuItem {
return {
+19 -6
View File
@@ -14,14 +14,16 @@ export type ContinuousCaptureOptions = {
/** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */
recordAs?: string;
/**
* Transcribe with this provider instead of the Transcription Provider
* setting, letting it detect the spoken language (Translate). While any
* client asks for one, every capture uses it, and a shared capture running
* another provider restarts on it.
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
* or "" for the provider to detect it), instead of the Transcription
* Provider setting (Translate). While any client asks for one, every
* capture uses it, and a shared capture running another restarts on it.
*/
multilingualProvider?: VoiceProvider;
transcription?: TranscriptionChoice;
};
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
@@ -100,6 +102,15 @@ export interface Layer {
readonly depth?: number;
paintParts?(): Plane[];
readonly paintOverBase?: boolean;
/**
* True when this layer stands in for `below`, a stacked layer directly
* beneath it (or beneath another layer it stands in for): when this layer
* calls paintBelow, the stack skips `below` and paints what is under it
* instead. A submenu replacing the menu it opened from uses this, so the
* hidden menu costs no plane and its dimUnderneath doesn't dim the content
* a second time; popping the submenu still returns to it.
*/
replacesLayerBelow?(below: Layer): boolean;
/**
* Dim everything painted beneath this layer: a 0..1 brightness factor
* applied to the pixels and glyphs of every plane below it in the stack
@@ -324,7 +335,9 @@ export class LayerStack {
return [ownPlane];
}
const dim = layer.dimUnderneath || 1;
const below = this.paintLayer(layer.paintOverBase ? 0 : index - 1, dimSoFar * dim, raw);
let belowIndex = layer.paintOverBase ? 0 : index - 1;
while (belowIndex > 0 && layer.replacesLayerBelow?.(this.layers[belowIndex]!)) belowIndex--;
const below = this.paintLayer(belowIndex, dimSoFar * dim, raw);
return [...(!raw && dim < 1 ? dimPlanes(below, dim) : below), ownPlane];
}
}
+92
View File
@@ -0,0 +1,92 @@
import { type GrayImage, type UiFont } from "../graphics/image";
import { truncateText } from "../graphics/textwrap";
import { voiceControlBridge } from "../native/voice-control";
import { lineStep } from "./metrics";
const HEADER_X = 24;
const HEADER_Y = 20;
const TITLE_GRAY = 200;
const STATUS_GRAY = 110;
const SEGMENTS = 8;
const SEGMENT_WIDTH = 4;
const SEGMENT_GAP = 2;
const METER_GAP = 10;
const LIT_GRAY = 200;
const UNLIT_GRAY = 45;
/** How often the meter may ask for a repaint (each one is a frame over BLE). */
const TICK_MS = 200;
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
const DECAY = 0.5;
/**
* A compact segmented mic-level meter for listening views. Levels arrive per
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
* asks for a repaint only when the number of lit segments changed and the
* window is on screen.
*/
export class MicLevelMeter {
readonly width = SEGMENTS * (SEGMENT_WIDTH + SEGMENT_GAP) - SEGMENT_GAP;
private peak = 0;
private shown = 0;
private lit = 0;
private timer: ReturnType<typeof setInterval> | null = null;
private unsubscribe: (() => void) | null = null;
start(requestRender: () => void, isVisible: () => boolean): void {
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
this.peak = Math.max(this.peak, level);
});
this.timer = setInterval(() => {
this.shown = Math.max(this.peak, this.shown * DECAY);
this.peak = 0;
const lit = Math.min(SEGMENTS, Math.round(this.shown * SEGMENTS));
if (lit === this.lit) return;
this.lit = lit;
if (isVisible()) requestRender();
}, TICK_MS);
}
stop(): void {
this.unsubscribe?.();
this.unsubscribe = null;
if (this.timer !== null) clearInterval(this.timer);
this.timer = null;
}
/** Draw the segments with their bottoms on `baseline`, `height` tall. */
paint(image: GrayImage, x: number, baseline: number, height: number): void {
for (let index = 0; index < SEGMENTS; index++) {
const segmentX = x + index * (SEGMENT_WIDTH + SEGMENT_GAP);
image.fillRect(segmentX, baseline - height, SEGMENT_WIDTH, height, index < this.lit ? LIT_GRAY : UNLIT_GRAY);
}
}
}
/**
* The one-line header of the listening views (Transcribe, Translate):
* "Title · status", the status truncated to fit, then the mic level meter at
* the line's end when one is given (while listening). Returns the y at which
* the view's body starts.
*/
export function paintListeningHeader(
image: GrayImage,
font: UiFont,
title: string,
status: string,
meter: MicLevelMeter | null,
): number {
let textRight = image.width - HEADER_X;
if (meter) {
const meterX = textRight - meter.width;
// Roughly cap height, sitting on the text's baseline.
meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6)));
textRight = meterX - METER_GAP;
}
image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY);
if (status) {
const statusX = HEADER_X + Math.ceil(font.measureText(title));
image.drawText(font, statusX, HEADER_Y, truncateText(font, ` · ${status}`, textRight - statusX), STATUS_GRAY);
}
return HEADER_Y + lineStep(font) + 2;
}
+118 -42
View File
@@ -9,12 +9,16 @@ import {
dismissNotification,
invokeNotificationAction,
readActiveNotifications,
readNotificationCustomView,
readNotificationIconByKey,
readNotificationLargeIcon,
readNotificationPicture,
type AndroidNotification,
type AndroidNotificationAction,
type NotificationImageResult,
} from "../native/notification-icons";
import { setNotificationSourceEnabled } from "../native/notification-sources";
import { NOTIFICATION_ICON_SIZE } from "../native/notification-types";
import { notificationEmptyMessage } from "../native/notification-access";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../util/render-freshness";
import { type InputEvent } from "./gestures";
@@ -26,7 +30,7 @@ const TITLE_X = 18;
const TITLE_Y = 10;
const LIST_TOP = 38;
const CARD_X = 20;
const ICON_SIZE = 24;
const ICON_SIZE = NOTIFICATION_ICON_SIZE;
const ICON_TEXT_GAP = 8;
const CARD_TEXT_X = 10 + ICON_SIZE + ICON_TEXT_GAP;
const CARD_GAP = 6;
@@ -45,13 +49,18 @@ const DETAIL_TITLE_Y = 21;
/** Gaps below the title's line box and below the icon/app-name header row. */
const DETAIL_HEADER_GAP = 3;
const DETAIL_BODY_GAP = 4;
/** Text lines an attached picture leaves room for above it (when there are that many); it takes the rest. */
/** Untitled text lines that attached images (custom layout, picture) leave room for above them, when there are that many; they take the rest. */
const DETAIL_PICTURE_TEXT_LINES = 2;
/** Gap between the text and an attached picture, and the space kept below the picture. */
const DETAIL_PICTURE_GAP = 6;
const DETAIL_PICTURE_BOTTOM_MARGIN = 12;
/** An attached picture with less height than this to fit in (a large UI font) is left out. */
const DETAIL_PICTURE_MIN_HEIGHT = 32;
/** Likewise a custom layout; Dexcom's reading is about 40 high. */
const DETAIL_CUSTOM_VIEW_MIN_HEIGHT = 16;
/** A large icon (sender's avatar, site's logo): at the right of a list card, left of the detail view's title. */
const LARGE_ICON_SIZE = 32;
const LARGE_ICON_GAP = 8;
// Disable-source confirmation: actions on top, the explanation below them.
const CONFIRM_X = 24;
const CONFIRM_Y = 8;
@@ -67,14 +76,27 @@ function iconForNotification(key: string): GrayImage | null {
return icon;
}
/** Attached picture for a paint pass, fitting maxWidth x maxHeight; allow-stale like iconForNotification. */
function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null {
if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null;
const { picture, stale } = readNotificationPicture(notification, maxWidth, maxHeight, renderPassAllowsStaleData());
if (stale) {
/** A notification image read for a paint pass (allow-stale like iconForNotification). */
function imageForPaint(
read: (allowStale: boolean) => NotificationImageResult,
): NotificationImageResult {
const result = read(renderPassAllowsStaleData());
if (result.stale) {
noteStaleDataUsed();
}
return picture;
return result;
}
/** Attached picture for a paint pass, fitting maxWidth x maxHeight. */
function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null {
if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null;
return imageForPaint((allowStale) => readNotificationPicture(notification, maxWidth, maxHeight, allowStale)).image;
}
/** Large icon for a paint pass, LARGE_ICON_SIZE square at most. */
function largeIconForNotification(notification: AndroidNotification): GrayImage | null {
if (!notification.hasLargeIcon) return null;
return imageForPaint((allowStale) => readNotificationLargeIcon(notification, LARGE_ICON_SIZE, allowStale)).image;
}
type CardLayout = {
@@ -152,7 +174,8 @@ export class NotificationsListLayer implements Layer {
// Icons are only resolved for cards actually drawn, so a long list
// does not fetch icons for everything below the fold.
const icon = iconForNotification(layout.notification.key);
drawNotificationCard(image, font, layout, CARD_X, cursorY, cardWidth, index === selectedIndex, focused, icon);
const largeIcon = largeIconForNotification(layout.notification);
drawNotificationCard(image, font, layout, CARD_X, cursorY, cardWidth, index === selectedIndex, focused, icon, largeIcon);
}
cursorY += layout.height + CARD_GAP;
if (cursorY > listBottom + 80) break;
@@ -356,8 +379,10 @@ function buildNotificationCardLayout(
font: UiFont,
notification: AndroidNotification,
selected: boolean,
cardTextWidth: number,
fullTextWidth: number,
): CardLayout {
// The large icon sits at the card's right, beside all its text.
const cardTextWidth = notification.hasLargeIcon ? fullTextWidth - LARGE_ICON_SIZE - LARGE_ICON_GAP : fullTextWidth;
const lines: string[] = [];
const time = formatRelativeTime(notification.postTime);
const title = notificationTitle(notification);
@@ -383,7 +408,7 @@ function buildNotificationCardLayout(
return {
notification,
lines,
height: Math.max(ICON_SIZE + 18, 12 + lines.length * lineStep(font)),
height: Math.max(ICON_SIZE + 18, notification.hasLargeIcon ? LARGE_ICON_SIZE + 16 : 0, 12 + lines.length * lineStep(font)),
};
}
@@ -397,6 +422,7 @@ function drawNotificationCard(
selected: boolean,
focused: boolean,
icon: GrayImage | null,
largeIcon: GrayImage | null,
): void {
// Match the shared menu highlight: faint border + black fill when
// unselected; bright border when selected; the non-black fill only appears
@@ -408,6 +434,9 @@ function drawNotificationCard(
if (icon) {
image.bitBlt(icon, x + 10, y + 8, { transparentZero: true });
}
if (largeIcon) {
image.bitBlt(largeIcon, x + width - 10 - LARGE_ICON_SIZE + ((LARGE_ICON_SIZE - largeIcon.width) >> 1), y + 8);
}
for (let index = 0; index < layout.lines.length; index++) {
const value = index === 0 ? 140 : selected ? 235 : 185;
image.drawText(font, x + CARD_TEXT_X, y + 7 + index * lineStep(font), layout.lines[index]!, value);
@@ -437,18 +466,35 @@ function drawDetailContent(
): void {
const contentX = NOTIFICATION_CONTENT_X;
const contentWidth = notificationContentWidth(width);
const bodyTop = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`);
const step = lineStep(font);
const bottom = height - DETAIL_PICTURE_BOTTOM_MARGIN;
let top = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`);
// The title, beside the large icon (a sender's avatar, a site's logo) when
// there is one. A picture or custom layout can be the whole notification
// (Dexcom's has no text), so either stands in for the "(untitled)" placeholder.
const hasImages = Boolean(notification.hasPicture || notification.hasCustomView);
const title = notification.title || (hasImages ? "" : "(untitled)");
const avatarSize = notification.hasLargeIcon ? LARGE_ICON_SIZE : 0;
const titleX = avatarSize ? contentX + avatarSize + LARGE_ICON_GAP : contentX;
const titleWidth = contentX + contentWidth - titleX;
const titleLines = title ? wrapText(font, title, titleWidth) : [];
let titleHeight = 0;
if (avatarSize) {
const avatar = largeIconForNotification(notification);
if (avatar) image.bitBlt(avatar, contentX + ((avatarSize - avatar.width) >> 1), top + ((avatarSize - avatar.height) >> 1));
titleHeight = avatarSize;
}
if (titleLines.length) {
// A title shorter than the avatar is centered on it.
const titleTop = top + Math.max(0, (avatarSize - titleLines.length * step) >> 1);
const shown = drawClippedLines(image, font, titleLines, titleX, titleTop, titleWidth, ((bottom - top) / step) | 0, 230);
titleHeight = Math.max(titleHeight, shown * step);
}
const lines: string[] = [];
// A picture can be the whole notification (Dexcom's has no text), so it
// stands in for the "(untitled)" placeholder.
const hasTitle = Boolean(notification.title) || !notification.hasPicture;
if (hasTitle) {
lines.push(...wrapText(font, notification.title || "(untitled)", contentWidth));
}
const body = detailNotificationBody(notification);
if (body) {
if (lines.length) lines.push("");
lines.push(...wrapText(font, body, contentWidth));
}
const meta = [notification.subText, notification.infoText, notification.summaryText].filter(Boolean).join(" ");
@@ -456,36 +502,65 @@ function drawDetailContent(
if (lines.length) lines.push("");
lines.push(...wrapText(font, meta, contentWidth));
}
// Text follows the title after a blank line (just a gap beside an avatar); images after a gap.
if (titleHeight) {
top += titleHeight + (lines.length && !avatarSize ? step : DETAIL_PICTURE_GAP);
}
const step = lineStep(font);
const textValue = (index: number) => index === 0 && hasTitle ? 230 : 190;
const reservedTextHeight = Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step;
const picture = pictureForNotification(notification, contentWidth,
height - DETAIL_PICTURE_BOTTOM_MARGIN - bodyTop - reservedTextHeight - (reservedTextHeight ? DETAIL_PICTURE_GAP : 0));
if (picture) {
// The text gets whatever the picture leaves, above it, with its last
// shown line marked when some is cut off.
const roomForText = height - DETAIL_PICTURE_BOTTOM_MARGIN - picture.height - DETAIL_PICTURE_GAP - bodyTop;
const shownLines = Math.min(lines.length, Math.max(0, (roomForText / step) | 0));
for (let index = 0; index < shownLines; index++) {
const line = index === shownLines - 1 && lines.length > shownLines
? truncateText(font, lines[index] + "...", contentWidth) : lines[index]!;
image.drawText(font, contentX, bodyTop + index * step, line, textValue(index));
// The custom layout, then the picture, take what is left below the title,
// or below the first few lines of text when there is no title.
const reservedTextHeight = titleHeight ? 0 : Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step;
const roomForImages = bottom - top - (reservedTextHeight ? reservedTextHeight + DETAIL_PICTURE_GAP : 0);
let customView: GrayImage | null = null;
if (notification.hasCustomView && roomForImages >= DETAIL_CUSTOM_VIEW_MIN_HEIGHT) {
const result = imageForPaint((allowStale) => readNotificationCustomView(notification, contentWidth, roomForImages, allowStale));
customView = result.image;
if (!customView && !result.stale && notification.customText?.length) {
// It couldn't be drawn: its text is better than nothing.
lines.unshift(...wrapText(font, notification.customText.join(" "), contentWidth), ...(lines.length ? [""] : []));
}
image.bitBlt(picture, contentX, bodyTop + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0));
return;
}
const picture = pictureForNotification(notification, contentWidth,
roomForImages - (customView ? customView.height + DETAIL_PICTURE_GAP : 0));
const maxLines = Math.max(1, ((height - bodyTop - step) / step) | 0);
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
const line = lines[index]!;
image.drawText(font, contentX, bodyTop + index * step, line, textValue(index));
// The text gets whatever the images leave, above them, with its last shown
// line marked when some is cut off.
const imagesHeight = (customView ? customView.height : 0) + (picture ? picture.height : 0)
+ (customView && picture ? DETAIL_PICTURE_GAP : 0);
const roomForText = bottom - top - (imagesHeight ? imagesHeight + DETAIL_PICTURE_GAP : 0);
const shownLines = drawClippedLines(image, font, lines, contentX, top, contentWidth, (roomForText / step) | 0, 190);
let imageTop = top + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0);
if (customView) {
image.bitBlt(customView, contentX, imageTop);
imageTop += customView.height + DETAIL_PICTURE_GAP;
}
if (lines.length > maxLines) {
image.drawText(font, contentX, height - 24, "...", 140);
if (picture) {
image.bitBlt(picture, contentX, imageTop);
}
}
/**
* Draw up to maxLines of lines from (x, y), marking the last one drawn with
* "..." when some are left out. Returns how many were drawn.
*/
function drawClippedLines(
image: GrayImage,
font: UiFont,
lines: string[],
x: number,
y: number,
width: number,
maxLines: number,
value: number,
): number {
const shown = Math.min(lines.length, Math.max(0, maxLines));
for (let index = 0; index < shown; index++) {
const line = index === shown - 1 && lines.length > shown ? truncateText(font, lines[index] + "...", width) : lines[index]!;
image.drawText(font, x, y + index * lineStep(font), line, value);
}
return shown;
}
/** Left edge of a notification detail view's content column. */
export const NOTIFICATION_CONTENT_X = DETAIL_CONTENT_X;
@@ -577,5 +652,6 @@ function detailNotificationBody(notification: AndroidNotification): string {
}
function notificationTitle(notification: AndroidNotification): string {
return notification.title || notification.text || notification.summaryText || notification.appName || notification.packageName || "(untitled)";
return notification.title || notification.text || notification.customText?.join(" ") || notification.summaryText
|| notification.appName || notification.packageName || "(untitled)";
}
+232
View File
@@ -0,0 +1,232 @@
import { MenuLayer, type MenuItem } from "./menu";
import { EditTextSettingLayer } from "./dashboard-settings";
import type { LayerContext } from "./layers";
import {
discoverOpenAiCompatibleModels,
type DiscoveredCompatibleModel,
} from "../native/openai-chat-models";
export type ModelDiscoverySettings = {
getBaseUrl(): string;
getApiKey(): string;
getModel(): string;
setModel(model: string): void;
};
type Discover = typeof discoverOpenAiCompatibleModels;
type PickerState = "loading" | "models" | "error";
const DISCOVERY_EXPLANATION =
"Discover models queries only this server's /models endpoint. Choose an exact model ID, or keep using manual entry. This does not send an inference request.";
/** A cancellable, on-demand model picker embedded in Faceclaw's existing menus. */
export class OpenAiCompatibleModelPicker extends MenuLayer {
private state: PickerState = "loading";
private models: DiscoveredCompatibleModel[] = [];
private error = "";
private showOffline = false;
private requestGeneration = 0;
private endpointKey = "";
private removed = false;
private readonly modelIdByRow = new Map<MenuItem, string>();
constructor(
private readonly settings: ModelDiscoverySettings,
private readonly modelSetting: ConstructorParameters<typeof EditTextSettingLayer>[0],
private readonly discover: Discover = discoverOpenAiCompatibleModels,
viewport: { width: number; height: number } = { width: 640, height: 480 },
) {
super("Discover models", [], discoveryLayout(viewport));
}
open(ctx: LayerContext): void {
ctx.stack.push(this);
void this.refresh(ctx);
}
private currentEndpoint(): string {
return `${this.settings.getBaseUrl().trim()}\n${this.settings.getApiKey().trim()}`;
}
private async refresh(ctx: LayerContext, forceRefresh = false): Promise<void> {
const generation = ++this.requestGeneration;
const endpointKey = this.currentEndpoint();
this.endpointKey = endpointKey;
this.state = "loading";
this.error = "";
this.updateRows();
try {
const models = await this.discover({
baseUrl: this.settings.getBaseUrl(),
apiKey: this.settings.getApiKey(),
forceRefresh,
});
if (!this.isActive(generation)) return;
if (endpointKey !== this.currentEndpoint()) {
this.error = "API URL or key changed; discover again";
this.state = "error";
} else {
this.models = models;
this.state = "models";
}
} catch (error) {
if (!this.isActive(generation)) return;
this.error = endpointKey !== this.currentEndpoint()
? "API URL or key changed; discover again"
: error instanceof Error ? error.message : String(error);
this.state = "error";
}
this.updateRows();
ctx.actions.requestRender();
}
private isActive(generation: number): boolean {
// A temporary overlay may sit above this still-open picker. Removal, not
// top-of-stack status, determines whether the pending result is obsolete.
return !this.removed && generation === this.requestGeneration;
}
private updateRows(): void {
this.modelIdByRow.clear();
const rows: MenuItem[] = [];
if (this.state === "loading") {
rows.push({ label: "Discovering models...", disabled: true, description: DISCOVERY_EXPLANATION, onSelect: () => {} });
rows.push({
label: "Cancel discovery",
description: "Ignore the pending result and return to Assistant settings.",
onSelect: (innerCtx) => innerCtx.stack.pop(),
});
} else if (this.state === "error") {
rows.push({
label: `Failed: ${truncateModelLabel(this.error || "Discovery failed", 40)}`,
disabled: true,
description: `${this.error}. Manual model entry is still available.`,
onSelect: () => {},
});
rows.push({
label: "Try again",
description: "Query this server's /models endpoint again without changing the saved model.",
onSelect: (innerCtx) => { void this.refresh(innerCtx, true); },
});
} else {
const saved = this.settings.getModel().trim();
const listed = this.models.filter((model) => this.showOffline || !model.offline || model.id === saved);
const savedModel = saved ? this.models.find((model) => model.id === saved) : undefined;
if (saved && !savedModel) {
rows.push(this.modelRow({
id: saved, name: saved, supportsTools: null, offline: false, deprecated: false, beta: false,
}, "saved; not listed"));
}
if (!listed.length) {
rows.push({
label: this.models.length ? "No online models listed" : "No models listed",
disabled: true,
description: "Keep the saved model or use manual entry.",
onSelect: () => {},
});
} else {
rows.push(...listed.map((model) => this.modelRow(model)));
}
if (this.models.some((model) => model.offline)) {
rows.push({
label: this.showOffline ? "Hide offline models" : `Show offline (${this.models.filter((model) => model.offline).length})`,
description: "Offline models normally fail inference until the provider restores them.",
onSelect: (innerCtx) => {
this.showOffline = !this.showOffline;
this.updateRows();
innerCtx.actions.requestRender();
},
});
}
rows.push({
label: "Refresh models",
description: "Fetch a fresh catalog. This never changes the selected model by itself.",
onSelect: (innerCtx) => { void this.refresh(innerCtx, true); },
});
}
rows.push({
label: "Enter model manually",
description: "Use an exact model ID when discovery is unavailable or the desired model is not listed.",
onSelect: (innerCtx) => {
this.requestGeneration++; // Any late discovery result belongs to the closed picker.
void innerCtx.actions.startTextSettingEdit(this.modelSetting);
innerCtx.stack.popThrough(this);
innerCtx.stack.push(new EditTextSettingLayer(this.modelSetting));
},
});
const current = this.settings.getModel().trim();
const selected = Math.max(0, rows.findIndex((row) => this.modelIdByRow.get(row) === current));
this.setItems(rows, selected);
}
private modelRow(model: DiscoveredCompatibleModel, extraLabel?: string): MenuItem {
const flags: string[] = [];
if (model.supportsTools === true) flags.push("tools");
else if (model.supportsTools === false) flags.push("no tools");
else flags.push("tools unknown");
if (model.offline) flags.push("offline");
if (model.deprecated) flags.push("deprecated");
if (model.beta) flags.push("beta");
if (extraLabel) flags.push(extraLabel);
const selected = this.settings.getModel().trim() === model.id ? " *" : "";
const displayName = model.name === model.id ? model.id : `${model.name} (${model.id})`;
const toolFlag = model.supportsTools === true ? "tools" : model.supportsTools === false ? "no tools" : "tools?";
const visibleFlags = [toolFlag];
if (model.offline) visibleFlags.push("offline");
if (model.deprecated) visibleFlags.push("old");
if (model.beta) visibleFlags.push("beta");
if (extraLabel) visibleFlags.push("saved only");
const flagSuffix = ` [${visibleFlags.join("/")}]`;
const row: MenuItem = {
label: `${truncateModelLabel(displayName, Math.max(12, 44 - flagSuffix.length))}${flagSuffix}${selected}`,
description: `${model.name}: ${model.id} — ${flags.join(", ")}. Tool support unknown means the server did not advertise it; test before relying on glasses actions.`,
onSelect: (innerCtx) => {
// Re-read immediately before saving; a late catalog never overwrites a
// selection made against a different endpoint.
if (this.settings.getBaseUrl().trim() + "\n" + this.settings.getApiKey().trim() !== this.endpointKey) return;
this.settings.setModel(model.id);
innerCtx.stack.popThrough(this);
},
};
this.modelIdByRow.set(row, model.id);
return row;
}
onRemoved(): void {
this.removed = true;
this.requestGeneration++; // Invalidate the pending result immediately.
}
}
/** Kept separate so tests can verify the normal Faceclaw modal sizing contract. */
function discoveryLayout(viewport: { width: number; height: number }): ConstructorParameters<typeof MenuLayer>[2] {
return {
x: "center",
y: 8,
width: Math.min(320, viewport.width - 40),
minHeight: Math.min(160, viewport.height - 40),
maxHeight: Math.min(240, viewport.height - 40),
depth: 4,
};
}
function truncateModelLabel(value: string, maxLength = 44): string {
if (value.length <= maxLength) return value;
const tailLength = Math.min(19, Math.max(4, maxLength - 8));
const headLength = Math.max(1, maxLength - tailLength - 3);
return `${value.slice(0, headLength)}...${value.slice(-tailLength)}`;
}
/** One stable Assistant settings row; discovery state belongs to the pushed picker. */
export function compatibleModelDiscoveryMenuItem(
settings: ModelDiscoverySettings,
modelSetting: ConstructorParameters<typeof EditTextSettingLayer>[0],
discover: Discover = discoverOpenAiCompatibleModels,
): MenuItem {
return {
label: "Discover models",
description: DISCOVERY_EXPLANATION,
disabled: () => !settings.getBaseUrl().trim(),
onSelect: (ctx) => new OpenAiCompatibleModelPicker(settings, modelSetting, discover, ctx.stack.getBaseSize()).open(ctx),
};
}
+1 -1
View File
@@ -7,6 +7,7 @@ import { truncateText } from "../../graphics/textwrap";
import { activeAmbientCards } from "./ambient-cards";
import { BATTERY_ICON_WIDTH, DENSE_BATTERY_BLOCK_HEIGHT, drawBattery, drawDenseBatteries, type BatteryDevice } from "../../graphics/battery";
import { ALL_NOTIFICATIONS, readActiveNotificationIcons } from "../../native/notification-icons";
import { NOTIFICATION_ICON_SIZE } from "../../native/notification-types";
import { readPhoneBatteryState } from "../../native/phone-battery";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
import { renderIcon, renderIconWithGlyph, type IconActivity, type IconName } from "../../graphics/icons";
@@ -430,7 +431,6 @@ function statusBarPlacement(state: ShellChromeState): StatusBarPlacement | null
return { left, right, top, contentTop: atBottom ? top + 1 : top, divider };
}
const NOTIFICATION_ICON_SIZE = 24;
const NOTIFICATION_ICON_STEP = NOTIFICATION_ICON_SIZE + 4;
// Notification icons' top, in the top bar's layout (where any status icon's
// selection box starts, a row above).
+5 -4
View File
@@ -192,12 +192,13 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
drawIcon: options.drawIcon ?? windowIcon(options.icon, options.iconLetter),
handleInput: async (event, frameId) => {
// The shell opened its system menu over this window; close our own
// context menu so the two never stack. Never forwarded to app layers.
// context menu (and any window submenus open from it) so the two never
// stack. Never forwarded to app layers.
if (event.type === "system-menu-opened") {
options.onSystemMenuOpened?.();
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
requestRender();
}
let closedMenu = false;
while (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) closedMenu = true;
if (closedMenu) requestRender();
return;
}
// The default tap-then-hold response: the window menu (or the system
+88 -6
View File
@@ -8,6 +8,16 @@ import { type VoiceSendTarget } from "./voice-input";
/** A menu row: a send target, or Discard (no target). */
type KeyboardMenuRow = InputDialogRow & { target?: VoiceSendTarget };
/**
* Where the keyboard dialog shows: on both ends, only the phone's typing panel
* (the glasses carry on as if no dialog were open), or only the glasses (the
* phone hides its copy of the text but keeps the IME typing into it).
*/
export type KeyboardInputVisibility = "both" | "phone" | "glasses";
/** Sent messages kept for the phone's history button. */
const MAX_HISTORY = 20;
/**
* The phone side's handle on an open keyboard-input dialog: the phone types
* into it and picks where the text goes. The layer itself implements this;
@@ -25,6 +35,10 @@ export type KeyboardInputSession = {
send(): void;
sendTo(targetId: string): void;
discard(): void;
readonly visibility: KeyboardInputVisibility;
setVisibility(visibility: KeyboardInputVisibility): void;
/** Messages sent from keyboard dialogs since the app started, newest first. */
readonly history: ReadonlyArray<string>;
};
export type KeyboardInputLayerOptions = {
@@ -33,6 +47,16 @@ export type KeyboardInputLayerOptions = {
onClosed: () => void;
/** Pop this layer off the shell stack. */
dismiss: () => void;
/**
* Put the layer back on the shell stack, or take it off without ending the
* session (phone-only visibility; onRemoved is ignored while hiding).
*/
setShownOnGlasses: (shown: boolean) => void;
/** Starting visibility; the shell pushes the layer only if it isn't "phone". */
visibility: KeyboardInputVisibility;
onVisibilityChanged: (visibility: KeyboardInputVisibility) => void;
/** Shared, newest-first list this layer adds its sent message to. */
history: string[];
/** Ordered send destinations shown as the first menu rows. */
sendTargets: VoiceSendTarget[];
/** Which send target is highlighted by default (entry-point dependent). */
@@ -52,10 +76,16 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
private text = "";
private readonly menu: Menu<KeyboardMenuRow>;
private closed = false;
private currentVisibility: KeyboardInputVisibility;
// Set while the shell takes the layer off the stack for phone-only typing.
private hiding = false;
private readonly actions: LayerActions;
private readonly onClosed: () => void;
private readonly dismiss: () => void;
private readonly setShownOnGlasses: (shown: boolean) => void;
private readonly onVisibilityChanged: (visibility: KeyboardInputVisibility) => void;
private readonly sentHistory: string[];
private readonly sendTargets: VoiceSendTarget[];
private readonly defaultTargetIndex: number;
@@ -63,6 +93,10 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
this.actions = options.actions;
this.onClosed = options.onClosed;
this.dismiss = options.dismiss;
this.setShownOnGlasses = options.setShownOnGlasses;
this.currentVisibility = options.visibility;
this.onVisibilityChanged = options.onVisibilityChanged;
this.sentHistory = options.history;
this.sendTargets = options.sendTargets;
const defaultIndex = options.defaultTargetIndex ?? 0;
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
@@ -73,6 +107,33 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
return this.sendTargets.map((target) => ({ id: target.id, label: target.label }));
}
get visibility(): KeyboardInputVisibility {
return this.currentVisibility;
}
get history(): ReadonlyArray<string> {
return this.sentHistory;
}
/** Whether the dialog is on the glasses (the shell stack) rather than phone-only. */
get shownOnGlasses(): boolean {
return this.currentVisibility !== "phone";
}
setVisibility(visibility: KeyboardInputVisibility): void {
if (this.closed || visibility === this.currentVisibility) return;
const wasShown = this.shownOnGlasses;
this.currentVisibility = visibility;
this.onVisibilityChanged(visibility);
if (this.shownOnGlasses === wasShown) return;
this.hiding = !this.shownOnGlasses;
try {
this.setShownOnGlasses(this.shownOnGlasses);
} finally {
this.hiding = false;
}
}
setText(text: string): void {
if (this.closed || this.text === text) return;
this.text = text;
@@ -90,7 +151,23 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
}
discard(): void {
if (!this.closed) this.dismiss();
this.close();
}
private close(): void {
if (this.closed) return;
// Phone-only, there is no stack entry to pop; end the session directly.
if (this.shownOnGlasses) {
this.dismiss();
} else {
this.finish();
}
}
private finish(): void {
if (this.closed) return;
this.closed = true;
this.onClosed();
}
private sendToTarget(target: VoiceSendTarget): void {
@@ -99,7 +176,11 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
// An empty message has nothing to deliver; the dialog stays up for the
// phone to keep typing (the glasses row is drawn dim for the same reason).
if (!text) return;
this.dismiss();
const previous = this.sentHistory.indexOf(text);
if (previous >= 0) this.sentHistory.splice(previous, 1);
this.sentHistory.unshift(text);
this.sentHistory.length = Math.min(this.sentHistory.length, MAX_HISTORY);
this.close();
target.onSend(text);
}
@@ -140,12 +221,12 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
if (row.target) {
this.sendToTarget(row.target);
} else {
this.dismiss();
this.close();
}
return;
}
case "double-click":
this.dismiss();
this.close();
return;
default:
return;
@@ -153,7 +234,8 @@ export class KeyboardInputLayer implements Layer, KeyboardInputSession {
}
onRemoved(): void {
this.closed = true;
this.onClosed();
// Taken off the glasses for phone-only typing: the session carries on.
if (this.hiding) return;
this.finish();
}
}
+55 -12
View File
@@ -20,7 +20,7 @@ import {
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input";
import { KeyboardInputLayer, type KeyboardInputSession, type KeyboardInputVisibility } from "./keyboard-input";
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
@@ -38,7 +38,11 @@ import {
assistantBridgePortSetting,
assistantBridgeTokenSetting,
assistantModelSetting,
assistantOpenAiCompatibleKeySetting,
assistantOpenAiCompatibleModelSetting,
assistantOpenAiCompatibleUrlSetting,
assistantSkipConfirmationSetting,
veniceWebSearchSetting,
batteryIndicatorSettingsKey,
brightnessSetting,
onAnySettingChanged,
@@ -310,6 +314,9 @@ const noopActions: LayerActions = noopLayerActions;
*/
const LONG_PRESS_ESCAPE_MENU_MS = 4000;
/** Settings-store key for the keyboard dialog's last-chosen KeyboardInputVisibility. */
const KEYBOARD_INPUT_VISIBILITY_KEY = "keyboardInput.visibility";
/** Shell-surface overlay menu (the system menu); closing it returns focus to the sidebar. */
class ShellOverlayMenuLayer extends MenuLayer {
/**
@@ -455,6 +462,8 @@ class Shell {
private readonly trayIcons = new Map<string, { icon: GrayImage; widget?: TrayWidget }>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
// Messages sent from keyboard dialogs, newest first; in memory only.
private readonly keyboardInputHistory: string[] = [];
// App-posted notifications that are open or queued, by key.
private readonly appNotifications = new Map<string, AppNotification>();
private readonly notificationModals = new NotificationModalQueue<NotificationPopup>({
@@ -1194,7 +1203,7 @@ class Shell {
const foreground = this.foregroundWindow()?.windowId ?? "none";
if (!this.screenOn) return `fg=${foreground} target=screen-off`;
if (this.activeVoiceLayer) return `fg=${foreground} target=voice`;
if (this.activeKeyboardLayer) return `fg=${foreground} target=keyboard`;
if (this.activeKeyboardLayer?.shownOnGlasses) return `fg=${foreground} target=keyboard`;
if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`;
return `fg=${foreground} target=${this.focus}`;
}
@@ -1583,12 +1592,16 @@ class Shell {
* continues that conversation; otherwise it goes to the usual destinations
* with Send to Assistant highlighted. Returns the session the phone types
* into (the already-open one if there is one), or null while a voice
* dialog is up.
* dialog is up. The phone can keep the dialog off the glasses (see
* KeyboardInputVisibility); that choice persists across dialogs.
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
if (!this.screenOn) this.wake("sidebar");
const storedVisibility = getStringSetting(KEYBOARD_INPUT_VISIBILITY_KEY, "both");
const visibility: KeyboardInputVisibility =
storedVisibility === "phone" || storedVisibility === "glasses" ? storedVisibility : "both";
if (visibility !== "phone" && !this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
let targets: VoiceSendTarget[];
@@ -1598,7 +1611,15 @@ class Shell {
{
id: "assistant",
label: "Send",
onSend: (text) => this.runAssistantTurn(assistantSession, assistantLayer, text),
// Phone-only, the overlay can close while the dialog is open; the
// text then starts a fresh overlay on the same conversation.
onSend: (text) => {
if (this.assistantLayer === assistantLayer) {
this.runAssistantTurn(assistantSession, assistantLayer, text);
} else {
this.sendToAssistant(text);
}
},
},
];
} else {
@@ -1616,16 +1637,29 @@ class Shell {
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === layer);
// Wherever it sits: a menu opened over the dialog stays open.
this.stack.remove(layer);
// A send or discard from the phone side arrives outside the input
// path, so nothing else asks for the repaint.
this.config.requestShellRender();
},
setShownOnGlasses: (shown) => {
if (shown) {
if (!this.screenOn) this.wake("sidebar");
this.stack.push(layer);
} else {
this.stack.remove(layer);
}
this.config.requestShellRender();
},
visibility,
onVisibilityChanged: (value) => setStringSetting(KEYBOARD_INPUT_VISIBILITY_KEY, value),
history: this.keyboardInputHistory,
sendTargets: targets,
defaultTargetIndex: defaultIndex,
});
this.activeKeyboardLayer = layer;
this.stack.push(layer);
if (layer.shownOnGlasses) this.stack.push(layer);
this.config.onKeyboardInputChanged?.(layer);
this.config.requestShellRender();
return layer;
@@ -1636,6 +1670,15 @@ class Shell {
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
}
/** What to set up when isAssistantAvailable() is false. */
assistantSetupHint(): string {
if (assistantBackendSetting.get() === "external") return "Configure the agent bridge host and token in Settings.";
const model = this.getAssistantConversations().current().model;
if (model === "hermes") return "Set the Hermes URL in Settings > Assistant.";
if (model === "openai-compatible") return "Set the compatible API URL and model ID in Settings > Assistant.";
return global.isIOS ? "Configure an assistant model in Settings." : "Set an API key or download the on-phone model in Settings.";
}
/** Observe assistant turns (see AssistantActivityEvent). */
onAssistantActivity(listener: (event: AssistantActivityEvent) => void): () => void {
this.assistantActivityListeners.add(listener);
@@ -1690,6 +1733,10 @@ class Shell {
openai: openAiApiKeySetting.get(),
hermesUrl: assistantHermesUrlSetting.get(),
hermesKey: assistantHermesKeySetting.get(),
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
veniceWebSearch: veniceWebSearchSetting.get(),
});
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null;
@@ -1716,11 +1763,7 @@ class Shell {
if (!text) return;
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert(
assistantBackendSetting.get() === "external"
? "Configure the agent bridge host and token in Settings."
: global.isIOS ? "Set an OpenAI or Anthropic API key in Settings." : "Set an API key or download the on-phone model in Settings.",
);
this.showAlert(this.assistantSetupHint());
return;
}
if (session.isTurnActive()) {
+10
View File
@@ -33,6 +33,16 @@ export class WindowMenuLayer extends MenuLayer {
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
: WINDOW_MENU_LAYOUT);
}
/**
* A window submenu (Settings, Spotify, ...) opens in place of the window
* menu it came from, which it would mostly cover anyway: the window content
* shows beneath it, dimmed once, rather than the parent menu dimming it a
* second time and costing a plane of its own.
*/
replacesLayerBelow(below: Layer): boolean {
return below instanceof WindowMenuLayer;
}
}
export type WindowMenuOptions = {