Merge main into ios-port and update the iOS CFW transport

This commit is contained in:
James Babcock committed 2026-09-17 23:33:51 -07:00
commit e8e144f3f2
156 files changed
+10528 -1679

No files matched your search

+25
View File
@@ -0,0 +1,25 @@
/**
* Clock/date text shared by the shell's top bar and the Glanceboard's System
* Card, so the two never disagree on the time-format setting or the date
* spelling.
*/
import { timeFormatSetting } from "./dashboard-settings";
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
/** Format a clock time, honoring the 12/24-hour setting ("14:05" / "2:05 PM"). */
export function formatClockTime(now: Date): string {
const minutes = String(now.getMinutes()).padStart(2, "0");
const hour24 = now.getHours();
if (timeFormatSetting.get() === "12h") {
const hour12 = ((hour24 + 11) % 12) + 1;
return `${hour12}:${minutes} ${hour24 < 12 ? "AM" : "PM"}`;
}
return `${hour24}:${minutes}`;
}
/** Short date as the top bar shows it: "Thu 11 Sep". */
export function formatClockDate(now: Date): string {
return `${WEEKDAYS[now.getDay()]} ${now.getDate()} ${MONTHS[now.getMonth()]}`;
}
+84 -12
View File
@@ -25,7 +25,9 @@ export type NightscoutSettings = {
siteUrl: string;
apiToken: string;
};
export type BatteryDisplayMode = "icon" | "percentage";
export type BatteryDisplayMode = "icon" | "percentage" | "stacked" | "stacked-percentage";
/** When a top-bar battery indicator is shown: always, only below 50%, or never. */
export type BatteryIndicatorVisibility = "always" | "low" | "never";
export type TimeFormat = "24h" | "12h";
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "3m" | "never";
// "auto" lets the glasses' ambient-light sensor drive brightness; the numeric
@@ -211,14 +213,68 @@ export class ConfigSettingString<TId extends string = string> extends ConfigSett
export const batteryDisplayModeSetting = new ConfigSettingEnum<BatteryDisplayMode>({
id: "batteryDisplayMode",
label: "Battery display",
label: "Style",
storageKey: "dashboard.systemCard.batteryDisplayMode",
defaultValue: "icon",
values: ["icon", "percentage"],
defaultValue: "stacked",
values: ["icon", "percentage", "stacked", "stacked-percentage"],
formatValue: batteryDisplayModeLabel,
description: "How the top bar shows the phone and glasses battery levels: a small gauge icon or an exact percentage.",
description: "How the top bar shows battery levels: a gauge icon or exact percentage beside the label, or a compact gauge or percentage with the label stacked above it.",
});
/** Below this charge level a "Below 50%" indicator becomes visible. */
export const BATTERY_LOW_VISIBILITY_THRESHOLD = 50;
function batteryVisibilitySetting(
id: string,
device: string,
storageKey: string,
): ConfigSettingEnum<BatteryIndicatorVisibility> {
return new ConfigSettingEnum<BatteryIndicatorVisibility>({
id,
label: device,
storageKey,
defaultValue: "always",
values: ["always", "low", "never"],
formatValue: batteryIndicatorVisibilityLabel,
description: `When the top bar shows the ${device} battery: always, only once it drops below ${BATTERY_LOW_VISIBILITY_THRESHOLD}%, or never.`,
});
}
export const phoneBatteryVisibilitySetting = batteryVisibilitySetting(
"phoneBatteryVisibility", "Phone", "display.battery.phoneVisibility",
);
export const glassesBatteryVisibilitySetting = batteryVisibilitySetting(
"glassesBatteryVisibility", "G2", "display.battery.glassesVisibility",
);
export const ringBatteryVisibilitySetting = batteryVisibilitySetting(
"ringBatteryVisibility", "R1", "display.battery.ringVisibility",
);
/** The Wear OS watch; the indicator only exists while a watch is reachable. */
export const watchBatteryVisibilitySetting = batteryVisibilitySetting(
"watchBatteryVisibility", "Watch", "display.battery.watchVisibility",
);
/** Whether an indicator with this visibility setting shows at the given charge. */
export function batteryIndicatorVisible(visibility: BatteryIndicatorVisibility, percent: number): boolean {
if (visibility === "never") return false;
if (visibility === "always") return true;
return percent < BATTERY_LOW_VISIBILITY_THRESHOLD;
}
/**
* One string summarizing every setting the top-bar battery block reads, so
* the shell can cheaply tell whether a settings change needs a repaint.
*/
export function batteryIndicatorSettingsKey(): string {
return [
batteryDisplayModeSetting.get(),
phoneBatteryVisibilitySetting.get(),
glassesBatteryVisibilitySetting.get(),
ringBatteryVisibilitySetting.get(),
watchBatteryVisibilitySetting.get(),
].join("|");
}
export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
id: "timeFormat",
label: "Time format",
@@ -447,7 +503,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
storageKey: "developer.useMicControl",
defaultValue: true,
description:
"Use the custom firmware's per-temple mic-control channel (caps token micctl) for the Microphones app's array capture. When off, behave as if the firmware doesn't have the feature and use the standard single mixed stream.",
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
});
export const showBleBandwidthSetting = new ConfigSettingBoolean({
@@ -472,12 +528,19 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
"How R1 ring input reaches the phone. Only via glasses: the ring's own link to the glasses carries its gestures, and the phone never opens a Bluetooth connection to the ring. Direct: also connect to the ring from the phone (currently unreliable). Takes effect on the next connection to the glasses.",
});
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox";
// "whisper" (no "onboard-" prefix) is OpenAI's CLOUD realtime model
// (gpt-realtime-whisper); "onboard-whisper" is the on-device sherpa-onnx
// Whisper backend. Same underlying model family, two different places it
// runs -- see the same note in native/voice-control.ts. The "whisper" value
// keeps its name (it's a persisted setting on real installs) but its label
// below now says "OpenAI" to tell the two apart in the picker.
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
const voiceProviderLabels: Record<VoiceProvider, string> = {
onboard: "On-device",
onboard: "On-device (Moonshine)",
"onboard-whisper": "On-device (Whisper)",
elevenlabs: "ElevenLabs",
whisper: "Whisper",
whisper: "OpenAI (Whisper)",
soniox: "Soniox",
};
@@ -486,7 +549,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
label: "Transcription Provider",
storageKey: "voice.provider",
defaultValue: "onboard",
values: ["onboard", "elevenlabs", "whisper", "soniox"],
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
formatValue: (value) => voiceProviderLabels[value] ?? value,
isDisabled: (value) => {
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
@@ -494,7 +557,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
return false;
},
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).",
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
});
const wakeWordActionLabels: Record<WakeWordAction, string> = {
@@ -934,7 +997,16 @@ export function screenTimeoutLabel(value: ScreenTimeoutSetting): string {
}
export function batteryDisplayModeLabel(value: BatteryDisplayMode): string {
return value === "icon" ? "Icon" : "Percentage";
if (value === "percentage") return "Percentage";
if (value === "stacked") return "Stacked";
if (value === "stacked-percentage") return "Stacked percentage";
return "Icon";
}
export function batteryIndicatorVisibilityLabel(value: BatteryIndicatorVisibility): string {
if (value === "never") return "Never";
if (value === "low") return `Below ${BATTERY_LOW_VISIBILITY_THRESHOLD}%`;
return "Always";
}
export function timeFormatLabel(value: TimeFormat): string {
+71 -28
View File
@@ -13,12 +13,13 @@ import {
startLocalModelDownload,
} from "../../native/llama";
import {
ASR_MODEL,
ASR_MODELS,
asrModelState,
cancelAsrModelDownload,
deleteAsrModel,
onAsrModelStateChanged,
startAsrModelDownload,
type AsrModelId,
} from "../../native/asr-model";
import { TextViewerLayer } from "../../apps/files/text-viewer";
import type { LayerContext } from "../layers";
@@ -35,6 +36,10 @@ import {
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
brightnessSetting,
glassesBatteryVisibilitySetting,
phoneBatteryVisibilitySetting,
ringBatteryVisibilitySetting,
watchBatteryVisibilitySetting,
displayModeSetting,
navigateDisplayModeSetting,
navigateVerticalPositionSetting,
@@ -75,7 +80,7 @@ import {
} from "../dashboard-settings";
import { clearRecentDestinations } from "../../apps/navigate/destinations";
import { wearBridge } from "../../native/wear-bridge";
import { SettingsPanelLayer, type SettingsSection } from "./settings-panel";
import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from "./settings-panel";
import { terminalFontPickerMenuItem, uiFontPickerMenuItem } from "../font-picker";
/** The Settings app's master-detail panel (sections on the left, contents on the right). */
@@ -102,8 +107,8 @@ function settingsSections(): SettingsSection[] {
enumSettingMenuItem(verticalPositionSetting),
// Band / tall / full-panel; the dashboard controller reflows windows.
enumSettingMenuItem(displayModeSetting),
// Controls the top-bar battery indicators (icon vs percentage).
enumSettingMenuItem(batteryDisplayModeSetting),
// Submenu: top-bar battery indicator style plus per-device visibility.
batteryIndicatorsMenuItem(),
// Controls the top-bar clock (24-hour vs 12-hour).
enumSettingMenuItem(timeFormatSetting),
// Opens the modal font picker (face, weight, size) for UI text.
@@ -115,7 +120,8 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
asrModelMenuItem(),
asrModelMenuItem("moonshine"),
asrModelMenuItem("whisper-base-en"),
],
},
{
@@ -253,6 +259,39 @@ function settingsSections(): SettingsSection[] {
});
}
/**
* The Display section's "Battery indicators" row: opens a modal submenu with
* the style (icon / percentage / stacked) and, per device, when its
* indicator is visible. The row itself shows the current style.
*/
function batteryIndicatorsMenuItem(): MenuItem {
return {
label: "Battery indicators",
description:
"Top-bar battery indicators for the phone, Wear OS watch, glasses, and ring: their style, and whether each shows always, only when low, or never.",
onSelect: (ctx) => {
openSettingsSubMenu(ctx, "Battery indicators", [
enumSettingMenuItem(batteryDisplayModeSetting),
enumSettingMenuItem(phoneBatteryVisibilitySetting),
enumSettingMenuItem(watchBatteryVisibilitySetting),
enumSettingMenuItem(glassesBatteryVisibilitySetting),
enumSettingMenuItem(ringBatteryVisibilitySetting),
]);
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(
image,
getDefaultSmallFont(),
x,
y,
width,
"Battery indicators",
batteryDisplayModeSetting.displayValue(),
);
},
};
}
const LOCAL_MODEL_GB = `${(LOCAL_MODEL.sizeBytes / 1e9).toFixed(1)}GB`;
// While a download is running, re-render on progress updates so the row's
@@ -324,48 +363,52 @@ function localModelMenuItem(): MenuItem {
};
}
const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`;
function asrModelMb(id: AsrModelId): string {
return `${Math.round(ASR_MODELS[id].totalBytes / 1e6)}MB`;
}
let asrModelRenderUnsub: (() => void) | null = null;
const asrModelRenderUnsub: Partial<Record<AsrModelId, () => void>> = {};
function watchAsrModelDownload(ctx: LayerContext): void {
asrModelRenderUnsub?.();
asrModelRenderUnsub = onAsrModelStateChanged((state) => {
function watchAsrModelDownload(id: AsrModelId, ctx: LayerContext): void {
asrModelRenderUnsub[id]?.();
asrModelRenderUnsub[id] = onAsrModelStateChanged(id, (state) => {
ctx.actions.requestRender();
if (state.status !== "downloading") {
asrModelRenderUnsub?.();
asrModelRenderUnsub = null;
asrModelRenderUnsub[id]?.();
asrModelRenderUnsub[id] = undefined;
}
});
}
function asrModelStatusText(): string {
const state = asrModelState();
function asrModelStatusText(id: AsrModelId): string {
const state = asrModelState(id);
if (state.status === "ready") return "downloaded";
if (state.status === "downloading") {
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
return `${pct}% of ${ASR_MODEL_MB}`;
return `${pct}% of ${asrModelMb(id)}`;
}
return "not downloaded";
}
/** Download/cancel/delete management for the on-device transcription model. */
function asrModelMenuItem(): MenuItem {
/** Download/cancel/delete management for one on-device transcription model. */
function asrModelMenuItem(id: AsrModelId): MenuItem {
const def = ASR_MODELS[id];
const rowLabel = `On-device model: ${def.label}`;
return {
label: "On-device voice model",
label: rowLabel,
description:
`${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` +
`${def.label} (${asrModelMb(id)} download). ` +
"Transcribes voice input on the phone itself, with no API key or cloud service. " +
"Required for the On-device transcription provider; the cloud providers work without it. " +
"Required for its matching Transcription Provider option; the other providers work without it. " +
"An interrupted download resumes where it left off.",
onSelect: (ctx) => {
const state = asrModelState();
const state = asrModelState(id);
const action: MenuItem =
state.status === "downloading"
? {
label: "Cancel download",
onSelect: (innerCtx) => {
cancelAsrModelDownload();
cancelAsrModelDownload(id);
innerCtx.stack.pop();
},
}
@@ -373,22 +416,22 @@ function asrModelMenuItem(): MenuItem {
? {
label: "Delete model",
onSelect: (innerCtx) => {
deleteAsrModel();
deleteAsrModel(id);
innerCtx.stack.pop();
},
}
: {
label: `Download (${ASR_MODEL_MB})`,
label: `Download (${asrModelMb(id)})`,
onSelect: (innerCtx) => {
startAsrModelDownload();
watchAsrModelDownload(innerCtx);
startAsrModelDownload(id);
watchAsrModelDownload(id, innerCtx);
innerCtx.stack.pop();
},
};
openModalMenu(ctx, "On-device voice model", [action], 0);
openModalMenu(ctx, rowLabel, [action], 0);
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText());
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, rowLabel, asrModelStatusText(id));
},
};
}
+1 -1
View File
@@ -28,7 +28,7 @@ export type LayerActions = {
* (hands-free); otherwise it runs until stopVoiceCapture.
*/
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
/** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */
stopVoiceCapture: () => Promise<void> | void;
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
startContinuousVoiceCapture: () => Promise<void> | void;
+66
View File
@@ -0,0 +1,66 @@
import type { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { clamp } from "../../util/numeric-util";
import { brightnessSetting, brightnessSettingToLevel, type BrightnessSetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../gestures";
import type { Layer, LayerContext, PaintBelow } from "../layers";
import { appViewportRect, SHELL_OPAQUE_BLACK } from "./geometry";
const DIALOG_WIDTH = 272;
const BAR_WIDTH = 12;
const BAR_HEIGHT = 100;
/** Manual brightness changes apply immediately so the wearer can judge the level. */
export class BrightnessPickerLayer implements Layer {
constructor(private readonly onClosed: () => void) {}
onRemoved(): void {
this.onClosed();
}
paint(_ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
const image = paintBelow();
const font = getDefaultSmallFont();
const viewport = appViewportRect("min");
const height = BAR_HEIGHT + 3 * font.lineHeight + 56;
const left = viewport.x + Math.round((viewport.width - DIALOG_WIDTH) / 2);
const top = viewport.y + Math.round((viewport.height - height) / 2);
image.fillRoundedRect(left, top, DIALOG_WIDTH, height, SHELL_OPAQUE_BLACK, 8);
image.drawRoundedRect(left, top, DIALOG_WIDTH, height, 110, 8);
const centeredText = (text: string, y: number, value: number) => {
image.drawText(font, left + Math.round((DIALOG_WIDTH - font.measureText(text)) / 2), y, text, value);
};
centeredText("Brightness", top + 12, 235);
const barX = left + Math.round((DIALOG_WIDTH - BAR_WIDTH) / 2);
const barY = top + font.lineHeight + 24;
const level = brightnessSettingToLevel(brightnessSetting.get());
image.drawRect(barX - 2, barY - 2, BAR_WIDTH + 4, BAR_HEIGHT + 4, 150);
const filledHeight = Math.round(BAR_HEIGHT * (level ?? 0) / 100);
image.fillRect(barX, barY + BAR_HEIGHT - filledHeight, BAR_WIDTH, filledHeight, 255);
image.drawText(font, barX + BAR_WIDTH + 12, barY + Math.round((BAR_HEIGHT - font.lineHeight) / 2),
level === null ? "Auto" : `${level}%`, 235);
centeredText(gestureHints([[GESTURE_SCROLL, "adjust"]]), barY + BAR_HEIGHT + 12, 150);
centeredText(gestureHints([[`${GESTURE_CLICK} / ${GESTURE_DOUBLE_CLICK}`, "confirm"]]),
barY + BAR_HEIGHT + font.lineHeight + 16, 150);
return image;
}
handleInput(event: InputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
case "scroll-down": {
const level = brightnessSettingToLevel(brightnessSetting.get());
// Respect Auto if it was enabled on the phone while this dialog was open.
if (level === null) return;
const next = clamp(level + (event.type === "scroll-up" ? 10 : -10), 0, 100);
if (next !== level) brightnessSetting.set(String(next) as BrightnessSetting);
return;
}
case "click":
case "double-click":
ctx.stack.pop();
return;
}
}
}
+116 -28
View File
@@ -7,7 +7,17 @@ import { readActiveNotificationIcons } from "../../native/notification-icons";
import { readPhoneBatteryState } from "../../native/phone-battery";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
import { renderIcon, renderIconWithGlyph, type IconActivity, type IconName } from "../../graphics/icons";
import { batteryDisplayModeSetting, timeFormatSetting } from "../dashboard-settings";
import {
batteryDisplayModeSetting,
batteryIndicatorVisible,
glassesBatteryVisibilitySetting,
phoneBatteryVisibilitySetting,
ringBatteryVisibilitySetting,
watchBatteryVisibilitySetting,
type BatteryIndicatorVisibility,
} from "../dashboard-settings";
import { formatClockDate, formatClockTime } from "../clock-format";
import { getFont } from "../../graphics/bdffont";
import { Layer } from "../layers";
import { scrollToKeepSelectionVisible } from "../menu";
import { lineStep } from "../metrics";
@@ -83,8 +93,6 @@ export function sidebarContentLeft(windowCount: number): number {
}
const NOTIFICATION_ICON_SIZE = 24;
const BORDER_VALUE = 40;
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
export type ShellChromeWindow = {
windowId: string;
@@ -101,7 +109,15 @@ export type ShellChromeState = {
/** Height mode of the foreground window; decides where its top bar sits. */
foregroundHeightMode: WindowHeightMode;
foregroundAppId?: string;
battery: { headset: number | null; headsetCharging: boolean | null };
battery: {
headset: number | null;
headsetCharging: boolean | null;
ring: number | null;
ringCharging: boolean | null;
/** The Wear OS watch (app/g2/wear-remote.ts); null when no watch is reachable. */
watch: number | null;
watchCharging: boolean | null;
};
/** App-provided tray images, drawn between notification icons and batteries. */
trayIcons: GrayImage[];
};
@@ -312,8 +328,7 @@ export class ShellChromeLayer implements Layer {
image.drawLine(barLeft, barTop + TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, barTop + TOP_BAR_HEIGHT - 1, BORDER_VALUE);
const now = new Date();
const clock = `${WEEKDAYS[now.getDay()]} ${now.getDate()} ${MONTHS[now.getMonth()]} ` +
formatClockTime(now);
const clock = `${formatClockDate(now)} ${formatClockTime(now)}`;
const clockX = barLeft + 10;
const textY = barTop + Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
image.drawText(font, clockX, textY, clock, 210);
@@ -336,35 +351,51 @@ export class ShellChromeLayer implements Layer {
}
/**
* Labelled battery indicators for the phone and the G2, right-aligned in
* the top bar, following the dashboard card's icon/percentage setting.
* Returns the left edge of the battery block.
* Labelled battery indicators for the phone, Wear OS watch, G2, and R1,
* right-aligned in the top bar. The watch one exists only while a watch
* running the Faceclaw watch app is reachable (no placeholder otherwise).
* The Settings > Display > Battery indicators submenu picks the style (label beside a gauge icon or percentage, or stacked above
* either) and, per device, whether the indicator shows always, only below
* 50%, or never. Returns the left edge of the battery block.
*/
private drawTopBarBatteries(image: GrayImage, state: ShellChromeState, barTop: number): number {
const font = getDefaultSmallFont();
const percentageMode = batteryDisplayModeSetting.get() === "percentage";
type BatteryItem = { label: string; percent: number; charging: boolean };
const mode = batteryDisplayModeSetting.get();
const items: BatteryItem[] = [];
const phone = readPhoneBatteryState();
if (phone.battery !== null && Number.isFinite(phone.battery)) {
items.push({ label: "Phone", percent: phone.battery, charging: Boolean(phone.charging) });
pushBatteryItem(items, phoneBatteryVisibilitySetting.get(), "Phone", phone.battery, Boolean(phone.charging));
}
if (state.battery.watch !== null && Number.isFinite(state.battery.watch)) {
pushBatteryItem(items, watchBatteryVisibilitySetting.get(), "Watch", state.battery.watch,
Boolean(state.battery.watchCharging));
}
if (state.battery.headset !== null && Number.isFinite(state.battery.headset)) {
items.push({ label: "G2", percent: state.battery.headset, charging: Boolean(state.battery.headsetCharging) });
pushBatteryItem(items, glassesBatteryVisibilitySetting.get(), "G2", state.battery.headset,
Boolean(state.battery.headsetCharging));
}
if (state.battery.ring !== null && Number.isInteger(state.battery.ring)
&& state.battery.ring >= 0 && state.battery.ring <= 100) {
pushBatteryItem(items, ringBatteryVisibilitySetting.get(), "R1", state.battery.ring,
Boolean(state.battery.ringCharging));
}
if (!items.length) return G2_LENS_WIDTH;
if (mode === "stacked" || mode === "stacked-percentage") {
return drawStackedBatteries(image, items, barTop, mode === "stacked-percentage");
}
const font = getDefaultSmallFont();
const percentageMode = mode === "percentage";
const labelGap = 5;
const itemGap = 12;
const textY = barTop + Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
let x = G2_LENS_WIDTH - 8;
let x = G2_LENS_WIDTH - BATTERY_BLOCK_RIGHT_MARGIN;
for (let index = items.length - 1; index >= 0; index--) {
const item = items[index]!;
const percentText = `${Math.max(0, Math.min(100, Math.round(item.percent)))}%`;
const percentText = `${item.percent}%`;
const valueWidth = percentageMode ? font.measureText(percentText) : BATTERY_ICON_WIDTH;
const labelWidth = font.measureText(item.label);
x -= labelWidth + labelGap + valueWidth;
image.drawText(font, x, textY, item.label, 150);
image.drawText(font, x, textY, item.label, BATTERY_LABEL_VALUE);
const valueX = x + labelWidth + labelGap;
if (percentageMode) {
if (item.charging) {
@@ -386,6 +417,74 @@ export class ShellChromeLayer implements Layer {
}
}
type BatteryItem = { label: string; percent: number; charging: boolean };
const BATTERY_BLOCK_RIGHT_MARGIN = 8;
const BATTERY_LABEL_VALUE = 150;
/** Append an indicator if its visibility setting shows it at this charge. */
function pushBatteryItem(
items: BatteryItem[],
visibility: BatteryIndicatorVisibility,
label: string,
percent: number,
charging: boolean,
): void {
const clamped = Math.max(0, Math.min(100, Math.round(percent)));
if (!batteryIndicatorVisible(visibility, clamped)) return;
items.push({ label, percent: clamped, charging });
}
// Stacked style geometry, in rows below the bar's top edge. The bar has 27
// usable rows above its bottom border line, too few for the user-sized UI
// font (line height up to 21px) over a 10px gauge, so both stacked lines use
// a fixed 12px bitmap face instead: TerminusV, proportional, with 8px
// capitals and digits on a 10px ascent. Ink then spans rows 3..22: label
// capitals at 3..10, a 2px gap, and the gauge at 13..22 (or the percentage
// digits at 13..20, whose charging highlight box covers 12..21), leaving 3
// rows above and 4 below (before the border) so the pair reads as one
// centered unit.
const STACKED_LABEL_BASELINE = 11;
const STACKED_ICON_TOP = 13;
const STACKED_PERCENT_BASELINE = 21;
const STACKED_ITEM_GAP = 10;
/**
* Stacked styles: each label centered above its gauge icon (or percentage
* text), right-aligned in the bar. Returns the left edge of the battery
* block.
*/
function drawStackedBatteries(image: GrayImage, items: BatteryItem[], barTop: number, percentage: boolean): number {
const font = getFont("terminusv12");
const labelTop = barTop + STACKED_LABEL_BASELINE - font.ascent;
const percentTop = barTop + STACKED_PERCENT_BASELINE - font.ascent;
let x = G2_LENS_WIDTH - BATTERY_BLOCK_RIGHT_MARGIN;
for (let index = items.length - 1; index >= 0; index--) {
const item = items[index]!;
const percentText = `${item.percent}%`;
const labelWidth = font.measureText(item.label);
const valueWidth = percentage ? font.measureText(percentText) : BATTERY_ICON_WIDTH;
const itemWidth = Math.max(labelWidth, valueWidth);
x -= itemWidth;
image.drawText(font, x + (((itemWidth - labelWidth) / 2) | 0), labelTop, item.label, BATTERY_LABEL_VALUE);
const valueX = x + (((itemWidth - valueWidth) / 2) | 0);
if (!percentage) {
const icon = drawBattery(item.percent, item.charging);
image.bitBlt(icon, valueX, barTop + STACKED_ICON_TOP, { transparentZero: true });
} else if (item.charging) {
// Inverted text marks charging, as in the side-by-side percentage style;
// the box hugs the digits' 8px ink rather than the full 12px line.
const inkTop = barTop + STACKED_PERCENT_BASELINE - 8;
image.fillRect(valueX - 2, inkTop - 1, valueWidth + 4, 10, 255);
image.drawText(font, valueX, percentTop, percentText, 1);
} else {
image.drawText(font, valueX, percentTop, percentText, 200);
}
x -= STACKED_ITEM_GAP;
}
return x + STACKED_ITEM_GAP;
}
// Ambient cards (encounter popups): compact and deliberately unobtrusive,
// anchored to the bottom-right of the window band and stacking upward, clear
// of the caption area's left-aligned text. Never interactive.
@@ -519,17 +618,6 @@ function tabLeftInset(yy: number, top: number, bottom: number, radius: number):
return Math.max(0, Math.round(radius - Math.sqrt(Math.max(0, radius * radius - dy * dy))));
}
/** Format the top-bar clock time, honoring the 12/24-hour setting. */
function formatClockTime(now: Date): string {
const minutes = String(now.getMinutes()).padStart(2, "0");
const hour24 = now.getHours();
if (timeFormatSetting.get() === "12h") {
const hour12 = ((hour24 + 11) % 12) + 1;
return `${hour12}:${minutes} ${hour24 < 12 ? "AM" : "PM"}`;
}
return `${hour24}:${minutes}`;
}
/** Small triangle marker for sidebar overflow; direction -1 = up, 1 = down. */
function drawChevron(image: GrayImage, centerX: number, y: number, direction: -1 | 1): void {
const half = 5;
+14 -2
View File
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
* has no menu of its own, and tap-then-hold opens the system menu instead.
*/
menuItems?: () => MenuItem[];
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
holdToTalk?: boolean;
isVoiceCapturing?: () => boolean;
onSystemMenuOpened?: () => void;
onAppMenuOpened?: () => void;
setScreenOn?: ShellWindow["setScreenOn"];
/** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions;
/**
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
shell.openSystemMenu(options.windowId);
return;
}
stack.push(new WindowMenuLayer(options.title, items));
options.onAppMenuOpened?.();
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
};
const window: ShellWindow = {
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// The window's own LayerStack decides per layer whether a swipe is
// directional or falls back to click / double-click.
acceptsDirectional: true,
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
setScreenOn: options.setScreenOn,
heightMode,
hasAppMenu: () => appMenuItems().length > 0,
close: () => {
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// The shell opened its system menu over this window; close our own
// context menu so the two never stack. Never forwarded to app layers.
if (event.type === "system-menu-opened") {
options.onSystemMenuOpened?.();
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
requestRender();
}
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// submenus and app content alike. A plain long-press never arrives:
// the shell keeps it for the system menu (no in-process window claims
// it).
if (event.type === "short-then-long-press") {
if ((!options.holdToTalk && event.type === "short-then-long-press") ||
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
openWindowMenu();
await render(frameId);
return;
+162 -49
View File
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
import { resolveAssistantModel } from "../../assistant/models";
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
@@ -32,7 +34,8 @@ import {
assistantBridgeTokenSetting,
assistantModelSetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
batteryIndicatorSettingsKey,
brightnessSetting,
onAnySettingChanged,
openAiApiKeySetting,
timeFormatSetting,
@@ -42,6 +45,7 @@ import { onAmbientCardsChanged } from "./ambient-cards";
import { ShellChromeLayer, sidebarContentLeft, type ShellChromeState, type ShellChromeWindow } from "./chrome-layer";
import { ShellModalLayer } from "./modal-layer";
import { ToolDebugMenuLayer } from "./tool-debug-layer";
import { BrightnessPickerLayer } from "./brightness-picker-layer";
import { toolRegistry } from "../../assistant/tool-registry";
import {
minWindowTop,
@@ -61,7 +65,7 @@ import {
* Input flow: every event enters via receiveInput. The shell consumes
* everything while the sidebar or a shell overlay has focus and forwards the
* rest to the focused window. Long-press opens the shell-owned system menu
* (Focus app switcher, Voice input, Close window, Debug) without reaching the
* (Focus app switcher, Voice input, Brightness, Close window, Debug) without reaching the
* app, so the shell keeps working when a window's handler hangs; a window
* that claims long-press for a move of its own gets it forwarded instead, and
* holding the press past the escape threshold still opens the system menu.
@@ -91,6 +95,10 @@ export type ShellWindow = {
* system menu; holding the press past the escape threshold still opens it.
*/
claimsLongPress?: () => boolean;
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
@@ -141,6 +149,13 @@ export type ShellWindow = {
onFocus?: (lastInput: InputEvent | null) => void;
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
setScreenOn?: (on: boolean) => void;
/**
* Input focus arrived at or left this window: it is (no longer) where
* ordinary input goes. Unlike setForeground / onFocus, this also tracks
* shell overlays (the system menu, a notification modal, the voice dialog)
* and screen-off, so a game can pause on any of them. Sent on change only.
*/
setInputFocus?: (focused: boolean) => void;
};
export type ShellConfig = {
@@ -283,24 +298,55 @@ class ShellAlertLayer implements Layer {
}
}
/** Every setting the top bar paints from, as one comparable string. */
function topBarSettingsKey(): string {
return `${batteryIndicatorSettingsKey()}|${timeFormatSetting.get()}`;
}
class Shell {
private windows: ShellWindow[] = [];
private selectedIndex = 0;
/** Window ids in most-recently-visible-first order; closing the visible window returns to the next entry. */
private mruWindowIds: string[] = [];
private focus: FocusKind = "sidebar";
/** The window last told it holds input focus (see syncInputFocus). */
private inputFocusedWindowId: string | null = null;
private screenOn = true;
private lastInputAtMs = Date.now();
/** The most recent input event received, for windows gaining focus (see ShellWindow.onFocus). */
private lastInput: InputEvent | null = null;
private battery: ShellChromeState["battery"] = { headset: null, headsetCharging: null };
private battery: ShellChromeState["battery"] = {
headset: null, headsetCharging: null, ring: null, ringCharging: null, watch: null, watchCharging: null,
};
private attention = new Map<string, boolean>();
// App-provided top-bar tray icons, keyed by owner id; drawn between the
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private conversations: AssistantConversations | null = null;
private get assistantSession(): AssistantSession | null {
return this.conversations?.current().session ?? null;
}
getAssistantConversations(): AssistantConversations {
if (!this.conversations) {
this.conversations = new AssistantConversations(
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
() => assistantModelSetting.get(),
(value) => setStringSetting("assistant.conversations", value),
getStringSetting("assistant.conversations", ""),
);
}
return this.conversations;
}
async prepareChatVoiceCapture(): Promise<boolean> {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
}
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>();
@@ -315,14 +361,21 @@ class Shell {
private readonly chrome = new ShellChromeLayer(() => this.chromeState());
private readonly stack = new LayerStack(this.chrome, this.actions);
// Top-bar settings we mirror into the chrome; a change to either repaints
// the shell surface so the top bar reflects it immediately.
// Top-bar settings we mirror into the chrome; a change to any of them
// repaints the shell surface so the top bar reflects it immediately.
private topBarSettingsSubscribed = false;
private lastBatteryDisplayMode: string | null = null;
private lastTimeFormat: string | null = null;
private lastTopBarSettingsKey: string | null = null;
configure(config: ShellConfig): void {
this.config = config;
// Nearly every state change ends in a shell render request, so it is
// the natural point to tell windows about input-focus changes too.
this.config = {
...config,
requestShellRender: () => {
this.syncInputFocus();
config.requestShellRender();
},
};
this.stack.setActions(config.actions);
this.subscribeToTopBarSettings();
this.subscribeToAmbientCards();
@@ -345,16 +398,11 @@ class Shell {
private subscribeToTopBarSettings(): void {
if (this.topBarSettingsSubscribed) return;
this.topBarSettingsSubscribed = true;
this.lastBatteryDisplayMode = batteryDisplayModeSetting.get();
this.lastTimeFormat = timeFormatSetting.get();
this.lastTopBarSettingsKey = topBarSettingsKey();
onAnySettingChanged(() => {
const batteryMode = batteryDisplayModeSetting.get();
const timeFormat = timeFormatSetting.get();
if (batteryMode === this.lastBatteryDisplayMode && timeFormat === this.lastTimeFormat) {
return;
}
this.lastBatteryDisplayMode = batteryMode;
this.lastTimeFormat = timeFormat;
const key = topBarSettingsKey();
if (key === this.lastTopBarSettingsKey) return;
this.lastTopBarSettingsKey = key;
this.config.requestShellRender();
});
}
@@ -468,7 +516,7 @@ class Shell {
return this.screenOn;
}
/** Current headset battery levels (for the assistant's get_state tool). */
/** Current G2, R1, and Wear OS watch battery levels (top bar, Glanceboard, assistant). */
getBatteryLevels(): ShellChromeState["battery"] {
return this.battery;
}
@@ -508,6 +556,34 @@ class Shell {
return !!window && this.focus === "window" && this.foregroundWindow() === window;
}
/**
* The window ordinary input reaches right now: the foreground window with
* focus in-window, the screen on, and no shell overlay (system menu,
* notification modal, voice dialog, alert) above it.
*/
private inputTargetWindow(): ShellWindow | undefined {
if (!this.screenOn || this.focus !== "window" || !this.stack.isAtBase() || this.activeVoiceLayer) {
return undefined;
}
return this.foregroundWindow();
}
/**
* Tell windows when they gain or lose input focus (ShellWindow.setInputFocus).
* Called from every shell render request and the input/wake/sleep paths,
* which between them follow every focus-affecting state change; it diffs
* against the last notification, so calling it often is cheap.
*/
private syncInputFocus(): void {
const target = this.inputTargetWindow();
const targetId = target?.windowId ?? null;
if (targetId === this.inputFocusedWindowId) return;
const previous = this.windows.find((w) => w.windowId === this.inputFocusedWindowId);
this.inputFocusedWindowId = targetId;
previous?.setInputFocus?.(false);
target?.setInputFocus?.(true);
}
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
wake(focus: FocusKind, nowMs = Date.now()): boolean {
this.lastInputAtMs = nowMs;
@@ -515,7 +591,10 @@ class Shell {
const alreadyFocused = this.isFocusTarget(gaining);
this.focus = focus;
if (gaining && !alreadyFocused) gaining.onFocus?.(this.lastInput);
if (this.screenOn) return false;
if (this.screenOn) {
this.syncInputFocus();
return false;
}
this.screenOn = true;
this.config.onScreenStateChanged(true);
for (const window of this.windows) {
@@ -524,6 +603,7 @@ class Shell {
// Refresh the foreground window; the compositor restored its retained
// frame, but its content may be stale (e.g. a running stopwatch).
this.foregroundWindow()?.requestRender();
this.syncInputFocus();
return true;
}
@@ -536,6 +616,7 @@ class Shell {
for (const window of this.windows) {
window.setScreenOn?.(false);
}
this.syncInputFocus();
this.config.onScreenStateChanged(false);
}
@@ -548,6 +629,7 @@ class Shell {
this.setSelectedIndex(index);
this.focus = "window";
if (!alreadyFocused) target?.onFocus?.(this.lastInput);
this.syncInputFocus();
}
/** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */
@@ -561,7 +643,7 @@ class Shell {
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -626,6 +708,16 @@ class Shell {
}
async receiveInput(event: InputEvent, frameId = 0): Promise<ShellInputOutcome> {
try {
return await this.routeInput(event, frameId);
} finally {
// Whatever the input did (opened an overlay, moved focus, slept the
// screen), windows learn about the resulting input-focus change.
this.syncInputFocus();
}
}
private async routeInput(event: InputEvent, frameId: number): Promise<ShellInputOutcome> {
const previous = this.lastInput;
this.lastInput = event;
// A visible window may paint a source-dependent indicator (see
@@ -647,6 +739,7 @@ class Shell {
// Even AI app never launches, so the firmware does not power the display
// for us either -- actions that need it wake the screen themselves.
if (event.type === "wakeword") {
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
const action = wakeWordActionSetting.get();
if (action === "off") {
return { shell: false, window: false };
@@ -698,11 +791,11 @@ class Shell {
if (!window) {
return { shell: true, window: false };
}
if (!window.claimsLongPress?.()) {
if (!window.holdToTalk && !window.claimsLongPress?.()) {
this.openEscapeMenu();
return { shell: true, window: false };
}
this.startEscapeMenuTimer();
if (!window.holdToTalk) this.startEscapeMenuTimer();
if (this.focus === "sidebar") {
this.focus = "window";
window.onFocus?.(this.lastInput);
@@ -718,6 +811,10 @@ class Shell {
// when it has none.
if (event.type === "short-then-long-press") {
const window = this.foregroundWindow();
if (window?.holdToTalk) {
this.openEscapeMenu();
return { shell: true, window: false };
}
// Over the open system menu it switches to the app's context menu:
// close the system menu and deliver the gesture to the window as
// usual. A window without a menu of its own would only ask for the
@@ -1027,7 +1124,7 @@ class Shell {
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer) return null;
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
if (!this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
@@ -1072,7 +1169,8 @@ class Shell {
}
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
const current = this.getAssistantConversations().current();
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
}
/** Observe assistant turns (see AssistantActivityEvent). */
@@ -1110,20 +1208,13 @@ class Shell {
}
private ensureAssistantSession(): AssistantSession | null {
const config = this.resolveAssistantConfiguration();
if (!config) return null;
if (
!this.assistantSession ||
this.assistantSession.isExpired() ||
!this.assistantSession.matchesConfiguration(config)
) {
this.assistantSession?.cancel();
this.assistantSession = new AssistantSession(config);
}
return this.assistantSession;
return this.getAssistantConversations().ensureSession();
}
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
private resolveAssistantConfiguration(
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
): AssistantBackendConfig | null {
if (assistantBackendSetting.get() === "external") {
const host = assistantBridgeHostSetting.get().trim();
const token = assistantBridgeTokenSetting.get();
@@ -1131,10 +1222,11 @@ class Shell {
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
return { kind: "external", bridge: { host, port, token } };
}
const llm = resolveAssistantModel(assistantModelSetting.get(), {
const llm = resolveAssistantModel(model, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null;
}
@@ -1154,7 +1246,9 @@ class Shell {
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay.
*/
sendToAssistant(text: string): void {
sendToAssistant(text: string, showOverlay = true): void {
text = text.trim();
if (!text) return;
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert(
@@ -1164,7 +1258,15 @@ class Shell {
);
return;
}
if (session.isTurnActive()) {
this.showAlert("The assistant is still working on the previous request");
return;
}
if (!this.screenOn) this.wake("sidebar");
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
this.runAssistantTurn(session, null, text);
return;
}
let layer = this.assistantLayer;
if (!layer) {
const created = new AssistantLayer(this.config.actions, {
@@ -1173,7 +1275,7 @@ class Shell {
onClose: () => this.closeAssistantLayer(),
onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the reference so a later query starts clean.
// stop the turn and drop the overlay reference; keep shared history.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" });
@@ -1187,23 +1289,24 @@ class Shell {
this.config.requestShellRender();
}
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
if (session.isTurnActive()) return;
layer?.startTurn();
let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar;
layer.onTextDelta(delta, textSoFar);
layer?.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
},
onToolActivity: (label) => layer.onToolActivity(label),
onToolActivity: (label) => layer?.onToolActivity(label),
onTurnDone: () => {
layer.onTurnDone();
layer?.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar });
},
onError: (message) => {
layer.onError(message);
layer?.onError(message);
this.emitAssistantActivity({ phase: "error", text: message });
},
});
@@ -1301,7 +1404,7 @@ class Shell {
/**
* The system/escape menu: the entries every window shares (Focus app
* switcher, Voice input, Close window) plus Debug. Shell-owned and
* switcher, Voice input, Brightness, Close window) plus Debug. Shell-owned and
* shell-drawn (never the app's), so an unresponsive app can always be
* closed. It opens for long-press (over the app's own menu too), after an
* extended hold in a window that claims long-press, and on a window's
@@ -1348,6 +1451,16 @@ class Shell {
},
}]),
);
if (brightnessSetting.get() !== "auto") {
items.push({
label: "Brightness",
onSelect: (ctx) => {
ctx.stack.pop();
ctx.stack.push(new BrightnessPickerLayer(() => this.yieldFocusToSidebar()));
this.config.requestShellRender();
},
});
}
items.push({
label: "Debug",
onSelect: (ctx) => {
@@ -1359,7 +1472,7 @@ class Shell {
// when the app has one (otherwise it opens this very menu, not worth a
// hint).
const footer = foreground.hasAppMenu?.()
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
: undefined;
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
layer.selectItem(initialSelection);
+70 -20
View File
@@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { paintInputDialog } from "./input-dialog";
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
// After native recognition finishes, a cloud final can still trail in.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
/**
@@ -64,7 +63,11 @@ export type VoiceInputLayerOptions = {
*/
export class VoiceInputLayer implements Layer {
private phase: VoicePhase = "capturing";
private status = "Listening...";
private status = "Starting microphone...";
/** Mirrors the bridge: only true while audio is actually being transcribed. */
private listening = false;
/** Bridge guidance shown in place of the empty transcript (e.g. "check your phone"). */
private detail = "";
// The active utterance. displayText() is what the dialog shows and what
// Send delivers; the refine flow also writes the merged result here.
private finalizedText = "";
@@ -76,6 +79,10 @@ export class VoiceInputLayer implements Layer {
private capturing = false;
/** Continuation stopped; waiting for the trailing final transcript. */
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
private pendingFollowup = false;
private stoppingCapture = false;
private stopGeneration = 0;
private acceptingTranscript = true;
private refineHandle: AnthropicStreamHandle | null = null;
private menuIndex = 0;
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
@@ -114,6 +121,8 @@ export class VoiceInputLayer implements Layer {
// The refine stage owns the status line ("Refining...", error text).
if (this.phase === "refining") return;
this.status = state.status;
this.listening = state.listening;
this.detail = state.detail;
this.actions.requestRender();
});
if (this.handsFree || global.isIOS) {
@@ -149,19 +158,48 @@ export class VoiceInputLayer implements Layer {
}
this.capturing = false;
this.phase = "menu";
void this.actions.stopVoiceCapture();
if (this.autoSend && this.sendTargets.length) {
// Skip-confirmation (wakeword): send to the default target as soon as the
// transcript finalizes, or after a short wait for the trailing final.
this.pendingAutoSend = true;
this.status = "Sending...";
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
} else if (this.status.startsWith("Listening")) {
} else if (this.status.endsWith("...")) {
// A progress status ("Starting microphone...", "Listening...") gives
// way to the menu prompt; an error (ending in ".") stays visible.
this.status = "Send, continue, or discard?";
}
this.stopCapture(() => {
if (this.pendingAutoSend) {
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
});
this.actions.requestRender();
}
/** Native stop can outlast its synchronous join while Whisper is decoding. */
private stopCapture(onStopped: () => void): void {
const generation = ++this.stopGeneration;
this.stoppingCapture = true;
const afterStop = () => {
if (generation !== this.stopGeneration) return;
this.stoppingCapture = false;
onStopped();
this.actions.requestRender();
};
const stopped = this.actions.stopVoiceCapture();
if (!stopped) {
afterStop();
return;
}
void stopped.then(afterStop, (error) => {
if (generation !== this.stopGeneration) return;
this.stoppingCapture = false;
this.pendingAutoSend = false;
this.acceptingTranscript = false;
this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error));
});
}
/** Fire the queued skip-confirmation send (or fall back to the menu). */
private performAutoSend(): void {
if (this.autoSendTimer !== null) {
@@ -185,7 +223,7 @@ export class VoiceInputLayer implements Layer {
/** The menu rows: one per send target, then Continue, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasText = text.length > 0 && !this.stoppingCapture;
const hasLlmKey = !global.isIOS && anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
@@ -193,6 +231,7 @@ export class VoiceInputLayer implements Layer {
label: target.label,
dim: !hasText,
onSelect: () => {
if (this.stoppingCapture) return;
this.dismiss();
if (hasText) target.onSend(text);
},
@@ -200,9 +239,9 @@ export class VoiceInputLayer implements Layer {
}
rows.push({
label: global.isIOS ? "Continue (Not available on iOS)" : hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
dim: !hasLlmKey || this.stoppingCapture,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();
},
});
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
@@ -213,8 +252,8 @@ export class VoiceInputLayer implements Layer {
const image = paintBelow();
const inMenu = this.phase === "menu";
paintInputDialog(image, {
title: this.capturing ? "Voice ●" : "Voice",
status: this.status,
title: this.capturing && this.listening ? "Voice ●" : "Voice",
status: this.stoppingCapture ? "Finishing transcription..." : this.status,
text: this.displayText() || this.placeholderText(),
rows: inMenu ? this.menuRows() : [],
selectedRow: this.menuIndex,
@@ -276,6 +315,7 @@ export class VoiceInputLayer implements Layer {
/** Continue selected: keep the message aside and record a follow-up. */
private startContinuation(): void {
this.acceptingTranscript = true;
this.baseText = this.displayText().trim();
this.finalizedText = "";
this.liveText = "";
@@ -290,15 +330,18 @@ export class VoiceInputLayer implements Layer {
private endContinuationCapture(): void {
if (!this.capturing) return;
this.capturing = false;
void this.actions.stopVoiceCapture();
this.status = "Refining...";
this.pendingFollowup = true;
this.status = "Finishing transcription...";
this.stopCapture(() => {
if (this.pendingFollowup) {
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
});
this.actions.requestRender();
// The provider's committed transcript arrives shortly after stop; refine
// when it does, or after a timeout with whatever partials we have.
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
}
private beginRefine(): void {
this.pendingFollowup = false;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -335,9 +378,10 @@ export class VoiceInputLayer implements Layer {
/** Abort a continuation (mic or LLM stage) and restore the prior message. */
private cancelContinuation(status: string): void {
this.acceptingTranscript = false;
if (this.capturing) {
this.capturing = false;
void this.actions.stopVoiceCapture();
this.stopCapture(() => {});
}
this.refineHandle?.cancel();
this.refineHandle = null;
@@ -345,6 +389,7 @@ export class VoiceInputLayer implements Layer {
}
private backToMenu(text: string, status: string): void {
this.pendingFollowup = false;
if (this.followupFinalizeTimer !== null) {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
@@ -358,6 +403,9 @@ export class VoiceInputLayer implements Layer {
}
onRemoved(): void {
++this.stopGeneration;
this.acceptingTranscript = false;
this.pendingFollowup = false;
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.unsubscribeStatus?.();
@@ -392,7 +440,9 @@ export class VoiceInputLayer implements Layer {
private placeholderText(): string {
switch (this.phase) {
case "capturing":
return "Listening...";
// Only claim to listen when the mic is actually running; otherwise
// the bridge's guidance (permission prompt on the phone, etc.), if any.
return this.listening ? "Listening..." : this.detail;
case "continuing":
return "Say more, or describe an edit...";
case "refining":
@@ -419,14 +469,14 @@ export class VoiceInputLayer implements Layer {
private onTranscript(event: VoiceTranscriptEvent): void {
// The refine stream owns the text buffers once it starts; a transcript
// that trails in after that point is stale.
if (this.phase === "refining") return;
if (!this.acceptingTranscript || this.phase === "refining") return;
if (event.isFinal) {
const finalText = event.text.trim() || this.liveText.trim();
if (finalText) {
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
}
this.liveText = "";
if (this.phase === "continuing" && this.followupFinalizeTimer !== null) {
if (this.phase === "continuing" && this.pendingFollowup) {
// The follow-up finalized; no need to keep waiting.
this.beginRefine();
return;
+38
View File
@@ -0,0 +1,38 @@
/**
* Main-thread mirror of small state that worker apps publish (the
* "publish-state" worker reply): the latest value per key, with change
* notifications, so main-thread consumers that are not the app's own windows
* (a Glanceboard widget, say) can show what a worker knows without a second
* data connection. Values cross postMessage, so they are plain JSON.
*/
const values = new Map<string, unknown>();
const listeners = new Map<string, Set<(value: unknown) => void>>();
export function publishWorkerState(key: string, value: unknown): void {
values.set(key, value);
for (const listener of Array.from(listeners.get(key) ?? [])) {
try {
listener(value);
} catch (error) {
console.warn(`worker-state listener for ${key} failed`, error);
}
}
}
/** The latest published value for `key`, or undefined when nothing was published yet. */
export function readWorkerState<T>(key: string): T | undefined {
return values.get(key) as T | undefined;
}
/** Subscribe to later publications of `key`; returns the unsubscribe. */
export function onWorkerStateChanged<T>(key: string, listener: (value: T) => void): () => void {
let set = listeners.get(key);
if (!set) {
set = new Set();
listeners.set(key, set);
}
set.add(listener as (value: unknown) => void);
return () => {
set!.delete(listener as (value: unknown) => void);
};
}
+22
View File
@@ -5,6 +5,7 @@ import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/to
import { appViewportSize, type WindowHeightMode } from "./geometry";
import * as frameTimings from "../../native/frame-timings";
import { shell, type ShellWindow } from "./shell";
import { publishWorkerState } from "./worker-state";
/**
* Messages between the shell (main thread) and an app worker. One worker
@@ -20,6 +21,12 @@ export type WorkerAppMessage =
| { type: "text-input"; windowId: string; text: string }
| { type: "render"; windowId: string; focused: boolean }
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
/**
* Input focus arrived at or left the window, counting shell overlays and
* screen-off (see ShellWindow.setInputFocus). Sent on change only; the
* per-message `focused` flags above stay the source of truth for painting.
*/
| { type: "input-focus"; windowId: string; focused: boolean }
| { type: "screen"; on: boolean }
/** Assistant tool invocation aimed at a window; reply with tool-result. */
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
@@ -135,6 +142,15 @@ export type WorkerAppReply =
type: "tool-result";
callId: string;
result: ToolResult;
}
| {
/**
* Publish a small piece of app state for main-thread consumers outside
* the app's windows (see app/ui/shell/worker-state.ts). JSON only.
*/
type: "publish-state";
key: string;
state: unknown;
};
export type WorkerWindowSpec = {
@@ -318,6 +334,9 @@ export class WorkerAppHost {
case "set-title":
// Titles are informational for now (sidebar shows icons only).
break;
case "publish-state":
publishWorkerState(message.key, message.state);
break;
case "set-tools":
// Only a window we actually have open may contribute tools.
if (this.openWindows.has(message.windowId)) {
@@ -420,6 +439,9 @@ export class WorkerAppHost {
// uniform; the worker treats it globally.
this.post({ type: "screen", on });
},
setInputFocus: (focused) => {
this.post({ type: "input-focus", windowId: spec.windowId, focused });
},
};
shell.registerWindow(window);
// Configure the surface before any foregrounding so the worker's first
+26
View File
@@ -0,0 +1,26 @@
/**
* The "Sound: on/off" entry in a game's context menu, persisted per app under
* `<appId>.soundOn` so each game remembers its own choice across launches.
* Reads default to on; storage failures fall back to on and never throw.
*/
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
function soundKey(appId: string): string {
return `${appId}.soundOn`;
}
export function loadSoundEnabled(appId: string): boolean {
try {
return getBooleanSetting(soundKey(appId), true);
} catch {
return true;
}
}
export function saveSoundEnabled(appId: string, enabled: boolean): void {
try {
setBooleanSetting(soundKey(appId), enabled);
} catch (error) {
console.warn(`${appId} sound setting save failed: ${error}`);
}
}
+5 -3
View File
@@ -1,6 +1,6 @@
import { GrayImage } from "../graphics/image";
import { singlePlane, type Plane } from "../graphics/plane";
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
import { type Layer, LayerStack, noopLayerActions } from "./layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
import type { WorkerAppReply } from "./shell/worker-window";
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
};
export class WindowMenuLayer extends MenuLayer {
constructor(title: string | null, items: MenuItem[]) {
super(title, items, WINDOW_MENU_LAYOUT);
constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
super(title, items, holdToTalk
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
: WINDOW_MENU_LAYOUT);
}
}