forked from Blendman974/faceclaw
Merge main into ios-port and update the iOS CFW transport
This commit is contained in:
commit
e8e144f3f2
156 files changed
+10528
-1679
No files matched your search
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Clock/date text shared by the shell's top bar and the Glanceboard's System
|
||||
* Card, so the two never disagree on the time-format setting or the date
|
||||
* spelling.
|
||||
*/
|
||||
import { timeFormatSetting } from "./dashboard-settings";
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
/** Format a clock time, honoring the 12/24-hour setting ("14:05" / "2:05 PM"). */
|
||||
export function formatClockTime(now: Date): string {
|
||||
const minutes = String(now.getMinutes()).padStart(2, "0");
|
||||
const hour24 = now.getHours();
|
||||
if (timeFormatSetting.get() === "12h") {
|
||||
const hour12 = ((hour24 + 11) % 12) + 1;
|
||||
return `${hour12}:${minutes} ${hour24 < 12 ? "AM" : "PM"}`;
|
||||
}
|
||||
return `${hour24}:${minutes}`;
|
||||
}
|
||||
|
||||
/** Short date as the top bar shows it: "Thu 11 Sep". */
|
||||
export function formatClockDate(now: Date): string {
|
||||
return `${WEEKDAYS[now.getDay()]} ${now.getDate()} ${MONTHS[now.getMonth()]}`;
|
||||
}
|
||||
@@ -25,7 +25,9 @@ export type NightscoutSettings = {
|
||||
siteUrl: string;
|
||||
apiToken: string;
|
||||
};
|
||||
export type BatteryDisplayMode = "icon" | "percentage";
|
||||
export type BatteryDisplayMode = "icon" | "percentage" | "stacked" | "stacked-percentage";
|
||||
/** When a top-bar battery indicator is shown: always, only below 50%, or never. */
|
||||
export type BatteryIndicatorVisibility = "always" | "low" | "never";
|
||||
export type TimeFormat = "24h" | "12h";
|
||||
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "3m" | "never";
|
||||
// "auto" lets the glasses' ambient-light sensor drive brightness; the numeric
|
||||
@@ -211,14 +213,68 @@ export class ConfigSettingString<TId extends string = string> extends ConfigSett
|
||||
|
||||
export const batteryDisplayModeSetting = new ConfigSettingEnum<BatteryDisplayMode>({
|
||||
id: "batteryDisplayMode",
|
||||
label: "Battery display",
|
||||
label: "Style",
|
||||
storageKey: "dashboard.systemCard.batteryDisplayMode",
|
||||
defaultValue: "icon",
|
||||
values: ["icon", "percentage"],
|
||||
defaultValue: "stacked",
|
||||
values: ["icon", "percentage", "stacked", "stacked-percentage"],
|
||||
formatValue: batteryDisplayModeLabel,
|
||||
description: "How the top bar shows the phone and glasses battery levels: a small gauge icon or an exact percentage.",
|
||||
description: "How the top bar shows battery levels: a gauge icon or exact percentage beside the label, or a compact gauge or percentage with the label stacked above it.",
|
||||
});
|
||||
|
||||
/** Below this charge level a "Below 50%" indicator becomes visible. */
|
||||
export const BATTERY_LOW_VISIBILITY_THRESHOLD = 50;
|
||||
|
||||
function batteryVisibilitySetting(
|
||||
id: string,
|
||||
device: string,
|
||||
storageKey: string,
|
||||
): ConfigSettingEnum<BatteryIndicatorVisibility> {
|
||||
return new ConfigSettingEnum<BatteryIndicatorVisibility>({
|
||||
id,
|
||||
label: device,
|
||||
storageKey,
|
||||
defaultValue: "always",
|
||||
values: ["always", "low", "never"],
|
||||
formatValue: batteryIndicatorVisibilityLabel,
|
||||
description: `When the top bar shows the ${device} battery: always, only once it drops below ${BATTERY_LOW_VISIBILITY_THRESHOLD}%, or never.`,
|
||||
});
|
||||
}
|
||||
|
||||
export const phoneBatteryVisibilitySetting = batteryVisibilitySetting(
|
||||
"phoneBatteryVisibility", "Phone", "display.battery.phoneVisibility",
|
||||
);
|
||||
export const glassesBatteryVisibilitySetting = batteryVisibilitySetting(
|
||||
"glassesBatteryVisibility", "G2", "display.battery.glassesVisibility",
|
||||
);
|
||||
export const ringBatteryVisibilitySetting = batteryVisibilitySetting(
|
||||
"ringBatteryVisibility", "R1", "display.battery.ringVisibility",
|
||||
);
|
||||
/** The Wear OS watch; the indicator only exists while a watch is reachable. */
|
||||
export const watchBatteryVisibilitySetting = batteryVisibilitySetting(
|
||||
"watchBatteryVisibility", "Watch", "display.battery.watchVisibility",
|
||||
);
|
||||
|
||||
/** Whether an indicator with this visibility setting shows at the given charge. */
|
||||
export function batteryIndicatorVisible(visibility: BatteryIndicatorVisibility, percent: number): boolean {
|
||||
if (visibility === "never") return false;
|
||||
if (visibility === "always") return true;
|
||||
return percent < BATTERY_LOW_VISIBILITY_THRESHOLD;
|
||||
}
|
||||
|
||||
/**
|
||||
* One string summarizing every setting the top-bar battery block reads, so
|
||||
* the shell can cheaply tell whether a settings change needs a repaint.
|
||||
*/
|
||||
export function batteryIndicatorSettingsKey(): string {
|
||||
return [
|
||||
batteryDisplayModeSetting.get(),
|
||||
phoneBatteryVisibilitySetting.get(),
|
||||
glassesBatteryVisibilitySetting.get(),
|
||||
ringBatteryVisibilitySetting.get(),
|
||||
watchBatteryVisibilitySetting.get(),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
|
||||
id: "timeFormat",
|
||||
label: "Time format",
|
||||
@@ -447,7 +503,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
|
||||
storageKey: "developer.useMicControl",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Use the custom firmware's per-temple mic-control channel (caps token micctl) for the Microphones app's array capture. When off, behave as if the firmware doesn't have the feature and use the standard single mixed stream.",
|
||||
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
|
||||
});
|
||||
|
||||
export const showBleBandwidthSetting = new ConfigSettingBoolean({
|
||||
@@ -472,12 +528,19 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
"How R1 ring input reaches the phone. Only via glasses: the ring's own link to the glasses carries its gestures, and the phone never opens a Bluetooth connection to the ring. Direct: also connect to the ring from the phone (currently unreliable). Takes effect on the next connection to the glasses.",
|
||||
});
|
||||
|
||||
export type VoiceProvider = "onboard" | "elevenlabs" | "whisper" | "soniox";
|
||||
// "whisper" (no "onboard-" prefix) is OpenAI's CLOUD realtime model
|
||||
// (gpt-realtime-whisper); "onboard-whisper" is the on-device sherpa-onnx
|
||||
// Whisper backend. Same underlying model family, two different places it
|
||||
// runs -- see the same note in native/voice-control.ts. The "whisper" value
|
||||
// keeps its name (it's a persisted setting on real installs) but its label
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox";
|
||||
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: "On-device",
|
||||
onboard: "On-device (Moonshine)",
|
||||
"onboard-whisper": "On-device (Whisper)",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "Whisper",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
soniox: "Soniox",
|
||||
};
|
||||
|
||||
@@ -486,7 +549,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
label: "Transcription Provider",
|
||||
storageKey: "voice.provider",
|
||||
defaultValue: "onboard",
|
||||
values: ["onboard", "elevenlabs", "whisper", "soniox"],
|
||||
values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
@@ -494,7 +557,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0;
|
||||
return false;
|
||||
},
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).",
|
||||
description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
@@ -934,7 +997,16 @@ export function screenTimeoutLabel(value: ScreenTimeoutSetting): string {
|
||||
}
|
||||
|
||||
export function batteryDisplayModeLabel(value: BatteryDisplayMode): string {
|
||||
return value === "icon" ? "Icon" : "Percentage";
|
||||
if (value === "percentage") return "Percentage";
|
||||
if (value === "stacked") return "Stacked";
|
||||
if (value === "stacked-percentage") return "Stacked percentage";
|
||||
return "Icon";
|
||||
}
|
||||
|
||||
export function batteryIndicatorVisibilityLabel(value: BatteryIndicatorVisibility): string {
|
||||
if (value === "never") return "Never";
|
||||
if (value === "low") return `Below ${BATTERY_LOW_VISIBILITY_THRESHOLD}%`;
|
||||
return "Always";
|
||||
}
|
||||
|
||||
export function timeFormatLabel(value: TimeFormat): string {
|
||||
|
||||
@@ -13,12 +13,13 @@ import {
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
import {
|
||||
ASR_MODEL,
|
||||
ASR_MODELS,
|
||||
asrModelState,
|
||||
cancelAsrModelDownload,
|
||||
deleteAsrModel,
|
||||
onAsrModelStateChanged,
|
||||
startAsrModelDownload,
|
||||
type AsrModelId,
|
||||
} from "../../native/asr-model";
|
||||
import { TextViewerLayer } from "../../apps/files/text-viewer";
|
||||
import type { LayerContext } from "../layers";
|
||||
@@ -35,6 +36,10 @@ import {
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
brightnessSetting,
|
||||
glassesBatteryVisibilitySetting,
|
||||
phoneBatteryVisibilitySetting,
|
||||
ringBatteryVisibilitySetting,
|
||||
watchBatteryVisibilitySetting,
|
||||
displayModeSetting,
|
||||
navigateDisplayModeSetting,
|
||||
navigateVerticalPositionSetting,
|
||||
@@ -75,7 +80,7 @@ import {
|
||||
} from "../dashboard-settings";
|
||||
import { clearRecentDestinations } from "../../apps/navigate/destinations";
|
||||
import { wearBridge } from "../../native/wear-bridge";
|
||||
import { SettingsPanelLayer, type SettingsSection } from "./settings-panel";
|
||||
import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from "./settings-panel";
|
||||
import { terminalFontPickerMenuItem, uiFontPickerMenuItem } from "../font-picker";
|
||||
|
||||
/** The Settings app's master-detail panel (sections on the left, contents on the right). */
|
||||
@@ -102,8 +107,8 @@ function settingsSections(): SettingsSection[] {
|
||||
enumSettingMenuItem(verticalPositionSetting),
|
||||
// Band / tall / full-panel; the dashboard controller reflows windows.
|
||||
enumSettingMenuItem(displayModeSetting),
|
||||
// Controls the top-bar battery indicators (icon vs percentage).
|
||||
enumSettingMenuItem(batteryDisplayModeSetting),
|
||||
// Submenu: top-bar battery indicator style plus per-device visibility.
|
||||
batteryIndicatorsMenuItem(),
|
||||
// Controls the top-bar clock (24-hour vs 12-hour).
|
||||
enumSettingMenuItem(timeFormatSetting),
|
||||
// Opens the modal font picker (face, weight, size) for UI text.
|
||||
@@ -115,7 +120,8 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
asrModelMenuItem(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -253,6 +259,39 @@ function settingsSections(): SettingsSection[] {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The Display section's "Battery indicators" row: opens a modal submenu with
|
||||
* the style (icon / percentage / stacked) and, per device, when its
|
||||
* indicator is visible. The row itself shows the current style.
|
||||
*/
|
||||
function batteryIndicatorsMenuItem(): MenuItem {
|
||||
return {
|
||||
label: "Battery indicators",
|
||||
description:
|
||||
"Top-bar battery indicators for the phone, Wear OS watch, glasses, and ring: their style, and whether each shows always, only when low, or never.",
|
||||
onSelect: (ctx) => {
|
||||
openSettingsSubMenu(ctx, "Battery indicators", [
|
||||
enumSettingMenuItem(batteryDisplayModeSetting),
|
||||
enumSettingMenuItem(phoneBatteryVisibilitySetting),
|
||||
enumSettingMenuItem(watchBatteryVisibilitySetting),
|
||||
enumSettingMenuItem(glassesBatteryVisibilitySetting),
|
||||
enumSettingMenuItem(ringBatteryVisibilitySetting),
|
||||
]);
|
||||
},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(
|
||||
image,
|
||||
getDefaultSmallFont(),
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
"Battery indicators",
|
||||
batteryDisplayModeSetting.displayValue(),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const LOCAL_MODEL_GB = `${(LOCAL_MODEL.sizeBytes / 1e9).toFixed(1)}GB`;
|
||||
|
||||
// While a download is running, re-render on progress updates so the row's
|
||||
@@ -324,48 +363,52 @@ function localModelMenuItem(): MenuItem {
|
||||
};
|
||||
}
|
||||
|
||||
const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`;
|
||||
function asrModelMb(id: AsrModelId): string {
|
||||
return `${Math.round(ASR_MODELS[id].totalBytes / 1e6)}MB`;
|
||||
}
|
||||
|
||||
let asrModelRenderUnsub: (() => void) | null = null;
|
||||
const asrModelRenderUnsub: Partial<Record<AsrModelId, () => void>> = {};
|
||||
|
||||
function watchAsrModelDownload(ctx: LayerContext): void {
|
||||
asrModelRenderUnsub?.();
|
||||
asrModelRenderUnsub = onAsrModelStateChanged((state) => {
|
||||
function watchAsrModelDownload(id: AsrModelId, ctx: LayerContext): void {
|
||||
asrModelRenderUnsub[id]?.();
|
||||
asrModelRenderUnsub[id] = onAsrModelStateChanged(id, (state) => {
|
||||
ctx.actions.requestRender();
|
||||
if (state.status !== "downloading") {
|
||||
asrModelRenderUnsub?.();
|
||||
asrModelRenderUnsub = null;
|
||||
asrModelRenderUnsub[id]?.();
|
||||
asrModelRenderUnsub[id] = undefined;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function asrModelStatusText(): string {
|
||||
const state = asrModelState();
|
||||
function asrModelStatusText(id: AsrModelId): string {
|
||||
const state = asrModelState(id);
|
||||
if (state.status === "ready") return "downloaded";
|
||||
if (state.status === "downloading") {
|
||||
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
|
||||
return `${pct}% of ${ASR_MODEL_MB}`;
|
||||
return `${pct}% of ${asrModelMb(id)}`;
|
||||
}
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-device transcription model. */
|
||||
function asrModelMenuItem(): MenuItem {
|
||||
/** Download/cancel/delete management for one on-device transcription model. */
|
||||
function asrModelMenuItem(id: AsrModelId): MenuItem {
|
||||
const def = ASR_MODELS[id];
|
||||
const rowLabel = `On-device model: ${def.label}`;
|
||||
return {
|
||||
label: "On-device voice model",
|
||||
label: rowLabel,
|
||||
description:
|
||||
`${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` +
|
||||
`${def.label} (${asrModelMb(id)} download). ` +
|
||||
"Transcribes voice input on the phone itself, with no API key or cloud service. " +
|
||||
"Required for the On-device transcription provider; the cloud providers work without it. " +
|
||||
"Required for its matching Transcription Provider option; the other providers work without it. " +
|
||||
"An interrupted download resumes where it left off.",
|
||||
onSelect: (ctx) => {
|
||||
const state = asrModelState();
|
||||
const state = asrModelState(id);
|
||||
const action: MenuItem =
|
||||
state.status === "downloading"
|
||||
? {
|
||||
label: "Cancel download",
|
||||
onSelect: (innerCtx) => {
|
||||
cancelAsrModelDownload();
|
||||
cancelAsrModelDownload(id);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
@@ -373,22 +416,22 @@ function asrModelMenuItem(): MenuItem {
|
||||
? {
|
||||
label: "Delete model",
|
||||
onSelect: (innerCtx) => {
|
||||
deleteAsrModel();
|
||||
deleteAsrModel(id);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
: {
|
||||
label: `Download (${ASR_MODEL_MB})`,
|
||||
label: `Download (${asrModelMb(id)})`,
|
||||
onSelect: (innerCtx) => {
|
||||
startAsrModelDownload();
|
||||
watchAsrModelDownload(innerCtx);
|
||||
startAsrModelDownload(id);
|
||||
watchAsrModelDownload(id, innerCtx);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
};
|
||||
openModalMenu(ctx, "On-device voice model", [action], 0);
|
||||
openModalMenu(ctx, rowLabel, [action], 0);
|
||||
},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText());
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, rowLabel, asrModelStatusText(id));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ export type LayerActions = {
|
||||
* (hands-free); otherwise it runs until stopVoiceCapture.
|
||||
*/
|
||||
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
|
||||
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
||||
/** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||
startContinuousVoiceCapture: () => Promise<void> | void;
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
import { brightnessSetting, brightnessSettingToLevel, type BrightnessSetting } from "../dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../gestures";
|
||||
import type { Layer, LayerContext, PaintBelow } from "../layers";
|
||||
import { appViewportRect, SHELL_OPAQUE_BLACK } from "./geometry";
|
||||
|
||||
const DIALOG_WIDTH = 272;
|
||||
const BAR_WIDTH = 12;
|
||||
const BAR_HEIGHT = 100;
|
||||
|
||||
/** Manual brightness changes apply immediately so the wearer can judge the level. */
|
||||
export class BrightnessPickerLayer implements Layer {
|
||||
constructor(private readonly onClosed: () => void) {}
|
||||
|
||||
onRemoved(): void {
|
||||
this.onClosed();
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const image = paintBelow();
|
||||
const font = getDefaultSmallFont();
|
||||
const viewport = appViewportRect("min");
|
||||
const height = BAR_HEIGHT + 3 * font.lineHeight + 56;
|
||||
const left = viewport.x + Math.round((viewport.width - DIALOG_WIDTH) / 2);
|
||||
const top = viewport.y + Math.round((viewport.height - height) / 2);
|
||||
image.fillRoundedRect(left, top, DIALOG_WIDTH, height, SHELL_OPAQUE_BLACK, 8);
|
||||
image.drawRoundedRect(left, top, DIALOG_WIDTH, height, 110, 8);
|
||||
|
||||
const centeredText = (text: string, y: number, value: number) => {
|
||||
image.drawText(font, left + Math.round((DIALOG_WIDTH - font.measureText(text)) / 2), y, text, value);
|
||||
};
|
||||
centeredText("Brightness", top + 12, 235);
|
||||
const barX = left + Math.round((DIALOG_WIDTH - BAR_WIDTH) / 2);
|
||||
const barY = top + font.lineHeight + 24;
|
||||
const level = brightnessSettingToLevel(brightnessSetting.get());
|
||||
image.drawRect(barX - 2, barY - 2, BAR_WIDTH + 4, BAR_HEIGHT + 4, 150);
|
||||
const filledHeight = Math.round(BAR_HEIGHT * (level ?? 0) / 100);
|
||||
image.fillRect(barX, barY + BAR_HEIGHT - filledHeight, BAR_WIDTH, filledHeight, 255);
|
||||
image.drawText(font, barX + BAR_WIDTH + 12, barY + Math.round((BAR_HEIGHT - font.lineHeight) / 2),
|
||||
level === null ? "Auto" : `${level}%`, 235);
|
||||
centeredText(gestureHints([[GESTURE_SCROLL, "adjust"]]), barY + BAR_HEIGHT + 12, 150);
|
||||
centeredText(gestureHints([[`${GESTURE_CLICK} / ${GESTURE_DOUBLE_CLICK}`, "confirm"]]),
|
||||
barY + BAR_HEIGHT + font.lineHeight + 16, 150);
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "scroll-down": {
|
||||
const level = brightnessSettingToLevel(brightnessSetting.get());
|
||||
// Respect Auto if it was enabled on the phone while this dialog was open.
|
||||
if (level === null) return;
|
||||
const next = clamp(level + (event.type === "scroll-up" ? 10 : -10), 0, 100);
|
||||
if (next !== level) brightnessSetting.set(String(next) as BrightnessSetting);
|
||||
return;
|
||||
}
|
||||
case "click":
|
||||
case "double-click":
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
+116
-28
@@ -7,7 +7,17 @@ import { readActiveNotificationIcons } from "../../native/notification-icons";
|
||||
import { readPhoneBatteryState } from "../../native/phone-battery";
|
||||
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../util/render-freshness";
|
||||
import { renderIcon, renderIconWithGlyph, type IconActivity, type IconName } from "../../graphics/icons";
|
||||
import { batteryDisplayModeSetting, timeFormatSetting } from "../dashboard-settings";
|
||||
import {
|
||||
batteryDisplayModeSetting,
|
||||
batteryIndicatorVisible,
|
||||
glassesBatteryVisibilitySetting,
|
||||
phoneBatteryVisibilitySetting,
|
||||
ringBatteryVisibilitySetting,
|
||||
watchBatteryVisibilitySetting,
|
||||
type BatteryIndicatorVisibility,
|
||||
} from "../dashboard-settings";
|
||||
import { formatClockDate, formatClockTime } from "../clock-format";
|
||||
import { getFont } from "../../graphics/bdffont";
|
||||
import { Layer } from "../layers";
|
||||
import { scrollToKeepSelectionVisible } from "../menu";
|
||||
import { lineStep } from "../metrics";
|
||||
@@ -83,8 +93,6 @@ export function sidebarContentLeft(windowCount: number): number {
|
||||
}
|
||||
const NOTIFICATION_ICON_SIZE = 24;
|
||||
const BORDER_VALUE = 40;
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
export type ShellChromeWindow = {
|
||||
windowId: string;
|
||||
@@ -101,7 +109,15 @@ export type ShellChromeState = {
|
||||
/** Height mode of the foreground window; decides where its top bar sits. */
|
||||
foregroundHeightMode: WindowHeightMode;
|
||||
foregroundAppId?: string;
|
||||
battery: { headset: number | null; headsetCharging: boolean | null };
|
||||
battery: {
|
||||
headset: number | null;
|
||||
headsetCharging: boolean | null;
|
||||
ring: number | null;
|
||||
ringCharging: boolean | null;
|
||||
/** The Wear OS watch (app/g2/wear-remote.ts); null when no watch is reachable. */
|
||||
watch: number | null;
|
||||
watchCharging: boolean | null;
|
||||
};
|
||||
/** App-provided tray images, drawn between notification icons and batteries. */
|
||||
trayIcons: GrayImage[];
|
||||
};
|
||||
@@ -312,8 +328,7 @@ export class ShellChromeLayer implements Layer {
|
||||
image.drawLine(barLeft, barTop + TOP_BAR_HEIGHT - 1, G2_LENS_WIDTH - 1, barTop + TOP_BAR_HEIGHT - 1, BORDER_VALUE);
|
||||
|
||||
const now = new Date();
|
||||
const clock = `${WEEKDAYS[now.getDay()]} ${now.getDate()} ${MONTHS[now.getMonth()]} ` +
|
||||
formatClockTime(now);
|
||||
const clock = `${formatClockDate(now)} ${formatClockTime(now)}`;
|
||||
const clockX = barLeft + 10;
|
||||
const textY = barTop + Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
|
||||
image.drawText(font, clockX, textY, clock, 210);
|
||||
@@ -336,35 +351,51 @@ export class ShellChromeLayer implements Layer {
|
||||
}
|
||||
|
||||
/**
|
||||
* Labelled battery indicators for the phone and the G2, right-aligned in
|
||||
* the top bar, following the dashboard card's icon/percentage setting.
|
||||
* Returns the left edge of the battery block.
|
||||
* Labelled battery indicators for the phone, Wear OS watch, G2, and R1,
|
||||
* right-aligned in the top bar. The watch one exists only while a watch
|
||||
* running the Faceclaw watch app is reachable (no placeholder otherwise).
|
||||
* The Settings > Display > Battery indicators submenu picks the style (label beside a gauge icon or percentage, or stacked above
|
||||
* either) and, per device, whether the indicator shows always, only below
|
||||
* 50%, or never. Returns the left edge of the battery block.
|
||||
*/
|
||||
private drawTopBarBatteries(image: GrayImage, state: ShellChromeState, barTop: number): number {
|
||||
const font = getDefaultSmallFont();
|
||||
const percentageMode = batteryDisplayModeSetting.get() === "percentage";
|
||||
type BatteryItem = { label: string; percent: number; charging: boolean };
|
||||
const mode = batteryDisplayModeSetting.get();
|
||||
const items: BatteryItem[] = [];
|
||||
const phone = readPhoneBatteryState();
|
||||
if (phone.battery !== null && Number.isFinite(phone.battery)) {
|
||||
items.push({ label: "Phone", percent: phone.battery, charging: Boolean(phone.charging) });
|
||||
pushBatteryItem(items, phoneBatteryVisibilitySetting.get(), "Phone", phone.battery, Boolean(phone.charging));
|
||||
}
|
||||
if (state.battery.watch !== null && Number.isFinite(state.battery.watch)) {
|
||||
pushBatteryItem(items, watchBatteryVisibilitySetting.get(), "Watch", state.battery.watch,
|
||||
Boolean(state.battery.watchCharging));
|
||||
}
|
||||
if (state.battery.headset !== null && Number.isFinite(state.battery.headset)) {
|
||||
items.push({ label: "G2", percent: state.battery.headset, charging: Boolean(state.battery.headsetCharging) });
|
||||
pushBatteryItem(items, glassesBatteryVisibilitySetting.get(), "G2", state.battery.headset,
|
||||
Boolean(state.battery.headsetCharging));
|
||||
}
|
||||
if (state.battery.ring !== null && Number.isInteger(state.battery.ring)
|
||||
&& state.battery.ring >= 0 && state.battery.ring <= 100) {
|
||||
pushBatteryItem(items, ringBatteryVisibilitySetting.get(), "R1", state.battery.ring,
|
||||
Boolean(state.battery.ringCharging));
|
||||
}
|
||||
if (!items.length) return G2_LENS_WIDTH;
|
||||
if (mode === "stacked" || mode === "stacked-percentage") {
|
||||
return drawStackedBatteries(image, items, barTop, mode === "stacked-percentage");
|
||||
}
|
||||
|
||||
const font = getDefaultSmallFont();
|
||||
const percentageMode = mode === "percentage";
|
||||
const labelGap = 5;
|
||||
const itemGap = 12;
|
||||
const textY = barTop + Math.max(0, ((TOP_BAR_HEIGHT - font.lineHeight) / 2) | 0);
|
||||
let x = G2_LENS_WIDTH - 8;
|
||||
let x = G2_LENS_WIDTH - BATTERY_BLOCK_RIGHT_MARGIN;
|
||||
for (let index = items.length - 1; index >= 0; index--) {
|
||||
const item = items[index]!;
|
||||
const percentText = `${Math.max(0, Math.min(100, Math.round(item.percent)))}%`;
|
||||
const percentText = `${item.percent}%`;
|
||||
const valueWidth = percentageMode ? font.measureText(percentText) : BATTERY_ICON_WIDTH;
|
||||
const labelWidth = font.measureText(item.label);
|
||||
x -= labelWidth + labelGap + valueWidth;
|
||||
image.drawText(font, x, textY, item.label, 150);
|
||||
image.drawText(font, x, textY, item.label, BATTERY_LABEL_VALUE);
|
||||
const valueX = x + labelWidth + labelGap;
|
||||
if (percentageMode) {
|
||||
if (item.charging) {
|
||||
@@ -386,6 +417,74 @@ export class ShellChromeLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
type BatteryItem = { label: string; percent: number; charging: boolean };
|
||||
|
||||
const BATTERY_BLOCK_RIGHT_MARGIN = 8;
|
||||
const BATTERY_LABEL_VALUE = 150;
|
||||
|
||||
/** Append an indicator if its visibility setting shows it at this charge. */
|
||||
function pushBatteryItem(
|
||||
items: BatteryItem[],
|
||||
visibility: BatteryIndicatorVisibility,
|
||||
label: string,
|
||||
percent: number,
|
||||
charging: boolean,
|
||||
): void {
|
||||
const clamped = Math.max(0, Math.min(100, Math.round(percent)));
|
||||
if (!batteryIndicatorVisible(visibility, clamped)) return;
|
||||
items.push({ label, percent: clamped, charging });
|
||||
}
|
||||
|
||||
// Stacked style geometry, in rows below the bar's top edge. The bar has 27
|
||||
// usable rows above its bottom border line, too few for the user-sized UI
|
||||
// font (line height up to 21px) over a 10px gauge, so both stacked lines use
|
||||
// a fixed 12px bitmap face instead: TerminusV, proportional, with 8px
|
||||
// capitals and digits on a 10px ascent. Ink then spans rows 3..22: label
|
||||
// capitals at 3..10, a 2px gap, and the gauge at 13..22 (or the percentage
|
||||
// digits at 13..20, whose charging highlight box covers 12..21), leaving 3
|
||||
// rows above and 4 below (before the border) so the pair reads as one
|
||||
// centered unit.
|
||||
const STACKED_LABEL_BASELINE = 11;
|
||||
const STACKED_ICON_TOP = 13;
|
||||
const STACKED_PERCENT_BASELINE = 21;
|
||||
const STACKED_ITEM_GAP = 10;
|
||||
|
||||
/**
|
||||
* Stacked styles: each label centered above its gauge icon (or percentage
|
||||
* text), right-aligned in the bar. Returns the left edge of the battery
|
||||
* block.
|
||||
*/
|
||||
function drawStackedBatteries(image: GrayImage, items: BatteryItem[], barTop: number, percentage: boolean): number {
|
||||
const font = getFont("terminusv12");
|
||||
const labelTop = barTop + STACKED_LABEL_BASELINE - font.ascent;
|
||||
const percentTop = barTop + STACKED_PERCENT_BASELINE - font.ascent;
|
||||
let x = G2_LENS_WIDTH - BATTERY_BLOCK_RIGHT_MARGIN;
|
||||
for (let index = items.length - 1; index >= 0; index--) {
|
||||
const item = items[index]!;
|
||||
const percentText = `${item.percent}%`;
|
||||
const labelWidth = font.measureText(item.label);
|
||||
const valueWidth = percentage ? font.measureText(percentText) : BATTERY_ICON_WIDTH;
|
||||
const itemWidth = Math.max(labelWidth, valueWidth);
|
||||
x -= itemWidth;
|
||||
image.drawText(font, x + (((itemWidth - labelWidth) / 2) | 0), labelTop, item.label, BATTERY_LABEL_VALUE);
|
||||
const valueX = x + (((itemWidth - valueWidth) / 2) | 0);
|
||||
if (!percentage) {
|
||||
const icon = drawBattery(item.percent, item.charging);
|
||||
image.bitBlt(icon, valueX, barTop + STACKED_ICON_TOP, { transparentZero: true });
|
||||
} else if (item.charging) {
|
||||
// Inverted text marks charging, as in the side-by-side percentage style;
|
||||
// the box hugs the digits' 8px ink rather than the full 12px line.
|
||||
const inkTop = barTop + STACKED_PERCENT_BASELINE - 8;
|
||||
image.fillRect(valueX - 2, inkTop - 1, valueWidth + 4, 10, 255);
|
||||
image.drawText(font, valueX, percentTop, percentText, 1);
|
||||
} else {
|
||||
image.drawText(font, valueX, percentTop, percentText, 200);
|
||||
}
|
||||
x -= STACKED_ITEM_GAP;
|
||||
}
|
||||
return x + STACKED_ITEM_GAP;
|
||||
}
|
||||
|
||||
// Ambient cards (encounter popups): compact and deliberately unobtrusive,
|
||||
// anchored to the bottom-right of the window band and stacking upward, clear
|
||||
// of the caption area's left-aligned text. Never interactive.
|
||||
@@ -519,17 +618,6 @@ function tabLeftInset(yy: number, top: number, bottom: number, radius: number):
|
||||
return Math.max(0, Math.round(radius - Math.sqrt(Math.max(0, radius * radius - dy * dy))));
|
||||
}
|
||||
|
||||
/** Format the top-bar clock time, honoring the 12/24-hour setting. */
|
||||
function formatClockTime(now: Date): string {
|
||||
const minutes = String(now.getMinutes()).padStart(2, "0");
|
||||
const hour24 = now.getHours();
|
||||
if (timeFormatSetting.get() === "12h") {
|
||||
const hour12 = ((hour24 + 11) % 12) + 1;
|
||||
return `${hour12}:${minutes} ${hour24 < 12 ? "AM" : "PM"}`;
|
||||
}
|
||||
return `${hour24}:${minutes}`;
|
||||
}
|
||||
|
||||
/** Small triangle marker for sidebar overflow; direction -1 = up, 1 = down. */
|
||||
function drawChevron(image: GrayImage, centerX: number, y: number, direction: -1 | 1): void {
|
||||
const half = 5;
|
||||
|
||||
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
|
||||
* has no menu of its own, and tap-then-hold opens the system menu instead.
|
||||
*/
|
||||
menuItems?: () => MenuItem[];
|
||||
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
|
||||
holdToTalk?: boolean;
|
||||
isVoiceCapturing?: () => boolean;
|
||||
onSystemMenuOpened?: () => void;
|
||||
onAppMenuOpened?: () => void;
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
actions: LayerActions;
|
||||
/**
|
||||
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
shell.openSystemMenu(options.windowId);
|
||||
return;
|
||||
}
|
||||
stack.push(new WindowMenuLayer(options.title, items));
|
||||
options.onAppMenuOpened?.();
|
||||
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
|
||||
};
|
||||
|
||||
const window: ShellWindow = {
|
||||
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// The window's own LayerStack decides per layer whether a swipe is
|
||||
// directional or falls back to click / double-click.
|
||||
acceptsDirectional: true,
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
close: () => {
|
||||
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// The shell opened its system menu over this window; close our own
|
||||
// context menu so the two never stack. Never forwarded to app layers.
|
||||
if (event.type === "system-menu-opened") {
|
||||
options.onSystemMenuOpened?.();
|
||||
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
|
||||
requestRender();
|
||||
}
|
||||
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// submenus and app content alike. A plain long-press never arrives:
|
||||
// the shell keeps it for the system menu (no in-process window claims
|
||||
// it).
|
||||
if (event.type === "short-then-long-press") {
|
||||
if ((!options.holdToTalk && event.type === "short-then-long-press") ||
|
||||
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
|
||||
openWindowMenu();
|
||||
await render(frameId);
|
||||
return;
|
||||
|
||||
+162
-49
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
|
||||
import { voiceActivity } from "./voice-activity";
|
||||
import { AssistantLayer } from "./assistant";
|
||||
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
|
||||
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
@@ -32,7 +34,8 @@ import {
|
||||
assistantBridgeTokenSetting,
|
||||
assistantModelSetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
batteryIndicatorSettingsKey,
|
||||
brightnessSetting,
|
||||
onAnySettingChanged,
|
||||
openAiApiKeySetting,
|
||||
timeFormatSetting,
|
||||
@@ -42,6 +45,7 @@ import { onAmbientCardsChanged } from "./ambient-cards";
|
||||
import { ShellChromeLayer, sidebarContentLeft, type ShellChromeState, type ShellChromeWindow } from "./chrome-layer";
|
||||
import { ShellModalLayer } from "./modal-layer";
|
||||
import { ToolDebugMenuLayer } from "./tool-debug-layer";
|
||||
import { BrightnessPickerLayer } from "./brightness-picker-layer";
|
||||
import { toolRegistry } from "../../assistant/tool-registry";
|
||||
import {
|
||||
minWindowTop,
|
||||
@@ -61,7 +65,7 @@ import {
|
||||
* Input flow: every event enters via receiveInput. The shell consumes
|
||||
* everything while the sidebar or a shell overlay has focus and forwards the
|
||||
* rest to the focused window. Long-press opens the shell-owned system menu
|
||||
* (Focus app switcher, Voice input, Close window, Debug) without reaching the
|
||||
* (Focus app switcher, Voice input, Brightness, Close window, Debug) without reaching the
|
||||
* app, so the shell keeps working when a window's handler hangs; a window
|
||||
* that claims long-press for a move of its own gets it forwarded instead, and
|
||||
* holding the press past the escape threshold still opens the system menu.
|
||||
@@ -91,6 +95,10 @@ export type ShellWindow = {
|
||||
* system menu; holding the press past the escape threshold still opens it.
|
||||
*/
|
||||
claimsLongPress?: () => boolean;
|
||||
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
|
||||
holdToTalk?: boolean;
|
||||
/** A window owns microphone capture outside the shell voice dialog. */
|
||||
isVoiceCapturing?: () => boolean;
|
||||
/**
|
||||
* True when the window gives swipe-left / swipe-right (watch directional
|
||||
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
||||
@@ -141,6 +149,13 @@ export type ShellWindow = {
|
||||
onFocus?: (lastInput: InputEvent | null) => void;
|
||||
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
|
||||
setScreenOn?: (on: boolean) => void;
|
||||
/**
|
||||
* Input focus arrived at or left this window: it is (no longer) where
|
||||
* ordinary input goes. Unlike setForeground / onFocus, this also tracks
|
||||
* shell overlays (the system menu, a notification modal, the voice dialog)
|
||||
* and screen-off, so a game can pause on any of them. Sent on change only.
|
||||
*/
|
||||
setInputFocus?: (focused: boolean) => void;
|
||||
};
|
||||
|
||||
export type ShellConfig = {
|
||||
@@ -283,24 +298,55 @@ class ShellAlertLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
/** Every setting the top bar paints from, as one comparable string. */
|
||||
function topBarSettingsKey(): string {
|
||||
return `${batteryIndicatorSettingsKey()}|${timeFormatSetting.get()}`;
|
||||
}
|
||||
|
||||
class Shell {
|
||||
private windows: ShellWindow[] = [];
|
||||
private selectedIndex = 0;
|
||||
/** Window ids in most-recently-visible-first order; closing the visible window returns to the next entry. */
|
||||
private mruWindowIds: string[] = [];
|
||||
private focus: FocusKind = "sidebar";
|
||||
/** The window last told it holds input focus (see syncInputFocus). */
|
||||
private inputFocusedWindowId: string | null = null;
|
||||
private screenOn = true;
|
||||
private lastInputAtMs = Date.now();
|
||||
/** The most recent input event received, for windows gaining focus (see ShellWindow.onFocus). */
|
||||
private lastInput: InputEvent | null = null;
|
||||
private battery: ShellChromeState["battery"] = { headset: null, headsetCharging: null };
|
||||
private battery: ShellChromeState["battery"] = {
|
||||
headset: null, headsetCharging: null, ring: null, ringCharging: null, watch: null, watchCharging: null,
|
||||
};
|
||||
private attention = new Map<string, boolean>();
|
||||
// App-provided top-bar tray icons, keyed by owner id; drawn between the
|
||||
// notification icons and the battery indicators.
|
||||
private readonly trayIcons = new Map<string, GrayImage>();
|
||||
private activeVoiceLayer: VoiceInputLayer | null = null;
|
||||
private activeKeyboardLayer: KeyboardInputLayer | null = null;
|
||||
private assistantSession: AssistantSession | null = null;
|
||||
private conversations: AssistantConversations | null = null;
|
||||
private get assistantSession(): AssistantSession | null {
|
||||
return this.conversations?.current().session ?? null;
|
||||
}
|
||||
|
||||
getAssistantConversations(): AssistantConversations {
|
||||
if (!this.conversations) {
|
||||
this.conversations = new AssistantConversations(
|
||||
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
|
||||
() => assistantModelSetting.get(),
|
||||
(value) => setStringSetting("assistant.conversations", value),
|
||||
getStringSetting("assistant.conversations", ""),
|
||||
);
|
||||
}
|
||||
return this.conversations;
|
||||
}
|
||||
|
||||
async prepareChatVoiceCapture(): Promise<boolean> {
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
|
||||
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
||||
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
|
||||
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
|
||||
}
|
||||
private assistantLayer: AssistantLayer | null = null;
|
||||
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
|
||||
private readonly alertListeners = new Set<(text: string) => void>();
|
||||
@@ -315,14 +361,21 @@ class Shell {
|
||||
private readonly chrome = new ShellChromeLayer(() => this.chromeState());
|
||||
private readonly stack = new LayerStack(this.chrome, this.actions);
|
||||
|
||||
// Top-bar settings we mirror into the chrome; a change to either repaints
|
||||
// the shell surface so the top bar reflects it immediately.
|
||||
// Top-bar settings we mirror into the chrome; a change to any of them
|
||||
// repaints the shell surface so the top bar reflects it immediately.
|
||||
private topBarSettingsSubscribed = false;
|
||||
private lastBatteryDisplayMode: string | null = null;
|
||||
private lastTimeFormat: string | null = null;
|
||||
private lastTopBarSettingsKey: string | null = null;
|
||||
|
||||
configure(config: ShellConfig): void {
|
||||
this.config = config;
|
||||
// Nearly every state change ends in a shell render request, so it is
|
||||
// the natural point to tell windows about input-focus changes too.
|
||||
this.config = {
|
||||
...config,
|
||||
requestShellRender: () => {
|
||||
this.syncInputFocus();
|
||||
config.requestShellRender();
|
||||
},
|
||||
};
|
||||
this.stack.setActions(config.actions);
|
||||
this.subscribeToTopBarSettings();
|
||||
this.subscribeToAmbientCards();
|
||||
@@ -345,16 +398,11 @@ class Shell {
|
||||
private subscribeToTopBarSettings(): void {
|
||||
if (this.topBarSettingsSubscribed) return;
|
||||
this.topBarSettingsSubscribed = true;
|
||||
this.lastBatteryDisplayMode = batteryDisplayModeSetting.get();
|
||||
this.lastTimeFormat = timeFormatSetting.get();
|
||||
this.lastTopBarSettingsKey = topBarSettingsKey();
|
||||
onAnySettingChanged(() => {
|
||||
const batteryMode = batteryDisplayModeSetting.get();
|
||||
const timeFormat = timeFormatSetting.get();
|
||||
if (batteryMode === this.lastBatteryDisplayMode && timeFormat === this.lastTimeFormat) {
|
||||
return;
|
||||
}
|
||||
this.lastBatteryDisplayMode = batteryMode;
|
||||
this.lastTimeFormat = timeFormat;
|
||||
const key = topBarSettingsKey();
|
||||
if (key === this.lastTopBarSettingsKey) return;
|
||||
this.lastTopBarSettingsKey = key;
|
||||
this.config.requestShellRender();
|
||||
});
|
||||
}
|
||||
@@ -468,7 +516,7 @@ class Shell {
|
||||
return this.screenOn;
|
||||
}
|
||||
|
||||
/** Current headset battery levels (for the assistant's get_state tool). */
|
||||
/** Current G2, R1, and Wear OS watch battery levels (top bar, Glanceboard, assistant). */
|
||||
getBatteryLevels(): ShellChromeState["battery"] {
|
||||
return this.battery;
|
||||
}
|
||||
@@ -508,6 +556,34 @@ class Shell {
|
||||
return !!window && this.focus === "window" && this.foregroundWindow() === window;
|
||||
}
|
||||
|
||||
/**
|
||||
* The window ordinary input reaches right now: the foreground window with
|
||||
* focus in-window, the screen on, and no shell overlay (system menu,
|
||||
* notification modal, voice dialog, alert) above it.
|
||||
*/
|
||||
private inputTargetWindow(): ShellWindow | undefined {
|
||||
if (!this.screenOn || this.focus !== "window" || !this.stack.isAtBase() || this.activeVoiceLayer) {
|
||||
return undefined;
|
||||
}
|
||||
return this.foregroundWindow();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tell windows when they gain or lose input focus (ShellWindow.setInputFocus).
|
||||
* Called from every shell render request and the input/wake/sleep paths,
|
||||
* which between them follow every focus-affecting state change; it diffs
|
||||
* against the last notification, so calling it often is cheap.
|
||||
*/
|
||||
private syncInputFocus(): void {
|
||||
const target = this.inputTargetWindow();
|
||||
const targetId = target?.windowId ?? null;
|
||||
if (targetId === this.inputFocusedWindowId) return;
|
||||
const previous = this.windows.find((w) => w.windowId === this.inputFocusedWindowId);
|
||||
this.inputFocusedWindowId = targetId;
|
||||
previous?.setInputFocus?.(false);
|
||||
target?.setInputFocus?.(true);
|
||||
}
|
||||
|
||||
/** Turn the screen on (if off) and set focus. Returns whether it was off. */
|
||||
wake(focus: FocusKind, nowMs = Date.now()): boolean {
|
||||
this.lastInputAtMs = nowMs;
|
||||
@@ -515,7 +591,10 @@ class Shell {
|
||||
const alreadyFocused = this.isFocusTarget(gaining);
|
||||
this.focus = focus;
|
||||
if (gaining && !alreadyFocused) gaining.onFocus?.(this.lastInput);
|
||||
if (this.screenOn) return false;
|
||||
if (this.screenOn) {
|
||||
this.syncInputFocus();
|
||||
return false;
|
||||
}
|
||||
this.screenOn = true;
|
||||
this.config.onScreenStateChanged(true);
|
||||
for (const window of this.windows) {
|
||||
@@ -524,6 +603,7 @@ class Shell {
|
||||
// Refresh the foreground window; the compositor restored its retained
|
||||
// frame, but its content may be stale (e.g. a running stopwatch).
|
||||
this.foregroundWindow()?.requestRender();
|
||||
this.syncInputFocus();
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -536,6 +616,7 @@ class Shell {
|
||||
for (const window of this.windows) {
|
||||
window.setScreenOn?.(false);
|
||||
}
|
||||
this.syncInputFocus();
|
||||
this.config.onScreenStateChanged(false);
|
||||
}
|
||||
|
||||
@@ -548,6 +629,7 @@ class Shell {
|
||||
this.setSelectedIndex(index);
|
||||
this.focus = "window";
|
||||
if (!alreadyFocused) target?.onFocus?.(this.lastInput);
|
||||
this.syncInputFocus();
|
||||
}
|
||||
|
||||
/** Idle timeout: sleep if the configured timeout elapsed. Returns whether it slept. */
|
||||
@@ -561,7 +643,7 @@ class Shell {
|
||||
// ring. An in-flight assistant turn suspends it for the same reason (a
|
||||
// tool loop can run for a while with no input); once the turn ends and
|
||||
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
@@ -626,6 +708,16 @@ class Shell {
|
||||
}
|
||||
|
||||
async receiveInput(event: InputEvent, frameId = 0): Promise<ShellInputOutcome> {
|
||||
try {
|
||||
return await this.routeInput(event, frameId);
|
||||
} finally {
|
||||
// Whatever the input did (opened an overlay, moved focus, slept the
|
||||
// screen), windows learn about the resulting input-focus change.
|
||||
this.syncInputFocus();
|
||||
}
|
||||
}
|
||||
|
||||
private async routeInput(event: InputEvent, frameId: number): Promise<ShellInputOutcome> {
|
||||
const previous = this.lastInput;
|
||||
this.lastInput = event;
|
||||
// A visible window may paint a source-dependent indicator (see
|
||||
@@ -647,6 +739,7 @@ class Shell {
|
||||
// Even AI app never launches, so the firmware does not power the display
|
||||
// for us either -- actions that need it wake the screen themselves.
|
||||
if (event.type === "wakeword") {
|
||||
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
|
||||
const action = wakeWordActionSetting.get();
|
||||
if (action === "off") {
|
||||
return { shell: false, window: false };
|
||||
@@ -698,11 +791,11 @@ class Shell {
|
||||
if (!window) {
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
if (!window.claimsLongPress?.()) {
|
||||
if (!window.holdToTalk && !window.claimsLongPress?.()) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
this.startEscapeMenuTimer();
|
||||
if (!window.holdToTalk) this.startEscapeMenuTimer();
|
||||
if (this.focus === "sidebar") {
|
||||
this.focus = "window";
|
||||
window.onFocus?.(this.lastInput);
|
||||
@@ -718,6 +811,10 @@ class Shell {
|
||||
// when it has none.
|
||||
if (event.type === "short-then-long-press") {
|
||||
const window = this.foregroundWindow();
|
||||
if (window?.holdToTalk) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
// Over the open system menu it switches to the app's context menu:
|
||||
// close the system menu and deliver the gesture to the window as
|
||||
// usual. A window without a menu of its own would only ask for the
|
||||
@@ -1027,7 +1124,7 @@ class Shell {
|
||||
*/
|
||||
startKeyboardInput(): KeyboardInputSession | null {
|
||||
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
|
||||
if (this.activeVoiceLayer) return null;
|
||||
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
const assistantLayer = this.assistantLayer;
|
||||
const assistantSession = this.assistantSession;
|
||||
@@ -1072,7 +1169,8 @@ class Shell {
|
||||
}
|
||||
|
||||
isAssistantAvailable(): boolean {
|
||||
return this.resolveAssistantConfiguration() !== null;
|
||||
const current = this.getAssistantConversations().current();
|
||||
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
|
||||
}
|
||||
|
||||
/** Observe assistant turns (see AssistantActivityEvent). */
|
||||
@@ -1110,20 +1208,13 @@ class Shell {
|
||||
}
|
||||
|
||||
private ensureAssistantSession(): AssistantSession | null {
|
||||
const config = this.resolveAssistantConfiguration();
|
||||
if (!config) return null;
|
||||
if (
|
||||
!this.assistantSession ||
|
||||
this.assistantSession.isExpired() ||
|
||||
!this.assistantSession.matchesConfiguration(config)
|
||||
) {
|
||||
this.assistantSession?.cancel();
|
||||
this.assistantSession = new AssistantSession(config);
|
||||
}
|
||||
return this.assistantSession;
|
||||
return this.getAssistantConversations().ensureSession();
|
||||
}
|
||||
|
||||
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
|
||||
private resolveAssistantConfiguration(
|
||||
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
|
||||
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
|
||||
): AssistantBackendConfig | null {
|
||||
if (assistantBackendSetting.get() === "external") {
|
||||
const host = assistantBridgeHostSetting.get().trim();
|
||||
const token = assistantBridgeTokenSetting.get();
|
||||
@@ -1131,10 +1222,11 @@ class Shell {
|
||||
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
||||
return { kind: "external", bridge: { host, port, token } };
|
||||
}
|
||||
const llm = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
const llm = resolveAssistantModel(model, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
});
|
||||
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
|
||||
return llm ? { kind: "direct", llm } : null;
|
||||
}
|
||||
|
||||
@@ -1154,7 +1246,9 @@ class Shell {
|
||||
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
||||
* existing session and overlay.
|
||||
*/
|
||||
sendToAssistant(text: string): void {
|
||||
sendToAssistant(text: string, showOverlay = true): void {
|
||||
text = text.trim();
|
||||
if (!text) return;
|
||||
const session = this.ensureAssistantSession();
|
||||
if (!session) {
|
||||
this.showAlert(
|
||||
@@ -1164,7 +1258,15 @@ class Shell {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (session.isTurnActive()) {
|
||||
this.showAlert("The assistant is still working on the previous request");
|
||||
return;
|
||||
}
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
|
||||
this.runAssistantTurn(session, null, text);
|
||||
return;
|
||||
}
|
||||
let layer = this.assistantLayer;
|
||||
if (!layer) {
|
||||
const created = new AssistantLayer(this.config.actions, {
|
||||
@@ -1173,7 +1275,7 @@ class Shell {
|
||||
onClose: () => this.closeAssistantLayer(),
|
||||
onRemoved: () => {
|
||||
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
||||
// stop the turn and drop the reference so a later query starts clean.
|
||||
// stop the turn and drop the overlay reference; keep shared history.
|
||||
this.assistantSession?.cancel();
|
||||
if (this.assistantLayer === created) this.assistantLayer = null;
|
||||
this.emitAssistantActivity({ phase: "closed", text: "" });
|
||||
@@ -1187,23 +1289,24 @@ class Shell {
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
|
||||
layer.startTurn();
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
|
||||
if (session.isTurnActive()) return;
|
||||
layer?.startTurn();
|
||||
let replySoFar = "";
|
||||
this.emitAssistantActivity({ phase: "thinking", text: "" });
|
||||
session.sendUtterance(text, this.buildAssistantContext(), {
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
replySoFar = textSoFar;
|
||||
layer.onTextDelta(delta, textSoFar);
|
||||
layer?.onTextDelta(delta, textSoFar);
|
||||
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
|
||||
},
|
||||
onToolActivity: (label) => layer.onToolActivity(label),
|
||||
onToolActivity: (label) => layer?.onToolActivity(label),
|
||||
onTurnDone: () => {
|
||||
layer.onTurnDone();
|
||||
layer?.onTurnDone();
|
||||
this.emitAssistantActivity({ phase: "done", text: replySoFar });
|
||||
},
|
||||
onError: (message) => {
|
||||
layer.onError(message);
|
||||
layer?.onError(message);
|
||||
this.emitAssistantActivity({ phase: "error", text: message });
|
||||
},
|
||||
});
|
||||
@@ -1301,7 +1404,7 @@ class Shell {
|
||||
|
||||
/**
|
||||
* The system/escape menu: the entries every window shares (Focus app
|
||||
* switcher, Voice input, Close window) plus Debug. Shell-owned and
|
||||
* switcher, Voice input, Brightness, Close window) plus Debug. Shell-owned and
|
||||
* shell-drawn (never the app's), so an unresponsive app can always be
|
||||
* closed. It opens for long-press (over the app's own menu too), after an
|
||||
* extended hold in a window that claims long-press, and on a window's
|
||||
@@ -1348,6 +1451,16 @@ class Shell {
|
||||
},
|
||||
}]),
|
||||
);
|
||||
if (brightnessSetting.get() !== "auto") {
|
||||
items.push({
|
||||
label: "Brightness",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new BrightnessPickerLayer(() => this.yieldFocusToSidebar()));
|
||||
this.config.requestShellRender();
|
||||
},
|
||||
});
|
||||
}
|
||||
items.push({
|
||||
label: "Debug",
|
||||
onSelect: (ctx) => {
|
||||
@@ -1359,7 +1472,7 @@ class Shell {
|
||||
// when the app has one (otherwise it opens this very menu, not worth a
|
||||
// hint).
|
||||
const footer = foreground.hasAppMenu?.()
|
||||
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
|
||||
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
|
||||
: undefined;
|
||||
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
|
||||
layer.selectItem(initialSelection);
|
||||
|
||||
+70
-20
@@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
|
||||
import { Layer, type LayerActions, type LayerContext } from "../layers";
|
||||
import { paintInputDialog } from "./input-dialog";
|
||||
// After the mic stops, the provider's final transcript can trail in (cloud
|
||||
// commit round-trip); wait this long for it before refining with what we have.
|
||||
// After native recognition finishes, a cloud final can still trail in.
|
||||
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
|
||||
|
||||
/**
|
||||
@@ -64,7 +63,11 @@ export type VoiceInputLayerOptions = {
|
||||
*/
|
||||
export class VoiceInputLayer implements Layer {
|
||||
private phase: VoicePhase = "capturing";
|
||||
private status = "Listening...";
|
||||
private status = "Starting microphone...";
|
||||
/** Mirrors the bridge: only true while audio is actually being transcribed. */
|
||||
private listening = false;
|
||||
/** Bridge guidance shown in place of the empty transcript (e.g. "check your phone"). */
|
||||
private detail = "";
|
||||
// The active utterance. displayText() is what the dialog shows and what
|
||||
// Send delivers; the refine flow also writes the merged result here.
|
||||
private finalizedText = "";
|
||||
@@ -76,6 +79,10 @@ export class VoiceInputLayer implements Layer {
|
||||
private capturing = false;
|
||||
/** Continuation stopped; waiting for the trailing final transcript. */
|
||||
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private pendingFollowup = false;
|
||||
private stoppingCapture = false;
|
||||
private stopGeneration = 0;
|
||||
private acceptingTranscript = true;
|
||||
private refineHandle: AnthropicStreamHandle | null = null;
|
||||
private menuIndex = 0;
|
||||
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
|
||||
@@ -114,6 +121,8 @@ export class VoiceInputLayer implements Layer {
|
||||
// The refine stage owns the status line ("Refining...", error text).
|
||||
if (this.phase === "refining") return;
|
||||
this.status = state.status;
|
||||
this.listening = state.listening;
|
||||
this.detail = state.detail;
|
||||
this.actions.requestRender();
|
||||
});
|
||||
if (this.handsFree || global.isIOS) {
|
||||
@@ -149,19 +158,48 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
this.capturing = false;
|
||||
this.phase = "menu";
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.autoSend && this.sendTargets.length) {
|
||||
// Skip-confirmation (wakeword): send to the default target as soon as the
|
||||
// transcript finalizes, or after a short wait for the trailing final.
|
||||
this.pendingAutoSend = true;
|
||||
this.status = "Sending...";
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
} else if (this.status.startsWith("Listening")) {
|
||||
} else if (this.status.endsWith("...")) {
|
||||
// A progress status ("Starting microphone...", "Listening...") gives
|
||||
// way to the menu prompt; an error (ending in ".") stays visible.
|
||||
this.status = "Send, continue, or discard?";
|
||||
}
|
||||
this.stopCapture(() => {
|
||||
if (this.pendingAutoSend) {
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
});
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Native stop can outlast its synchronous join while Whisper is decoding. */
|
||||
private stopCapture(onStopped: () => void): void {
|
||||
const generation = ++this.stopGeneration;
|
||||
this.stoppingCapture = true;
|
||||
const afterStop = () => {
|
||||
if (generation !== this.stopGeneration) return;
|
||||
this.stoppingCapture = false;
|
||||
onStopped();
|
||||
this.actions.requestRender();
|
||||
};
|
||||
const stopped = this.actions.stopVoiceCapture();
|
||||
if (!stopped) {
|
||||
afterStop();
|
||||
return;
|
||||
}
|
||||
void stopped.then(afterStop, (error) => {
|
||||
if (generation !== this.stopGeneration) return;
|
||||
this.stoppingCapture = false;
|
||||
this.pendingAutoSend = false;
|
||||
this.acceptingTranscript = false;
|
||||
this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error));
|
||||
});
|
||||
}
|
||||
|
||||
/** Fire the queued skip-confirmation send (or fall back to the menu). */
|
||||
private performAutoSend(): void {
|
||||
if (this.autoSendTimer !== null) {
|
||||
@@ -185,7 +223,7 @@ export class VoiceInputLayer implements Layer {
|
||||
/** The menu rows: one per send target, then Continue, then Discard. */
|
||||
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
|
||||
const text = this.displayText().trim();
|
||||
const hasText = text.length > 0;
|
||||
const hasText = text.length > 0 && !this.stoppingCapture;
|
||||
const hasLlmKey = !global.isIOS && anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
|
||||
for (const target of this.sendTargets) {
|
||||
@@ -193,6 +231,7 @@ export class VoiceInputLayer implements Layer {
|
||||
label: target.label,
|
||||
dim: !hasText,
|
||||
onSelect: () => {
|
||||
if (this.stoppingCapture) return;
|
||||
this.dismiss();
|
||||
if (hasText) target.onSend(text);
|
||||
},
|
||||
@@ -200,9 +239,9 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
rows.push({
|
||||
label: global.isIOS ? "Continue (Not available on iOS)" : hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
dim: !hasLlmKey,
|
||||
dim: !hasLlmKey || this.stoppingCapture,
|
||||
onSelect: () => {
|
||||
if (hasLlmKey) this.startContinuation();
|
||||
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();
|
||||
},
|
||||
});
|
||||
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
|
||||
@@ -213,8 +252,8 @@ export class VoiceInputLayer implements Layer {
|
||||
const image = paintBelow();
|
||||
const inMenu = this.phase === "menu";
|
||||
paintInputDialog(image, {
|
||||
title: this.capturing ? "Voice ●" : "Voice",
|
||||
status: this.status,
|
||||
title: this.capturing && this.listening ? "Voice ●" : "Voice",
|
||||
status: this.stoppingCapture ? "Finishing transcription..." : this.status,
|
||||
text: this.displayText() || this.placeholderText(),
|
||||
rows: inMenu ? this.menuRows() : [],
|
||||
selectedRow: this.menuIndex,
|
||||
@@ -276,6 +315,7 @@ export class VoiceInputLayer implements Layer {
|
||||
|
||||
/** Continue selected: keep the message aside and record a follow-up. */
|
||||
private startContinuation(): void {
|
||||
this.acceptingTranscript = true;
|
||||
this.baseText = this.displayText().trim();
|
||||
this.finalizedText = "";
|
||||
this.liveText = "";
|
||||
@@ -290,15 +330,18 @@ export class VoiceInputLayer implements Layer {
|
||||
private endContinuationCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.status = "Refining...";
|
||||
this.pendingFollowup = true;
|
||||
this.status = "Finishing transcription...";
|
||||
this.stopCapture(() => {
|
||||
if (this.pendingFollowup) {
|
||||
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
});
|
||||
this.actions.requestRender();
|
||||
// The provider's committed transcript arrives shortly after stop; refine
|
||||
// when it does, or after a timeout with whatever partials we have.
|
||||
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private beginRefine(): void {
|
||||
this.pendingFollowup = false;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -335,9 +378,10 @@ export class VoiceInputLayer implements Layer {
|
||||
|
||||
/** Abort a continuation (mic or LLM stage) and restore the prior message. */
|
||||
private cancelContinuation(status: string): void {
|
||||
this.acceptingTranscript = false;
|
||||
if (this.capturing) {
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.stopCapture(() => {});
|
||||
}
|
||||
this.refineHandle?.cancel();
|
||||
this.refineHandle = null;
|
||||
@@ -345,6 +389,7 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
private backToMenu(text: string, status: string): void {
|
||||
this.pendingFollowup = false;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -358,6 +403,9 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
++this.stopGeneration;
|
||||
this.acceptingTranscript = false;
|
||||
this.pendingFollowup = false;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
@@ -392,7 +440,9 @@ export class VoiceInputLayer implements Layer {
|
||||
private placeholderText(): string {
|
||||
switch (this.phase) {
|
||||
case "capturing":
|
||||
return "Listening...";
|
||||
// Only claim to listen when the mic is actually running; otherwise
|
||||
// the bridge's guidance (permission prompt on the phone, etc.), if any.
|
||||
return this.listening ? "Listening..." : this.detail;
|
||||
case "continuing":
|
||||
return "Say more, or describe an edit...";
|
||||
case "refining":
|
||||
@@ -419,14 +469,14 @@ export class VoiceInputLayer implements Layer {
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
// The refine stream owns the text buffers once it starts; a transcript
|
||||
// that trails in after that point is stale.
|
||||
if (this.phase === "refining") return;
|
||||
if (!this.acceptingTranscript || this.phase === "refining") return;
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
if (this.phase === "continuing" && this.followupFinalizeTimer !== null) {
|
||||
if (this.phase === "continuing" && this.pendingFollowup) {
|
||||
// The follow-up finalized; no need to keep waiting.
|
||||
this.beginRefine();
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Main-thread mirror of small state that worker apps publish (the
|
||||
* "publish-state" worker reply): the latest value per key, with change
|
||||
* notifications, so main-thread consumers that are not the app's own windows
|
||||
* (a Glanceboard widget, say) can show what a worker knows without a second
|
||||
* data connection. Values cross postMessage, so they are plain JSON.
|
||||
*/
|
||||
const values = new Map<string, unknown>();
|
||||
const listeners = new Map<string, Set<(value: unknown) => void>>();
|
||||
|
||||
export function publishWorkerState(key: string, value: unknown): void {
|
||||
values.set(key, value);
|
||||
for (const listener of Array.from(listeners.get(key) ?? [])) {
|
||||
try {
|
||||
listener(value);
|
||||
} catch (error) {
|
||||
console.warn(`worker-state listener for ${key} failed`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The latest published value for `key`, or undefined when nothing was published yet. */
|
||||
export function readWorkerState<T>(key: string): T | undefined {
|
||||
return values.get(key) as T | undefined;
|
||||
}
|
||||
|
||||
/** Subscribe to later publications of `key`; returns the unsubscribe. */
|
||||
export function onWorkerStateChanged<T>(key: string, listener: (value: T) => void): () => void {
|
||||
let set = listeners.get(key);
|
||||
if (!set) {
|
||||
set = new Set();
|
||||
listeners.set(key, set);
|
||||
}
|
||||
set.add(listener as (value: unknown) => void);
|
||||
return () => {
|
||||
set!.delete(listener as (value: unknown) => void);
|
||||
};
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/to
|
||||
import { appViewportSize, type WindowHeightMode } from "./geometry";
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { shell, type ShellWindow } from "./shell";
|
||||
import { publishWorkerState } from "./worker-state";
|
||||
|
||||
/**
|
||||
* Messages between the shell (main thread) and an app worker. One worker
|
||||
@@ -20,6 +21,12 @@ export type WorkerAppMessage =
|
||||
| { type: "text-input"; windowId: string; text: string }
|
||||
| { type: "render"; windowId: string; focused: boolean }
|
||||
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
|
||||
/**
|
||||
* Input focus arrived at or left the window, counting shell overlays and
|
||||
* screen-off (see ShellWindow.setInputFocus). Sent on change only; the
|
||||
* per-message `focused` flags above stay the source of truth for painting.
|
||||
*/
|
||||
| { type: "input-focus"; windowId: string; focused: boolean }
|
||||
| { type: "screen"; on: boolean }
|
||||
/** Assistant tool invocation aimed at a window; reply with tool-result. */
|
||||
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
|
||||
@@ -135,6 +142,15 @@ export type WorkerAppReply =
|
||||
type: "tool-result";
|
||||
callId: string;
|
||||
result: ToolResult;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* Publish a small piece of app state for main-thread consumers outside
|
||||
* the app's windows (see app/ui/shell/worker-state.ts). JSON only.
|
||||
*/
|
||||
type: "publish-state";
|
||||
key: string;
|
||||
state: unknown;
|
||||
};
|
||||
|
||||
export type WorkerWindowSpec = {
|
||||
@@ -318,6 +334,9 @@ export class WorkerAppHost {
|
||||
case "set-title":
|
||||
// Titles are informational for now (sidebar shows icons only).
|
||||
break;
|
||||
case "publish-state":
|
||||
publishWorkerState(message.key, message.state);
|
||||
break;
|
||||
case "set-tools":
|
||||
// Only a window we actually have open may contribute tools.
|
||||
if (this.openWindows.has(message.windowId)) {
|
||||
@@ -420,6 +439,9 @@ export class WorkerAppHost {
|
||||
// uniform; the worker treats it globally.
|
||||
this.post({ type: "screen", on });
|
||||
},
|
||||
setInputFocus: (focused) => {
|
||||
this.post({ type: "input-focus", windowId: spec.windowId, focused });
|
||||
},
|
||||
};
|
||||
shell.registerWindow(window);
|
||||
// Configure the surface before any foregrounding so the worker's first
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* The "Sound: on/off" entry in a game's context menu, persisted per app under
|
||||
* `<appId>.soundOn` so each game remembers its own choice across launches.
|
||||
* Reads default to on; storage failures fall back to on and never throw.
|
||||
*/
|
||||
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
|
||||
|
||||
function soundKey(appId: string): string {
|
||||
return `${appId}.soundOn`;
|
||||
}
|
||||
|
||||
export function loadSoundEnabled(appId: string): boolean {
|
||||
try {
|
||||
return getBooleanSetting(soundKey(appId), true);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export function saveSoundEnabled(appId: string, enabled: boolean): void {
|
||||
try {
|
||||
setBooleanSetting(soundKey(appId), enabled);
|
||||
} catch (error) {
|
||||
console.warn(`${appId} sound setting save failed: ${error}`);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { singlePlane, type Plane } from "../graphics/plane";
|
||||
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||
import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||
import { type Layer, LayerStack, noopLayerActions } from "./layers";
|
||||
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
|
||||
import type { WorkerAppReply } from "./shell/worker-window";
|
||||
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
|
||||
};
|
||||
|
||||
export class WindowMenuLayer extends MenuLayer {
|
||||
constructor(title: string | null, items: MenuItem[]) {
|
||||
super(title, items, WINDOW_MENU_LAYOUT);
|
||||
constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
|
||||
super(title, items, holdToTalk
|
||||
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
|
||||
: WINDOW_MENU_LAYOUT);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user