Merge branch 'main' into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-10-25 01:06:38 -07:00
commit 4f8c082b52
368 files changed
+26238 -3401

No files matched your search

+232 -90
View File
@@ -15,8 +15,9 @@ import { startForegroundNotification, stopForegroundNotification, updateForegrou
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout-bridge";
import { weatherBridge } from "../native/weather";
import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, readActiveNotifications } from "../native/notification-icons";
import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, onAndroidNotificationRemoved, readActiveNotifications } from "../native/notification-icons";
import { shouldShowNotificationOnGlasses } from "../native/notification-sources";
import { watchNotificationListenerBinding } from "../native/notification-access";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { defaultRingName, setGlassesRingLink } from "../native/ring-status";
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
@@ -47,17 +48,20 @@ const MIRROR_TOUCH_GESTURES: Record<Exclude<MirrorTouchKind, "tap">, WearRemoteI
"swipe-left": "swipe-left",
"swipe-right": "swipe-right",
};
import { findSoundEffect, playSoundEffect } from "../ui/sound-effects";
import { GlanceHost } from "./glance-host";
import { mirrorToPanelFraction, phonePreviewCrop } from "./phone-preview-crop";
import { type GlanceEvent } from "./glance-state";
import { isPreviewOnlyMode, isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state";
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { voiceActivity } from "../ui/shell/voice-activity";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
import { prepareFrameDraws } from "../graphics/glyph-wire";
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen";
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
import { NotificationPopupDelay, notificationPopupDelayMs } from "../ui/shell/notification-popup-delay";
import { type InputEvent } from "../ui/gestures";
import { registerSystemTools } from "../assistant/system-tools";
import { updateGlassesPresence } from "./glasses-presence";
@@ -71,10 +75,10 @@ import { ALL_APPS } from "../apps/all-apps";
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry";
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayModeSetting, navigateDisplayModeSetting, navigateVerticalPositionSetting, terminalDisplayModeSetting, terminalVerticalPositionSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -94,7 +98,7 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
import { wearerVerificationOptions } from "../apps/microphones/speakers";
import { micSession } from "../apps/microphones/mic-session";
import { glassesDisplayLabel } from "./glasses-display-state";
import { glassesDisplayAsleep, glassesDisplayLabel, type GlassesDisplayState } from "./glasses-display-state";
import { PreviewDisplayTarget, type DisplayTarget } from "../native/preview-display";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
@@ -164,6 +168,9 @@ const SHELL_REFRESH_INTERVAL_MS = 60_000;
// schedules a refresh), so this just covers anything that slips through.
const PREVIEW_INTERVAL_MS = 1_000;
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
// voiceActivity holder for continuous capture (Transcribe, Calculator's
// continuous mode, the Teleprompter reader).
const CONTINUOUS_CAPTURE_HOLDER = "continuous-capture";
// The lock screen is a short notice, not something to read at leisure; it
// ignores the screen-timeout setting (including "never").
const LOCK_SCREEN_TIMEOUT_MS = 15_000;
@@ -186,24 +193,26 @@ const EVEN_APP_DETECTED_MESSAGE =
const LAUNCHABLE_APPS = ALL_APPS.filter((app) => app.showInLauncher !== false);
/**
* The global settings window viewport sizes depend on: the display mode,
* whether the app switcher takes width beside windows, height below them
* or neither (a popup), and the chrome rows each window loses to it: below
* them, whether the status bar joins the row (dropping the top bar); with a
* popup, whether the status bar keeps rows of its own and the window frame
* (where the status bar goes only moves windows).
* The global settings window viewport sizes depend on: the display mode's
* size (a band's position only moves windows), whether the app switcher
* takes width beside windows, height below them or neither (a popup), and
* the chrome rows each window loses to it: below them, whether the status
* bar joins the row (dropping the top bar); with a popup, whether the
* status bar keeps rows of its own and the window frame (where the status
* bar goes only moves windows).
*/
function viewportSizesKey(): string {
const position = appSwitcherPositionSetting.get();
const switcher = position === "bottom" ? `row|${statusBarPositionSetting.get()}`
: position === "popup" ? `popup|${statusBarVisibilitySetting.get()}|${windowBorderSetting.get()}`
: "strip";
return `${displayModeSetting.get()}|${switcher}`;
const layout = displayLayoutSetting.get();
return `${layout.startsWith("band-") ? "band" : layout}|${switcher}`;
}
/** The global settings that move window surfaces without resizing them. */
function windowPlacementKey(): string {
return `${verticalPositionSetting.get()}|${appSwitcherPositionSetting.get()}|${statusBarPositionSetting.get()}`;
return `${displayLayoutSetting.get()}|${appSwitcherPositionSetting.get()}|${statusBarPositionSetting.get()}`;
}
function createInitialDisplayPreview(): ImageSource | null {
@@ -276,6 +285,8 @@ class DashboardController {
private extractedFontsConfirmed = false;
private screenRecordingActive = false;
private displayPreview: ImageSource | null = createInitialDisplayPreview();
/** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */
private displayPreviewCrop: ScreenRect | null = null;
private silentMode = false;
// Brightness value last pushed to the glasses; see pushBrightness.
private lastPushedBrightness: string | null = null;
@@ -283,9 +294,6 @@ class DashboardController {
// drop-off right after a low reading can be explained as a flat battery.
private lastHeadsetBattery: number | null = null;
private readonly listeners = new Set<DashboardListener>();
// Set at connect time from the persisted flag; the one-time post-onboarding
// welcome sound plays on the first rendered frame (proof the session is warm).
private welcomeSoundArmed = false;
private communicator: FaceclawCommunicatorBridge | null = null;
// Headless stand-in for the compositor when no glasses are paired
@@ -343,6 +351,9 @@ class DashboardController {
private offFirmwareInfo: (() => void) | null = null;
private offVoiceStatus: (() => void) | null = null;
private offAndroidNotification: (() => void) | null = null;
private offAndroidNotificationRemoved: (() => void) | null = null;
/** New notifications wait here briefly before popping up (see NotificationPopupDelay). */
private readonly notificationPopups = new NotificationPopupDelay((notificationKey) => this.popUpAndroidNotification(notificationKey));
private lastInput = "waiting...";
private lastSys = "none yet";
private shellRenderInProgress = false;
@@ -389,8 +400,9 @@ class DashboardController {
endTextSettingEdit: () => this.endTextSettingEdit(),
startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => this.stopVoiceCapture(),
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
this.startContinuousVoiceCapture(client, options),
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
};
this.sharedActions = sharedActions;
@@ -431,6 +443,7 @@ class DashboardController {
},
openNotificationsWindow: () =>
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
onSwitcherBarHeightChanged: () => this.applySwitcherBarHeight(),
onScreenStateChanged: (on) => {
// Any wake of the regular UI replaces a showing Glanceboard.
if (on) this.glance.dismiss();
@@ -445,11 +458,9 @@ class DashboardController {
app.boot?.(this.buildAppContext(app));
}
timerEngine.onChange(() => this.refreshAlarmReliabilityWarning());
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => {
void this.handleAndroidNotificationPosted(notificationKey).catch((error) => {
this.appendLog(`notification wake failed: ${this.formatError(error)}`);
});
});
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => this.handleAndroidNotificationPosted(notificationKey));
this.offAndroidNotificationRemoved = onAndroidNotificationRemoved((notificationKey) => this.handleAndroidNotificationRemoved(notificationKey));
watchNotificationListenerBinding();
readActiveNotifications(ALL_NOTIFICATIONS);
// Settings toggled from the glasses can change what the phone UI shows
// (e.g. the text-setting editor), so re-emit the snapshot on any change.
@@ -459,15 +470,16 @@ class DashboardController {
this.pushFirmwareDebugFlags();
this.pushBrightness();
this.syncEvenHubScreenOffSetting();
// Display mode first: a reflow repositions every surface too, so the
// vertical-position check after it then has nothing left to do.
// Display mode size first: a reflow repositions every surface too, so
// the band-position check after it then has nothing left to do.
this.applyDisplayModeIfChanged();
this.applyVerticalPositionIfChanged();
this.applyAppLayoutsIfChanged();
this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged();
// A Preview color change should show on the mirror at once, not at the
// next composited frame (cheap: floor-limited, dedup'd on the Java side).
// A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on
// the Java side).
this.schedulePreviewUpdate();
});
// Connect to the external agent bridge at boot if configured; the
@@ -540,7 +552,7 @@ class DashboardController {
});
}
// Vertical-position changes (and moving the app switcher between the left
// Band position changes (and moving the app switcher between the left
// and right edges, or a popup switcher's status bar between the top and
// bottom) move every window surface and the chrome that aligns with them;
// the values are tracked so unrelated setting changes don't trigger a full
@@ -550,7 +562,7 @@ class DashboardController {
private lastViewportSizes = viewportSizesKey();
/**
* Display mode changed (Settings > Display, or the phone page's picker),
* Display mode size changed (Settings > Display, or the phone page's picker),
* or the app switcher or its chrome changed what windows lose to them
* (see viewportSizesKey): every window's viewport size changes. In-process windows re-measure in place; workers
* that support resizing receive the new viewport. Other worker windows are
@@ -592,12 +604,35 @@ class DashboardController {
});
}
/**
* The bottom switcher's layout changed its height (shell.ts
* onSwitcherBarHeightChanged): window bands move where the slack under
* them runs out, and full-height windows resize; the rest stay as they
* are. Windows that can't resize keep the surface they have, the switcher
* covering any of it that it now overlaps, rather than being relaunched
* for it.
*/
private applySwitcherBarHeight(): void {
void (async () => {
for (const window of Array.from(shell.getWindows())) {
const rect = JSON.stringify(appViewportRect(window.heightMode, window.appId));
if (!window.relayout || this.windowSurfaceRects.get(window.surfaceId) === rect) continue;
window.relayout();
await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode);
}
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
})().catch((error) => {
this.appendLog(`switcher height change failed: ${this.formatError(error)}`);
});
}
private lastAppLayouts = this.appLayouts();
private appLayouts(): Record<string, string> {
return {
navigate: `${navigateDisplayModeSetting.get()}:${navigateVerticalPositionSetting.get()}`,
terminal: `${terminalDisplayModeSetting.get()}:${terminalVerticalPositionSetting.get()}`,
navigate: navigateDisplayLayoutSetting.get(),
terminal: terminalDisplayLayoutSetting.get(),
};
}
@@ -1170,14 +1205,23 @@ class DashboardController {
/** What occupies the foreground-title line on the watch-style controls. */
glassesDisplayLabel(): string {
return glassesDisplayLabel({
return glassesDisplayLabel(this.glassesDisplayState());
}
/** Whether that line reads "Display off" (the watch face adds a wake hint under it). */
glassesDisplayAsleep(): boolean {
return glassesDisplayAsleep(this.glassesDisplayState());
}
private glassesDisplayState(): GlassesDisplayState {
return {
phase: this.phase,
silentMode: this.silentMode,
screenOn: shell.isScreenOn(),
battery: this.lastHeadsetBattery,
foregroundTitle: shell.getForegroundApp()?.title ?? null,
previewMode: this.isPreviewDisplayActive(),
});
};
}
/**
@@ -1439,7 +1483,6 @@ class DashboardController {
this.teardownPreviewDisplay();
this.lastInput = "waiting...";
this.lastSys = "none yet";
this.welcomeSoundArmed = isWelcomeSoundPending();
this.firmwareWarningMessage = "";
this.glassesWorn = null;
this.wakeAfterChargingReconnect = false;
@@ -1614,11 +1657,12 @@ class DashboardController {
// parked by the previous session (or by an EvenHub suspend) resumes.
this.resumeVoiceCapture();
// A rendered frame means the session is warmed up (fixedLayoutCreated),
// so the buzzer won't be dropped. Play the one-time welcome sound now.
if (this.welcomeSoundArmed) {
this.welcomeSoundArmed = false;
setWelcomeSoundPending(false);
void this.playWelcomeSound();
// so the onboarding's welcome jingle won't be dropped. Open the
// on-glasses onboarding if this version hasn't been through it.
if (claimGlassesOnboardingAutoLaunch()) {
this.launchApp(ONBOARDING_APP_ID).catch((error) => {
this.appendLog(`onboarding launch failed: ${this.formatError(error)}`);
});
}
}
});
@@ -1731,6 +1775,10 @@ class DashboardController {
await nightscoutBridge.stop().catch(() => {});
weatherBridge.setSessionActive(false);
voiceControlBridge.stop();
// The parked continuous capture went with it; the next session starts
// it afresh for any window still asking.
this.continuousAttemptedOn = undefined;
this.syncContinuousClaim();
if (communicator) {
await communicator.setFaceclawWakeLeaseEnabled(false).catch(() => false);
await communicator.close().catch(() => {});
@@ -1966,13 +2014,14 @@ class DashboardController {
await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch");
return;
}
const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH);
const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT);
const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny);
const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH);
const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT);
const foreground = shell.foregroundWindow();
const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId);
this.appendLog(`mirror tap at ${x},${y}`);
if (!shell.hasOverlay() && shell.focusNotificationAt(x, y)) {
this.appendLog("mirror tap: notification icon");
if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) {
this.appendLog("mirror tap: status bar item");
this.requestShellRender();
return;
}
@@ -2054,12 +2103,73 @@ class DashboardController {
return voiceControlBridge.stopPushToTalk();
}
private startContinuousVoiceCapture(): void {
void this.beginVoiceCapture("continuous");
// Windows that want continuous capture, and what each needs of it. They
// share one stream, which runs while any of them wants it: one window
// letting go must not cut off another.
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>();
// The connection a continuous-capture start was last tried on (undefined:
// none since the last teardown), so a window waiting on the glasses is
// retried once per new session rather than on every frame.
private continuousAttemptedOn: FaceclawCommunicatorBridge | null | undefined = undefined;
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options);
this.retargetContinuousCapture();
this.ensureContinuousCapture(true);
}
private stopContinuousVoiceCapture(): void {
voiceControlBridge.stopContinuousCapture();
private stopContinuousVoiceCapture(client: string): void {
if (!this.continuousClients.delete(client)) return;
if (this.continuousClients.size > 0) {
// The window that needed its own provider may be the one that left.
this.retargetContinuousCapture();
return;
}
try {
voiceControlBridge.stopContinuousCapture();
} finally {
this.syncContinuousClaim();
}
}
/**
* Start continuous capture for the windows asking for it, unless it is
* running or parked across a session outage. A window restored at launch
* asks before the glasses connect, so this is retried on each new session
* (from resumeVoiceCapture); `force` retries on the same one.
*/
private ensureContinuousCapture(force = false): void {
if (this.continuousClients.size === 0 || voiceControlBridge.holdsCapture("continuous")) return;
if (!force && this.continuousAttemptedOn === this.communicator) return;
this.continuousAttemptedOn = this.communicator;
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
}
/** Move a running continuous capture onto the provider its windows now call for. */
private retargetContinuousCapture(): void {
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
}
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
private multilingualProvider(): VoiceProvider | undefined {
for (const options of this.continuousClients.values()) {
if (options.multilingualProvider) return options.multilingualProvider;
}
return undefined;
}
/**
* Continuous capture's claim on the mic (the Microphones app and EvenHub
* mic apps stand down while it is held) follows whether the capture really
* holds the mic, not merely whether a window asked: a start that never
* happened (no glasses yet, permission refused) must not idle every other
* mic user.
*/
private syncContinuousClaim(): void {
voiceActivity.setActive(
this.continuousClients.size > 0 && voiceControlBridge.holdsCapture("continuous"),
CONTINUOUS_CAPTURE_HOLDER,
);
}
/**
@@ -2068,13 +2178,21 @@ class DashboardController {
* stands in for the G2 mic.
*/
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
// A window that needs a multilingual provider sets it for every capture
// while it is open, so push-to-talk still shares its stream.
const multilingual = this.multilingualProvider();
return {
communicator: communicator?.getNativeCommunicator() ?? null,
usePhoneMic: communicator === null,
provider: voiceProviderSetting.get(),
provider: multilingual ?? voiceProviderSetting.get(),
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(),
sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(),
// Whisper detects the language itself when the speech may be in any.
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
// "My voice only" (Microphones app): verify command utterances
// against the enrolled wearer voice-print; non-matching speakers'
@@ -2087,15 +2205,19 @@ class DashboardController {
}
/**
* Restart the mic for a capture that a previous session left holding. Cheap
* to call per frame: it does nothing unless one is actually parked.
* Restart the mic for a capture that a previous session left holding, and
* start a continuous capture asked for before this session. Cheap to call
* per frame: it does nothing unless one is actually waiting.
*/
private resumeVoiceCapture(): void {
if (!voiceControlBridge.hasSuspendedCapture()) return;
const communicator = this.communicator;
if (!communicator || this.phase !== "connected") return;
this.appendLog("resuming voice capture on the new glasses session");
voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator));
if (voiceControlBridge.hasSuspendedCapture()) {
this.appendLog("resuming voice capture on the new glasses session");
voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator));
}
// A continuous capture asked for before this session existed.
this.ensureContinuousCapture();
}
/**
@@ -2140,6 +2262,11 @@ class DashboardController {
if (kind === "ptt") {
voiceControlBridge.startPushToTalk(options);
} else {
// Every window may have let go while permission was pending.
if (this.continuousClients.size === 0) return;
// Claimed before the capture starts so the Microphones app and
// EvenHub mic apps stand down first, as they do for the voice dialog.
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER);
voiceControlBridge.startContinuousCapture(options);
}
})
@@ -2294,7 +2421,7 @@ class DashboardController {
let shellPreWoke = false;
if (wakewordShouldWake || displayShouldWake) {
if (!shell.isScreenOn()) {
shellPreWoke = shell.wake("sidebar");
shellPreWoke = shell.wake(displayShouldWake ? shell.userWakeFocus() : "sidebar");
}
if (shellPreWoke || this.evenHubSessionSuspended || this.evenHubResumePromise) {
const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () =>
@@ -2394,7 +2521,7 @@ class DashboardController {
* An in-process singleton app's window, opened in the background (neither
* foregrounded nor focused) unless it is open already: for the shell to
* bring forward itself (the Notifications window under a notification
* selected in the switcher). The bridge runs calls in order, so the
* selected in the switcher, or an app widget's window under the widget). The bridge runs calls in order, so the
* shell may foreground it at once: its surface's visibility is settled
* when the configure call is queued, ahead of any frame.
*/
@@ -2483,6 +2610,8 @@ class DashboardController {
launchApp: (appId, params) => this.launchApp(appId, params),
uninstallApp: (appId) => this.uninstallApp(appId),
launchInProcessApp: (windowId, surfaceId, create) => this.launchInProcessApp(windowId, surfaceId, create),
openInProcessAppInBackground: (windowId, surfaceId, create) =>
this.openInProcessAppInBackground(windowId, surfaceId, create),
ensureWorkerHost: (createWorker) => this.ensureWorkerHost(app.appId, createWorker),
submitWindowFrame: (surfaceId, planes, paintMs, frameId) =>
this.submitWindowFrame(surfaceId, planes, paintMs, frameId),
@@ -2600,20 +2729,22 @@ class DashboardController {
target: DisplayTarget | null = this.display,
): Promise<void> {
if (!target) return;
await target.configureSurface(surfaceId, {
...appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId),
zOrder: 0,
transparency: "opaque",
});
const rect = appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId);
this.windowSurfaceRects.set(surfaceId, JSON.stringify(rect));
await target.configureSurface(surfaceId, { ...rect, zOrder: 0, transparency: "opaque" });
await target.setSurfaceVisible(surfaceId, typeof visible === "function" ? visible() : visible);
}
/** Each window surface's rect as last configured (applySwitcherBarHeight skips those that stay). */
private readonly windowSurfaceRects = new Map<string, string>();
/** A late answer to "is this window in front?", for configureWindowSurface. */
private isForegroundWindow(windowId: string): () => boolean {
return () => shell.foregroundWindow()?.windowId === windowId;
}
private removeWindowSurface(surfaceId: string): void {
this.windowSurfaceRects.delete(surfaceId);
const display = this.display;
if (!display) return;
void display.removeSurface(surfaceId).catch((error) => {
@@ -2751,23 +2882,47 @@ class DashboardController {
this.schedulePreviewUpdate();
}
private async handleAndroidNotificationPosted(notificationKey: string): Promise<void> {
private handleAndroidNotificationPosted(notificationKey: string): void {
// The tray shows the notification at once; its popup waits out a short
// delay, so one withdrawn right away (see NotificationPopupDelay) never shows.
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) {
this.requestShellRender();
return;
if (notification && shouldShowNotificationOnGlasses(notification.packageName)) {
this.notificationPopups.post(notificationKey, notificationPopupDelayMs(notification));
}
// New notifications open a shell modal over the app viewport; if the
// screen was off, wake for it and go back to sleep when it is closed.
// Waking while already on would steal focus, so only wake from sleep.
const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar");
if (wokeScreen) {
this.appendLog("android notification woke the screen");
}
shell.openNotificationModal(notificationKey, wokeScreen);
this.requestShellRender();
}
private handleAndroidNotificationRemoved(notificationKey: string): void {
if (this.notificationPopups.withdraw(notificationKey)) {
this.appendLog(`android notification ${notificationKey} withdrawn before its popup`);
}
try {
shell.notificationRemoved(notificationKey);
} catch (error) {
this.appendLog(`closing a removed notification's popup failed: ${this.formatError(error)}`);
}
this.requestShellRender();
}
/** A posted notification waited out its popup delay: show it, if it is still there. */
private popUpAndroidNotification(notificationKey: string): void {
try {
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) return;
// New notifications open a shell modal over the app viewport; if the
// screen was off, wake for it and go back to sleep when it is closed.
// Waking while already on would steal focus, so only wake from sleep.
const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar");
if (wokeScreen) {
this.appendLog("android notification woke the screen");
}
shell.openNotificationModal(notificationKey, wokeScreen);
this.requestShellRender();
} catch (error) {
this.appendLog(`notification popup failed: ${this.formatError(error)}`);
}
}
private async playBuzzerSequence(payload: Uint8Array): Promise<void> {
if (this.phase !== "connected" || !this.communicator) {
return;
@@ -2775,21 +2930,6 @@ class DashboardController {
await this.communicator.playBuzzerSequence(payload);
}
/** One-time celebratory jingle on the first connection after onboarding. */
private async playWelcomeSound(): Promise<void> {
const effect = findSoundEffect("questcomplete");
if (!effect) return;
try {
await playSoundEffect(
effect,
(payload) => this.playBuzzerSequence(payload),
(ms) => new Promise((resolve) => setTimeout(resolve, ms)),
);
} catch (error) {
this.appendLog(`welcome sound failed: ${this.formatError(error)}`);
}
}
private clearDashboardTimer(): void {
this.cancelEvenHubSuspendTimer();
if (this.shellRefreshTimer) {
@@ -2933,8 +3073,10 @@ class DashboardController {
const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager;
if (!power?.isInteractive()) return;
}
const preview = display.getCompositePreview(previewColorSetting.get() === "green");
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect());
const preview = display.getCompositePreview(previewColorSetting.get() === "green", crop);
if (preview) {
this.displayPreviewCrop = crop;
this.setDisplayPreview(preview);
}
}
+9 -1
View File
@@ -12,7 +12,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire";
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
import * as frameTimings from "../native/frame-timings";
import { type DisplayTarget } from "../native/preview-display";
import { minWindowTop } from "../ui/shell/geometry";
import { minWindowTop, type ScreenRect } from "../ui/shell/geometry";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition";
import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state";
@@ -59,6 +59,8 @@ export class GlanceHost {
private rendering = false;
private renderPending = false;
private nextRenderWantsFreshData = false;
/** Where the last render put the board on screen. */
private boardRect: ScreenRect | null = null;
constructor(private readonly options: GlanceHostOptions) {}
@@ -66,6 +68,11 @@ export class GlanceHost {
return this.state.visible;
}
/** The board's area of the screen while it shows (for the phone mirror's Scaled preview), else null. */
screenRect(): ScreenRect | null {
return this.state.visible ? this.boardRect : null;
}
/** Whether sleep-time gestures should reach the board at all. */
isEnabled(): boolean {
return this.options.getProvider()?.isEnabled() ?? false;
@@ -249,6 +256,7 @@ export class GlanceHost {
const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0);
const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop()));
boardImage.composeInto(full, x, y);
this.boardRect = { x, y, width: boardImage.width, height: boardImage.height };
return full;
}),
);
+14 -1
View File
@@ -23,12 +23,25 @@ export function glassesDisplayLabel(state: GlassesDisplayState): string {
if (state.silentMode) {
return "Silent mode";
}
if (!state.screenOn) {
if (glassesDisplayAsleep(state)) {
return "Display off";
}
return state.foregroundTitle || "Launcher";
}
/**
* True when the label reads "Display off": the screen is dark but the glasses
* (or the preview standing in for them) would light it on a wake gesture.
*/
export function glassesDisplayAsleep(state: GlassesDisplayState): boolean {
return (
state.phase !== "charging" &&
(state.phase === "connected" || state.previewMode) &&
!state.silentMode &&
!state.screenOn
);
}
export function chargingDisplayLabel(battery: number | null): string {
return validBatteryLevel(battery) ? `Charging · G2 ${battery}%` : "Charging";
}
+202 -37
View File
@@ -7,7 +7,7 @@ import { launcherEntries } from '../apps/launcher'
import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
import { isInstalledPackagePresent } from '../apps/evenhub/updates'
import { openEvenHubStoreForPackage } from '../apps/evenhub'
import { launchInstalledPackage, closeRunningPackage } from '../apps/evenhub/manager'
import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager'
import { registerSystemTools } from '../assistant/system-tools'
import { registerWindowTools } from '../assistant/window-tools'
import { registerNavigateTools } from '../assistant/navigate-tools'
@@ -16,6 +16,7 @@ import { IosNavigationSensors } from '../native/ios-navigation-sensors'
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
import { iosVoiceInput } from '../native/ios-voice-input'
import { voiceActivity } from '../ui/shell/voice-activity'
import { nightscoutBridge } from '../native/nightscout-bridge'
import { weatherBridge } from '../native/weather'
import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios'
@@ -25,6 +26,7 @@ import { GlanceHost, type GlanceDisplay } from './glance-host'
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen'
import { OsEventTypeList } from './events'
import { loadDeviceAddresses } from './device-addresses'
import { updateGlassesPresence } from './glasses-presence'
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
import { deviceAddressError } from './ios-peripheral-identity'
@@ -39,18 +41,23 @@ import { flattenPlanesWithDraws, planesFingerprint, type Plane } from '../graphi
import { prepareFrameDraws } from '../graphics/glyph-wire'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
import { noopLayerActions, type LayerActions } from '../ui/layers'
import { noopLayerActions, type ContinuousCaptureOptions, type LayerActions } from '../ui/layers'
import { TextViewerLayer } from '../apps/files/text-viewer'
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry'
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry'
import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings'
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { isWelcomeSoundPending, setWelcomeSoundPending } from '../phone-ui/onboarding-state'
import { findSoundEffect, playSoundEffect } from '../ui/sound-effects'
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
const SHELL_SURFACE_ID = 'shell'
// voiceActivity holder for continuous capture (Transcribe), as on Android.
const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture'
const SHELL_SURFACE_Z_ORDER = 1
const LOCK_SURFACE_Z_ORDER = 1000
/** The shell render loop waits this long for the glasses to take a frame before moving on. */
@@ -114,6 +121,8 @@ export class IosPreviewController {
appendLog: message => this.logBluetooth(message),
})
private previewTimer: ReturnType<typeof setTimeout> | null = null
/** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */
private previewCrop: ScreenRect | null = null
private clockTimer: ReturnType<typeof setInterval> | null = null
private offSettings: (() => void) | null = null
private lastBrightness: string | null = null
@@ -126,6 +135,8 @@ export class IosPreviewController {
private nextWorkerFrame = 0
private inputQueue: Promise<void> = Promise.resolve()
private lastLayout = ''
/** The phone-UI button last pushed to the page (onEvenHubPhoneUiChanged). */
private evenHubPhoneUi: PhoneUiButton | null = null
private readonly appHosts = new Map<string, WorkerAppHost>()
private readonly inProcessApps = new Map<string, InProcessWindow>()
private readonly batteryObservers: any[] = []
@@ -137,6 +148,20 @@ export class IosPreviewController {
private acknowledgedFrames = 0
private logLines: string[] = []
private logTimer: ReturnType<typeof setTimeout> | null = null
// Windows that want continuous capture, and what each needs of it (a
// recording, a multilingual provider). They share one capture, which runs
// while any of them wants it.
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
// transcriptionKey of the running continuous capture's provider; one that no
// longer matches what its windows need is drained and restarted.
private continuousTranscription = ''
// Continuous capture should (re)start once audio is available: a window
// asked, the glasses connected or disconnected, or the glasses lock or the
// app lifecycle cut it short. A capture that fails doesn't set it, so a
// broken start is retried on the next such change rather than in a loop.
private continuousPending = false
private continuousCaptureToken = 0
private sessionConnected = false
private readonly actions: LayerActions = {
...noopLayerActions,
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
@@ -144,7 +169,8 @@ export class IosPreviewController {
disconnect: () => this.disconnect(),
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options),
stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client),
startTextSettingEdit: async setting => { await this.editSetting(setting) },
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
@@ -160,7 +186,8 @@ export class IosPreviewController {
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
private readonly onError: (message: string) => void,
private readonly onConnectionState: (state: SessionState) => void = () => {},
private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {}) {
private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {},
private readonly onEvenHubPhoneUiChanged: (button: PhoneUiButton | null) => void = () => {}) {
this.ensurePreviewDisplay()
shell.configure({
actions: this.actions,
@@ -175,7 +202,12 @@ export class IosPreviewController {
prepareVoiceCapture: () => this.prepareVoiceCapture(),
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
onSwitcherBarHeightChanged: () => {
this.relayout()
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
},
onScreenStateChanged: on => {
if (on) this.glance.dismiss()
void this.display?.setScreenBlanked(!on).catch(error => this.fail(error))
@@ -218,11 +250,15 @@ export class IosPreviewController {
assistantAvailable: () => shell.isAssistantAvailable(),
assistant: text => shell.sendToAssistant(text),
})
// The header's EvenHub phone-UI button hides while that UI is showing.
onPhoneShownChanged(() => this.syncEvenHubPhoneUi())
registerSystemTools()
registerWindowTools({ apps: ALL_APPS.filter(app => !iosAppUnavailableReason(app.appId)),
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
registerNavigateTools(id => this.launchApp(id))
registerRoamTools(id => this.launchApp(id))
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
}
@@ -277,16 +313,35 @@ export class IosPreviewController {
this.relayout()
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
this.syncEvenHubPhoneUi()
this.syncContinuousCapture()
}
pause(): void {
if (!this.active) return
this.active = false
if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true
iosVoiceInput.stopPhoneCapture()
// The phone preview is hidden; the glasses still own their screen and input.
this.syncRuntime()
this.logBluetooth(`Phone background; glasses ${this.state.phase}`)
this.flushBluetoothLog()
}
/** Overlay an EvenHub app's phone UI (the header's app-icon button). */
showEvenHubPhoneUi(windowId: string): void {
showOnPhone(windowId)
}
/**
* Tell the page when the foreground window's EvenHub phone-UI button
* changes. A paused controller skips this and catches up on resume.
*/
private syncEvenHubPhoneUi(): void {
if (!this.active) return
const button = phoneUiButton()
const current = this.evenHubPhoneUi
if (current?.windowId === button?.windowId && current?.icon === button?.icon) return
this.evenHubPhoneUi = button
this.onEvenHubPhoneUiChanged(button)
}
private get runtimeNeeded(): boolean {
return this.active || ['connected', 'connecting', 'retrying', 'disconnecting'].includes(this.state.phase)
}
@@ -373,6 +428,7 @@ export class IosPreviewController {
// a session's first snapshot) wakes the screen, to the lock notice if locked.
const putOn = wearing && this.glassesWorn === false
this.glassesWorn = wearing
this.syncPresence()
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
if (putOn && this.runtimeRunning && !shell.isScreenOn()) shell.wake('sidebar')
@@ -382,8 +438,10 @@ export class IosPreviewController {
this.glassesLocked = locked
if (locked) {
this.glance.dismiss()
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.')
}
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
'Glasses locked. Transcription resumes when you unlock your phone.')
} else this.syncContinuousCapture()
void (async () => {
await this.displayReady
const display = this.display
@@ -418,10 +476,11 @@ export class IosPreviewController {
}
}
/** Submit a painted frame for an in-process window (e.g. the launcher). */
private async submit(id: string, planes: Plane[]): Promise<void> {
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
await this.displayReady
const display = this.display
if (!display) return
// The window can close (removing its surface) while this frame waits.
if (!display || !live()) return
const fingerprint = planesFingerprint(planes)
const { image, draws } = flattenPlanesWithDraws(planes)
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
@@ -481,8 +540,10 @@ export class IosPreviewController {
this.previewTimer = null
if (!this.active) return
try {
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green')
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect())
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop)
if (!image) return
this.previewCrop = crop
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
} catch (error) { this.fail(error) }
@@ -536,7 +597,8 @@ export class IosPreviewController {
await shell.receiveInput(event)
this.requestShellRender()
}
private async mirrorTap(nx: number, ny: number): Promise<void> {
private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise<void> {
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
const window = shell.foregroundWindow()
@@ -610,7 +672,7 @@ export class IosPreviewController {
// The explicit render below supplies a fresh frame after configuration.
let surfaceReady = false
const app = create({
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(),
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
reconfigureSurface: () => {
@@ -657,11 +719,6 @@ export class IosPreviewController {
this.appHosts.set(appId, host)
return host
}
cycleDisplayMode(): void {
const values = DISPLAY_MODE_VALUES
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
}
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
get connectionState(): SessionState | null { return this.sessionCreated ? this.state : null }
get connectionDetails(): string {
const state = this.connectionState
@@ -672,13 +729,19 @@ export class IosPreviewController {
this.logBluetooth(status)
this.emitState()
}
/** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */
private syncPresence(): void {
updateGlassesPresence({ connected: this.state.phase === 'connected', worn: this.glassesWorn, charging: this.state.charging ?? false })
}
private emitState(): void {
if (this.state.phase !== 'connected') resetRingInputFilter()
this.pushBrightness()
this.maybePlayWelcomeSound(this.state)
this.maybeLaunchOnboarding(this.state)
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
this.syncContinuousSession()
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
this.syncPresence()
this.syncRuntime()
if (this.active) this.onConnectionState({ ...this.state })
this.requestShellRender()
@@ -695,7 +758,10 @@ export class IosPreviewController {
this.firmwareCompatible = false
this.update('connecting', 'Finding configured devices…')
let identifiers
try { identifiers = await resolveIosPeripherals(addresses) }
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
// fails and retries every few seconds, starving the glasses link.
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
catch (failure) {
this.update('error', failure instanceof Error ? failure.message : String(failure))
return
@@ -763,7 +829,7 @@ export class IosPreviewController {
if (this.communicator !== communicator) return
this.state = { ...this.state, frames: this.state.frames + 1 }
this.onActivity()
this.maybePlayWelcomeSound(this.state)
this.maybeLaunchOnboarding(this.state)
if (this.active) this.onConnectionState({ ...this.state })
this.schedulePreviewUpdate()
}),
@@ -871,23 +937,20 @@ export class IosPreviewController {
this.glance.reset()
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
}
private maybePlayWelcomeSound(state: SessionState): void {
private maybeLaunchOnboarding(state: SessionState): void {
const firstNewFrame = state.frames > this.acknowledgedFrames
this.acknowledgedFrames = state.frames
// Match Android: wait for an acknowledged frame before consuming the jingle.
if (!firstNewFrame || state.phase !== 'connected' || !isWelcomeSoundPending()) return
setWelcomeSoundPending(false)
const effect = findSoundEffect('questcomplete')
if (effect) void playSoundEffect(effect, payload => this.actions.playBuzzerSequence(payload),
ms => new Promise(resolve => setTimeout(resolve, ms)))
.catch(error => this.logBluetooth(`Welcome sound failed: ${error}`))
// Match Android: wait for an acknowledged frame (a warm session, so the
// onboarding's welcome jingle isn't dropped) before opening it.
if (!firstNewFrame || state.phase !== 'connected' || !claimGlassesOnboardingAutoLaunch()) return
void this.launchApp(ONBOARDING_APP_ID)
}
startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() }
private async prepareVoiceCapture(): Promise<boolean> {
if (this.glassesLocked) return false
const usePhoneMic = this.state.phase !== 'connected'
if (usePhoneMic && !this.active) return false
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic)
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider())
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active)
}
@@ -895,14 +958,116 @@ export class IosPreviewController {
if (this.glassesLocked) return
const log = (message: string) => this.logBluetooth(message)
const communicator = this.communicator
const voice = this.voiceProvider()
if (communicator && this.state.phase === 'connected') {
await iosVoiceInput.startGlassesCapture(communicator, log, endpointing)
await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing)
} else if (this.active) {
// Recheck microphone permission if the glasses disconnected after prepare.
if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return
await iosVoiceInput.startPhoneCapture(log, endpointing)
if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
}
}
/**
* Continuous capture on behalf of `client` (one id per window: Transcribe,
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
* Transcribe audio is on.
*/
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options)
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
this.continuousClients.delete(client)
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
return
}
this.syncContinuousRecording()
// A listening capture is shared as it is; otherwise start one (after any drain).
if (!iosVoiceInput.continuousSource) this.continuousPending = true
this.syncContinuousCapture()
}
private stopContinuousVoiceCapture(client: string): void {
if (!this.continuousClients.delete(client)) return
this.syncContinuousRecording()
// The window that needed its own provider may be the one that left.
if (this.continuousClients.size > 0) { this.syncContinuousCapture(); return }
// A start still awaiting permission or the mic backs off.
++this.continuousCaptureToken
this.continuousPending = false
this.syncContinuousClaim()
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
}
/** Connecting or losing the glasses moves continuous capture to the new audio source. */
private syncContinuousSession(): void {
const connected = this.state.phase === 'connected'
if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true
this.sessionConnected = connected
this.syncContinuousCapture()
}
/**
* Keep continuous capture running while a window wants it: start a pending
* one once audio is available, and move a phone capture to the glasses once
* they connect. A glasses session supplies the audio, even while it
* reconnects (as on Android); the phone mic stands in only without one, while
* Faceclaw is in the foreground. Cheap enough to call on every state change.
*/
private syncContinuousCapture(): void {
this.syncContinuousClaim()
if (this.continuousClients.size === 0 || this.glassesLocked) return
const glassesReady = !!this.communicator && this.state.phase === 'connected'
if (iosVoiceInput.continuousActive) {
// Draining or listening already. A phone capture from before the glasses
// connected is drained here, and its end restarts it on the glasses; so
// is one transcribing with another provider than its windows now need.
const retarget = this.continuousTranscription !== transcriptionKey(this.voiceProvider())
if (iosVoiceInput.continuousSource && (retarget || glassesReady && iosVoiceInput.continuousSource === 'phone')) {
this.continuousPending = true
void iosVoiceInput.stopContinuousCapture()
}
return
}
if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return
this.continuousPending = false
void this.beginContinuousCapture()
}
private async beginContinuousCapture(): Promise<void> {
const voice = this.voiceProvider()
if (!iosVoiceInput.supportsContinuous(voice)) return
this.continuousTranscription = transcriptionKey(voice)
// A stop during an await below bumps the token; this start then backs off.
const token = ++this.continuousCaptureToken
const current = () => token === this.continuousCaptureToken
const log = (message: string) => this.logBluetooth(message)
// Claimed before the capture starts so other mic users stand down first, as on Android.
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
const communicator = this.communicator
if (communicator) {
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true)
} else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) {
await iosVoiceInput.startPhoneCapture(voice, log, false, true)
}
if (current()) this.syncContinuousClaim()
}
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
private syncContinuousClaim(): void {
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
}
/** Record to the first client's requested name while Save Transcribe audio is on. */
private syncContinuousRecording(): void {
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
iosVoiceInput.recordTo(name)
}
/**
* The transcription provider setting and the keys the cloud ones need. A
* window that needs a multilingual provider (Translate) sets it for every
* capture while it is open, with Whisper left to detect the language.
*/
private voiceProvider(): VoiceProviderSelection {
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
}
private logBluetooth(message: string): void {
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
+33
View File
@@ -0,0 +1,33 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image";
import { previewScalingSetting } from "../ui/dashboard-settings";
import { phonePreviewRect, usedScreenRect, type ScreenRect } from "../ui/shell/geometry";
import { shell } from "../ui/shell/shell";
/**
* The screen area the phone's mirror shows (Settings > Phone display >
* Preview scaling), or null for the whole panel. Scaled zooms to the
* Glanceboard while it shows, else to the area the shell uses around the
* foreground window, centred at the panel's aspect. A locked display gets
* the whole panel: its message box is centred in the panel, not the band.
*/
export function phonePreviewCrop(locked: boolean, glanceRect: ScreenRect | null): ScreenRect | null {
if (locked || previewScalingSetting.get() === "raw") return null;
const window = shell.foregroundWindow();
const crop = phonePreviewRect(glanceRect ?? usedScreenRect(window?.heightMode ?? "min", window?.appId));
const wholePanel = crop.x === 0 && crop.y === 0 && crop.width === G2_LENS_WIDTH && crop.height === G2_LENS_HEIGHT;
return wholePanel ? null : crop;
}
/**
* A point on the mirror, as fractions of the mirror's width and height,
* as fractions of the panel's, given the crop the mirror is showing.
* Points in the black margins past the panel's edges come out below 0 or
* above 1.
*/
export function mirrorToPanelFraction(crop: ScreenRect | null, nx: number, ny: number): { nx: number; ny: number } {
if (!crop) return { nx, ny };
return {
nx: (crop.x + nx * crop.width) / G2_LENS_WIDTH,
ny: (crop.y + ny * crop.height) / G2_LENS_HEIGHT,
};
}
+3 -15
View File
@@ -23,12 +23,11 @@
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the
* message shapes are documented in wear/PROTOCOL.md.
*/
import { flagAnalyticsEvent } from "../analytics/analytics";
import { type AppDefinition } from "../apps/app-definition";
import { getInstalledEvenHubApps, installedEvenHubAppId } from "../apps/evenhub/installed-apps";
import { WEAR_PATHS, wearBridge, type WearMessage } from "../native/wear-bridge";
import {
DISPLAY_MODE_VALUES,
displayModeSetting,
onAnySettingChanged,
watchCanUnlockSetting,
watchCrownClockwiseNextSetting,
@@ -87,7 +86,6 @@ const COMMANDS = [
"connect",
"disconnect",
"close-assistant",
"display-mode",
] as const;
type WearCommand = (typeof COMMANDS)[number];
@@ -167,6 +165,7 @@ export class WearRemote {
// with it (no placeholder).
if (changed) this.watchBatteryPollsUnanswered = 0;
if (connection.reachable) {
flagAnalyticsEvent("minimal", "device.wear.reachable");
this.publishNow(true);
this.pollWatchBattery(true);
} else if (changed) {
@@ -275,7 +274,6 @@ export class WearRemote {
acceptsText: typeof window.receiveTextInput === "function",
})),
apps: this.listApps(),
displayMode: displayModeSetting.get(),
remoteEnabled: watchRemoteEnabledSetting.get(),
crownClockwiseNext: watchCrownClockwiseNextSetting.get(),
canUnlock: watchCanUnlockSetting.get(),
@@ -326,6 +324,7 @@ export class WearRemote {
return;
}
flagAnalyticsEvent("minimal", "device.wear.used");
switch (message.path) {
case WEAR_PATHS.input:
await this.handleInput(message.payload, ack);
@@ -519,17 +518,6 @@ export class WearRemote {
// but tell the watch which of the two happened.
ack(true, shell.closeAssistant() ? "" : "The assistant was not open.");
return;
case "display-mode": {
const value = String(payload.value ?? "");
if (!(DISPLAY_MODE_VALUES as readonly string[]).includes(value)) {
ack(false, `Unknown display mode: ${value}`);
return;
}
displayModeSetting.set(value as (typeof DISPLAY_MODE_VALUES)[number]);
this.schedulePublish();
ack(true, `Display mode: ${value}`);
return;
}
default:
ack(false, `Unhandled command: ${command}`);
}