Merge branch 'main' into glanceboard-tilt-angle-and-glance-down

This commit is contained in:
James Babcock committed 2026-11-15 22:38:39 -08:00
commit 37d81d0308
416 files changed
+34629 -3885

No files matched your search

+236 -65
View File
@@ -16,7 +16,6 @@ import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout-bridge";
import { weatherBridge } from "../native/weather";
import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, onAndroidNotificationRemoved, readActiveNotifications } from "../native/notification-icons";
import { shouldShowNotificationOnGlasses } from "../native/notification-sources";
import { watchNotificationListenerBinding } from "../native/notification-access";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { defaultRingName, setGlassesRingLink } from "../native/ring-status";
@@ -26,6 +25,8 @@ import { GLANCE_TILT_MAX_MS, glanceTiltMaxMs } from "./glance-state";
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from "./firmware-compat";
import { keyboardService } from "./keyboard/service";
import type { KeyEvent } from "./keyboard/keys";
import { hasExtractedEvenHubFonts } from "./firmware-builder";
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy";
import { WearRemote, type WearRemoteInputKind } from "./wear-remote";
@@ -57,9 +58,9 @@ import { type GlanceEvent } from "./glance-state";
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { VoiceReplay, voiceControlBridge } from "../native/voice-control";
import { voiceActivity } from "../ui/shell/voice-activity";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { GrayImage } from "../graphics/image";
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
import { prepareFrameDraws } from "../graphics/glyph-wire";
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen";
@@ -78,10 +79,11 @@ import { ALL_APPS } from "../apps/all-apps";
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, windowEdges, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { sameEdges } from "../ui/viewport";
import { type ContinuousCaptureOptions, type LayerActions, type MicCount, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -101,6 +103,8 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
import { wearerVerificationOptions } from "../apps/microphones/speakers";
import { micSession } from "../apps/microphones/mic-session";
import { micArrayCaptureConfig, micControlSupported } from "../apps/microphones/mic-control";
import { encodeMicControl } from "../apps/microphones/mic-protocol";
import {
chargingPreviewMessage,
glassesDisplayAsleep,
@@ -114,8 +118,6 @@ type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging"
export type DashboardSnapshot = {
phase: ConnectionPhase;
/** Last-known battery levels: the glasses, their charging case, the R1 ring and a Wear OS watch. */
batteries: ReturnType<typeof shell.getBatteryLevels>;
status: string;
displayPreview: ImageSource | null;
/**
@@ -158,7 +160,7 @@ export type DashboardSnapshot = {
previewMode: boolean;
/**
* The foreground glasses window's EvenHub phone UI, when it can be opened
* (the action bar's app-icon button); null otherwise.
* (the app-icon button under the mirror); null otherwise.
*/
evenHubPhoneUi: PhoneUiButton | null;
};
@@ -198,25 +200,27 @@ const LOW_BATTERY_PERCENT = 5;
const EVEN_APP_DETECTED_MESSAGE =
"The Even Realities app appears to be running. If Faceclaw has trouble connecting, open its app settings and force stop it.";
// The launcher grid's app list; also fixes the app ids apps.launch accepts.
const LAUNCHABLE_APPS = ALL_APPS.filter((app) => app.showInLauncher !== false);
// The launcher grid's app list for the paired glasses; also fixes the app ids apps.launch accepts.
const launchableApps = () => ALL_APPS.filter((app) => app.showInLauncher !== false && app.isSupported?.() !== false);
/**
* The global settings window viewport sizes depend on: the display mode's
* size (a band's position only moves windows), whether the app switcher
* The global settings window viewport sizes depend on: the paired glasses'
* screen, the display mode's size (a band's position only moves windows), whether the app switcher
* takes width beside windows, height below them or neither (a popup), and
* the chrome rows each window loses to it: below them, whether the status
* bar joins the row (dropping the top bar); with a popup, whether the
* status bar keeps rows of its own and the window frame (where the status
* bar goes only moves windows).
* bar joins the row (dropping the top bar, for the window frame's top row
* if it has a frame); with a popup, whether the status bar keeps rows of its
* own and the window frame (where the status bar goes only moves windows).
*/
function viewportSizesKey(): string {
const position = appSwitcherPositionSetting.get();
const switcher = position === "bottom" ? `row|${statusBarPositionSetting.get()}`
: position === "popup" ? `popup|${statusBarVisibilitySetting.get()}|${windowBorderSetting.get()}`
const statusBar = statusBarPositionSetting.get();
const framed = windowBorderSetting.get() === "all";
const switcher = position === "bottom" ? `row|${statusBar}|${statusBar === "bottom" && framed}`
: position === "popup" ? `popup|${statusBarVisibilitySetting.get()}|${framed}`
: "strip";
const layout = displayLayoutSetting.get();
return `${layout.startsWith("band-") ? "band" : layout}|${switcher}`;
return `${screenWidth()}x${screenHeight()}|${layout.startsWith("band-") ? "band" : layout}|${switcher}`;
}
/** The global settings that move window surfaces without resizing them. */
@@ -225,7 +229,7 @@ function windowPlacementKey(): string {
}
function createInitialDisplayPreview(): ImageSource | null {
return grayImageToPreviewSource(new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0));
return grayImageToPreviewSource(new GrayImage(screenWidth(), screenHeight(), 0));
}
/** Echo the controller's appendLog() trace to the console. */
@@ -414,6 +418,7 @@ class DashboardController {
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
this.startContinuousVoiceCapture(client, options),
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
openVoiceReplay: (path: string) => this.openVoiceReplay(path),
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
};
this.sharedActions = sharedActions;
@@ -426,11 +431,11 @@ class DashboardController {
registerRoamTools((appId) => this.launchApp(appId));
// apps.* tools mirror the launcher grid and sidebar (launch, focus, close).
registerWindowTools({
apps: LAUNCHABLE_APPS,
apps: launchableApps,
launchApp: (appId) => this.launchApp(appId),
requestShellRender: () => this.requestShellRender(),
});
// The action bar's EvenHub phone-UI button hides while that UI is showing.
// The main page's EvenHub phone-UI button hides while that UI is showing.
onPhoneShownChanged(() => this.emit());
shell.configure({
actions: {
@@ -453,13 +458,15 @@ class DashboardController {
this.emit();
},
openNotificationsWindow: () =>
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID,
(options) => createNotificationsAppWindow(options, (section) => this.openSettings(section))),
onSwitcherBarHeightChanged: () => this.applySwitcherBarHeight(),
onScreenStateChanged: (on) => {
// Any wake of the regular UI replaces a showing Glanceboard.
if (on) this.glance.dismiss();
this.handleScreenStateChanged(on);
if (on) this.requestShellRender();
if (on) keyboardService.retryNow();
this.emit();
},
});
@@ -481,14 +488,18 @@ class DashboardController {
this.pushFirmwareDebugFlags();
this.pushBrightness();
this.syncEvenHubScreenOffSetting();
this.syncVoiceWake();
// Display mode size first: a reflow repositions every surface too, so
// the band-position check after it then has nothing left to do.
this.applyDisplayModeIfChanged();
const resized = this.applyDisplayModeIfChanged();
this.applyVerticalPositionIfChanged();
this.applyAppLayoutsIfChanged();
const appsResized = this.applyAppLayoutsIfChanged();
// Either reflow hands windows their new edges along with their size.
if (!resized && !appsResized) this.applyWindowEdgesIfChanged();
this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged();
this.applyGlanceAngleIfChanged();
this.syncContinuousRecording();
// A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on
// the Java side).
@@ -509,8 +520,31 @@ class DashboardController {
assistantAvailable: () => shell.isAssistantAvailable(),
assistant: text => shell.sendToAssistant(text),
});
// A keyboard paired through the glasses (CFW keyboard bridge). The
// service decides what each key means (app/g2/keyboard/key-router.ts);
// this is how it reaches the display and the input pipeline.
keyboardService.setHost({
ready: () => this.phase === "connected",
locked: () => this.glassesLocked,
wake: () => {
void this.handleInputEvent({
kind: "display-wake",
containerName: "",
eventType: OsEventTypeList.DOUBLE_CLICK_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL,
systemExitReasonCode: 0,
frameId: 0,
}).catch((error) => this.appendLog(`keyboard wake failed: ${this.formatError(error)}`));
},
gesture: (gesture, origin) => this.injectSyntheticRingInput(gesture, origin),
log: (line) => this.appendLog(line),
// A keyboard that is switched off would otherwise keep the glasses'
// radio looking for it around the clock. One still connected when the
// display sleeps stays connected, and its next key wakes the display.
shouldReconnect: () => this.phase === "connected" && shell.isScreenOn(),
});
this.wearRemote = new WearRemote({
apps: LAUNCHABLE_APPS,
apps: launchableApps,
injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"),
launchApp: (appId) => this.launchApp(appId),
connect: () => this.connect(),
@@ -580,9 +614,9 @@ class DashboardController {
* that support resizing receive the new viewport. Other worker windows are
* closed and launched again at the new size.
*/
private applyDisplayModeIfChanged(): void {
private applyDisplayModeIfChanged(): boolean {
const sizes = viewportSizesKey();
if (sizes === this.lastViewportSizes) return;
if (sizes === this.lastViewportSizes) return false;
this.lastViewportSizes = sizes;
// The reflow below reconfigures every surface, covering any move too.
this.lastWindowPlacement = windowPlacementKey();
@@ -614,6 +648,22 @@ class DashboardController {
})().catch((error) => {
this.appendLog(`display mode change failed: ${this.formatError(error)}`);
});
return true;
}
/**
* A setting changed what some windows' edges touch (geometry.ts
* windowEdges: a border line, or blank space) without resizing them, as
* the Border setting does under a top status bar: those windows lay out
* again with the padding their new edges suggest. Their surfaces stay as
* they are.
*/
private applyWindowEdgesIfChanged(): void {
for (const window of Array.from(shell.getWindows())) {
const laidOut = window.layoutSize?.();
if (!laidOut || !window.relayout || sameEdges(laidOut.edges, windowEdges(window.appId))) continue;
window.relayout();
}
}
/**
@@ -623,14 +673,28 @@ class DashboardController {
* are. Windows that can't resize keep the surface they have, the switcher
* covering any of it that it now overlaps, rather than being relaunched
* for it.
*
* A window is judged by its own size as well as its surface's rect: other
* paths (connect, a launch) configure surfaces from the geometry without
* resizing windows, so a window laid out before the row's height was known
* (restored at startup, at the default height) can sit behind a surface
* that already matches, and its frames would not fit it.
*/
private applySwitcherBarHeight(): void {
void (async () => {
for (const window of Array.from(shell.getWindows())) {
const rect = JSON.stringify(appViewportRect(window.heightMode, window.appId));
if (!window.relayout || this.windowSurfaceRects.get(window.surfaceId) === rect) continue;
window.relayout();
await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode);
if (!window.relayout) continue;
const rect = appViewportRect(window.heightMode, window.appId);
const size = window.layoutSize?.();
const resized = !!size && (size.width !== rect.width || size.height !== rect.height);
if (!resized && this.windowSurfaceRects.get(window.surfaceId) === JSON.stringify(rect)) continue;
try {
window.relayout();
await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode);
} catch (error) {
// The rest still need to follow the row.
this.appendLog(`switcher height change failed for ${window.windowId}: ${this.formatError(error)}`);
}
}
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
@@ -648,10 +712,10 @@ class DashboardController {
};
}
private applyAppLayoutsIfChanged(): void {
private applyAppLayoutsIfChanged(): boolean {
const layouts = this.appLayouts();
const changed = Object.keys(layouts).filter((appId) => layouts[appId] !== this.lastAppLayouts[appId]);
if (!changed.length) return;
if (!changed.length) return false;
this.lastAppLayouts = layouts;
void (async () => {
for (const window of Array.from(shell.getWindows())) {
@@ -662,6 +726,7 @@ class DashboardController {
shell.foregroundWindow()?.requestRender();
this.requestShellRender();
})().catch((error) => this.appendLog(`app layout change failed: ${this.formatError(error)}`));
return true;
}
private applyVerticalPositionIfChanged(): void {
@@ -821,8 +886,8 @@ class DashboardController {
await communicator.configureSurface(LOCK_SCREEN_SURFACE_ID, {
x: 0,
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
width: screenWidth(),
height: screenHeight(),
zOrder: 1000,
transparency: "opaque",
});
@@ -930,6 +995,24 @@ class DashboardController {
return applied;
}
/** What the communicator was last told about the glasses' own wake word (null: not yet). */
private voiceWakeSent: boolean | null = null;
/**
* The glasses' own wake word follows the wake action: on a RayNeo iO, its
* "Hey RayNeo" switch (G2 glasses use the wake lease, syncFaceclawWakeLease).
*/
private syncVoiceWake(communicator = this.communicator): void {
if (!communicator) return;
const enabled = wakeWordActionSetting.get() !== "off";
if (this.voiceWakeSent === enabled) return;
this.voiceWakeSent = enabled;
void communicator.setVoiceWakeEnabled(enabled).catch((error) => {
this.voiceWakeSent = null;
this.appendLog(`wake word sync failed: ${this.formatError(error)}`);
});
}
private syncEvenHubScreenOffSetting(): void {
void this.syncFaceclawWakeLease().catch((error) => {
this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`);
@@ -1047,9 +1130,11 @@ class DashboardController {
/**
* A Direct-mode session may have unpaired the R1 from the glasses (see
* GlassesSessionRing.releaseRingFromGlassesLocked). The first session in
* another mode asks the glasses to connect to it again; the ring's own
* glasses targets were cleared, so it accepts them. Called on connect.
* GlassesSessionRing.releaseRingFromGlassesLocked), and a RayNeo iO
* session (always Direct) may have cleared the ring's glasses targets.
* The first G2 session in another mode asks the glasses to connect to it
* again; the ring's own glasses targets were cleared, so it accepts them.
* Called on connect.
*/
private handRingBackToGlassesIfNeeded(): void {
if (this.ringModeAtConnect === "direct" || !getBooleanSetting(RING_TAKEN_FROM_GLASSES_KEY, false)) return;
@@ -1170,11 +1255,10 @@ class DashboardController {
alarmReliabilityMessage: this.alarmReliabilityMessage,
previewMode: this.isPreviewDisplayActive(),
evenHubPhoneUi: phoneUiButton(),
batteries: shell.getBatteryLevels(),
};
}
/** Overlay an EvenHub app's phone UI (the action bar's app-icon button). */
/** Overlay an EvenHub app's phone UI (the app-icon button under the mirror). */
showEvenHubPhoneUi(windowId: string): void {
showEvenHubPhoneUi(windowId);
}
@@ -1237,6 +1321,7 @@ class DashboardController {
silentMode: this.silentMode,
screenOn: shell.isScreenOn(),
battery: this.lastHeadsetBattery,
glassesModel: glassesModelName(),
caseBattery: batteries.case,
caseCharging: batteries.caseCharging,
foregroundTitle: shell.getForegroundApp()?.title ?? null,
@@ -1409,12 +1494,12 @@ class DashboardController {
// is in place before any real render can reach the compositor.
this.previewTarget = target;
try {
await target.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT);
await target.configureCompositorScreen(screenWidth(), screenHeight());
await target.configureSurface(SHELL_SURFACE_ID, {
x: 0,
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
width: screenWidth(),
height: screenHeight(),
zOrder: SHELL_SURFACE_Z_ORDER,
transparency: "color-key",
});
@@ -1497,17 +1582,21 @@ class DashboardController {
this.refreshEvenAppStatus();
this.setPhase("connecting");
this.setStatus("Connecting to the glasses...");
// "Only via glasses" (the default) means the phone never opens its own
// link to the ring: the glasses relay its gestures to us anyway. An empty
// address disables every direct-ring code path in the communicator.
const ringMode = ringConnectionModeSetting.get();
// "Only via glasses" (the G2's default) means the phone never opens its
// own link to the ring: the glasses relay its gestures to us anyway. An
// empty address disables every direct-ring code path in the communicator.
// A RayNeo iO (one link, its address in both arm slots) can't relay a
// ring, so with one the phone always takes the ring directly.
const ringMode = effectiveRingConnectionMode();
this.ringModeAtConnect = ringMode;
const glassesType = glassesTypeSetting.get();
const ringAddress = ringMode !== "glasses" ? addresses.ring : "";
// Direct mode may unpair the ring from the glasses; remember to hand it
// back once a later session runs in another mode.
if (ringMode === "direct" && ringAddress) setBooleanSetting(RING_TAKEN_FROM_GLASSES_KEY, true);
this.appendLog(
`Using configured arms: R=${addresses.right} L=${addresses.left}${ringAddress ? ` ring=${ringAddress}` : ""}`,
this.appendLog((glassesType === "rayneo-io"
? `Using configured RayNeo iO: ${addresses.right}`
: `Using configured arms: R=${addresses.right} L=${addresses.left}`) + (ringAddress ? ` ring=${ringAddress}` : ""),
);
let communicator: FaceclawCommunicatorBridge | null = null;
@@ -1529,6 +1618,7 @@ class DashboardController {
// sync needs the phone role, which the session borrows briefly for
// each pull (GlassesSessionRing.beginRingHealthWindowLocked).
ringRole: ringMode === "direct" ? "glasses" : "phone",
glassesType,
});
this.communicator = communicator;
// Size the compositor before anything else is awaited: windows opened
@@ -1536,8 +1626,11 @@ class DashboardController {
// configure their surfaces through this.communicator, and the Java call
// queue is FIFO, so enqueuing the screen size first guarantees it lands
// ahead of them. Mirrors ensurePreviewDisplay.
await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT);
// The paired glasses' screen: the G2's 640x480 or the RayNeo iO's 540x220.
await communicator.configureCompositorScreen(screenWidth(), screenHeight());
await communicator.configureBrightness(getBrightnessPreferences());
this.voiceWakeSent = null;
this.syncVoiceWake(communicator);
this.offState = communicator.onStateChange((state) => {
if (state.phase !== "connected") resetRingInputFilter();
if (isHaltedSessionPhase(state.phase)) {
@@ -1706,6 +1799,7 @@ class DashboardController {
this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`);
});
this.ensureWearStateTracking();
this.syncKeyboardLink();
}
// The shared session halts on incompatible firmware by itself
// ("incompatible-firmware" phase); this only explains why.
@@ -1731,8 +1825,8 @@ class DashboardController {
await communicator.configureSurface(SHELL_SURFACE_ID, {
x: 0,
y: 0,
width: G2_LENS_WIDTH,
height: G2_LENS_HEIGHT,
width: screenWidth(),
height: screenHeight(),
zOrder: SHELL_SURFACE_Z_ORDER,
transparency: "color-key",
});
@@ -2040,8 +2134,8 @@ class DashboardController {
return;
}
const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny);
const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH);
const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT);
const x = Math.round(Math.min(1, Math.max(0, at.nx)) * screenWidth());
const y = Math.round(Math.min(1, Math.max(0, at.ny)) * screenHeight());
const foreground = shell.foregroundWindow();
const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId);
this.appendLog(`mirror tap at ${x},${y}`);
@@ -2139,12 +2233,14 @@ class DashboardController {
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options);
this.syncContinuousRecording();
this.retargetContinuousCapture();
this.ensureContinuousCapture(true);
}
private stopContinuousVoiceCapture(client: string): void {
if (!this.continuousClients.delete(client)) return;
this.syncContinuousRecording();
if (this.continuousClients.size > 0) {
// The window that needed its own provider may be the one that left.
this.retargetContinuousCapture();
@@ -2170,6 +2266,18 @@ class DashboardController {
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
}
/**
* Record to the first client's requested name while Save Recording is on
* (it is toggled from those windows' menus, so this also runs when a
* setting changes).
*/
private syncContinuousRecording(): void {
const name = saveRecordingSetting.get()
? Array.from(this.continuousClients.values(), (options) => options.recordAs).find(Boolean) ?? ""
: "";
voiceControlBridge.recordTo(name);
}
/** Move a running continuous capture onto the provider its windows now call for. */
private retargetContinuousCapture(): void {
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
@@ -2183,6 +2291,30 @@ class DashboardController {
return undefined;
}
/**
* How many of the glasses' microphones the continuous-capture windows call
* for (the most any asks for), where the glasses can stream them
* separately: G2 with the microphone channel in use. Otherwise 1, the
* mixed stream.
*/
private continuousMics(): MicCount {
if (glassesTypeSetting.get() !== "g2" || !micControlSupported()) return 1;
let mics: MicCount = 1;
for (const options of this.continuousClients.values()) {
if ((options.mics ?? 1) > mics) mics = options.mics!;
}
return mics;
}
/**
* A recording as a capture of its own (see VoiceReplay), transcribed with
* the provider and processing settings a microphone capture would get.
* "My voice only" is left out: a recording is of whoever spoke.
*/
private openVoiceReplay(path: string): VoiceReplay {
return new VoiceReplay(path, () => ({ ...this.voiceCaptureOptions(null), speakerVerification: undefined }));
}
/**
* Continuous capture's claim on the mic (the Microphones app and EvenHub
* mic apps stand down while it is held) follows whether the capture really
@@ -2208,9 +2340,26 @@ class DashboardController {
// Otherwise only Whisper server is told a language (its own setting).
const choice = this.transcriptionChoice();
const provider = choice?.provider ?? voiceProviderSetting.get();
// Separate microphones only come from connected glasses.
const mics = communicator ? this.continuousMics() : 1;
const micConfig = mics === 1 ? null : micArrayCaptureConfig(mics);
return {
communicator: communicator?.getNativeCommunicator() ?? null,
usePhoneMic: communicator === null,
micArray: micConfig ? { configure: encodeMicControl("configure", micConfig) } : null,
// Stored in a recording of the capture (see Save Recording).
captureInfo: {
glasses: communicator ? glassesModelName() : null,
mics,
micConfig: micConfig && {
source: micConfig.source,
channelMask: micConfig.channelMask,
codec: micConfig.codec,
format: micConfig.format,
sampleRateHz: micConfig.sampleRateHz,
directionDetection: micConfig.beamform,
},
},
provider,
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
@@ -2219,7 +2368,6 @@ class DashboardController {
whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
// "My voice only" (Microphones app): verify command utterances
// against the enrolled wearer voice-print; non-matching speakers'
// finals are suppressed by the bridge.
@@ -2530,6 +2678,23 @@ class DashboardController {
applyGlanceAngle();
}
/**
* Tell the keyboard bridge whether the session can host a keyboard: a G2
* on firmware with the bridge, connected and in use. While the glasses
* charge in their case the link stays up but nobody is typing, so the
* firmware is told to let the keyboard go.
*/
private syncKeyboardLink(): void {
const up =
this.phase === "connected" &&
this.customFirmwareConfirmed &&
glassesTypeSetting.get() === "g2";
// Charging, or a disconnect the user asked for: the link is still up, so
// the DISABLE gets out ahead of everything the teardown sends.
if (up) keyboardService.linkReady();
else keyboardService.linkLost(this.phase === "charging" || this.phase === "disconnecting");
}
/**
* The Glanceboard event a sleep-time input means, or null when the shell
* should see the input as usual. A display-wake carries the head-tilt
@@ -2636,11 +2801,7 @@ class DashboardController {
setSurfaceVisible: (surfaceId, visible) => this.setWindowSurfaceVisible(surfaceId, visible),
removeSurface: (surfaceId) => this.removeWindowSurface(surfaceId),
requestShellRender: () => this.requestShellRender(),
openSettings: (section) => {
void this.launchApp("settings", { section }).catch((error) => {
this.appendLog(`settings launch failed: ${this.formatError(error)}`);
});
},
openSettings: (section) => this.openSettings(section),
startTextSettingEdit: (settingId) => {
const setting = getStringSettingById(settingId);
if (setting) {
@@ -2650,6 +2811,7 @@ class DashboardController {
}
},
endTextSettingEdit: () => this.endTextSettingEdit(),
unhandledKey: (event) => keyboardService.handleUnhandledKey(event as KeyEvent),
});
this.appHosts.set(appId, host);
return host;
@@ -2683,10 +2845,11 @@ class DashboardController {
if (!this.openAppsRestored || this.suppressOpenAppsPersist) return;
const open: string[] = [];
for (const window of shell.getWindows()) {
if (window.appId === "launcher") continue;
if (window.appId === "launcher" || window.restoresApp === false) continue;
if (!open.includes(window.appId)) open.push(window.appId);
}
savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null });
const foreground = shell.foregroundWindow();
savePersistedOpenApps({ open, foreground: foreground && foreground.restoresApp !== false ? foreground.appId : null });
}
/**
@@ -2724,6 +2887,13 @@ class DashboardController {
this.requestShellRender();
}
/** Open or focus the Settings app, optionally on a section (by label). */
private openSettings(section?: string): void {
void this.launchApp("settings", { section }).catch((error) => {
this.appendLog(`settings launch failed: ${this.formatError(error)}`);
});
}
/**
* Launch an app by id: the launcher grid, assistant tools, window-menu
* Settings item, and open-apps restore all come through here. Apps with an
@@ -2940,7 +3110,7 @@ class DashboardController {
// The tray shows the notification at once; its popup waits out a short
// delay, so one withdrawn right away (see NotificationPopupDelay) never shows.
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
if (notification && shouldShowNotificationOnGlasses(notification.packageName)) {
if (notification && shouldPopUpNewNotification(notification.packageName)) {
this.notificationPopups.post(notificationKey, notificationPopupDelayMs(notification));
}
this.requestShellRender();
@@ -2962,7 +3132,7 @@ class DashboardController {
private popUpAndroidNotification(notificationKey: string): void {
try {
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) return;
if (!notification || !shouldPopUpNewNotification(notification.packageName)) return;
// New notifications open a shell modal over the app viewport; if the
// screen was off, wake for it and go back to sleep when it is closed.
// Waking while already on would steal focus, so only wake from sleep.
@@ -3007,6 +3177,7 @@ class DashboardController {
private setPhase(phase: ConnectionPhase): void {
if (this.phase === phase) return;
this.phase = phase;
this.syncKeyboardLink();
if (phase !== "connected" && phase !== "charging") {
shell.setBatteryLevels({ ring: null, ringCharging: null });
}