forked from Blendman974/faceclaw
Merge branch 'main' into ring-health-fixes
This commit is contained in:
commit
4f8c082b52
368 files changed
+26238
-3401
No files matched your search
+232
-90
@@ -15,8 +15,9 @@ import { startForegroundNotification, stopForegroundNotification, updateForegrou
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { nightscoutBridge } from "../native/nightscout-bridge";
|
||||
import { weatherBridge } from "../native/weather";
|
||||
import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, readActiveNotifications } from "../native/notification-icons";
|
||||
import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, onAndroidNotificationRemoved, readActiveNotifications } from "../native/notification-icons";
|
||||
import { shouldShowNotificationOnGlasses } from "../native/notification-sources";
|
||||
import { watchNotificationListenerBinding } from "../native/notification-access";
|
||||
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
|
||||
import { defaultRingName, setGlassesRingLink } from "../native/ring-status";
|
||||
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
|
||||
@@ -47,17 +48,20 @@ const MIRROR_TOUCH_GESTURES: Record<Exclude<MirrorTouchKind, "tap">, WearRemoteI
|
||||
"swipe-left": "swipe-left",
|
||||
"swipe-right": "swipe-right",
|
||||
};
|
||||
import { findSoundEffect, playSoundEffect } from "../ui/sound-effects";
|
||||
import { GlanceHost } from "./glance-host";
|
||||
import { mirrorToPanelFraction, phonePreviewCrop } from "./phone-preview-crop";
|
||||
import { type GlanceEvent } from "./glance-state";
|
||||
import { isPreviewOnlyMode, isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state";
|
||||
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
|
||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceActivity } from "../ui/shell/voice-activity";
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||
import { prepareFrameDraws } from "../graphics/glyph-wire";
|
||||
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen";
|
||||
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
|
||||
import { NotificationPopupDelay, notificationPopupDelayMs } from "../ui/shell/notification-popup-delay";
|
||||
import { type InputEvent } from "../ui/gestures";
|
||||
import { registerSystemTools } from "../assistant/system-tools";
|
||||
import { updateGlassesPresence } from "./glasses-presence";
|
||||
@@ -71,10 +75,10 @@ import { ALL_APPS } from "../apps/all-apps";
|
||||
import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition";
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayModeSetting, navigateDisplayModeSetting, navigateVerticalPositionSetting, terminalDisplayModeSetting, terminalVerticalPositionSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -94,7 +98,7 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
|
||||
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
|
||||
import { wearerVerificationOptions } from "../apps/microphones/speakers";
|
||||
import { micSession } from "../apps/microphones/mic-session";
|
||||
import { glassesDisplayLabel } from "./glasses-display-state";
|
||||
import { glassesDisplayAsleep, glassesDisplayLabel, type GlassesDisplayState } from "./glasses-display-state";
|
||||
import { PreviewDisplayTarget, type DisplayTarget } from "../native/preview-display";
|
||||
|
||||
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
|
||||
@@ -164,6 +168,9 @@ const SHELL_REFRESH_INTERVAL_MS = 60_000;
|
||||
// schedules a refresh), so this just covers anything that slips through.
|
||||
const PREVIEW_INTERVAL_MS = 1_000;
|
||||
const SCREEN_TIMEOUT_CHECK_MS = 1_000;
|
||||
// voiceActivity holder for continuous capture (Transcribe, Calculator's
|
||||
// continuous mode, the Teleprompter reader).
|
||||
const CONTINUOUS_CAPTURE_HOLDER = "continuous-capture";
|
||||
// The lock screen is a short notice, not something to read at leisure; it
|
||||
// ignores the screen-timeout setting (including "never").
|
||||
const LOCK_SCREEN_TIMEOUT_MS = 15_000;
|
||||
@@ -186,24 +193,26 @@ const EVEN_APP_DETECTED_MESSAGE =
|
||||
const LAUNCHABLE_APPS = ALL_APPS.filter((app) => app.showInLauncher !== false);
|
||||
|
||||
/**
|
||||
* The global settings window viewport sizes depend on: the display mode,
|
||||
* whether the app switcher takes width beside windows, height below them
|
||||
* or neither (a popup), and the chrome rows each window loses to it: below
|
||||
* them, whether the status bar joins the row (dropping the top bar); with a
|
||||
* popup, whether the status bar keeps rows of its own and the window frame
|
||||
* (where the status bar goes only moves windows).
|
||||
* The global settings window viewport sizes depend on: the display mode's
|
||||
* size (a band's position only moves windows), whether the app switcher
|
||||
* takes width beside windows, height below them or neither (a popup), and
|
||||
* the chrome rows each window loses to it: below them, whether the status
|
||||
* bar joins the row (dropping the top bar); with a popup, whether the
|
||||
* status bar keeps rows of its own and the window frame (where the status
|
||||
* bar goes only moves windows).
|
||||
*/
|
||||
function viewportSizesKey(): string {
|
||||
const position = appSwitcherPositionSetting.get();
|
||||
const switcher = position === "bottom" ? `row|${statusBarPositionSetting.get()}`
|
||||
: position === "popup" ? `popup|${statusBarVisibilitySetting.get()}|${windowBorderSetting.get()}`
|
||||
: "strip";
|
||||
return `${displayModeSetting.get()}|${switcher}`;
|
||||
const layout = displayLayoutSetting.get();
|
||||
return `${layout.startsWith("band-") ? "band" : layout}|${switcher}`;
|
||||
}
|
||||
|
||||
/** The global settings that move window surfaces without resizing them. */
|
||||
function windowPlacementKey(): string {
|
||||
return `${verticalPositionSetting.get()}|${appSwitcherPositionSetting.get()}|${statusBarPositionSetting.get()}`;
|
||||
return `${displayLayoutSetting.get()}|${appSwitcherPositionSetting.get()}|${statusBarPositionSetting.get()}`;
|
||||
}
|
||||
|
||||
function createInitialDisplayPreview(): ImageSource | null {
|
||||
@@ -276,6 +285,8 @@ class DashboardController {
|
||||
private extractedFontsConfirmed = false;
|
||||
private screenRecordingActive = false;
|
||||
private displayPreview: ImageSource | null = createInitialDisplayPreview();
|
||||
/** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */
|
||||
private displayPreviewCrop: ScreenRect | null = null;
|
||||
private silentMode = false;
|
||||
// Brightness value last pushed to the glasses; see pushBrightness.
|
||||
private lastPushedBrightness: string | null = null;
|
||||
@@ -283,9 +294,6 @@ class DashboardController {
|
||||
// drop-off right after a low reading can be explained as a flat battery.
|
||||
private lastHeadsetBattery: number | null = null;
|
||||
private readonly listeners = new Set<DashboardListener>();
|
||||
// Set at connect time from the persisted flag; the one-time post-onboarding
|
||||
// welcome sound plays on the first rendered frame (proof the session is warm).
|
||||
private welcomeSoundArmed = false;
|
||||
|
||||
private communicator: FaceclawCommunicatorBridge | null = null;
|
||||
// Headless stand-in for the compositor when no glasses are paired
|
||||
@@ -343,6 +351,9 @@ class DashboardController {
|
||||
private offFirmwareInfo: (() => void) | null = null;
|
||||
private offVoiceStatus: (() => void) | null = null;
|
||||
private offAndroidNotification: (() => void) | null = null;
|
||||
private offAndroidNotificationRemoved: (() => void) | null = null;
|
||||
/** New notifications wait here briefly before popping up (see NotificationPopupDelay). */
|
||||
private readonly notificationPopups = new NotificationPopupDelay((notificationKey) => this.popUpAndroidNotification(notificationKey));
|
||||
private lastInput = "waiting...";
|
||||
private lastSys = "none yet";
|
||||
private shellRenderInProgress = false;
|
||||
@@ -389,8 +400,9 @@ class DashboardController {
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => this.stopVoiceCapture(),
|
||||
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
|
||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
|
||||
this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
this.sharedActions = sharedActions;
|
||||
@@ -431,6 +443,7 @@ class DashboardController {
|
||||
},
|
||||
openNotificationsWindow: () =>
|
||||
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
|
||||
onSwitcherBarHeightChanged: () => this.applySwitcherBarHeight(),
|
||||
onScreenStateChanged: (on) => {
|
||||
// Any wake of the regular UI replaces a showing Glanceboard.
|
||||
if (on) this.glance.dismiss();
|
||||
@@ -445,11 +458,9 @@ class DashboardController {
|
||||
app.boot?.(this.buildAppContext(app));
|
||||
}
|
||||
timerEngine.onChange(() => this.refreshAlarmReliabilityWarning());
|
||||
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => {
|
||||
void this.handleAndroidNotificationPosted(notificationKey).catch((error) => {
|
||||
this.appendLog(`notification wake failed: ${this.formatError(error)}`);
|
||||
});
|
||||
});
|
||||
this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => this.handleAndroidNotificationPosted(notificationKey));
|
||||
this.offAndroidNotificationRemoved = onAndroidNotificationRemoved((notificationKey) => this.handleAndroidNotificationRemoved(notificationKey));
|
||||
watchNotificationListenerBinding();
|
||||
readActiveNotifications(ALL_NOTIFICATIONS);
|
||||
// Settings toggled from the glasses can change what the phone UI shows
|
||||
// (e.g. the text-setting editor), so re-emit the snapshot on any change.
|
||||
@@ -459,15 +470,16 @@ class DashboardController {
|
||||
this.pushFirmwareDebugFlags();
|
||||
this.pushBrightness();
|
||||
this.syncEvenHubScreenOffSetting();
|
||||
// Display mode first: a reflow repositions every surface too, so the
|
||||
// vertical-position check after it then has nothing left to do.
|
||||
// Display mode size first: a reflow repositions every surface too, so
|
||||
// the band-position check after it then has nothing left to do.
|
||||
this.applyDisplayModeIfChanged();
|
||||
this.applyVerticalPositionIfChanged();
|
||||
this.applyAppLayoutsIfChanged();
|
||||
this.syncAssistantBridgeIfChanged();
|
||||
this.syncLockScreenSettingIfChanged();
|
||||
// A Preview color change should show on the mirror at once, not at the
|
||||
// next composited frame (cheap: floor-limited, dedup'd on the Java side).
|
||||
// A Preview color or scaling change should show on the mirror at once,
|
||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||
// the Java side).
|
||||
this.schedulePreviewUpdate();
|
||||
});
|
||||
// Connect to the external agent bridge at boot if configured; the
|
||||
@@ -540,7 +552,7 @@ class DashboardController {
|
||||
});
|
||||
}
|
||||
|
||||
// Vertical-position changes (and moving the app switcher between the left
|
||||
// Band position changes (and moving the app switcher between the left
|
||||
// and right edges, or a popup switcher's status bar between the top and
|
||||
// bottom) move every window surface and the chrome that aligns with them;
|
||||
// the values are tracked so unrelated setting changes don't trigger a full
|
||||
@@ -550,7 +562,7 @@ class DashboardController {
|
||||
private lastViewportSizes = viewportSizesKey();
|
||||
|
||||
/**
|
||||
* Display mode changed (Settings > Display, or the phone page's picker),
|
||||
* Display mode size changed (Settings > Display, or the phone page's picker),
|
||||
* or the app switcher or its chrome changed what windows lose to them
|
||||
* (see viewportSizesKey): every window's viewport size changes. In-process windows re-measure in place; workers
|
||||
* that support resizing receive the new viewport. Other worker windows are
|
||||
@@ -592,12 +604,35 @@ class DashboardController {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The bottom switcher's layout changed its height (shell.ts
|
||||
* onSwitcherBarHeightChanged): window bands move where the slack under
|
||||
* them runs out, and full-height windows resize; the rest stay as they
|
||||
* are. Windows that can't resize keep the surface they have, the switcher
|
||||
* covering any of it that it now overlaps, rather than being relaunched
|
||||
* for it.
|
||||
*/
|
||||
private applySwitcherBarHeight(): void {
|
||||
void (async () => {
|
||||
for (const window of Array.from(shell.getWindows())) {
|
||||
const rect = JSON.stringify(appViewportRect(window.heightMode, window.appId));
|
||||
if (!window.relayout || this.windowSurfaceRects.get(window.surfaceId) === rect) continue;
|
||||
window.relayout();
|
||||
await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode);
|
||||
}
|
||||
shell.foregroundWindow()?.requestRender();
|
||||
this.requestShellRender();
|
||||
})().catch((error) => {
|
||||
this.appendLog(`switcher height change failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
|
||||
private lastAppLayouts = this.appLayouts();
|
||||
|
||||
private appLayouts(): Record<string, string> {
|
||||
return {
|
||||
navigate: `${navigateDisplayModeSetting.get()}:${navigateVerticalPositionSetting.get()}`,
|
||||
terminal: `${terminalDisplayModeSetting.get()}:${terminalVerticalPositionSetting.get()}`,
|
||||
navigate: navigateDisplayLayoutSetting.get(),
|
||||
terminal: terminalDisplayLayoutSetting.get(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1170,14 +1205,23 @@ class DashboardController {
|
||||
|
||||
/** What occupies the foreground-title line on the watch-style controls. */
|
||||
glassesDisplayLabel(): string {
|
||||
return glassesDisplayLabel({
|
||||
return glassesDisplayLabel(this.glassesDisplayState());
|
||||
}
|
||||
|
||||
/** Whether that line reads "Display off" (the watch face adds a wake hint under it). */
|
||||
glassesDisplayAsleep(): boolean {
|
||||
return glassesDisplayAsleep(this.glassesDisplayState());
|
||||
}
|
||||
|
||||
private glassesDisplayState(): GlassesDisplayState {
|
||||
return {
|
||||
phase: this.phase,
|
||||
silentMode: this.silentMode,
|
||||
screenOn: shell.isScreenOn(),
|
||||
battery: this.lastHeadsetBattery,
|
||||
foregroundTitle: shell.getForegroundApp()?.title ?? null,
|
||||
previewMode: this.isPreviewDisplayActive(),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1439,7 +1483,6 @@ class DashboardController {
|
||||
this.teardownPreviewDisplay();
|
||||
this.lastInput = "waiting...";
|
||||
this.lastSys = "none yet";
|
||||
this.welcomeSoundArmed = isWelcomeSoundPending();
|
||||
this.firmwareWarningMessage = "";
|
||||
this.glassesWorn = null;
|
||||
this.wakeAfterChargingReconnect = false;
|
||||
@@ -1614,11 +1657,12 @@ class DashboardController {
|
||||
// parked by the previous session (or by an EvenHub suspend) resumes.
|
||||
this.resumeVoiceCapture();
|
||||
// A rendered frame means the session is warmed up (fixedLayoutCreated),
|
||||
// so the buzzer won't be dropped. Play the one-time welcome sound now.
|
||||
if (this.welcomeSoundArmed) {
|
||||
this.welcomeSoundArmed = false;
|
||||
setWelcomeSoundPending(false);
|
||||
void this.playWelcomeSound();
|
||||
// so the onboarding's welcome jingle won't be dropped. Open the
|
||||
// on-glasses onboarding if this version hasn't been through it.
|
||||
if (claimGlassesOnboardingAutoLaunch()) {
|
||||
this.launchApp(ONBOARDING_APP_ID).catch((error) => {
|
||||
this.appendLog(`onboarding launch failed: ${this.formatError(error)}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1731,6 +1775,10 @@ class DashboardController {
|
||||
await nightscoutBridge.stop().catch(() => {});
|
||||
weatherBridge.setSessionActive(false);
|
||||
voiceControlBridge.stop();
|
||||
// The parked continuous capture went with it; the next session starts
|
||||
// it afresh for any window still asking.
|
||||
this.continuousAttemptedOn = undefined;
|
||||
this.syncContinuousClaim();
|
||||
if (communicator) {
|
||||
await communicator.setFaceclawWakeLeaseEnabled(false).catch(() => false);
|
||||
await communicator.close().catch(() => {});
|
||||
@@ -1966,13 +2014,14 @@ class DashboardController {
|
||||
await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch");
|
||||
return;
|
||||
}
|
||||
const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH);
|
||||
const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT);
|
||||
const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny);
|
||||
const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH);
|
||||
const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT);
|
||||
const foreground = shell.foregroundWindow();
|
||||
const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId);
|
||||
this.appendLog(`mirror tap at ${x},${y}`);
|
||||
if (!shell.hasOverlay() && shell.focusNotificationAt(x, y)) {
|
||||
this.appendLog("mirror tap: notification icon");
|
||||
if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) {
|
||||
this.appendLog("mirror tap: status bar item");
|
||||
this.requestShellRender();
|
||||
return;
|
||||
}
|
||||
@@ -2054,12 +2103,73 @@ class DashboardController {
|
||||
return voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
private startContinuousVoiceCapture(): void {
|
||||
void this.beginVoiceCapture("continuous");
|
||||
// Windows that want continuous capture, and what each needs of it. They
|
||||
// share one stream, which runs while any of them wants it: one window
|
||||
// letting go must not cut off another.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>();
|
||||
// The connection a continuous-capture start was last tried on (undefined:
|
||||
// none since the last teardown), so a window waiting on the glasses is
|
||||
// retried once per new session rather than on every frame.
|
||||
private continuousAttemptedOn: FaceclawCommunicatorBridge | null | undefined = undefined;
|
||||
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options);
|
||||
this.retargetContinuousCapture();
|
||||
this.ensureContinuousCapture(true);
|
||||
}
|
||||
|
||||
private stopContinuousVoiceCapture(): void {
|
||||
voiceControlBridge.stopContinuousCapture();
|
||||
private stopContinuousVoiceCapture(client: string): void {
|
||||
if (!this.continuousClients.delete(client)) return;
|
||||
if (this.continuousClients.size > 0) {
|
||||
// The window that needed its own provider may be the one that left.
|
||||
this.retargetContinuousCapture();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
voiceControlBridge.stopContinuousCapture();
|
||||
} finally {
|
||||
this.syncContinuousClaim();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start continuous capture for the windows asking for it, unless it is
|
||||
* running or parked across a session outage. A window restored at launch
|
||||
* asks before the glasses connect, so this is retried on each new session
|
||||
* (from resumeVoiceCapture); `force` retries on the same one.
|
||||
*/
|
||||
private ensureContinuousCapture(force = false): void {
|
||||
if (this.continuousClients.size === 0 || voiceControlBridge.holdsCapture("continuous")) return;
|
||||
if (!force && this.continuousAttemptedOn === this.communicator) return;
|
||||
this.continuousAttemptedOn = this.communicator;
|
||||
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
|
||||
}
|
||||
|
||||
/** Move a running continuous capture onto the provider its windows now call for. */
|
||||
private retargetContinuousCapture(): void {
|
||||
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
||||
}
|
||||
|
||||
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
|
||||
private multilingualProvider(): VoiceProvider | undefined {
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if (options.multilingualProvider) return options.multilingualProvider;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Continuous capture's claim on the mic (the Microphones app and EvenHub
|
||||
* mic apps stand down while it is held) follows whether the capture really
|
||||
* holds the mic, not merely whether a window asked: a start that never
|
||||
* happened (no glasses yet, permission refused) must not idle every other
|
||||
* mic user.
|
||||
*/
|
||||
private syncContinuousClaim(): void {
|
||||
voiceActivity.setActive(
|
||||
this.continuousClients.size > 0 && voiceControlBridge.holdsCapture("continuous"),
|
||||
CONTINUOUS_CAPTURE_HOLDER,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2068,13 +2178,21 @@ class DashboardController {
|
||||
* stands in for the G2 mic.
|
||||
*/
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs a multilingual provider sets it for every capture
|
||||
// while it is open, so push-to-talk still shares its stream.
|
||||
const multilingual = this.multilingualProvider();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider: voiceProviderSetting.get(),
|
||||
provider: multilingual ?? voiceProviderSetting.get(),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
// Whisper detects the language itself when the speech may be in any.
|
||||
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
// against the enrolled wearer voice-print; non-matching speakers'
|
||||
@@ -2087,15 +2205,19 @@ class DashboardController {
|
||||
}
|
||||
|
||||
/**
|
||||
* Restart the mic for a capture that a previous session left holding. Cheap
|
||||
* to call per frame: it does nothing unless one is actually parked.
|
||||
* Restart the mic for a capture that a previous session left holding, and
|
||||
* start a continuous capture asked for before this session. Cheap to call
|
||||
* per frame: it does nothing unless one is actually waiting.
|
||||
*/
|
||||
private resumeVoiceCapture(): void {
|
||||
if (!voiceControlBridge.hasSuspendedCapture()) return;
|
||||
const communicator = this.communicator;
|
||||
if (!communicator || this.phase !== "connected") return;
|
||||
this.appendLog("resuming voice capture on the new glasses session");
|
||||
voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator));
|
||||
if (voiceControlBridge.hasSuspendedCapture()) {
|
||||
this.appendLog("resuming voice capture on the new glasses session");
|
||||
voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator));
|
||||
}
|
||||
// A continuous capture asked for before this session existed.
|
||||
this.ensureContinuousCapture();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2140,6 +2262,11 @@ class DashboardController {
|
||||
if (kind === "ptt") {
|
||||
voiceControlBridge.startPushToTalk(options);
|
||||
} else {
|
||||
// Every window may have let go while permission was pending.
|
||||
if (this.continuousClients.size === 0) return;
|
||||
// Claimed before the capture starts so the Microphones app and
|
||||
// EvenHub mic apps stand down first, as they do for the voice dialog.
|
||||
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER);
|
||||
voiceControlBridge.startContinuousCapture(options);
|
||||
}
|
||||
})
|
||||
@@ -2294,7 +2421,7 @@ class DashboardController {
|
||||
let shellPreWoke = false;
|
||||
if (wakewordShouldWake || displayShouldWake) {
|
||||
if (!shell.isScreenOn()) {
|
||||
shellPreWoke = shell.wake("sidebar");
|
||||
shellPreWoke = shell.wake(displayShouldWake ? shell.userWakeFocus() : "sidebar");
|
||||
}
|
||||
if (shellPreWoke || this.evenHubSessionSuspended || this.evenHubResumePromise) {
|
||||
const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () =>
|
||||
@@ -2394,7 +2521,7 @@ class DashboardController {
|
||||
* An in-process singleton app's window, opened in the background (neither
|
||||
* foregrounded nor focused) unless it is open already: for the shell to
|
||||
* bring forward itself (the Notifications window under a notification
|
||||
* selected in the switcher). The bridge runs calls in order, so the
|
||||
* selected in the switcher, or an app widget's window under the widget). The bridge runs calls in order, so the
|
||||
* shell may foreground it at once: its surface's visibility is settled
|
||||
* when the configure call is queued, ahead of any frame.
|
||||
*/
|
||||
@@ -2483,6 +2610,8 @@ class DashboardController {
|
||||
launchApp: (appId, params) => this.launchApp(appId, params),
|
||||
uninstallApp: (appId) => this.uninstallApp(appId),
|
||||
launchInProcessApp: (windowId, surfaceId, create) => this.launchInProcessApp(windowId, surfaceId, create),
|
||||
openInProcessAppInBackground: (windowId, surfaceId, create) =>
|
||||
this.openInProcessAppInBackground(windowId, surfaceId, create),
|
||||
ensureWorkerHost: (createWorker) => this.ensureWorkerHost(app.appId, createWorker),
|
||||
submitWindowFrame: (surfaceId, planes, paintMs, frameId) =>
|
||||
this.submitWindowFrame(surfaceId, planes, paintMs, frameId),
|
||||
@@ -2600,20 +2729,22 @@ class DashboardController {
|
||||
target: DisplayTarget | null = this.display,
|
||||
): Promise<void> {
|
||||
if (!target) return;
|
||||
await target.configureSurface(surfaceId, {
|
||||
...appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId),
|
||||
zOrder: 0,
|
||||
transparency: "opaque",
|
||||
});
|
||||
const rect = appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId);
|
||||
this.windowSurfaceRects.set(surfaceId, JSON.stringify(rect));
|
||||
await target.configureSurface(surfaceId, { ...rect, zOrder: 0, transparency: "opaque" });
|
||||
await target.setSurfaceVisible(surfaceId, typeof visible === "function" ? visible() : visible);
|
||||
}
|
||||
|
||||
/** Each window surface's rect as last configured (applySwitcherBarHeight skips those that stay). */
|
||||
private readonly windowSurfaceRects = new Map<string, string>();
|
||||
|
||||
/** A late answer to "is this window in front?", for configureWindowSurface. */
|
||||
private isForegroundWindow(windowId: string): () => boolean {
|
||||
return () => shell.foregroundWindow()?.windowId === windowId;
|
||||
}
|
||||
|
||||
private removeWindowSurface(surfaceId: string): void {
|
||||
this.windowSurfaceRects.delete(surfaceId);
|
||||
const display = this.display;
|
||||
if (!display) return;
|
||||
void display.removeSurface(surfaceId).catch((error) => {
|
||||
@@ -2751,23 +2882,47 @@ class DashboardController {
|
||||
this.schedulePreviewUpdate();
|
||||
}
|
||||
|
||||
private async handleAndroidNotificationPosted(notificationKey: string): Promise<void> {
|
||||
private handleAndroidNotificationPosted(notificationKey: string): void {
|
||||
// The tray shows the notification at once; its popup waits out a short
|
||||
// delay, so one withdrawn right away (see NotificationPopupDelay) never shows.
|
||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
|
||||
if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) {
|
||||
this.requestShellRender();
|
||||
return;
|
||||
if (notification && shouldShowNotificationOnGlasses(notification.packageName)) {
|
||||
this.notificationPopups.post(notificationKey, notificationPopupDelayMs(notification));
|
||||
}
|
||||
// New notifications open a shell modal over the app viewport; if the
|
||||
// screen was off, wake for it and go back to sleep when it is closed.
|
||||
// Waking while already on would steal focus, so only wake from sleep.
|
||||
const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar");
|
||||
if (wokeScreen) {
|
||||
this.appendLog("android notification woke the screen");
|
||||
}
|
||||
shell.openNotificationModal(notificationKey, wokeScreen);
|
||||
this.requestShellRender();
|
||||
}
|
||||
|
||||
private handleAndroidNotificationRemoved(notificationKey: string): void {
|
||||
if (this.notificationPopups.withdraw(notificationKey)) {
|
||||
this.appendLog(`android notification ${notificationKey} withdrawn before its popup`);
|
||||
}
|
||||
try {
|
||||
shell.notificationRemoved(notificationKey);
|
||||
} catch (error) {
|
||||
this.appendLog(`closing a removed notification's popup failed: ${this.formatError(error)}`);
|
||||
}
|
||||
this.requestShellRender();
|
||||
}
|
||||
|
||||
/** A posted notification waited out its popup delay: show it, if it is still there. */
|
||||
private popUpAndroidNotification(notificationKey: string): void {
|
||||
try {
|
||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey);
|
||||
if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) return;
|
||||
// New notifications open a shell modal over the app viewport; if the
|
||||
// screen was off, wake for it and go back to sleep when it is closed.
|
||||
// Waking while already on would steal focus, so only wake from sleep.
|
||||
const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar");
|
||||
if (wokeScreen) {
|
||||
this.appendLog("android notification woke the screen");
|
||||
}
|
||||
shell.openNotificationModal(notificationKey, wokeScreen);
|
||||
this.requestShellRender();
|
||||
} catch (error) {
|
||||
this.appendLog(`notification popup failed: ${this.formatError(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async playBuzzerSequence(payload: Uint8Array): Promise<void> {
|
||||
if (this.phase !== "connected" || !this.communicator) {
|
||||
return;
|
||||
@@ -2775,21 +2930,6 @@ class DashboardController {
|
||||
await this.communicator.playBuzzerSequence(payload);
|
||||
}
|
||||
|
||||
/** One-time celebratory jingle on the first connection after onboarding. */
|
||||
private async playWelcomeSound(): Promise<void> {
|
||||
const effect = findSoundEffect("questcomplete");
|
||||
if (!effect) return;
|
||||
try {
|
||||
await playSoundEffect(
|
||||
effect,
|
||||
(payload) => this.playBuzzerSequence(payload),
|
||||
(ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
);
|
||||
} catch (error) {
|
||||
this.appendLog(`welcome sound failed: ${this.formatError(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private clearDashboardTimer(): void {
|
||||
this.cancelEvenHubSuspendTimer();
|
||||
if (this.shellRefreshTimer) {
|
||||
@@ -2933,8 +3073,10 @@ class DashboardController {
|
||||
const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager;
|
||||
if (!power?.isInteractive()) return;
|
||||
}
|
||||
const preview = display.getCompositePreview(previewColorSetting.get() === "green");
|
||||
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect());
|
||||
const preview = display.getCompositePreview(previewColorSetting.get() === "green", crop);
|
||||
if (preview) {
|
||||
this.displayPreviewCrop = crop;
|
||||
this.setDisplayPreview(preview);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire";
|
||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||
import * as frameTimings from "../native/frame-timings";
|
||||
import { type DisplayTarget } from "../native/preview-display";
|
||||
import { minWindowTop } from "../ui/shell/geometry";
|
||||
import { minWindowTop, type ScreenRect } from "../ui/shell/geometry";
|
||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition";
|
||||
import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state";
|
||||
@@ -59,6 +59,8 @@ export class GlanceHost {
|
||||
private rendering = false;
|
||||
private renderPending = false;
|
||||
private nextRenderWantsFreshData = false;
|
||||
/** Where the last render put the board on screen. */
|
||||
private boardRect: ScreenRect | null = null;
|
||||
|
||||
constructor(private readonly options: GlanceHostOptions) {}
|
||||
|
||||
@@ -66,6 +68,11 @@ export class GlanceHost {
|
||||
return this.state.visible;
|
||||
}
|
||||
|
||||
/** The board's area of the screen while it shows (for the phone mirror's Scaled preview), else null. */
|
||||
screenRect(): ScreenRect | null {
|
||||
return this.state.visible ? this.boardRect : null;
|
||||
}
|
||||
|
||||
/** Whether sleep-time gestures should reach the board at all. */
|
||||
isEnabled(): boolean {
|
||||
return this.options.getProvider()?.isEnabled() ?? false;
|
||||
@@ -249,6 +256,7 @@ export class GlanceHost {
|
||||
const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0);
|
||||
const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop()));
|
||||
boardImage.composeInto(full, x, y);
|
||||
this.boardRect = { x, y, width: boardImage.width, height: boardImage.height };
|
||||
return full;
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -23,12 +23,25 @@ export function glassesDisplayLabel(state: GlassesDisplayState): string {
|
||||
if (state.silentMode) {
|
||||
return "Silent mode";
|
||||
}
|
||||
if (!state.screenOn) {
|
||||
if (glassesDisplayAsleep(state)) {
|
||||
return "Display off";
|
||||
}
|
||||
return state.foregroundTitle || "Launcher";
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the label reads "Display off": the screen is dark but the glasses
|
||||
* (or the preview standing in for them) would light it on a wake gesture.
|
||||
*/
|
||||
export function glassesDisplayAsleep(state: GlassesDisplayState): boolean {
|
||||
return (
|
||||
state.phase !== "charging" &&
|
||||
(state.phase === "connected" || state.previewMode) &&
|
||||
!state.silentMode &&
|
||||
!state.screenOn
|
||||
);
|
||||
}
|
||||
|
||||
export function chargingDisplayLabel(battery: number | null): string {
|
||||
return validBatteryLevel(battery) ? `Charging · G2 ${battery}%` : "Charging";
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import { launcherEntries } from '../apps/launcher'
|
||||
import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
|
||||
import { isInstalledPackagePresent } from '../apps/evenhub/updates'
|
||||
import { openEvenHubStoreForPackage } from '../apps/evenhub'
|
||||
import { launchInstalledPackage, closeRunningPackage } from '../apps/evenhub/manager'
|
||||
import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager'
|
||||
import { registerSystemTools } from '../assistant/system-tools'
|
||||
import { registerWindowTools } from '../assistant/window-tools'
|
||||
import { registerNavigateTools } from '../assistant/navigate-tools'
|
||||
@@ -16,6 +16,7 @@ import { IosNavigationSensors } from '../native/ios-navigation-sensors'
|
||||
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
|
||||
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { voiceActivity } from '../ui/shell/voice-activity'
|
||||
import { nightscoutBridge } from '../native/nightscout-bridge'
|
||||
import { weatherBridge } from '../native/weather'
|
||||
import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios'
|
||||
@@ -25,6 +26,7 @@ import { GlanceHost, type GlanceDisplay } from './glance-host'
|
||||
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen'
|
||||
import { OsEventTypeList } from './events'
|
||||
import { loadDeviceAddresses } from './device-addresses'
|
||||
import { updateGlassesPresence } from './glasses-presence'
|
||||
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
|
||||
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
@@ -39,18 +41,23 @@ import { flattenPlanesWithDraws, planesFingerprint, type Plane } from '../graphi
|
||||
import { prepareFrameDraws } from '../graphics/glyph-wire'
|
||||
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
||||
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
|
||||
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
||||
import { noopLayerActions, type ContinuousCaptureOptions, type LayerActions } from '../ui/layers'
|
||||
import { TextViewerLayer } from '../apps/files/text-viewer'
|
||||
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry'
|
||||
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry'
|
||||
import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings'
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { isWelcomeSoundPending, setWelcomeSoundPending } from '../phone-ui/onboarding-state'
|
||||
import { findSoundEffect, playSoundEffect } from '../ui/sound-effects'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
|
||||
const SHELL_SURFACE_ID = 'shell'
|
||||
// voiceActivity holder for continuous capture (Transcribe), as on Android.
|
||||
const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture'
|
||||
const SHELL_SURFACE_Z_ORDER = 1
|
||||
const LOCK_SURFACE_Z_ORDER = 1000
|
||||
/** The shell render loop waits this long for the glasses to take a frame before moving on. */
|
||||
@@ -114,6 +121,8 @@ export class IosPreviewController {
|
||||
appendLog: message => this.logBluetooth(message),
|
||||
})
|
||||
private previewTimer: ReturnType<typeof setTimeout> | null = null
|
||||
/** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */
|
||||
private previewCrop: ScreenRect | null = null
|
||||
private clockTimer: ReturnType<typeof setInterval> | null = null
|
||||
private offSettings: (() => void) | null = null
|
||||
private lastBrightness: string | null = null
|
||||
@@ -126,6 +135,8 @@ export class IosPreviewController {
|
||||
private nextWorkerFrame = 0
|
||||
private inputQueue: Promise<void> = Promise.resolve()
|
||||
private lastLayout = ''
|
||||
/** The phone-UI button last pushed to the page (onEvenHubPhoneUiChanged). */
|
||||
private evenHubPhoneUi: PhoneUiButton | null = null
|
||||
private readonly appHosts = new Map<string, WorkerAppHost>()
|
||||
private readonly inProcessApps = new Map<string, InProcessWindow>()
|
||||
private readonly batteryObservers: any[] = []
|
||||
@@ -137,6 +148,20 @@ export class IosPreviewController {
|
||||
private acknowledgedFrames = 0
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Windows that want continuous capture, and what each needs of it (a
|
||||
// recording, a multilingual provider). They share one capture, which runs
|
||||
// while any of them wants it.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
|
||||
// transcriptionKey of the running continuous capture's provider; one that no
|
||||
// longer matches what its windows need is drained and restarted.
|
||||
private continuousTranscription = ''
|
||||
// Continuous capture should (re)start once audio is available: a window
|
||||
// asked, the glasses connected or disconnected, or the glasses lock or the
|
||||
// app lifecycle cut it short. A capture that fails doesn't set it, so a
|
||||
// broken start is retried on the next such change rather than in a loop.
|
||||
private continuousPending = false
|
||||
private continuousCaptureToken = 0
|
||||
private sessionConnected = false
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
||||
@@ -144,7 +169,8 @@ export class IosPreviewController {
|
||||
disconnect: () => this.disconnect(),
|
||||
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
||||
startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."),
|
||||
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client),
|
||||
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
||||
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
||||
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
|
||||
@@ -160,7 +186,8 @@ export class IosPreviewController {
|
||||
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
|
||||
private readonly onError: (message: string) => void,
|
||||
private readonly onConnectionState: (state: SessionState) => void = () => {},
|
||||
private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {}) {
|
||||
private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {},
|
||||
private readonly onEvenHubPhoneUiChanged: (button: PhoneUiButton | null) => void = () => {}) {
|
||||
this.ensurePreviewDisplay()
|
||||
shell.configure({
|
||||
actions: this.actions,
|
||||
@@ -175,7 +202,12 @@ export class IosPreviewController {
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
|
||||
onSwitcherBarHeightChanged: () => {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
},
|
||||
onScreenStateChanged: on => {
|
||||
if (on) this.glance.dismiss()
|
||||
void this.display?.setScreenBlanked(!on).catch(error => this.fail(error))
|
||||
@@ -218,11 +250,15 @@ export class IosPreviewController {
|
||||
assistantAvailable: () => shell.isAssistantAvailable(),
|
||||
assistant: text => shell.sendToAssistant(text),
|
||||
})
|
||||
// The header's EvenHub phone-UI button hides while that UI is showing.
|
||||
onPhoneShownChanged(() => this.syncEvenHubPhoneUi())
|
||||
registerSystemTools()
|
||||
registerWindowTools({ apps: ALL_APPS.filter(app => !iosAppUnavailableReason(app.appId)),
|
||||
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
|
||||
registerNavigateTools(id => this.launchApp(id))
|
||||
registerRoamTools(id => this.launchApp(id))
|
||||
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
|
||||
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
|
||||
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
|
||||
}
|
||||
|
||||
@@ -277,16 +313,35 @@ export class IosPreviewController {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
this.syncEvenHubPhoneUi()
|
||||
this.syncContinuousCapture()
|
||||
}
|
||||
pause(): void {
|
||||
if (!this.active) return
|
||||
this.active = false
|
||||
if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true
|
||||
iosVoiceInput.stopPhoneCapture()
|
||||
// The phone preview is hidden; the glasses still own their screen and input.
|
||||
this.syncRuntime()
|
||||
this.logBluetooth(`Phone background; glasses ${this.state.phase}`)
|
||||
this.flushBluetoothLog()
|
||||
}
|
||||
/** Overlay an EvenHub app's phone UI (the header's app-icon button). */
|
||||
showEvenHubPhoneUi(windowId: string): void {
|
||||
showOnPhone(windowId)
|
||||
}
|
||||
/**
|
||||
* Tell the page when the foreground window's EvenHub phone-UI button
|
||||
* changes. A paused controller skips this and catches up on resume.
|
||||
*/
|
||||
private syncEvenHubPhoneUi(): void {
|
||||
if (!this.active) return
|
||||
const button = phoneUiButton()
|
||||
const current = this.evenHubPhoneUi
|
||||
if (current?.windowId === button?.windowId && current?.icon === button?.icon) return
|
||||
this.evenHubPhoneUi = button
|
||||
this.onEvenHubPhoneUiChanged(button)
|
||||
}
|
||||
private get runtimeNeeded(): boolean {
|
||||
return this.active || ['connected', 'connecting', 'retrying', 'disconnecting'].includes(this.state.phase)
|
||||
}
|
||||
@@ -373,6 +428,7 @@ export class IosPreviewController {
|
||||
// a session's first snapshot) wakes the screen, to the lock notice if locked.
|
||||
const putOn = wearing && this.glassesWorn === false
|
||||
this.glassesWorn = wearing
|
||||
this.syncPresence()
|
||||
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
|
||||
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
|
||||
if (putOn && this.runtimeRunning && !shell.isScreenOn()) shell.wake('sidebar')
|
||||
@@ -382,8 +438,10 @@ export class IosPreviewController {
|
||||
this.glassesLocked = locked
|
||||
if (locked) {
|
||||
this.glance.dismiss()
|
||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.')
|
||||
}
|
||||
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
|
||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
||||
'Glasses locked. Transcription resumes when you unlock your phone.')
|
||||
} else this.syncContinuousCapture()
|
||||
void (async () => {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
@@ -418,10 +476,11 @@ export class IosPreviewController {
|
||||
}
|
||||
}
|
||||
/** Submit a painted frame for an in-process window (e.g. the launcher). */
|
||||
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
if (!display) return
|
||||
// The window can close (removing its surface) while this frame waits.
|
||||
if (!display || !live()) return
|
||||
const fingerprint = planesFingerprint(planes)
|
||||
const { image, draws } = flattenPlanesWithDraws(planes)
|
||||
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
|
||||
@@ -481,8 +540,10 @@ export class IosPreviewController {
|
||||
this.previewTimer = null
|
||||
if (!this.active) return
|
||||
try {
|
||||
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green')
|
||||
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect())
|
||||
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop)
|
||||
if (!image) return
|
||||
this.previewCrop = crop
|
||||
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
|
||||
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
||||
} catch (error) { this.fail(error) }
|
||||
@@ -536,7 +597,8 @@ export class IosPreviewController {
|
||||
await shell.receiveInput(event)
|
||||
this.requestShellRender()
|
||||
}
|
||||
private async mirrorTap(nx: number, ny: number): Promise<void> {
|
||||
private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise<void> {
|
||||
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
|
||||
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||
const window = shell.foregroundWindow()
|
||||
@@ -610,7 +672,7 @@ export class IosPreviewController {
|
||||
// The explicit render below supplies a fresh frame after configuration.
|
||||
let surfaceReady = false
|
||||
const app = create({
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(),
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
|
||||
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
|
||||
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
|
||||
reconfigureSurface: () => {
|
||||
@@ -657,11 +719,6 @@ export class IosPreviewController {
|
||||
this.appHosts.set(appId, host)
|
||||
return host
|
||||
}
|
||||
cycleDisplayMode(): void {
|
||||
const values = DISPLAY_MODE_VALUES
|
||||
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
|
||||
}
|
||||
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
|
||||
get connectionState(): SessionState | null { return this.sessionCreated ? this.state : null }
|
||||
get connectionDetails(): string {
|
||||
const state = this.connectionState
|
||||
@@ -672,13 +729,19 @@ export class IosPreviewController {
|
||||
this.logBluetooth(status)
|
||||
this.emitState()
|
||||
}
|
||||
/** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */
|
||||
private syncPresence(): void {
|
||||
updateGlassesPresence({ connected: this.state.phase === 'connected', worn: this.glassesWorn, charging: this.state.charging ?? false })
|
||||
}
|
||||
private emitState(): void {
|
||||
if (this.state.phase !== 'connected') resetRingInputFilter()
|
||||
this.pushBrightness()
|
||||
this.maybePlayWelcomeSound(this.state)
|
||||
this.maybeLaunchOnboarding(this.state)
|
||||
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
||||
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
this.syncContinuousSession()
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
||||
this.syncPresence()
|
||||
this.syncRuntime()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
@@ -695,7 +758,10 @@ export class IosPreviewController {
|
||||
this.firmwareCompatible = false
|
||||
this.update('connecting', 'Finding configured devices…')
|
||||
let identifiers
|
||||
try { identifiers = await resolveIosPeripherals(addresses) }
|
||||
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
|
||||
// fails and retries every few seconds, starving the glasses link.
|
||||
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
|
||||
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
|
||||
catch (failure) {
|
||||
this.update('error', failure instanceof Error ? failure.message : String(failure))
|
||||
return
|
||||
@@ -763,7 +829,7 @@ export class IosPreviewController {
|
||||
if (this.communicator !== communicator) return
|
||||
this.state = { ...this.state, frames: this.state.frames + 1 }
|
||||
this.onActivity()
|
||||
this.maybePlayWelcomeSound(this.state)
|
||||
this.maybeLaunchOnboarding(this.state)
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.schedulePreviewUpdate()
|
||||
}),
|
||||
@@ -871,23 +937,20 @@ export class IosPreviewController {
|
||||
this.glance.reset()
|
||||
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
||||
}
|
||||
private maybePlayWelcomeSound(state: SessionState): void {
|
||||
private maybeLaunchOnboarding(state: SessionState): void {
|
||||
const firstNewFrame = state.frames > this.acknowledgedFrames
|
||||
this.acknowledgedFrames = state.frames
|
||||
// Match Android: wait for an acknowledged frame before consuming the jingle.
|
||||
if (!firstNewFrame || state.phase !== 'connected' || !isWelcomeSoundPending()) return
|
||||
setWelcomeSoundPending(false)
|
||||
const effect = findSoundEffect('questcomplete')
|
||||
if (effect) void playSoundEffect(effect, payload => this.actions.playBuzzerSequence(payload),
|
||||
ms => new Promise(resolve => setTimeout(resolve, ms)))
|
||||
.catch(error => this.logBluetooth(`Welcome sound failed: ${error}`))
|
||||
// Match Android: wait for an acknowledged frame (a warm session, so the
|
||||
// onboarding's welcome jingle isn't dropped) before opening it.
|
||||
if (!firstNewFrame || state.phase !== 'connected' || !claimGlassesOnboardingAutoLaunch()) return
|
||||
void this.launchApp(ONBOARDING_APP_ID)
|
||||
}
|
||||
startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() }
|
||||
private async prepareVoiceCapture(): Promise<boolean> {
|
||||
if (this.glassesLocked) return false
|
||||
const usePhoneMic = this.state.phase !== 'connected'
|
||||
if (usePhoneMic && !this.active) return false
|
||||
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic)
|
||||
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider())
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active)
|
||||
}
|
||||
@@ -895,14 +958,116 @@ export class IosPreviewController {
|
||||
if (this.glassesLocked) return
|
||||
const log = (message: string) => this.logBluetooth(message)
|
||||
const communicator = this.communicator
|
||||
const voice = this.voiceProvider()
|
||||
if (communicator && this.state.phase === 'connected') {
|
||||
await iosVoiceInput.startGlassesCapture(communicator, log, endpointing)
|
||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing)
|
||||
} else if (this.active) {
|
||||
// Recheck microphone permission if the glasses disconnected after prepare.
|
||||
if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return
|
||||
await iosVoiceInput.startPhoneCapture(log, endpointing)
|
||||
if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
||||
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
||||
* Transcribe audio is on.
|
||||
*/
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options)
|
||||
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||
this.continuousClients.delete(client)
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
this.syncContinuousRecording()
|
||||
// A listening capture is shared as it is; otherwise start one (after any drain).
|
||||
if (!iosVoiceInput.continuousSource) this.continuousPending = true
|
||||
this.syncContinuousCapture()
|
||||
}
|
||||
private stopContinuousVoiceCapture(client: string): void {
|
||||
if (!this.continuousClients.delete(client)) return
|
||||
this.syncContinuousRecording()
|
||||
// The window that needed its own provider may be the one that left.
|
||||
if (this.continuousClients.size > 0) { this.syncContinuousCapture(); return }
|
||||
// A start still awaiting permission or the mic backs off.
|
||||
++this.continuousCaptureToken
|
||||
this.continuousPending = false
|
||||
this.syncContinuousClaim()
|
||||
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
|
||||
}
|
||||
/** Connecting or losing the glasses moves continuous capture to the new audio source. */
|
||||
private syncContinuousSession(): void {
|
||||
const connected = this.state.phase === 'connected'
|
||||
if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true
|
||||
this.sessionConnected = connected
|
||||
this.syncContinuousCapture()
|
||||
}
|
||||
/**
|
||||
* Keep continuous capture running while a window wants it: start a pending
|
||||
* one once audio is available, and move a phone capture to the glasses once
|
||||
* they connect. A glasses session supplies the audio, even while it
|
||||
* reconnects (as on Android); the phone mic stands in only without one, while
|
||||
* Faceclaw is in the foreground. Cheap enough to call on every state change.
|
||||
*/
|
||||
private syncContinuousCapture(): void {
|
||||
this.syncContinuousClaim()
|
||||
if (this.continuousClients.size === 0 || this.glassesLocked) return
|
||||
const glassesReady = !!this.communicator && this.state.phase === 'connected'
|
||||
if (iosVoiceInput.continuousActive) {
|
||||
// Draining or listening already. A phone capture from before the glasses
|
||||
// connected is drained here, and its end restarts it on the glasses; so
|
||||
// is one transcribing with another provider than its windows now need.
|
||||
const retarget = this.continuousTranscription !== transcriptionKey(this.voiceProvider())
|
||||
if (iosVoiceInput.continuousSource && (retarget || glassesReady && iosVoiceInput.continuousSource === 'phone')) {
|
||||
this.continuousPending = true
|
||||
void iosVoiceInput.stopContinuousCapture()
|
||||
}
|
||||
return
|
||||
}
|
||||
if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return
|
||||
this.continuousPending = false
|
||||
void this.beginContinuousCapture()
|
||||
}
|
||||
private async beginContinuousCapture(): Promise<void> {
|
||||
const voice = this.voiceProvider()
|
||||
if (!iosVoiceInput.supportsContinuous(voice)) return
|
||||
this.continuousTranscription = transcriptionKey(voice)
|
||||
// A stop during an await below bumps the token; this start then backs off.
|
||||
const token = ++this.continuousCaptureToken
|
||||
const current = () => token === this.continuousCaptureToken
|
||||
const log = (message: string) => this.logBluetooth(message)
|
||||
// Claimed before the capture starts so other mic users stand down first, as on Android.
|
||||
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
|
||||
const communicator = this.communicator
|
||||
if (communicator) {
|
||||
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true)
|
||||
} else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) {
|
||||
await iosVoiceInput.startPhoneCapture(voice, log, false, true)
|
||||
}
|
||||
if (current()) this.syncContinuousClaim()
|
||||
}
|
||||
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
||||
private syncContinuousClaim(): void {
|
||||
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||
}
|
||||
/** Record to the first client's requested name while Save Transcribe audio is on. */
|
||||
private syncContinuousRecording(): void {
|
||||
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
|
||||
iosVoiceInput.recordTo(name)
|
||||
}
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs a multilingual provider (Translate) sets it for every
|
||||
* capture while it is open, with Whisper left to detect the language.
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
|
||||
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image";
|
||||
import { previewScalingSetting } from "../ui/dashboard-settings";
|
||||
import { phonePreviewRect, usedScreenRect, type ScreenRect } from "../ui/shell/geometry";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
|
||||
/**
|
||||
* The screen area the phone's mirror shows (Settings > Phone display >
|
||||
* Preview scaling), or null for the whole panel. Scaled zooms to the
|
||||
* Glanceboard while it shows, else to the area the shell uses around the
|
||||
* foreground window, centred at the panel's aspect. A locked display gets
|
||||
* the whole panel: its message box is centred in the panel, not the band.
|
||||
*/
|
||||
export function phonePreviewCrop(locked: boolean, glanceRect: ScreenRect | null): ScreenRect | null {
|
||||
if (locked || previewScalingSetting.get() === "raw") return null;
|
||||
const window = shell.foregroundWindow();
|
||||
const crop = phonePreviewRect(glanceRect ?? usedScreenRect(window?.heightMode ?? "min", window?.appId));
|
||||
const wholePanel = crop.x === 0 && crop.y === 0 && crop.width === G2_LENS_WIDTH && crop.height === G2_LENS_HEIGHT;
|
||||
return wholePanel ? null : crop;
|
||||
}
|
||||
|
||||
/**
|
||||
* A point on the mirror, as fractions of the mirror's width and height,
|
||||
* as fractions of the panel's, given the crop the mirror is showing.
|
||||
* Points in the black margins past the panel's edges come out below 0 or
|
||||
* above 1.
|
||||
*/
|
||||
export function mirrorToPanelFraction(crop: ScreenRect | null, nx: number, ny: number): { nx: number; ny: number } {
|
||||
if (!crop) return { nx, ny };
|
||||
return {
|
||||
nx: (crop.x + nx * crop.width) / G2_LENS_WIDTH,
|
||||
ny: (crop.y + ny * crop.height) / G2_LENS_HEIGHT,
|
||||
};
|
||||
}
|
||||
+3
-15
@@ -23,12 +23,11 @@
|
||||
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the
|
||||
* message shapes are documented in wear/PROTOCOL.md.
|
||||
*/
|
||||
import { flagAnalyticsEvent } from "../analytics/analytics";
|
||||
import { type AppDefinition } from "../apps/app-definition";
|
||||
import { getInstalledEvenHubApps, installedEvenHubAppId } from "../apps/evenhub/installed-apps";
|
||||
import { WEAR_PATHS, wearBridge, type WearMessage } from "../native/wear-bridge";
|
||||
import {
|
||||
DISPLAY_MODE_VALUES,
|
||||
displayModeSetting,
|
||||
onAnySettingChanged,
|
||||
watchCanUnlockSetting,
|
||||
watchCrownClockwiseNextSetting,
|
||||
@@ -87,7 +86,6 @@ const COMMANDS = [
|
||||
"connect",
|
||||
"disconnect",
|
||||
"close-assistant",
|
||||
"display-mode",
|
||||
] as const;
|
||||
type WearCommand = (typeof COMMANDS)[number];
|
||||
|
||||
@@ -167,6 +165,7 @@ export class WearRemote {
|
||||
// with it (no placeholder).
|
||||
if (changed) this.watchBatteryPollsUnanswered = 0;
|
||||
if (connection.reachable) {
|
||||
flagAnalyticsEvent("minimal", "device.wear.reachable");
|
||||
this.publishNow(true);
|
||||
this.pollWatchBattery(true);
|
||||
} else if (changed) {
|
||||
@@ -275,7 +274,6 @@ export class WearRemote {
|
||||
acceptsText: typeof window.receiveTextInput === "function",
|
||||
})),
|
||||
apps: this.listApps(),
|
||||
displayMode: displayModeSetting.get(),
|
||||
remoteEnabled: watchRemoteEnabledSetting.get(),
|
||||
crownClockwiseNext: watchCrownClockwiseNextSetting.get(),
|
||||
canUnlock: watchCanUnlockSetting.get(),
|
||||
@@ -326,6 +324,7 @@ export class WearRemote {
|
||||
return;
|
||||
}
|
||||
|
||||
flagAnalyticsEvent("minimal", "device.wear.used");
|
||||
switch (message.path) {
|
||||
case WEAR_PATHS.input:
|
||||
await this.handleInput(message.payload, ack);
|
||||
@@ -519,17 +518,6 @@ export class WearRemote {
|
||||
// but tell the watch which of the two happened.
|
||||
ack(true, shell.closeAssistant() ? "" : "The assistant was not open.");
|
||||
return;
|
||||
case "display-mode": {
|
||||
const value = String(payload.value ?? "");
|
||||
if (!(DISPLAY_MODE_VALUES as readonly string[]).includes(value)) {
|
||||
ack(false, `Unknown display mode: ${value}`);
|
||||
return;
|
||||
}
|
||||
displayModeSetting.set(value as (typeof DISPLAY_MODE_VALUES)[number]);
|
||||
this.schedulePublish();
|
||||
ack(true, `Display mode: ${value}`);
|
||||
return;
|
||||
}
|
||||
default:
|
||||
ack(false, `Unhandled command: ${command}`);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user