Merge branch 'main' into rayneo-io-support

This commit is contained in:
James Babcock committed 2026-11-04 03:11:15 -08:00
commit 739e81c8a6
47 files changed
+3967 -90

No files matched your search

+51 -5
View File
@@ -29,7 +29,8 @@ import { OsEventTypeList } from './events'
import { loadDeviceAddresses } from './device-addresses'
import { updateGlassesPresence } from './glasses-presence'
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy'
import { WearRemote, type WearRemoteInputKind } from './wear-remote'
import { deviceAddressError } from './ios-peripheral-identity'
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app'
@@ -167,6 +168,8 @@ export class IosPreviewController {
private continuousPending = false
private continuousCaptureToken = 0
private sessionConnected = false
/** The Apple Watch remote; constructed last so it sees a fully wired shell. */
private wearRemote: WearRemote | null = null
private readonly actions: LayerActions = {
...noopLayerActions,
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
@@ -210,7 +213,7 @@ export class IosPreviewController {
prepareVoiceCapture: () => this.prepareVoiceCapture(),
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi(); this.wearRemote?.schedulePublish() },
openNotificationsWindow: () =>
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
onSwitcherBarHeightChanged: () => {
@@ -227,6 +230,7 @@ export class IosPreviewController {
this.unblankAfterShellRender = on
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
this.requestShellRender()
this.wearRemote?.schedulePublish()
},
})
const launcher = createLauncherWindow({
@@ -275,6 +279,20 @@ export class IosPreviewController {
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
this.wearRemote = new WearRemote({
apps: ALL_APPS.filter(app => app.showInLauncher !== false && !iosAppUnavailableReason(app.appId)),
injectInput: kind => this.injectWatchInput(kind),
launchApp: id => this.launchApp(id),
// The watch's Connect/Disconnect are explicit user choices, like the phone's own buttons.
connect: () => { resumeAutoReconnect(); return this.connect(true) },
disconnect: () => { suppressAutoReconnect(); return this.disconnect() },
setGlassesLocked: locked => this.setGlassesLocked(locked),
requestShellRender: () => this.requestShellRender(),
previewInputActive: () => !this.communicator && this.runtimeRunning,
getState: () => ({ phase: this.state.phase, status: this.state.status, glassesLocked: this.glassesLocked,
glassesWorn: this.glassesWorn, silentMode: false, lastHeadsetBattery: this.state.battery }),
appendLog: line => this.logBluetooth(line),
})
}
/** The compositor frames go to: the live session, else the headless preview. */
@@ -443,6 +461,7 @@ export class IosPreviewController {
// a session's first snapshot) wakes the screen, to the lock notice if locked.
const putOn = wearing && this.glassesWorn === false
this.glassesWorn = wearing
this.wearRemote?.schedulePublish()
this.syncPresence()
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
@@ -451,6 +470,7 @@ export class IosPreviewController {
private setGlassesLocked(locked: boolean): void {
if (locked === this.glassesLocked) return
this.glassesLocked = locked
this.wearRemote?.schedulePublish()
if (locked) {
this.glance.dismiss()
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
@@ -599,7 +619,25 @@ export class IosPreviewController {
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
}).catch(error => this.fail(error))
}
/** Input from the Apple Watch: shares the controller queue with physical input. */
private injectWatchInput(kind: WearRemoteInputKind): Promise<void> {
const task = this.inputQueue.then(async () => {
this.onActivity()
// As on Android, and because the watch may be acting on a stale state snapshot: a dark
// display takes a double-click (wake) or a tap/hold (Glanceboard), and scrolls mean nothing.
if (!shell.isScreenOn() && (kind === 'scroll-up' || kind === 'scroll-down' || kind === 'short-then-long-press')) return
// A real hold releases when the finger lifts; a discrete long-press releases at once.
const type = kind === 'long-press-start' ? 'long-press' : kind
await this.receiveInput(makeInputEvent({ type, source: 'watch' } as InputEventPayload))
if (kind === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: 'watch' }))
console.log(`[ios-preview] apple-watch ${kind}: ${shell.describeInputTarget()}`)
})
this.inputQueue = task.catch(error => this.fail(error))
return task
}
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
// Input can change the foreground app or the display state, and the shell doesn't announce those.
this.wearRemote?.schedulePublish()
if (!acceptInput(event)) return
if (this.glassesLocked) {
// Preserve the locked display's sleep/wake controls without dispatching
@@ -807,6 +845,7 @@ export class IosPreviewController {
this.syncRuntime()
if (this.active) this.onConnectionState({ ...this.state })
this.requestShellRender()
this.wearRemote?.schedulePublish()
}
/** The ring level arrives through the glasses, so like Android it is shown only while connected. */
private pushBatteryLevels(): void {
@@ -814,11 +853,16 @@ export class IosPreviewController {
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging,
ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null })
}
async connect(): Promise<void> {
if (!this.active || this.connecting || this.stopping) return
/** `fromBackground`: the watch asked, and the phone may be in a pocket with the app suspended. */
async connect(fromBackground = false): Promise<void> {
if ((!this.active && !fromBackground) || this.connecting || this.stopping) return
if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return
const addresses = loadDeviceAddresses(), error = deviceAddressError(addresses)
if (error) { this.onError(error); return }
if (error) {
if (this.active) this.onError(error)
else this.logBluetooth(error)
return
}
this.connecting = true
this.sessionCreated = true
try {
@@ -882,6 +926,7 @@ export class IosPreviewController {
this.pushBatteryLevels()
if (this.active) this.onConnectionState({ ...this.state })
this.requestShellRender()
this.wearRemote?.schedulePublish()
}),
communicator.onCaseState(state => {
if (this.communicator !== communicator) return
@@ -889,6 +934,7 @@ export class IosPreviewController {
// the case's own charging is only known while the glasses are docked.
shell.setBatteryLevels({ ...(state.battery !== null ? { case: state.battery } : {}),
caseCharging: state.docked ? state.charging : null })
this.wearRemote?.schedulePublish()
}),
communicator.onFirmwareInfo(info => {
if (this.communicator !== communicator) return
+51 -13
View File
@@ -1,5 +1,6 @@
/**
* The Wear OS watch as an input device and status mirror for the glasses.
* The watch (Wear OS or Apple Watch) as an input device and status mirror for
* the glasses.
*
* Inbound, the watch sends ring-style gestures (tap, double-tap, hold,
* swipe/crown scrolls, "Hey Even") which feed the exact synthetic-ring path
@@ -17,11 +18,14 @@
* The watch's own battery flows the other way for the glasses' top-bar and
* Glanceboard indicators: the phone polls it (the watch answers from a
* listener service even while its app is closed) and the open watch app
* pushes changes unprompted. The level lives in shell.getBatteryLevels()
* beside the G2 and R1 ones and is cleared when no watch is reachable.
* pushes changes unprompted. An Apple Watch app can't be woken to answer, so
* it reports when it can instead (see wear/PROTOCOL.md). The level lives in
* shell.getBatteryLevels() beside the G2 and R1 ones and is cleared when no
* watch is reachable or the last report has gone stale.
*
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the
* message shapes are documented in wear/PROTOCOL.md.
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge) or
* wear-bridge.ios.ts (FaceclawWatchBridge); the message shapes are documented
* in wear/PROTOCOL.md.
*/
import { flagAnalyticsEvent } from "../analytics/analytics";
import { type AppDefinition } from "../apps/app-definition";
@@ -33,6 +37,8 @@ import {
watchCrownClockwiseNextSetting,
watchMirrorAssistantSetting,
watchRemoteEnabledSetting,
watchSoundEffectsSetting,
watchVibrateOnInputSetting,
} from "../ui/dashboard-settings";
import { shell, type AssistantActivityEvent } from "../ui/shell/shell";
import { voiceActivity } from "../ui/shell/voice-activity";
@@ -102,6 +108,10 @@ const WATCH_BATTERY_POLL_INTERVAL_MS = 5 * 60_000;
// glasses connecting) would otherwise trigger back to back.
const WATCH_BATTERY_POLL_MIN_GAP_MS = 20_000;
const WATCH_BATTERY_URGENT_POLL_LIMIT = 3;
// Wear OS answers every poll. A closed Apple Watch app only reports when the
// system gives it background time, which can be hours apart, so a level this
// old says more about the watch being gone than about its charge.
const WATCH_BATTERY_MAX_AGE_MS = 4 * 60 * 60_000;
/** What the dashboard controller lends the watch remote. */
export type WearRemoteHost = {
@@ -113,6 +123,11 @@ export type WearRemoteHost = {
disconnect: () => Promise<void>;
setGlassesLocked: (locked: boolean, reason: string) => void;
requestShellRender: () => void;
/**
* The shell is running on the phone's preview with no glasses involved, so
* gestures have somewhere to go even though nothing is connected.
*/
previewInputActive?: () => boolean;
getState: () => {
phase: string;
status: string;
@@ -136,6 +151,7 @@ export class WearRemote {
private lastAssistantSentAtMs = 0;
private lastWatchBatteryPollAtMs = 0;
private watchBatteryKnown = false;
private watchBatteryReportedAtMs = 0;
// Urgent polls that got no answer; a watch app too old to answer them
// must not be poked every publish, so after a few the cadence drops back.
private watchBatteryPollsUnanswered = 0;
@@ -143,7 +159,7 @@ export class WearRemote {
constructor(private readonly host: WearRemoteHost) {
this.active = wearBridge.isAvailable();
if (!this.active) {
host.appendLog("watch: Google Play services unavailable; watch remote disabled");
host.appendLog(`watch: ${global.isIOS ? "WatchConnectivity" : "Google Play services"} unavailable; watch remote disabled`);
return;
}
wearBridge.onMessage((message) => {
@@ -183,10 +199,25 @@ export class WearRemote {
// Catch-all refresh for state the controller does not announce (cheap:
// identical states are dropped on the Java side).
setInterval(() => this.schedulePublish(), STATE_REFRESH_INTERVAL_MS);
setInterval(() => this.pollWatchBattery(false), WATCH_BATTERY_POLL_INTERVAL_MS);
setInterval(() => {
this.expireWatchBattery();
this.pollWatchBattery(false);
}, WATCH_BATTERY_POLL_INTERVAL_MS);
this.schedulePublish();
}
/** Gestures have a display to act on: the glasses, or the phone's preview without them. */
private takesInput(): boolean {
const phase = this.host.getState().phase;
return phase === "connected" || phase === "charging" || this.host.previewInputActive?.() === true;
}
private expireWatchBattery(): void {
if (this.watchBatteryKnown && Date.now() - this.watchBatteryReportedAtMs > WATCH_BATTERY_MAX_AGE_MS) {
this.setWatchBattery(null, null);
}
}
/**
* Ask the reachable watch for its battery. `urgent` skips the poll gap for
* a level that is not known yet (the glasses just connected, the watch
@@ -257,6 +288,7 @@ export class WearRemote {
phase: state.phase,
status: state.status,
connected: state.phase === "connected" || state.phase === "charging",
preview: this.host.previewInputActive?.() === true,
screenOn: shell.isScreenOn(),
locked: state.glassesLocked,
worn: state.glassesWorn,
@@ -278,6 +310,8 @@ export class WearRemote {
apps: this.listApps(),
remoteEnabled: watchRemoteEnabledSetting.get(),
crownClockwiseNext: watchCrownClockwiseNextSetting.get(),
vibrateOnInput: watchVibrateOnInputSetting.get(),
soundEffects: watchSoundEffectsSetting.get(),
canUnlock: watchCanUnlockSetting.get(),
mirrorAssistant: watchMirrorAssistantSetting.get(),
assistantAvailable: shell.isAssistantAvailable(),
@@ -307,7 +341,11 @@ export class WearRemote {
// acked (the watch sends it fire-and-forget, without a seq).
this.watchBatteryPollsUnanswered = 0;
const battery = readNumber(message.payload.battery, -1);
const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100;
// A report the watch left behind earlier says when it was taken; a live one is current.
const reportedAtMs = readNumber(message.payload.at, Date.now());
const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100 &&
Date.now() - reportedAtMs <= WATCH_BATTERY_MAX_AGE_MS;
this.watchBatteryReportedAtMs = reportedAtMs;
this.setWatchBattery(valid ? battery : null, valid ? message.payload.charging === true : null);
return;
}
@@ -315,8 +353,7 @@ export class WearRemote {
// Touch-down ahead of the gesture that follows it. Fire-and-forget like
// the battery (no seq, never acked): wherever that gesture would be
// refused, the press is dropped silently and the gesture's ack says why.
const phase = this.host.getState().phase;
if (watchRemoteEnabledSetting.get() && (phase === "connected" || phase === "charging")) {
if (watchRemoteEnabledSetting.get() && this.takesInput()) {
await this.host.injectInput("ring-press");
}
return;
@@ -378,8 +415,7 @@ export class WearRemote {
return;
}
const kind = gesture as WearRemoteInputKind;
const state = this.host.getState();
if (state.phase !== "connected" && state.phase !== "charging") {
if (!this.takesInput()) {
ack(false, "The glasses are not connected.");
return;
}
@@ -422,7 +458,7 @@ export class WearRemote {
default:
break;
}
if (!connected) {
if (!this.takesInput()) {
ack(false, "The glasses are not connected.");
return;
}
@@ -561,6 +597,8 @@ export class WearRemote {
}
function readNumber(value: unknown, fallback: number): number {
// Number(null) is 0: a watch reporting an unknown battery would read as empty.
if (value === null) return fallback;
const numeric = typeof value === "number" ? value : Number(value);
return Number.isFinite(numeric) ? numeric : fallback;
}