forked from Blendman974/faceclaw
Add an Apple Watch app that can control the glasses
This commit is contained in:
1 parent
c896ce9ac8
commit
2696f3b76c
46 files changed
+3877
-88
No files matched your search
@@ -29,7 +29,8 @@ import { OsEventTypeList } from './events'
|
||||
import { loadDeviceAddresses } from './device-addresses'
|
||||
import { updateGlassesPresence } from './glasses-presence'
|
||||
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
|
||||
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
|
||||
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy'
|
||||
import { WearRemote, type WearRemoteInputKind } from './wear-remote'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
||||
import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app'
|
||||
@@ -167,6 +168,8 @@ export class IosPreviewController {
|
||||
private continuousPending = false
|
||||
private continuousCaptureToken = 0
|
||||
private sessionConnected = false
|
||||
/** The Apple Watch remote; constructed last so it sees a fully wired shell. */
|
||||
private wearRemote: WearRemote | null = null
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
||||
@@ -210,7 +213,7 @@ export class IosPreviewController {
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
|
||||
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi(); this.wearRemote?.schedulePublish() },
|
||||
openNotificationsWindow: () =>
|
||||
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
|
||||
onSwitcherBarHeightChanged: () => {
|
||||
@@ -227,6 +230,7 @@ export class IosPreviewController {
|
||||
this.unblankAfterShellRender = on
|
||||
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
|
||||
this.requestShellRender()
|
||||
this.wearRemote?.schedulePublish()
|
||||
},
|
||||
})
|
||||
const launcher = createLauncherWindow({
|
||||
@@ -275,6 +279,20 @@ export class IosPreviewController {
|
||||
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
|
||||
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
|
||||
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
|
||||
this.wearRemote = new WearRemote({
|
||||
apps: ALL_APPS.filter(app => app.showInLauncher !== false && !iosAppUnavailableReason(app.appId)),
|
||||
injectInput: kind => this.injectWatchInput(kind),
|
||||
launchApp: id => this.launchApp(id),
|
||||
// The watch's Connect/Disconnect are explicit user choices, like the phone's own buttons.
|
||||
connect: () => { resumeAutoReconnect(); return this.connect(true) },
|
||||
disconnect: () => { suppressAutoReconnect(); return this.disconnect() },
|
||||
setGlassesLocked: locked => this.setGlassesLocked(locked),
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
previewInputActive: () => !this.communicator && this.runtimeRunning,
|
||||
getState: () => ({ phase: this.state.phase, status: this.state.status, glassesLocked: this.glassesLocked,
|
||||
glassesWorn: this.glassesWorn, silentMode: false, lastHeadsetBattery: this.state.battery }),
|
||||
appendLog: line => this.logBluetooth(line),
|
||||
})
|
||||
}
|
||||
|
||||
/** The compositor frames go to: the live session, else the headless preview. */
|
||||
@@ -443,6 +461,7 @@ export class IosPreviewController {
|
||||
// a session's first snapshot) wakes the screen, to the lock notice if locked.
|
||||
const putOn = wearing && this.glassesWorn === false
|
||||
this.glassesWorn = wearing
|
||||
this.wearRemote?.schedulePublish()
|
||||
this.syncPresence()
|
||||
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
|
||||
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
|
||||
@@ -451,6 +470,7 @@ export class IosPreviewController {
|
||||
private setGlassesLocked(locked: boolean): void {
|
||||
if (locked === this.glassesLocked) return
|
||||
this.glassesLocked = locked
|
||||
this.wearRemote?.schedulePublish()
|
||||
if (locked) {
|
||||
this.glance.dismiss()
|
||||
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
|
||||
@@ -599,7 +619,25 @@ export class IosPreviewController {
|
||||
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
|
||||
}).catch(error => this.fail(error))
|
||||
}
|
||||
/** Input from the Apple Watch: shares the controller queue with physical input. */
|
||||
private injectWatchInput(kind: WearRemoteInputKind): Promise<void> {
|
||||
const task = this.inputQueue.then(async () => {
|
||||
this.onActivity()
|
||||
// As on Android, and because the watch may be acting on a stale state snapshot: a dark
|
||||
// display takes a double-click (wake) or a tap/hold (Glanceboard), and scrolls mean nothing.
|
||||
if (!shell.isScreenOn() && (kind === 'scroll-up' || kind === 'scroll-down' || kind === 'short-then-long-press')) return
|
||||
// A real hold releases when the finger lifts; a discrete long-press releases at once.
|
||||
const type = kind === 'long-press-start' ? 'long-press' : kind
|
||||
await this.receiveInput(makeInputEvent({ type, source: 'watch' } as InputEventPayload))
|
||||
if (kind === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: 'watch' }))
|
||||
console.log(`[ios-preview] apple-watch ${kind}: ${shell.describeInputTarget()}`)
|
||||
})
|
||||
this.inputQueue = task.catch(error => this.fail(error))
|
||||
return task
|
||||
}
|
||||
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
|
||||
// Input can change the foreground app or the display state, and the shell doesn't announce those.
|
||||
this.wearRemote?.schedulePublish()
|
||||
if (!acceptInput(event)) return
|
||||
if (this.glassesLocked) {
|
||||
// Preserve the locked display's sleep/wake controls without dispatching
|
||||
@@ -807,6 +845,7 @@ export class IosPreviewController {
|
||||
this.syncRuntime()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
this.wearRemote?.schedulePublish()
|
||||
}
|
||||
/** The ring level arrives through the glasses, so like Android it is shown only while connected. */
|
||||
private pushBatteryLevels(): void {
|
||||
@@ -814,11 +853,16 @@ export class IosPreviewController {
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging,
|
||||
ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null })
|
||||
}
|
||||
async connect(): Promise<void> {
|
||||
if (!this.active || this.connecting || this.stopping) return
|
||||
/** `fromBackground`: the watch asked, and the phone may be in a pocket with the app suspended. */
|
||||
async connect(fromBackground = false): Promise<void> {
|
||||
if ((!this.active && !fromBackground) || this.connecting || this.stopping) return
|
||||
if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return
|
||||
const addresses = loadDeviceAddresses(), error = deviceAddressError(addresses)
|
||||
if (error) { this.onError(error); return }
|
||||
if (error) {
|
||||
if (this.active) this.onError(error)
|
||||
else this.logBluetooth(error)
|
||||
return
|
||||
}
|
||||
this.connecting = true
|
||||
this.sessionCreated = true
|
||||
try {
|
||||
@@ -882,6 +926,7 @@ export class IosPreviewController {
|
||||
this.pushBatteryLevels()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
this.wearRemote?.schedulePublish()
|
||||
}),
|
||||
communicator.onCaseState(state => {
|
||||
if (this.communicator !== communicator) return
|
||||
@@ -889,6 +934,7 @@ export class IosPreviewController {
|
||||
// the case's own charging is only known while the glasses are docked.
|
||||
shell.setBatteryLevels({ ...(state.battery !== null ? { case: state.battery } : {}),
|
||||
caseCharging: state.docked ? state.charging : null })
|
||||
this.wearRemote?.schedulePublish()
|
||||
}),
|
||||
communicator.onFirmwareInfo(info => {
|
||||
if (this.communicator !== communicator) return
|
||||
|
||||
+47
-13
@@ -1,5 +1,6 @@
|
||||
/**
|
||||
* The Wear OS watch as an input device and status mirror for the glasses.
|
||||
* The watch (Wear OS or Apple Watch) as an input device and status mirror for
|
||||
* the glasses.
|
||||
*
|
||||
* Inbound, the watch sends ring-style gestures (tap, double-tap, hold,
|
||||
* swipe/crown scrolls, "Hey Even") which feed the exact synthetic-ring path
|
||||
@@ -17,11 +18,14 @@
|
||||
* The watch's own battery flows the other way for the glasses' top-bar and
|
||||
* Glanceboard indicators: the phone polls it (the watch answers from a
|
||||
* listener service even while its app is closed) and the open watch app
|
||||
* pushes changes unprompted. The level lives in shell.getBatteryLevels()
|
||||
* beside the G2 and R1 ones and is cleared when no watch is reachable.
|
||||
* pushes changes unprompted. An Apple Watch app can't be woken to answer, so
|
||||
* it reports when it can instead (see wear/PROTOCOL.md). The level lives in
|
||||
* shell.getBatteryLevels() beside the G2 and R1 ones and is cleared when no
|
||||
* watch is reachable or the last report has gone stale.
|
||||
*
|
||||
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the
|
||||
* message shapes are documented in wear/PROTOCOL.md.
|
||||
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge) or
|
||||
* wear-bridge.ios.ts (FaceclawWatchBridge); the message shapes are documented
|
||||
* in wear/PROTOCOL.md.
|
||||
*/
|
||||
import { flagAnalyticsEvent } from "../analytics/analytics";
|
||||
import { type AppDefinition } from "../apps/app-definition";
|
||||
@@ -102,6 +106,10 @@ const WATCH_BATTERY_POLL_INTERVAL_MS = 5 * 60_000;
|
||||
// glasses connecting) would otherwise trigger back to back.
|
||||
const WATCH_BATTERY_POLL_MIN_GAP_MS = 20_000;
|
||||
const WATCH_BATTERY_URGENT_POLL_LIMIT = 3;
|
||||
// Wear OS answers every poll. A closed Apple Watch app only reports when the
|
||||
// system gives it background time, which can be hours apart, so a level this
|
||||
// old says more about the watch being gone than about its charge.
|
||||
const WATCH_BATTERY_MAX_AGE_MS = 4 * 60 * 60_000;
|
||||
|
||||
/** What the dashboard controller lends the watch remote. */
|
||||
export type WearRemoteHost = {
|
||||
@@ -113,6 +121,11 @@ export type WearRemoteHost = {
|
||||
disconnect: () => Promise<void>;
|
||||
setGlassesLocked: (locked: boolean, reason: string) => void;
|
||||
requestShellRender: () => void;
|
||||
/**
|
||||
* The shell is running on the phone's preview with no glasses involved, so
|
||||
* gestures have somewhere to go even though nothing is connected.
|
||||
*/
|
||||
previewInputActive?: () => boolean;
|
||||
getState: () => {
|
||||
phase: string;
|
||||
status: string;
|
||||
@@ -136,6 +149,7 @@ export class WearRemote {
|
||||
private lastAssistantSentAtMs = 0;
|
||||
private lastWatchBatteryPollAtMs = 0;
|
||||
private watchBatteryKnown = false;
|
||||
private watchBatteryReportedAtMs = 0;
|
||||
// Urgent polls that got no answer; a watch app too old to answer them
|
||||
// must not be poked every publish, so after a few the cadence drops back.
|
||||
private watchBatteryPollsUnanswered = 0;
|
||||
@@ -143,7 +157,7 @@ export class WearRemote {
|
||||
constructor(private readonly host: WearRemoteHost) {
|
||||
this.active = wearBridge.isAvailable();
|
||||
if (!this.active) {
|
||||
host.appendLog("watch: Google Play services unavailable; watch remote disabled");
|
||||
host.appendLog(`watch: ${global.isIOS ? "WatchConnectivity" : "Google Play services"} unavailable; watch remote disabled`);
|
||||
return;
|
||||
}
|
||||
wearBridge.onMessage((message) => {
|
||||
@@ -183,10 +197,25 @@ export class WearRemote {
|
||||
// Catch-all refresh for state the controller does not announce (cheap:
|
||||
// identical states are dropped on the Java side).
|
||||
setInterval(() => this.schedulePublish(), STATE_REFRESH_INTERVAL_MS);
|
||||
setInterval(() => this.pollWatchBattery(false), WATCH_BATTERY_POLL_INTERVAL_MS);
|
||||
setInterval(() => {
|
||||
this.expireWatchBattery();
|
||||
this.pollWatchBattery(false);
|
||||
}, WATCH_BATTERY_POLL_INTERVAL_MS);
|
||||
this.schedulePublish();
|
||||
}
|
||||
|
||||
/** Gestures have a display to act on: the glasses, or the phone's preview without them. */
|
||||
private takesInput(): boolean {
|
||||
const phase = this.host.getState().phase;
|
||||
return phase === "connected" || phase === "charging" || this.host.previewInputActive?.() === true;
|
||||
}
|
||||
|
||||
private expireWatchBattery(): void {
|
||||
if (this.watchBatteryKnown && Date.now() - this.watchBatteryReportedAtMs > WATCH_BATTERY_MAX_AGE_MS) {
|
||||
this.setWatchBattery(null, null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the reachable watch for its battery. `urgent` skips the poll gap for
|
||||
* a level that is not known yet (the glasses just connected, the watch
|
||||
@@ -257,6 +286,7 @@ export class WearRemote {
|
||||
phase: state.phase,
|
||||
status: state.status,
|
||||
connected: state.phase === "connected" || state.phase === "charging",
|
||||
preview: this.host.previewInputActive?.() === true,
|
||||
screenOn: shell.isScreenOn(),
|
||||
locked: state.glassesLocked,
|
||||
worn: state.glassesWorn,
|
||||
@@ -307,7 +337,11 @@ export class WearRemote {
|
||||
// acked (the watch sends it fire-and-forget, without a seq).
|
||||
this.watchBatteryPollsUnanswered = 0;
|
||||
const battery = readNumber(message.payload.battery, -1);
|
||||
const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100;
|
||||
// A report the watch left behind earlier says when it was taken; a live one is current.
|
||||
const reportedAtMs = readNumber(message.payload.at, Date.now());
|
||||
const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100 &&
|
||||
Date.now() - reportedAtMs <= WATCH_BATTERY_MAX_AGE_MS;
|
||||
this.watchBatteryReportedAtMs = reportedAtMs;
|
||||
this.setWatchBattery(valid ? battery : null, valid ? message.payload.charging === true : null);
|
||||
return;
|
||||
}
|
||||
@@ -315,8 +349,7 @@ export class WearRemote {
|
||||
// Touch-down ahead of the gesture that follows it. Fire-and-forget like
|
||||
// the battery (no seq, never acked): wherever that gesture would be
|
||||
// refused, the press is dropped silently and the gesture's ack says why.
|
||||
const phase = this.host.getState().phase;
|
||||
if (watchRemoteEnabledSetting.get() && (phase === "connected" || phase === "charging")) {
|
||||
if (watchRemoteEnabledSetting.get() && this.takesInput()) {
|
||||
await this.host.injectInput("ring-press");
|
||||
}
|
||||
return;
|
||||
@@ -378,8 +411,7 @@ export class WearRemote {
|
||||
return;
|
||||
}
|
||||
const kind = gesture as WearRemoteInputKind;
|
||||
const state = this.host.getState();
|
||||
if (state.phase !== "connected" && state.phase !== "charging") {
|
||||
if (!this.takesInput()) {
|
||||
ack(false, "The glasses are not connected.");
|
||||
return;
|
||||
}
|
||||
@@ -422,7 +454,7 @@ export class WearRemote {
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (!connected) {
|
||||
if (!this.takesInput()) {
|
||||
ack(false, "The glasses are not connected.");
|
||||
return;
|
||||
}
|
||||
@@ -561,6 +593,8 @@ export class WearRemote {
|
||||
}
|
||||
|
||||
function readNumber(value: unknown, fallback: number): number {
|
||||
// Number(null) is 0: a watch reporting an unknown battery would read as empty.
|
||||
if (value === null) return fallback;
|
||||
const numeric = typeof value === "number" ? value : Number(value);
|
||||
return Number.isFinite(numeric) ? numeric : fallback;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user