From 344e140e920fd91717e64db0e457d6026c7a9de2 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 15 Nov 2026 22:21:12 -0800 Subject: [PATCH] Configurable custom keybindings for bluetooth keyboards paired with the glasses --- app/apps/terminal/terminal-app.worker.ts | 16 +- app/g2/dashboard-controller.ts | 51 ++-- app/g2/ios-preview-controller.ts | 39 +++ app/g2/keyboard/bindings.ts | 266 +++++++++++++++++++ app/g2/keyboard/key-router.ts | 144 ++++++++++ app/g2/keyboard/keys.ts | 30 --- app/g2/keyboard/service.ts | 97 ++++++- app/ui/dashboard/settings-menus.ts | 3 +- app/ui/keybinding-settings.ts | 147 ++++++++++ app/ui/shell/shell.ts | 7 +- app/ui/shell/worker-window.ts | 22 +- tests/assistant-settings-visibility.test.cjs | 3 +- tests/ios-glanceboard.test.cjs | 56 +++- tests/keybinding-settings.test.cjs | 160 +++++++++++ tests/keyboard-bindings.test.cjs | 245 +++++++++++++++++ tests/keyboard-keys.test.cjs | 15 +- tests/tsconfig.json | 2 + 17 files changed, 1203 insertions(+), 100 deletions(-) create mode 100644 app/g2/keyboard/bindings.ts create mode 100644 app/g2/keyboard/key-router.ts create mode 100644 app/ui/keybinding-settings.ts create mode 100644 tests/keybinding-settings.test.cjs create mode 100644 tests/keyboard-bindings.test.cjs diff --git a/app/apps/terminal/terminal-app.worker.ts b/app/apps/terminal/terminal-app.worker.ts index 99917edd..2445895f 100644 --- a/app/apps/terminal/terminal-app.worker.ts +++ b/app/apps/terminal/terminal-app.worker.ts @@ -55,7 +55,7 @@ import { import * as frameTimings from "../../native/frame-timings"; import { getActiveDisplay } from "../../native/active-display"; import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures"; -import { navigationGestureForKey, type KeyEvent } from "../../g2/keyboard/keys"; +import type { KeyEvent } from "../../g2/keyboard/keys"; import { encodeTerminalKey, terminalScrollbackKey } from "./terminal-keys"; import { G2MirrorClient, type G2MirrorClientOptions, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client"; import { onSettingsStoreChanged } from "../../native/settings-store"; @@ -1270,11 +1270,12 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number) } /** - * A key from a hardware keyboard (the glasses' keyboard bridge). A session - * with nothing over it gets the key as its terminal would, every key - * included: Escape and the arrows belong to the program, so leaving is still - * the ring's double tap. Anywhere else (the hub's lists, an open menu) the - * navigation keys act as the ring gestures they stand for. + * A key from a hardware keyboard (the glasses' keyboard bridge) that no + * keybinding claimed. A session with nothing over it gets the key as its + * terminal would: Escape and the arrows belong to the program, and the way + * out is Escape with Shift, Ctrl or Alt, which never arrives here (it is a + * built-in keybinding for "back"). Anywhere else (the hub's lists, an open + * menu) the key is handed back, to act as navigation. */ function handleKey(window: TerminalWindow, event: KeyEvent): void { if (window.kind === "view" && !window.menu?.isOpen()) { @@ -1294,8 +1295,7 @@ function handleKey(window: TerminalWindow, event: KeyEvent): void { window.client.sendInput(bytes); return; } - const gesture = navigationGestureForKey(event); - if (gesture) handleInput(window, { type: gesture, source: "ring", timestampMs: event.timestampMs }, 0); + post({ type: "key-unhandled", windowId: window.windowId, event }); } /** Top visible absolute line index when following the live bottom. */ diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 7988161f..d98ae1c5 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -23,7 +23,7 @@ import { getBooleanSetting, setBooleanSetting } from "../native/settings-store"; import { grayImageToPreviewSource } from "../native/gray-image-preview"; import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from "./firmware-compat"; import { keyboardService } from "./keyboard/service"; -import { navigationGestureForKey, type KeyEvent } from "./keyboard/keys"; +import type { KeyEvent } from "./keyboard/keys"; import { hasExtractedEvenHubFonts } from "./firmware-builder"; import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy"; import { WearRemote, type WearRemoteInputKind } from "./wear-remote"; @@ -515,10 +515,23 @@ class DashboardController { assistantAvailable: () => shell.isAssistantAvailable(), assistant: text => shell.sendToAssistant(text), }); - // A keyboard paired through the glasses (CFW keyboard bridge). Its keys - // go to the focused window when that takes keys, else act as navigation. + // A keyboard paired through the glasses (CFW keyboard bridge). The + // service decides what each key means (app/g2/keyboard/key-router.ts); + // this is how it reaches the display and the input pipeline. keyboardService.setHost({ - onKey: (event) => this.handleKeyboardKey(event), + ready: () => this.phase === "connected", + locked: () => this.glassesLocked, + wake: () => { + void this.handleInputEvent({ + kind: "display-wake", + containerName: "", + eventType: OsEventTypeList.DOUBLE_CLICK_EVENT, + eventSource: EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL, + systemExitReasonCode: 0, + frameId: 0, + }).catch((error) => this.appendLog(`keyboard wake failed: ${this.formatError(error)}`)); + }, + gesture: (gesture, origin) => this.injectSyntheticRingInput(gesture, origin), log: (line) => this.appendLog(line), // A keyboard that is switched off would otherwise keep the glasses' // radio looking for it around the clock. One still connected when the @@ -2656,35 +2669,6 @@ class DashboardController { else keyboardService.linkLost(this.phase === "charging" || this.phase === "disconnecting"); } - /** - * A key from the keyboard paired through the glasses. A window that takes - * keys (a terminal session) gets it as a key; everywhere else the few - * navigation keys act as the ring gestures they stand for. A dark display - * takes no typing: the first press only wakes it, as the ring's double tap - * would, and a locked one stays locked. - */ - private handleKeyboardKey(event: KeyEvent): void { - if (this.phase !== "connected") return; - if (this.glassesLocked || !shell.isScreenOn()) { - if (event.phase !== "down" || event.repeat || shell.isScreenOn()) return; - void this.handleInputEvent({ - kind: "display-wake", - containerName: "", - eventType: OsEventTypeList.DOUBLE_CLICK_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL, - systemExitReasonCode: 0, - frameId: 0, - }).catch((error) => this.appendLog(`keyboard wake failed: ${this.formatError(error)}`)); - return; - } - if (shell.receiveKeyEvent(event)) return; - const gesture = navigationGestureForKey(event); - if (!gesture) return; - void this.injectSyntheticRingInput(gesture).catch((error) => - this.appendLog(`keyboard navigation failed: ${this.formatError(error)}`), - ); - } - /** * The Glanceboard event a sleep-time input means, or null when the shell * should see the input as usual. A display-wake carries the head-tilt @@ -2792,6 +2776,7 @@ class DashboardController { } }, endTextSettingEdit: () => this.endTextSettingEdit(), + unhandledKey: (event) => keyboardService.handleUnhandledKey(event as KeyEvent), }); this.appHosts.set(appId, host); return host; diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 2e605374..e661c8ef 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -29,6 +29,8 @@ import { OsEventTypeList } from './events' import { loadDeviceAddresses } from './device-addresses' import { updateGlassesPresence } from './glasses-presence' import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat' +import { keyboardService } from './keyboard/service' +import type { KeyEvent } from './keyboard/keys' import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy' import { WearRemote, type WearRemoteInputKind } from './wear-remote' import { deviceAddressError } from './ios-peripheral-identity' @@ -232,6 +234,7 @@ export class IosPreviewController { if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error)) this.requestShellRender() this.wearRemote?.schedulePublish() + if (on) keyboardService.retryNow() }, }) const launcher = createLauncherWindow({ @@ -270,6 +273,28 @@ export class IosPreviewController { assistantAvailable: () => shell.isAssistantAvailable(), assistant: text => shell.sendToAssistant(text), }) + // A keyboard paired through the glasses (the CFW keyboard bridge). The service decides what + // each key means (app/g2/keyboard/key-router.ts); its gestures share the controller queue + // with physical input, like the watch's. + const queueKeyboardInput = (run: () => Promise): Promise => { + const task = this.inputQueue.then(() => { this.onActivity(); return run() }) + this.inputQueue = task.catch(error => this.fail(error)) + return task + } + keyboardService.setHost({ + ready: () => this.state.phase === 'connected', + locked: () => this.glassesLocked, + wake: () => { void queueKeyboardInput(() => this.receiveInput(makeInputEvent({ type: 'display-wake' }))).catch(() => {}) }, + gesture: (gesture, origin) => queueKeyboardInput(async () => { + await this.receiveInput(makeInputEvent({ type: gesture, source: origin } as InputEventPayload)) + // A discrete hold includes its release. + if (gesture === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: origin })) + }), + log: line => this.logBluetooth(line), + // Only while the display is on: a keyboard that is switched off would otherwise keep the + // glasses' radio looking for it around the clock. + shouldReconnect: () => this.state.phase === 'connected' && shell.isScreenOn(), + }) // The header's EvenHub phone-UI button hides while that UI is showing. onPhoneShownChanged(() => this.syncEvenHubPhoneUi()) registerSystemTools() @@ -809,6 +834,7 @@ export class IosPreviewController { }, setSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible), removeSurface: id => this.removeWindowSurface(id), + unhandledKey: event => keyboardService.handleUnhandledKey(event as KeyEvent), submitPixels: (id, pixels, width, height, draws) => { void this.submitWorkerPixels(id, pixels, width, height, draws).catch(error => this.fail(error)) }, @@ -830,6 +856,18 @@ export class IosPreviewController { this.state = { ...this.state, phase, status } this.logBluetooth(status) this.emitState() + this.syncKeyboardLink() + } + /** + * Tell the keyboard bridge whether the session can host a keyboard: a G2 on firmware with + * the bridge, connected. On a disconnect the user asked for, the link is still up when the + * phase turns, so the firmware is told to let the keyboard go first. + */ + private syncKeyboardLink(): void { + const up = !!this.communicator && this.state.phase === 'connected' && this.firmwareCompatible + && glassesTypeSetting.get() === 'g2' + if (up) keyboardService.linkReady() + else keyboardService.linkLost(this.state.phase === 'disconnecting') } /** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */ private syncPresence(): void { @@ -949,6 +987,7 @@ export class IosPreviewController { const firmwareWarning = firmwareIncompatibilityMessage(info) ?? '' if (firmwareWarning && firmwareWarning !== this.state.firmwareWarning) this.logBluetooth(`Firmware compatibility warning: ${firmwareWarning}`) this.firmwareCompatible = !firmwareWarning && hasCompatibleFirmware(info) + this.syncKeyboardLink() this.state = { ...this.state, leftVersion: info.leftVersion || this.state.leftVersion, rightVersion: info.rightVersion || this.state.rightVersion, capabilities: info.extension || this.state.capabilities, firmwareWarning } diff --git a/app/g2/keyboard/bindings.ts b/app/g2/keyboard/bindings.ts new file mode 100644 index 00000000..fc70a4cf --- /dev/null +++ b/app/g2/keyboard/bindings.ts @@ -0,0 +1,266 @@ +/** + * What keys from a hardware keyboard do, apart from typing: the built-in + * defaults, the user's custom keybindings, and how the two combine. + * + * A key becomes one of the gestures the ring and the watch already have, so + * nothing downstream knows about keyboards. Three layers, first match wins: + * + * 1. Custom keybindings (Settings > Keyboard > Keybindings). These apply + * everywhere, including where the focused window takes keys for itself + * (a terminal session), since that is the point of binding a remote's + * few buttons. + * 2. Built-in bindings that also apply there: Escape with Shift, Ctrl or + * Alt goes back, which is the way out of a terminal session. + * 3. Built-in navigation, only for keys no window took: the arrows are the + * watch's four swipes, Enter selects, Escape goes back. + * + * No platform imports: this runs under node in the tests. + */ +import type { KeyEvent } from "./keys"; + +/** A gesture a key can stand for, named as the input pipeline names it. */ +export type KeyGesture = + | "swipe-up" + | "swipe-down" + | "swipe-left" + | "swipe-right" + | "scroll-up" + | "scroll-down" + | "click" + | "double-click" + | "long-press" + | "short-then-long-press"; + +/** + * What a custom keybinding does. "none" turns off the key's built-in + * navigation meaning; it does not stop the key being typed into a window + * that takes keys. + */ +export type KeyAction = KeyGesture | "none"; + +/** The actions, in the order the picker lists them. */ +export const KEY_ACTIONS: readonly { action: KeyAction; label: string }[] = [ + { action: "click", label: "Select" }, + { action: "double-click", label: "Back" }, + { action: "swipe-up", label: "Swipe up" }, + { action: "swipe-down", label: "Swipe down" }, + { action: "swipe-left", label: "Swipe left" }, + { action: "swipe-right", label: "Swipe right" }, + { action: "scroll-up", label: "Scroll up" }, + { action: "scroll-down", label: "Scroll down" }, + { action: "long-press", label: "System menu" }, + { action: "short-then-long-press", label: "App menu" }, + { action: "none", label: "Nothing" }, +]; + +export function keyActionLabel(action: KeyAction): string { + return KEY_ACTIONS.find((entry) => entry.action === action)?.label ?? action; +} + +function isKeyAction(value: unknown): value is KeyAction { + return KEY_ACTIONS.some((entry) => entry.action === value); +} + +/** A physical key with the modifiers held: what a keybinding is keyed on. */ +export type KeyChord = { code: string; shift: boolean; ctrl: boolean; alt: boolean; meta: boolean }; +export type KeyBinding = { chord: KeyChord; action: KeyAction }; + +const MODIFIER_CODES = new Set([ + "ControlLeft", "ControlRight", "ShiftLeft", "ShiftRight", "AltLeft", "AltRight", "MetaLeft", "MetaRight", +]); + +/** Shift, Ctrl, Alt and Meta themselves: they qualify a chord rather than being one. */ +export function isModifierCode(code: string): boolean { + return MODIFIER_CODES.has(code); +} + +export function chordOf(event: KeyEvent): KeyChord { + const { shift, ctrl, alt, meta } = event.modifiers; + return { code: event.code, shift, ctrl, alt, meta }; +} + +/** Canonical text of a chord ("Ctrl+Shift+Escape"): the stored form, and the lookup key. */ +export function chordId(chord: KeyChord): string { + return [chord.ctrl && "Ctrl", chord.alt && "Alt", chord.shift && "Shift", chord.meta && "Meta", chord.code] + .filter(Boolean).join("+"); +} + +/** The inverse of chordId, or null for anything it could not have produced. */ +export function parseChordId(text: string): KeyChord | null { + const parts = text.split("+"); + const code = parts.pop() ?? ""; + if (!/^[A-Za-z][A-Za-z0-9]{0,39}$/.test(code) || isModifierCode(code)) return null; + const chord: KeyChord = { code, shift: false, ctrl: false, alt: false, meta: false }; + for (const part of parts) { + if (part === "Ctrl" && !chord.ctrl) chord.ctrl = true; + else if (part === "Alt" && !chord.alt) chord.alt = true; + else if (part === "Shift" && !chord.shift) chord.shift = true; + else if (part === "Meta" && !chord.meta) chord.meta = true; + else return null; + } + return chord; +} + +const CODE_LABELS: Record = { + ArrowUp: "Up", ArrowDown: "Down", ArrowLeft: "Left", ArrowRight: "Right", + Escape: "Esc", PageUp: "Page Up", PageDown: "Page Down", CapsLock: "Caps Lock", + PrintScreen: "Print Screen", ScrollLock: "Scroll Lock", NumLock: "Num Lock", ContextMenu: "Menu", + Minus: "-", Equal: "=", BracketLeft: "[", BracketRight: "]", Backslash: "\\", IntlBackslash: "\\", + IntlHash: "#", Semicolon: ";", Quote: "'", Backquote: "`", Comma: ",", Period: ".", Slash: "/", + AudioVolumeUp: "Volume Up", AudioVolumeDown: "Volume Down", AudioVolumeMute: "Mute", + MediaPlayPause: "Play/Pause", MediaPlay: "Play", MediaPause: "Pause", MediaStop: "Stop", + MediaTrackNext: "Next Track", MediaTrackPrevious: "Previous Track", + BrightnessUp: "Brightness Up", BrightnessDown: "Brightness Down", + BrowserSearch: "Search", BrowserHome: "Browser Home", BrowserBack: "Browser Back", BrowserForward: "Browser Forward", + LaunchMail: "Mail", LaunchApplication1: "App 1", LaunchApplication2: "App 2", +}; + +/** A key as a person would name it. Unknown keys keep their HID usage number, which is all there is. */ +function codeLabel(code: string): string { + const known = CODE_LABELS[code]; + if (known) return known; + let match = /^Key([A-Z])$/.exec(code); + if (match) return match[1]!; + match = /^Digit([0-9])$/.exec(code); + if (match) return match[1]!; + match = /^Numpad(.+)$/.exec(code); + if (match) return `Numpad ${match[1]}`; + match = /^Usage([0-9a-f]+)$/.exec(code); + if (match) return `Key 0x${match[1]}`; + match = /^Consumer([0-9a-f]+)$/.exec(code); + if (match) return `Media key 0x${match[1]}`; + return code; +} + +/** A chord for display: "Shift+Esc", "Left", "Volume Up". */ +export function describeChord(chord: KeyChord): string { + return [chord.ctrl && "Ctrl", chord.alt && "Alt", chord.shift && "Shift", chord.meta && "Meta", codeLabel(chord.code)] + .filter(Boolean).join("+"); +} + +/** The most custom keybindings kept; far more than a keyboard has uses for. */ +export const MAX_KEY_BINDINGS = 64; + +/** + * Custom keybindings from their stored JSON. Lenient: whatever doesn't make + * sense is dropped, and a later entry for a chord replaces an earlier one. + */ +export function parseKeyBindings(raw: string): KeyBinding[] { + let parsed: unknown; + try { + parsed = raw ? JSON.parse(raw) : []; + } catch { + return []; + } + if (!Array.isArray(parsed)) return []; + const bindings = new Map(); + for (const entry of parsed) { + if (!entry || typeof entry !== "object") continue; + const { key, action } = entry as { key?: unknown; action?: unknown }; + const chord = typeof key === "string" ? parseChordId(key) : null; + if (!chord || !isKeyAction(action)) continue; + bindings.set(chordId(chord), { chord, action }); + if (bindings.size >= MAX_KEY_BINDINGS) break; + } + return [...bindings.values()]; +} + +export function serializeKeyBindings(bindings: readonly KeyBinding[]): string { + return JSON.stringify(bindings.map((binding) => ({ key: chordId(binding.chord), action: binding.action }))); +} + +/** Custom keybindings as the lookup resolveKey takes. */ +export function keyBindingMap(bindings: readonly KeyBinding[]): Map { + return new Map(bindings.map((binding) => [chordId(binding.chord), binding.action])); +} + +/** + * The built-in meaning of a key press, or null when it has none. + * `overridesTyping`: it holds even where the focused window takes keys. + * Built-ins go by the logical key, so both Enter keys select. + */ +export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overridesTyping: boolean } | null { + const { shift, ctrl, alt, meta } = event.modifiers; + if (meta) return null; + if (event.key === "Escape") { + // Plain Escape belongs to whatever takes keys (it is a keystroke in a + // terminal); with a modifier it always means "back out of here". + return { action: "double-click", overridesTyping: shift || ctrl || alt }; + } + if (ctrl || alt) return null; + switch (event.key) { + case "ArrowUp": + return { action: "swipe-up", overridesTyping: false }; + case "ArrowDown": + return { action: "swipe-down", overridesTyping: false }; + case "ArrowLeft": + return { action: "swipe-left", overridesTyping: false }; + case "ArrowRight": + return { action: "swipe-right", overridesTyping: false }; + case "Enter": + return { action: "click", overridesTyping: false }; + default: + return null; + } +} + +/** The built-ins, for showing to the user. Keep in step with defaultKeyAction. */ +export const DEFAULT_KEYS_HELP = + "Arrow keys: swipe up, down, left and right, as on the watch. Enter: select. Esc: back.\n\n" + + "In a terminal session those keys are typed instead, and Shift+Esc (or Ctrl+Esc, Alt+Esc) goes back.\n\n" + + "A custom keybinding replaces these for its key and works everywhere, terminals included. " + + "\"Nothing\" turns a key's meaning here off without stopping it being typed."; + +/** + * Whether holding the key repeats the gesture. Moving up and down a list + * does. Sideways swipes don't: depending on the layout they enter and leave + * things, and a held key would flap between the two. Selecting, backing out + * and menus act once per press. + */ +export function gestureRepeats(gesture: KeyGesture): boolean { + return gesture === "scroll-up" || gesture === "scroll-down" || gesture === "swipe-up" || gesture === "swipe-down"; +} + +export type KeyResolution = { + /** The key is spoken for: don't offer it to the focused window. */ + consume: boolean; + /** + * The gesture to perform: at once when `consume`, otherwise only if no + * window takes the key. Null for none (including a held key's repeats of + * a gesture that doesn't repeat). + */ + gesture: KeyGesture | null; +}; + +/** What one key press (or auto-repeat) means, given the custom keybindings. */ +export function resolveKey(event: KeyEvent, custom: ReadonlyMap): KeyResolution { + const repeatable = (gesture: KeyGesture): KeyGesture | null => + event.repeat && !gestureRepeats(gesture) ? null : gesture; + const bound = custom.get(chordId(chordOf(event))); + if (bound !== undefined && bound !== "none") return { consume: true, gesture: repeatable(bound) }; + // "none" takes the built-in meaning away and leaves the key to be typed. + if (bound === "none") return { consume: false, gesture: null }; + const builtIn = defaultKeyAction(event); + if (!builtIn) return { consume: false, gesture: null }; + return { consume: builtIn.overridesTyping, gesture: repeatable(builtIn.action) }; +} + +export type GestureOrigin = "ring" | "watch"; + +/** + * Which device a keyboard's gesture is presented as. Swipes are the watch's + * and scrolls the ring's; the two schemes differ in places (an icon grid + * moves by cell for the watch, by row and then cell for the ring). A tap, a + * double tap or a hold exists in both, so it goes with whichever the + * keyboard last moved with: arrow keys make the keyboard a watch, a remote + * bound to scroll makes it a ring. Before any movement, the watch. + */ +export class GestureScheme { + private scheme: GestureOrigin = "watch"; + + originFor(gesture: KeyGesture): GestureOrigin { + if (gesture.startsWith("swipe-")) this.scheme = "watch"; + else if (gesture.startsWith("scroll-")) this.scheme = "ring"; + return this.scheme; + } +} diff --git a/app/g2/keyboard/key-router.ts b/app/g2/keyboard/key-router.ts new file mode 100644 index 00000000..cbda2ac2 --- /dev/null +++ b/app/g2/keyboard/key-router.ts @@ -0,0 +1,144 @@ +/** + * Where a key from the hardware keyboard goes: to a keybinding being set up, + * to a gesture it is bound to, to the window with input focus, or to the + * built-in navigation. The meanings are in bindings.ts; this is the order of + * asking, and the bookkeeping around it. Shared by the Android and iOS + * controllers through KeyRouterHost. + * No platform imports: this runs under node in the tests. + */ +import { + GestureScheme, + chordOf, + isModifierCode, + resolveKey, + type GestureOrigin, + type KeyAction, + type KeyChord, + type KeyGesture, +} from "./bindings"; +import type { KeyEvent } from "./keys"; + +export type KeyRouterHost = { + /** Connected to glasses that are in use. */ + ready(): boolean; + locked(): boolean; + screenOn(): boolean; + /** Turn a dark display on, and nothing else. */ + wake(): void; + /** Offer a key to the window with input focus; true when that window takes keys. */ + offerToWindow(event: KeyEvent): boolean; + /** Perform a gesture as if it came from the ring or the watch. */ + gesture(gesture: KeyGesture, origin: GestureOrigin): Promise; + /** The custom keybindings, by chord id. */ + bindings(): ReadonlyMap; + /** A key was used for something other than a gesture or typing (count it against the screen timeout). */ + noteActivity(): void; + log(line: string): void; + now?(): number; +}; + +/** + * A held key repeats twenty times a second, faster than a list can repaint + * over the glasses' link. Repeats are dropped while a gesture is still being + * handled, and closer together than this. + */ +const REPEAT_MIN_INTERVAL_MS = 100; + +export class KeyRouter { + private capture: ((chord: KeyChord) => void) | null = null; + /** Keys whose press was spoken for, so their release is too. */ + private readonly consumed = new Set(); + private readonly scheme = new GestureScheme(); + private gesturesInFlight = 0; + private lastGestureAtMs = 0; + + constructor(private readonly host: KeyRouterHost) {} + + /** One key transition from the keyboard. */ + handleKey(event: KeyEvent): void { + const host = this.host; + if (!host.ready()) return; + if (host.locked() || !host.screenOn()) { + // A dark display takes no typing: the first press only wakes it, as + // the ring's double tap would, and a locked one stays locked. + if (event.phase === "down" && !event.repeat && !host.screenOn()) host.wake(); + return; + } + if (this.capture) { + this.captureKey(event); + return; + } + if (event.phase === "up") { + if (!this.consumed.delete(event.code)) host.offerToWindow(event); + return; + } + const resolution = resolveKey(event, host.bindings()); + if (resolution.consume) { + this.consumed.add(event.code); + if (resolution.gesture) this.perform(resolution.gesture, event.repeat); + return; + } + if (host.offerToWindow(event)) return; + if (resolution.gesture) this.perform(resolution.gesture, event.repeat); + } + + /** + * A key the focused window was offered and had no use for (a terminal + * showing its session list rather than a session): it gets the built-in + * navigation meaning after all. + */ + handleUnhandledKey(event: KeyEvent): void { + const host = this.host; + if (event.phase !== "down" || this.capture || !host.ready() || host.locked() || !host.screenOn()) return; + const resolution = resolveKey(event, host.bindings()); + if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat); + } + + /** + * Take the next key press instead of acting on it (setting up a + * keybinding). Modifiers held with it are part of the chord; pressed alone + * they are waited through. Returns a function that calls the capture off. + */ + captureNextKey(onChord: (chord: KeyChord) => void): () => void { + this.capture = onChord; + return () => { + if (this.capture === onChord) this.capture = null; + }; + } + + /** The keyboard went away: nothing is held any more. */ + reset(): void { + this.consumed.clear(); + } + + private captureKey(event: KeyEvent): void { + this.host.noteActivity(); + if (event.phase === "up") { + this.consumed.delete(event.code); + return; + } + if (event.repeat || isModifierCode(event.code)) return; + const capture = this.capture!; + this.capture = null; + this.consumed.add(event.code); + capture(chordOf(event)); + } + + private perform(gesture: KeyGesture, repeat: boolean): void { + const now = this.host.now ? this.host.now() : Date.now(); + if (repeat && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return; + this.lastGestureAtMs = now; + void this.run(gesture, this.scheme.originFor(gesture)); + } + + private async run(gesture: KeyGesture, origin: GestureOrigin): Promise { + this.gesturesInFlight++; + try { + await this.host.gesture(gesture, origin); + } catch (error) { + this.host.log(`keyboard gesture ${gesture} failed: ${error instanceof Error ? error.message : String(error)}`); + } finally { + this.gesturesInFlight--; + } + } +} diff --git a/app/g2/keyboard/keys.ts b/app/g2/keyboard/keys.ts index 0fd1d314..1a4c35e8 100644 --- a/app/g2/keyboard/keys.ts +++ b/app/g2/keyboard/keys.ts @@ -283,33 +283,3 @@ export class KeyTranslator { this.repeating = null; } } - -/** The ring gestures a keyboard can stand in for. */ -export type KeyNavigationGesture = "scroll-up" | "scroll-down" | "click" | "double-click"; - -/** - * What a key does where nothing takes keys as keys: the ring's vocabulary, - * with the same reading of left and right as a watch swipe (right selects, - * left backs out). Held arrows keep scrolling; Enter and Escape act once per - * press. Chords and every other key do nothing. This is a deliberately small - * default, not a button-mapping scheme. - */ -export function navigationGestureForKey(event: KeyEvent): KeyNavigationGesture | null { - if (event.phase !== "down") return null; - const { ctrl, alt, meta } = event.modifiers; - if (ctrl || alt || meta) return null; - switch (event.key) { - case "ArrowUp": - return "scroll-up"; - case "ArrowDown": - return "scroll-down"; - case "Enter": - case "ArrowRight": - return event.repeat ? null : "click"; - case "Escape": - case "ArrowLeft": - return event.repeat ? null : "double-click"; - default: - return null; - } -} diff --git a/app/g2/keyboard/service.ts b/app/g2/keyboard/service.ts index 3241a3c6..333704ff 100644 --- a/app/g2/keyboard/service.ts +++ b/app/g2/keyboard/service.ts @@ -1,13 +1,29 @@ /** * The keyboard bridge as the app uses it: one KeyboardBridge for the life of * the app, fed by whichever glasses session is live, its remembered keyboard - * kept in the settings store, and its key transitions turned into KeyEvents. - * The dashboard controller reports the link coming and going and takes the - * keys; the Settings app drives pairing through `bridge`. + * and the custom keybindings kept in the settings store, and its key + * transitions turned into KeyEvents and routed. The platform's controller + * reports the link coming and going and performs the gestures keys stand + * for; the Settings app drives pairing through `bridge` and edits the + * keybindings. */ import { addKeyboardEventListener, keyboardLinkSession, sendKeyboardControlRecord } from "../../native/keyboard-link"; import { getStringSetting, setStringSetting } from "../../native/settings-store"; +import { shell } from "../../ui/shell/shell"; +import { + MAX_KEY_BINDINGS, + chordId, + keyBindingMap, + parseKeyBindings, + serializeKeyBindings, + type GestureOrigin, + type KeyAction, + type KeyBinding, + type KeyChord, + type KeyGesture, +} from "./bindings"; import { KeyboardBridge, type KeyboardBridgeSnapshot, type SavedKeyboard } from "./bridge"; +import { KeyRouter } from "./key-router"; import { KeyTranslator, type KeyEvent } from "./keys"; declare const java: any; @@ -15,9 +31,18 @@ declare const arc4random_uniform: ((bound: number) => number) | undefined; /** JSON of the remembered keyboard ({ address, addressType, name }), or "" for none. */ const SAVED_KEYBOARD_KEY = "keyboard.saved"; +/** JSON list of the custom keybindings ({ key, action }); see bindings.ts. */ +const KEY_BINDINGS_KEY = "keyboard.bindings"; +/** What the platform's controller does for the keyboard. */ export type KeyboardServiceHost = { - onKey(event: KeyEvent): void; + /** Connected to glasses that are in use (not charging in their case). */ + ready(): boolean; + locked(): boolean; + /** Turn a dark display on, and nothing else (a locked display stays locked). */ + wake(): void; + /** Perform a gesture as if it came from the ring or the watch. */ + gesture(gesture: KeyGesture, origin: GestureOrigin): Promise; /** Routine lines only (failed background reconnects); the rest go to the device log directly. */ log(line: string): void; /** Whether an absent keyboard is worth looking for now (see KeyboardBridgeHost). */ @@ -52,7 +77,21 @@ class KeyboardService { private session: unknown = null; private offEvents: (() => void) | null = null; private readonly listeners = new Set<() => void>(); - private readonly translator = new KeyTranslator({ emit: (event) => this.host?.onKey(event) }); + private bindings: KeyBinding[] = parseKeyBindings(getStringSetting(KEY_BINDINGS_KEY, "")); + private bindingMap = keyBindingMap(this.bindings); + + private readonly router = new KeyRouter({ + ready: () => this.host?.ready() ?? false, + locked: () => this.host?.locked() ?? false, + screenOn: () => shell.isScreenOn(), + wake: () => this.host?.wake(), + offerToWindow: (event) => shell.receiveKeyEvent(event), + gesture: (gesture, origin) => this.host?.gesture(gesture, origin) ?? Promise.resolve(), + bindings: () => this.bindingMap, + noteActivity: () => shell.noteUserActivity(), + log: (line) => console.log(`[keyboard] ${line}`), + }); + private readonly translator = new KeyTranslator({ emit: (event) => this.router.handleKey(event) }); readonly bridge = new KeyboardBridge({ send: (record) => { @@ -67,7 +106,10 @@ class KeyboardService { loadSaved: loadSavedKeyboard, storeSaved: (saved) => setStringSetting(SAVED_KEYBOARD_KEY, saved ? JSON.stringify(saved) : ""), onKeys: (events) => this.translator.handle(events), - onKeyboardGone: () => this.translator.reset(), + onKeyboardGone: () => { + this.translator.reset(); + this.router.reset(); + }, onChanged: () => { for (const listener of [...this.listeners]) listener(); }, @@ -126,6 +168,49 @@ class KeyboardService { this.offEvents = null; this.session = null; } + + /** A key a worker's window was sent and had no use for (the "key-unhandled" reply). */ + handleUnhandledKey(event: KeyEvent): void { + this.router.handleUnhandledKey(event); + } + + // ---------- Custom keybindings ---------- + + /** The custom keybindings, in the order they were added. */ + keyBindings(): readonly KeyBinding[] { + return this.bindings; + } + + /** Bind a chord, replacing what it was bound to. False when the list is full. */ + setKeyBinding(chord: KeyChord, action: KeyAction): boolean { + const id = chordId(chord); + const index = this.bindings.findIndex((binding) => chordId(binding.chord) === id); + if (index < 0 && this.bindings.length >= MAX_KEY_BINDINGS) return false; + const next = [...this.bindings]; + if (index < 0) next.push({ chord, action }); + else next[index] = { chord, action }; + this.storeBindings(next); + return true; + } + + removeKeyBinding(chord: KeyChord): void { + const id = chordId(chord); + this.storeBindings(this.bindings.filter((binding) => chordId(binding.chord) !== id)); + } + + /** + * Take the next key pressed on the keyboard instead of acting on it. Returns + * a function that calls the capture off. + */ + captureNextKey(onChord: (chord: KeyChord) => void): () => void { + return this.router.captureNextKey(onChord); + } + + private storeBindings(bindings: KeyBinding[]): void { + this.bindings = bindings; + this.bindingMap = keyBindingMap(bindings); + setStringSetting(KEY_BINDINGS_KEY, serializeKeyBindings(bindings)); + } } export const keyboardService = new KeyboardService(); diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 6351f82a..567ef335 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -104,6 +104,7 @@ import { uiFontPickerMenuItem } from "../font-picker"; import { NotificationFilterLayer } from "../notification-filter"; import { compatibleModelDiscoveryMenuItem } from "../openai-compatible-model-picker"; import { keyboardSettingsMenuItem } from "../keyboard-settings"; +import { keybindingsMenuItem } from "../keybinding-settings"; import { isVeniceApiBaseUrl } from "../../native/openai-chat"; /** The Settings section the Notifications app's "Settings" menu item opens. */ @@ -248,7 +249,7 @@ function settingsSections(): SettingsSection[] { // out: the Settings window can be open across a change of glasses. { label: "Keyboard", - items: [keyboardSettingsMenuItem()], + items: [keyboardSettingsMenuItem(), keybindingsMenuItem()], hidden: () => glassesTypeSetting.get() !== "g2", }, { diff --git a/app/ui/keybinding-settings.ts b/app/ui/keybinding-settings.ts new file mode 100644 index 00000000..8c813ed1 --- /dev/null +++ b/app/ui/keybinding-settings.ts @@ -0,0 +1,147 @@ +import { getDefaultSmallFont } from "../graphics/ui-fonts"; +import { + DEFAULT_KEYS_HELP, + KEY_ACTIONS, + chordId, + describeChord, + keyActionLabel, + type KeyChord, +} from "../g2/keyboard/bindings"; +import { keyboardService } from "../g2/keyboard/service"; +import { TextViewerLayer } from "../apps/files/text-viewer"; +import type { LayerContext } from "./layers"; +import { drawRightValueMenuItem, MenuLayer, openModalMenu, submenuItem, TextPageLayer, type MenuItem } from "./menu"; + +/** + * Settings > Keyboard > Keybindings: the custom keybindings (a key, with + * any modifiers held, to a gesture), which come before the built-in + * meanings in app/g2/keyboard/bindings.ts. Adding one asks for the key on + * the keyboard itself, so it works for a remote whose buttons send who + * knows what. + */ + +/** Waits for the key to bind. Double tap backs out, as from any page. */ +class KeyCaptureLayer extends TextPageLayer { + private cancel: (() => void) | null = null; + + constructor(private readonly onChord: (chord: KeyChord, ctx: LayerContext) => void) { + super( + "Add keybinding", + "Press the key to bind, on the keyboard. Hold Shift, Ctrl or Alt with it to bind the combination.", + ); + } + + open(ctx: LayerContext): void { + ctx.stack.push(this); + this.cancel = keyboardService.captureNextKey((chord) => { + this.cancel = null; + ctx.stack.popThrough(this); + this.onChord(chord, ctx); + ctx.actions.requestRender(); + }); + } + + onRemoved(): void { + this.cancel?.(); + this.cancel = null; + } +} + +class KeybindingsLayer extends MenuLayer { + constructor(viewport: { width: number; height: number }) { + super("Keybindings", [], { + x: "center", + y: 8, + width: Math.min(320, viewport.width - 40), + minHeight: Math.min(160, viewport.height - 40), + maxHeight: Math.min(240, viewport.height - 40), + depth: 4, + }); + } + + open(ctx: LayerContext): void { + this.updateRows(); + ctx.stack.push(this); + } + + /** Rebuild the rows, with the highlight on `selectId`'s keybinding when given. */ + private updateRows(selectId?: string): void { + const bindings = keyboardService.keyBindings(); + const rows: MenuItem[] = bindings.map((binding) => { + const label = describeChord(binding.chord); + return { + label, + onSelect: (ctx: LayerContext) => this.pickAction(ctx, binding.chord), + render: ({ image, x, y, width }) => { + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, keyActionLabel(binding.action)); + }, + }; + }); + rows.push( + { label: "Add keybinding", onSelect: (ctx) => this.add(ctx) }, + // Scrolls, where a page would cut the text off in a short window. + submenuItem("Default keys", (ctx) => ctx.stack.push(new TextViewerLayer(DEFAULT_KEYS_HELP, "Default keys"))), + ); + const selected = selectId === undefined ? -1 : bindings.findIndex((binding) => chordId(binding.chord) === selectId); + if (selected >= 0) this.setItems(rows, selected); + else this.setItems(rows); + } + + private add(ctx: LayerContext): void { + if (!keyboardService.bridge.isConnected()) { + ctx.stack.push(new TextPageLayer( + "Add keybinding", + "No keyboard is connected. Pair or reconnect one under Bluetooth keyboard, then come back: the key to bind is read from the keyboard.", + )); + return; + } + new KeyCaptureLayer((chord, captureCtx) => this.pickAction(captureCtx, chord)).open(ctx); + } + + /** Choose what `chord` does: for a new keybinding, or to change or remove one. */ + private pickAction(ctx: LayerContext, chord: KeyChord): void { + const id = chordId(chord); + const current = keyboardService.keyBindings().find((binding) => chordId(binding.chord) === id)?.action; + const done = (inner: LayerContext): void => { + inner.stack.pop(); + this.updateRows(id); + inner.actions.requestRender(); + }; + const rows: MenuItem[] = KEY_ACTIONS.map((entry) => ({ + label: entry.action === current ? `${entry.label} *` : entry.label, + onSelect: (inner: LayerContext) => { + if (!keyboardService.setKeyBinding(chord, entry.action)) { + inner.stack.pop(); + inner.stack.push(new TextPageLayer("Keybindings", "There is no room for another keybinding. Remove one first.")); + return; + } + done(inner); + }, + })); + if (current !== undefined) { + rows.push({ + label: "Remove keybinding", + onSelect: (inner) => { + keyboardService.removeKeyBinding(chord); + done(inner); + }, + }); + } + const selected = Math.max(0, KEY_ACTIONS.findIndex((entry) => entry.action === current)); + openModalMenu(ctx, describeChord(chord), rows, selected); + } +} + +/** The Settings row that opens the keybindings menu. */ +export function keybindingsMenuItem(): MenuItem { + return { + label: "Keybindings", + description: + "Choose what keys do: arrows, Enter and Esc navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.", + onSelect: (ctx) => new KeybindingsLayer(ctx.stack.getBaseSize()).open(ctx), + render: ({ image, x, y, width }) => { + const count = keyboardService.keyBindings().length; + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "Keybindings", count ? `${count} custom` : "Default"); + }, + }; +} diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 8989a943..e8792750 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -196,9 +196,10 @@ export type ShellWindow = { /** * Take a key from a hardware keyboard (app/g2/keyboard) as a key. Optional: * only windows that give keys a meaning of their own (the terminal) - * implement it, and they get every key while they have input focus, - * presses and releases alike. For the rest the controller turns the few - * navigation keys into ring gestures (navigationGestureForKey). + * implement it. They get the keys no keybinding claimed while they have + * input focus, presses and releases alike; the rest of the time, and for + * every other window, the navigation keys act as gestures (see + * app/g2/keyboard/bindings.ts). */ handleKeyEvent?: (event: KeyEvent) => void; /** Foreground state changed: this window's surface is (not) the visible one. */ diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index 403dc69f..b6b3e9ad 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -58,6 +58,16 @@ export type WorkerAppReply = type: "worker-ready"; } | { type: "yield-focus"; windowId: string } + | { + /** + * A "key" message the window had no use for as a key (a terminal + * showing its session list rather than a session). The host gives it + * the keyboard's navigation meaning instead. + */ + type: "key-unhandled"; + windowId: string; + event: unknown; + } | { /** Foreground and focus one of the app's existing windows. */ type: "focus-window"; @@ -202,9 +212,9 @@ export type WorkerWindowSpec = { acceptsDirectional?: boolean; /** * Deliver hardware-keyboard keys to the worker as "key" messages while the - * window has input focus, instead of the navigation-gesture fallback. The - * worker then owns what every key means in every window state, navigation - * included (see ShellWindow.handleKeyEvent). + * window has input focus (see ShellWindow.handleKeyEvent). The worker + * answers "key-unhandled" for a key it doesn't want as a key just then, + * which gets the keyboard's navigation meaning instead. */ acceptsKeys?: boolean; }; @@ -234,6 +244,8 @@ export type WorkerAppHostOptions = { startTextSettingEdit: (settingId: string) => void; /** Close the phone text editor opened by startTextSettingEdit. */ endTextSettingEdit: () => void; + /** A key a window was sent and handed back (the "key-unhandled" reply); a KeyEvent. */ + unhandledKey?: (event: unknown) => void; }; /** @@ -323,6 +335,10 @@ export class WorkerAppHost { shell.yieldFocusToSidebar(); } break; + case "key-unhandled": + // Stale once focus has moved on: the key was meant for that window. + if (shell.isWindowFocused(message.windowId)) this.options.unhandledKey?.(message.event); + break; case "focus-window": if (this.openWindows.has(message.windowId)) { shell.focusWindow(message.windowId); diff --git a/tests/assistant-settings-visibility.test.cjs b/tests/assistant-settings-visibility.test.cjs index 388a272e..1e8ed8e3 100644 --- a/tests/assistant-settings-visibility.test.cjs +++ b/tests/assistant-settings-visibility.test.cjs @@ -75,6 +75,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible', glasse '../font-picker': { uiFontPickerMenuItem: () => ({ label: 'Font' }) }, '../notification-filter': { NotificationFilterLayer: class {} }, '../keyboard-settings': { keyboardSettingsMenuItem: () => ({ label: 'Bluetooth keyboard' }) }, + '../keybinding-settings': { keybindingsMenuItem: () => ({ label: 'Keybindings' }) }, '../openai-compatible-model-picker': { compatibleModelDiscoveryMenuItem: () => ({ label: 'Discover models' }), }, @@ -160,7 +161,7 @@ test('the Keyboard section, with its pairing row, follows the glasses type while // options; whether it shows is asked again on every paint. assert.equal(labels.indexOf('Keyboard'), labels.indexOf('Watch') + 1); const keyboard = sections.find(section => section.label === 'Keyboard'); - assert.equal(keyboard.items.map(item => item.label).join(), 'Bluetooth keyboard'); + assert.equal(keyboard.items.map(item => item.label).join(), 'Bluetooth keyboard,Keybindings'); assert.equal(keyboard.hidden(), true); glassesType = 'rayneo-io'; assert.equal(keyboard.hidden(), true); diff --git a/tests/ios-glanceboard.test.cjs b/tests/ios-glanceboard.test.cjs index b9d55157..e4fa9d70 100644 --- a/tests/ios-glanceboard.test.cjs +++ b/tests/ios-glanceboard.test.cjs @@ -30,6 +30,9 @@ function fixture() { let now = 1000, nextTask = 0, screenOn = true, shellOptions; const battery = {}; const tasks = new Map(), sent = [], previews = [], received = [], errors = []; + // The keyboard bridge's service, as the controller sees it: what it was told, and the host it was given. + const keyboard = { host: null, calls: [], setHost(host) { this.host = host; }, linkReady() { this.calls.push('ready'); }, + linkLost(release) { this.calls.push(release ? 'lost+release' : 'lost'); }, retryNow() { this.calls.push('retry'); }, handleUnhandledKey() {} }; const boardStats = { starts: 0, stops: 0, paints: 0 }; const registered = [], inProcessApps = []; const settings = { lock: true, enabled: true, tap: true, hold: true, tilt: true, duration: 3000, depth: 0 }; @@ -165,6 +168,7 @@ function fixture() { './device-addresses': { loadDeviceAddresses: () => ({ right: 'AA', left: 'BB', ring: '' }) }, './ios-peripheral-identity': { deviceAddressError: () => null }, './wear-remote': { WearRemote: class { schedulePublish() {} } }, './firmware-compat': load('app/g2/firmware-compat.ts', {}), './reconnect-policy': load('app/g2/reconnect-policy.ts', {}), + './keyboard/service': { keyboardService: keyboard }, '../apps/launcher': { launcherEntries: () => [] }, '../apps/evenhub/installed-apps': {}, '../apps/evenhub/manager': { onPhoneShownChanged: () => () => {}, phoneUiButton: () => null }, '../apps/evenhub/updates': {}, '../apps/evenhub': {}, @@ -223,7 +227,7 @@ function fixture() { } async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); } async function connect() { await controller.connect(); await render(); } - return { relay, controller, shell, battery, remoteHost, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, + return { relay, controller, shell, battery, remoteHost, keyboard, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, get shellOptions() { return shellOptions; }, registered, inProcessApps, get bridge() { return bridge; }, connect, wear: wearing => bridge.emitWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render }; } @@ -348,3 +352,53 @@ test('the switcher can open the Notifications window in the background, once, wi assert.equal(f.shellOptions.openNotificationsWindow().opened, true); assert.equal(f.inProcessApps.length, 2); }); + +test('a paired keyboard: the bridge follows the G2 link, and its keys\' gestures go in with physical input', async () => { + const f = fixture(); + await f.connect(); + // Connected is not enough: the firmware has to be one with the bridge. + f.bridge.emitState('connected'); + assert.ok(!f.keyboard.calls.includes('ready')); + f.bridge.listeners.firmware({ leftVersion: '2.3.0.24', rightVersion: '2.3.0.24', extension: 'Faceclaw/38' }); + assert.ok(!f.keyboard.calls.includes('ready')); + f.bridge.listeners.firmware({ leftVersion: '2.3.0.24', rightVersion: '2.3.0.24', extension: 'Faceclaw/39' }); + assert.equal(f.keyboard.calls.at(-1), 'ready'); + assert.equal(f.keyboard.host.ready(), true); + // The link dropping on its own ends it without a word to the firmware; coming back restores it. + f.bridge.emitState('retrying', 'Reconnecting'); + assert.equal(f.keyboard.calls.at(-1), 'lost'); assert.equal(f.keyboard.host.ready(), false); + f.bridge.emitState('connected'); + assert.equal(f.keyboard.calls.at(-1), 'ready'); + + // A key's gesture is ordinary input from there on, tagged with the device it stands in for. + f.received.length = 0; + await f.keyboard.host.gesture('swipe-left', 'watch'); await f.keyboard.host.gesture('click', 'ring'); + await f.keyboard.host.gesture('long-press', 'watch'); + assert.deepEqual(f.received.map(input => `${input.type}/${input.source}`), + ['swipe-left/watch', 'click/ring', 'long-press/watch', 'long-press-release/watch']); + // A dark display: the bridge stops looking for an absent keyboard, a key wakes it, and it looks again. + f.shell.sleep(); + assert.equal(f.keyboard.host.shouldReconnect(), false); + f.keyboard.calls.length = 0; f.received.length = 0; + f.keyboard.host.wake(); await f.render(); + assert.deepEqual(f.received.map(input => input.type), ['display-wake']); + assert.equal(f.shell.isScreenOn(), true); assert.ok(f.keyboard.calls.includes('retry')); + assert.equal(f.keyboard.host.shouldReconnect(), true); + + // Disconnecting on purpose lets the keyboard go while the link is still up. + f.keyboard.calls.length = 0; + await f.controller.disconnect(); + assert.equal(f.keyboard.calls[0], 'lost+release'); + assert.equal(f.keyboard.host.ready(), false); +}); + +test('a keyboard needs a G2: other glasses never bring the bridge up', async () => { + glassesType = 'g1'; + try { + const f = fixture(); + await f.connect(); + f.bridge.emitState('connected'); + f.bridge.listeners.firmware({ leftVersion: 'g1v0 0.5.66', rightVersion: 'g1v0 0.5.66', extension: 'Faceclaw/39' }); + assert.ok(!f.keyboard.calls.includes('ready'), f.keyboard.calls.join()); + } finally { glassesType = 'g2'; } +}); diff --git a/tests/keybinding-settings.test.cjs b/tests/keybinding-settings.test.cjs new file mode 100644 index 00000000..ebd8a8cc --- /dev/null +++ b/tests/keybinding-settings.test.cjs @@ -0,0 +1,160 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, modules) { + const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); + const context = { exports: {}, console, require(name) { assert.ok(name in modules, name); return modules[name]; } }; + vm.runInNewContext(ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, context); + return context.exports; +} + +const graphics = require('../.test-build/app/graphics/image.js'); +const metrics = require('../.test-build/app/ui/metrics.js'); +const menuCore = require('../.test-build/app/ui/menu-core.js'); +const bindings = require('../.test-build/app/g2/keyboard/bindings.js'); +const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); +const font = BdfFont.parse(fs.readFileSync(path.join(__dirname, '../app/fonts/terminus/ter-u12n.bdf'), 'utf8')); +const textwrap = load('app/graphics/textwrap.ts', {}); +const fonts = { getDefaultSmallFont: () => font }; +const menu = load('app/ui/menu.ts', { + './menu-animation-pref': require('../.test-build/app/ui/menu-animation-pref.js'), './animation-speed': require('../.test-build/app/ui/animation-speed.js'), '../native/settings-store': { getStringSetting: (_key, fallback) => fallback }, '../graphics/screen-size': { screenWidth: () => 640, screenHeight: () => 480 }, + '../graphics/image': graphics, '../graphics/textwrap': textwrap, '../graphics/ui-fonts': fonts, + '../util/numeric-util': { clamp: (n, lo, hi) => Math.max(lo, Math.min(hi, n)) }, './gestures': {}, + './metrics': metrics, './menu-core': menuCore, +}); + +/** The settings menu over a fake keyboard service and layer stack. */ +function fixture({ connected = true, stored = [], room = true } = {}) { + const service = { + list: [...stored], capture: null, cancelled: 0, + bridge: { isConnected: () => connected }, + keyBindings() { return this.list; }, + setKeyBinding(chord, action) { + if (!room) return false; + const id = bindings.chordId(chord); + const index = this.list.findIndex(b => bindings.chordId(b.chord) === id); + const next = [...this.list]; + if (index < 0) next.push({ chord, action }); else next[index] = { chord, action }; + this.list = next; + return true; + }, + removeKeyBinding(chord) { this.list = this.list.filter(b => bindings.chordId(b.chord) !== bindings.chordId(chord)); }, + captureNextKey(onChord) { this.capture = onChord; return () => { if (this.capture === onChord) { this.capture = null; this.cancelled++; } }; }, + }; + class TextViewerLayer { constructor(text, title) { this.text = text; this.title = title; } handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); } } + const { keybindingsMenuItem } = load('app/ui/keybinding-settings.ts', { + '../graphics/ui-fonts': fonts, '../g2/keyboard/bindings': bindings, '../g2/keyboard/service': { keyboardService: service }, + '../apps/files/text-viewer': { TextViewerLayer }, './menu': menu, + }); + const layers = [{ base: true }]; + let renders = 0; + const removed = layer => layer.onRemoved?.(); + const ctx = { + actions: { requestRender: () => { renders++; } }, + stack: { + getBaseSize: () => ({ width: 576, height: 260, padding: { top: 0, right: 0, bottom: 0, left: 0 } }), isFocused: () => true, + push: layer => { layers.push(layer); }, + pop: () => { if (layers.length > 1) removed(layers.pop()); }, + popThrough: layer => { + const index = layers.indexOf(layer); + if (index <= 0) return false; + for (const gone of layers.splice(index).reverse()) removed(gone); + return true; + }, + }, + }; + const top = () => layers[layers.length - 1]; + const input = type => top().handleInput({ type, timestampMs: 0 }, ctx); + const labels = () => top().items.map(item => item.label); + /** Move the highlight of the menu on top to the row with this label, and select it. */ + async function choose(label) { + const layer = top(); + for (let i = 0; i <= layer.items.length && layer.selectedItem?.label !== label; i++) await input('scroll-down'); + assert.equal(layer.selectedItem?.label, label, `${label} in ${labels().join(' | ')}`); + await input('click'); + } + const item = keybindingsMenuItem(); + return { service, item, ctx, layers, top, input, labels, choose, renders: () => renders, + open: () => item.onSelect(ctx), chord: (code, mods = {}) => ({ code, shift: false, ctrl: false, alt: false, meta: false, ...mods }) }; +} + +test('adding a keybinding: the key is read from the keyboard, then an action is picked', async () => { + const f = fixture(); + f.open(); + assert.deepEqual(f.labels(), ['Add keybinding', 'Default keys']); + await f.choose('Add keybinding'); + assert.ok(f.service.capture, 'waiting for a key'); + assert.match(f.top().body ?? f.top().constructor.name, /Press the key|KeyCaptureLayer/); + f.service.capture(f.chord('AudioVolumeUp')); + // The waiting page is gone and the picker for that key is up, with every action in it. + assert.equal(f.layers.length, 3); assert.equal(f.top().title, 'Volume Up'); + assert.deepEqual(f.labels(), ['Select', 'Back', 'Swipe up', 'Swipe down', 'Swipe left', 'Swipe right', + 'Scroll up', 'Scroll down', 'System menu', 'App menu', 'Nothing']); + await f.choose('Scroll up'); + assert.deepEqual(f.service.list, [{ chord: f.chord('AudioVolumeUp'), action: 'scroll-up' }]); + // Back on the list, on the new row. + assert.equal(f.layers.length, 2); + assert.deepEqual(f.labels(), ['Volume Up', 'Add keybinding', 'Default keys']); + assert.equal(f.top().selectedItem.label, 'Volume Up'); + assert.ok(f.renders() > 0); +}); + +test('a keybinding can be changed or removed, and a chord is one keybinding however often it is added', async () => { + const f = fixture(); + f.service.list = [{ chord: f.chord('Escape', { shift: true }), action: 'none' }, { chord: f.chord('KeyQ', { ctrl: true }), action: 'long-press' }]; + f.open(); + assert.deepEqual(f.labels(), ['Shift+Esc', 'Ctrl+Q', 'Add keybinding', 'Default keys']); + await f.choose('Ctrl+Q'); + // The picker opens on the current action, marks it, and offers removal. + assert.equal(f.top().selectedItem.label, 'System menu *'); assert.equal(f.labels().at(-1), 'Remove keybinding'); + await f.choose('App menu'); + assert.equal(f.service.list[1].action, 'short-then-long-press'); assert.equal(f.service.list.length, 2); + // Adding the same chord again edits it rather than making a second one. + await f.choose('Add keybinding'); + f.service.capture(f.chord('KeyQ', { ctrl: true })); + assert.equal(f.top().selectedItem.label, 'App menu *'); + await f.choose('Back'); + assert.deepEqual(f.service.list.map(b => b.action), ['none', 'double-click']); + await f.choose('Shift+Esc'); + await f.choose('Remove keybinding'); + assert.deepEqual(f.labels(), ['Ctrl+Q', 'Add keybinding', 'Default keys']); +}); + +test('backing out of the wait calls the capture off; with no keyboard there is nothing to wait for', async () => { + const f = fixture(); + f.open(); + await f.choose('Add keybinding'); + await f.input('double-click'); + assert.equal(f.service.capture, null); assert.equal(f.service.cancelled, 1); assert.equal(f.layers.length, 2); + // A key pressed after that is not taken for a keybinding. + assert.deepEqual(f.service.list, []); + + const away = fixture({ connected: false }); + away.open(); + await away.choose('Add keybinding'); + assert.equal(away.service.capture, null); + assert.match(away.top().body, /No keyboard is connected/); + await away.input('double-click'); + assert.equal(away.layers.length, 2); +}); + +test('the defaults are a page of their own, and a full list says so', async () => { + const f = fixture(); + f.open(); + await f.choose('Default keys'); + assert.equal(f.top().title, 'Default keys'); + for (const phrase of ['Arrow keys', 'Enter', 'Shift+Esc', 'terminal']) assert.ok(f.top().text.includes(phrase), phrase); + + const full = fixture({ room: false }); + full.open(); + await full.choose('Add keybinding'); + full.service.capture(full.chord('F5')); + await full.choose('Select'); + assert.match(full.top().body, /no room/); assert.deepEqual(full.service.list, []); +}); diff --git a/tests/keyboard-bindings.test.cjs b/tests/keyboard-bindings.test.cjs new file mode 100644 index 00000000..d95752df --- /dev/null +++ b/tests/keyboard-bindings.test.cjs @@ -0,0 +1,245 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const { + KEY_ACTIONS, MAX_KEY_BINDINGS, GestureScheme, chordId, chordOf, defaultKeyAction, describeChord, gestureRepeats, + isModifierCode, keyActionLabel, keyBindingMap, parseChordId, parseKeyBindings, resolveKey, serializeKeyBindings, +} = require('../.test-build/app/g2/keyboard/bindings.js') +const { KeyRouter } = require('../.test-build/app/g2/keyboard/key-router.js') + +const key = (k, overrides = {}) => ({ + phase: 'down', key: k, code: overrides.code ?? k, repeat: false, timestampMs: 0, + ...overrides, modifiers: { shift: false, ctrl: false, alt: false, meta: false, ...overrides.modifiers }, +}) +const chord = (code, mods = {}) => ({ code, shift: false, ctrl: false, alt: false, meta: false, ...mods }) +const none = new Map() + +test('built-in keys: arrows are the watch\'s swipes, Enter selects, Esc backs out', () => { + const action = (k, overrides) => defaultKeyAction(key(k, overrides)) + assert.deepEqual(action('ArrowLeft'), { action: 'swipe-left', overridesTyping: false }) + assert.deepEqual(action('ArrowRight'), { action: 'swipe-right', overridesTyping: false }) + assert.deepEqual(action('ArrowUp'), { action: 'swipe-up', overridesTyping: false }) + assert.deepEqual(action('ArrowDown'), { action: 'swipe-down', overridesTyping: false }) + assert.deepEqual(action('Enter'), { action: 'click', overridesTyping: false }) + assert.deepEqual(action('Enter', { code: 'NumpadEnter' }), { action: 'click', overridesTyping: false }) + assert.deepEqual(action('Escape'), { action: 'double-click', overridesTyping: false }) + // With a modifier, Escape means "back" even where keys are being typed. + for (const modifier of ['shift', 'ctrl', 'alt']) { + assert.deepEqual(action('Escape', { modifiers: { [modifier]: true } }), { action: 'double-click', overridesTyping: true }) + } + assert.equal(action('Escape', { modifiers: { meta: true } }), null) + assert.equal(action('ArrowLeft', { modifiers: { ctrl: true } }), null) + assert.equal(action('Enter', { modifiers: { alt: true } }), null) + for (const k of ['a', ' ', 'Backspace', 'Tab', 'F1', 'AudioVolumeUp']) assert.equal(action(k), null) +}) + +test('chords: canonical ids round-trip, and read naturally', () => { + const c = chord('Escape', { shift: true, ctrl: true }) + assert.equal(chordId(c), 'Ctrl+Shift+Escape'); assert.deepEqual(parseChordId('Ctrl+Shift+Escape'), c) + assert.deepEqual(parseChordId('ArrowLeft'), chord('ArrowLeft')) + assert.deepEqual(chordOf(key('A', { code: 'KeyA', modifiers: { shift: true } })), chord('KeyA', { shift: true })) + for (const bad of ['', 'Shift+', 'Shift+Shift+KeyA', 'Hyper+KeyA', 'ShiftLeft', 'Ctrl+ControlRight', 'Key A', '+'.repeat(3)]) { + assert.equal(parseChordId(bad), null, bad) + } + assert.equal(isModifierCode('AltRight'), true); assert.equal(isModifierCode('CapsLock'), false) + const said = (code, mods) => describeChord(chord(code, mods)) + assert.equal(said('Escape', { shift: true }), 'Shift+Esc'); assert.equal(said('ArrowLeft'), 'Left') + assert.equal(said('KeyA', { ctrl: true, alt: true }), 'Ctrl+Alt+A'); assert.equal(said('Digit7'), '7') + assert.equal(said('AudioVolumeUp'), 'Volume Up'); assert.equal(said('MediaPlayPause'), 'Play/Pause') + assert.equal(said('NumpadEnter'), 'Numpad Enter'); assert.equal(said('F5'), 'F5'); assert.equal(said('Slash'), '/') + // A remote's unnamed buttons keep the only name they have. + assert.equal(said('Consumer1cb'), 'Media key 0x1cb'); assert.equal(said('Usagea5'), 'Key 0xa5') +}) + +test('custom keybindings survive storage; junk in storage is dropped', () => { + const bindings = [ + { chord: chord('AudioVolumeUp'), action: 'scroll-up' }, + { chord: chord('KeyQ', { ctrl: true }), action: 'long-press' }, + { chord: chord('ArrowLeft'), action: 'none' }, + ] + assert.deepEqual(parseKeyBindings(serializeKeyBindings(bindings)), bindings) + assert.deepEqual([...keyBindingMap(bindings)], [['AudioVolumeUp', 'scroll-up'], ['Ctrl+KeyQ', 'long-press'], ['ArrowLeft', 'none']]) + assert.deepEqual(parseKeyBindings(''), []); assert.deepEqual(parseKeyBindings('{'), []); assert.deepEqual(parseKeyBindings('{"a":1}'), []) + assert.deepEqual(parseKeyBindings(JSON.stringify([ + null, 7, { key: 'KeyA' }, { key: 'KeyA', action: 'explode' }, { key: 'Shift+ShiftLeft', action: 'click' }, + { key: 'KeyB', action: 'click' }, { key: 'KeyB', action: 'double-click' }, + ])), [{ chord: chord('KeyB'), action: 'double-click' }]) + const many = Array.from({ length: 200 }, (_, i) => ({ key: `F${i + 1}`, action: 'click' })) + assert.equal(parseKeyBindings(JSON.stringify(many)).length, MAX_KEY_BINDINGS) + // Every action has a label for the picker, and the list has no duplicates. + assert.equal(new Set(KEY_ACTIONS.map(a => a.action)).size, KEY_ACTIONS.length) + assert.equal(keyActionLabel('double-click'), 'Back'); assert.equal(keyActionLabel('none'), 'Nothing') +}) + +test('resolution: custom first and everywhere, then the built-ins; "Nothing" only removes the built-in', () => { + const custom = keyBindingMap([ + { chord: chord('AudioVolumeUp'), action: 'scroll-up' }, + { chord: chord('ArrowLeft'), action: 'double-click' }, + { chord: chord('ArrowRight'), action: 'none' }, + { chord: chord('Escape', { shift: true }), action: 'none' }, + ]) + // Unbound keys: typed where something takes keys, otherwise the built-in. + assert.deepEqual(resolveKey(key('a', { code: 'KeyA' }), custom), { consume: false, gesture: null }) + assert.deepEqual(resolveKey(key('ArrowUp'), custom), { consume: false, gesture: 'swipe-up' }) + assert.deepEqual(resolveKey(key('Escape'), custom), { consume: false, gesture: 'double-click' }) + assert.deepEqual(resolveKey(key('Escape', { modifiers: { ctrl: true } }), custom), { consume: true, gesture: 'double-click' }) + // Custom: spoken for even in a terminal. + assert.deepEqual(resolveKey(key('AudioVolumeUp'), custom), { consume: true, gesture: 'scroll-up' }) + assert.deepEqual(resolveKey(key('ArrowLeft'), custom), { consume: true, gesture: 'double-click' }) + // The chord is exact: Shift+Left is not the Left keybinding. + assert.deepEqual(resolveKey(key('ArrowLeft', { modifiers: { shift: true } }), custom), { consume: false, gesture: 'swipe-left' }) + // "Nothing": no navigation, still typed; and it can take a built-in override away. + assert.deepEqual(resolveKey(key('ArrowRight'), custom), { consume: false, gesture: null }) + assert.deepEqual(resolveKey(key('Escape', { modifiers: { shift: true } }), custom), { consume: false, gesture: null }) + // Held keys: moving repeats, the rest acts once but stays spoken for. + assert.deepEqual(resolveKey(key('AudioVolumeUp', { repeat: true }), custom), { consume: true, gesture: 'scroll-up' }) + assert.deepEqual(resolveKey(key('ArrowLeft', { repeat: true }), custom), { consume: true, gesture: null }) + assert.deepEqual(resolveKey(key('ArrowDown', { repeat: true }), none), { consume: false, gesture: 'swipe-down' }) + assert.deepEqual(resolveKey(key('ArrowRight', { repeat: true }), none), { consume: false, gesture: null }) + assert.deepEqual(resolveKey(key('Enter', { repeat: true }), none), { consume: false, gesture: null }) + assert.deepEqual(['swipe-left', 'swipe-right', 'click', 'double-click', 'long-press'].filter(gestureRepeats), []) +}) + +test('taps go with whichever scheme the keyboard last moved by', () => { + const scheme = new GestureScheme() + assert.equal(scheme.originFor('click'), 'watch') + assert.equal(scheme.originFor('scroll-down'), 'ring'); assert.equal(scheme.originFor('click'), 'ring') + assert.equal(scheme.originFor('double-click'), 'ring'); assert.equal(scheme.originFor('long-press'), 'ring') + assert.equal(scheme.originFor('swipe-left'), 'watch'); assert.equal(scheme.originFor('click'), 'watch') + assert.equal(scheme.originFor('short-then-long-press'), 'watch') +}) + +function router(options = {}) { + const state = { + ready: true, locked: false, screenOn: true, windowTakesKeys: false, now: 1000, + bindings: keyBindingMap(options.bindings ?? []), gestures: [], offered: [], wakes: 0, activity: 0, pending: [], + hold: false, + } + const r = new KeyRouter({ + ready: () => state.ready, locked: () => state.locked, screenOn: () => state.screenOn, + wake: () => { state.wakes++ }, + offerToWindow: event => { state.offered.push(`${event.phase}:${event.key}`); return state.windowTakesKeys }, + gesture: (gesture, origin) => { + state.gestures.push(`${gesture}/${origin}`) + return state.hold ? new Promise(resolve => state.pending.push(resolve)) : Promise.resolve() + }, + bindings: () => state.bindings, noteActivity: () => { state.activity++ }, log: () => {}, now: () => state.now, + }) + return { r, state } +} +const up = (k, overrides) => key(k, { ...overrides, phase: 'up' }) +const flush = () => new Promise(resolve => setImmediate(resolve)) + +test('router: navigation where nothing takes keys, typing where something does, Shift+Esc out of either', () => { + const { r, state } = router() + r.handleKey(key('ArrowLeft')); r.handleKey(up('ArrowLeft')); r.handleKey(key('Enter')); r.handleKey(key('Escape')) + assert.deepEqual(state.gestures, ['swipe-left/watch', 'click/watch', 'double-click/watch']) + // A terminal session: the same keys are its own, Escape included. + state.windowTakesKeys = true; state.gestures.length = 0; state.offered.length = 0 + for (const k of ['ArrowLeft', 'Enter', 'Escape', 'a']) { r.handleKey(key(k)); r.handleKey(up(k)) } + assert.deepEqual(state.gestures, []) + assert.deepEqual(state.offered, ['down:ArrowLeft', 'up:ArrowLeft', 'down:Enter', 'up:Enter', 'down:Escape', 'up:Escape', 'down:a', 'up:a']) + // Shift+Esc (or Ctrl, or Alt) backs out, and neither its press nor its release is typed. + state.offered.length = 0 + for (const modifier of ['shift', 'ctrl', 'alt']) { + r.handleKey(key('Escape', { modifiers: { [modifier]: true } })); r.handleKey(up('Escape')) + } + assert.deepEqual(state.gestures, ['double-click/watch', 'double-click/watch', 'double-click/watch']) + assert.deepEqual(state.offered, []) +}) + +test('router: a key the window hands back gets its navigation meaning; one it kept does not', () => { + const { r, state } = router() + state.windowTakesKeys = true + r.handleKey(key('ArrowDown')) + assert.deepEqual(state.gestures, []) + // The terminal was showing its session list: the key comes back. + r.handleUnhandledKey(key('ArrowDown')) + assert.deepEqual(state.gestures, ['swipe-down/watch']) + r.handleUnhandledKey(key('a')); r.handleUnhandledKey(up('ArrowDown')) + assert.equal(state.gestures.length, 1) + // Stale by the time it comes back: the display went dark, or locked. + state.screenOn = false; r.handleUnhandledKey(key('ArrowDown')) + state.screenOn = true; state.locked = true; r.handleUnhandledKey(key('ArrowDown')) + assert.equal(state.gestures.length, 1); assert.equal(state.wakes, 0) +}) + +test('router: custom keybindings work in a terminal too, and "Nothing" leaves the key to be typed', () => { + const { r, state } = router({ bindings: [ + { chord: chord('AudioVolumeUp'), action: 'scroll-up' }, + { chord: chord('AudioVolumeDown'), action: 'scroll-down' }, + { chord: chord('MediaPlayPause'), action: 'click' }, + { chord: chord('ArrowLeft'), action: 'none' }, + ] }) + state.windowTakesKeys = true + r.handleKey(key('AudioVolumeDown')); r.handleKey(up('AudioVolumeDown')); r.handleKey(key('MediaPlayPause')); r.handleKey(up('MediaPlayPause')) + // A remote bound to scroll is a ring: its select is a ring's tap. + assert.deepEqual(state.gestures, ['scroll-down/ring', 'click/ring']); assert.deepEqual(state.offered, []) + r.handleKey(key('ArrowLeft')) + assert.deepEqual(state.offered, ['down:ArrowLeft']) + // And where nothing takes keys, "Nothing" is nothing. + state.windowTakesKeys = false; state.gestures.length = 0 + r.handleKey(key('ArrowLeft')); r.handleUnhandledKey(key('ArrowLeft')) + assert.deepEqual(state.gestures, []) +}) + +test('router: a dark display only wakes, a locked one stays locked, and nothing happens off the glasses', () => { + const { r, state } = router() + state.screenOn = false + r.handleKey(key('a')); r.handleKey(key('a', { repeat: true })); r.handleKey(up('a')) + assert.equal(state.wakes, 1); assert.deepEqual(state.gestures, []); assert.deepEqual(state.offered, []) + state.locked = true + r.handleKey(key('Enter')) + assert.equal(state.wakes, 2) + state.screenOn = true + r.handleKey(key('Enter')); r.handleKey(key('Escape', { modifiers: { shift: true } })) + assert.equal(state.wakes, 2); assert.deepEqual(state.gestures, []) + state.locked = false; state.ready = false + r.handleKey(key('Enter')) + assert.deepEqual(state.gestures, []) +}) + +test('router: a held key repeats movement at a rate the display can follow, and never queues behind itself', async () => { + const { r, state } = router() + r.handleKey(key('ArrowDown')) + for (let i = 1; i <= 20; i++) { await flush(); state.now = 1400 + i * 50; r.handleKey(key('ArrowDown', { repeat: true })) } + // Twenty repeats a second arrive; every other one is acted on. + assert.equal(state.gestures.length, 1 + 10) + // While a gesture is still being handled, repeats are dropped rather than queued. + state.gestures.length = 0; state.hold = true; state.now += 1000 + r.handleKey(key('ArrowUp')) + for (let i = 1; i <= 5; i++) { state.now += 200; r.handleKey(key('ArrowUp', { repeat: true })) } + assert.equal(state.gestures.length, 1) + state.pending.splice(0).forEach(resolve => resolve()); await flush() + state.now += 200; r.handleKey(key('ArrowUp', { repeat: true })) + assert.equal(state.gestures.length, 2) + // A fresh press is never dropped, even on the heels of another gesture. + r.handleKey(up('ArrowUp')); r.handleKey(key('Enter')) + assert.equal(state.gestures.at(-1), 'click/watch') +}) + +test('router: capturing a key for a new keybinding takes the chord and nothing else happens', () => { + const { r, state } = router() + const captured = [] + r.captureNextKey(c => captured.push(c)) + // Modifiers alone are waited through; the key pressed with them is the chord. + r.handleKey(key('Shift', { code: 'ShiftLeft', modifiers: { shift: true } })) + assert.deepEqual(captured, []) + r.handleKey(key('Escape', { modifiers: { shift: true } })) + assert.deepEqual(captured, [chord('Escape', { shift: true })]) + // Its release, and the modifier's, do nothing; the next key is ordinary again. + r.handleKey(up('Escape')); r.handleKey(up('Shift', { code: 'ShiftLeft' })) + assert.deepEqual(state.gestures, []); assert.ok(state.activity >= 2) + r.handleKey(key('Enter')) + assert.deepEqual(state.gestures, ['click/watch']) + // Called off: keys go back to meaning what they mean. + const cancel = r.captureNextKey(c => captured.push(c)) + cancel() + r.handleKey(key('ArrowUp')) + assert.equal(captured.length, 1); assert.equal(state.gestures.at(-1), 'swipe-up/watch') + // A second capture replaces the first, and the first one's cancel can't undo it. + const first = r.captureNextKey(() => captured.push('first')) + r.captureNextKey(c => captured.push(c)) + first() + r.handleKey(key('x', { code: 'KeyX' })) + assert.deepEqual(captured.at(-1), chord('KeyX')) +}) diff --git a/tests/keyboard-keys.test.cjs b/tests/keyboard-keys.test.cjs index e198dcab..b6730c1e 100644 --- a/tests/keyboard-keys.test.cjs +++ b/tests/keyboard-keys.test.cjs @@ -1,6 +1,6 @@ const test = require('node:test') const assert = require('node:assert/strict') -const { KeyTranslator, translateUsage, navigationGestureForKey } = require('../.test-build/app/g2/keyboard/keys.js') +const { KeyTranslator, translateUsage } = require('../.test-build/app/g2/keyboard/keys.js') const { encodeTerminalKey, terminalScrollbackKey } = require('../.test-build/app/apps/terminal/terminal-keys.js') const K = 7 @@ -98,19 +98,6 @@ test('reset forgets held keys and modifiers without emitting, and stops the repe assert.deepEqual([events.at(-1).key, events.at(-1).modifiers.shift], ['a', false]) }) -test('navigation keys stand in for ring gestures; everything else does nothing', () => { - const gesture = (k, extra = {}) => navigationGestureForKey(key({ key: k, text: undefined, ...extra })) - assert.equal(gesture('ArrowUp'), 'scroll-up'); assert.equal(gesture('ArrowDown'), 'scroll-down') - assert.equal(gesture('Enter'), 'click'); assert.equal(gesture('ArrowRight'), 'click') - assert.equal(gesture('Escape'), 'double-click'); assert.equal(gesture('ArrowLeft'), 'double-click') - // Holding an arrow keeps scrolling; holding Enter or Escape acts once. - assert.equal(gesture('ArrowDown', { repeat: true }), 'scroll-down') - assert.equal(gesture('Enter', { repeat: true }), null); assert.equal(gesture('Escape', { repeat: true }), null) - assert.equal(gesture('Enter', { phase: 'up' }), null) - assert.equal(gesture('Enter', { modifiers: { ctrl: true } }), null) - assert.equal(gesture('a'), null); assert.equal(gesture(' '), null); assert.equal(gesture('Backspace'), null) -}) - test('terminal: printable keys, Enter, Tab, Backspace, Escape', () => { const enc = (overrides, appCursor = false) => encodeTerminalKey(key(overrides), appCursor) assert.equal(enc({}), 'a') diff --git a/tests/tsconfig.json b/tests/tsconfig.json index cf0a2f81..d08f54c1 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -21,6 +21,8 @@ "../app/g2/keyboard/client.ts", "../app/g2/keyboard/bridge.ts", "../app/g2/keyboard/keys.ts", + "../app/g2/keyboard/bindings.ts", + "../app/g2/keyboard/key-router.ts", "../app/apps/terminal/terminal-keys.ts", "../app/phone-ui/ble-bandwidth-meter.ts", "../app/apps/terminal/hub-header.ts",