From ab692741f712d5eba639750df7327b9a96a16d69 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 15 Nov 2026 23:57:33 -0800 Subject: [PATCH] Add support for a BLE mouse (ie, the touchpad/mouse buttons in a handheld mouse/keyboard combo like the Rii K28) --- app/apps/developer/developer-app.ts | 8 + app/apps/developer/mouse-test.ts | 206 ++++++++++++++++++ app/g2/keyboard/bindings.ts | 27 ++- app/g2/keyboard/bridge.ts | 11 +- app/g2/keyboard/client.ts | 85 ++++++-- app/g2/keyboard/hid.ts | 104 ++++++++- app/g2/keyboard/key-router.ts | 18 +- app/g2/keyboard/keys.ts | 25 ++- app/g2/keyboard/pointer.ts | 83 +++++++ app/g2/keyboard/service.ts | 19 ++ app/ui/keybinding-settings.ts | 4 +- .../g2protocol/session/GlassesSessionCore.kt | 5 +- tests/keyboard-bindings.test.cjs | 57 +++++ tests/keyboard-bridge.test.cjs | 90 +++++++- tests/keyboard-keys.test.cjs | 21 ++ tests/keyboard-pointer.test.cjs | 58 +++++ tests/mouse-test-page.test.cjs | 136 ++++++++++++ tests/tsconfig.json | 1 + 18 files changed, 901 insertions(+), 57 deletions(-) create mode 100644 app/apps/developer/mouse-test.ts create mode 100644 app/g2/keyboard/pointer.ts create mode 100644 tests/keyboard-pointer.test.cjs create mode 100644 tests/mouse-test-page.test.cjs diff --git a/app/apps/developer/developer-app.ts b/app/apps/developer/developer-app.ts index 276990b3..02615a0a 100644 --- a/app/apps/developer/developer-app.ts +++ b/app/apps/developer/developer-app.ts @@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo"; import { AccelerometerDemoLayer } from "./accelerometer-demo"; import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark"; import { LightSensorDemoLayer } from "./light-sensor-demo"; +import { MouseTestLayer } from "./mouse-test"; import { MicrophonesMonitorLayer } from "./microphones-monitor"; import { ResourceUsageLayer } from "./resource-usage"; import { RingStatusLayer } from "./ring-status"; @@ -70,6 +71,13 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M label: "Microphones", onSelect: openMicrophones, }, + { + // The touchpad or mouse of a keyboard paired through the glasses. + label: "Mouse cursor", + onSelect: (ctx) => { + ctx.stack.push(new MouseTestLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender)); + }, + }, { label: "BLE bandwidth", onSelect: (ctx) => { diff --git a/app/apps/developer/mouse-test.ts b/app/apps/developer/mouse-test.ts new file mode 100644 index 00000000..a2233036 --- /dev/null +++ b/app/apps/developer/mouse-test.ts @@ -0,0 +1,206 @@ +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { GrayImage } from "../../graphics/image"; +import type { HidPointerSample } from "../../g2/keyboard/hid"; +import { keyboardService } from "../../g2/keyboard/service"; +import { type InputEvent } from "../../ui/gestures"; +import { Layer, type LayerContext } from "../../ui/layers"; +import { shell } from "../../ui/shell/shell"; +import { clamp } from "../../util/numeric-util"; + +// How often to reconcile holding the pointer with window visibility. +const RECONCILE_INTERVAL_MS = 400; +/** A touchpad reports a hundred times a second; the page repaints at most this often. */ +const RENDER_INTERVAL_MS = 33; +/** Cursor pixels per count the touchpad reports. */ +const POINTER_GAIN = 1; +/** Left-click marks kept on the page. */ +const MAX_MARKS = 12; + +/** The arrow, tip at its top left: '#' is the outline, '.' the fill. */ +const CURSOR = [ + "#", + "##", + "#.#", + "#..#", + "#...#", + "#....#", + "#.....#", + "#......#", + "#.......#", + "#........#", + "#.....#####", + "#..#..#", + "#.# #..#", + "## #..#", + "# #..#", + " #..#", + " ##", +]; + +/** + * Mouse cursor test: moves an arrow with the touchpad or mouse of the + * keyboard paired through the glasses (app/g2/keyboard/), and shows its + * buttons and wheel. The main UI has no cursor; this is where one is tried + * out. + * + * While the page is visible it takes the pointer, so the buttons and wheel + * show here instead of acting as select, back and scroll. A left click + * leaves a mark where the arrow is and a right click clears them. Leave + * with Esc, or a double tap on the ring or a temple. + */ +export class MouseTestLayer implements Layer { + private x = -1; + private y = -1; + private size = { width: 0, height: 0 }; + private buttons = 0; + private wheel = 0; + private marks: Array<{ x: number; y: number }> = []; + private samples = 0; + private rate = 0; + private rateWindowStart = Date.now(); + private removed = false; + private release: (() => void) | null = null; + private reconcileTimer: ReturnType | null = null; + private renderTimer: ReturnType | null = null; + + constructor( + private readonly windowId: string, + private readonly requestRender: () => void, + ) { + this.reconcileTimer = setInterval(() => this.reconcile(), RECONCILE_INTERVAL_MS); + this.reconcile(); + } + + /** Hold the pointer only while the page can be seen; elsewhere its buttons go back to being keys. */ + private reconcile(): void { + if (this.removed) return; + const visible = shell.isWindowVisible(this.windowId); + if (visible && !this.release) { + this.release = keyboardService.takePointer((sample) => this.onSample(sample)); + } else if (!visible && this.release) { + this.release(); + this.release = null; + this.buttons = 0; + } + // The rate reads zero once the pointer has been still for a second. + const now = Date.now(); + if (now - this.rateWindowStart >= 1000) { + const rate = this.samples; + this.samples = 0; + this.rateWindowStart = now; + if (rate !== this.rate) { + this.rate = rate; + this.scheduleRender(); + } + } + } + + private onSample(sample: HidPointerSample): void { + if (this.removed) return; + this.samples++; + const { width, height } = this.size; + if (width > 0 && height > 0) { + if (this.x < 0) { + this.x = width / 2; + this.y = height / 2; + } + this.x = clamp(this.x + sample.dx * POINTER_GAIN, 0, width - 1); + this.y = clamp(this.y + sample.dy * POINTER_GAIN, 0, height - 1); + } + const pressed = sample.buttons & ~this.buttons; + this.buttons = sample.buttons; + this.wheel += sample.wheel; + if (pressed & 1 && this.x >= 0) { + this.marks.push({ x: Math.round(this.x), y: Math.round(this.y) }); + if (this.marks.length > MAX_MARKS) this.marks.shift(); + } + if (pressed & 2) this.marks = []; + this.scheduleRender(); + } + + private scheduleRender(): void { + if (this.renderTimer !== null || this.removed) return; + this.renderTimer = setTimeout(() => { + this.renderTimer = null; + if (!this.removed) this.requestRender(); + }, RENDER_INTERVAL_MS); + } + + onRemoved(): void { + this.removed = true; + if (this.reconcileTimer !== null) clearInterval(this.reconcileTimer); + this.reconcileTimer = null; + if (this.renderTimer !== null) clearTimeout(this.renderTimer); + this.renderTimer = null; + this.release?.(); + this.release = null; + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const { width, height, padding } = ctx.stack.getBaseSize(); + this.size = { width, height }; + const image = new GrayImage(width, height, 0); + const left = padding.left; + const right = width - padding.right; + const top = padding.top; + const line = font.lineHeight + 4; + + image.drawText(font, left, top, "Mouse cursor", 220); + const snapshot = keyboardService.snapshot(); + const problem = snapshot.state !== "connected" + ? "No keyboard is connected." + : !snapshot.pointer ? "This keyboard has no touchpad or mouse buttons." : null; + if (problem) { + image.drawText(font, left, top + 2 * line, problem, 190); + image.drawText(font, left, top + 3 * line, "Pair one under Settings > Keyboard.", 120); + return image; + } + + const position = this.x < 0 ? "move the pointer" : `${Math.round(this.x)}, ${Math.round(this.y)}`; + image.drawText(font, right - font.measureText(position), top, position, 150); + // Buttons: a box each, filled while held. + let x = left; + const boxY = top + line; + for (const [index, label] of ["L", "M", "R"].entries()) { + // HID numbers them left, right, middle; they sit left, middle, right. + const held = !!(this.buttons & [1, 4, 2][index]!); + const boxW = font.measureText(label) + 10; + if (held) image.fillRect(x, boxY, boxW, font.lineHeight + 2, 200); + else image.drawRect(x, boxY, boxW, font.lineHeight + 2, 90); + image.drawText(font, x + 5, boxY + 1, label, held ? 1 : 170); + x += boxW + 4; + } + image.drawText(font, x + 8, boxY + 1, `wheel ${this.wheel}`, 150); + const rate = `${this.rate}/s`; + image.drawText(font, right - font.measureText(rate), boxY + 1, rate, 110); + const hint = "Left click marks, right click clears. Esc or double tap: back"; + image.drawText(font, left, height - padding.bottom - font.lineHeight, hint, 90); + + for (const mark of this.marks) { + image.drawLine(mark.x - 4, mark.y, mark.x + 4, mark.y, 140); + image.drawLine(mark.x, mark.y - 4, mark.x, mark.y + 4, 140); + } + if (this.x >= 0) drawCursor(image, Math.round(this.x), Math.round(this.y)); + return image; + } + + async handleInput(event: InputEvent, ctx: LayerContext): Promise { + if (event.type === "double-click") { + ctx.stack.pop(); + } + } +} + +function drawCursor(image: GrayImage, x: number, y: number): void { + for (const [row, pixels] of CURSOR.entries()) { + for (let col = 0; col < pixels.length; col++) { + const pixel = pixels[col]; + if (pixel === " ") continue; + const px = x + col; + const py = y + row; + if (px < 0 || py < 0 || px >= image.width || py >= image.height) continue; + image.setPixel(px, py, pixel === "#" ? 255 : 70); + } + } +} diff --git a/app/g2/keyboard/bindings.ts b/app/g2/keyboard/bindings.ts index fc70a4cf..85606bfe 100644 --- a/app/g2/keyboard/bindings.ts +++ b/app/g2/keyboard/bindings.ts @@ -10,10 +10,15 @@ * (a terminal session), since that is the point of binding a remote's * few buttons. * 2. Built-in bindings that also apply there: Escape with Shift, Ctrl or - * Alt goes back, which is the way out of a terminal session. + * Alt goes back, which is the way out of a terminal session; and a + * mouse's buttons and wheel, which are never typed. * 3. Built-in navigation, only for keys no window took: the arrows are the * watch's four swipes, Enter selects, Escape goes back. * + * A keyboard's touchpad or mouse arrives here as keys too: "MouseButton1" + * (left), "MouseButton2" (right), "MouseButton3" (middle) and so on, and + * "WheelUp" / "WheelDown" / "WheelLeft" / "WheelRight" (see pointer.ts). + * * No platform imports: this runs under node in the tests. */ import type { KeyEvent } from "./keys"; @@ -113,6 +118,8 @@ const CODE_LABELS: Record = { BrightnessUp: "Brightness Up", BrightnessDown: "Brightness Down", BrowserSearch: "Search", BrowserHome: "Browser Home", BrowserBack: "Browser Back", BrowserForward: "Browser Forward", LaunchMail: "Mail", LaunchApplication1: "App 1", LaunchApplication2: "App 2", + MouseButton1: "Left button", MouseButton2: "Right button", MouseButton3: "Middle button", + WheelUp: "Wheel up", WheelDown: "Wheel down", WheelLeft: "Wheel left", WheelRight: "Wheel right", }; /** A key as a person would name it. Unknown keys keep their HID usage number, which is all there is. */ @@ -125,6 +132,8 @@ function codeLabel(code: string): string { if (match) return match[1]!; match = /^Numpad(.+)$/.exec(code); if (match) return `Numpad ${match[1]}`; + match = /^MouseButton([0-9]+)$/.exec(code); + if (match) return `Mouse button ${match[1]}`; match = /^Usage([0-9a-f]+)$/.exec(code); if (match) return `Key 0x${match[1]}`; match = /^Consumer([0-9a-f]+)$/.exec(code); @@ -174,12 +183,26 @@ export function keyBindingMap(bindings: readonly KeyBinding[]): Map [chordId(binding.chord), binding.action])); } +/** What a mouse does unless bound otherwise: the ring's tap, double tap, hold and scroll. */ +const MOUSE_DEFAULTS: Record = { + MouseButton1: "click", + MouseButton2: "double-click", + MouseButton3: "long-press", + WheelUp: "scroll-up", + WheelDown: "scroll-down", +}; + /** * The built-in meaning of a key press, or null when it has none. * `overridesTyping`: it holds even where the focused window takes keys. * Built-ins go by the logical key, so both Enter keys select. */ export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overridesTyping: boolean } | null { + // A mouse's buttons and wheel are the ring's gestures: tap, double tap, + // hold, and scroll. Nothing types them, so they hold everywhere, whatever + // is held on the keyboard. + const mouse = MOUSE_DEFAULTS[event.key]; + if (mouse) return { action: mouse, overridesTyping: true }; const { shift, ctrl, alt, meta } = event.modifiers; if (meta) return null; if (event.key === "Escape") { @@ -208,6 +231,8 @@ export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overrid export const DEFAULT_KEYS_HELP = "Arrow keys: swipe up, down, left and right, as on the watch. Enter: select. Esc: back.\n\n" + "In a terminal session those keys are typed instead, and Shift+Esc (or Ctrl+Esc, Alt+Esc) goes back.\n\n" + + "Mouse or touchpad: the left button selects, the right button goes back, the middle button opens the system menu, " + + "and the wheel scrolls. These work in a terminal session too.\n\n" + "A custom keybinding replaces these for its key and works everywhere, terminals included. " + "\"Nothing\" turns a key's meaning here off without stopping it being typed."; diff --git a/app/g2/keyboard/bridge.ts b/app/g2/keyboard/bridge.ts index 5dcfa78d..bfe53593 100644 --- a/app/g2/keyboard/bridge.ts +++ b/app/g2/keyboard/bridge.ts @@ -12,7 +12,7 @@ * No platform imports: this runs under node in the tests. */ import { KeyboardClient, KeyboardClientError, type KeyboardStatus } from "./client"; -import type { HidKeyEvent } from "./hid"; +import type { HidKeyEvent, HidPointerSample } from "./hid"; import { KeyboardDm, KeyboardError, @@ -56,6 +56,8 @@ export type KeyboardBridgeSnapshot = { busy: boolean; /** The remembered keyboard is away and the bridge keeps trying to bring it back. */ waiting: boolean; + /** The connected keyboard has a touchpad or mouse buttons. */ + pointer: boolean; }; export type KeyboardBridgeHost = { @@ -65,6 +67,8 @@ export type KeyboardBridgeHost = { storeSaved(saved: SavedKeyboard | null): void; /** Key transitions from the keyboard (including forced releases). */ onKeys(events: HidKeyEvent[]): void; + /** A report from the keyboard's touchpad or mouse (including a forced release of its buttons). */ + onPointer(sample: HidPointerSample): void; /** The keyboard stopped being usable; drop held-key state without releases. */ onKeyboardGone(): void; /** Something in snapshot() changed. */ @@ -129,6 +133,7 @@ export class KeyboardBridge { scanResults: sortAdverts(this.scanResults), busy: this.busy, waiting: this.available && !!this.saved && !this.ready && !this.userDisconnected && !this.busy, + pointer: this.ready && this.client?.configuration?.pointer === true, }; } @@ -328,6 +333,7 @@ export class KeyboardBridge { (record) => live() && this.host.send(record), { keys: (events) => { if (live() && this.ready) this.host.onKeys(events); }, + pointer: (sample) => { if (live() && this.ready) this.host.onPointer(sample); }, advert: (advert) => { if (live()) this.addAdvert(advert); }, status: (status) => { if (live()) this.onStatus(status); }, dm: (event, status) => { if (live()) this.onDm(event, status); }, @@ -380,7 +386,8 @@ export class KeyboardBridge { } this.host.log( `keyboard ready: ${keyboard.name || keyboard.address}, ${config.watched.length} of ` + - `${config.reports.filter((r) => r.type === 1).length} input reports subscribed, epoch ${config.epoch}`); + `${config.reports.filter((r) => r.type === 1).length} input reports subscribed` + + `${config.pointer ? ", with a pointer" : ""}, epoch ${config.epoch}`); this.setState("connected"); } catch (e) { if (!live()) return; diff --git a/app/g2/keyboard/client.ts b/app/g2/keyboard/client.ts index 474a8058..627fc14c 100644 --- a/app/g2/keyboard/client.ts +++ b/app/g2/keyboard/client.ts @@ -10,7 +10,16 @@ * numbers found by configure() are only good for one connection epoch. * No platform imports: this runs under node in the tests. */ -import { HidKeys, parseReportMap, reportHasKeys, type HidKeyEvent, type HidReport } from "./hid"; +import { + HidKeys, + parseReportMap, + reportHasKeys, + reportHasPointer, + reportPointer, + type HidKeyEvent, + type HidPointerSample, + type HidReport, +} from "./hid"; import { KeyboardDm, KeyboardError, @@ -42,6 +51,12 @@ export type KeyboardStatus = { export type KeyboardClientEvents = { /** Key transitions from one input report (or the releases a gap forces). */ keys?(events: HidKeyEvent[]): void; + /** + * One report from the keyboard's touchpad or mouse: the buttons now held + * (across all its pointer reports) and the movement and wheel since the + * last one. Also sent with everything zero when a gap forces a release. + */ + pointer?(sample: HidPointerSample): void; advert?(advert: KeyboardAdvert): void; status?(status: KeyboardStatus): void; /** The stream may have lost records; held keys were released. */ @@ -78,8 +93,10 @@ export type KeyboardConfiguration = { epoch: number; /** Every report found, input or not. */ reports: KeyboardReport[]; - /** Handles of the input reports subscribed for keys. */ + /** Handles of the input reports subscribed: those that carry keys, or a pointer's buttons and movement. */ watched: number[]; + /** Whether any of them is a pointer's (a touchpad, mouse buttons, a wheel). */ + pointer: boolean; serviceChanged?: number; }; @@ -109,7 +126,7 @@ type Pending = { timer: ReturnType; }; type Waiter = { test: () => boolean; resolve: () => void; reject: (e: Error) => void; timer: ReturnType }; -type Input = { report: HidReport; id: number; service: number }; +type Input = { report: HidReport; id: number; service: number; keys: boolean; pointer: boolean }; const COMMAND_TIMEOUT_MS = 32_000; const MAX_PENDING = 8; @@ -159,6 +176,8 @@ export class KeyboardClient { private readonly pending = new Map(); private readonly waiters = new Set(); private readonly keys = new HidKeys(); + /** Pointer buttons held, by the report that says so (as HidKeys does for keys). */ + private readonly pointerButtons = new Map(); private inputs = new Map(); private readonly records = new KeyboardRecords(() => this.gap()); private readonly randomPasskey: () => number; @@ -248,18 +267,35 @@ export class KeyboardClient { this.gap(); return; } - try { - const events = this.keys.update(r.handle, input.report, r.data); - if (events.length) this.events.keys?.(events); - } catch (e) { - this.release(); - this.events.error?.(messageOf(e)); + if (input.keys) { + try { + const events = this.keys.update(r.handle, input.report, r.data); + if (events.length) this.events.keys?.(events); + } catch (e) { + this.release(); + this.events.error?.(messageOf(e)); + } + } + if (input.pointer) { + const sample = reportPointer(input.report, r.data); + this.pointerButtons.set(r.handle, sample.buttons); + this.events.pointer?.({ ...sample, buttons: this.heldButtons() }); } } + private heldButtons(): number { + let buttons = 0; + for (const held of this.pointerButtons.values()) buttons |= held; + return buttons; + } + + /** Nothing is held any more, as far as anyone can tell: say so. */ private release(): void { const events = this.keys.reset(); if (events.length) this.events.keys?.(events); + const buttons = this.heldButtons(); + this.pointerButtons.clear(); + if (buttons) this.events.pointer?.({ buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }); } private gap(): void { @@ -524,13 +560,13 @@ export class KeyboardClient { /** * Discover every HID service, its Report Map and each report's Report * Reference; select Report Protocol and subscribe the input reports that - * can carry keys. Needs an encrypted link. No handle is assumed. + * carry keys or a pointer's buttons and movement. Needs an encrypted link. + * No handle is assumed. * - * Only what typing needs is required of the keyboard: a Report Map that - * parses, and at least one key-bearing input report with a Report - * Reference and a CCC. A report that can't be placed is skipped and - * logged, as is anything wrong with an external report (the battery - * level, usually). + * Only what input needs is required of the device: a Report Map that + * parses, and at least one such input report with a Report Reference and + * a CCC. A report that can't be placed is skipped and logged, as is + * anything wrong with an external report (the battery level, usually). */ async configure(): Promise { const epoch = this.epoch; @@ -543,6 +579,7 @@ export class KeyboardClient { log(`gatt ${attrs.map((a) => `${a.handle}=${typeof a.uuid === "number" ? a.uuid.toString(16) : a.uuid}`).join(" ")}`); const reports: KeyboardReport[] = []; const watched: KeyboardReport[] = []; + let pointerFound = false; const skipped: string[] = []; const descriptorsFor = (handle: number): Attribute[] => { const end = attrs.find((a) => a.handle > handle && @@ -576,8 +613,14 @@ export class KeyboardClient { const layout = parsed.get(id); if (!layout) return skip(a, `input report id ${id} is not in the Report Map`); if (ccc === undefined || (properties !== undefined && !(properties & 0x10))) return skip(a, "input report can't notify"); - if (reportHasKeys(layout)) watched.push(report); - else log(`report ${a.handle} (id ${id}) carries no keys; not subscribed`); + const keys = reportHasKeys(layout); + const pointer = reportHasPointer(layout); + if (keys || pointer) watched.push(report); + else log(`report ${a.handle} (id ${id}) carries no keys or pointer; not subscribed`); + if (pointer) { + pointerFound = true; + log(`report ${a.handle} (id ${id}) is a pointer's${keys ? ", with keys" : ""}`); + } } reports.push(report); log(`report ${a.handle}: id ${id} type ${type} ccc ${ccc ?? "-"} properties ${properties ?? "?"}`); @@ -625,7 +668,7 @@ export class KeyboardClient { if (this.closed || this.epoch !== epoch) throw new KeyboardClientError("The keyboard link changed during setup"); if (!watched.length) { throw new KeyboardClientError(reports.length || skipped.length - ? `No usable keyboard input report (${skipped.join("; ") || "none carries keys"})` + ? `No usable keyboard input report (${skipped.join("; ") || "none carries keys or a pointer"})` : "This device has no HID service"); } if (watched.length > MAX_WATCHES) throw new KeyboardClientError("Keyboard has more than 16 input reports"); @@ -634,7 +677,9 @@ export class KeyboardClient { const watches = watched.map((r) => r.handle); if (changed && changedCcc && watches.length < MAX_WATCHES) watches.push(changed.handle); const serviceChanged = changed && changedCcc && watches.includes(changed.handle) ? changed.handle : undefined; - const config: KeyboardConfiguration = { epoch: this.epoch, reports, watched: watched.map((r) => r.handle), serviceChanged }; + const config: KeyboardConfiguration = { + epoch: this.epoch, reports, watched: watched.map((r) => r.handle), pointer: pointerFound, serviceChanged, + }; this.restore(config); try { await this.command(KeyboardOp.watch, [watches.length, ...watches.flatMap(le16)]); @@ -671,7 +716,7 @@ export class KeyboardClient { if (!r.map.length || r.map.length > MAX_VALUE) throw new KeyboardClientError("Invalid restored Report Map"); const report = parseReportMap(r.map).get(r.id); if (!report) throw new KeyboardClientError("Restored Report ID missing"); - inputs.set(r.handle, { report, id: r.id, service: r.service }); + inputs.set(r.handle, { report, id: r.id, service: r.service, keys: reportHasKeys(report), pointer: reportHasPointer(report) }); } if (!inputs.size || inputs.size > MAX_WATCHES) throw new KeyboardClientError("Invalid restored input count"); this.release(); diff --git a/app/g2/keyboard/hid.ts b/app/g2/keyboard/hid.ts index 566d1491..809f4ca5 100644 --- a/app/g2/keyboard/hid.ts +++ b/app/g2/keyboard/hid.ts @@ -6,8 +6,9 @@ * * HID over GATT carries the Report ID out of band (in each characteristic's * Report Reference descriptor), so byte zero of a report value is data: it is - * never stripped as USB HID would. Only the keyboard/keypad page (7) and the - * consumer page (12) produce keys; pointer axes and padding are ignored. + * never stripped as USB HID would. The keyboard/keypad page (7) and the + * consumer page (12) produce keys. A mouse's or touchpad's report (buttons, + * relative X/Y, wheel) is read by reportPointer. * No platform imports: this runs under node in the tests. */ @@ -20,6 +21,8 @@ type Field = { size: number; count: number; variable: boolean; + /** Reports a change since the last report (a mouse's axes and wheel), not a state. */ + relative: boolean; min: number; max: number; usages: HidUsage[]; @@ -90,7 +93,8 @@ export function parseReportMap(bytes: Uint8Array): Map { reports.set(g.id, r); } const field: Field = { - bit: r.bits, size: g.size, count: g.count, variable: !!(v & 2), min: g.min, max: g.max, usages: [...usages], + bit: r.bits, size: g.size, count: g.count, variable: !!(v & 2), relative: !!(v & 4), + min: g.min, max: g.max, usages: [...usages], }; if (lo || hi) { if (!lo || !hi || lo.page !== hi.page || hi.usage < lo.usage) fail("usage range"); @@ -98,8 +102,8 @@ export function parseReportMap(bytes: Uint8Array): Map { } r.bits += g.size * g.count; if (r.bits > 4096) fail("report exceeds 512 bytes"); - // Constant padding and relative pointer axes are not keys. - if (!(v & 1) && !(v & 4)) r.fields.push(field); + // Constant fields are padding. + if (!(v & 1)) r.fields.push(field); } else if (tag !== 9 && tag !== 11) fail("unknown main item"); usages = []; lo = hi = undefined; @@ -130,7 +134,86 @@ function usageAt(f: Field, index: number): HidUsage | undefined { */ export function reportHasKeys(report: HidReport): boolean { const isKeyPage = (u: HidUsage): boolean => u.page === HID_PAGE_KEYBOARD || u.page === HID_PAGE_CONSUMER; - return report.fields.some((f) => f.usages.some(isKeyPage) || (f.range !== undefined && isKeyPage(f.range[0]))); + return report.fields.some((f) => !f.relative && (f.usages.some(isKeyPage) || (f.range !== undefined && isKeyPage(f.range[0])))); +} + +/** One field's `index`th value, sign-extended when the field is signed. Bits past the data read as zero. */ +function fieldValue(f: Field, index: number, data: Uint8Array): number { + let value = 0; + for (let j = 0; j < f.size; j++) { + const bit = f.bit + index * f.size + j; + if ((data[bit >>> 3] ?? 0) & (1 << (bit & 7))) value += 2 ** j; + } + if (f.min < 0 && value >= 2 ** (f.size - 1)) value -= 2 ** f.size; + return value; +} + +/** What one report from a mouse or touchpad says. */ +export type HidPointerSample = { + /** Buttons held, bit 0 for button 1 (left), bit 1 for button 2 (right), bit 2 for button 3 (middle). */ + buttons: number; + /** Movement since the last report, in the device's counts; +x right, +y down. */ + dx: number; + dy: number; + /** Wheel detents since the last report; positive is away from the user ("up"). */ + wheel: number; + /** Horizontal wheel (AC Pan); positive is right. */ + pan: number; +}; + +const HID_PAGE_GENERIC_DESKTOP = 1; +const HID_PAGE_BUTTON = 9; +const USAGE_X = 0x30; +const USAGE_Y = 0x31; +const USAGE_WHEEL = 0x38; +const USAGE_AC_PAN = 0x238; +/** Buttons past this are not tracked; a mouse with more is rare and has nothing to bind them to anyway. */ +const MAX_POINTER_BUTTONS = 8; + +/** Which part of a pointer sample a field's usage feeds, if any. */ +function pointerRole(f: Field, u: HidUsage | undefined): "button" | "dx" | "dy" | "wheel" | "pan" | null { + if (!u || !f.variable) return null; + if (u.page === HID_PAGE_BUTTON) return !f.relative && u.usage >= 1 && u.usage <= MAX_POINTER_BUTTONS ? "button" : null; + if (!f.relative) return null; // an absolute X/Y is a touchscreen or a digitizer, not a mouse + if (u.page === HID_PAGE_GENERIC_DESKTOP) { + if (u.usage === USAGE_X) return "dx"; + if (u.usage === USAGE_Y) return "dy"; + if (u.usage === USAGE_WHEEL) return "wheel"; + } + if (u.page === HID_PAGE_CONSUMER && u.usage === USAGE_AC_PAN) return "pan"; + return null; +} + +/** Whether an input report carries a mouse's buttons, movement or wheel. */ +export function reportHasPointer(report: HidReport): boolean { + return report.fields.some((f) => { + for (let i = 0; i < f.count; i++) if (pointerRole(f, usageAt(f, i))) return true; + return false; + }); +} + +/** + * Read a mouse or touchpad report. Lenient about length, unlike reportKeys: + * some devices leave the wheel bytes off when there is nothing to say, and a + * missing byte is zero either way. + */ +export function reportPointer(report: HidReport, data: Uint8Array): HidPointerSample { + const sample: HidPointerSample = { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }; + for (const f of report.fields) { + for (let i = 0; i < f.count; i++) { + const u = usageAt(f, i); + const role = pointerRole(f, u); + if (!role) continue; + const value = fieldValue(f, i, data); + if (value < f.min || value > f.max) continue; // out of range: no reading + if (role === "button") { + if (value) sample.buttons |= 1 << (u!.usage - 1); + } else { + sample[role] += value; + } + } + } + return sample; } /** The keys one input report says are held. Throws on a length mismatch or a rollover report. */ @@ -138,13 +221,10 @@ export function reportKeys(report: HidReport, data: Uint8Array): HidUsage[] { if (data.length !== Math.ceil(report.bits / 8)) throw new Error("HID input length does not match Report Map"); const keys = new Map(); for (const f of report.fields) { + // A relative field is a movement, not a key that is down. + if (f.relative) continue; for (let i = 0; i < f.count; i++) { - let value = 0; - for (let j = 0; j < f.size; j++) { - const bit = f.bit + i * f.size + j; - if (data[bit >>> 3]! & (1 << (bit & 7))) value += 2 ** j; - } - if (f.min < 0 && value >= 2 ** (f.size - 1)) value -= 2 ** f.size; + const value = fieldValue(f, i, data); if (value < f.min || value > f.max) continue; // null state const u = usageAt(f, f.variable ? i : value - f.min); if (!u || (f.variable && value === 0) || !u.usage) continue; diff --git a/app/g2/keyboard/key-router.ts b/app/g2/keyboard/key-router.ts index cbda2ac2..385f0a04 100644 --- a/app/g2/keyboard/key-router.ts +++ b/app/g2/keyboard/key-router.ts @@ -38,9 +38,10 @@ export type KeyRouterHost = { }; /** - * A held key repeats twenty times a second, faster than a list can repaint - * over the glasses' link. Repeats are dropped while a gesture is still being - * handled, and closer together than this. + * A held key repeats twenty times a second, and a wheel or a two-finger + * scroll reports faster still: more than a list can repaint over the + * glasses' link. Those are dropped while a gesture is still being handled, + * and closer together than this. */ const REPEAT_MIN_INTERVAL_MS = 100; @@ -73,13 +74,14 @@ export class KeyRouter { return; } const resolution = resolveKey(event, host.bindings()); + const droppable = event.repeat || event.burst === true; if (resolution.consume) { this.consumed.add(event.code); - if (resolution.gesture) this.perform(resolution.gesture, event.repeat); + if (resolution.gesture) this.perform(resolution.gesture, droppable); return; } if (host.offerToWindow(event)) return; - if (resolution.gesture) this.perform(resolution.gesture, event.repeat); + if (resolution.gesture) this.perform(resolution.gesture, droppable); } /** @@ -91,7 +93,7 @@ export class KeyRouter { const host = this.host; if (event.phase !== "down" || this.capture || !host.ready() || host.locked() || !host.screenOn()) return; const resolution = resolveKey(event, host.bindings()); - if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat); + if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat || event.burst === true); } /** @@ -124,9 +126,9 @@ export class KeyRouter { capture(chordOf(event)); } - private perform(gesture: KeyGesture, repeat: boolean): void { + private perform(gesture: KeyGesture, droppable: boolean): void { const now = this.host.now ? this.host.now() : Date.now(); - if (repeat && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return; + if (droppable && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return; this.lastGestureAtMs = now; void this.run(gesture, this.scheme.originFor(gesture)); } diff --git a/app/g2/keyboard/keys.ts b/app/g2/keyboard/keys.ts index 1a4c35e8..1ab9f146 100644 --- a/app/g2/keyboard/keys.ts +++ b/app/g2/keyboard/keys.ts @@ -24,12 +24,19 @@ export type KeyEvent = { modifiers: KeyModifiers; /** A synthesized auto-repeat of a held key. */ repeat: boolean; + /** + * One of a stream that can arrive faster than it can be acted on (a wheel + * turning): whoever acts on it may drop some rather than fall behind. + */ + burst?: boolean; /** Date.now() when the phone saw the transition. */ timestampMs: number; }; const HID_PAGE_KEYBOARD = 7; const HID_PAGE_CONSUMER = 12; +/** A mouse's buttons, which arrive here as keys: usage 1 is the left button, 2 the right, 3 the middle. */ +export const HID_PAGE_BUTTON = 9; const USAGE_CAPS_LOCK = 57; /** [unshifted, shifted, code] for US-layout punctuation on the keyboard page. */ @@ -159,6 +166,7 @@ export function translateUsage(page: number, usage: number, shift: boolean, caps const name = CONSUMER[usage]; return name ? { key: name, code: name } : { key: "Unidentified", code: `Consumer${usage.toString(16)}` }; } + if (page === HID_PAGE_BUTTON) return { key: `MouseButton${usage}`, code: `MouseButton${usage}` }; return { key: "Unidentified", code: `Page${page.toString(16)}Usage${usage.toString(16)}` }; } @@ -217,8 +225,9 @@ export class KeyTranslator { if (modifier) this.modifiers[modifier] = true; if (event.page === HID_PAGE_KEYBOARD && event.usage === USAGE_CAPS_LOCK) this.capsLock = !this.capsLock; this.emit("down", event.page, event.usage, false); - // The newest non-modifier key is the one that repeats, as on a PC. - if (!modifier) this.startRepeat(event.page, event.usage); + // The newest non-modifier key is the one that repeats, as on a PC. A + // mouse button neither repeats nor takes the repeat from a held key. + if (!modifier && event.page !== HID_PAGE_BUTTON) this.startRepeat(event.page, event.usage); } else { if (!this.held.delete(id)) return; if (modifier) this.modifiers[modifier] = this.otherHeld(event.usage, modifier); @@ -227,6 +236,18 @@ export class KeyTranslator { } } + /** + * A press and release in one, for input that has no "held" (a wheel's + * detent): `name` is both the key and the code, e.g. "WheelUp". Carries + * the modifiers held on the keyboard, like any key. + */ + pulse(name: string): void { + const timestampMs = this.now(); + for (const phase of ["down", "up"] as const) { + this.emitEvent({ phase, key: name, code: name, modifiers: { ...this.modifiers }, repeat: false, burst: true, timestampMs }); + } + } + /** The keyboard went away: forget everything without emitting releases. */ reset(): void { this.stopRepeat(); diff --git a/app/g2/keyboard/pointer.ts b/app/g2/keyboard/pointer.ts new file mode 100644 index 00000000..e1d13ede --- /dev/null +++ b/app/g2/keyboard/pointer.ts @@ -0,0 +1,83 @@ +/** + * A keyboard's touchpad or mouse, as the rest of the keyboard support sees + * it. Its buttons and wheel become keys ("MouseButton1", "WheelUp"), so + * they get default meanings and custom keybindings like any key + * (bindings.ts). Its movement goes nowhere unless something has taken the + * pointer (the cursor test page): there is no cursor in the main UI. + * No platform imports: this runs under node in the tests. + */ +import type { HidKeyEvent, HidPointerSample } from "./hid"; +import { HID_PAGE_BUTTON } from "./keys"; + +/** The names a wheel's detents arrive under, as keys. */ +export type WheelKey = "WheelUp" | "WheelDown" | "WheelLeft" | "WheelRight"; + +export type PointerInputHost = { + /** Button transitions, as HID usages on the Button page. */ + buttons(events: HidKeyEvent[]): void; + /** One wheel report in a direction, however many detents it claimed. */ + wheel(key: WheelKey): void; +}; + +export class PointerInput { + /** The buttons the host has been told are down. */ + private reported = 0; + /** The buttons actually down, as of the last sample. */ + private held = 0; + /** Buttons that went down while the pointer was taken: not keys until they have come up. */ + private ignored = 0; + private grab: ((sample: HidPointerSample) => void) | null = null; + + constructor(private readonly host: PointerInputHost) {} + + /** One report from the pointer (see KeyboardClientEvents.pointer). */ + handle(sample: HidPointerSample): void { + this.held = sample.buttons; + if (this.grab) { + this.grab(sample); + return; + } + this.ignored &= sample.buttons; + this.report(sample.buttons & ~this.ignored); + // One step per report: a touchpad's two-finger scroll sends many small + // ones, and the gesture they turn into is rate-limited downstream. + if (sample.wheel) this.host.wheel(sample.wheel > 0 ? "WheelUp" : "WheelDown"); + if (sample.pan) this.host.wheel(sample.pan > 0 ? "WheelRight" : "WheelLeft"); + } + + /** + * Take the pointer: every sample goes to `listener`, movement included, + * and its buttons and wheel stop acting as keys. Buttons held at that + * moment are released first, so nothing is left down. Returns a function + * that gives the pointer back. + */ + take(listener: (sample: HidPointerSample) => void): () => void { + this.report(0); + this.grab = listener; + return () => { + if (this.grab !== listener) return; + this.grab = null; + // A button still down was pressed for the listener, not as a key: its + // release is nobody's business either. + this.ignored = this.held; + }; + } + + /** The keyboard went away: nothing is held, and nobody is told. */ + reset(): void { + this.reported = 0; + this.held = 0; + this.ignored = 0; + } + + private report(buttons: number): void { + const changed = buttons ^ this.reported; + if (!changed) return; + const events: HidKeyEvent[] = []; + for (let bit = 0; bit < 8; bit++) { + if (changed & (1 << bit)) events.push({ page: HID_PAGE_BUTTON, usage: bit + 1, down: !!(buttons & (1 << bit)) }); + } + this.reported = buttons; + this.host.buttons(events); + } +} diff --git a/app/g2/keyboard/service.ts b/app/g2/keyboard/service.ts index 333704ff..84dfdcea 100644 --- a/app/g2/keyboard/service.ts +++ b/app/g2/keyboard/service.ts @@ -23,8 +23,10 @@ import { type KeyGesture, } from "./bindings"; import { KeyboardBridge, type KeyboardBridgeSnapshot, type SavedKeyboard } from "./bridge"; +import type { HidPointerSample } from "./hid"; import { KeyRouter } from "./key-router"; import { KeyTranslator, type KeyEvent } from "./keys"; +import { PointerInput } from "./pointer"; declare const java: any; declare const arc4random_uniform: ((bound: number) => number) | undefined; @@ -92,6 +94,11 @@ class KeyboardService { log: (line) => console.log(`[keyboard] ${line}`), }); private readonly translator = new KeyTranslator({ emit: (event) => this.router.handleKey(event) }); + /** The touchpad's buttons and wheel go in as keys; see pointer.ts. */ + private readonly pointer = new PointerInput({ + buttons: (events) => this.translator.handle(events), + wheel: (key) => this.translator.pulse(key), + }); readonly bridge = new KeyboardBridge({ send: (record) => { @@ -106,8 +113,10 @@ class KeyboardService { loadSaved: loadSavedKeyboard, storeSaved: (saved) => setStringSetting(SAVED_KEYBOARD_KEY, saved ? JSON.stringify(saved) : ""), onKeys: (events) => this.translator.handle(events), + onPointer: (sample) => this.pointer.handle(sample), onKeyboardGone: () => { this.translator.reset(); + this.pointer.reset(); this.router.reset(); }, onChanged: () => { @@ -206,6 +215,16 @@ class KeyboardService { return this.router.captureNextKey(onChord); } + /** + * Take the keyboard's touchpad or mouse: every report goes to `listener`, + * movement included, and its buttons and wheel stop acting as keys until + * the returned function is called. For the cursor test page; the main UI + * has no cursor. + */ + takePointer(listener: (sample: HidPointerSample) => void): () => void { + return this.pointer.take(listener); + } + private storeBindings(bindings: KeyBinding[]): void { this.bindings = bindings; this.bindingMap = keyBindingMap(bindings); diff --git a/app/ui/keybinding-settings.ts b/app/ui/keybinding-settings.ts index 8c813ed1..37717f17 100644 --- a/app/ui/keybinding-settings.ts +++ b/app/ui/keybinding-settings.ts @@ -27,7 +27,7 @@ class KeyCaptureLayer extends TextPageLayer { constructor(private readonly onChord: (chord: KeyChord, ctx: LayerContext) => void) { super( "Add keybinding", - "Press the key to bind, on the keyboard. Hold Shift, Ctrl or Alt with it to bind the combination.", + "Press the key to bind, on the keyboard: a key, a mouse button, or a turn of the wheel. Hold Shift, Ctrl or Alt with it to bind the combination.", ); } @@ -137,7 +137,7 @@ export function keybindingsMenuItem(): MenuItem { return { label: "Keybindings", description: - "Choose what keys do: arrows, Enter and Esc navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.", + "Choose what keys do: arrows, Enter, Esc and a mouse's buttons and wheel navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.", onSelect: (ctx) => new KeybindingsLayer(ctx.stack.getBaseSize()).open(ctx), render: ({ image, x, y, width }) => { const count = keyboardService.keyBindings().length; diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt index b14ac538..2620c5de 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -2001,13 +2001,13 @@ class GlassesSessionCore( tap(data.copyOf()) return } - logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size) val frame = BleProtocol.parseFrame(data) if (frame.ok && frame.sid == BleProtocol.SID_UI_SETTING) { // CFW keyboard-bridge records (field 131) come from whichever temple hosts the // keyboard. They are their own stream: neither an ACK nor glasses input, and the // once-a-second status record among them must not count as user activity (which - // would hold off battery polling). + // would hold off battery polling). A touchpad sends a hundred of them a second + // while a finger is on it, so they are not logged one by one either. val keyboardEvent = BleProtocol.parseFaceclawKeyboardEvent(frame.pb) if (keyboardEvent != null) { monitor.withLock { lastIncomingAtMs = now() } @@ -2015,6 +2015,7 @@ class GlassesSessionCore( return } } + logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size) val decodedWearState = BleProtocol.parseWearState(frame) val compassEvent = if (address.equals(rightAddress, ignoreCase = true)) BleProtocol.parseCompassEvent(frame) diff --git a/tests/keyboard-bindings.test.cjs b/tests/keyboard-bindings.test.cjs index d95752df..8c5359db 100644 --- a/tests/keyboard-bindings.test.cjs +++ b/tests/keyboard-bindings.test.cjs @@ -243,3 +243,60 @@ test('router: capturing a key for a new keybinding takes the chord and nothing e r.handleKey(key('x', { code: 'KeyX' })) assert.deepEqual(captured.at(-1), chord('KeyX')) }) + +test('a mouse by default: left taps, right double taps, middle holds, the wheel is the ring\'s scroll', () => { + const action = (k, overrides) => defaultKeyAction(key(k, overrides)) + // Nothing types a mouse button, so these hold in a terminal session too. + assert.deepEqual(action('MouseButton1'), { action: 'click', overridesTyping: true }) + assert.deepEqual(action('MouseButton2'), { action: 'double-click', overridesTyping: true }) + assert.deepEqual(action('MouseButton3'), { action: 'long-press', overridesTyping: true }) + assert.deepEqual(action('WheelUp'), { action: 'scroll-up', overridesTyping: true }) + assert.deepEqual(action('WheelDown'), { action: 'scroll-down', overridesTyping: true }) + // Whatever is held on the keyboard. + assert.deepEqual(action('MouseButton1', { modifiers: { ctrl: true, shift: true } }), { action: 'click', overridesTyping: true }) + // Extra buttons and the horizontal wheel mean nothing until bound. + for (const k of ['MouseButton4', 'MouseButton5', 'WheelLeft', 'WheelRight']) assert.equal(action(k), null) + const said = code => describeChord(chord(code)) + assert.equal(said('MouseButton1'), 'Left button'); assert.equal(said('MouseButton2'), 'Right button') + assert.equal(said('MouseButton3'), 'Middle button'); assert.equal(said('MouseButton4'), 'Mouse button 4') + assert.equal(said('WheelUp'), 'Wheel up'); assert.equal(describeChord(chord('WheelDown', { ctrl: true })), 'Ctrl+Wheel down') + assert.deepEqual(parseChordId('Shift+MouseButton2'), chord('MouseButton2', { shift: true })) +}) + +test('router: the mouse works where keys are being typed, its wheel is paced, and its buttons can be rebound', async () => { + const { r, state } = router({ bindings: [{ chord: chord('MouseButton4'), action: 'short-then-long-press' }, { chord: chord('MouseButton2'), action: 'none' }] }) + const wheel = k => { r.handleKey(key(k, { burst: true })); r.handleKey(up(k, { burst: true })) } + state.windowTakesKeys = true + r.handleKey(key('MouseButton1')); r.handleKey(up('MouseButton1')) + r.handleKey(key('MouseButton3')); r.handleKey(up('MouseButton3')) + await flush(); state.now += 500 + wheel('WheelDown') + // The wheel is the ring's scroll, and what follows it is the ring's too. + await flush(); state.now += 500 + r.handleKey(key('MouseButton1')); r.handleKey(up('MouseButton1')) + assert.deepEqual(state.gestures, ['click/watch', 'long-press/watch', 'scroll-down/ring', 'click/ring']) + assert.deepEqual(state.offered, [], 'never typed') + // A custom keybinding for an extra button; "Nothing" on the right button leaves it inert. + state.gestures.length = 0 + r.handleKey(key('MouseButton4')); r.handleKey(up('MouseButton4')); r.handleKey(key('MouseButton2')); r.handleKey(up('MouseButton2')) + assert.deepEqual(state.gestures, ['short-then-long-press/ring']) + // A two-finger scroll reports far faster than the display can follow: every 100 ms at most. + state.gestures.length = 0; state.offered.length = 0 + for (let i = 0; i < 40; i++) { await flush(); state.now += 25; wheel('WheelUp') } + assert.equal(state.gestures.length, 10); assert.ok(state.gestures.every(g => g === 'scroll-up/ring')) + // On a dark display a click wakes it and does nothing else. + state.screenOn = false; state.gestures.length = 0 + r.handleKey(key('MouseButton1')); r.handleKey(up('MouseButton1')) + assert.equal(state.wakes, 1); assert.deepEqual(state.gestures, []) +}) + +test('router: a mouse button or a turn of the wheel can be captured for a keybinding', () => { + const { r, state } = router() + const captured = [] + r.captureNextKey(c => captured.push(c)) + r.handleKey(key('MouseButton2')); r.handleKey(up('MouseButton2')) + r.captureNextKey(c => captured.push(c)) + r.handleKey(key('WheelUp', { burst: true, modifiers: { shift: true } })); r.handleKey(up('WheelUp', { burst: true })) + assert.deepEqual(captured, [chord('MouseButton2'), chord('WheelUp', { shift: true })]) + assert.deepEqual(state.gestures, []) +}) diff --git a/tests/keyboard-bridge.test.cjs b/tests/keyboard-bridge.test.cjs index 5fbdfedd..427f97ef 100644 --- a/tests/keyboard-bridge.test.cjs +++ b/tests/keyboard-bridge.test.cjs @@ -1,6 +1,6 @@ const test = require('node:test') const assert = require('node:assert/strict') -const { HidKeys, parseReportMap, reportKeys } = require('../.test-build/app/g2/keyboard/hid.js') +const { HidKeys, parseReportMap, reportHasKeys, reportHasPointer, reportKeys, reportPointer } = require('../.test-build/app/g2/keyboard/hid.js') const { KeyboardRecords, keyboardControl, le16, u16, parseKeyboardAdvert, mergeKeyboardAdverts, parseKeyboardAddress, formatKeyboardAddress, @@ -69,6 +69,41 @@ test('descriptor resource bounds and malformed items', () => { } }) +// A wheel mouse as most are described: three buttons, 8-bit X/Y and wheel. +const mouseMap = Uint8Array.from([0x05,1,0x09,2,0xa1,1,0x09,1,0xa1,0,0x05,9,0x19,1,0x29,3,0x15,0,0x25,1,0x75,1,0x95,3,0x81,2, + 0x75,5,0x95,1,0x81,1,0x05,1,0x09,0x30,0x09,0x31,0x09,0x38,0x15,0x81,0x25,0x7f,0x75,8,0x95,3,0x81,6,0xc0,0xc0]) +test('a mouse report: buttons, movement and wheel; not keys', () => { + const r = parseReportMap(mouseMap).get(0) + assert.equal(reportHasPointer(r), true); assert.equal(reportHasKeys(r), false) + assert.deepEqual(reportPointer(r, Uint8Array.of(0, 0, 0, 0)), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }) + assert.deepEqual(reportPointer(r, Uint8Array.of(0b101, 10, 0xf6, 1)), { buttons: 5, dx: 10, dy: -10, wheel: 1, pan: 0 }) + assert.deepEqual(reportPointer(r, Uint8Array.of(0b010, 0, 0, 0xff)), { buttons: 2, dx: 0, dy: 0, wheel: -1, pan: 0 }) + // Some devices leave trailing bytes off; what is missing is zero. Extra bytes are ignored. + assert.deepEqual(reportPointer(r, Uint8Array.of(1, 3)), { buttons: 1, dx: 3, dy: 0, wheel: 0, pan: 0 }) + assert.deepEqual(reportPointer(r, Uint8Array.of(0, 1, 1, 1, 9, 9)), { buttons: 0, dx: 1, dy: 1, wheel: 1, pan: 0 }) + // -128 is outside the declared range: no reading, rather than a jump. + assert.deepEqual(reportPointer(r, Uint8Array.of(0, 0x80, 0, 0)), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }) + assert.deepEqual(reportKeys(r, Uint8Array.of(7, 1, 1, 1)), []) + // A keyboard's report is not a pointer's. + assert.equal(reportHasPointer(parseReportMap(boot).get(0)), false) +}) +test('a touchpad with 12-bit axes and a horizontal wheel; an absolute digitizer is not a mouse', () => { + // Five buttons, 3 bits of padding, X/Y packed 12 bits each, wheel, then AC Pan from the consumer page. + const map = Uint8Array.from([0x05,1,0x09,2,0xa1,1,0x05,9,0x19,1,0x29,5,0x15,0,0x25,1,0x75,1,0x95,5,0x81,2, + 0x75,3,0x95,1,0x81,1,0x05,1,0x09,0x30,0x09,0x31,0x16,0x01,0xf8,0x26,0xff,0x07,0x75,12,0x95,2,0x81,6, + 0x09,0x38,0x15,0x81,0x25,0x7f,0x75,8,0x95,1,0x81,6,0x05,0x0c,0x0a,0x38,0x02,0x81,6,0xc0]) + const r = parseReportMap(map).get(0) + assert.equal(reportHasPointer(r), true) + // X = 0x123 (291), Y = 0xfff (-1): packed little endian, 12 bits each. + assert.deepEqual(reportPointer(r, Uint8Array.of(0b10000, 0x23, 0xf1, 0xff, 2, 0xfe)), { buttons: 16, dx: 291, dy: -1, wheel: 2, pan: -2 }) + // The pan field is on the consumer page, but it is a movement, not a key going down. + assert.equal(reportHasKeys(r), false); assert.deepEqual(reportKeys(r, Uint8Array.of(0, 0, 0, 0, 0, 5)), []) + // Absolute X/Y (a touchscreen, a pen) is something else, and is left alone. + const absolute = parseReportMap(Uint8Array.of(0x05,1,0x09,0x30,0x09,0x31,0x15,0,0x26,0xff,0x0f,0x75,16,0x95,2,0x81,2)).get(0) + assert.equal(reportHasPointer(absolute), false) + assert.deepEqual(reportPointer(absolute, Uint8Array.of(1, 2, 3, 4)), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }) +}) + test('control records, UUIDs and addresses', () => { assert.deepEqual([...keyboardControl(4,0x1234,0x5678,[1,2,3,4,5,6,7])], [75,66,1,4,0x34,0x12,0x78,0x56,1,2,3,4,5,6,7]) assert.throws(() => keyboardControl(13,1,1)) @@ -238,23 +273,39 @@ function makeCombo(g) { g.attrs.clear(); for (const [handle, attr] of sorted) g.attrs.set(handle, attr) } for (const declarations of ['short', 'refused']) { - test(`client: a keyboard with a trackpad whose stack answers declaration reads ${declarations} still types`, async () => { - const events = [], logs = [] + test(`client: a keyboard with a trackpad whose stack answers declaration reads ${declarations} types and points`, async () => { + const events = [], pointer = [], logs = [] let client const g = fakeGlasses(payload => client.accept(payload)) makeCombo(g); g.declarations = declarations - client = new KeyboardClient(g.send, { keys: e => events.push(...e), log: line => logs.push(line) }) + client = new KeyboardClient(g.send, { keys: e => events.push(...e), pointer: p => pointer.push(p), log: line => logs.push(line) }) await client.enable(); await client.connect(1, new Uint8Array(6)); const config = await client.configure() - // Keys and media keys are subscribed; the pointer and the battery are not. - assert.deepEqual(config.watched, [27,31]); assert.deepEqual(g.watches, [27,31,3]); assert.deepEqual(g.writes, [25,4,28,32]) + // Keys, media keys and the trackpad are subscribed; the battery level is not. + assert.deepEqual(config.watched, [27,31,35]); assert.equal(config.pointer, true) + assert.deepEqual(g.watches, [27,31,35,3]); assert.deepEqual(g.writes, [25,4,28,32,36]) assert.deepEqual(config.reports.map(r => [r.handle, r.id]), [[27,1],[31,2],[35,4]]) assert.deepEqual(config.reports.map(r => r.properties), declarations === 'short' ? [0x12,0x12,0x12] : [undefined,undefined,undefined]) g.input(27, [0,0,4,0,0,0,0,0]) assert.deepEqual(events.map(e => [e.usage,e.down]), [[4,true]]) + // The trackpad: movement and buttons, and never a key. + g.input(35, [0, 5, 0xfd]); g.input(35, [1, 0, 0]); g.input(35, [5, 0x81, 0x7f]) + assert.deepEqual(pointer, [ + { buttons: 0, dx: 5, dy: -3, wheel: 0, pan: 0 }, + { buttons: 1, dx: 0, dy: 0, wheel: 0, pan: 0 }, + { buttons: 5, dx: -127, dy: 127, wheel: 0, pan: 0 }, + ]) + assert.equal(events.length, 1) + // A record lost on the way: the held buttons are let go along with the held keys. + g.seq++; g.status() + assert.deepEqual(pointer.at(-1), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }) + assert.deepEqual([events.at(-1).usage, events.at(-1).down], [4, false]) + // With nothing held, another gap has nothing to say. + const count = pointer.length; g.seq++; g.status() + assert.equal(pointer.length, count) // What was skipped, and why, is in the log; so is the table it came from. assert.ok(logs.some(line => /^gatt 1=2800 2=2803 /.test(line)), logs.join('\n')) assert.ok(logs.some(line => /report 42 skipped: no Report Reference/.test(line)), logs.join('\n')) - assert.ok(logs.some(line => /report 35 \(id 4\) carries no keys/.test(line)), logs.join('\n')) + assert.ok(logs.some(line => /report 35 \(id 4\) is a pointer's/.test(line)), logs.join('\n')) assert.ok(logs.some(line => line.startsWith('report map 22: 0501'))) client.close() }) @@ -313,7 +364,7 @@ test('client: firmware errors carry their code, and a dead link rejects at once' }) function bridgeSetup(saved = null) { - const host = { keys: [], gone: 0, changes: 0, logs: [], prompts: [], saved, reconnect: true } + const host = { keys: [], pointer: [], gone: 0, changes: 0, logs: [], prompts: [], saved, reconnect: true } let bridge const g = fakeGlasses(payload => bridge.accept(payload)) bridge = new KeyboardBridge({ @@ -321,6 +372,7 @@ function bridgeSetup(saved = null) { loadSaved: () => host.saved, storeSaved: value => { host.saved = value }, onKeys: events => host.keys.push(...events), + onPointer: sample => host.pointer.push(sample), onKeyboardGone: () => { host.gone++ }, onChanged: () => { host.changes++ @@ -479,6 +531,28 @@ test('bridge: a scan started by hand takes over from an attempt in flight, and n assert.deepEqual(g.ops, []) bridge.linkLost() }) +test('bridge: a keyboard\'s touchpad is passed on while it is in use, and the snapshot says it has one', async () => { + const plain = bridgeSetup() + plain.bridge.linkReady(); await plain.bridge.connect(K380) + assert.equal(plain.bridge.snapshot().pointer, false) + plain.bridge.linkLost() + + const { bridge, g, host } = bridgeSetup() + makeCombo(g) + bridge.linkReady() + assert.equal(bridge.snapshot().pointer, false) + await bridge.connect(K380) + assert.equal(bridge.snapshot().pointer, true) + assert.ok(host.logs.some(line => /keyboard ready: K380, 3 of 3 input reports subscribed, with a pointer/.test(line)), host.logs.join('\n')) + g.input(35, [2, 1, 1]) + assert.deepEqual(host.pointer, [{ buttons: 2, dx: 1, dy: 1, wheel: 0, pan: 0 }]) + // The keyboard drops with a button down: the button is let go, as held keys + // are, before the host hears the keyboard is gone; and the snapshot follows. + g.close(); await settle() + assert.deepEqual(host.pointer.at(-1), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }); assert.equal(host.pointer.length, 2) + assert.equal(host.gone, 1); assert.equal(bridge.snapshot().pointer, false) + bridge.linkLost() +}) test('bridge: leaving on purpose tells the glasses to let the keyboard go', async (t) => { t.mock.timers.enable({ apis: ['setTimeout'] }) const { bridge, g } = bridgeSetup(K380) diff --git a/tests/keyboard-keys.test.cjs b/tests/keyboard-keys.test.cjs index b6730c1e..d6faa494 100644 --- a/tests/keyboard-keys.test.cjs +++ b/tests/keyboard-keys.test.cjs @@ -142,3 +142,24 @@ test('terminal: Shift+PageUp and Shift+PageDown page the scrollback instead', () assert.equal(page('PageUp', {}), null); assert.equal(page('PageUp', { shift: true, ctrl: true }), null) assert.equal(page('PageUp', { shift: true }, 'up'), null); assert.equal(page('ArrowUp', { shift: true }), null) }) + +test('mouse buttons are keys that neither repeat nor take the repeat from a held key; a wheel detent is a press and release', () => { + const { t, events, advance } = translator() + t.handle([down(A)]) + t.handle([{ page: 9, usage: 1, down: true }]) + assert.deepEqual([events.at(-1).key, events.at(-1).code, events.at(-1).text], ['MouseButton1', 'MouseButton1', undefined]) + // The held key goes on repeating; the held button never does. + advance(500) + assert.ok(events.filter(e => e.repeat).every(e => e.key === 'a')); assert.ok(events.some(e => e.repeat)) + t.handle([up(A), { page: 9, usage: 1, down: false }]) + assert.deepEqual([events.at(-1).phase, events.at(-1).key], ['up', 'MouseButton1']) + assert.equal(translateUsage(9, 3, false, false).code, 'MouseButton3') + + // The wheel: down and up at once, marked as droppable, with the keyboard's modifiers. + events.length = 0 + t.handle([down(SHIFT)]); t.pulse('WheelUp') + assert.deepEqual(events.slice(1).map(e => [e.phase, e.key, e.code, e.burst, e.repeat, e.modifiers.shift]), + [['down', 'WheelUp', 'WheelUp', true, false, true], ['up', 'WheelUp', 'WheelUp', true, false, true]]) + const count = events.length; advance(2000) + assert.equal(events.length, count) +}) diff --git a/tests/keyboard-pointer.test.cjs b/tests/keyboard-pointer.test.cjs new file mode 100644 index 00000000..e71a36e4 --- /dev/null +++ b/tests/keyboard-pointer.test.cjs @@ -0,0 +1,58 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const { PointerInput } = require('../.test-build/app/g2/keyboard/pointer.js') + +const sample = (buttons, extra = {}) => ({ buttons, dx: 0, dy: 0, wheel: 0, pan: 0, ...extra }) +function pointer() { + const out = [] + const input = new PointerInput({ + buttons: events => out.push(...events.map(e => `${e.down ? 'down' : 'up'}:${e.page}:${e.usage}`)), + wheel: key => out.push(key), + }) + return { input, out } +} + +test('buttons become key transitions; the wheel a key per report; movement nothing', () => { + const { input, out } = pointer() + input.handle(sample(0, { dx: 40, dy: -3 })) + assert.deepEqual(out, []) + input.handle(sample(0b001)); input.handle(sample(0b001, { dx: 2 })); input.handle(sample(0b110)); input.handle(sample(0)) + assert.deepEqual(out, ['down:9:1', 'up:9:1', 'down:9:2', 'down:9:3', 'up:9:2', 'up:9:3']) + out.length = 0 + // However many detents a report claims, it is one step; a two-finger scroll sends many. + input.handle(sample(0, { wheel: 1 })); input.handle(sample(0, { wheel: -3 })); input.handle(sample(0, { pan: 2 })); input.handle(sample(0, { pan: -1 })) + assert.deepEqual(out, ['WheelUp', 'WheelDown', 'WheelRight', 'WheelLeft']) +}) + +test('taking the pointer: samples go to the taker, held buttons are let go first, and its clicks are nobody else\'s', () => { + const { input, out } = pointer() + const taken = [] + input.handle(sample(0b001)) + const release = input.take(s => taken.push(s)) + assert.deepEqual(out, ['down:9:1', 'up:9:1']) + out.length = 0 + input.handle(sample(0b001, { dx: 5 })); input.handle(sample(0b011, { wheel: 1 })); input.handle(sample(0b010)) + assert.deepEqual(taken, [sample(0b001, { dx: 5 }), sample(0b011, { wheel: 1 }), sample(0b010)]) + assert.deepEqual(out, []) + // Given back with the right button still down: its release is not a key-up for anyone. + release() + input.handle(sample(0)); input.handle(sample(0b100)); input.handle(sample(0)) + assert.deepEqual(out, ['down:9:3', 'up:9:3']) + // A stale release function can't take the pointer away from a later taker. + const second = [] + const releaseSecond = input.take(s => second.push(s)) + release() + input.handle(sample(0, { dx: 1 })) + assert.equal(second.length, 1) + releaseSecond() +}) + +test('the keyboard going away forgets held buttons without reporting releases', () => { + const { input, out } = pointer() + input.handle(sample(0b001)) + input.reset() + assert.deepEqual(out, ['down:9:1']) + // The next keyboard starts clean: a press is a press. + input.handle(sample(0b001)) + assert.deepEqual(out, ['down:9:1', 'down:9:1']) +}) diff --git a/tests/mouse-test-page.test.cjs b/tests/mouse-test-page.test.cjs new file mode 100644 index 00000000..f1804c46 --- /dev/null +++ b/tests/mouse-test-page.test.cjs @@ -0,0 +1,136 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, modules, globals = {}) { + const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); + const context = { exports: {}, console, ...globals, require(name) { assert.ok(name in modules, name); return modules[name]; } }; + vm.runInNewContext(ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, context); + return context.exports; +} + +const graphics = require('../.test-build/app/graphics/image.js'); +const { bordered } = require('./helpers/viewport.cjs'); +const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); +const font = BdfFont.parse(fs.readFileSync(path.join(__dirname, '../app/fonts/terminus/ter-u12n.bdf'), 'utf8')); + +/** The page over a fake keyboard service, shell and clock. */ +function fixture({ state = 'connected', pointer = true } = {}) { + let now = 10_000, visible = true, renders = 0, nextTimer = 1; + const timers = new Map(); + const service = { + listener: null, takes: 0, releases: 0, + snapshot: () => ({ state, pointer }), + takePointer(listener) { + this.takes++; this.listener = listener; + return () => { if (this.listener === listener) this.listener = null; this.releases++; }; + }, + }; + const globals = { + Date: { now: () => now }, + setInterval: (fn, ms) => { timers.set(nextTimer, { fn, ms, at: now + ms, repeat: true }); return nextTimer++; }, + setTimeout: (fn, ms) => { timers.set(nextTimer, { fn, ms, at: now + ms, repeat: false }); return nextTimer++; }, + clearInterval: id => timers.delete(id), clearTimeout: id => timers.delete(id), + }; + const { MouseTestLayer } = load('app/apps/developer/mouse-test.ts', { + '../../graphics/ui-fonts': { getDefaultSmallFont: () => font }, '../../graphics/image': graphics, + '../../g2/keyboard/service': { keyboardService: service }, + '../../ui/shell/shell': { shell: { isWindowVisible: () => visible } }, + '../../util/numeric-util': { clamp: (n, lo, hi) => Math.max(lo, Math.min(hi, n)) }, + '../../ui/gestures': {}, '../../ui/layers': {}, + }, globals); + const layer = new MouseTestLayer('developer', () => { renders++; }); + const viewport = bordered(576, 260); + const popped = []; + const ctx = { stack: { getBaseSize: () => viewport, pop: () => popped.push(1) } }; + function advance(ms) { + const end = now + ms; + for (;;) { + const due = [...timers].filter(([, t]) => t.at <= end).sort((a, b) => a[1].at - b[1].at)[0]; + if (!due) break; + const [id, timer] = due; + now = timer.at; + if (timer.repeat) timer.at += timer.ms; else timers.delete(id); + timer.fn(); + } + now = end; + } + const sample = (buttons, extra = {}) => service.listener({ buttons, dx: 0, dy: 0, wheel: 0, pan: 0, ...extra }); + return { layer, service, ctx, viewport, timers, popped, advance, sample, paint: () => layer.paint(ctx), + renders: () => renders, setVisible: value => { visible = value; } }; +} +/** Whether the arrow's tip (its top-left pixel, full white) is at (x, y) and nowhere it shouldn't be. */ +const tipAt = (image, x, y) => image.getPixel(x, y) === 255 && image.getPixel(x, y + 1) === 255 && image.getPixel(x + 1, y + 2) === 70; + +test('the arrow follows the touchpad, stays on the page, and repaints at a rate the link can carry', () => { + const f = fixture(); + assert.equal(f.service.takes, 1, 'takes the pointer while the page shows'); + f.paint(); + const { width, height } = f.viewport; + // The first movement starts from the middle of the page. + f.sample(0, { dx: 10, dy: -20 }); + assert.ok(tipAt(f.paint(), width / 2 + 10, height / 2 - 20)); + // A hundred reports in a second are one repaint per 33 ms, not a hundred. + const before = f.renders(); + for (let i = 0; i < 100; i++) { f.sample(0, { dx: 1 }); f.advance(10); } + assert.ok(f.renders() - before >= 20 && f.renders() - before <= 31, String(f.renders() - before)); + assert.ok(tipAt(f.paint(), width / 2 + 110, height / 2 - 20)); + // Pushed past an edge it stops there. + f.sample(0, { dx: 5000, dy: 5000 }); + let image = f.paint(); + assert.equal(image.getPixel(width - 1, height - 1), 255); + f.sample(0, { dx: -9000, dy: -9000 }); + assert.ok(tipAt(f.paint(), 0, 0)); +}); + +test('buttons and wheel show on the page; a left click leaves a mark and a right click clears them', () => { + const f = fixture(); + f.paint(); + f.sample(0, { dx: 100, dy: 40 }); + const { width, height } = f.viewport; + const at = { x: width / 2 + 100, y: height / 2 + 40 }; + f.sample(1); + f.sample(0, { dx: -60, wheel: 2 }); + let image = f.paint(); + // The mark: a small cross where the click was, after the arrow has moved off it. + assert.equal(image.getPixel(at.x - 4, at.y), 140); assert.equal(image.getPixel(at.x, at.y + 4), 140); + assert.ok(tipAt(image, at.x - 60, at.y)); + f.sample(2); + image = f.paint(); + assert.equal(image.getPixel(at.x - 4, at.y), 0); + f.sample(0); + assert.doesNotThrow(() => f.paint()); +}); + +test('the pointer is only held while the page can be seen, and given back when the page goes', () => { + const f = fixture(); + f.paint(); + f.sample(1); + // Another window comes to the front: the buttons go back to being keys. + f.setVisible(false); f.advance(400); + assert.equal(f.service.releases, 1); assert.equal(f.service.listener, null); + f.setVisible(true); f.advance(400); + assert.equal(f.service.takes, 2); + // Double tap (or Esc, which is one) leaves; leaving gives the pointer back and stops the timers. + f.layer.handleInput({ type: 'double-click' }, f.ctx); + assert.equal(f.popped.length, 1); + f.layer.onRemoved(); + assert.equal(f.service.releases, 2); assert.equal(f.timers.size, 0); + const renders = f.renders(); + f.advance(5000); + assert.equal(f.renders(), renders); +}); + +test('without a keyboard, or with one that has no pointer, the page says so', () => { + for (const options of [{ state: 'idle' }, { state: 'connected', pointer: false }]) { + const f = fixture(options); + const image = f.paint(); + assert.ok(image instanceof graphics.GrayImage); + f.layer.onRemoved(); + } +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 09e25a29..8f812beb 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -23,6 +23,7 @@ "../app/g2/keyboard/keys.ts", "../app/g2/keyboard/bindings.ts", "../app/g2/keyboard/key-router.ts", + "../app/g2/keyboard/pointer.ts", "../app/apps/terminal/terminal-keys.ts", "../app/phone-ui/ble-bandwidth-meter.ts", "../app/apps/terminal/hub-header.ts",