Add support for a BLE mouse (ie, the touchpad/mouse buttons in a handheld mouse/keyboard combo like the Rii K28)

This commit is contained in:
James Babcock committed 2026-11-15 23:57:33 -08:00
1 parent e09f8eae96
commit ab692741f7
18 files changed
+901 -57

No files matched your search

+8
View File
@@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo";
import { AccelerometerDemoLayer } from "./accelerometer-demo";
import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark";
import { LightSensorDemoLayer } from "./light-sensor-demo";
import { MouseTestLayer } from "./mouse-test";
import { MicrophonesMonitorLayer } from "./microphones-monitor";
import { ResourceUsageLayer } from "./resource-usage";
import { RingStatusLayer } from "./ring-status";
@@ -70,6 +71,13 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
label: "Microphones",
onSelect: openMicrophones,
},
{
// The touchpad or mouse of a keyboard paired through the glasses.
label: "Mouse cursor",
onSelect: (ctx) => {
ctx.stack.push(new MouseTestLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
},
},
{
label: "BLE bandwidth",
onSelect: (ctx) => {
+206
View File
@@ -0,0 +1,206 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import type { HidPointerSample } from "../../g2/keyboard/hid";
import { keyboardService } from "../../g2/keyboard/service";
import { type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { shell } from "../../ui/shell/shell";
import { clamp } from "../../util/numeric-util";
// How often to reconcile holding the pointer with window visibility.
const RECONCILE_INTERVAL_MS = 400;
/** A touchpad reports a hundred times a second; the page repaints at most this often. */
const RENDER_INTERVAL_MS = 33;
/** Cursor pixels per count the touchpad reports. */
const POINTER_GAIN = 1;
/** Left-click marks kept on the page. */
const MAX_MARKS = 12;
/** The arrow, tip at its top left: '#' is the outline, '.' the fill. */
const CURSOR = [
"#",
"##",
"#.#",
"#..#",
"#...#",
"#....#",
"#.....#",
"#......#",
"#.......#",
"#........#",
"#.....#####",
"#..#..#",
"#.# #..#",
"## #..#",
"# #..#",
" #..#",
" ##",
];
/**
* Mouse cursor test: moves an arrow with the touchpad or mouse of the
* keyboard paired through the glasses (app/g2/keyboard/), and shows its
* buttons and wheel. The main UI has no cursor; this is where one is tried
* out.
*
* While the page is visible it takes the pointer, so the buttons and wheel
* show here instead of acting as select, back and scroll. A left click
* leaves a mark where the arrow is and a right click clears them. Leave
* with Esc, or a double tap on the ring or a temple.
*/
export class MouseTestLayer implements Layer {
private x = -1;
private y = -1;
private size = { width: 0, height: 0 };
private buttons = 0;
private wheel = 0;
private marks: Array<{ x: number; y: number }> = [];
private samples = 0;
private rate = 0;
private rateWindowStart = Date.now();
private removed = false;
private release: (() => void) | null = null;
private reconcileTimer: ReturnType<typeof setInterval> | null = null;
private renderTimer: ReturnType<typeof setTimeout> | null = null;
constructor(
private readonly windowId: string,
private readonly requestRender: () => void,
) {
this.reconcileTimer = setInterval(() => this.reconcile(), RECONCILE_INTERVAL_MS);
this.reconcile();
}
/** Hold the pointer only while the page can be seen; elsewhere its buttons go back to being keys. */
private reconcile(): void {
if (this.removed) return;
const visible = shell.isWindowVisible(this.windowId);
if (visible && !this.release) {
this.release = keyboardService.takePointer((sample) => this.onSample(sample));
} else if (!visible && this.release) {
this.release();
this.release = null;
this.buttons = 0;
}
// The rate reads zero once the pointer has been still for a second.
const now = Date.now();
if (now - this.rateWindowStart >= 1000) {
const rate = this.samples;
this.samples = 0;
this.rateWindowStart = now;
if (rate !== this.rate) {
this.rate = rate;
this.scheduleRender();
}
}
}
private onSample(sample: HidPointerSample): void {
if (this.removed) return;
this.samples++;
const { width, height } = this.size;
if (width > 0 && height > 0) {
if (this.x < 0) {
this.x = width / 2;
this.y = height / 2;
}
this.x = clamp(this.x + sample.dx * POINTER_GAIN, 0, width - 1);
this.y = clamp(this.y + sample.dy * POINTER_GAIN, 0, height - 1);
}
const pressed = sample.buttons & ~this.buttons;
this.buttons = sample.buttons;
this.wheel += sample.wheel;
if (pressed & 1 && this.x >= 0) {
this.marks.push({ x: Math.round(this.x), y: Math.round(this.y) });
if (this.marks.length > MAX_MARKS) this.marks.shift();
}
if (pressed & 2) this.marks = [];
this.scheduleRender();
}
private scheduleRender(): void {
if (this.renderTimer !== null || this.removed) return;
this.renderTimer = setTimeout(() => {
this.renderTimer = null;
if (!this.removed) this.requestRender();
}, RENDER_INTERVAL_MS);
}
onRemoved(): void {
this.removed = true;
if (this.reconcileTimer !== null) clearInterval(this.reconcileTimer);
this.reconcileTimer = null;
if (this.renderTimer !== null) clearTimeout(this.renderTimer);
this.renderTimer = null;
this.release?.();
this.release = null;
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height, padding } = ctx.stack.getBaseSize();
this.size = { width, height };
const image = new GrayImage(width, height, 0);
const left = padding.left;
const right = width - padding.right;
const top = padding.top;
const line = font.lineHeight + 4;
image.drawText(font, left, top, "Mouse cursor", 220);
const snapshot = keyboardService.snapshot();
const problem = snapshot.state !== "connected"
? "No keyboard is connected."
: !snapshot.pointer ? "This keyboard has no touchpad or mouse buttons." : null;
if (problem) {
image.drawText(font, left, top + 2 * line, problem, 190);
image.drawText(font, left, top + 3 * line, "Pair one under Settings > Keyboard.", 120);
return image;
}
const position = this.x < 0 ? "move the pointer" : `${Math.round(this.x)}, ${Math.round(this.y)}`;
image.drawText(font, right - font.measureText(position), top, position, 150);
// Buttons: a box each, filled while held.
let x = left;
const boxY = top + line;
for (const [index, label] of ["L", "M", "R"].entries()) {
// HID numbers them left, right, middle; they sit left, middle, right.
const held = !!(this.buttons & [1, 4, 2][index]!);
const boxW = font.measureText(label) + 10;
if (held) image.fillRect(x, boxY, boxW, font.lineHeight + 2, 200);
else image.drawRect(x, boxY, boxW, font.lineHeight + 2, 90);
image.drawText(font, x + 5, boxY + 1, label, held ? 1 : 170);
x += boxW + 4;
}
image.drawText(font, x + 8, boxY + 1, `wheel ${this.wheel}`, 150);
const rate = `${this.rate}/s`;
image.drawText(font, right - font.measureText(rate), boxY + 1, rate, 110);
const hint = "Left click marks, right click clears. Esc or double tap: back";
image.drawText(font, left, height - padding.bottom - font.lineHeight, hint, 90);
for (const mark of this.marks) {
image.drawLine(mark.x - 4, mark.y, mark.x + 4, mark.y, 140);
image.drawLine(mark.x, mark.y - 4, mark.x, mark.y + 4, 140);
}
if (this.x >= 0) drawCursor(image, Math.round(this.x), Math.round(this.y));
return image;
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
ctx.stack.pop();
}
}
}
function drawCursor(image: GrayImage, x: number, y: number): void {
for (const [row, pixels] of CURSOR.entries()) {
for (let col = 0; col < pixels.length; col++) {
const pixel = pixels[col];
if (pixel === " ") continue;
const px = x + col;
const py = y + row;
if (px < 0 || py < 0 || px >= image.width || py >= image.height) continue;
image.setPixel(px, py, pixel === "#" ? 255 : 70);
}
}
}
+26 -1
View File
@@ -10,10 +10,15 @@
* (a terminal session), since that is the point of binding a remote's
* few buttons.
* 2. Built-in bindings that also apply there: Escape with Shift, Ctrl or
* Alt goes back, which is the way out of a terminal session.
* Alt goes back, which is the way out of a terminal session; and a
* mouse's buttons and wheel, which are never typed.
* 3. Built-in navigation, only for keys no window took: the arrows are the
* watch's four swipes, Enter selects, Escape goes back.
*
* A keyboard's touchpad or mouse arrives here as keys too: "MouseButton1"
* (left), "MouseButton2" (right), "MouseButton3" (middle) and so on, and
* "WheelUp" / "WheelDown" / "WheelLeft" / "WheelRight" (see pointer.ts).
*
* No platform imports: this runs under node in the tests.
*/
import type { KeyEvent } from "./keys";
@@ -113,6 +118,8 @@ const CODE_LABELS: Record<string, string> = {
BrightnessUp: "Brightness Up", BrightnessDown: "Brightness Down",
BrowserSearch: "Search", BrowserHome: "Browser Home", BrowserBack: "Browser Back", BrowserForward: "Browser Forward",
LaunchMail: "Mail", LaunchApplication1: "App 1", LaunchApplication2: "App 2",
MouseButton1: "Left button", MouseButton2: "Right button", MouseButton3: "Middle button",
WheelUp: "Wheel up", WheelDown: "Wheel down", WheelLeft: "Wheel left", WheelRight: "Wheel right",
};
/** A key as a person would name it. Unknown keys keep their HID usage number, which is all there is. */
@@ -125,6 +132,8 @@ function codeLabel(code: string): string {
if (match) return match[1]!;
match = /^Numpad(.+)$/.exec(code);
if (match) return `Numpad ${match[1]}`;
match = /^MouseButton([0-9]+)$/.exec(code);
if (match) return `Mouse button ${match[1]}`;
match = /^Usage([0-9a-f]+)$/.exec(code);
if (match) return `Key 0x${match[1]}`;
match = /^Consumer([0-9a-f]+)$/.exec(code);
@@ -174,12 +183,26 @@ export function keyBindingMap(bindings: readonly KeyBinding[]): Map<string, KeyA
return new Map(bindings.map((binding) => [chordId(binding.chord), binding.action]));
}
/** What a mouse does unless bound otherwise: the ring's tap, double tap, hold and scroll. */
const MOUSE_DEFAULTS: Record<string, KeyGesture | undefined> = {
MouseButton1: "click",
MouseButton2: "double-click",
MouseButton3: "long-press",
WheelUp: "scroll-up",
WheelDown: "scroll-down",
};
/**
* The built-in meaning of a key press, or null when it has none.
* `overridesTyping`: it holds even where the focused window takes keys.
* Built-ins go by the logical key, so both Enter keys select.
*/
export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overridesTyping: boolean } | null {
// A mouse's buttons and wheel are the ring's gestures: tap, double tap,
// hold, and scroll. Nothing types them, so they hold everywhere, whatever
// is held on the keyboard.
const mouse = MOUSE_DEFAULTS[event.key];
if (mouse) return { action: mouse, overridesTyping: true };
const { shift, ctrl, alt, meta } = event.modifiers;
if (meta) return null;
if (event.key === "Escape") {
@@ -208,6 +231,8 @@ export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overrid
export const DEFAULT_KEYS_HELP =
"Arrow keys: swipe up, down, left and right, as on the watch. Enter: select. Esc: back.\n\n" +
"In a terminal session those keys are typed instead, and Shift+Esc (or Ctrl+Esc, Alt+Esc) goes back.\n\n" +
"Mouse or touchpad: the left button selects, the right button goes back, the middle button opens the system menu, " +
"and the wheel scrolls. These work in a terminal session too.\n\n" +
"A custom keybinding replaces these for its key and works everywhere, terminals included. " +
"\"Nothing\" turns a key's meaning here off without stopping it being typed.";
+9 -2
View File
@@ -12,7 +12,7 @@
* No platform imports: this runs under node in the tests.
*/
import { KeyboardClient, KeyboardClientError, type KeyboardStatus } from "./client";
import type { HidKeyEvent } from "./hid";
import type { HidKeyEvent, HidPointerSample } from "./hid";
import {
KeyboardDm,
KeyboardError,
@@ -56,6 +56,8 @@ export type KeyboardBridgeSnapshot = {
busy: boolean;
/** The remembered keyboard is away and the bridge keeps trying to bring it back. */
waiting: boolean;
/** The connected keyboard has a touchpad or mouse buttons. */
pointer: boolean;
};
export type KeyboardBridgeHost = {
@@ -65,6 +67,8 @@ export type KeyboardBridgeHost = {
storeSaved(saved: SavedKeyboard | null): void;
/** Key transitions from the keyboard (including forced releases). */
onKeys(events: HidKeyEvent[]): void;
/** A report from the keyboard's touchpad or mouse (including a forced release of its buttons). */
onPointer(sample: HidPointerSample): void;
/** The keyboard stopped being usable; drop held-key state without releases. */
onKeyboardGone(): void;
/** Something in snapshot() changed. */
@@ -129,6 +133,7 @@ export class KeyboardBridge {
scanResults: sortAdverts(this.scanResults),
busy: this.busy,
waiting: this.available && !!this.saved && !this.ready && !this.userDisconnected && !this.busy,
pointer: this.ready && this.client?.configuration?.pointer === true,
};
}
@@ -328,6 +333,7 @@ export class KeyboardBridge {
(record) => live() && this.host.send(record),
{
keys: (events) => { if (live() && this.ready) this.host.onKeys(events); },
pointer: (sample) => { if (live() && this.ready) this.host.onPointer(sample); },
advert: (advert) => { if (live()) this.addAdvert(advert); },
status: (status) => { if (live()) this.onStatus(status); },
dm: (event, status) => { if (live()) this.onDm(event, status); },
@@ -380,7 +386,8 @@ export class KeyboardBridge {
}
this.host.log(
`keyboard ready: ${keyboard.name || keyboard.address}, ${config.watched.length} of ` +
`${config.reports.filter((r) => r.type === 1).length} input reports subscribed, epoch ${config.epoch}`);
`${config.reports.filter((r) => r.type === 1).length} input reports subscribed` +
`${config.pointer ? ", with a pointer" : ""}, epoch ${config.epoch}`);
this.setState("connected");
} catch (e) {
if (!live()) return;
+65 -20
View File
@@ -10,7 +10,16 @@
* numbers found by configure() are only good for one connection epoch.
* No platform imports: this runs under node in the tests.
*/
import { HidKeys, parseReportMap, reportHasKeys, type HidKeyEvent, type HidReport } from "./hid";
import {
HidKeys,
parseReportMap,
reportHasKeys,
reportHasPointer,
reportPointer,
type HidKeyEvent,
type HidPointerSample,
type HidReport,
} from "./hid";
import {
KeyboardDm,
KeyboardError,
@@ -42,6 +51,12 @@ export type KeyboardStatus = {
export type KeyboardClientEvents = {
/** Key transitions from one input report (or the releases a gap forces). */
keys?(events: HidKeyEvent[]): void;
/**
* One report from the keyboard's touchpad or mouse: the buttons now held
* (across all its pointer reports) and the movement and wheel since the
* last one. Also sent with everything zero when a gap forces a release.
*/
pointer?(sample: HidPointerSample): void;
advert?(advert: KeyboardAdvert): void;
status?(status: KeyboardStatus): void;
/** The stream may have lost records; held keys were released. */
@@ -78,8 +93,10 @@ export type KeyboardConfiguration = {
epoch: number;
/** Every report found, input or not. */
reports: KeyboardReport[];
/** Handles of the input reports subscribed for keys. */
/** Handles of the input reports subscribed: those that carry keys, or a pointer's buttons and movement. */
watched: number[];
/** Whether any of them is a pointer's (a touchpad, mouse buttons, a wheel). */
pointer: boolean;
serviceChanged?: number;
};
@@ -109,7 +126,7 @@ type Pending = {
timer: ReturnType<typeof setTimeout>;
};
type Waiter = { test: () => boolean; resolve: () => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> };
type Input = { report: HidReport; id: number; service: number };
type Input = { report: HidReport; id: number; service: number; keys: boolean; pointer: boolean };
const COMMAND_TIMEOUT_MS = 32_000;
const MAX_PENDING = 8;
@@ -159,6 +176,8 @@ export class KeyboardClient {
private readonly pending = new Map<number, Pending>();
private readonly waiters = new Set<Waiter>();
private readonly keys = new HidKeys();
/** Pointer buttons held, by the report that says so (as HidKeys does for keys). */
private readonly pointerButtons = new Map<number, number>();
private inputs = new Map<number, Input>();
private readonly records = new KeyboardRecords(() => this.gap());
private readonly randomPasskey: () => number;
@@ -248,18 +267,35 @@ export class KeyboardClient {
this.gap();
return;
}
try {
const events = this.keys.update(r.handle, input.report, r.data);
if (events.length) this.events.keys?.(events);
} catch (e) {
this.release();
this.events.error?.(messageOf(e));
if (input.keys) {
try {
const events = this.keys.update(r.handle, input.report, r.data);
if (events.length) this.events.keys?.(events);
} catch (e) {
this.release();
this.events.error?.(messageOf(e));
}
}
if (input.pointer) {
const sample = reportPointer(input.report, r.data);
this.pointerButtons.set(r.handle, sample.buttons);
this.events.pointer?.({ ...sample, buttons: this.heldButtons() });
}
}
private heldButtons(): number {
let buttons = 0;
for (const held of this.pointerButtons.values()) buttons |= held;
return buttons;
}
/** Nothing is held any more, as far as anyone can tell: say so. */
private release(): void {
const events = this.keys.reset();
if (events.length) this.events.keys?.(events);
const buttons = this.heldButtons();
this.pointerButtons.clear();
if (buttons) this.events.pointer?.({ buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 });
}
private gap(): void {
@@ -524,13 +560,13 @@ export class KeyboardClient {
/**
* Discover every HID service, its Report Map and each report's Report
* Reference; select Report Protocol and subscribe the input reports that
* can carry keys. Needs an encrypted link. No handle is assumed.
* carry keys or a pointer's buttons and movement. Needs an encrypted link.
* No handle is assumed.
*
* Only what typing needs is required of the keyboard: a Report Map that
* parses, and at least one key-bearing input report with a Report
* Reference and a CCC. A report that can't be placed is skipped and
* logged, as is anything wrong with an external report (the battery
* level, usually).
* Only what input needs is required of the device: a Report Map that
* parses, and at least one such input report with a Report Reference and
* a CCC. A report that can't be placed is skipped and logged, as is
* anything wrong with an external report (the battery level, usually).
*/
async configure(): Promise<KeyboardConfiguration> {
const epoch = this.epoch;
@@ -543,6 +579,7 @@ export class KeyboardClient {
log(`gatt ${attrs.map((a) => `${a.handle}=${typeof a.uuid === "number" ? a.uuid.toString(16) : a.uuid}`).join(" ")}`);
const reports: KeyboardReport[] = [];
const watched: KeyboardReport[] = [];
let pointerFound = false;
const skipped: string[] = [];
const descriptorsFor = (handle: number): Attribute[] => {
const end = attrs.find((a) => a.handle > handle &&
@@ -576,8 +613,14 @@ export class KeyboardClient {
const layout = parsed.get(id);
if (!layout) return skip(a, `input report id ${id} is not in the Report Map`);
if (ccc === undefined || (properties !== undefined && !(properties & 0x10))) return skip(a, "input report can't notify");
if (reportHasKeys(layout)) watched.push(report);
else log(`report ${a.handle} (id ${id}) carries no keys; not subscribed`);
const keys = reportHasKeys(layout);
const pointer = reportHasPointer(layout);
if (keys || pointer) watched.push(report);
else log(`report ${a.handle} (id ${id}) carries no keys or pointer; not subscribed`);
if (pointer) {
pointerFound = true;
log(`report ${a.handle} (id ${id}) is a pointer's${keys ? ", with keys" : ""}`);
}
}
reports.push(report);
log(`report ${a.handle}: id ${id} type ${type} ccc ${ccc ?? "-"} properties ${properties ?? "?"}`);
@@ -625,7 +668,7 @@ export class KeyboardClient {
if (this.closed || this.epoch !== epoch) throw new KeyboardClientError("The keyboard link changed during setup");
if (!watched.length) {
throw new KeyboardClientError(reports.length || skipped.length
? `No usable keyboard input report (${skipped.join("; ") || "none carries keys"})`
? `No usable keyboard input report (${skipped.join("; ") || "none carries keys or a pointer"})`
: "This device has no HID service");
}
if (watched.length > MAX_WATCHES) throw new KeyboardClientError("Keyboard has more than 16 input reports");
@@ -634,7 +677,9 @@ export class KeyboardClient {
const watches = watched.map((r) => r.handle);
if (changed && changedCcc && watches.length < MAX_WATCHES) watches.push(changed.handle);
const serviceChanged = changed && changedCcc && watches.includes(changed.handle) ? changed.handle : undefined;
const config: KeyboardConfiguration = { epoch: this.epoch, reports, watched: watched.map((r) => r.handle), serviceChanged };
const config: KeyboardConfiguration = {
epoch: this.epoch, reports, watched: watched.map((r) => r.handle), pointer: pointerFound, serviceChanged,
};
this.restore(config);
try {
await this.command(KeyboardOp.watch, [watches.length, ...watches.flatMap(le16)]);
@@ -671,7 +716,7 @@ export class KeyboardClient {
if (!r.map.length || r.map.length > MAX_VALUE) throw new KeyboardClientError("Invalid restored Report Map");
const report = parseReportMap(r.map).get(r.id);
if (!report) throw new KeyboardClientError("Restored Report ID missing");
inputs.set(r.handle, { report, id: r.id, service: r.service });
inputs.set(r.handle, { report, id: r.id, service: r.service, keys: reportHasKeys(report), pointer: reportHasPointer(report) });
}
if (!inputs.size || inputs.size > MAX_WATCHES) throw new KeyboardClientError("Invalid restored input count");
this.release();
+92 -12
View File
@@ -6,8 +6,9 @@
*
* HID over GATT carries the Report ID out of band (in each characteristic's
* Report Reference descriptor), so byte zero of a report value is data: it is
* never stripped as USB HID would. Only the keyboard/keypad page (7) and the
* consumer page (12) produce keys; pointer axes and padding are ignored.
* never stripped as USB HID would. The keyboard/keypad page (7) and the
* consumer page (12) produce keys. A mouse's or touchpad's report (buttons,
* relative X/Y, wheel) is read by reportPointer.
* No platform imports: this runs under node in the tests.
*/
@@ -20,6 +21,8 @@ type Field = {
size: number;
count: number;
variable: boolean;
/** Reports a change since the last report (a mouse's axes and wheel), not a state. */
relative: boolean;
min: number;
max: number;
usages: HidUsage[];
@@ -90,7 +93,8 @@ export function parseReportMap(bytes: Uint8Array): Map<number, HidReport> {
reports.set(g.id, r);
}
const field: Field = {
bit: r.bits, size: g.size, count: g.count, variable: !!(v & 2), min: g.min, max: g.max, usages: [...usages],
bit: r.bits, size: g.size, count: g.count, variable: !!(v & 2), relative: !!(v & 4),
min: g.min, max: g.max, usages: [...usages],
};
if (lo || hi) {
if (!lo || !hi || lo.page !== hi.page || hi.usage < lo.usage) fail("usage range");
@@ -98,8 +102,8 @@ export function parseReportMap(bytes: Uint8Array): Map<number, HidReport> {
}
r.bits += g.size * g.count;
if (r.bits > 4096) fail("report exceeds 512 bytes");
// Constant padding and relative pointer axes are not keys.
if (!(v & 1) && !(v & 4)) r.fields.push(field);
// Constant fields are padding.
if (!(v & 1)) r.fields.push(field);
} else if (tag !== 9 && tag !== 11) fail("unknown main item");
usages = [];
lo = hi = undefined;
@@ -130,7 +134,86 @@ function usageAt(f: Field, index: number): HidUsage | undefined {
*/
export function reportHasKeys(report: HidReport): boolean {
const isKeyPage = (u: HidUsage): boolean => u.page === HID_PAGE_KEYBOARD || u.page === HID_PAGE_CONSUMER;
return report.fields.some((f) => f.usages.some(isKeyPage) || (f.range !== undefined && isKeyPage(f.range[0])));
return report.fields.some((f) => !f.relative && (f.usages.some(isKeyPage) || (f.range !== undefined && isKeyPage(f.range[0]))));
}
/** One field's `index`th value, sign-extended when the field is signed. Bits past the data read as zero. */
function fieldValue(f: Field, index: number, data: Uint8Array): number {
let value = 0;
for (let j = 0; j < f.size; j++) {
const bit = f.bit + index * f.size + j;
if ((data[bit >>> 3] ?? 0) & (1 << (bit & 7))) value += 2 ** j;
}
if (f.min < 0 && value >= 2 ** (f.size - 1)) value -= 2 ** f.size;
return value;
}
/** What one report from a mouse or touchpad says. */
export type HidPointerSample = {
/** Buttons held, bit 0 for button 1 (left), bit 1 for button 2 (right), bit 2 for button 3 (middle). */
buttons: number;
/** Movement since the last report, in the device's counts; +x right, +y down. */
dx: number;
dy: number;
/** Wheel detents since the last report; positive is away from the user ("up"). */
wheel: number;
/** Horizontal wheel (AC Pan); positive is right. */
pan: number;
};
const HID_PAGE_GENERIC_DESKTOP = 1;
const HID_PAGE_BUTTON = 9;
const USAGE_X = 0x30;
const USAGE_Y = 0x31;
const USAGE_WHEEL = 0x38;
const USAGE_AC_PAN = 0x238;
/** Buttons past this are not tracked; a mouse with more is rare and has nothing to bind them to anyway. */
const MAX_POINTER_BUTTONS = 8;
/** Which part of a pointer sample a field's usage feeds, if any. */
function pointerRole(f: Field, u: HidUsage | undefined): "button" | "dx" | "dy" | "wheel" | "pan" | null {
if (!u || !f.variable) return null;
if (u.page === HID_PAGE_BUTTON) return !f.relative && u.usage >= 1 && u.usage <= MAX_POINTER_BUTTONS ? "button" : null;
if (!f.relative) return null; // an absolute X/Y is a touchscreen or a digitizer, not a mouse
if (u.page === HID_PAGE_GENERIC_DESKTOP) {
if (u.usage === USAGE_X) return "dx";
if (u.usage === USAGE_Y) return "dy";
if (u.usage === USAGE_WHEEL) return "wheel";
}
if (u.page === HID_PAGE_CONSUMER && u.usage === USAGE_AC_PAN) return "pan";
return null;
}
/** Whether an input report carries a mouse's buttons, movement or wheel. */
export function reportHasPointer(report: HidReport): boolean {
return report.fields.some((f) => {
for (let i = 0; i < f.count; i++) if (pointerRole(f, usageAt(f, i))) return true;
return false;
});
}
/**
* Read a mouse or touchpad report. Lenient about length, unlike reportKeys:
* some devices leave the wheel bytes off when there is nothing to say, and a
* missing byte is zero either way.
*/
export function reportPointer(report: HidReport, data: Uint8Array): HidPointerSample {
const sample: HidPointerSample = { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 };
for (const f of report.fields) {
for (let i = 0; i < f.count; i++) {
const u = usageAt(f, i);
const role = pointerRole(f, u);
if (!role) continue;
const value = fieldValue(f, i, data);
if (value < f.min || value > f.max) continue; // out of range: no reading
if (role === "button") {
if (value) sample.buttons |= 1 << (u!.usage - 1);
} else {
sample[role] += value;
}
}
}
return sample;
}
/** The keys one input report says are held. Throws on a length mismatch or a rollover report. */
@@ -138,13 +221,10 @@ export function reportKeys(report: HidReport, data: Uint8Array): HidUsage[] {
if (data.length !== Math.ceil(report.bits / 8)) throw new Error("HID input length does not match Report Map");
const keys = new Map<string, HidUsage>();
for (const f of report.fields) {
// A relative field is a movement, not a key that is down.
if (f.relative) continue;
for (let i = 0; i < f.count; i++) {
let value = 0;
for (let j = 0; j < f.size; j++) {
const bit = f.bit + i * f.size + j;
if (data[bit >>> 3]! & (1 << (bit & 7))) value += 2 ** j;
}
if (f.min < 0 && value >= 2 ** (f.size - 1)) value -= 2 ** f.size;
const value = fieldValue(f, i, data);
if (value < f.min || value > f.max) continue; // null state
const u = usageAt(f, f.variable ? i : value - f.min);
if (!u || (f.variable && value === 0) || !u.usage) continue;
+10 -8
View File
@@ -38,9 +38,10 @@ export type KeyRouterHost = {
};
/**
* A held key repeats twenty times a second, faster than a list can repaint
* over the glasses' link. Repeats are dropped while a gesture is still being
* handled, and closer together than this.
* A held key repeats twenty times a second, and a wheel or a two-finger
* scroll reports faster still: more than a list can repaint over the
* glasses' link. Those are dropped while a gesture is still being handled,
* and closer together than this.
*/
const REPEAT_MIN_INTERVAL_MS = 100;
@@ -73,13 +74,14 @@ export class KeyRouter {
return;
}
const resolution = resolveKey(event, host.bindings());
const droppable = event.repeat || event.burst === true;
if (resolution.consume) {
this.consumed.add(event.code);
if (resolution.gesture) this.perform(resolution.gesture, event.repeat);
if (resolution.gesture) this.perform(resolution.gesture, droppable);
return;
}
if (host.offerToWindow(event)) return;
if (resolution.gesture) this.perform(resolution.gesture, event.repeat);
if (resolution.gesture) this.perform(resolution.gesture, droppable);
}
/**
@@ -91,7 +93,7 @@ export class KeyRouter {
const host = this.host;
if (event.phase !== "down" || this.capture || !host.ready() || host.locked() || !host.screenOn()) return;
const resolution = resolveKey(event, host.bindings());
if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat);
if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat || event.burst === true);
}
/**
@@ -124,9 +126,9 @@ export class KeyRouter {
capture(chordOf(event));
}
private perform(gesture: KeyGesture, repeat: boolean): void {
private perform(gesture: KeyGesture, droppable: boolean): void {
const now = this.host.now ? this.host.now() : Date.now();
if (repeat && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return;
if (droppable && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return;
this.lastGestureAtMs = now;
void this.run(gesture, this.scheme.originFor(gesture));
}
+23 -2
View File
@@ -24,12 +24,19 @@ export type KeyEvent = {
modifiers: KeyModifiers;
/** A synthesized auto-repeat of a held key. */
repeat: boolean;
/**
* One of a stream that can arrive faster than it can be acted on (a wheel
* turning): whoever acts on it may drop some rather than fall behind.
*/
burst?: boolean;
/** Date.now() when the phone saw the transition. */
timestampMs: number;
};
const HID_PAGE_KEYBOARD = 7;
const HID_PAGE_CONSUMER = 12;
/** A mouse's buttons, which arrive here as keys: usage 1 is the left button, 2 the right, 3 the middle. */
export const HID_PAGE_BUTTON = 9;
const USAGE_CAPS_LOCK = 57;
/** [unshifted, shifted, code] for US-layout punctuation on the keyboard page. */
@@ -159,6 +166,7 @@ export function translateUsage(page: number, usage: number, shift: boolean, caps
const name = CONSUMER[usage];
return name ? { key: name, code: name } : { key: "Unidentified", code: `Consumer${usage.toString(16)}` };
}
if (page === HID_PAGE_BUTTON) return { key: `MouseButton${usage}`, code: `MouseButton${usage}` };
return { key: "Unidentified", code: `Page${page.toString(16)}Usage${usage.toString(16)}` };
}
@@ -217,8 +225,9 @@ export class KeyTranslator {
if (modifier) this.modifiers[modifier] = true;
if (event.page === HID_PAGE_KEYBOARD && event.usage === USAGE_CAPS_LOCK) this.capsLock = !this.capsLock;
this.emit("down", event.page, event.usage, false);
// The newest non-modifier key is the one that repeats, as on a PC.
if (!modifier) this.startRepeat(event.page, event.usage);
// The newest non-modifier key is the one that repeats, as on a PC. A
// mouse button neither repeats nor takes the repeat from a held key.
if (!modifier && event.page !== HID_PAGE_BUTTON) this.startRepeat(event.page, event.usage);
} else {
if (!this.held.delete(id)) return;
if (modifier) this.modifiers[modifier] = this.otherHeld(event.usage, modifier);
@@ -227,6 +236,18 @@ export class KeyTranslator {
}
}
/**
* A press and release in one, for input that has no "held" (a wheel's
* detent): `name` is both the key and the code, e.g. "WheelUp". Carries
* the modifiers held on the keyboard, like any key.
*/
pulse(name: string): void {
const timestampMs = this.now();
for (const phase of ["down", "up"] as const) {
this.emitEvent({ phase, key: name, code: name, modifiers: { ...this.modifiers }, repeat: false, burst: true, timestampMs });
}
}
/** The keyboard went away: forget everything without emitting releases. */
reset(): void {
this.stopRepeat();
+83
View File
@@ -0,0 +1,83 @@
/**
* A keyboard's touchpad or mouse, as the rest of the keyboard support sees
* it. Its buttons and wheel become keys ("MouseButton1", "WheelUp"), so
* they get default meanings and custom keybindings like any key
* (bindings.ts). Its movement goes nowhere unless something has taken the
* pointer (the cursor test page): there is no cursor in the main UI.
* No platform imports: this runs under node in the tests.
*/
import type { HidKeyEvent, HidPointerSample } from "./hid";
import { HID_PAGE_BUTTON } from "./keys";
/** The names a wheel's detents arrive under, as keys. */
export type WheelKey = "WheelUp" | "WheelDown" | "WheelLeft" | "WheelRight";
export type PointerInputHost = {
/** Button transitions, as HID usages on the Button page. */
buttons(events: HidKeyEvent[]): void;
/** One wheel report in a direction, however many detents it claimed. */
wheel(key: WheelKey): void;
};
export class PointerInput {
/** The buttons the host has been told are down. */
private reported = 0;
/** The buttons actually down, as of the last sample. */
private held = 0;
/** Buttons that went down while the pointer was taken: not keys until they have come up. */
private ignored = 0;
private grab: ((sample: HidPointerSample) => void) | null = null;
constructor(private readonly host: PointerInputHost) {}
/** One report from the pointer (see KeyboardClientEvents.pointer). */
handle(sample: HidPointerSample): void {
this.held = sample.buttons;
if (this.grab) {
this.grab(sample);
return;
}
this.ignored &= sample.buttons;
this.report(sample.buttons & ~this.ignored);
// One step per report: a touchpad's two-finger scroll sends many small
// ones, and the gesture they turn into is rate-limited downstream.
if (sample.wheel) this.host.wheel(sample.wheel > 0 ? "WheelUp" : "WheelDown");
if (sample.pan) this.host.wheel(sample.pan > 0 ? "WheelRight" : "WheelLeft");
}
/**
* Take the pointer: every sample goes to `listener`, movement included,
* and its buttons and wheel stop acting as keys. Buttons held at that
* moment are released first, so nothing is left down. Returns a function
* that gives the pointer back.
*/
take(listener: (sample: HidPointerSample) => void): () => void {
this.report(0);
this.grab = listener;
return () => {
if (this.grab !== listener) return;
this.grab = null;
// A button still down was pressed for the listener, not as a key: its
// release is nobody's business either.
this.ignored = this.held;
};
}
/** The keyboard went away: nothing is held, and nobody is told. */
reset(): void {
this.reported = 0;
this.held = 0;
this.ignored = 0;
}
private report(buttons: number): void {
const changed = buttons ^ this.reported;
if (!changed) return;
const events: HidKeyEvent[] = [];
for (let bit = 0; bit < 8; bit++) {
if (changed & (1 << bit)) events.push({ page: HID_PAGE_BUTTON, usage: bit + 1, down: !!(buttons & (1 << bit)) });
}
this.reported = buttons;
this.host.buttons(events);
}
}
+19
View File
@@ -23,8 +23,10 @@ import {
type KeyGesture,
} from "./bindings";
import { KeyboardBridge, type KeyboardBridgeSnapshot, type SavedKeyboard } from "./bridge";
import type { HidPointerSample } from "./hid";
import { KeyRouter } from "./key-router";
import { KeyTranslator, type KeyEvent } from "./keys";
import { PointerInput } from "./pointer";
declare const java: any;
declare const arc4random_uniform: ((bound: number) => number) | undefined;
@@ -92,6 +94,11 @@ class KeyboardService {
log: (line) => console.log(`[keyboard] ${line}`),
});
private readonly translator = new KeyTranslator({ emit: (event) => this.router.handleKey(event) });
/** The touchpad's buttons and wheel go in as keys; see pointer.ts. */
private readonly pointer = new PointerInput({
buttons: (events) => this.translator.handle(events),
wheel: (key) => this.translator.pulse(key),
});
readonly bridge = new KeyboardBridge({
send: (record) => {
@@ -106,8 +113,10 @@ class KeyboardService {
loadSaved: loadSavedKeyboard,
storeSaved: (saved) => setStringSetting(SAVED_KEYBOARD_KEY, saved ? JSON.stringify(saved) : ""),
onKeys: (events) => this.translator.handle(events),
onPointer: (sample) => this.pointer.handle(sample),
onKeyboardGone: () => {
this.translator.reset();
this.pointer.reset();
this.router.reset();
},
onChanged: () => {
@@ -206,6 +215,16 @@ class KeyboardService {
return this.router.captureNextKey(onChord);
}
/**
* Take the keyboard's touchpad or mouse: every report goes to `listener`,
* movement included, and its buttons and wheel stop acting as keys until
* the returned function is called. For the cursor test page; the main UI
* has no cursor.
*/
takePointer(listener: (sample: HidPointerSample) => void): () => void {
return this.pointer.take(listener);
}
private storeBindings(bindings: KeyBinding[]): void {
this.bindings = bindings;
this.bindingMap = keyBindingMap(bindings);
+2 -2
View File
@@ -27,7 +27,7 @@ class KeyCaptureLayer extends TextPageLayer {
constructor(private readonly onChord: (chord: KeyChord, ctx: LayerContext) => void) {
super(
"Add keybinding",
"Press the key to bind, on the keyboard. Hold Shift, Ctrl or Alt with it to bind the combination.",
"Press the key to bind, on the keyboard: a key, a mouse button, or a turn of the wheel. Hold Shift, Ctrl or Alt with it to bind the combination.",
);
}
@@ -137,7 +137,7 @@ export function keybindingsMenuItem(): MenuItem {
return {
label: "Keybindings",
description:
"Choose what keys do: arrows, Enter and Esc navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.",
"Choose what keys do: arrows, Enter, Esc and a mouse's buttons and wheel navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.",
onSelect: (ctx) => new KeybindingsLayer(ctx.stack.getBaseSize()).open(ctx),
render: ({ image, x, y, width }) => {
const count = keyboardService.keyBindings().length;
@@ -2001,13 +2001,13 @@ class GlassesSessionCore(
tap(data.copyOf())
return
}
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
val frame = BleProtocol.parseFrame(data)
if (frame.ok && frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW keyboard-bridge records (field 131) come from whichever temple hosts the
// keyboard. They are their own stream: neither an ACK nor glasses input, and the
// once-a-second status record among them must not count as user activity (which
// would hold off battery polling).
// would hold off battery polling). A touchpad sends a hundred of them a second
// while a finger is on it, so they are not logged one by one either.
val keyboardEvent = BleProtocol.parseFaceclawKeyboardEvent(frame.pb)
if (keyboardEvent != null) {
monitor.withLock { lastIncomingAtMs = now() }
@@ -2015,6 +2015,7 @@ class GlassesSessionCore(
return
}
}
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
val decodedWearState = BleProtocol.parseWearState(frame)
val compassEvent = if (address.equals(rightAddress, ignoreCase = true))
BleProtocol.parseCompassEvent(frame)
+57
View File
@@ -243,3 +243,60 @@ test('router: capturing a key for a new keybinding takes the chord and nothing e
r.handleKey(key('x', { code: 'KeyX' }))
assert.deepEqual(captured.at(-1), chord('KeyX'))
})
test('a mouse by default: left taps, right double taps, middle holds, the wheel is the ring\'s scroll', () => {
const action = (k, overrides) => defaultKeyAction(key(k, overrides))
// Nothing types a mouse button, so these hold in a terminal session too.
assert.deepEqual(action('MouseButton1'), { action: 'click', overridesTyping: true })
assert.deepEqual(action('MouseButton2'), { action: 'double-click', overridesTyping: true })
assert.deepEqual(action('MouseButton3'), { action: 'long-press', overridesTyping: true })
assert.deepEqual(action('WheelUp'), { action: 'scroll-up', overridesTyping: true })
assert.deepEqual(action('WheelDown'), { action: 'scroll-down', overridesTyping: true })
// Whatever is held on the keyboard.
assert.deepEqual(action('MouseButton1', { modifiers: { ctrl: true, shift: true } }), { action: 'click', overridesTyping: true })
// Extra buttons and the horizontal wheel mean nothing until bound.
for (const k of ['MouseButton4', 'MouseButton5', 'WheelLeft', 'WheelRight']) assert.equal(action(k), null)
const said = code => describeChord(chord(code))
assert.equal(said('MouseButton1'), 'Left button'); assert.equal(said('MouseButton2'), 'Right button')
assert.equal(said('MouseButton3'), 'Middle button'); assert.equal(said('MouseButton4'), 'Mouse button 4')
assert.equal(said('WheelUp'), 'Wheel up'); assert.equal(describeChord(chord('WheelDown', { ctrl: true })), 'Ctrl+Wheel down')
assert.deepEqual(parseChordId('Shift+MouseButton2'), chord('MouseButton2', { shift: true }))
})
test('router: the mouse works where keys are being typed, its wheel is paced, and its buttons can be rebound', async () => {
const { r, state } = router({ bindings: [{ chord: chord('MouseButton4'), action: 'short-then-long-press' }, { chord: chord('MouseButton2'), action: 'none' }] })
const wheel = k => { r.handleKey(key(k, { burst: true })); r.handleKey(up(k, { burst: true })) }
state.windowTakesKeys = true
r.handleKey(key('MouseButton1')); r.handleKey(up('MouseButton1'))
r.handleKey(key('MouseButton3')); r.handleKey(up('MouseButton3'))
await flush(); state.now += 500
wheel('WheelDown')
// The wheel is the ring's scroll, and what follows it is the ring's too.
await flush(); state.now += 500
r.handleKey(key('MouseButton1')); r.handleKey(up('MouseButton1'))
assert.deepEqual(state.gestures, ['click/watch', 'long-press/watch', 'scroll-down/ring', 'click/ring'])
assert.deepEqual(state.offered, [], 'never typed')
// A custom keybinding for an extra button; "Nothing" on the right button leaves it inert.
state.gestures.length = 0
r.handleKey(key('MouseButton4')); r.handleKey(up('MouseButton4')); r.handleKey(key('MouseButton2')); r.handleKey(up('MouseButton2'))
assert.deepEqual(state.gestures, ['short-then-long-press/ring'])
// A two-finger scroll reports far faster than the display can follow: every 100 ms at most.
state.gestures.length = 0; state.offered.length = 0
for (let i = 0; i < 40; i++) { await flush(); state.now += 25; wheel('WheelUp') }
assert.equal(state.gestures.length, 10); assert.ok(state.gestures.every(g => g === 'scroll-up/ring'))
// On a dark display a click wakes it and does nothing else.
state.screenOn = false; state.gestures.length = 0
r.handleKey(key('MouseButton1')); r.handleKey(up('MouseButton1'))
assert.equal(state.wakes, 1); assert.deepEqual(state.gestures, [])
})
test('router: a mouse button or a turn of the wheel can be captured for a keybinding', () => {
const { r, state } = router()
const captured = []
r.captureNextKey(c => captured.push(c))
r.handleKey(key('MouseButton2')); r.handleKey(up('MouseButton2'))
r.captureNextKey(c => captured.push(c))
r.handleKey(key('WheelUp', { burst: true, modifiers: { shift: true } })); r.handleKey(up('WheelUp', { burst: true }))
assert.deepEqual(captured, [chord('MouseButton2'), chord('WheelUp', { shift: true })])
assert.deepEqual(state.gestures, [])
})
+82 -8
View File
@@ -1,6 +1,6 @@
const test = require('node:test')
const assert = require('node:assert/strict')
const { HidKeys, parseReportMap, reportKeys } = require('../.test-build/app/g2/keyboard/hid.js')
const { HidKeys, parseReportMap, reportHasKeys, reportHasPointer, reportKeys, reportPointer } = require('../.test-build/app/g2/keyboard/hid.js')
const {
KeyboardRecords, keyboardControl, le16, u16, parseKeyboardAdvert, mergeKeyboardAdverts,
parseKeyboardAddress, formatKeyboardAddress,
@@ -69,6 +69,41 @@ test('descriptor resource bounds and malformed items', () => {
}
})
// A wheel mouse as most are described: three buttons, 8-bit X/Y and wheel.
const mouseMap = Uint8Array.from([0x05,1,0x09,2,0xa1,1,0x09,1,0xa1,0,0x05,9,0x19,1,0x29,3,0x15,0,0x25,1,0x75,1,0x95,3,0x81,2,
0x75,5,0x95,1,0x81,1,0x05,1,0x09,0x30,0x09,0x31,0x09,0x38,0x15,0x81,0x25,0x7f,0x75,8,0x95,3,0x81,6,0xc0,0xc0])
test('a mouse report: buttons, movement and wheel; not keys', () => {
const r = parseReportMap(mouseMap).get(0)
assert.equal(reportHasPointer(r), true); assert.equal(reportHasKeys(r), false)
assert.deepEqual(reportPointer(r, Uint8Array.of(0, 0, 0, 0)), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 })
assert.deepEqual(reportPointer(r, Uint8Array.of(0b101, 10, 0xf6, 1)), { buttons: 5, dx: 10, dy: -10, wheel: 1, pan: 0 })
assert.deepEqual(reportPointer(r, Uint8Array.of(0b010, 0, 0, 0xff)), { buttons: 2, dx: 0, dy: 0, wheel: -1, pan: 0 })
// Some devices leave trailing bytes off; what is missing is zero. Extra bytes are ignored.
assert.deepEqual(reportPointer(r, Uint8Array.of(1, 3)), { buttons: 1, dx: 3, dy: 0, wheel: 0, pan: 0 })
assert.deepEqual(reportPointer(r, Uint8Array.of(0, 1, 1, 1, 9, 9)), { buttons: 0, dx: 1, dy: 1, wheel: 1, pan: 0 })
// -128 is outside the declared range: no reading, rather than a jump.
assert.deepEqual(reportPointer(r, Uint8Array.of(0, 0x80, 0, 0)), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 })
assert.deepEqual(reportKeys(r, Uint8Array.of(7, 1, 1, 1)), [])
// A keyboard's report is not a pointer's.
assert.equal(reportHasPointer(parseReportMap(boot).get(0)), false)
})
test('a touchpad with 12-bit axes and a horizontal wheel; an absolute digitizer is not a mouse', () => {
// Five buttons, 3 bits of padding, X/Y packed 12 bits each, wheel, then AC Pan from the consumer page.
const map = Uint8Array.from([0x05,1,0x09,2,0xa1,1,0x05,9,0x19,1,0x29,5,0x15,0,0x25,1,0x75,1,0x95,5,0x81,2,
0x75,3,0x95,1,0x81,1,0x05,1,0x09,0x30,0x09,0x31,0x16,0x01,0xf8,0x26,0xff,0x07,0x75,12,0x95,2,0x81,6,
0x09,0x38,0x15,0x81,0x25,0x7f,0x75,8,0x95,1,0x81,6,0x05,0x0c,0x0a,0x38,0x02,0x81,6,0xc0])
const r = parseReportMap(map).get(0)
assert.equal(reportHasPointer(r), true)
// X = 0x123 (291), Y = 0xfff (-1): packed little endian, 12 bits each.
assert.deepEqual(reportPointer(r, Uint8Array.of(0b10000, 0x23, 0xf1, 0xff, 2, 0xfe)), { buttons: 16, dx: 291, dy: -1, wheel: 2, pan: -2 })
// The pan field is on the consumer page, but it is a movement, not a key going down.
assert.equal(reportHasKeys(r), false); assert.deepEqual(reportKeys(r, Uint8Array.of(0, 0, 0, 0, 0, 5)), [])
// Absolute X/Y (a touchscreen, a pen) is something else, and is left alone.
const absolute = parseReportMap(Uint8Array.of(0x05,1,0x09,0x30,0x09,0x31,0x15,0,0x26,0xff,0x0f,0x75,16,0x95,2,0x81,2)).get(0)
assert.equal(reportHasPointer(absolute), false)
assert.deepEqual(reportPointer(absolute, Uint8Array.of(1, 2, 3, 4)), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 })
})
test('control records, UUIDs and addresses', () => {
assert.deepEqual([...keyboardControl(4,0x1234,0x5678,[1,2,3,4,5,6,7])], [75,66,1,4,0x34,0x12,0x78,0x56,1,2,3,4,5,6,7])
assert.throws(() => keyboardControl(13,1,1))
@@ -238,23 +273,39 @@ function makeCombo(g) {
g.attrs.clear(); for (const [handle, attr] of sorted) g.attrs.set(handle, attr)
}
for (const declarations of ['short', 'refused']) {
test(`client: a keyboard with a trackpad whose stack answers declaration reads ${declarations} still types`, async () => {
const events = [], logs = []
test(`client: a keyboard with a trackpad whose stack answers declaration reads ${declarations} types and points`, async () => {
const events = [], pointer = [], logs = []
let client
const g = fakeGlasses(payload => client.accept(payload))
makeCombo(g); g.declarations = declarations
client = new KeyboardClient(g.send, { keys: e => events.push(...e), log: line => logs.push(line) })
client = new KeyboardClient(g.send, { keys: e => events.push(...e), pointer: p => pointer.push(p), log: line => logs.push(line) })
await client.enable(); await client.connect(1, new Uint8Array(6)); const config = await client.configure()
// Keys and media keys are subscribed; the pointer and the battery are not.
assert.deepEqual(config.watched, [27,31]); assert.deepEqual(g.watches, [27,31,3]); assert.deepEqual(g.writes, [25,4,28,32])
// Keys, media keys and the trackpad are subscribed; the battery level is not.
assert.deepEqual(config.watched, [27,31,35]); assert.equal(config.pointer, true)
assert.deepEqual(g.watches, [27,31,35,3]); assert.deepEqual(g.writes, [25,4,28,32,36])
assert.deepEqual(config.reports.map(r => [r.handle, r.id]), [[27,1],[31,2],[35,4]])
assert.deepEqual(config.reports.map(r => r.properties), declarations === 'short' ? [0x12,0x12,0x12] : [undefined,undefined,undefined])
g.input(27, [0,0,4,0,0,0,0,0])
assert.deepEqual(events.map(e => [e.usage,e.down]), [[4,true]])
// The trackpad: movement and buttons, and never a key.
g.input(35, [0, 5, 0xfd]); g.input(35, [1, 0, 0]); g.input(35, [5, 0x81, 0x7f])
assert.deepEqual(pointer, [
{ buttons: 0, dx: 5, dy: -3, wheel: 0, pan: 0 },
{ buttons: 1, dx: 0, dy: 0, wheel: 0, pan: 0 },
{ buttons: 5, dx: -127, dy: 127, wheel: 0, pan: 0 },
])
assert.equal(events.length, 1)
// A record lost on the way: the held buttons are let go along with the held keys.
g.seq++; g.status()
assert.deepEqual(pointer.at(-1), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 })
assert.deepEqual([events.at(-1).usage, events.at(-1).down], [4, false])
// With nothing held, another gap has nothing to say.
const count = pointer.length; g.seq++; g.status()
assert.equal(pointer.length, count)
// What was skipped, and why, is in the log; so is the table it came from.
assert.ok(logs.some(line => /^gatt 1=2800 2=2803 /.test(line)), logs.join('\n'))
assert.ok(logs.some(line => /report 42 skipped: no Report Reference/.test(line)), logs.join('\n'))
assert.ok(logs.some(line => /report 35 \(id 4\) carries no keys/.test(line)), logs.join('\n'))
assert.ok(logs.some(line => /report 35 \(id 4\) is a pointer's/.test(line)), logs.join('\n'))
assert.ok(logs.some(line => line.startsWith('report map 22: 0501')))
client.close()
})
@@ -313,7 +364,7 @@ test('client: firmware errors carry their code, and a dead link rejects at once'
})
function bridgeSetup(saved = null) {
const host = { keys: [], gone: 0, changes: 0, logs: [], prompts: [], saved, reconnect: true }
const host = { keys: [], pointer: [], gone: 0, changes: 0, logs: [], prompts: [], saved, reconnect: true }
let bridge
const g = fakeGlasses(payload => bridge.accept(payload))
bridge = new KeyboardBridge({
@@ -321,6 +372,7 @@ function bridgeSetup(saved = null) {
loadSaved: () => host.saved,
storeSaved: value => { host.saved = value },
onKeys: events => host.keys.push(...events),
onPointer: sample => host.pointer.push(sample),
onKeyboardGone: () => { host.gone++ },
onChanged: () => {
host.changes++
@@ -479,6 +531,28 @@ test('bridge: a scan started by hand takes over from an attempt in flight, and n
assert.deepEqual(g.ops, [])
bridge.linkLost()
})
test('bridge: a keyboard\'s touchpad is passed on while it is in use, and the snapshot says it has one', async () => {
const plain = bridgeSetup()
plain.bridge.linkReady(); await plain.bridge.connect(K380)
assert.equal(plain.bridge.snapshot().pointer, false)
plain.bridge.linkLost()
const { bridge, g, host } = bridgeSetup()
makeCombo(g)
bridge.linkReady()
assert.equal(bridge.snapshot().pointer, false)
await bridge.connect(K380)
assert.equal(bridge.snapshot().pointer, true)
assert.ok(host.logs.some(line => /keyboard ready: K380, 3 of 3 input reports subscribed, with a pointer/.test(line)), host.logs.join('\n'))
g.input(35, [2, 1, 1])
assert.deepEqual(host.pointer, [{ buttons: 2, dx: 1, dy: 1, wheel: 0, pan: 0 }])
// The keyboard drops with a button down: the button is let go, as held keys
// are, before the host hears the keyboard is gone; and the snapshot follows.
g.close(); await settle()
assert.deepEqual(host.pointer.at(-1), { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 }); assert.equal(host.pointer.length, 2)
assert.equal(host.gone, 1); assert.equal(bridge.snapshot().pointer, false)
bridge.linkLost()
})
test('bridge: leaving on purpose tells the glasses to let the keyboard go', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] })
const { bridge, g } = bridgeSetup(K380)
+21
View File
@@ -142,3 +142,24 @@ test('terminal: Shift+PageUp and Shift+PageDown page the scrollback instead', ()
assert.equal(page('PageUp', {}), null); assert.equal(page('PageUp', { shift: true, ctrl: true }), null)
assert.equal(page('PageUp', { shift: true }, 'up'), null); assert.equal(page('ArrowUp', { shift: true }), null)
})
test('mouse buttons are keys that neither repeat nor take the repeat from a held key; a wheel detent is a press and release', () => {
const { t, events, advance } = translator()
t.handle([down(A)])
t.handle([{ page: 9, usage: 1, down: true }])
assert.deepEqual([events.at(-1).key, events.at(-1).code, events.at(-1).text], ['MouseButton1', 'MouseButton1', undefined])
// The held key goes on repeating; the held button never does.
advance(500)
assert.ok(events.filter(e => e.repeat).every(e => e.key === 'a')); assert.ok(events.some(e => e.repeat))
t.handle([up(A), { page: 9, usage: 1, down: false }])
assert.deepEqual([events.at(-1).phase, events.at(-1).key], ['up', 'MouseButton1'])
assert.equal(translateUsage(9, 3, false, false).code, 'MouseButton3')
// The wheel: down and up at once, marked as droppable, with the keyboard's modifiers.
events.length = 0
t.handle([down(SHIFT)]); t.pulse('WheelUp')
assert.deepEqual(events.slice(1).map(e => [e.phase, e.key, e.code, e.burst, e.repeat, e.modifiers.shift]),
[['down', 'WheelUp', 'WheelUp', true, false, true], ['up', 'WheelUp', 'WheelUp', true, false, true]])
const count = events.length; advance(2000)
assert.equal(events.length, count)
})
+58
View File
@@ -0,0 +1,58 @@
const test = require('node:test')
const assert = require('node:assert/strict')
const { PointerInput } = require('../.test-build/app/g2/keyboard/pointer.js')
const sample = (buttons, extra = {}) => ({ buttons, dx: 0, dy: 0, wheel: 0, pan: 0, ...extra })
function pointer() {
const out = []
const input = new PointerInput({
buttons: events => out.push(...events.map(e => `${e.down ? 'down' : 'up'}:${e.page}:${e.usage}`)),
wheel: key => out.push(key),
})
return { input, out }
}
test('buttons become key transitions; the wheel a key per report; movement nothing', () => {
const { input, out } = pointer()
input.handle(sample(0, { dx: 40, dy: -3 }))
assert.deepEqual(out, [])
input.handle(sample(0b001)); input.handle(sample(0b001, { dx: 2 })); input.handle(sample(0b110)); input.handle(sample(0))
assert.deepEqual(out, ['down:9:1', 'up:9:1', 'down:9:2', 'down:9:3', 'up:9:2', 'up:9:3'])
out.length = 0
// However many detents a report claims, it is one step; a two-finger scroll sends many.
input.handle(sample(0, { wheel: 1 })); input.handle(sample(0, { wheel: -3 })); input.handle(sample(0, { pan: 2 })); input.handle(sample(0, { pan: -1 }))
assert.deepEqual(out, ['WheelUp', 'WheelDown', 'WheelRight', 'WheelLeft'])
})
test('taking the pointer: samples go to the taker, held buttons are let go first, and its clicks are nobody else\'s', () => {
const { input, out } = pointer()
const taken = []
input.handle(sample(0b001))
const release = input.take(s => taken.push(s))
assert.deepEqual(out, ['down:9:1', 'up:9:1'])
out.length = 0
input.handle(sample(0b001, { dx: 5 })); input.handle(sample(0b011, { wheel: 1 })); input.handle(sample(0b010))
assert.deepEqual(taken, [sample(0b001, { dx: 5 }), sample(0b011, { wheel: 1 }), sample(0b010)])
assert.deepEqual(out, [])
// Given back with the right button still down: its release is not a key-up for anyone.
release()
input.handle(sample(0)); input.handle(sample(0b100)); input.handle(sample(0))
assert.deepEqual(out, ['down:9:3', 'up:9:3'])
// A stale release function can't take the pointer away from a later taker.
const second = []
const releaseSecond = input.take(s => second.push(s))
release()
input.handle(sample(0, { dx: 1 }))
assert.equal(second.length, 1)
releaseSecond()
})
test('the keyboard going away forgets held buttons without reporting releases', () => {
const { input, out } = pointer()
input.handle(sample(0b001))
input.reset()
assert.deepEqual(out, ['down:9:1'])
// The next keyboard starts clean: a press is a press.
input.handle(sample(0b001))
assert.deepEqual(out, ['down:9:1', 'down:9:1'])
})
+136
View File
@@ -0,0 +1,136 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
function load(file, modules, globals = {}) {
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
const context = { exports: {}, console, ...globals, require(name) { assert.ok(name in modules, name); return modules[name]; } };
vm.runInNewContext(ts.transpileModule(source, {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText, context);
return context.exports;
}
const graphics = require('../.test-build/app/graphics/image.js');
const { bordered } = require('./helpers/viewport.cjs');
const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} });
const font = BdfFont.parse(fs.readFileSync(path.join(__dirname, '../app/fonts/terminus/ter-u12n.bdf'), 'utf8'));
/** The page over a fake keyboard service, shell and clock. */
function fixture({ state = 'connected', pointer = true } = {}) {
let now = 10_000, visible = true, renders = 0, nextTimer = 1;
const timers = new Map();
const service = {
listener: null, takes: 0, releases: 0,
snapshot: () => ({ state, pointer }),
takePointer(listener) {
this.takes++; this.listener = listener;
return () => { if (this.listener === listener) this.listener = null; this.releases++; };
},
};
const globals = {
Date: { now: () => now },
setInterval: (fn, ms) => { timers.set(nextTimer, { fn, ms, at: now + ms, repeat: true }); return nextTimer++; },
setTimeout: (fn, ms) => { timers.set(nextTimer, { fn, ms, at: now + ms, repeat: false }); return nextTimer++; },
clearInterval: id => timers.delete(id), clearTimeout: id => timers.delete(id),
};
const { MouseTestLayer } = load('app/apps/developer/mouse-test.ts', {
'../../graphics/ui-fonts': { getDefaultSmallFont: () => font }, '../../graphics/image': graphics,
'../../g2/keyboard/service': { keyboardService: service },
'../../ui/shell/shell': { shell: { isWindowVisible: () => visible } },
'../../util/numeric-util': { clamp: (n, lo, hi) => Math.max(lo, Math.min(hi, n)) },
'../../ui/gestures': {}, '../../ui/layers': {},
}, globals);
const layer = new MouseTestLayer('developer', () => { renders++; });
const viewport = bordered(576, 260);
const popped = [];
const ctx = { stack: { getBaseSize: () => viewport, pop: () => popped.push(1) } };
function advance(ms) {
const end = now + ms;
for (;;) {
const due = [...timers].filter(([, t]) => t.at <= end).sort((a, b) => a[1].at - b[1].at)[0];
if (!due) break;
const [id, timer] = due;
now = timer.at;
if (timer.repeat) timer.at += timer.ms; else timers.delete(id);
timer.fn();
}
now = end;
}
const sample = (buttons, extra = {}) => service.listener({ buttons, dx: 0, dy: 0, wheel: 0, pan: 0, ...extra });
return { layer, service, ctx, viewport, timers, popped, advance, sample, paint: () => layer.paint(ctx),
renders: () => renders, setVisible: value => { visible = value; } };
}
/** Whether the arrow's tip (its top-left pixel, full white) is at (x, y) and nowhere it shouldn't be. */
const tipAt = (image, x, y) => image.getPixel(x, y) === 255 && image.getPixel(x, y + 1) === 255 && image.getPixel(x + 1, y + 2) === 70;
test('the arrow follows the touchpad, stays on the page, and repaints at a rate the link can carry', () => {
const f = fixture();
assert.equal(f.service.takes, 1, 'takes the pointer while the page shows');
f.paint();
const { width, height } = f.viewport;
// The first movement starts from the middle of the page.
f.sample(0, { dx: 10, dy: -20 });
assert.ok(tipAt(f.paint(), width / 2 + 10, height / 2 - 20));
// A hundred reports in a second are one repaint per 33 ms, not a hundred.
const before = f.renders();
for (let i = 0; i < 100; i++) { f.sample(0, { dx: 1 }); f.advance(10); }
assert.ok(f.renders() - before >= 20 && f.renders() - before <= 31, String(f.renders() - before));
assert.ok(tipAt(f.paint(), width / 2 + 110, height / 2 - 20));
// Pushed past an edge it stops there.
f.sample(0, { dx: 5000, dy: 5000 });
let image = f.paint();
assert.equal(image.getPixel(width - 1, height - 1), 255);
f.sample(0, { dx: -9000, dy: -9000 });
assert.ok(tipAt(f.paint(), 0, 0));
});
test('buttons and wheel show on the page; a left click leaves a mark and a right click clears them', () => {
const f = fixture();
f.paint();
f.sample(0, { dx: 100, dy: 40 });
const { width, height } = f.viewport;
const at = { x: width / 2 + 100, y: height / 2 + 40 };
f.sample(1);
f.sample(0, { dx: -60, wheel: 2 });
let image = f.paint();
// The mark: a small cross where the click was, after the arrow has moved off it.
assert.equal(image.getPixel(at.x - 4, at.y), 140); assert.equal(image.getPixel(at.x, at.y + 4), 140);
assert.ok(tipAt(image, at.x - 60, at.y));
f.sample(2);
image = f.paint();
assert.equal(image.getPixel(at.x - 4, at.y), 0);
f.sample(0);
assert.doesNotThrow(() => f.paint());
});
test('the pointer is only held while the page can be seen, and given back when the page goes', () => {
const f = fixture();
f.paint();
f.sample(1);
// Another window comes to the front: the buttons go back to being keys.
f.setVisible(false); f.advance(400);
assert.equal(f.service.releases, 1); assert.equal(f.service.listener, null);
f.setVisible(true); f.advance(400);
assert.equal(f.service.takes, 2);
// Double tap (or Esc, which is one) leaves; leaving gives the pointer back and stops the timers.
f.layer.handleInput({ type: 'double-click' }, f.ctx);
assert.equal(f.popped.length, 1);
f.layer.onRemoved();
assert.equal(f.service.releases, 2); assert.equal(f.timers.size, 0);
const renders = f.renders();
f.advance(5000);
assert.equal(f.renders(), renders);
});
test('without a keyboard, or with one that has no pointer, the page says so', () => {
for (const options of [{ state: 'idle' }, { state: 'connected', pointer: false }]) {
const f = fixture(options);
const image = f.paint();
assert.ok(image instanceof graphics.GrayImage);
f.layer.onRemoved();
}
});
+1
View File
@@ -23,6 +23,7 @@
"../app/g2/keyboard/keys.ts",
"../app/g2/keyboard/bindings.ts",
"../app/g2/keyboard/key-router.ts",
"../app/g2/keyboard/pointer.ts",
"../app/apps/terminal/terminal-keys.ts",
"../app/phone-ui/ble-bandwidth-meter.ts",
"../app/apps/terminal/hub-header.ts",