forked from Blendman974/faceclaw
Add bluetooth keyboard support
G2 only: You can pair the glasses with a bluetooth keyboard. It will relay key events, and you can use it to type in some apps and to navigate the UI.
This commit is contained in:
1 parent
9372b2742c
commit
fd3c26333f
46 files changed
+3867
-41
No files matched your search
@@ -103,6 +103,7 @@ import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from ".
|
||||
import { uiFontPickerMenuItem } from "../font-picker";
|
||||
import { NotificationFilterLayer } from "../notification-filter";
|
||||
import { compatibleModelDiscoveryMenuItem } from "../openai-compatible-model-picker";
|
||||
import { keyboardSettingsMenuItem } from "../keyboard-settings";
|
||||
import { isVeniceApiBaseUrl } from "../../native/openai-chat";
|
||||
|
||||
/** The Settings section the Notifications app's "Settings" menu item opens. */
|
||||
@@ -242,6 +243,14 @@ function settingsSections(): SettingsSection[] {
|
||||
],
|
||||
renderDetail: renderWatchStatus,
|
||||
},
|
||||
// A Bluetooth keyboard paired with the glasses themselves (the G2's
|
||||
// custom firmware hosts it; app/g2/keyboard/). Hidden rather than left
|
||||
// out: the Settings window can be open across a change of glasses.
|
||||
{
|
||||
label: "Keyboard",
|
||||
items: [keyboardSettingsMenuItem()],
|
||||
hidden: () => glassesTypeSetting.get() !== "g2",
|
||||
},
|
||||
{
|
||||
label: "Developer",
|
||||
items: [
|
||||
|
||||
@@ -28,6 +28,13 @@ export type SettingsSection = {
|
||||
height: number;
|
||||
ctx: LayerContext;
|
||||
}) => number | void;
|
||||
/**
|
||||
* Leave the section out of the list while this returns true. Re-read on
|
||||
* every paint and input, for a section that only applies with some other
|
||||
* setting (the paired glasses' type): the Settings window outlives changes
|
||||
* to those.
|
||||
*/
|
||||
hidden?: () => boolean;
|
||||
};
|
||||
|
||||
const LEFT_W = 150;
|
||||
@@ -66,7 +73,7 @@ export class SettingsPanelLayer implements Layer {
|
||||
|
||||
constructor(private readonly sections: SettingsSection[]) {
|
||||
this.leftMenu = new Menu<SettingsSection>({
|
||||
items: sections,
|
||||
items: sections.filter((section) => !section.hidden?.()),
|
||||
wrap: true,
|
||||
rowGap: ROW_GAP,
|
||||
highlight: { radius: 6 },
|
||||
@@ -108,9 +115,28 @@ export class SettingsPanelLayer implements Layer {
|
||||
return this.leftMenu.selectedItem ?? this.sections[0]!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring the left column in line with which sections are hidden now. The
|
||||
* highlight stays on its section; if that one just went away it moves to
|
||||
* the list's start, with focus back in the left column.
|
||||
*/
|
||||
private syncSections(): void {
|
||||
const visible = this.sections.filter((section) => !section.hidden?.());
|
||||
const shown = this.leftMenu.items;
|
||||
if (visible.length === shown.length && visible.every((section, index) => section === shown[index])) return;
|
||||
const selected = this.leftMenu.selectedItem;
|
||||
const index = selected ? visible.indexOf(selected) : -1;
|
||||
this.leftMenu.setItems(visible, Math.max(0, index));
|
||||
if (index < 0) {
|
||||
this.focus = "left";
|
||||
this.resetRight();
|
||||
}
|
||||
}
|
||||
|
||||
/** Select a left-column section by label (deep link, e.g. from an app's menu). */
|
||||
focusSection(label: string): void {
|
||||
const index = this.sections.findIndex((section) => section.label === label);
|
||||
this.syncSections();
|
||||
const index = this.leftMenu.items.findIndex((section) => section.label === label);
|
||||
if (index < 0) return;
|
||||
this.leftMenu.select(index);
|
||||
this.focus = "left";
|
||||
@@ -123,6 +149,7 @@ export class SettingsPanelLayer implements Layer {
|
||||
const { width, height, padding } = viewport;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const appFocused = ctx.stack.isFocused();
|
||||
this.syncSections();
|
||||
const section = this.section();
|
||||
const rightItems = section.items;
|
||||
this.rightMenu.setItems(rightItems);
|
||||
@@ -228,6 +255,7 @@ export class SettingsPanelLayer implements Layer {
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
this.syncSections();
|
||||
const scroll = scrollEventFor(event);
|
||||
if (this.focus === "left") {
|
||||
if (scroll) {
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
import { getDefaultSmallFont } from "../graphics/ui-fonts";
|
||||
import type { KeyboardBridgeSnapshot } from "../g2/keyboard/bridge";
|
||||
import type { KeyboardAdvert } from "../g2/keyboard/protocol";
|
||||
import { keyboardService } from "../g2/keyboard/service";
|
||||
import { ConfigSettingString, textSettingMenuItem } from "./dashboard-settings";
|
||||
import type { LayerContext } from "./layers";
|
||||
import { drawRightValueMenuItem, MenuLayer, type MenuItem } from "./menu";
|
||||
|
||||
/**
|
||||
* Settings > Keyboard: pairing a Bluetooth keyboard with the glasses (the
|
||||
* CFW keyboard bridge, app/g2/keyboard/). The keyboard pairs with the
|
||||
* glasses, not the phone, so all of it happens here, on the glasses: scan,
|
||||
* pick, and for a keyboard that wants one, the passkey to type on it.
|
||||
*/
|
||||
|
||||
/** The code a keyboard with a display shows, typed on the phone. Scratch, not a preference. */
|
||||
const pairingCodeSetting = new ConfigSettingString({
|
||||
id: "keyboardPairingCode",
|
||||
storageKey: "keyboard.pairingCode",
|
||||
label: "Code on keyboard",
|
||||
defaultValue: "",
|
||||
normalize: (value) => (value ?? "").replace(/\D/g, "").slice(0, 6),
|
||||
});
|
||||
|
||||
const sixDigits = (value: number): string => {
|
||||
const text = String(value).padStart(6, "0");
|
||||
return `${text.slice(0, 3)} ${text.slice(3)}`;
|
||||
};
|
||||
|
||||
function advertLabel(advert: KeyboardAdvert): string {
|
||||
const name = advert.name.trim() || advert.address;
|
||||
return name.length > 30 ? `${name.slice(0, 27)}...` : name;
|
||||
}
|
||||
|
||||
/** One line saying where the keyboard stands, for the Settings row and the menu's first row. */
|
||||
export function keyboardStatusText(snapshot: KeyboardBridgeSnapshot): string {
|
||||
const name = snapshot.activeName || snapshot.saved?.name || "keyboard";
|
||||
switch (snapshot.state) {
|
||||
case "unavailable":
|
||||
return snapshot.saved ? "Glasses not connected" : "Not set up";
|
||||
case "scanning":
|
||||
return "Scanning...";
|
||||
case "connecting":
|
||||
return `Connecting to ${name}...`;
|
||||
case "pairing":
|
||||
return `Pairing with ${name}...`;
|
||||
case "configuring":
|
||||
return `Setting up ${name}...`;
|
||||
case "connected":
|
||||
return `Connected: ${name}`;
|
||||
case "error":
|
||||
return snapshot.error || "Failed";
|
||||
default:
|
||||
if (snapshot.waiting) return `Waiting for ${name}`;
|
||||
return snapshot.saved ? `${name} (disconnected)` : "Not set up";
|
||||
}
|
||||
}
|
||||
|
||||
/** The short form for the right edge of the Settings row. */
|
||||
function keyboardStatusBrief(snapshot: KeyboardBridgeSnapshot): string {
|
||||
switch (snapshot.state) {
|
||||
case "connected":
|
||||
return "Connected";
|
||||
case "scanning":
|
||||
return "Scanning";
|
||||
case "connecting":
|
||||
case "pairing":
|
||||
case "configuring":
|
||||
return "Connecting";
|
||||
case "error":
|
||||
return "Failed";
|
||||
case "unavailable":
|
||||
return snapshot.saved ? "Offline" : "Off";
|
||||
default:
|
||||
return snapshot.waiting ? "Waiting" : snapshot.saved ? "Disconnected" : "Off";
|
||||
}
|
||||
}
|
||||
|
||||
const info = (label: string): MenuItem => ({ label, disabled: true, onSelect: () => {} });
|
||||
|
||||
class KeyboardSettingsLayer extends MenuLayer {
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private showOthers = false;
|
||||
/** What kind of rows are showing; a change moves the highlight to the first row that does something. */
|
||||
private shape = "";
|
||||
|
||||
constructor(viewport: { width: number; height: number }) {
|
||||
super("Bluetooth keyboard", [], {
|
||||
x: "center",
|
||||
y: 8,
|
||||
width: Math.min(320, viewport.width - 40),
|
||||
minHeight: Math.min(160, viewport.height - 40),
|
||||
maxHeight: Math.min(240, viewport.height - 40),
|
||||
depth: 4,
|
||||
});
|
||||
}
|
||||
|
||||
open(ctx: LayerContext): void {
|
||||
ctx.stack.push(this);
|
||||
this.updateRows();
|
||||
this.unsubscribe = keyboardService.subscribe(() => {
|
||||
this.updateRows();
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
/** Run a bridge command; its failure shows up in the snapshot, which repaints the rows. */
|
||||
private run(action: () => Promise<unknown>): void {
|
||||
void action().catch(() => {});
|
||||
}
|
||||
|
||||
private updateRows(): void {
|
||||
const snapshot = keyboardService.snapshot();
|
||||
const bridge = keyboardService.bridge;
|
||||
const rows: MenuItem[] = [info(keyboardStatusText(snapshot))];
|
||||
let shape: string = snapshot.state;
|
||||
|
||||
const scanRow = (label: string): MenuItem => ({
|
||||
label,
|
||||
disabled: snapshot.busy,
|
||||
onSelect: () => {
|
||||
this.showOthers = false;
|
||||
this.run(() => bridge.scan());
|
||||
},
|
||||
});
|
||||
|
||||
if (snapshot.state === "unavailable") {
|
||||
rows.push(info("Connect G2 glasses to pair one."));
|
||||
if (snapshot.saved) rows.push({ label: `Forget ${snapshot.saved.name || "keyboard"}`, onSelect: () => this.run(() => bridge.forget()) });
|
||||
} else if (snapshot.pairing) {
|
||||
const pairing = snapshot.pairing;
|
||||
shape = `pairing:${pairing.kind}`;
|
||||
if (pairing.kind === "display") {
|
||||
rows.push(info(`Type ${sixDigits(pairing.pin)} on the keyboard,`), info("then press Enter."));
|
||||
} else if (pairing.kind === "compare") {
|
||||
rows.push(
|
||||
info(`Does the keyboard show ${sixDigits(pairing.value)}?`),
|
||||
{ label: "Yes, pair", onSelect: () => this.run(() => bridge.answerCompare(true)) },
|
||||
{ label: "No", onSelect: () => this.run(() => bridge.answerCompare(false)) },
|
||||
);
|
||||
} else {
|
||||
rows.push(
|
||||
info("Enter the code the keyboard shows."),
|
||||
textSettingMenuItem(pairingCodeSetting),
|
||||
{
|
||||
label: "Pair with this code",
|
||||
disabled: () => pairingCodeSetting.get().length !== 6,
|
||||
onSelect: () => {
|
||||
const code = pairingCodeSetting.get();
|
||||
pairingCodeSetting.set("");
|
||||
this.run(() => bridge.answerPasskey(parseInt(code, 10)));
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
rows.push({ label: "Cancel pairing", onSelect: () => this.run(() => bridge.disconnect()) });
|
||||
} else if (snapshot.state === "connecting" || snapshot.state === "pairing" || snapshot.state === "configuring") {
|
||||
rows.push({ label: "Cancel", onSelect: () => this.run(() => bridge.disconnect()) });
|
||||
} else if (snapshot.state === "scanning" || (snapshot.scanResults.length > 0 && snapshot.state !== "connected")) {
|
||||
shape = "results";
|
||||
const hid = snapshot.scanResults.filter((advert) => advert.hid);
|
||||
const others = snapshot.scanResults.filter((advert) => !advert.hid);
|
||||
const pick = (advert: KeyboardAdvert): MenuItem => ({
|
||||
label: advertLabel(advert),
|
||||
disabled: snapshot.busy,
|
||||
onSelect: () => {
|
||||
this.run(() => bridge.connect({ address: advert.address, addressType: advert.addressType, name: advert.name.trim() || advert.address }));
|
||||
},
|
||||
});
|
||||
rows.push(...hid.map(pick));
|
||||
if (!hid.length && snapshot.state !== "scanning") rows.push(info("No keyboards found."), info("Put it in pairing mode."));
|
||||
if (others.length && this.showOthers) rows.push(...others.map(pick));
|
||||
else if (others.length) {
|
||||
rows.push({
|
||||
label: `Show ${others.length} other device${others.length === 1 ? "" : "s"}`,
|
||||
onSelect: (ctx) => {
|
||||
this.showOthers = true;
|
||||
this.updateRows();
|
||||
ctx.actions.requestRender();
|
||||
},
|
||||
});
|
||||
}
|
||||
if (snapshot.state !== "scanning") rows.push(scanRow("Scan again"));
|
||||
if (snapshot.saved && snapshot.state !== "scanning") {
|
||||
rows.push({ label: `Reconnect ${snapshot.saved.name || "keyboard"}`, disabled: snapshot.busy, onSelect: () => this.run(() => bridge.reconnect()) });
|
||||
}
|
||||
} else if (snapshot.state === "connected") {
|
||||
rows.push(
|
||||
{ label: "Disconnect", onSelect: () => this.run(() => bridge.disconnect()) },
|
||||
{ label: "Forget this keyboard", onSelect: () => this.run(() => bridge.forget()) },
|
||||
scanRow("Pair a different keyboard"),
|
||||
);
|
||||
} else if (snapshot.saved) {
|
||||
shape = snapshot.waiting ? "waiting" : "saved";
|
||||
rows.push({ label: "Reconnect now", disabled: snapshot.busy, onSelect: () => this.run(() => bridge.reconnect()) });
|
||||
if (snapshot.waiting) rows.push({ label: "Stop reconnecting", onSelect: () => this.run(() => bridge.disconnect()) });
|
||||
rows.push(
|
||||
{ label: "Forget this keyboard", onSelect: () => this.run(() => bridge.forget()) },
|
||||
scanRow("Pair a different keyboard"),
|
||||
);
|
||||
} else {
|
||||
shape = "empty";
|
||||
rows.push(info("Put the keyboard in pairing mode,"), info("then scan."), scanRow("Scan for keyboards"));
|
||||
}
|
||||
|
||||
const firstAction = rows.findIndex((row) => row.disabled !== true);
|
||||
if (shape !== this.shape) {
|
||||
this.shape = shape;
|
||||
this.setItems(rows, Math.max(0, firstAction));
|
||||
} else {
|
||||
this.setItems(rows);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The Settings row that opens the keyboard menu. */
|
||||
export function keyboardSettingsMenuItem(): MenuItem {
|
||||
return {
|
||||
label: "Bluetooth keyboard",
|
||||
description:
|
||||
"Pair a Bluetooth keyboard with the glasses. It types into Terminal sessions, and its arrow keys, Enter and Escape work everywhere as scroll, select and back.",
|
||||
onSelect: (ctx) => new KeyboardSettingsLayer(ctx.stack.getBaseSize()).open(ctx),
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "Bluetooth keyboard", keyboardStatusBrief(keyboardService.snapshot()));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
isWatchInput,
|
||||
makeInputEvent,
|
||||
} from "../gestures";
|
||||
import type { KeyEvent } from "../../g2/keyboard/keys";
|
||||
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
|
||||
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
|
||||
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
|
||||
@@ -192,6 +193,14 @@ export type ShellWindow = {
|
||||
* only windows that consume typed text (the terminal) implement it.
|
||||
*/
|
||||
receiveTextInput?: (text: string, options?: { submit?: boolean }) => void;
|
||||
/**
|
||||
* Take a key from a hardware keyboard (app/g2/keyboard) as a key. Optional:
|
||||
* only windows that give keys a meaning of their own (the terminal)
|
||||
* implement it, and they get every key while they have input focus,
|
||||
* presses and releases alike. For the rest the controller turns the few
|
||||
* navigation keys into ring gestures (navigationGestureForKey).
|
||||
*/
|
||||
handleKeyEvent?: (event: KeyEvent) => void;
|
||||
/** Foreground state changed: this window's surface is (not) the visible one. */
|
||||
setForeground?: (foreground: boolean) => void;
|
||||
/**
|
||||
@@ -1606,6 +1615,21 @@ class Shell {
|
||||
this.foregroundWindow()?.receiveTextInput?.(text, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Offer a hardware-keyboard key to the window that has input focus. True
|
||||
* when that window takes keys (ShellWindow.handleKeyEvent) and got this
|
||||
* one; false when the key is the caller's to interpret: the screen is off,
|
||||
* the sidebar or a shell overlay has focus, or the window has no use for
|
||||
* keys. Typing counts as activity for the screen timeout.
|
||||
*/
|
||||
receiveKeyEvent(event: KeyEvent): boolean {
|
||||
const window = this.inputTargetWindow();
|
||||
if (!window?.handleKeyEvent) return false;
|
||||
if (event.phase === "down") this.lastInputAtMs = Date.now();
|
||||
window.handleKeyEvent(event);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the voice dialog aimed at the foreground window. Called when the
|
||||
* user picks Voice input from the system menu (or an app asks via a
|
||||
|
||||
@@ -28,6 +28,8 @@ export type WorkerAppMessage =
|
||||
| { type: "close-window"; windowId: string }
|
||||
| { type: "input"; windowId: string; event: unknown; frameId: number; focused: boolean }
|
||||
| { type: "text-input"; windowId: string; text: string; submit?: boolean }
|
||||
/** A hardware-keyboard key (a KeyEvent), for windows opened with acceptsKeys. */
|
||||
| { type: "key"; windowId: string; event: unknown; focused: boolean }
|
||||
| { type: "render"; windowId: string; focused: boolean }
|
||||
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
|
||||
/**
|
||||
@@ -81,6 +83,8 @@ export type WorkerAppReply =
|
||||
focus?: boolean;
|
||||
/** Window height: the standard 288px band ("min", default) or full screen ("max"). */
|
||||
heightMode?: WindowHeightMode;
|
||||
/** Send the window hardware-keyboard keys (see WorkerWindowSpec). */
|
||||
acceptsKeys?: boolean;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
@@ -196,6 +200,13 @@ export type WorkerWindowSpec = {
|
||||
* of all four directions in every window state (see ShellWindow).
|
||||
*/
|
||||
acceptsDirectional?: boolean;
|
||||
/**
|
||||
* Deliver hardware-keyboard keys to the worker as "key" messages while the
|
||||
* window has input focus, instead of the navigation-gesture fallback. The
|
||||
* worker then owns what every key means in every window state, navigation
|
||||
* included (see ShellWindow.handleKeyEvent).
|
||||
*/
|
||||
acceptsKeys?: boolean;
|
||||
};
|
||||
|
||||
export type WorkerAppHostOptions = {
|
||||
@@ -335,6 +346,7 @@ export class WorkerAppHost {
|
||||
iconGlyph: message.iconGlyph,
|
||||
focus: message.focus,
|
||||
heightMode: message.heightMode,
|
||||
acceptsKeys: message.acceptsKeys,
|
||||
});
|
||||
break;
|
||||
case "set-height-mode":
|
||||
@@ -522,6 +534,13 @@ export class WorkerAppHost {
|
||||
receiveTextInput: (text, options) => {
|
||||
this.post({ type: "text-input", windowId: spec.windowId, text, ...(options?.submit === undefined ? {} : { submit: options.submit }) });
|
||||
},
|
||||
...(spec.acceptsKeys
|
||||
? {
|
||||
handleKeyEvent: (event) => {
|
||||
this.post({ type: "key", windowId: spec.windowId, event, focused: shell.isWindowFocused(spec.windowId) });
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
setForeground: (foreground) => {
|
||||
this.options.setSurfaceVisible(surfaceId, foreground);
|
||||
if (foreground) {
|
||||
|
||||
Reference in new issue
Block a user