Add keyboard-input button to the phone home screen

This commit is contained in:
James Babcock committed 2026-09-03 16:06:46 -07:00
1 parent 00dd546b1c
commit aef6194687
12 files changed
+574 -85

No files matched your search

+90
View File
@@ -0,0 +1,90 @@
import { G2_LENS_WIDTH, type GrayImage } from "../../graphics/image";
import { wrapText, truncateText } from "../../graphics/textwrap";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { drawSelectionHighlight } from "../menu";
import { listRowHeight } from "../metrics";
import { MIN_WINDOW_HEIGHT, minWindowTop } from "./geometry";
/**
* The shared look of the shell's text-entry dialogs (voice input and phone
* keyboard input): a solid box over whatever is on screen with a title line,
* a status line, the message so far, and either a menu of destinations at
* the bottom or a gesture hint.
*/
const DIALOG_X = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
// The dialog fits inside the min-height window band (like the other shell
// overlays), wherever the vertical position setting puts it.
const DIALOG_MARGIN_Y = 24;
const DIALOG_H = MIN_WINDOW_HEIGHT - 2 * DIALOG_MARGIN_Y;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
/** Dialog top edge; band-relative, so computed per paint. */
function dialogY(): number {
return minWindowTop() + DIALOG_MARGIN_Y;
}
export type InputDialogRow = {
label: string;
/** Drawn faint: the row is not currently selectable (e.g. nothing to send). */
dim: boolean;
};
export type InputDialogContent = {
title: string;
status: string;
/** The message body (a placeholder when nothing has been captured yet). */
text: string;
/** Menu rows along the bottom edge; empty for no menu. */
rows: readonly InputDialogRow[];
selectedRow: number;
/** Gesture hint drawn along the bottom edge when there is no menu. */
hint?: string;
};
/** Paint the dialog onto `image` (an already-painted canvas of the layers below). */
export function paintInputDialog(image: GrayImage, content: InputDialogContent): void {
const font = getDefaultSmallFont();
const menuRowH = listRowHeight(font);
const top = dialogY();
// Solid dialog box over the underlying UI. Fill 1, not 0: identical after
// 4bpp quantization, but 0 is transparent on the color-key shell surface.
image.fillRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, top + 12, content.title, 220);
image.drawText(font, left, top + 30, truncateText(font, content.status, TEXT_MAX_WIDTH), 130);
const rows = content.rows;
const hasMenu = rows.length > 0;
// Reserve space for the actual number of rows this menu has.
const textBottom = hasMenu ? top + DIALOG_H - rows.length * menuRowH - 8 : top + DIALOG_H - 8;
const textTop = top + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
// The tail of a long message stays in view: it is what was said last (or
// where the phone keyboard is typing).
const wrapped = wrapText(font, content.text, TEXT_MAX_WIDTH);
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (hasMenu) {
const menuTop = top + DIALOG_H - rows.length * menuRowH - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * menuRowH;
const selected = i === content.selectedRow;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, menuRowH - 2, true, 6);
}
const row = rows[i]!;
image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
}
} else if (content.hint) {
image.drawText(font, left, top + DIALOG_H - 14, content.hint, 110 - font.lineHeight);
}
}
+159
View File
@@ -0,0 +1,159 @@
import { type GrayImage } from "../../graphics/image";
import { type InputEvent } from "../gestures";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { paintInputDialog } from "./input-dialog";
import { type VoiceSendTarget } from "./voice-input";
/**
* The phone side's handle on an open keyboard-input dialog: the phone types
* into it and picks where the text goes. The layer itself implements this;
* the controller hands it to the phone UI while the dialog is up.
*/
export type KeyboardInputSession = {
/** The send destinations, in the glasses menu's row order. */
readonly targets: ReadonlyArray<{ id: string; label: string }>;
/** Replace the message with what the phone's text field now holds. */
setText(text: string): void;
/**
* Send to the destination highlighted on the glasses (the entry point's
* default unless the user scrolled the menu). The IME's send key.
*/
send(): void;
sendTo(targetId: string): void;
discard(): void;
};
export type KeyboardInputLayerOptions = {
actions: LayerActions;
/** Post-removal cleanup (also fires when the screen turns off). */
onClosed: () => void;
/** Pop this layer off the shell stack. */
dismiss: () => void;
/** Ordered send destinations shown as the first menu rows. */
sendTargets: VoiceSendTarget[];
/** Which send target is highlighted by default (entry-point dependent). */
defaultTargetIndex?: number;
};
/**
* The voice dialog's keyboard twin: the same box on the glasses, but the
* message is typed on the phone (its IME, opened by the keyboard button next
* to the mic button) instead of transcribed. The glasses mirror the text as
* it is typed and show the same send / discard menu, which works from either
* end: the ring or watch scrolls and clicks it, the phone has a button per
* row plus the IME's send key. There is no Continue row — the phone can
* simply edit the text.
*/
export class KeyboardInputLayer implements Layer, KeyboardInputSession {
private text = "";
private menuIndex: number;
private closed = false;
private readonly actions: LayerActions;
private readonly onClosed: () => void;
private readonly dismiss: () => void;
private readonly sendTargets: VoiceSendTarget[];
private readonly defaultTargetIndex: number;
constructor(options: KeyboardInputLayerOptions) {
this.actions = options.actions;
this.onClosed = options.onClosed;
this.dismiss = options.dismiss;
this.sendTargets = options.sendTargets;
const defaultIndex = options.defaultTargetIndex ?? 0;
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
this.menuIndex = this.defaultTargetIndex;
}
get targets(): ReadonlyArray<{ id: string; label: string }> {
return this.sendTargets.map((target) => ({ id: target.id, label: target.label }));
}
setText(text: string): void {
if (this.closed || this.text === text) return;
this.text = text;
this.actions.requestRender();
}
send(): void {
const row = this.menuRows()[this.menuIndex];
const target = row?.target ?? this.sendTargets[this.defaultTargetIndex];
if (target) this.sendToTarget(target);
}
sendTo(targetId: string): void {
const target = this.sendTargets.find((candidate) => candidate.id === targetId);
if (target) this.sendToTarget(target);
}
discard(): void {
if (!this.closed) this.dismiss();
}
private sendToTarget(target: VoiceSendTarget): void {
if (this.closed) return;
const text = this.text.trim();
// An empty message has nothing to deliver; the dialog stays up for the
// phone to keep typing (the glasses row is drawn dim for the same reason).
if (!text) return;
this.dismiss();
target.onSend(text);
}
/** The menu rows: one per send target, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; target?: VoiceSendTarget }> {
const hasText = this.text.trim().length > 0;
const rows: Array<{ label: string; dim: boolean; target?: VoiceSendTarget }> = [];
for (const target of this.sendTargets) {
rows.push({ label: target.label, dim: !hasText, target });
}
rows.push({ label: "Discard", dim: false });
return rows;
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const image = paintBelow();
paintInputDialog(image, {
title: "Keyboard",
status: "Type on the phone",
text: this.text || "(nothing typed yet)",
rows: this.menuRows(),
selectedRow: this.menuIndex,
});
return image;
}
handleInput(event: InputEvent, _ctx: LayerContext): void {
const rowCount = this.menuRows().length;
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % rowCount;
this.actions.requestRender();
return;
case "click": {
const row = this.menuRows()[this.menuIndex];
if (!row) return;
if (row.target) {
this.sendToTarget(row.target);
} else {
this.dismiss();
}
return;
}
case "double-click":
this.dismiss();
return;
default:
return;
}
}
onRemoved(): void {
this.closed = true;
this.onClosed();
}
}
+71 -7
View File
@@ -17,6 +17,7 @@ import {
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input";
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
@@ -153,6 +154,11 @@ export type ShellConfig = {
* when RECORD_AUDIO isn't granted yet (the phone mic is the source there).
*/
prepareVoiceCapture?: () => Promise<boolean>;
/**
* The keyboard dialog opened (with the session the phone types into) or
* closed (null) by any path; the phone UI shows/hides its typing panel.
*/
onKeyboardInputChanged?: (session: KeyboardInputSession | null) => void;
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
onScreenStateChanged: (on: boolean) => void;
/** Window registered/removed or foreground changed (persists the open-app list). */
@@ -291,6 +297,7 @@ class Shell {
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
@@ -548,10 +555,11 @@ class Shell {
// An open voice dialog suspends the timeout: a long dictation or refine
// has no button presses, but the screen must stay on for it. Sliding
// lastInputAtMs forward also restarts the full timeout when it closes.
// An in-flight assistant turn suspends it for the same reason (a tool loop
// can run for a while with no input); once the turn ends and the
// Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
// The keyboard dialog likewise: typing happens on the phone, not the
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -644,7 +652,7 @@ class Shell {
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake("sidebar");
if (action === "voice-input" && !this.activeVoiceLayer) {
if (action === "voice-input" && !this.activeVoiceLayer && !this.activeKeyboardLayer) {
if (this.assistantLayer) {
// The assistant overlay is up; a wakeword continues that conversation.
this.startAssistantFollowUp(true);
@@ -796,6 +804,7 @@ class Shell {
const foreground = this.foregroundWindow()?.windowId ?? "none";
if (!this.screenOn) return `fg=${foreground} target=screen-off`;
if (this.activeVoiceLayer) return `fg=${foreground} target=voice`;
if (this.activeKeyboardLayer) return `fg=${foreground} target=keyboard`;
if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`;
return `fg=${foreground} target=${this.focus}`;
}
@@ -904,7 +913,7 @@ class Shell {
}
// Re-checked after the await: another path may have opened a dialog
// (or torn down the base state) while a permission prompt was up.
if (!ready || this.activeVoiceLayer) return;
if (!ready || this.activeVoiceLayer || this.activeKeyboardLayer) return;
this.openVoiceDialogNow(options);
this.config.requestShellRender();
})();
@@ -1003,6 +1012,61 @@ class Shell {
this.config.requestShellRender();
}
/**
* Open the keyboard dialog: the voice dialog's typed twin, driven by the
* phone's keyboard button (beside the mic button, whose wakeword this
* mirrors: it wakes a dark screen too). Over the assistant overlay the text
* continues that conversation; otherwise it goes to the usual destinations
* with Send to Assistant highlighted. Returns the session the phone types
* into (the already-open one if there is one), or null while a voice
* dialog is up.
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer) return null;
if (!this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
let targets: VoiceSendTarget[];
let defaultIndex = 0;
if (assistantLayer && assistantSession) {
targets = [
{
id: "assistant",
label: "Send",
onSend: (text) => this.runAssistantTurn(assistantSession, assistantLayer, text),
},
];
} else {
targets = this.buildVoiceSendTargets();
defaultIndex = Math.max(0, targets.findIndex((target) => target.id === "assistant"));
}
const layer = new KeyboardInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeKeyboardLayer === layer) {
this.activeKeyboardLayer = null;
this.config.onKeyboardInputChanged?.(null);
// The idle countdown restarts in full once keyboard input ends.
this.noteUserActivity();
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === layer);
// A send or discard from the phone side arrives outside the input
// path, so nothing else asks for the repaint.
this.config.requestShellRender();
},
sendTargets: targets,
defaultTargetIndex: defaultIndex,
});
this.activeKeyboardLayer = layer;
this.stack.push(layer);
this.config.onKeyboardInputChanged?.(layer);
this.config.requestShellRender();
return layer;
}
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
}
@@ -1149,7 +1213,7 @@ class Shell {
private startAssistantFollowUp(handsFree = false): void {
const layer = this.assistantLayer;
const session = this.assistantSession;
if (!layer || !session || this.activeVoiceLayer) return;
if (!layer || !session || this.activeVoiceLayer || this.activeKeyboardLayer) return;
const voice = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
+10 -60
View File
@@ -1,28 +1,10 @@
import { G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { wrapText, truncateText } from "../../graphics/textwrap";
import { type BdfFont } from "../../graphics/bdffont";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { type GrayImage } from "../../graphics/image";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { refineDictation, type AnthropicStreamHandle } from "../../native/anthropic";
import { anthropicApiKeySetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
import { drawSelectionHighlight } from "../menu";
import { listRowHeight } from "../metrics";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { MIN_WINDOW_HEIGHT, minWindowTop } from "./geometry";
const DIALOG_X = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
// The dialog fits inside the min-height window band (like the other shell
// overlays), wherever the vertical position setting puts it.
const DIALOG_MARGIN_Y = 24;
const DIALOG_H = MIN_WINDOW_HEIGHT - 2 * DIALOG_MARGIN_Y;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
/** Dialog top edge; band-relative, so computed per paint. */
function dialogY(): number {
return minWindowTop() + DIALOG_MARGIN_Y;
}
import { paintInputDialog } from "./input-dialog";
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
@@ -228,48 +210,16 @@ export class VoiceInputLayer implements Layer {
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const menuRowH = listRowHeight(font);
const image = paintBelow();
const top = dialogY();
// Solid dialog box over the underlying UI. Fill 1, not 0: identical after
// 4bpp quantization, but 0 is transparent on the color-key shell surface.
image.fillRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, top + 12, this.capturing ? "Voice ●" : "Voice", 220);
image.drawText(font, left, top + 30, truncateText(font, this.status, TEXT_MAX_WIDTH), 130);
const inMenu = this.phase === "menu";
const rows = inMenu ? this.menuRows() : [];
// Reserve space for the actual number of rows this menu has.
const textBottom = inMenu ? top + DIALOG_H - rows.length * menuRowH - 8 : top + DIALOG_H - 8;
const textTop = top + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
const text = this.displayText() || this.placeholderText();
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (inMenu) {
const menuTop = top + DIALOG_H - rows.length * menuRowH - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * menuRowH;
const selected = i === this.menuIndex;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, menuRowH - 2, true, 6);
}
const row = rows[i]!;
image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
}
} else {
image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110-font.lineHeight);
}
paintInputDialog(image, {
title: this.capturing ? "Voice ●" : "Voice",
status: this.status,
text: this.displayText() || this.placeholderText(),
rows: inMenu ? this.menuRows() : [],
selectedRow: this.menuIndex,
hint: inMenu ? undefined : this.hintText(),
});
return image;
}