Add keyboard-input button to the phone home screen

This commit is contained in:
James Babcock committed 2026-09-03 16:06:46 -07:00
1 parent 00dd546b1c
commit aef6194687
12 files changed
+574 -85

No files matched your search

@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Keyboard icon for the main page's tab-bar keyboard-dialog button (beside
the mic button). Material Design "keyboard" glyph (Apache 2.0); base fill
is overridden by tint-color in app.css. -->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="24"
android:viewportHeight="24">
<path
android:fillColor="#FF333333"
android:pathData="M20 5H4c-1.1 0-1.99.9-1.99 2L2 17c0 1.1.9 2 2 2h16c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm-9 3h2v2h-2V8zm0 3h2v2h-2v-2zM8 8h2v2H8V8zm0 3h2v2H8v-2zm-1 2H5v-2h2v2zm0-3H5V8h2v2zm9 7H8v-2h8v2zm0-4h-2v-2h2v2zm0-3h-2V8h2v2zm3 3h-2v-2h2v2zm0-3h-2V8h2v2z" />
</vector>
+1
View File
@@ -6,6 +6,7 @@
* Move some items from app context menus to the system context menu
* Long-press now opens the system context menu and tap-then-hold the app context menu (games keep long-press as a move while playing)
* Content beneath an open context menu is dimmed
* Add a keyboard-input button to the phone home screen
* Navigate app: Location indicator rotates based on the compass
* Navigate app: Darken the map background color
+8 -7
View File
@@ -182,11 +182,12 @@ Button.-primary {
background-color: #424242;
color: #ffffff;
}
/* The wakeword button at the tab bar's right edge: icon-only, narrower than
* the tabs, and never in the selected state (it isn't a tab). It is a
* button-styled GridLayout (Buttons can't hold images), so it borrows the
* .tab-button look and adds the elevation MaterialButtons get for free. */
.tab-mic-button {
/* The wakeword and keyboard buttons at the tab bar's right edge: icon-only,
* narrower than the tabs, and never in the selected state (they aren't tabs).
* Each is a button-styled GridLayout (Buttons can't hold images), so it
* borrows the .tab-button look and adds the elevation MaterialButtons get for
* free. */
.tab-icon-button {
width: 56;
/* The real MaterialButtons render 48dp tall inside a 60dp layout box (6dp
* insets top and bottom); the margins and explicit height reproduce that.
@@ -196,11 +197,11 @@ Button.-primary {
margin: 6 2;
android-elevation: 2;
}
.tab-mic-icon {
.tab-icon {
height: 20;
tint-color: #333333;
}
.ns-dark .tab-mic-icon {
.ns-dark .tab-icon {
tint-color: #cccccc;
}
.ns-dark .tab-button {
+50
View File
@@ -64,6 +64,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry";
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, displayModeSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
@@ -102,6 +103,13 @@ export type DashboardSnapshot = {
activeTextEditorToggleLabel: string;
activeTextEditorToggleValue: boolean;
activeTextEditorToggleVisible: boolean;
/**
* The keyboard dialog is up on the glasses (opened by the phone's keyboard
* button): the phone shows its typing panel, with a send button per
* destination in the glasses menu's row order.
*/
keyboardInputActive: boolean;
keyboardInputTargets: ReadonlyArray<{ id: string; label: string }>;
evenAppConflictMessage: string;
evenAppConflictWarningVisible: boolean;
firmwareWarningMessage: string;
@@ -274,6 +282,9 @@ class DashboardController {
// The window hosting the on-glasses text-setting editor (the Settings app
// registers itself); edit flows from the phone UI reach it through this.
private textEditorHost: TextEditorHost | null = null;
// The open keyboard dialog's session (see Shell.startKeyboardInput); the
// phone's typing panel feeds it.
private keyboardInput: KeyboardInputSession | null = null;
// Other in-process singleton apps, keyed by windowId.
private readonly inProcessApps = new Map<string, InProcessWindow>();
private sharedActions!: Omit<LayerActions, "requestRender">;
@@ -333,6 +344,10 @@ class DashboardController {
getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()),
requestShellRender: () => this.requestShellRender(),
prepareVoiceCapture: () => this.prepareVoiceCapture(),
onKeyboardInputChanged: (session) => {
this.keyboardInput = session;
this.emit();
},
onWindowsChanged: () => {
this.persistOpenApps();
// The foreground title is mirrored on both remote-control faces.
@@ -909,6 +924,8 @@ class DashboardController {
activeTextEditorToggleLabel: this.activeTextEditorToggle?.label ?? "",
activeTextEditorToggleValue: this.activeTextEditorToggle?.setting.get() ?? false,
activeTextEditorToggleVisible: this.activeTextEditorToggle !== null,
keyboardInputActive: this.keyboardInput !== null,
keyboardInputTargets: this.keyboardInput?.targets ?? [],
evenAppConflictMessage: this.evenAppConflictMessage,
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
firmwareWarningMessage: this.firmwareWarningMessage,
@@ -1045,6 +1062,39 @@ class DashboardController {
this.previewOrRenderAfterTextSettingChange();
}
/**
* The phone's keyboard button: open the keyboard dialog on the glasses
* (the typed twin of the mic button's wakeword). Needs a display to draw
* on and, like ring input, does nothing while the glasses are locked.
*/
startKeyboardInput(): void {
if (this.phase !== "connected" && this.phase !== "charging" && !this.isPreviewDisplayActive()) return;
if (this.glassesLocked) {
this.appendLog("keyboard input ignored while the glasses are locked");
return;
}
shell.startKeyboardInput();
}
/** The phone's typing panel changed: mirror the text on the glasses. */
setKeyboardInputText(text: string): void {
shell.noteUserActivity();
this.keyboardInput?.setText(text);
}
/** Send to one destination, or (no id: the IME's send key) the one highlighted on the glasses. */
sendKeyboardInput(targetId?: string): void {
if (targetId) {
this.keyboardInput?.sendTo(targetId);
} else {
this.keyboardInput?.send();
}
}
discardKeyboardInput(): void {
this.keyboardInput?.discard();
}
setActiveTextEditorToggleValue(value: boolean): void {
if (!this.activeTextEditorToggle || this.activeTextEditorToggle.setting.get() === value) return;
this.activeTextEditorToggle.setting.set(value);
+32 -2
View File
@@ -39,7 +39,12 @@ function cleanupPage(page: Page): void {
setPageState(page, undefined)
}
function applySettingsTextFieldContrast(textField: TextField): void {
/**
* Readable colours for the light editor panels in both themes. selectAllOnFocus
* suits replacing an existing setting value; a message being composed keeps
* its caret where the user taps.
*/
function applySettingsTextFieldContrast(textField: TextField, selectAllOnFocus = true): void {
textField.color = SETTINGS_TEXT_COLOR
textField.backgroundColor = SETTINGS_BACKGROUND_COLOR
textField.placeholderColor = SETTINGS_PLACEHOLDER_COLOR
@@ -57,7 +62,7 @@ function applySettingsTextFieldContrast(textField: TextField): void {
nativeTextField.setHintTextColor(android.graphics.Color.rgb(102, 102, 102))
// Editing an existing value usually means replacing it: select all on focus
// so typing starts fresh but the current value stays visible.
nativeTextField.setSelectAllOnFocus(true)
nativeTextField.setSelectAllOnFocus(selectAllOnFocus)
}
/**
@@ -83,6 +88,22 @@ function focusTextEditor(page: Page): void {
}, 0)
}
/** The keyboard dialog closed on the glasses (sent, discarded, or screen off). */
function dismissKeyboardInputKeyboard(page: Page): void {
page.getViewById<TextField>('keyboardInputField')?.dismissSoftInput()
}
/** The keyboard dialog opened: bring up the IME on its field right away. */
function focusKeyboardInput(page: Page): void {
setTimeout(() => {
const textField = page.getViewById<TextField>('keyboardInputField')
if (textField) {
applySettingsTextFieldContrast(textField, false)
textField.focus()
}
}, 0)
}
export function loaded(args: EventData) {
const page = args.object as Page
cleanupPage(page)
@@ -121,6 +142,12 @@ export function loaded(args: EventData) {
} else {
dismissTextEditorKeyboard(page)
}
} else if (propertyArgs.propertyName === 'keyboardInputActive') {
if (model.keyboardInputActive) {
focusKeyboardInput(page)
} else {
dismissKeyboardInputKeyboard(page)
}
}
},
}
@@ -131,6 +158,9 @@ export function loaded(args: EventData) {
if (model.isTextSettingEditorActive) {
focusTextEditor(page)
}
if (model.keyboardInputActive) {
focusKeyboardInput(page)
}
}
export function unloaded(args: EventData) {
+26
View File
@@ -146,5 +146,31 @@
checkedChange="{{ onTextSettingToggleChange }}" />
</GridLayout>
</StackLayout>
<!-- Keyboard-input panel (the keyboard button beside the mic button):
the phone's end of the keyboard dialog on the glasses
(ui/shell/keyboard-input.ts). Same placement and collapsing rules
as the setting editor above. The IME's send key goes to whichever
destination is highlighted on the glasses; the buttons pick one
directly. -->
<StackLayout
class="system-name-modal"
visibility="{{ keyboardInputVisibility }}"
horizontalAlignment="stretch"
verticalAlignment="bottom">
<Label text="Keyboard input" class="h2" />
<TextField
id="keyboardInputField"
text="{{ keyboardInputText }}"
hint="Type a message"
textChange="{{ onKeyboardInputTextChange }}"
returnPress="{{ onKeyboardInputReturnPress }}"
returnKeyType="send" />
<!-- Wraps onto a second line when the labels don't fit side by side. -->
<FlexboxLayout flexWrap="wrap" class="m-t-8">
<Button text="{{ keyboardInputPrimaryTargetLabel }}" tap="{{ onKeyboardInputPrimarySendTap }}" visibility="{{ keyboardInputPrimaryTargetVisibility }}" flexGrow="1" class="-primary" />
<Button text="{{ keyboardInputSecondaryTargetLabel }}" tap="{{ onKeyboardInputSecondarySendTap }}" visibility="{{ keyboardInputSecondaryTargetVisibility }}" flexGrow="1" />
<Button text="Discard" tap="{{ onKeyboardInputDiscardTap }}" flexGrow="1" />
</FlexboxLayout>
</StackLayout>
</GridLayout>
</Page>
+101 -1
View File
@@ -56,6 +56,9 @@ export class MainViewModel extends Observable {
private _activeTextEditorToggleLabel = "";
private _activeTextEditorToggleValue = false;
private _activeTextEditorToggleVisible = false;
private _keyboardInputActive = false;
private _keyboardInputTargets: ReadonlyArray<{ id: string; label: string }> = [];
private _keyboardInputText = "";
private _evenAppConflictMessage = "";
private _evenAppConflictWarningVisible = false;
private _firmwareWarningMessage = "";
@@ -103,6 +106,9 @@ export class MainViewModel extends Observable {
this.activeTextEditorToggleLabel = snapshot.activeTextEditorToggleLabel;
this.activeTextEditorToggleValue = snapshot.activeTextEditorToggleValue;
this.activeTextEditorToggleVisible = snapshot.activeTextEditorToggleVisible;
// Targets before active: the panel's buttons are labelled when it appears.
this.keyboardInputTargets = snapshot.keyboardInputTargets;
this.keyboardInputActive = snapshot.keyboardInputActive;
this.evenAppConflictMessage = snapshot.evenAppConflictMessage;
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
this.firmwareWarningMessage = snapshot.firmwareWarningMessage;
@@ -466,6 +472,100 @@ export class MainViewModel extends Observable {
return this.isTextSettingEditorActive ? "visible" : "collapse";
}
// ---- keyboard-input panel (the keyboard button beside the mic button) ----
//
// The dialog lives on the glasses (ui/shell/keyboard-input.ts); this panel
// is the phone's end of it: the text field the IME types into and a send
// button per destination. The field's text is the panel's own state, never
// echoed back from the controller (see setActiveTextSettingValue for why);
// it is cleared whenever a dialog opens or closes.
get keyboardInputActive(): boolean {
return this._keyboardInputActive;
}
set keyboardInputActive(value: boolean) {
if (this._keyboardInputActive === value) return;
this._keyboardInputActive = value;
this.keyboardInputText = "";
this.notifyPropertyChange("keyboardInputActive", value);
this.notifyPropertyChange("keyboardInputVisibility", this.keyboardInputVisibility);
this.notifyPropertyChange("controlsVisibility", this.controlsVisibility);
}
get keyboardInputVisibility(): "visible" | "collapse" {
return this._keyboardInputActive ? "visible" : "collapse";
}
get keyboardInputText(): string {
return this._keyboardInputText;
}
set keyboardInputText(value: string) {
if (this._keyboardInputText === value) return;
this._keyboardInputText = value;
this.notifyPropertyChange("keyboardInputText", value);
}
set keyboardInputTargets(value: ReadonlyArray<{ id: string; label: string }>) {
this._keyboardInputTargets = value;
this.notifyPropertyChange("keyboardInputPrimaryTargetLabel", this.keyboardInputPrimaryTargetLabel);
this.notifyPropertyChange("keyboardInputPrimaryTargetVisibility", this.keyboardInputPrimaryTargetVisibility);
this.notifyPropertyChange("keyboardInputSecondaryTargetLabel", this.keyboardInputSecondaryTargetLabel);
this.notifyPropertyChange("keyboardInputSecondaryTargetVisibility", this.keyboardInputSecondaryTargetVisibility);
}
// The glasses menu has at most two destinations (assistant, app), so the
// panel has a fixed pair of buttons rather than a repeater.
get keyboardInputPrimaryTargetLabel(): string {
return this._keyboardInputTargets[0]?.label ?? "";
}
get keyboardInputPrimaryTargetVisibility(): "visible" | "collapse" {
return this._keyboardInputTargets[0] ? "visible" : "collapse";
}
get keyboardInputSecondaryTargetLabel(): string {
return this._keyboardInputTargets[1]?.label ?? "";
}
get keyboardInputSecondaryTargetVisibility(): "visible" | "collapse" {
return this._keyboardInputTargets[1] ? "visible" : "collapse";
}
onKeyboardTap(): void {
dashboardController.startKeyboardInput();
}
onKeyboardInputTextChange(args: { value?: string; object?: { text?: string } }): void {
dashboardController.setKeyboardInputText(args.object?.text ?? args.value ?? "");
}
/** The IME's send key: the destination highlighted on the glasses. */
onKeyboardInputReturnPress(args: { object?: { text?: string } }): void {
// Commit the field's actual text at send-time, in case the final
// keystroke's textChange hadn't landed yet.
const text = args?.object?.text;
if (typeof text === "string") {
dashboardController.setKeyboardInputText(text);
}
dashboardController.sendKeyboardInput();
}
onKeyboardInputPrimarySendTap(): void {
const target = this._keyboardInputTargets[0];
if (target) dashboardController.sendKeyboardInput(target.id);
}
onKeyboardInputSecondarySendTap(): void {
const target = this._keyboardInputTargets[1];
if (target) dashboardController.sendKeyboardInput(target.id);
}
onKeyboardInputDiscardTap(): void {
dashboardController.discardKeyboardInput();
}
get evenAppConflictMessage(): string {
return this._evenAppConflictMessage;
}
@@ -1016,7 +1116,7 @@ export class MainViewModel extends Observable {
private _controlsTab: ControlsTab = lastControlsTab;
get controlsVisibility(): "visible" | "collapse" {
return this.isTextSettingEditorActive ? "collapse" : "visible";
return this.isTextSettingEditorActive || this._keyboardInputActive ? "collapse" : "visible";
}
get settingsTabVisibility(): "visible" | "collapse" {
+13 -8
View File
@@ -4,10 +4,12 @@
scheme as the watch pad - tap = select, double-tap or two fingers = back,
hold = menu, swipes navigate), and Ring (a touchpad on the ring's own
scheme: vertical swipes scroll, tap/double-tap/hold = click/double-click/
long-press). Tap-then-hold on either pad is the G2 system-menu gesture. The narrow mic button at the tab bar's right edge fires the
wakeword without changing the selected tab. Used by main-page.xml in both
orientations; binds to the page's MainViewModel, which collapses the whole
area while a text setting is being edited (keyboard-input mode).
long-press). Tap-then-hold on either pad is the G2 system-menu gesture. The
narrow mic and keyboard buttons at the tab bar's right edge fire the
wakeword / open the keyboard dialog without changing the selected tab. Used
by main-page.xml in both orientations; binds to the page's MainViewModel,
which collapses the whole area while a text setting is being edited or the
keyboard dialog is up (keyboard-input mode).
The root is a rows="auto,*" grid so the active section can use the leftover
height (the Watch face centers in it); the three sections share the star
@@ -17,15 +19,18 @@
<!-- rows="auto" everywhere below: a rows-less GridLayout's implicit row
acts as star, and this component sits in a bounded (star-row) parent,
where that would balloon the first grid to fill all remaining height. -->
<GridLayout row="0" rows="auto" columns="*,*,*,auto" class="controls-tab-bar">
<GridLayout row="0" rows="auto" columns="*,*,*,auto,auto" class="controls-tab-bar">
<Button col="0" text="Settings" tap="{{ onSettingsTabTap }}" class="{{ settingsTabClass }}" />
<Button col="1" text="Watch" tap="{{ onWatchTabTap }}" class="{{ watchTabClass }}" />
<Button col="2" text="Ring" tap="{{ onRingTabTap }}" class="{{ ringTabClass }}" />
<!-- A Button can't hold an image, so this is a button-styled tile. The
<!-- A Button can't hold an image, so these are button-styled tiles. The
explicit star row makes the cell fill the tile's height so the icon's
center alignment has room to work in. -->
<GridLayout col="3" rows="*" tap="{{ onSyntheticMicTap }}" class="tab-button tab-mic-button">
<Image src="res://ic_tab_mic" class="tab-mic-icon" verticalAlignment="center" horizontalAlignment="center" />
<GridLayout col="3" rows="*" tap="{{ onSyntheticMicTap }}" class="tab-button tab-icon-button">
<Image src="res://ic_tab_mic" class="tab-icon" verticalAlignment="center" horizontalAlignment="center" />
</GridLayout>
<GridLayout col="4" rows="*" tap="{{ onKeyboardTap }}" class="tab-button tab-icon-button">
<Image src="res://ic_tab_keyboard" class="tab-icon" verticalAlignment="center" horizontalAlignment="center" />
</GridLayout>
</GridLayout>
+90
View File
@@ -0,0 +1,90 @@
import { G2_LENS_WIDTH, type GrayImage } from "../../graphics/image";
import { wrapText, truncateText } from "../../graphics/textwrap";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { drawSelectionHighlight } from "../menu";
import { listRowHeight } from "../metrics";
import { MIN_WINDOW_HEIGHT, minWindowTop } from "./geometry";
/**
* The shared look of the shell's text-entry dialogs (voice input and phone
* keyboard input): a solid box over whatever is on screen with a title line,
* a status line, the message so far, and either a menu of destinations at
* the bottom or a gesture hint.
*/
const DIALOG_X = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
// The dialog fits inside the min-height window band (like the other shell
// overlays), wherever the vertical position setting puts it.
const DIALOG_MARGIN_Y = 24;
const DIALOG_H = MIN_WINDOW_HEIGHT - 2 * DIALOG_MARGIN_Y;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
/** Dialog top edge; band-relative, so computed per paint. */
function dialogY(): number {
return minWindowTop() + DIALOG_MARGIN_Y;
}
export type InputDialogRow = {
label: string;
/** Drawn faint: the row is not currently selectable (e.g. nothing to send). */
dim: boolean;
};
export type InputDialogContent = {
title: string;
status: string;
/** The message body (a placeholder when nothing has been captured yet). */
text: string;
/** Menu rows along the bottom edge; empty for no menu. */
rows: readonly InputDialogRow[];
selectedRow: number;
/** Gesture hint drawn along the bottom edge when there is no menu. */
hint?: string;
};
/** Paint the dialog onto `image` (an already-painted canvas of the layers below). */
export function paintInputDialog(image: GrayImage, content: InputDialogContent): void {
const font = getDefaultSmallFont();
const menuRowH = listRowHeight(font);
const top = dialogY();
// Solid dialog box over the underlying UI. Fill 1, not 0: identical after
// 4bpp quantization, but 0 is transparent on the color-key shell surface.
image.fillRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, top + 12, content.title, 220);
image.drawText(font, left, top + 30, truncateText(font, content.status, TEXT_MAX_WIDTH), 130);
const rows = content.rows;
const hasMenu = rows.length > 0;
// Reserve space for the actual number of rows this menu has.
const textBottom = hasMenu ? top + DIALOG_H - rows.length * menuRowH - 8 : top + DIALOG_H - 8;
const textTop = top + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
// The tail of a long message stays in view: it is what was said last (or
// where the phone keyboard is typing).
const wrapped = wrapText(font, content.text, TEXT_MAX_WIDTH);
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (hasMenu) {
const menuTop = top + DIALOG_H - rows.length * menuRowH - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * menuRowH;
const selected = i === content.selectedRow;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, menuRowH - 2, true, 6);
}
const row = rows[i]!;
image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
}
} else if (content.hint) {
image.drawText(font, left, top + DIALOG_H - 14, content.hint, 110 - font.lineHeight);
}
}
+159
View File
@@ -0,0 +1,159 @@
import { type GrayImage } from "../../graphics/image";
import { type InputEvent } from "../gestures";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { paintInputDialog } from "./input-dialog";
import { type VoiceSendTarget } from "./voice-input";
/**
* The phone side's handle on an open keyboard-input dialog: the phone types
* into it and picks where the text goes. The layer itself implements this;
* the controller hands it to the phone UI while the dialog is up.
*/
export type KeyboardInputSession = {
/** The send destinations, in the glasses menu's row order. */
readonly targets: ReadonlyArray<{ id: string; label: string }>;
/** Replace the message with what the phone's text field now holds. */
setText(text: string): void;
/**
* Send to the destination highlighted on the glasses (the entry point's
* default unless the user scrolled the menu). The IME's send key.
*/
send(): void;
sendTo(targetId: string): void;
discard(): void;
};
export type KeyboardInputLayerOptions = {
actions: LayerActions;
/** Post-removal cleanup (also fires when the screen turns off). */
onClosed: () => void;
/** Pop this layer off the shell stack. */
dismiss: () => void;
/** Ordered send destinations shown as the first menu rows. */
sendTargets: VoiceSendTarget[];
/** Which send target is highlighted by default (entry-point dependent). */
defaultTargetIndex?: number;
};
/**
* The voice dialog's keyboard twin: the same box on the glasses, but the
* message is typed on the phone (its IME, opened by the keyboard button next
* to the mic button) instead of transcribed. The glasses mirror the text as
* it is typed and show the same send / discard menu, which works from either
* end: the ring or watch scrolls and clicks it, the phone has a button per
* row plus the IME's send key. There is no Continue row — the phone can
* simply edit the text.
*/
export class KeyboardInputLayer implements Layer, KeyboardInputSession {
private text = "";
private menuIndex: number;
private closed = false;
private readonly actions: LayerActions;
private readonly onClosed: () => void;
private readonly dismiss: () => void;
private readonly sendTargets: VoiceSendTarget[];
private readonly defaultTargetIndex: number;
constructor(options: KeyboardInputLayerOptions) {
this.actions = options.actions;
this.onClosed = options.onClosed;
this.dismiss = options.dismiss;
this.sendTargets = options.sendTargets;
const defaultIndex = options.defaultTargetIndex ?? 0;
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
this.menuIndex = this.defaultTargetIndex;
}
get targets(): ReadonlyArray<{ id: string; label: string }> {
return this.sendTargets.map((target) => ({ id: target.id, label: target.label }));
}
setText(text: string): void {
if (this.closed || this.text === text) return;
this.text = text;
this.actions.requestRender();
}
send(): void {
const row = this.menuRows()[this.menuIndex];
const target = row?.target ?? this.sendTargets[this.defaultTargetIndex];
if (target) this.sendToTarget(target);
}
sendTo(targetId: string): void {
const target = this.sendTargets.find((candidate) => candidate.id === targetId);
if (target) this.sendToTarget(target);
}
discard(): void {
if (!this.closed) this.dismiss();
}
private sendToTarget(target: VoiceSendTarget): void {
if (this.closed) return;
const text = this.text.trim();
// An empty message has nothing to deliver; the dialog stays up for the
// phone to keep typing (the glasses row is drawn dim for the same reason).
if (!text) return;
this.dismiss();
target.onSend(text);
}
/** The menu rows: one per send target, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; target?: VoiceSendTarget }> {
const hasText = this.text.trim().length > 0;
const rows: Array<{ label: string; dim: boolean; target?: VoiceSendTarget }> = [];
for (const target of this.sendTargets) {
rows.push({ label: target.label, dim: !hasText, target });
}
rows.push({ label: "Discard", dim: false });
return rows;
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const image = paintBelow();
paintInputDialog(image, {
title: "Keyboard",
status: "Type on the phone",
text: this.text || "(nothing typed yet)",
rows: this.menuRows(),
selectedRow: this.menuIndex,
});
return image;
}
handleInput(event: InputEvent, _ctx: LayerContext): void {
const rowCount = this.menuRows().length;
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % rowCount;
this.actions.requestRender();
return;
case "click": {
const row = this.menuRows()[this.menuIndex];
if (!row) return;
if (row.target) {
this.sendToTarget(row.target);
} else {
this.dismiss();
}
return;
}
case "double-click":
this.dismiss();
return;
default:
return;
}
}
onRemoved(): void {
this.closed = true;
this.onClosed();
}
}
+71 -7
View File
@@ -17,6 +17,7 @@ import {
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input";
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
@@ -153,6 +154,11 @@ export type ShellConfig = {
* when RECORD_AUDIO isn't granted yet (the phone mic is the source there).
*/
prepareVoiceCapture?: () => Promise<boolean>;
/**
* The keyboard dialog opened (with the session the phone types into) or
* closed (null) by any path; the phone UI shows/hides its typing panel.
*/
onKeyboardInputChanged?: (session: KeyboardInputSession | null) => void;
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
onScreenStateChanged: (on: boolean) => void;
/** Window registered/removed or foreground changed (persists the open-app list). */
@@ -291,6 +297,7 @@ class Shell {
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
@@ -548,10 +555,11 @@ class Shell {
// An open voice dialog suspends the timeout: a long dictation or refine
// has no button presses, but the screen must stay on for it. Sliding
// lastInputAtMs forward also restarts the full timeout when it closes.
// An in-flight assistant turn suspends it for the same reason (a tool loop
// can run for a while with no input); once the turn ends and the
// Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
// The keyboard dialog likewise: typing happens on the phone, not the
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -644,7 +652,7 @@ class Shell {
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake("sidebar");
if (action === "voice-input" && !this.activeVoiceLayer) {
if (action === "voice-input" && !this.activeVoiceLayer && !this.activeKeyboardLayer) {
if (this.assistantLayer) {
// The assistant overlay is up; a wakeword continues that conversation.
this.startAssistantFollowUp(true);
@@ -796,6 +804,7 @@ class Shell {
const foreground = this.foregroundWindow()?.windowId ?? "none";
if (!this.screenOn) return `fg=${foreground} target=screen-off`;
if (this.activeVoiceLayer) return `fg=${foreground} target=voice`;
if (this.activeKeyboardLayer) return `fg=${foreground} target=keyboard`;
if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`;
return `fg=${foreground} target=${this.focus}`;
}
@@ -904,7 +913,7 @@ class Shell {
}
// Re-checked after the await: another path may have opened a dialog
// (or torn down the base state) while a permission prompt was up.
if (!ready || this.activeVoiceLayer) return;
if (!ready || this.activeVoiceLayer || this.activeKeyboardLayer) return;
this.openVoiceDialogNow(options);
this.config.requestShellRender();
})();
@@ -1003,6 +1012,61 @@ class Shell {
this.config.requestShellRender();
}
/**
* Open the keyboard dialog: the voice dialog's typed twin, driven by the
* phone's keyboard button (beside the mic button, whose wakeword this
* mirrors: it wakes a dark screen too). Over the assistant overlay the text
* continues that conversation; otherwise it goes to the usual destinations
* with Send to Assistant highlighted. Returns the session the phone types
* into (the already-open one if there is one), or null while a voice
* dialog is up.
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer) return null;
if (!this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
let targets: VoiceSendTarget[];
let defaultIndex = 0;
if (assistantLayer && assistantSession) {
targets = [
{
id: "assistant",
label: "Send",
onSend: (text) => this.runAssistantTurn(assistantSession, assistantLayer, text),
},
];
} else {
targets = this.buildVoiceSendTargets();
defaultIndex = Math.max(0, targets.findIndex((target) => target.id === "assistant"));
}
const layer = new KeyboardInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeKeyboardLayer === layer) {
this.activeKeyboardLayer = null;
this.config.onKeyboardInputChanged?.(null);
// The idle countdown restarts in full once keyboard input ends.
this.noteUserActivity();
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === layer);
// A send or discard from the phone side arrives outside the input
// path, so nothing else asks for the repaint.
this.config.requestShellRender();
},
sendTargets: targets,
defaultTargetIndex: defaultIndex,
});
this.activeKeyboardLayer = layer;
this.stack.push(layer);
this.config.onKeyboardInputChanged?.(layer);
this.config.requestShellRender();
return layer;
}
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
}
@@ -1149,7 +1213,7 @@ class Shell {
private startAssistantFollowUp(handsFree = false): void {
const layer = this.assistantLayer;
const session = this.assistantSession;
if (!layer || !session || this.activeVoiceLayer) return;
if (!layer || !session || this.activeVoiceLayer || this.activeKeyboardLayer) return;
const voice = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
+10 -60
View File
@@ -1,28 +1,10 @@
import { G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { wrapText, truncateText } from "../../graphics/textwrap";
import { type BdfFont } from "../../graphics/bdffont";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { type GrayImage } from "../../graphics/image";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { refineDictation, type AnthropicStreamHandle } from "../../native/anthropic";
import { anthropicApiKeySetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
import { drawSelectionHighlight } from "../menu";
import { listRowHeight } from "../metrics";
import { Layer, type LayerActions, type LayerContext } from "../layers";
import { MIN_WINDOW_HEIGHT, minWindowTop } from "./geometry";
const DIALOG_X = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
// The dialog fits inside the min-height window band (like the other shell
// overlays), wherever the vertical position setting puts it.
const DIALOG_MARGIN_Y = 24;
const DIALOG_H = MIN_WINDOW_HEIGHT - 2 * DIALOG_MARGIN_Y;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
/** Dialog top edge; band-relative, so computed per paint. */
function dialogY(): number {
return minWindowTop() + DIALOG_MARGIN_Y;
}
import { paintInputDialog } from "./input-dialog";
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
@@ -228,48 +210,16 @@ export class VoiceInputLayer implements Layer {
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const menuRowH = listRowHeight(font);
const image = paintBelow();
const top = dialogY();
// Solid dialog box over the underlying UI. Fill 1, not 0: identical after
// 4bpp quantization, but 0 is transparent on the color-key shell surface.
image.fillRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, top + 12, this.capturing ? "Voice ●" : "Voice", 220);
image.drawText(font, left, top + 30, truncateText(font, this.status, TEXT_MAX_WIDTH), 130);
const inMenu = this.phase === "menu";
const rows = inMenu ? this.menuRows() : [];
// Reserve space for the actual number of rows this menu has.
const textBottom = inMenu ? top + DIALOG_H - rows.length * menuRowH - 8 : top + DIALOG_H - 8;
const textTop = top + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
const text = this.displayText() || this.placeholderText();
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (inMenu) {
const menuTop = top + DIALOG_H - rows.length * menuRowH - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * menuRowH;
const selected = i === this.menuIndex;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, menuRowH - 2, true, 6);
}
const row = rows[i]!;
image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
}
} else {
image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110-font.lineHeight);
}
paintInputDialog(image, {
title: this.capturing ? "Voice ●" : "Voice",
status: this.status,
text: this.displayText() || this.placeholderText(),
rows: inMenu ? this.menuRows() : [],
selectedRow: this.menuIndex,
hint: inMenu ? undefined : this.hintText(),
});
return image;
}