diff --git a/App_Resources/Android/src/main/res/drawable/ic_tab_keyboard.xml b/App_Resources/Android/src/main/res/drawable/ic_tab_keyboard.xml
new file mode 100644
index 00000000..4e531df8
--- /dev/null
+++ b/App_Resources/Android/src/main/res/drawable/ic_tab_keyboard.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
diff --git a/CHANGELOG b/CHANGELOG
index 7c3535de..3ed1b5b0 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,7 @@
* Move some items from app context menus to the system context menu
* Long-press now opens the system context menu and tap-then-hold the app context menu (games keep long-press as a move while playing)
* Content beneath an open context menu is dimmed
+* Add a keyboard-input button to the phone home screen
* Navigate app: Location indicator rotates based on the compass
* Navigate app: Darken the map background color
diff --git a/app/app.css b/app/app.css
index dd7b3f05..1d96f325 100644
--- a/app/app.css
+++ b/app/app.css
@@ -182,11 +182,12 @@ Button.-primary {
background-color: #424242;
color: #ffffff;
}
-/* The wakeword button at the tab bar's right edge: icon-only, narrower than
- * the tabs, and never in the selected state (it isn't a tab). It is a
- * button-styled GridLayout (Buttons can't hold images), so it borrows the
- * .tab-button look and adds the elevation MaterialButtons get for free. */
-.tab-mic-button {
+/* The wakeword and keyboard buttons at the tab bar's right edge: icon-only,
+ * narrower than the tabs, and never in the selected state (they aren't tabs).
+ * Each is a button-styled GridLayout (Buttons can't hold images), so it
+ * borrows the .tab-button look and adds the elevation MaterialButtons get for
+ * free. */
+.tab-icon-button {
width: 56;
/* The real MaterialButtons render 48dp tall inside a 60dp layout box (6dp
* insets top and bottom); the margins and explicit height reproduce that.
@@ -196,11 +197,11 @@ Button.-primary {
margin: 6 2;
android-elevation: 2;
}
-.tab-mic-icon {
+.tab-icon {
height: 20;
tint-color: #333333;
}
-.ns-dark .tab-mic-icon {
+.ns-dark .tab-icon {
tint-color: #cccccc;
}
.ns-dark .tab-button {
diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts
index cfddeca8..2028fde6 100644
--- a/app/g2/dashboard-controller.ts
+++ b/app/g2/dashboard-controller.ts
@@ -64,6 +64,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry";
import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
+import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, displayModeSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
@@ -102,6 +103,13 @@ export type DashboardSnapshot = {
activeTextEditorToggleLabel: string;
activeTextEditorToggleValue: boolean;
activeTextEditorToggleVisible: boolean;
+ /**
+ * The keyboard dialog is up on the glasses (opened by the phone's keyboard
+ * button): the phone shows its typing panel, with a send button per
+ * destination in the glasses menu's row order.
+ */
+ keyboardInputActive: boolean;
+ keyboardInputTargets: ReadonlyArray<{ id: string; label: string }>;
evenAppConflictMessage: string;
evenAppConflictWarningVisible: boolean;
firmwareWarningMessage: string;
@@ -274,6 +282,9 @@ class DashboardController {
// The window hosting the on-glasses text-setting editor (the Settings app
// registers itself); edit flows from the phone UI reach it through this.
private textEditorHost: TextEditorHost | null = null;
+ // The open keyboard dialog's session (see Shell.startKeyboardInput); the
+ // phone's typing panel feeds it.
+ private keyboardInput: KeyboardInputSession | null = null;
// Other in-process singleton apps, keyed by windowId.
private readonly inProcessApps = new Map();
private sharedActions!: Omit;
@@ -333,6 +344,10 @@ class DashboardController {
getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()),
requestShellRender: () => this.requestShellRender(),
prepareVoiceCapture: () => this.prepareVoiceCapture(),
+ onKeyboardInputChanged: (session) => {
+ this.keyboardInput = session;
+ this.emit();
+ },
onWindowsChanged: () => {
this.persistOpenApps();
// The foreground title is mirrored on both remote-control faces.
@@ -909,6 +924,8 @@ class DashboardController {
activeTextEditorToggleLabel: this.activeTextEditorToggle?.label ?? "",
activeTextEditorToggleValue: this.activeTextEditorToggle?.setting.get() ?? false,
activeTextEditorToggleVisible: this.activeTextEditorToggle !== null,
+ keyboardInputActive: this.keyboardInput !== null,
+ keyboardInputTargets: this.keyboardInput?.targets ?? [],
evenAppConflictMessage: this.evenAppConflictMessage,
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
firmwareWarningMessage: this.firmwareWarningMessage,
@@ -1045,6 +1062,39 @@ class DashboardController {
this.previewOrRenderAfterTextSettingChange();
}
+ /**
+ * The phone's keyboard button: open the keyboard dialog on the glasses
+ * (the typed twin of the mic button's wakeword). Needs a display to draw
+ * on and, like ring input, does nothing while the glasses are locked.
+ */
+ startKeyboardInput(): void {
+ if (this.phase !== "connected" && this.phase !== "charging" && !this.isPreviewDisplayActive()) return;
+ if (this.glassesLocked) {
+ this.appendLog("keyboard input ignored while the glasses are locked");
+ return;
+ }
+ shell.startKeyboardInput();
+ }
+
+ /** The phone's typing panel changed: mirror the text on the glasses. */
+ setKeyboardInputText(text: string): void {
+ shell.noteUserActivity();
+ this.keyboardInput?.setText(text);
+ }
+
+ /** Send to one destination, or (no id: the IME's send key) the one highlighted on the glasses. */
+ sendKeyboardInput(targetId?: string): void {
+ if (targetId) {
+ this.keyboardInput?.sendTo(targetId);
+ } else {
+ this.keyboardInput?.send();
+ }
+ }
+
+ discardKeyboardInput(): void {
+ this.keyboardInput?.discard();
+ }
+
setActiveTextEditorToggleValue(value: boolean): void {
if (!this.activeTextEditorToggle || this.activeTextEditorToggle.setting.get() === value) return;
this.activeTextEditorToggle.setting.set(value);
diff --git a/app/phone-ui/main-page.ts b/app/phone-ui/main-page.ts
index 4298a5e7..01ed68fe 100644
--- a/app/phone-ui/main-page.ts
+++ b/app/phone-ui/main-page.ts
@@ -39,7 +39,12 @@ function cleanupPage(page: Page): void {
setPageState(page, undefined)
}
-function applySettingsTextFieldContrast(textField: TextField): void {
+/**
+ * Readable colours for the light editor panels in both themes. selectAllOnFocus
+ * suits replacing an existing setting value; a message being composed keeps
+ * its caret where the user taps.
+ */
+function applySettingsTextFieldContrast(textField: TextField, selectAllOnFocus = true): void {
textField.color = SETTINGS_TEXT_COLOR
textField.backgroundColor = SETTINGS_BACKGROUND_COLOR
textField.placeholderColor = SETTINGS_PLACEHOLDER_COLOR
@@ -57,7 +62,7 @@ function applySettingsTextFieldContrast(textField: TextField): void {
nativeTextField.setHintTextColor(android.graphics.Color.rgb(102, 102, 102))
// Editing an existing value usually means replacing it: select all on focus
// so typing starts fresh but the current value stays visible.
- nativeTextField.setSelectAllOnFocus(true)
+ nativeTextField.setSelectAllOnFocus(selectAllOnFocus)
}
/**
@@ -83,6 +88,22 @@ function focusTextEditor(page: Page): void {
}, 0)
}
+/** The keyboard dialog closed on the glasses (sent, discarded, or screen off). */
+function dismissKeyboardInputKeyboard(page: Page): void {
+ page.getViewById('keyboardInputField')?.dismissSoftInput()
+}
+
+/** The keyboard dialog opened: bring up the IME on its field right away. */
+function focusKeyboardInput(page: Page): void {
+ setTimeout(() => {
+ const textField = page.getViewById('keyboardInputField')
+ if (textField) {
+ applySettingsTextFieldContrast(textField, false)
+ textField.focus()
+ }
+ }, 0)
+}
+
export function loaded(args: EventData) {
const page = args.object as Page
cleanupPage(page)
@@ -121,6 +142,12 @@ export function loaded(args: EventData) {
} else {
dismissTextEditorKeyboard(page)
}
+ } else if (propertyArgs.propertyName === 'keyboardInputActive') {
+ if (model.keyboardInputActive) {
+ focusKeyboardInput(page)
+ } else {
+ dismissKeyboardInputKeyboard(page)
+ }
}
},
}
@@ -131,6 +158,9 @@ export function loaded(args: EventData) {
if (model.isTextSettingEditorActive) {
focusTextEditor(page)
}
+ if (model.keyboardInputActive) {
+ focusKeyboardInput(page)
+ }
}
export function unloaded(args: EventData) {
diff --git a/app/phone-ui/main-page.xml b/app/phone-ui/main-page.xml
index 9b8584fe..7ce43171 100644
--- a/app/phone-ui/main-page.xml
+++ b/app/phone-ui/main-page.xml
@@ -146,5 +146,31 @@
checkedChange="{{ onTextSettingToggleChange }}" />
+
+
+
+
+
+
+
+
+
+
+
diff --git a/app/phone-ui/main-view-model.ts b/app/phone-ui/main-view-model.ts
index 1644b01f..a08ac942 100644
--- a/app/phone-ui/main-view-model.ts
+++ b/app/phone-ui/main-view-model.ts
@@ -56,6 +56,9 @@ export class MainViewModel extends Observable {
private _activeTextEditorToggleLabel = "";
private _activeTextEditorToggleValue = false;
private _activeTextEditorToggleVisible = false;
+ private _keyboardInputActive = false;
+ private _keyboardInputTargets: ReadonlyArray<{ id: string; label: string }> = [];
+ private _keyboardInputText = "";
private _evenAppConflictMessage = "";
private _evenAppConflictWarningVisible = false;
private _firmwareWarningMessage = "";
@@ -103,6 +106,9 @@ export class MainViewModel extends Observable {
this.activeTextEditorToggleLabel = snapshot.activeTextEditorToggleLabel;
this.activeTextEditorToggleValue = snapshot.activeTextEditorToggleValue;
this.activeTextEditorToggleVisible = snapshot.activeTextEditorToggleVisible;
+ // Targets before active: the panel's buttons are labelled when it appears.
+ this.keyboardInputTargets = snapshot.keyboardInputTargets;
+ this.keyboardInputActive = snapshot.keyboardInputActive;
this.evenAppConflictMessage = snapshot.evenAppConflictMessage;
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
this.firmwareWarningMessage = snapshot.firmwareWarningMessage;
@@ -466,6 +472,100 @@ export class MainViewModel extends Observable {
return this.isTextSettingEditorActive ? "visible" : "collapse";
}
+ // ---- keyboard-input panel (the keyboard button beside the mic button) ----
+ //
+ // The dialog lives on the glasses (ui/shell/keyboard-input.ts); this panel
+ // is the phone's end of it: the text field the IME types into and a send
+ // button per destination. The field's text is the panel's own state, never
+ // echoed back from the controller (see setActiveTextSettingValue for why);
+ // it is cleared whenever a dialog opens or closes.
+
+ get keyboardInputActive(): boolean {
+ return this._keyboardInputActive;
+ }
+
+ set keyboardInputActive(value: boolean) {
+ if (this._keyboardInputActive === value) return;
+ this._keyboardInputActive = value;
+ this.keyboardInputText = "";
+ this.notifyPropertyChange("keyboardInputActive", value);
+ this.notifyPropertyChange("keyboardInputVisibility", this.keyboardInputVisibility);
+ this.notifyPropertyChange("controlsVisibility", this.controlsVisibility);
+ }
+
+ get keyboardInputVisibility(): "visible" | "collapse" {
+ return this._keyboardInputActive ? "visible" : "collapse";
+ }
+
+ get keyboardInputText(): string {
+ return this._keyboardInputText;
+ }
+
+ set keyboardInputText(value: string) {
+ if (this._keyboardInputText === value) return;
+ this._keyboardInputText = value;
+ this.notifyPropertyChange("keyboardInputText", value);
+ }
+
+ set keyboardInputTargets(value: ReadonlyArray<{ id: string; label: string }>) {
+ this._keyboardInputTargets = value;
+ this.notifyPropertyChange("keyboardInputPrimaryTargetLabel", this.keyboardInputPrimaryTargetLabel);
+ this.notifyPropertyChange("keyboardInputPrimaryTargetVisibility", this.keyboardInputPrimaryTargetVisibility);
+ this.notifyPropertyChange("keyboardInputSecondaryTargetLabel", this.keyboardInputSecondaryTargetLabel);
+ this.notifyPropertyChange("keyboardInputSecondaryTargetVisibility", this.keyboardInputSecondaryTargetVisibility);
+ }
+
+ // The glasses menu has at most two destinations (assistant, app), so the
+ // panel has a fixed pair of buttons rather than a repeater.
+ get keyboardInputPrimaryTargetLabel(): string {
+ return this._keyboardInputTargets[0]?.label ?? "";
+ }
+
+ get keyboardInputPrimaryTargetVisibility(): "visible" | "collapse" {
+ return this._keyboardInputTargets[0] ? "visible" : "collapse";
+ }
+
+ get keyboardInputSecondaryTargetLabel(): string {
+ return this._keyboardInputTargets[1]?.label ?? "";
+ }
+
+ get keyboardInputSecondaryTargetVisibility(): "visible" | "collapse" {
+ return this._keyboardInputTargets[1] ? "visible" : "collapse";
+ }
+
+ onKeyboardTap(): void {
+ dashboardController.startKeyboardInput();
+ }
+
+ onKeyboardInputTextChange(args: { value?: string; object?: { text?: string } }): void {
+ dashboardController.setKeyboardInputText(args.object?.text ?? args.value ?? "");
+ }
+
+ /** The IME's send key: the destination highlighted on the glasses. */
+ onKeyboardInputReturnPress(args: { object?: { text?: string } }): void {
+ // Commit the field's actual text at send-time, in case the final
+ // keystroke's textChange hadn't landed yet.
+ const text = args?.object?.text;
+ if (typeof text === "string") {
+ dashboardController.setKeyboardInputText(text);
+ }
+ dashboardController.sendKeyboardInput();
+ }
+
+ onKeyboardInputPrimarySendTap(): void {
+ const target = this._keyboardInputTargets[0];
+ if (target) dashboardController.sendKeyboardInput(target.id);
+ }
+
+ onKeyboardInputSecondarySendTap(): void {
+ const target = this._keyboardInputTargets[1];
+ if (target) dashboardController.sendKeyboardInput(target.id);
+ }
+
+ onKeyboardInputDiscardTap(): void {
+ dashboardController.discardKeyboardInput();
+ }
+
get evenAppConflictMessage(): string {
return this._evenAppConflictMessage;
}
@@ -1016,7 +1116,7 @@ export class MainViewModel extends Observable {
private _controlsTab: ControlsTab = lastControlsTab;
get controlsVisibility(): "visible" | "collapse" {
- return this.isTextSettingEditorActive ? "collapse" : "visible";
+ return this.isTextSettingEditorActive || this._keyboardInputActive ? "collapse" : "visible";
}
get settingsTabVisibility(): "visible" | "collapse" {
diff --git a/app/phone-ui/remote-controls.xml b/app/phone-ui/remote-controls.xml
index ff41d8ef..02108a82 100644
--- a/app/phone-ui/remote-controls.xml
+++ b/app/phone-ui/remote-controls.xml
@@ -4,10 +4,12 @@
scheme as the watch pad - tap = select, double-tap or two fingers = back,
hold = menu, swipes navigate), and Ring (a touchpad on the ring's own
scheme: vertical swipes scroll, tap/double-tap/hold = click/double-click/
- long-press). Tap-then-hold on either pad is the G2 system-menu gesture. The narrow mic button at the tab bar's right edge fires the
- wakeword without changing the selected tab. Used by main-page.xml in both
- orientations; binds to the page's MainViewModel, which collapses the whole
- area while a text setting is being edited (keyboard-input mode).
+ long-press). Tap-then-hold on either pad is the G2 system-menu gesture. The
+ narrow mic and keyboard buttons at the tab bar's right edge fire the
+ wakeword / open the keyboard dialog without changing the selected tab. Used
+ by main-page.xml in both orientations; binds to the page's MainViewModel,
+ which collapses the whole area while a text setting is being edited or the
+ keyboard dialog is up (keyboard-input mode).
The root is a rows="auto,*" grid so the active section can use the leftover
height (the Watch face centers in it); the three sections share the star
@@ -17,15 +19,18 @@
-
+
-
-
-
+
+
+
+
+
diff --git a/app/ui/shell/input-dialog.ts b/app/ui/shell/input-dialog.ts
new file mode 100644
index 00000000..7b36d9d1
--- /dev/null
+++ b/app/ui/shell/input-dialog.ts
@@ -0,0 +1,90 @@
+import { G2_LENS_WIDTH, type GrayImage } from "../../graphics/image";
+import { wrapText, truncateText } from "../../graphics/textwrap";
+import { getDefaultSmallFont } from "../../graphics/ui-fonts";
+import { drawSelectionHighlight } from "../menu";
+import { listRowHeight } from "../metrics";
+import { MIN_WINDOW_HEIGHT, minWindowTop } from "./geometry";
+
+/**
+ * The shared look of the shell's text-entry dialogs (voice input and phone
+ * keyboard input): a solid box over whatever is on screen with a title line,
+ * a status line, the message so far, and either a menu of destinations at
+ * the bottom or a gesture hint.
+ */
+
+const DIALOG_X = 40;
+const DIALOG_W = G2_LENS_WIDTH - 80;
+// The dialog fits inside the min-height window band (like the other shell
+// overlays), wherever the vertical position setting puts it.
+const DIALOG_MARGIN_Y = 24;
+const DIALOG_H = MIN_WINDOW_HEIGHT - 2 * DIALOG_MARGIN_Y;
+const TEXT_MAX_WIDTH = DIALOG_W - 32;
+
+/** Dialog top edge; band-relative, so computed per paint. */
+function dialogY(): number {
+ return minWindowTop() + DIALOG_MARGIN_Y;
+}
+
+export type InputDialogRow = {
+ label: string;
+ /** Drawn faint: the row is not currently selectable (e.g. nothing to send). */
+ dim: boolean;
+};
+
+export type InputDialogContent = {
+ title: string;
+ status: string;
+ /** The message body (a placeholder when nothing has been captured yet). */
+ text: string;
+ /** Menu rows along the bottom edge; empty for no menu. */
+ rows: readonly InputDialogRow[];
+ selectedRow: number;
+ /** Gesture hint drawn along the bottom edge when there is no menu. */
+ hint?: string;
+};
+
+/** Paint the dialog onto `image` (an already-painted canvas of the layers below). */
+export function paintInputDialog(image: GrayImage, content: InputDialogContent): void {
+ const font = getDefaultSmallFont();
+ const menuRowH = listRowHeight(font);
+ const top = dialogY();
+
+ // Solid dialog box over the underlying UI. Fill 1, not 0: identical after
+ // 4bpp quantization, but 0 is transparent on the color-key shell surface.
+ image.fillRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 1, 10);
+ image.drawRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 90, 10);
+
+ const left = DIALOG_X + 16;
+ image.drawText(font, left, top + 12, content.title, 220);
+ image.drawText(font, left, top + 30, truncateText(font, content.status, TEXT_MAX_WIDTH), 130);
+
+ const rows = content.rows;
+ const hasMenu = rows.length > 0;
+ // Reserve space for the actual number of rows this menu has.
+ const textBottom = hasMenu ? top + DIALOG_H - rows.length * menuRowH - 8 : top + DIALOG_H - 8;
+ const textTop = top + 56;
+ const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
+
+ // The tail of a long message stays in view: it is what was said last (or
+ // where the phone keyboard is typing).
+ const wrapped = wrapText(font, content.text, TEXT_MAX_WIDTH);
+ const firstLine = Math.max(0, wrapped.length - maxLines);
+ for (let index = firstLine; index < wrapped.length; index++) {
+ image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
+ }
+
+ if (hasMenu) {
+ const menuTop = top + DIALOG_H - rows.length * menuRowH - 2;
+ for (let i = 0; i < rows.length; i++) {
+ const rowY = menuTop + i * menuRowH;
+ const selected = i === content.selectedRow;
+ if (selected) {
+ drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, menuRowH - 2, true, 6);
+ }
+ const row = rows[i]!;
+ image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
+ }
+ } else if (content.hint) {
+ image.drawText(font, left, top + DIALOG_H - 14, content.hint, 110 - font.lineHeight);
+ }
+}
diff --git a/app/ui/shell/keyboard-input.ts b/app/ui/shell/keyboard-input.ts
new file mode 100644
index 00000000..22f51014
--- /dev/null
+++ b/app/ui/shell/keyboard-input.ts
@@ -0,0 +1,159 @@
+import { type GrayImage } from "../../graphics/image";
+import { type InputEvent } from "../gestures";
+import { Layer, type LayerActions, type LayerContext } from "../layers";
+import { paintInputDialog } from "./input-dialog";
+import { type VoiceSendTarget } from "./voice-input";
+
+/**
+ * The phone side's handle on an open keyboard-input dialog: the phone types
+ * into it and picks where the text goes. The layer itself implements this;
+ * the controller hands it to the phone UI while the dialog is up.
+ */
+export type KeyboardInputSession = {
+ /** The send destinations, in the glasses menu's row order. */
+ readonly targets: ReadonlyArray<{ id: string; label: string }>;
+ /** Replace the message with what the phone's text field now holds. */
+ setText(text: string): void;
+ /**
+ * Send to the destination highlighted on the glasses (the entry point's
+ * default unless the user scrolled the menu). The IME's send key.
+ */
+ send(): void;
+ sendTo(targetId: string): void;
+ discard(): void;
+};
+
+export type KeyboardInputLayerOptions = {
+ actions: LayerActions;
+ /** Post-removal cleanup (also fires when the screen turns off). */
+ onClosed: () => void;
+ /** Pop this layer off the shell stack. */
+ dismiss: () => void;
+ /** Ordered send destinations shown as the first menu rows. */
+ sendTargets: VoiceSendTarget[];
+ /** Which send target is highlighted by default (entry-point dependent). */
+ defaultTargetIndex?: number;
+};
+
+/**
+ * The voice dialog's keyboard twin: the same box on the glasses, but the
+ * message is typed on the phone (its IME, opened by the keyboard button next
+ * to the mic button) instead of transcribed. The glasses mirror the text as
+ * it is typed and show the same send / discard menu, which works from either
+ * end: the ring or watch scrolls and clicks it, the phone has a button per
+ * row plus the IME's send key. There is no Continue row — the phone can
+ * simply edit the text.
+ */
+export class KeyboardInputLayer implements Layer, KeyboardInputSession {
+ private text = "";
+ private menuIndex: number;
+ private closed = false;
+
+ private readonly actions: LayerActions;
+ private readonly onClosed: () => void;
+ private readonly dismiss: () => void;
+ private readonly sendTargets: VoiceSendTarget[];
+ private readonly defaultTargetIndex: number;
+
+ constructor(options: KeyboardInputLayerOptions) {
+ this.actions = options.actions;
+ this.onClosed = options.onClosed;
+ this.dismiss = options.dismiss;
+ this.sendTargets = options.sendTargets;
+ const defaultIndex = options.defaultTargetIndex ?? 0;
+ this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
+ this.menuIndex = this.defaultTargetIndex;
+ }
+
+ get targets(): ReadonlyArray<{ id: string; label: string }> {
+ return this.sendTargets.map((target) => ({ id: target.id, label: target.label }));
+ }
+
+ setText(text: string): void {
+ if (this.closed || this.text === text) return;
+ this.text = text;
+ this.actions.requestRender();
+ }
+
+ send(): void {
+ const row = this.menuRows()[this.menuIndex];
+ const target = row?.target ?? this.sendTargets[this.defaultTargetIndex];
+ if (target) this.sendToTarget(target);
+ }
+
+ sendTo(targetId: string): void {
+ const target = this.sendTargets.find((candidate) => candidate.id === targetId);
+ if (target) this.sendToTarget(target);
+ }
+
+ discard(): void {
+ if (!this.closed) this.dismiss();
+ }
+
+ private sendToTarget(target: VoiceSendTarget): void {
+ if (this.closed) return;
+ const text = this.text.trim();
+ // An empty message has nothing to deliver; the dialog stays up for the
+ // phone to keep typing (the glasses row is drawn dim for the same reason).
+ if (!text) return;
+ this.dismiss();
+ target.onSend(text);
+ }
+
+ /** The menu rows: one per send target, then Discard. */
+ private menuRows(): Array<{ label: string; dim: boolean; target?: VoiceSendTarget }> {
+ const hasText = this.text.trim().length > 0;
+ const rows: Array<{ label: string; dim: boolean; target?: VoiceSendTarget }> = [];
+ for (const target of this.sendTargets) {
+ rows.push({ label: target.label, dim: !hasText, target });
+ }
+ rows.push({ label: "Discard", dim: false });
+ return rows;
+ }
+
+ paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
+ const image = paintBelow();
+ paintInputDialog(image, {
+ title: "Keyboard",
+ status: "Type on the phone",
+ text: this.text || "(nothing typed yet)",
+ rows: this.menuRows(),
+ selectedRow: this.menuIndex,
+ });
+ return image;
+ }
+
+ handleInput(event: InputEvent, _ctx: LayerContext): void {
+ const rowCount = this.menuRows().length;
+ switch (event.type) {
+ case "scroll-up":
+ this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
+ this.actions.requestRender();
+ return;
+ case "scroll-down":
+ this.menuIndex = (this.menuIndex + 1) % rowCount;
+ this.actions.requestRender();
+ return;
+ case "click": {
+ const row = this.menuRows()[this.menuIndex];
+ if (!row) return;
+ if (row.target) {
+ this.sendToTarget(row.target);
+ } else {
+ this.dismiss();
+ }
+ return;
+ }
+ case "double-click":
+ this.dismiss();
+ return;
+ default:
+ return;
+ }
+ }
+
+ onRemoved(): void {
+ this.closed = true;
+ this.onClosed();
+ }
+}
diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts
index 6083c34b..0c35cdf4 100644
--- a/app/ui/shell/shell.ts
+++ b/app/ui/shell/shell.ts
@@ -17,6 +17,7 @@ import {
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
+import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input";
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
@@ -153,6 +154,11 @@ export type ShellConfig = {
* when RECORD_AUDIO isn't granted yet (the phone mic is the source there).
*/
prepareVoiceCapture?: () => Promise;
+ /**
+ * The keyboard dialog opened (with the session the phone types into) or
+ * closed (null) by any path; the phone UI shows/hides its typing panel.
+ */
+ onKeyboardInputChanged?: (session: KeyboardInputSession | null) => void;
/** Screen on/off changed: the controller blanks/unblanks the compositor. */
onScreenStateChanged: (on: boolean) => void;
/** Window registered/removed or foreground changed (persists the open-app list). */
@@ -291,6 +297,7 @@ class Shell {
// notification icons and the battery indicators.
private readonly trayIcons = new Map();
private activeVoiceLayer: VoiceInputLayer | null = null;
+ private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
@@ -548,10 +555,11 @@ class Shell {
// An open voice dialog suspends the timeout: a long dictation or refine
// has no button presses, but the screen must stay on for it. Sliding
// lastInputAtMs forward also restarts the full timeout when it closes.
- // An in-flight assistant turn suspends it for the same reason (a tool loop
- // can run for a while with no input); once the turn ends and the
- // Follow-up/Done menu is showing, the normal idle timeout resumes.
- if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
+ // The keyboard dialog likewise: typing happens on the phone, not the
+ // ring. An in-flight assistant turn suspends it for the same reason (a
+ // tool loop can run for a while with no input); once the turn ends and
+ // the Follow-up/Done menu is showing, the normal idle timeout resumes.
+ if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -644,7 +652,7 @@ class Shell {
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake("sidebar");
- if (action === "voice-input" && !this.activeVoiceLayer) {
+ if (action === "voice-input" && !this.activeVoiceLayer && !this.activeKeyboardLayer) {
if (this.assistantLayer) {
// The assistant overlay is up; a wakeword continues that conversation.
this.startAssistantFollowUp(true);
@@ -796,6 +804,7 @@ class Shell {
const foreground = this.foregroundWindow()?.windowId ?? "none";
if (!this.screenOn) return `fg=${foreground} target=screen-off`;
if (this.activeVoiceLayer) return `fg=${foreground} target=voice`;
+ if (this.activeKeyboardLayer) return `fg=${foreground} target=keyboard`;
if (!this.stack.isAtBase()) return `fg=${foreground} target=shell-overlay`;
return `fg=${foreground} target=${this.focus}`;
}
@@ -904,7 +913,7 @@ class Shell {
}
// Re-checked after the await: another path may have opened a dialog
// (or torn down the base state) while a permission prompt was up.
- if (!ready || this.activeVoiceLayer) return;
+ if (!ready || this.activeVoiceLayer || this.activeKeyboardLayer) return;
this.openVoiceDialogNow(options);
this.config.requestShellRender();
})();
@@ -1003,6 +1012,61 @@ class Shell {
this.config.requestShellRender();
}
+ /**
+ * Open the keyboard dialog: the voice dialog's typed twin, driven by the
+ * phone's keyboard button (beside the mic button, whose wakeword this
+ * mirrors: it wakes a dark screen too). Over the assistant overlay the text
+ * continues that conversation; otherwise it goes to the usual destinations
+ * with Send to Assistant highlighted. Returns the session the phone types
+ * into (the already-open one if there is one), or null while a voice
+ * dialog is up.
+ */
+ startKeyboardInput(): KeyboardInputSession | null {
+ if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
+ if (this.activeVoiceLayer) return null;
+ if (!this.screenOn) this.wake("sidebar");
+ const assistantLayer = this.assistantLayer;
+ const assistantSession = this.assistantSession;
+ let targets: VoiceSendTarget[];
+ let defaultIndex = 0;
+ if (assistantLayer && assistantSession) {
+ targets = [
+ {
+ id: "assistant",
+ label: "Send",
+ onSend: (text) => this.runAssistantTurn(assistantSession, assistantLayer, text),
+ },
+ ];
+ } else {
+ targets = this.buildVoiceSendTargets();
+ defaultIndex = Math.max(0, targets.findIndex((target) => target.id === "assistant"));
+ }
+ const layer = new KeyboardInputLayer({
+ actions: this.config.actions,
+ onClosed: () => {
+ if (this.activeKeyboardLayer === layer) {
+ this.activeKeyboardLayer = null;
+ this.config.onKeyboardInputChanged?.(null);
+ // The idle countdown restarts in full once keyboard input ends.
+ this.noteUserActivity();
+ }
+ },
+ dismiss: () => {
+ this.stack.popIfTop((top) => top === layer);
+ // A send or discard from the phone side arrives outside the input
+ // path, so nothing else asks for the repaint.
+ this.config.requestShellRender();
+ },
+ sendTargets: targets,
+ defaultTargetIndex: defaultIndex,
+ });
+ this.activeKeyboardLayer = layer;
+ this.stack.push(layer);
+ this.config.onKeyboardInputChanged?.(layer);
+ this.config.requestShellRender();
+ return layer;
+ }
+
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
}
@@ -1149,7 +1213,7 @@ class Shell {
private startAssistantFollowUp(handsFree = false): void {
const layer = this.assistantLayer;
const session = this.assistantSession;
- if (!layer || !session || this.activeVoiceLayer) return;
+ if (!layer || !session || this.activeVoiceLayer || this.activeKeyboardLayer) return;
const voice = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
diff --git a/app/ui/shell/voice-input.ts b/app/ui/shell/voice-input.ts
index bb0a16df..ed5f88c8 100644
--- a/app/ui/shell/voice-input.ts
+++ b/app/ui/shell/voice-input.ts
@@ -1,28 +1,10 @@
-import { G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
-import { wrapText, truncateText } from "../../graphics/textwrap";
-import { type BdfFont } from "../../graphics/bdffont";
-import { getDefaultSmallFont } from "../../graphics/ui-fonts";
+import { type GrayImage } from "../../graphics/image";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { refineDictation, type AnthropicStreamHandle } from "../../native/anthropic";
import { anthropicApiKeySetting } from "../dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
-import { drawSelectionHighlight } from "../menu";
-import { listRowHeight } from "../metrics";
import { Layer, type LayerActions, type LayerContext } from "../layers";
-import { MIN_WINDOW_HEIGHT, minWindowTop } from "./geometry";
-
-const DIALOG_X = 40;
-const DIALOG_W = G2_LENS_WIDTH - 80;
-// The dialog fits inside the min-height window band (like the other shell
-// overlays), wherever the vertical position setting puts it.
-const DIALOG_MARGIN_Y = 24;
-const DIALOG_H = MIN_WINDOW_HEIGHT - 2 * DIALOG_MARGIN_Y;
-const TEXT_MAX_WIDTH = DIALOG_W - 32;
-
-/** Dialog top edge; band-relative, so computed per paint. */
-function dialogY(): number {
- return minWindowTop() + DIALOG_MARGIN_Y;
-}
+import { paintInputDialog } from "./input-dialog";
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
@@ -228,48 +210,16 @@ export class VoiceInputLayer implements Layer {
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
- const font = getDefaultSmallFont();
- const menuRowH = listRowHeight(font);
const image = paintBelow();
- const top = dialogY();
-
- // Solid dialog box over the underlying UI. Fill 1, not 0: identical after
- // 4bpp quantization, but 0 is transparent on the color-key shell surface.
- image.fillRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 1, 10);
- image.drawRoundedRect(DIALOG_X, top, DIALOG_W, DIALOG_H, 90, 10);
-
- const left = DIALOG_X + 16;
- image.drawText(font, left, top + 12, this.capturing ? "Voice ●" : "Voice", 220);
- image.drawText(font, left, top + 30, truncateText(font, this.status, TEXT_MAX_WIDTH), 130);
-
const inMenu = this.phase === "menu";
- const rows = inMenu ? this.menuRows() : [];
- // Reserve space for the actual number of rows this menu has.
- const textBottom = inMenu ? top + DIALOG_H - rows.length * menuRowH - 8 : top + DIALOG_H - 8;
- const textTop = top + 56;
- const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
-
- const text = this.displayText() || this.placeholderText();
- const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
- const firstLine = Math.max(0, wrapped.length - maxLines);
- for (let index = firstLine; index < wrapped.length; index++) {
- image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
- }
-
- if (inMenu) {
- const menuTop = top + DIALOG_H - rows.length * menuRowH - 2;
- for (let i = 0; i < rows.length; i++) {
- const rowY = menuTop + i * menuRowH;
- const selected = i === this.menuIndex;
- if (selected) {
- drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, menuRowH - 2, true, 6);
- }
- const row = rows[i]!;
- image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200);
- }
- } else {
- image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110-font.lineHeight);
- }
+ paintInputDialog(image, {
+ title: this.capturing ? "Voice ●" : "Voice",
+ status: this.status,
+ text: this.displayText() || this.placeholderText(),
+ rows: inMenu ? this.menuRows() : [],
+ selectedRow: this.menuIndex,
+ hint: inMenu ? undefined : this.hintText(),
+ });
return image;
}