Add voice input support to terminal

This commit is contained in:
James Babcock committed 2026-07-13 00:27:03 -07:00
1 parent b2c90bc043
commit 7ca3735c23
5 files changed
+110 -12

No files matched your search

+16
View File
@@ -2,6 +2,7 @@ import { toUint8Array } from "../util/array-util";
declare const com: any;
declare const android: any;
declare const java: any;
/**
* Client for the g2mirror terminal-mirroring protocol (see
@@ -190,6 +191,16 @@ export class G2MirrorClient {
this.send({ type: "view" });
}
/**
* Write bytes to the attached app's terminal, as a terminal emulator would
* for keystrokes. `text` is sent as UTF-8; include a trailing "\r" to submit
* (the Enter key). No-op unless a session is attached.
*/
sendInput(text: string): void {
if (this.phase !== "attached" || !text) return;
this.send({ type: "input", data: encodeBase64Utf8(text) });
}
unview(): void {
if (this.phase !== "attached") return;
this.send({ type: "unview" });
@@ -357,6 +368,11 @@ function decodeBase64(data: string): Uint8Array {
return toUint8Array(android.util.Base64.decode(data, android.util.Base64.DEFAULT));
}
function encodeBase64Utf8(text: string): string {
const bytes = new java.lang.String(text).getBytes("UTF-8");
return String(android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP));
}
function shortenError(message: string): string {
const collapsed = message.replace(/\s+/g, " ").trim();
return collapsed.length <= 120 ? collapsed : `${collapsed.slice(0, 117)}...`;
+62 -7
View File
@@ -1,6 +1,8 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { GESTURE_DOUBLE_CLICK } from "../gestures";
import { drawSelectionHighlight } from "../menu";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
const DIALOG_X = 40;
@@ -8,23 +10,30 @@ const DIALOG_Y = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
/**
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
* The mic runs while the button is held (long-press); releasing it stops the
* mic and finalizes. The dialog stays up until dismissed with a double-click.
* mic and shows a Send / Discard menu. Send delivers the transcript to the
* foreground window (e.g. the terminal); Discard (or double-click) closes.
*/
export class VoiceInputLayer implements Layer {
private status = "Listening...";
private finalizedText = "";
private liveText = "";
private capturing = false;
// Set once the button is released: the mic has stopped and the Send/Discard
// menu is active.
private finished = false;
private menuIndex = 0;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
constructor(
private readonly actions: LayerActions,
private readonly onClosed: () => void,
private readonly onSend: (text: string) => void,
) {}
startCapture(): void {
@@ -39,13 +48,14 @@ export class VoiceInputLayer implements Layer {
this.actions.requestRender();
}
/** Button released: stop the mic and finalize, but keep the dialog open. */
/** Button released: stop the mic, finalize, and show the Send/Discard menu. */
endCapture(): void {
if (!this.capturing) return;
this.capturing = false;
this.finished = true;
void this.actions.stopVoiceCapture();
if (this.status.startsWith("Listening")) {
this.status = "Double-click to close.";
this.status = "Send or discard?";
}
this.actions.requestRender();
}
@@ -63,19 +73,64 @@ export class VoiceInputLayer implements Layer {
image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220);
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
const hasText = this.displayText().trim().length > 0;
// Leave room for the two-row menu when finished.
const textBottom = this.finished ? DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)");
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
const maxLines = 8;
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235);
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (this.finished) {
const labels = ["Send", "Discard"];
const menuTop = DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 2;
for (let i = 0; i < labels.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
}
// Dim "Send" when there is nothing to send.
const dim = i === 0 && !hasText;
image.drawText(font, left + 4, rowY + 2, labels[i]!, dim ? 90 : selected ? 255 : 200);
}
} else {
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} close`, 110);
}
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "double-click") {
ctx.stack.pop();
if (!this.finished) {
if (event.type === "double-click") {
ctx.stack.pop();
}
return;
}
switch (event.type) {
case "scroll-up":
case "scroll-down":
// Two options; either scroll direction toggles the selection.
this.menuIndex = (this.menuIndex + 1) % 2;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
const text = this.displayText().trim();
if (text) this.onSend(text);
}
ctx.stack.pop();
return;
case "double-click":
ctx.stack.pop();
return;
default:
return;
}
}
+19 -5
View File
@@ -42,6 +42,11 @@ export type ShellWindow = {
handleInput: (event: DashboardInputEvent, frameId: number) => Promise<void> | void;
/** Repaint and resubmit this window's surface. */
requestRender: () => void;
/**
* Deliver a text string to the window (e.g. finalized voice input). Optional:
* only windows that consume typed text (the terminal) implement it.
*/
receiveTextInput?: (text: string) => void;
/** Foreground state changed: this window's surface is (not) the visible one. */
setForeground?: (foreground: boolean) => void;
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
@@ -418,16 +423,25 @@ class Shell {
}
private openVoiceDialog(): void {
const layer = new VoiceInputLayer(this.config.actions, () => {
if (this.activeVoiceLayer === layer) {
this.activeVoiceLayer = null;
}
});
const layer = new VoiceInputLayer(
this.config.actions,
() => {
if (this.activeVoiceLayer === layer) {
this.activeVoiceLayer = null;
}
},
(text) => this.sendTextToForegroundWindow(text),
);
this.activeVoiceLayer = layer;
this.stack.push(layer);
layer.startCapture();
}
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
sendTextToForegroundWindow(text: string): void {
this.foregroundWindow()?.receiveTextInput?.(text);
}
private openOverlayMenu(): void {
const foreground = this.foregroundWindow();
const items: MenuItem[] = [
+4
View File
@@ -13,6 +13,7 @@ export type WorkerAppMessage =
| { type: "open-window"; windowId: string; surfaceId: string; viewport: { width: number; height: number } }
| { type: "close-window"; windowId: string }
| { type: "input"; windowId: string; event: unknown; frameId: number }
| { type: "text-input"; windowId: string; text: string }
| { type: "render"; windowId: string }
| { type: "foreground"; windowId: string; foreground: boolean }
| { type: "screen"; on: boolean };
@@ -146,6 +147,9 @@ export class WorkerAppHost {
requestRender: () => {
this.post({ type: "render", windowId: spec.windowId });
},
receiveTextInput: (text) => {
this.post({ type: "text-input", windowId: spec.windowId, text });
},
setForeground: (foreground) => {
this.options.setSurfaceVisible(surfaceId, foreground);
if (foreground) {
+9
View File
@@ -109,6 +109,15 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
handleInput(window, message.event as DashboardInputEvent, message.frameId);
break;
}
case "text-input": {
const window = windows.get(message.windowId);
// Only attached view windows have a terminal to type into. Append "\r"
// (the Enter key) so voice-dictated text is submitted as a command.
if (window && window.kind === "view") {
window.client.sendInput(`${message.text}\r`);
}
break;
}
case "render": {
const window = windows.get(message.windowId);
if (window) renderAndSubmit(window, 0);