diff --git a/app/native/g2mirror-client.ts b/app/native/g2mirror-client.ts index 96ac05e3..05e79999 100644 --- a/app/native/g2mirror-client.ts +++ b/app/native/g2mirror-client.ts @@ -2,6 +2,7 @@ import { toUint8Array } from "../util/array-util"; declare const com: any; declare const android: any; +declare const java: any; /** * Client for the g2mirror terminal-mirroring protocol (see @@ -190,6 +191,16 @@ export class G2MirrorClient { this.send({ type: "view" }); } + /** + * Write bytes to the attached app's terminal, as a terminal emulator would + * for keystrokes. `text` is sent as UTF-8; include a trailing "\r" to submit + * (the Enter key). No-op unless a session is attached. + */ + sendInput(text: string): void { + if (this.phase !== "attached" || !text) return; + this.send({ type: "input", data: encodeBase64Utf8(text) }); + } + unview(): void { if (this.phase !== "attached") return; this.send({ type: "unview" }); @@ -357,6 +368,11 @@ function decodeBase64(data: string): Uint8Array { return toUint8Array(android.util.Base64.decode(data, android.util.Base64.DEFAULT)); } +function encodeBase64Utf8(text: string): string { + const bytes = new java.lang.String(text).getBytes("UTF-8"); + return String(android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP)); +} + function shortenError(message: string): string { const collapsed = message.replace(/\s+/g, " ").trim(); return collapsed.length <= 120 ? collapsed : `${collapsed.slice(0, 117)}...`; diff --git a/app/ui/apps/voice-input.ts b/app/ui/apps/voice-input.ts index 51496038..99cc9e47 100644 --- a/app/ui/apps/voice-input.ts +++ b/app/ui/apps/voice-input.ts @@ -1,6 +1,8 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; +import { GESTURE_DOUBLE_CLICK } from "../gestures"; +import { drawSelectionHighlight } from "../menu"; import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; const DIALOG_X = 40; @@ -8,23 +10,30 @@ const DIALOG_Y = 40; const DIALOG_W = G2_LENS_WIDTH - 80; const DIALOG_H = G2_LENS_HEIGHT - 80; const TEXT_MAX_WIDTH = DIALOG_W - 32; +const MENU_ROW_H = 20; /** * Push-to-talk voice dialog, drawn on top of whatever is already on screen. * The mic runs while the button is held (long-press); releasing it stops the - * mic and finalizes. The dialog stays up until dismissed with a double-click. + * mic and shows a Send / Discard menu. Send delivers the transcript to the + * foreground window (e.g. the terminal); Discard (or double-click) closes. */ export class VoiceInputLayer implements Layer { private status = "Listening..."; private finalizedText = ""; private liveText = ""; private capturing = false; + // Set once the button is released: the mic has stopped and the Send/Discard + // menu is active. + private finished = false; + private menuIndex = 0; private unsubscribeTranscript: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; constructor( private readonly actions: LayerActions, private readonly onClosed: () => void, + private readonly onSend: (text: string) => void, ) {} startCapture(): void { @@ -39,13 +48,14 @@ export class VoiceInputLayer implements Layer { this.actions.requestRender(); } - /** Button released: stop the mic and finalize, but keep the dialog open. */ + /** Button released: stop the mic, finalize, and show the Send/Discard menu. */ endCapture(): void { if (!this.capturing) return; this.capturing = false; + this.finished = true; void this.actions.stopVoiceCapture(); if (this.status.startsWith("Listening")) { - this.status = "Double-click to close."; + this.status = "Send or discard?"; } this.actions.requestRender(); } @@ -63,19 +73,64 @@ export class VoiceInputLayer implements Layer { image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220); image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130); + const hasText = this.displayText().trim().length > 0; + // Leave room for the two-row menu when finished. + const textBottom = this.finished ? DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8; + const textTop = DIALOG_Y + 56; + const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0); + const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)"); const wrapped = wrapText(font, text, TEXT_MAX_WIDTH); - const maxLines = 8; const firstLine = Math.max(0, wrapped.length - maxLines); for (let index = firstLine; index < wrapped.length; index++) { - image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235); + image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235); + } + + if (this.finished) { + const labels = ["Send", "Discard"]; + const menuTop = DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 2; + for (let i = 0; i < labels.length; i++) { + const rowY = menuTop + i * MENU_ROW_H; + const selected = i === this.menuIndex; + if (selected) { + drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6); + } + // Dim "Send" when there is nothing to send. + const dim = i === 0 && !hasText; + image.drawText(font, left + 4, rowY + 2, labels[i]!, dim ? 90 : selected ? 255 : 200); + } + } else { + image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} close`, 110); } return image; } handleInput(event: DashboardInputEvent, ctx: LayerContext): void { - if (event.type === "double-click") { - ctx.stack.pop(); + if (!this.finished) { + if (event.type === "double-click") { + ctx.stack.pop(); + } + return; + } + switch (event.type) { + case "scroll-up": + case "scroll-down": + // Two options; either scroll direction toggles the selection. + this.menuIndex = (this.menuIndex + 1) % 2; + this.actions.requestRender(); + return; + case "click": + if (this.menuIndex === 0) { + const text = this.displayText().trim(); + if (text) this.onSend(text); + } + ctx.stack.pop(); + return; + case "double-click": + ctx.stack.pop(); + return; + default: + return; } } diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 8c46e2dc..10b68685 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -42,6 +42,11 @@ export type ShellWindow = { handleInput: (event: DashboardInputEvent, frameId: number) => Promise | void; /** Repaint and resubmit this window's surface. */ requestRender: () => void; + /** + * Deliver a text string to the window (e.g. finalized voice input). Optional: + * only windows that consume typed text (the terminal) implement it. + */ + receiveTextInput?: (text: string) => void; /** Foreground state changed: this window's surface is (not) the visible one. */ setForeground?: (foreground: boolean) => void; /** Screen turned on/off; hidden or screen-off windows should stop painting. */ @@ -418,16 +423,25 @@ class Shell { } private openVoiceDialog(): void { - const layer = new VoiceInputLayer(this.config.actions, () => { - if (this.activeVoiceLayer === layer) { - this.activeVoiceLayer = null; - } - }); + const layer = new VoiceInputLayer( + this.config.actions, + () => { + if (this.activeVoiceLayer === layer) { + this.activeVoiceLayer = null; + } + }, + (text) => this.sendTextToForegroundWindow(text), + ); this.activeVoiceLayer = layer; this.stack.push(layer); layer.startCapture(); } + /** Deliver a text string to the foreground window (e.g. finalized voice input). */ + sendTextToForegroundWindow(text: string): void { + this.foregroundWindow()?.receiveTextInput?.(text); + } + private openOverlayMenu(): void { const foreground = this.foregroundWindow(); const items: MenuItem[] = [ diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index 99a0e82c..895385bf 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -13,6 +13,7 @@ export type WorkerAppMessage = | { type: "open-window"; windowId: string; surfaceId: string; viewport: { width: number; height: number } } | { type: "close-window"; windowId: string } | { type: "input"; windowId: string; event: unknown; frameId: number } + | { type: "text-input"; windowId: string; text: string } | { type: "render"; windowId: string } | { type: "foreground"; windowId: string; foreground: boolean } | { type: "screen"; on: boolean }; @@ -146,6 +147,9 @@ export class WorkerAppHost { requestRender: () => { this.post({ type: "render", windowId: spec.windowId }); }, + receiveTextInput: (text) => { + this.post({ type: "text-input", windowId: spec.windowId, text }); + }, setForeground: (foreground) => { this.options.setSurfaceVisible(surfaceId, foreground); if (foreground) { diff --git a/app/workers/terminal-app.worker.ts b/app/workers/terminal-app.worker.ts index e05d01dd..f57230b7 100644 --- a/app/workers/terminal-app.worker.ts +++ b/app/workers/terminal-app.worker.ts @@ -109,6 +109,15 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { handleInput(window, message.event as DashboardInputEvent, message.frameId); break; } + case "text-input": { + const window = windows.get(message.windowId); + // Only attached view windows have a terminal to type into. Append "\r" + // (the Enter key) so voice-dictated text is submitted as a command. + if (window && window.kind === "view") { + window.client.sendInput(`${message.text}\r`); + } + break; + } case "render": { const window = windows.get(message.windowId); if (window) renderAndSubmit(window, 0);