Add voice input support to terminal
This commit is contained in:
1 parent
b2c90bc043
commit
7ca3735c23
5 files changed
+110
-12
No files matched your search
@@ -2,6 +2,7 @@ import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* Client for the g2mirror terminal-mirroring protocol (see
|
||||
@@ -190,6 +191,16 @@ export class G2MirrorClient {
|
||||
this.send({ type: "view" });
|
||||
}
|
||||
|
||||
/**
|
||||
* Write bytes to the attached app's terminal, as a terminal emulator would
|
||||
* for keystrokes. `text` is sent as UTF-8; include a trailing "\r" to submit
|
||||
* (the Enter key). No-op unless a session is attached.
|
||||
*/
|
||||
sendInput(text: string): void {
|
||||
if (this.phase !== "attached" || !text) return;
|
||||
this.send({ type: "input", data: encodeBase64Utf8(text) });
|
||||
}
|
||||
|
||||
unview(): void {
|
||||
if (this.phase !== "attached") return;
|
||||
this.send({ type: "unview" });
|
||||
@@ -357,6 +368,11 @@ function decodeBase64(data: string): Uint8Array {
|
||||
return toUint8Array(android.util.Base64.decode(data, android.util.Base64.DEFAULT));
|
||||
}
|
||||
|
||||
function encodeBase64Utf8(text: string): string {
|
||||
const bytes = new java.lang.String(text).getBytes("UTF-8");
|
||||
return String(android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP));
|
||||
}
|
||||
|
||||
function shortenError(message: string): string {
|
||||
const collapsed = message.replace(/\s+/g, " ").trim();
|
||||
return collapsed.length <= 120 ? collapsed : `${collapsed.slice(0, 117)}...`;
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { GESTURE_DOUBLE_CLICK } from "../gestures";
|
||||
import { drawSelectionHighlight } from "../menu";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
const DIALOG_X = 40;
|
||||
@@ -8,23 +10,30 @@ const DIALOG_Y = 40;
|
||||
const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
const MENU_ROW_H = 20;
|
||||
|
||||
/**
|
||||
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
|
||||
* The mic runs while the button is held (long-press); releasing it stops the
|
||||
* mic and finalizes. The dialog stays up until dismissed with a double-click.
|
||||
* mic and shows a Send / Discard menu. Send delivers the transcript to the
|
||||
* foreground window (e.g. the terminal); Discard (or double-click) closes.
|
||||
*/
|
||||
export class VoiceInputLayer implements Layer {
|
||||
private status = "Listening...";
|
||||
private finalizedText = "";
|
||||
private liveText = "";
|
||||
private capturing = false;
|
||||
// Set once the button is released: the mic has stopped and the Send/Discard
|
||||
// menu is active.
|
||||
private finished = false;
|
||||
private menuIndex = 0;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly onClosed: () => void,
|
||||
private readonly onSend: (text: string) => void,
|
||||
) {}
|
||||
|
||||
startCapture(): void {
|
||||
@@ -39,13 +48,14 @@ export class VoiceInputLayer implements Layer {
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Button released: stop the mic and finalize, but keep the dialog open. */
|
||||
/** Button released: stop the mic, finalize, and show the Send/Discard menu. */
|
||||
endCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
this.capturing = false;
|
||||
this.finished = true;
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.status.startsWith("Listening")) {
|
||||
this.status = "Double-click to close.";
|
||||
this.status = "Send or discard?";
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
@@ -63,19 +73,64 @@ export class VoiceInputLayer implements Layer {
|
||||
image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220);
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
|
||||
|
||||
const hasText = this.displayText().trim().length > 0;
|
||||
// Leave room for the two-row menu when finished.
|
||||
const textBottom = this.finished ? DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const textTop = DIALOG_Y + 56;
|
||||
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
|
||||
|
||||
const text = this.displayText() || (this.capturing ? "Listening..." : "(no speech detected)");
|
||||
const wrapped = wrapText(font, text, TEXT_MAX_WIDTH);
|
||||
const maxLines = 8;
|
||||
const firstLine = Math.max(0, wrapped.length - maxLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
image.drawText(font, left, DIALOG_Y + 56 + (index - firstLine) * 16, wrapped[index]!, 235);
|
||||
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
|
||||
}
|
||||
|
||||
if (this.finished) {
|
||||
const labels = ["Send", "Discard"];
|
||||
const menuTop = DIALOG_Y + DIALOG_H - 2 * MENU_ROW_H - 2;
|
||||
for (let i = 0; i < labels.length; i++) {
|
||||
const rowY = menuTop + i * MENU_ROW_H;
|
||||
const selected = i === this.menuIndex;
|
||||
if (selected) {
|
||||
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
|
||||
}
|
||||
// Dim "Send" when there is nothing to send.
|
||||
const dim = i === 0 && !hasText;
|
||||
image.drawText(font, left + 4, rowY + 2, labels[i]!, dim ? 90 : selected ? 255 : 200);
|
||||
}
|
||||
} else {
|
||||
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} close`, 110);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
if (!this.finished) {
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
return;
|
||||
}
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "scroll-down":
|
||||
// Two options; either scroll direction toggles the selection.
|
||||
this.menuIndex = (this.menuIndex + 1) % 2;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "click":
|
||||
if (this.menuIndex === 0) {
|
||||
const text = this.displayText().trim();
|
||||
if (text) this.onSend(text);
|
||||
}
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
case "double-click":
|
||||
ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+19
-5
@@ -42,6 +42,11 @@ export type ShellWindow = {
|
||||
handleInput: (event: DashboardInputEvent, frameId: number) => Promise<void> | void;
|
||||
/** Repaint and resubmit this window's surface. */
|
||||
requestRender: () => void;
|
||||
/**
|
||||
* Deliver a text string to the window (e.g. finalized voice input). Optional:
|
||||
* only windows that consume typed text (the terminal) implement it.
|
||||
*/
|
||||
receiveTextInput?: (text: string) => void;
|
||||
/** Foreground state changed: this window's surface is (not) the visible one. */
|
||||
setForeground?: (foreground: boolean) => void;
|
||||
/** Screen turned on/off; hidden or screen-off windows should stop painting. */
|
||||
@@ -418,16 +423,25 @@ class Shell {
|
||||
}
|
||||
|
||||
private openVoiceDialog(): void {
|
||||
const layer = new VoiceInputLayer(this.config.actions, () => {
|
||||
if (this.activeVoiceLayer === layer) {
|
||||
this.activeVoiceLayer = null;
|
||||
}
|
||||
});
|
||||
const layer = new VoiceInputLayer(
|
||||
this.config.actions,
|
||||
() => {
|
||||
if (this.activeVoiceLayer === layer) {
|
||||
this.activeVoiceLayer = null;
|
||||
}
|
||||
},
|
||||
(text) => this.sendTextToForegroundWindow(text),
|
||||
);
|
||||
this.activeVoiceLayer = layer;
|
||||
this.stack.push(layer);
|
||||
layer.startCapture();
|
||||
}
|
||||
|
||||
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
|
||||
sendTextToForegroundWindow(text: string): void {
|
||||
this.foregroundWindow()?.receiveTextInput?.(text);
|
||||
}
|
||||
|
||||
private openOverlayMenu(): void {
|
||||
const foreground = this.foregroundWindow();
|
||||
const items: MenuItem[] = [
|
||||
|
||||
@@ -13,6 +13,7 @@ export type WorkerAppMessage =
|
||||
| { type: "open-window"; windowId: string; surfaceId: string; viewport: { width: number; height: number } }
|
||||
| { type: "close-window"; windowId: string }
|
||||
| { type: "input"; windowId: string; event: unknown; frameId: number }
|
||||
| { type: "text-input"; windowId: string; text: string }
|
||||
| { type: "render"; windowId: string }
|
||||
| { type: "foreground"; windowId: string; foreground: boolean }
|
||||
| { type: "screen"; on: boolean };
|
||||
@@ -146,6 +147,9 @@ export class WorkerAppHost {
|
||||
requestRender: () => {
|
||||
this.post({ type: "render", windowId: spec.windowId });
|
||||
},
|
||||
receiveTextInput: (text) => {
|
||||
this.post({ type: "text-input", windowId: spec.windowId, text });
|
||||
},
|
||||
setForeground: (foreground) => {
|
||||
this.options.setSurfaceVisible(surfaceId, foreground);
|
||||
if (foreground) {
|
||||
|
||||
@@ -109,6 +109,15 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
handleInput(window, message.event as DashboardInputEvent, message.frameId);
|
||||
break;
|
||||
}
|
||||
case "text-input": {
|
||||
const window = windows.get(message.windowId);
|
||||
// Only attached view windows have a terminal to type into. Append "\r"
|
||||
// (the Enter key) so voice-dictated text is submitted as a command.
|
||||
if (window && window.kind === "view") {
|
||||
window.client.sendInput(`${message.text}\r`);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "render": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (window) renderAndSubmit(window, 0);
|
||||
|
||||
Reference in new issue
Block a user