Implement a basic voice assistant

This commit is contained in:
James Babcock committed 2026-07-25 00:44:04 -07:00
1 parent d2c987d48a
commit 0aeb24ed00
14 files changed
+1923 -80

No files matched your search

+190
View File
@@ -0,0 +1,190 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { GESTURE_DOUBLE_CLICK } from "../gestures";
import { drawSelectionHighlight } from "../menu";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
const DIALOG_X = 40;
const DIALOG_Y = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
const MENU_ROWS = 2;
/** thinking: turn running; done/error: finished, showing the Follow-up/Done menu. */
type AssistantPhase = "thinking" | "done" | "error";
export type AssistantLayerCallbacks = {
/** Record another utterance and continue this conversation. */
onFollowUp: () => void;
/** Abort the in-flight turn (double-click while thinking). */
onCancel: () => void;
/** The user closed the overlay (Done). */
onClose: () => void;
/** The layer left the stack by any path (Done, or the screen sleeping). */
onRemoved?: () => void;
};
/**
* The assistant overlay: streamed reply text with a status line for tool
* activity, and a Follow-up / Done menu once the turn ends. Double-click
* cancels an in-flight turn. The shell owns the AssistantSession and drives
* this layer's state through the on* methods as the turn streams.
*/
export class AssistantLayer implements Layer {
private phase: AssistantPhase = "thinking";
private replyText = "";
private status = "Thinking...";
private menuIndex = 0;
constructor(
private readonly actions: LayerActions,
private readonly callbacks: AssistantLayerCallbacks,
) {}
/** Shell: a new turn (initial utterance or follow-up) is starting. */
startTurn(): void {
this.phase = "thinking";
this.replyText = "";
this.status = "Thinking...";
this.menuIndex = 0;
this.actions.requestRender();
}
onTextDelta(_delta: string, textSoFar: string): void {
this.replyText = textSoFar;
if (this.phase === "thinking") this.status = "Thinking...";
this.actions.requestRender();
}
onToolActivity(label: string): void {
if (this.phase === "thinking") this.status = `→ ${label}`;
this.actions.requestRender();
}
onTurnDone(): void {
this.phase = "done";
this.status = this.replyText.trim() ? "" : "(no reply)";
this.menuIndex = 0;
this.actions.requestRender();
}
onError(message: string): void {
this.phase = "error";
this.status = message;
this.menuIndex = 0;
this.actions.requestRender();
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
// Solid dialog box (fill 1, matching the voice dialog: opaque after 4bpp
// quantization, but not the color-key transparent 0).
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, DIALOG_Y + 12, this.phase === "thinking" ? "Assistant ●" : "Assistant", 220);
if (this.status) {
const statusValue = this.phase === "error" ? 200 : 130;
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), statusValue);
}
const inMenu = this.phase !== "thinking";
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
const wrapped = wrapText(font, this.replyText, TEXT_MAX_WIDTH);
// Keep the tail visible as text streams past the bottom.
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (inMenu) {
const rows = ["Follow-up", "Done"];
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
}
image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200);
}
} else {
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110);
}
return image;
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
if (this.phase === "thinking") {
if (event.type === "double-click") {
this.callbacks.onCancel();
this.phase = "done";
this.status = "Cancelled";
this.menuIndex = 0;
this.actions.requestRender();
}
return;
}
// done / error: the Follow-up / Done menu.
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
this.callbacks.onFollowUp();
} else {
this.callbacks.onClose();
}
return;
case "double-click":
this.callbacks.onClose();
return;
default:
return;
}
}
onRemoved(): void {
this.callbacks.onRemoved?.();
}
}
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) lines.push(line);
return lines.length ? lines : [""];
}
function truncate(font: BdfFont, text: string, maxWidth: number): string {
if (font.measureText(text) <= maxWidth) return text;
let out = text;
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
out = out.slice(0, -1);
}
return `${out}...`;
}
+136 -42
View File
@@ -13,19 +13,47 @@ const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
const MENU_ROWS = 3;
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
/**
* The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or
* click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing"
* click-to-finish); "menu" is the send / Continue / Discard menu; "continuing"
* records a follow-up utterance (always click-to-finish); "refining" streams
* the LLM-merged text, then returns to "menu".
*/
type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
/**
* A destination the captured text can be sent to. The shell supplies these per
* entry point: "Send to Assistant" (hand off to the voice assistant) and/or
* "Type Into App" (deliver to the foreground window's text input).
*/
export type VoiceSendTarget = {
id: string;
label: string;
onSend: (text: string) => void;
};
export type VoiceInputLayerOptions = {
actions: LayerActions;
/** Post-removal cleanup (also fires when the screen turns off). */
onClosed: () => void;
/** Pop this layer off the shell stack. */
dismiss: () => void;
/** Ordered send destinations shown as the first menu rows. */
sendTargets: VoiceSendTarget[];
/** Which send target is highlighted by default (entry-point dependent). */
defaultTargetIndex?: number;
/** A single click ends the utterance (opened from a menu, no held button). */
finishOnClick?: boolean;
/** Wakeword flow: mic starts immediately and ends on detected silence. */
handsFree?: boolean;
/** Skip the menu: on capture end, send straight to the default target. */
autoSend?: boolean;
};
/**
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
* The mic runs while the button is held (long-press); releasing it stops the
@@ -58,17 +86,34 @@ export class VoiceInputLayer implements Layer {
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
private refineHandle: AnthropicStreamHandle | null = null;
private menuIndex = 0;
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
private pendingAutoSend = false;
private autoSendTimer: ReturnType<typeof setTimeout> | null = null;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private unsubscribeSpeechEnd: (() => void) | null = null;
constructor(
private readonly actions: LayerActions,
private readonly onClosed: () => void,
private readonly onSend: (text: string) => void,
private readonly finishOnClick = false,
private readonly handsFree = false,
) {}
private readonly actions: LayerActions;
private readonly onClosed: () => void;
private readonly dismiss: () => void;
private readonly sendTargets: VoiceSendTarget[];
private readonly defaultTargetIndex: number;
private readonly finishOnClick: boolean;
private readonly handsFree: boolean;
private readonly autoSend: boolean;
constructor(options: VoiceInputLayerOptions) {
this.actions = options.actions;
this.onClosed = options.onClosed;
this.dismiss = options.dismiss;
this.sendTargets = options.sendTargets;
this.finishOnClick = options.finishOnClick ?? false;
this.handsFree = options.handsFree ?? false;
this.autoSend = options.autoSend ?? false;
const defaultIndex = options.defaultTargetIndex ?? 0;
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
this.menuIndex = this.defaultTargetIndex;
}
startCapture(): void {
if (this.capturing) return;
@@ -113,13 +158,66 @@ export class VoiceInputLayer implements Layer {
this.capturing = false;
this.phase = "menu";
void this.actions.stopVoiceCapture();
if (this.status.startsWith("Listening")) {
if (this.autoSend && this.sendTargets.length) {
// Skip-confirmation (wakeword): send to the default target as soon as the
// transcript finalizes, or after a short wait for the trailing final.
this.pendingAutoSend = true;
this.status = "Sending...";
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
} else if (this.status.startsWith("Listening")) {
this.status = "Send, continue, or discard?";
}
this.actions.requestRender();
}
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
/** Fire the queued skip-confirmation send (or fall back to the menu). */
private performAutoSend(): void {
if (this.autoSendTimer !== null) {
clearTimeout(this.autoSendTimer);
this.autoSendTimer = null;
}
if (!this.pendingAutoSend) return;
this.pendingAutoSend = false;
const text = this.displayText().trim();
const target = this.sendTargets[this.defaultTargetIndex];
if (text && target) {
this.dismiss();
target.onSend(text);
} else {
// Nothing heard; fall back to the menu so the user can retry or discard.
this.status = "Send, continue, or discard?";
this.actions.requestRender();
}
}
/** The menu rows: one per send target, then Continue, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
rows.push({
label: target.label,
dim: !hasText,
onSelect: () => {
this.dismiss();
if (hasText) target.onSend(text);
},
});
}
rows.push({
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
},
});
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
return rows;
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
@@ -133,9 +231,9 @@ export class VoiceInputLayer implements Layer {
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
const inMenu = this.phase === "menu";
const hasText = this.displayText().trim().length > 0;
// Leave room for the three-row menu when it is showing.
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const rows = inMenu ? this.menuRows() : [];
// Reserve space for the actual number of rows this menu has.
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
@@ -147,14 +245,7 @@ export class VoiceInputLayer implements Layer {
}
if (inMenu) {
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean }> = [
// Dim "Send" when there is nothing to send.
{ label: "Send", dim: !hasText },
{ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey },
{ label: "Discard", dim: false },
];
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
const menuTop = DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
@@ -170,11 +261,11 @@ export class VoiceInputLayer implements Layer {
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
switch (this.phase) {
case "capturing":
if (event.type === "double-click") {
ctx.stack.pop();
this.dismiss();
} else if (this.clickEndsCapture && event.type === "click") {
this.endCapture();
}
@@ -192,37 +283,29 @@ export class VoiceInputLayer implements Layer {
}
return;
case "menu":
this.handleMenuInput(event, ctx);
this.handleMenuInput(event);
return;
}
}
private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void {
private handleMenuInput(event: DashboardInputEvent): void {
const rowCount = this.menuRows().length;
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
this.menuIndex = (this.menuIndex + 1) % rowCount;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
const text = this.displayText().trim();
if (text) this.onSend(text);
ctx.stack.pop();
} else if (this.menuIndex === 1) {
if (anthropicApiKeySetting.get().trim()) {
this.startContinuation();
}
// No key: the row is disabled; the click does nothing.
} else {
ctx.stack.pop();
}
case "click": {
const row = this.menuRows()[this.menuIndex];
row?.onSelect();
return;
}
case "double-click":
ctx.stack.pop();
this.dismiss();
return;
default:
return;
@@ -323,6 +406,11 @@ export class VoiceInputLayer implements Layer {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
}
if (this.autoSendTimer !== null) {
clearTimeout(this.autoSendTimer);
this.autoSendTimer = null;
}
this.pendingAutoSend = false;
this.refineHandle?.cancel();
this.refineHandle = null;
if (this.capturing) {
@@ -380,6 +468,12 @@ export class VoiceInputLayer implements Layer {
this.beginRefine();
return;
}
if (this.pendingAutoSend) {
// The utterance finalized; skip-confirmation can fire without waiting
// out the fallback timer.
this.performAutoSend();
return;
}
} else {
this.liveText = event.text.trim();
}