import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image"; import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; import { refineDictation, type AnthropicStreamHandle } from "../../native/anthropic"; import { anthropicApiKeySetting } from "../dashboard-settings"; import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints } from "../gestures"; import { drawSelectionHighlight } from "../menu"; import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers"; const DIALOG_X = 40; const DIALOG_Y = 40; const DIALOG_W = G2_LENS_WIDTH - 80; const DIALOG_H = G2_LENS_HEIGHT - 80; const TEXT_MAX_WIDTH = DIALOG_W - 32; const MENU_ROW_H = 20; const MENU_ROWS = 3; // After the mic stops, the provider's final transcript can trail in (cloud // commit round-trip); wait this long for it before refining with what we have. const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200; /** * The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or * click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing" * records a follow-up utterance (always click-to-finish); "refining" streams * the LLM-merged text, then returns to "menu". */ type VoicePhase = "capturing" | "menu" | "continuing" | "refining"; /** * Push-to-talk voice dialog, drawn on top of whatever is already on screen. * The mic runs while the button is held (long-press); releasing it stops the * mic and shows a Send / Continue / Discard menu. Send delivers the transcript * to the foreground window (e.g. the terminal); Discard (or double-click) * closes. Continue records another utterance — extra content or a spoken edit * ("change X to Y", "insert ... after ...") — and merges it into the message * with an LLM, landing back on the same menu. * * When opened from a menu instead of a held button (finishOnClick), there is * no release event to end the capture, so a single click stops the mic. * * When opened by the "Hey Even" wakeword (handsFree), there is likewise no * button involved: the mic starts immediately and stops when the speaker does, * via endpoint detection on the decoded PCM. A click still ends it early. */ export class VoiceInputLayer implements Layer { private phase: VoicePhase = "capturing"; private status = "Listening..."; // The active utterance. displayText() is what the dialog shows and what // Send delivers; the refine flow also writes the merged result here. private finalizedText = ""; private liveText = ""; // Snapshot of the message when Continue starts, so a failed or cancelled // continuation can fall back to it. private baseText = ""; /** Whether the mic is running (capture bookkeeping, not UI state). */ private capturing = false; /** Continuation stopped; waiting for the trailing final transcript. */ private followupFinalizeTimer: ReturnType | null = null; private refineHandle: AnthropicStreamHandle | null = null; private menuIndex = 0; private unsubscribeTranscript: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; private unsubscribeSpeechEnd: (() => void) | null = null; constructor( private readonly actions: LayerActions, private readonly onClosed: () => void, private readonly onSend: (text: string) => void, private readonly finishOnClick = false, private readonly handsFree = false, ) {} startCapture(): void { if (this.capturing) return; this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { // The refine stage owns the status line ("Refining...", error text). if (this.phase === "refining") return; this.status = state.status; this.actions.requestRender(); }); if (this.handsFree) { // No button is held, so the mic has to stop itself. endCapture() is // idempotent, and a click still ends the utterance early. this.unsubscribeSpeechEnd = voiceControlBridge.onSpeechEnd(() => { if (this.phase === "capturing") { this.endCapture(); } }); } this.capturing = true; void this.actions.startVoiceCapture(this.handsFree); this.actions.requestRender(); } /** * Whether a single click ends the utterance. True when there is no held * button to release: opened from the overlay menu (finishOnClick), or * hands-free from the wakeword — where it also lets the user cut the * silence-detection wait short. */ private get clickEndsCapture(): boolean { return this.finishOnClick || this.handsFree; } /** Button released (or click when clickEndsCapture): stop the mic. */ endCapture(): void { if (!this.capturing) return; if (this.phase === "continuing") { this.endContinuationCapture(); return; } this.capturing = false; this.phase = "menu"; void this.actions.stopVoiceCapture(); if (this.status.startsWith("Listening")) { this.status = "Send, continue, or discard?"; } this.actions.requestRender(); } paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage { const font = getDefaultSmallFont(); const image = paintBelow(); // Solid dialog box over the underlying UI. Fill 1, not 0: identical after // 4bpp quantization, but 0 is transparent on the color-key shell surface. image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10); image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10); const left = DIALOG_X + 16; image.drawText(font, left, DIALOG_Y + 12, this.capturing ? "Voice ●" : "Voice", 220); image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130); const inMenu = this.phase === "menu"; const hasText = this.displayText().trim().length > 0; // Leave room for the three-row menu when it is showing. const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8; const textTop = DIALOG_Y + 56; const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0); const text = this.displayText() || this.placeholderText(); const wrapped = wrapText(font, text, TEXT_MAX_WIDTH); const firstLine = Math.max(0, wrapped.length - maxLines); for (let index = firstLine; index < wrapped.length; index++) { image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235); } if (inMenu) { const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0; const rows: Array<{ label: string; dim: boolean }> = [ // Dim "Send" when there is nothing to send. { label: "Send", dim: !hasText }, { label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey }, { label: "Discard", dim: false }, ]; const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2; for (let i = 0; i < rows.length; i++) { const rowY = menuTop + i * MENU_ROW_H; const selected = i === this.menuIndex; if (selected) { drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6); } const row = rows[i]!; image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200); } } else { image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, this.hintText(), 110); } return image; } handleInput(event: DashboardInputEvent, ctx: LayerContext): void { switch (this.phase) { case "capturing": if (event.type === "double-click") { ctx.stack.pop(); } else if (this.clickEndsCapture && event.type === "click") { this.endCapture(); } return; case "continuing": if (event.type === "click") { this.endContinuationCapture(); } else if (event.type === "double-click") { this.cancelContinuation("Continuation cancelled"); } return; case "refining": if (event.type === "double-click") { this.cancelContinuation("Refinement cancelled"); } return; case "menu": this.handleMenuInput(event, ctx); return; } } private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void { switch (event.type) { case "scroll-up": this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS; this.actions.requestRender(); return; case "scroll-down": this.menuIndex = (this.menuIndex + 1) % MENU_ROWS; this.actions.requestRender(); return; case "click": if (this.menuIndex === 0) { const text = this.displayText().trim(); if (text) this.onSend(text); ctx.stack.pop(); } else if (this.menuIndex === 1) { if (anthropicApiKeySetting.get().trim()) { this.startContinuation(); } // No key: the row is disabled; the click does nothing. } else { ctx.stack.pop(); } return; case "double-click": ctx.stack.pop(); return; default: return; } } /** Continue selected: keep the message aside and record a follow-up. */ private startContinuation(): void { this.baseText = this.displayText().trim(); this.finalizedText = ""; this.liveText = ""; this.phase = "continuing"; this.status = "Listening..."; this.capturing = true; void this.actions.startVoiceCapture(); this.actions.requestRender(); } /** Follow-up tap: stop the mic, then refine once the transcript finalizes. */ private endContinuationCapture(): void { if (!this.capturing) return; this.capturing = false; void this.actions.stopVoiceCapture(); this.status = "Refining..."; this.actions.requestRender(); // The provider's committed transcript arrives shortly after stop; refine // when it does, or after a timeout with whatever partials we have. this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS); } private beginRefine(): void { if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; } if (this.phase !== "continuing") return; const followup = this.displayText().trim(); if (!followup) { this.backToMenu(this.baseText, "No follow-up heard"); return; } this.phase = "refining"; this.finalizedText = ""; this.liveText = ""; this.status = "Refining..."; this.actions.requestRender(); this.refineHandle = refineDictation({ apiKey: anthropicApiKeySetting.get(), original: this.baseText, followup, onTextDelta: (_delta, textSoFar) => { this.finalizedText = textSoFar; this.actions.requestRender(); }, onDone: (text) => { this.refineHandle = null; this.backToMenu(text, "Send, continue, or discard?"); }, onError: (message) => { this.refineHandle = null; this.backToMenu(this.baseText, message); }, }); } /** Abort a continuation (mic or LLM stage) and restore the prior message. */ private cancelContinuation(status: string): void { if (this.capturing) { this.capturing = false; void this.actions.stopVoiceCapture(); } this.refineHandle?.cancel(); this.refineHandle = null; this.backToMenu(this.baseText, status); } private backToMenu(text: string, status: string): void { if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; } this.finalizedText = text; this.liveText = ""; this.phase = "menu"; this.menuIndex = 0; this.status = status; this.actions.requestRender(); } onRemoved(): void { this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); this.unsubscribeStatus = null; this.unsubscribeSpeechEnd?.(); this.unsubscribeSpeechEnd = null; if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; } this.refineHandle?.cancel(); this.refineHandle = null; if (this.capturing) { this.capturing = false; void this.actions.stopVoiceCapture(); } this.onClosed(); } private displayText(): string { if (!this.finalizedText) return this.liveText; if (!this.liveText) return this.finalizedText; return `${this.finalizedText} ${this.liveText}`; } private placeholderText(): string { switch (this.phase) { case "capturing": return "Listening..."; case "continuing": return "Say more, or describe an edit..."; case "refining": return "Refining..."; case "menu": return "(no speech detected)"; } } private hintText(): string { switch (this.phase) { case "capturing": return this.clickEndsCapture ? gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "close"]]) : `${GESTURE_DOUBLE_CLICK} close`; case "continuing": return gestureHints([[GESTURE_CLICK, "done"], [GESTURE_DOUBLE_CLICK, "cancel"]]); case "refining": default: return `${GESTURE_DOUBLE_CLICK} cancel`; } } private onTranscript(event: VoiceTranscriptEvent): void { // The refine stream owns the text buffers once it starts; a transcript // that trails in after that point is stale. if (this.phase === "refining") return; if (event.isFinal) { const finalText = event.text.trim() || this.liveText.trim(); if (finalText) { this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; } this.liveText = ""; if (this.phase === "continuing" && this.followupFinalizeTimer !== null) { // The follow-up finalized; no need to keep waiting. this.beginRefine(); return; } } else { this.liveText = event.text.trim(); } this.actions.requestRender(); } } function wrapText(font: BdfFont, text: string, maxWidth: number): string[] { const words = text.split(/\s+/).filter(Boolean); const lines: string[] = []; let line = ""; for (const word of words) { const candidate = line ? `${line} ${word}` : word; if (line && font.measureText(candidate) > maxWidth) { lines.push(line); line = word; } else { line = candidate; } } if (line) lines.push(line); return lines.length ? lines : [""]; } function truncate(font: BdfFont, text: string, maxWidth: number): string { if (font.measureText(text) <= maxWidth) return text; let out = text; while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) { out = out.slice(0, -1); } return `${out}...`; }