// The calculator app layer for FaceClaw: a spoken (or typed) problem in, an // exact answer on the lens. // // Deliberately thin. Everything mathematical belongs to math/ and everything // about the standing problem, follow-ups, and long runs belongs to // MathCoordinator. This layer owns only what is genuinely the app's: which // mode a fresh capture lands in, what the viewport shows, and how the // touchpad and the microphone are shared. import { GrayImage, type UiFont } from "../../graphics/image"; import { getDefaultLargeFont, getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts"; import { wrapText, truncateText } from "../../graphics/textwrap"; import { lineStep } from "../../ui/metrics"; import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures"; import { type Layer, type LayerActions, type LayerContext } from "../../ui/layers"; import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings"; import { shell } from "../../ui/shell/shell"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; import { voiceActivity } from "../../ui/shell/voice-activity"; import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator"; import { type LongRunOffer, type LongRunProgress, offerGlassesPrompt, progressText } from "./math/workload"; import { plain } from "./math/text-renderer"; import { drawGraph } from "./calculator-graph"; import { type CalculatorListening, type CalculatorMode, CALCULATOR_MODES, cycleMode, isDictationCancellation, isWorthAnswering, listeningLabel, modeCommand, modeLabel } from "./calculator-commands"; // --------------------------------------------------------------------------- // Settings /** The mode a fresh capture lands in. Also what scrolling cycles. */ export const calculatorModeSetting = new ConfigSettingEnum({ id: "calculator-mode", label: "Default mode", storageKey: "calculator.defaultMode", defaultValue: "solve", values: [...CALCULATOR_MODES], formatValue: modeLabel, description: "What a freshly stated problem does. On the glasses, scroll applies another mode — or walks Explain steps — and tap toggles the mic.", }); export const calculatorListeningSetting = new ConfigSettingEnum({ id: "calculator-listening", label: "Listening", storageKey: "calculator.listening", defaultValue: "tap", values: ["tap", "continuous"], formatValue: listeningLabel, description: "Tap to talk captures one problem per tap. Continuous listens the whole time the calculator is in front, answering anything that sounds like maths.", }); export const calculatorDegreesSetting = new ConfigSettingBoolean({ id: "calculator-degrees", label: "Trigonometry in degrees", storageKey: "calculator.usesDegrees", defaultValue: false, description: "Evaluate sin/cos/tan in degrees instead of radians.", }); // The calculator's id for shared continuous capture, and its claim on the mic // while a dictation is armed. const CONTINUOUS_CLIENT = "calculator"; const DICTATION_HOLDER = "calculator-dictation"; // --------------------------------------------------------------------------- // Layout /** * The column the view is laid out in: the window's width inside the padding * suggested for its side edges (viewport.ts). Tabs, text, the plot and the * footer all start at its left; centred text is centred in it. */ type Column = { x: number; width: number }; /** Below the tabs' text top: their underline, and where the body starts. */ const TAB_UNDERLINE_OFFSET = 20; const BODY_TOP_OFFSET = 31; /** Indent, on both sides, of wrapped body text (a failure's reason, a step's detail, an answer). */ const BODY_INDENT = 8; /** The same for the prompt of an empty view, which keeps well clear of the sides. */ const PROMPT_INDENT = 24; // --------------------------------------------------------------------------- // Layer export class CalculatorLayer implements Layer { /** Rebound by the window factory once the window exists. */ requestRender: () => void = () => {}; /** * The last thing said to the calculator, so the view can show what it * heard next to what it made of it — the only way a wearer can tell a * mis-heard problem from a wrong answer. */ private lastInput = ""; private lastAnswer = ""; private lastFailure: string | null = null; private working = false; /** True while one finalized utterance is armed for capture. */ private dictating = false; private foreground = true; private continuousActive = false; private unsubscribeTranscript: (() => void) | null = null; private lastDeepAnswer: MathBranchOutcome | null = null; constructor( private readonly coordinator: MathCoordinator, private readonly actions: LayerActions, private readonly textInput?: () => Promise, ) { this.coordinator.setUsesDegrees(calculatorDegreesSetting.get()); this.coordinator.onChanged = () => this.handleCoordinatorChanged(); if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); } private get mode(): CalculatorMode { return calculatorModeSetting.get(); } private get listening(): CalculatorListening { return calculatorListeningSetting.get(); } // ------------------------------------------------------------------------- // Lifecycle onRemoved(): void { this.stopListening(); this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.coordinator.onChanged = null; } /** Foreground/background transitions, forwarded by the window factory. */ setForeground(foreground: boolean): void { if (this.foreground === foreground) return; this.foreground = foreground; if (!foreground) { this.stopListening(); } else { this.refreshListening(); } } /** Apply a settings change: degrees and the listening mode both act immediately. */ applySettings(): void { this.coordinator.setUsesDegrees(calculatorDegreesSetting.get()); this.refreshListening(); this.requestRender(); } private refreshListening(): void { if (this.textInput) return; if (!this.foreground) return; if (this.listening === "continuous") { if (!this.continuousActive) { this.continuousActive = true; void this.actions.startContinuousVoiceCapture(CONTINUOUS_CLIENT); } } else if (this.continuousActive) { this.continuousActive = false; void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT); } } private stopListening(): void { if (this.dictating) { void this.actions.stopVoiceCapture(); this.setDictating(false); } if (this.continuousActive) { this.continuousActive = false; void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT); } } // ------------------------------------------------------------------------- // Asking /** * Put a problem (or a follow-up command) to the calculator. * * Command first, problem second. "graph it" contains no operator and would * fail as an expression, and a wearer who says it expects the plot of what * is already on the lens rather than an error about the word "it". */ submit(text: string): void { const trimmed = text.trim(); if (!trimmed) return; this.lastInput = trimmed; this.lastFailure = null; const selected = modeCommand(trimmed); if (selected) { this.performFollowUp(selected); return; } const target = this.mode; this.working = true; this.requestRender(); void this.coordinator.answerSpoken(trimmed).then((outcome) => { // A fresh problem lands in the app's mode: solving is the default, but // a wearer whose default is "graph" said the problem in order to see it // plotted, not to read its root. if (outcome.kind === "value" && target !== "solve") { outcome = this.coordinator.run(target, ""); } this.working = false; this.applyOutcome(outcome); }); } /** * Re-run the standing problem through a different presentation branch. * This is the one action used by scroll cycling, spoken "… it" commands, * and the window menu, so changing mode can never mean "change a setting * but leave the old answer up". */ performFollowUp(selected: CalculatorMode): void { calculatorModeSetting.set(selected); if (!this.coordinator.hasProblem) { this.lastFailure = "No problem yet"; this.requestRender(); return; } this.working = false; this.lastFailure = null; this.applyOutcome(this.coordinator.run(selected, "")); } clear(): void { this.working = false; this.lastInput = ""; this.lastAnswer = ""; this.lastFailure = null; this.coordinator.handleRingAction("exit"); this.requestRender(); } private applyOutcome(outcome: MathBranchOutcome | null): void { if (outcome?.kind === "value") { this.lastAnswer = outcome.text; this.lastFailure = null; } else if (outcome?.kind === "failure") { this.lastFailure = outcome.reason; } this.requestRender(); } private handleCoordinatorChanged(): void { // The answer a deeper pass produced arrives on its own channel; fold it // into the standing answer the moment it lands. const deep = this.coordinator.deepAnswer; if (deep && deep !== this.lastDeepAnswer) { this.lastDeepAnswer = deep; this.applyOutcome(deep); return; } this.requestRender(); } // ------------------------------------------------------------------------- // Dictation /** * Arm exactly one finalized utterance. One, not a listening window: the * indicator on the lens is what makes this honest, and a calculator that * quietly held the microphone open would be indistinguishable from one * that did not. */ beginDictation(): void { if (this.textInput) { void this.textInput(); return; } if (this.dictating) return; this.setDictating(true); void this.actions.startVoiceCapture(true); this.requestRender(); } /** * Arm or disarm dictation, holding the mic claim for it as the voice dialog * does: the Microphones app and EvenHub mic apps stand down meanwhile (in * 4-mic mode their stream would otherwise reach the recognizer). */ private setDictating(dictating: boolean): void { this.dictating = dictating; voiceActivity.setActive(dictating, DICTATION_HOLDER); } cancelDictation(): void { if (!this.dictating) return; void this.actions.stopVoiceCapture(); this.setDictating(false); this.requestRender(); } /** * A finalized utterance reached the app. * * In tap mode this only ever arrives because the wearer armed it, so it is * always taken. In continuous mode it also arrives from ordinary room * speech, and there it has to clear the looks-like-maths gate first — * otherwise every sentence in the room becomes a failed calculation on the * lens. */ private onTranscript(event: VoiceTranscriptEvent): void { if (!event.isFinal) return; const text = event.text.trim(); if (!text) return; const armed = this.dictating; const continuous = this.continuousActive && this.foreground; if (!armed && !continuous) return; if (armed) { void this.actions.stopVoiceCapture(); this.setDictating(false); } // A standing offer or a run in flight owns the next yes/no said in the // room ("keep going", "never mind", "stop"). const longRunAnswer = this.coordinator.handleLongRunUtterance(text); if (longRunAnswer) { this.applyOutcome(longRunAnswer); return; } if (armed && isDictationCancellation(text)) { this.requestRender(); return; } if (!armed && !isWorthAnswering(text)) return; this.submit(text); } // ------------------------------------------------------------------------- // Input receiveTextInput(text: string): void { this.submit(text); } handleInput(event: InputEvent, _ctx: LayerContext): void { switch (event.type) { case "scroll-up": case "scroll-down": { const forward = event.type === "scroll-down"; // While Explain is showing a step sequence the scroll walks it — the // coordinator owns that, and asking it first is what keeps one // gesture from meaning two things at once. Otherwise scroll cycles // the MODE, because the mode is what the wearer changes between // problems and the steps only exist inside one of the three modes. if (this.mode === "explain" && this.coordinator.stepPosition !== null) { this.applyOutcome(this.coordinator.handleRingAction(forward ? "forward" : "back")); return; } const next = cycleMode(this.mode, forward); if (this.coordinator.hasProblem) { this.performFollowUp(next); } else { calculatorModeSetting.set(next); this.requestRender(); } return; } case "click": { // A standing offer owns the tap: accept and start the long run. if (this.coordinator.hasLongRunInteraction) { this.applyOutcome(this.coordinator.handleRingAction("commit")); return; } if (this.dictating) { this.cancelDictation(); } else { this.beginDictation(); } return; } case "double-click": { if (this.dictating) { this.cancelDictation(); return; } if (this.coordinator.isRunningLong || this.coordinator.pendingLongRun) { // Stop the run, or turn the offer down; the shallow answer stands. this.applyOutcome(this.coordinator.handleRingAction("exit")); return; } // The standard leave-the-app gesture. shell.yieldFocusToSidebar(); return; } default: return; } } // ------------------------------------------------------------------------- // Paint paint(ctx: LayerContext): GrayImage { const { width, height, padding } = ctx.stack.getBaseSize(); const image = new GrayImage(width, height, 0); const small = getDefaultSmallFont(); const column: Column = { x: padding.left, width: Math.max(0, width - padding.left - padding.right) }; // The tabs sit at the top edge's padding and the footer at the bottom's; // the body is what lies between. this.paintTabs(image, small, column, padding.top); const footerY = height - padding.bottom - small.lineHeight; const bodyTop = padding.top + BODY_TOP_OFFSET; let bodyBottom = footerY - 4; // A standing offer or a running deep pass takes the rows just above the // footer, whatever view is behind it. const offer = this.coordinator.pendingLongRun; const progress = this.coordinator.longRunProgress; if (offer || progress) { bodyBottom = this.paintLongRunPanel(image, small, column, bodyBottom, offer, progress); } if (this.working) { this.drawCentered(image, small, column, Math.round((bodyTop + bodyBottom) / 2) - 8, "Working…", 180); } else if (this.lastFailure !== null) { this.paintFailure(image, small, column, bodyTop, bodyBottom); } else if (this.mode === "explain" && this.coordinator.stepPosition !== null) { this.paintExplain(image, column, bodyTop, bodyBottom); } else if (this.mode === "graph" && this.coordinator.lastGraph) { this.paintGraph(image, small, column, bodyTop, bodyBottom); } else if (this.lastAnswer) { this.paintAnswer(image, column, bodyTop, bodyBottom); } else { this.paintEmpty(image, small, column, bodyTop, bodyBottom); } image.drawText(small, column.x, footerY, this.footerHints(), 110); return image; } /** The three modes as tabs from `y` down, plus the deg/listening status chips. */ private paintTabs(image: GrayImage, font: UiFont, column: Column, y: number): void { let x = column.x; const underlineY = y + TAB_UNDERLINE_OFFSET; for (const mode of CALCULATOR_MODES) { const label = modeLabel(mode); const active = mode === this.mode; image.drawText(font, x, y, label, active ? 245 : 105); image.drawLine(x, underlineY, x + font.measureText(label), underlineY, active ? 200 : 30); x += font.measureText(label) + 18; } const chips: string[] = []; if (calculatorDegreesSetting.get()) chips.push("deg"); if (this.dictating) chips.push("listening…"); else if (this.continuousActive && this.foreground) chips.push("mic on"); if (chips.length > 0) { const text = chips.join(" · "); image.drawText(font, column.x + column.width - font.measureText(text), y, text, this.dictating ? 220 : 110); } } private paintFailure( image: GrayImage, font: UiFont, column: Column, bodyTop: number, bodyBottom: number, ): void { this.paintHeardLine(image, font, column, bodyTop); const top = this.lastInput ? bodyTop + lineStep(font) + 6 : bodyTop + 8; const lines = wrapText(font, this.lastFailure ?? "", column.width - 2 * BODY_INDENT); const step = lineStep(font); const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step)); for (let index = 0; index < Math.min(lines.length, maxLines); index++) { image.drawText(font, column.x + BODY_INDENT, top + index * step, lines[index]!, 200); } } private paintExplain(image: GrayImage, column: Column, bodyTop: number, bodyBottom: number): void { const small = getDefaultSmallFont(); const medium = getDefaultMediumFont(); const position = this.coordinator.stepPosition; const steps = this.coordinator.explanationSteps; const step = position ? steps[position.index] : undefined; if (!position || !step) return; let y = bodyTop + 2; image.drawText(small, column.x, y, `Step ${position.index + 1} of ${position.total}`, 130); y += lineStep(small) + 2; image.drawText(medium, column.x, y, truncateText(medium, step.title, column.width), 245); y += lineStep(medium) + 2; image.drawText(medium, column.x, y, truncateText(medium, plain(step.expression), column.width), 220); y += lineStep(medium) + 6; const detailLines = wrapText(small, step.detail, column.width - 2 * BODY_INDENT); const stepPitch = lineStep(small); const maxLines = Math.max(0, Math.floor((bodyBottom - y) / stepPitch)); for (let index = 0; index < Math.min(detailLines.length, maxLines); index++) { image.drawText(small, column.x + BODY_INDENT, y + index * stepPitch, detailLines[index]!, 150); } } private paintGraph( image: GrayImage, font: UiFont, column: Column, bodyTop: number, bodyBottom: number, ): void { const graph = this.coordinator.lastGraph; if (!graph) return; const headline = this.lastAnswer.split("\n")[0] ?? ""; image.drawText(font, column.x, bodyTop, truncateText(font, headline, column.width), 200); // The plot's frame spans the column, so it meets the side padding itself. const plotTop = bodyTop + lineStep(font) + 4; drawGraph(image, graph, { x: column.x, y: plotTop, width: column.width, height: Math.max(2, bodyBottom - plotTop), }); } private paintAnswer(image: GrayImage, column: Column, bodyTop: number, bodyBottom: number): void { const small = getDefaultSmallFont(); this.paintHeardLine(image, small, column, bodyTop); const top = this.lastInput ? bodyTop + lineStep(small) + 4 : bodyTop; // A short single-line answer earns the large font; anything longer drops // to the medium font and wraps. const lines = this.lastAnswer.split("\n"); const large = getDefaultLargeFont(); const maxWidth = column.width - 2 * BODY_INDENT; if (lines.length === 1 && large.measureText(lines[0]!) <= maxWidth) { const y = Math.round((top + bodyBottom - large.lineHeight) / 2); this.drawCentered(image, large, column, y, lines[0]!, 245); return; } const medium = getDefaultMediumFont(); const wrapped = lines.flatMap((line) => wrapText(medium, line, maxWidth)); const step = lineStep(medium); const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step)); const shown = wrapped.slice(0, maxLines); let y = Math.max(top, Math.round((top + bodyBottom - shown.length * step) / 2)); for (const line of shown) { this.drawCentered(image, medium, column, y, line, 235); y += step; } } private paintEmpty( image: GrayImage, font: UiFont, column: Column, bodyTop: number, bodyBottom: number, ): void { const prompt = this.textInput ? "Tap to type a problem on your phone" : this.listening === "continuous" ? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"` : "Tap to say a problem — or use Voice input to type one"; const lines = wrapText(font, prompt, column.width - 2 * PROMPT_INDENT); const step = lineStep(font); let y = Math.round((bodyTop + bodyBottom - lines.length * step) / 2); for (const line of lines) { this.drawCentered(image, font, column, y, line, 150); y += step; } } private paintHeardLine(image: GrayImage, font: UiFont, column: Column, y: number): void { if (!this.lastInput) return; const label = this.textInput ? "Entered" : "Heard"; image.drawText(font, column.x, y, label, 110); const x = column.x + font.measureText(label) + 10; image.drawText(font, x, y, truncateText(font, this.lastInput, column.x + column.width - x), 160); } /** * The standing offer, or the live progress of an accepted run, in the two * rows above the footer, under a rule across the column. Returns the new * body bottom. */ private paintLongRunPanel( image: GrayImage, font: UiFont, column: Column, bodyBottom: number, offer: LongRunOffer | null, progress: LongRunProgress | null, ): number { const step = lineStep(font); const lines = progress ? [`${progressText(progress)} · ${Math.round(progress.elapsedSeconds)}s`] : offer ? offerGlassesPrompt(offer).split("\n") : []; const panelTop = bodyBottom - lines.length * step - 6; if (column.width > 0) image.drawLine(column.x, panelTop, column.x + column.width - 1, panelTop, 40); for (let index = 0; index < lines.length; index++) { image.drawText(font, column.x, panelTop + 4 + index * step, truncateText(font, lines[index]!, column.width), 190); } return panelTop - 2; } private footerHints(): string { if (this.coordinator.longRunProgress) { return gestureHints([[GESTURE_DOUBLE_CLICK, "stop"]]); } if (this.coordinator.pendingLongRun) { return gestureHints([ [GESTURE_CLICK, "keep going"], [GESTURE_DOUBLE_CLICK, "leave it"], ]); } if (this.dictating) { return gestureHints([ [GESTURE_CLICK, "cancel mic"], [GESTURE_DOUBLE_CLICK, "cancel"], ]); } const scrollTarget = this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode"; return gestureHints([ [GESTURE_SCROLL, scrollTarget], [GESTURE_CLICK, this.textInput ? "type" : "mic"], [GESTURE_DOUBLE_CLICK, "back"], ]); } private drawCentered(image: GrayImage, font: UiFont, column: Column, y: number, text: string, value: number): void { const x = column.x + Math.max(0, Math.round((column.width - font.measureText(text)) / 2)); image.drawText(font, x, y, text, value); } }