From 0765e943fb4d06976d082b397d3be9ac2edbd35a Mon Sep 17 00:00:00 2001 From: James Babcock Date: Tue, 8 Sep 2026 23:15:29 -0700 Subject: [PATCH] Add "AI Chat" app. --- CHANGELOG | 1 + app/apps/ai-chat/ai-chat-app.ts | 166 ++++++++++++++++++++++++++++++ app/apps/ai-chat/index.ts | 10 ++ app/apps/ai-chat/voice-draft.ts | 125 ++++++++++++++++++++++ app/apps/all-apps.ts | 2 + app/assistant/conversations.ts | 95 +++++++++++++++++ app/assistant/direct-backend.ts | 6 +- app/assistant/session.ts | 160 +++++++++++++++++++--------- app/g2/dashboard-controller.ts | 15 ++- app/graphics/icons.ts | 2 + app/ui/shell/in-process-window.ts | 16 ++- app/ui/shell/shell.ts | 97 +++++++++++------ app/ui/window-menu.ts | 8 +- 13 files changed, 616 insertions(+), 87 deletions(-) create mode 100644 app/apps/ai-chat/ai-chat-app.ts create mode 100644 app/apps/ai-chat/index.ts create mode 100644 app/apps/ai-chat/voice-draft.ts create mode 100644 app/assistant/conversations.ts diff --git a/CHANGELOG b/CHANGELOG index 8825e13a..07b5c26f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ * Transcribe app: Scrolling; automatic reconnection; insert newlines * Notifications: Add notification filters * Phone screen preview: Add rotation option; improve landscape layout +* Add "AI Chat" app (conversational-style use of the voice assistant with hold-to-talk) ## 0.6.5 - 2026-09-04 * Changed context menu gestures: — for system, ●— for app context menu. diff --git a/app/apps/ai-chat/ai-chat-app.ts b/app/apps/ai-chat/ai-chat-app.ts new file mode 100644 index 00000000..195477a2 --- /dev/null +++ b/app/apps/ai-chat/ai-chat-app.ts @@ -0,0 +1,166 @@ +import { GrayImage, type UiFont } from "../../graphics/image"; +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { truncateText, wrapText } from "../../graphics/textwrap"; +import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models"; +import type { AssistantTranscriptEntry } from "../../assistant/session"; +import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations"; +import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings"; +import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures"; +import type { Layer, LayerContext } from "../../ui/layers"; +import type { MenuItem } from "../../ui/menu"; +import { WindowMenuLayer } from "../../ui/window-menu"; +import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window"; +import { shell } from "../../ui/shell/shell"; +import { VoiceDraft } from "./voice-draft"; + +class AiChatLayer implements Layer { + readonly acceptsDirectional = true; + private firstLine: number | null = null; // null follows the streaming tail + private maxFirstLine = 0; + private selectedId = ""; + private requestRender = () => {}; + private readonly wrappedEntries = new WeakMap(); + readonly draft: VoiceDraft; + + constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) { + this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(), + () => this.requestRender(), (text) => this.send(text)); + } + start(render: () => void): () => void { + this.requestRender = render; + return this.conversations.onChanged(() => render()); + } + send(text: string): void { + this.firstLine = null; + shell.sendToAssistant(text, false); + } + paint(ctx: LayerContext): GrayImage { + const size = ctx.stack.getBaseSize(); + const image = new GrayImage(size.width, size.height); + const font = getDefaultSmallFont(); + const step = font.lineHeight + 3; + const left = 12; + const width = size.width - 28; + const record = this.conversations.current(); + if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; } + const transcript = record.session?.transcript ?? record.history?.transcript ?? []; + const title = this.conversations.title(); + image.drawText(font, left, 6, truncateText(font, title, width), 180); + const label = assistantBackendSetting.get() === "external" ? "External agent" : + `${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`; + const footer = wrapText(font, `${label} ${gestureHints([ + [GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"], + ])}`, width); + const footerY = size.height - step * footer.length; + const footerTop = footerY - step * 3 - 12; + const historyTop = step + 16; + const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step)); + const lines: { text: string; color: number }[] = []; + for (const entry of transcript) { + if (!entry.text) continue; + let wrapped = this.wrappedEntries.get(entry); + if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) { + wrapped = { font, width, text: entry.text, + lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) }; + this.wrappedEntries.set(entry, wrapped); + } + for (const text of wrapped.lines) { + lines.push({ text, color: entry.role === "user" ? 175 : 245 }); + } + lines.push({ text: "", color: 0 }); + } + if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 }, + { text: "Voice and chat share this conversation.", color: 150 }); + this.maxFirstLine = Math.max(0, lines.length - visible); + const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine); + for (let row = 0; row < visible; row++) { + const line = lines[first + row]; + if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color); + } + if (this.maxFirstLine > 0) { + const track = visible * step; + const thumb = Math.max(8, Math.floor(track * visible / lines.length)); + image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110); + } + image.fillRect(left, footerTop, width, 1, 65); + const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status; + if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160); + const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width); + pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220)); + footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120)); + return image; + } + handleInput(event: InputEvent): void { + switch (event.type) { + case "scroll-up": case "swipe-up": + this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return; + case "scroll-down": case "swipe-down": { + const next = (this.firstLine ?? this.maxFirstLine) + 3; + this.firstLine = next >= this.maxFirstLine ? null : next; return; + } + case "long-press": + if (!this.conversations.current().session?.isTurnActive()) { + if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; } + void this.draft.start(); + } + return; + case "long-press-release": this.draft.release(); return; + case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return; + } + } + menuItems(): MenuItem[] { + const record = this.conversations.current(); + const busy = () => this.draft.active || Boolean(record.session?.isTurnActive()); + const external = assistantBackendSetting.get() === "external"; + const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true)); + return [ + ...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []), + { label: "New session", disabled: () => external || busy(), onSelect: (ctx) => { + if (this.conversations.create()) ctx.stack.clearToBase(); + } }, + { label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions", + [...this.conversations.list()].reverse().map((item) => ({ + label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`, + onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); }, + }))) }, + { label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model", + ASSISTANT_MODEL_VALUES.map((model) => ({ + label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`, + disabled: () => !this.conversations.available(model, record.reasoning), + onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); }, + }))) }, + { label: `Reasoning: ${record.reasoning}`, disabled: () => { + const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() }); + return external || busy() || !llm?.effort; + }, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({ + label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`, + onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); }, + }))) }, + ...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => { + record.session?.cancel(); ctx.stack.clearToBase(); + } }] : []), + ]; + } +} + +export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow { + const layer = new AiChatLayer(shell.getAssistantConversations(), options); + let unsubscribe = () => {}; + const app = createInProcessWindow({ + ...options, + appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true, + baseLayer: layer, holdToTalk: true, + isVoiceCapturing: () => layer.draft.active, + onSystemMenuOpened: () => layer.draft.cancel(), + onAppMenuOpened: () => layer.draft.cancel(), + setScreenOn: (on) => { if (!on) layer.draft.cancel(); }, + setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); }, + menuItems: () => layer.menuItems(), + receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); }, + onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); }, + }); + unsubscribe = layer.start(app.requestRender); + return app; +} diff --git a/app/apps/ai-chat/index.ts b/app/apps/ai-chat/index.ts new file mode 100644 index 00000000..0ac4210a --- /dev/null +++ b/app/apps/ai-chat/index.ts @@ -0,0 +1,10 @@ +import type { AppDefinition } from "../app-definition"; +import { createAiChatWindow } from "./ai-chat-app"; + +const aiChatApp: AppDefinition = { + appId: "ai-chat", + title: "AI Chat", + icon: "message-circle", + launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow), +}; +export default aiChatApp; diff --git a/app/apps/ai-chat/voice-draft.ts b/app/apps/ai-chat/voice-draft.ts new file mode 100644 index 00000000..0cb6854c --- /dev/null +++ b/app/apps/ai-chat/voice-draft.ts @@ -0,0 +1,125 @@ +import { voiceControlBridge } from "../../native/voice-control"; +import { voiceActivity } from "../../ui/shell/voice-activity"; +import type { LayerActions } from "../../ui/layers"; + +/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */ +export class VoiceDraft { + phase: "idle" | "preparing" | "listening" | "finishing" = "idle"; + text = ""; + status = ""; + private committed = ""; + private live = ""; + private generation = 0; + private started = false; + private ownsVoiceActivity = false; + private starting = false; + private unsubscribe: (() => void)[] = []; + private timer: ReturnType | null = null; + + constructor( + private readonly actions: Pick, + private readonly prepare: () => Promise, + private readonly changed: () => void, + private readonly send: (text: string) => void, + ) {} + + get active(): boolean { return this.phase !== "idle" || this.starting; } + + async start(): Promise { + if (this.active) return; + const generation = ++this.generation; + this.text = this.committed = this.live = ""; + this.phase = "preparing"; + this.status = "Preparing microphone..."; + this.changed(); + try { + const ready = await this.prepare(); + if (generation !== this.generation) return; + if (!ready) { this.cancel("Microphone unavailable"); return; } + this.phase = "listening"; + this.status = "Listening... release to send"; + this.ownsVoiceActivity = true; + voiceActivity.setActive(true); + this.unsubscribe.push(voiceControlBridge.onTranscript((event) => { + if (!this.active || generation !== this.generation) return; + if (event.isFinal) { + const text = event.text.trim(); + if (text) this.committed = [this.committed, text].filter(Boolean).join(" "); + this.live = ""; + } else this.live = event.text.trim(); + this.text = [this.committed, this.live].filter(Boolean).join(" "); + if (event.isFinal && this.phase === "finishing") this.finish(); + else this.changed(); + })); + // onStatus immediately replays the previous capture's status. + let subscribed = false; + this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => { + if (!subscribed) return; + if (generation !== this.generation || !this.active) return; + if (/ignored|failed|error|denied/i.test(status)) this.cancel(status); + })); + subscribed = true; + if (generation !== this.generation) return; + this.starting = true; + this.changed(); + await this.actions.startVoiceCapture(); + this.starting = false; + if (generation !== this.generation) { + await this.actions.stopVoiceCapture(); + this.changed(); + return; + } + this.started = true; + if (this.isFinishing()) this.stopAndFinalize(); + } catch (error) { + this.starting = false; + if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`); + } + } + + private isFinishing(): boolean { return this.phase === "finishing"; } + + release(): void { + if (this.phase === "preparing") { this.cancel(); return; } + if (this.phase !== "listening") return; + this.phase = "finishing"; + this.status = "Finishing transcription..."; + this.changed(); + if (!this.starting) this.stopAndFinalize(); + } + + private stopAndFinalize(): void { + // Install fallback first: stop may synchronously deliver the final. + this.timer = setTimeout(() => this.finish(), 1500); + if (this.started) { + this.started = false; + void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error))); + } + } + + private finish(): void { + if (this.phase !== "finishing") return; + const text = this.text.trim(); + this.cancel(text ? "" : "No speech detected"); + if (text) this.send(text); + } + + cancel(status = ""): void { + ++this.generation; + this.phase = "idle"; + this.status = status; + this.text = this.live = this.committed = ""; + if (this.timer !== null) clearTimeout(this.timer); + this.timer = null; + for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe(); + if (this.started) { + this.started = false; + void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {}); + } + if (this.ownsVoiceActivity) { + this.ownsVoiceActivity = false; + voiceActivity.setActive(false); + } + this.changed(); + } +} diff --git a/app/apps/all-apps.ts b/app/apps/all-apps.ts index 6dbb80c1..07b6dbb4 100644 --- a/app/apps/all-apps.ts +++ b/app/apps/all-apps.ts @@ -1,4 +1,5 @@ import { type AppDefinition } from "./app-definition"; +import aiChatApp from "./ai-chat"; import launcherApp from "./launcher"; import timerApp from "./timer"; import calculatorApp from "./calculator"; @@ -30,6 +31,7 @@ import settingsApp from "./settings"; */ export const ALL_APPS: readonly AppDefinition[] = [ launcherApp, + aiChatApp, timerApp, calculatorApp, terminalApp, diff --git a/app/assistant/conversations.ts b/app/assistant/conversations.ts new file mode 100644 index 00000000..7e5df8f8 --- /dev/null +++ b/app/assistant/conversations.ts @@ -0,0 +1,95 @@ +import { AssistantSession, type AssistantBackendConfig, type AssistantSessionHistory } from "./session"; +import { ASSISTANT_MODEL_VALUES, type AssistantModel } from "./models"; + +export type ReasoningLevel = "default" | "low" | "medium" | "high"; +export type Conversation = { + id: string; + model: AssistantModel; + reasoning: ReasoningLevel; + session: AssistantSession | null; + history?: AssistantSessionHistory; +}; +type SavedConversation = Omit; +type SavedConversations = { selectedId: string; conversations: SavedConversation[] }; + +/** Shared selection and history for every assistant entry point. No credentials are saved. */ +export class AssistantConversations { + private records: Conversation[] = []; + private selectedId = ""; + private readonly listeners = new Set<() => void>(); + + constructor( + private readonly resolve: (model: AssistantModel, reasoning: ReasoningLevel) => AssistantBackendConfig | null, + private readonly defaultModel: () => AssistantModel, + private readonly persist: (value: string) => void, + saved = "", + ) { + try { + const data: SavedConversations = JSON.parse(saved); + if (Array.isArray(data.conversations)) { + this.records = data.conversations.filter((record) => + typeof record.id === "string" && ASSISTANT_MODEL_VALUES.includes(record.model) && + ["default", "low", "medium", "high"].includes(record.reasoning) && + Array.isArray(record.history?.messages) && Array.isArray(record.history?.transcript), + ).map((record) => ({ ...record, session: null })); + this.selectedId = data.selectedId; + } + } catch { /* First launch or invalid saved state. */ } + if (!this.records.length) this.records.push(this.makeConversation()); + if (!this.records.some((record) => record.id === this.selectedId)) this.selectedId = this.records[0]!.id; + } + + current(): Conversation { return this.records.find((record) => record.id === this.selectedId)!; } + list(): readonly Conversation[] { return this.records; } + title(record = this.current()): string { + const transcript = record.session?.transcript ?? record.history?.transcript ?? []; + return transcript.find((entry) => entry.role === "user")?.text.replace(/\s+/g, " ").slice(0, 60) || "New session"; + } + onChanged(listener: () => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + private changed(save = true): void { + if (save) this.persist(JSON.stringify({ selectedId: this.selectedId, conversations: this.records.map( + ({ session, ...record }) => ({ ...record, history: session?.history() ?? record.history ?? { messages: [], transcript: [] } }), + ) })); + for (const listener of this.listeners) listener(); + } + private makeConversation(): Conversation { + return { id: `${Date.now()}-${Math.random().toString(36).slice(2)}`, model: this.defaultModel(), reasoning: "default", session: null }; + } + create(): boolean { + if (this.current().session?.isTurnActive()) return false; + const record = this.makeConversation(); + this.records.push(record); + this.selectedId = record.id; + this.changed(); + return true; + } + select(id: string): boolean { + if (this.current().session?.isTurnActive() || !this.records.some((record) => record.id === id)) return false; + this.selectedId = id; + this.changed(); + return true; + } + configure(model: AssistantModel, reasoning: ReasoningLevel): boolean { + const record = this.current(); + if (record.session?.isTurnActive() || !this.resolve(model, reasoning)) return false; + record.model = model; + record.reasoning = reasoning; + this.ensureSession(); + this.changed(); + return true; + } + available(model: AssistantModel, reasoning: ReasoningLevel): boolean { return this.resolve(model, reasoning) !== null; } + ensureSession(): AssistantSession | null { + const record = this.current(); + const config = this.resolve(record.model, record.reasoning); + if (!config) return null; + if (!record.session) { + record.session = new AssistantSession(config, undefined, record.history); + record.session.onChanged(() => this.changed(!record.session!.isTurnActive())); + } else record.session.configure(config); + return record.session; + } +} diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index 12e5cbbc..451abdf6 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -50,6 +50,7 @@ export class DirectAssistantBackend { const startMs = Date.now(); let cancelled = false; let streamHandle: LlmStreamHandle | null = null; + let priorText = ""; const finishError = (message: string) => { if (cancelled) return; @@ -78,7 +79,7 @@ export class DirectAssistantBackend { tools: options.buildTools(), onTextDelta: (delta, textSoFar) => { if (cancelled) return; - options.callbacks.onTextDelta(delta, textSoFar); + options.callbacks.onTextDelta(delta, priorText + textSoFar); }, onDone: (result) => { streamHandle = null; @@ -87,6 +88,9 @@ export class DirectAssistantBackend { finishError("The assistant declined this request"); return; } + const fullText = priorText + result.text; + options.callbacks.onTextDelta("", fullText); + priorText = fullText ? `${fullText}\n\n` : ""; // Record what the model produced this iteration. const assistantContent: LlmContentBlock[] = result.content.length ? result.content diff --git a/app/assistant/session.ts b/app/assistant/session.ts index 604095a9..69a915fc 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -15,26 +15,20 @@ import type { AssistantTurnHandle, } from "./types"; -/** - * One assistant conversation. Owns the message history and turn state and - * drives either the on-phone provider loop (direct) or the external agent - * bridge. The UI talks only to this surface, so AssistantLayer is - * backend-agnostic. - * - * Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh - * one for a new wakeword and reuses the current one for a follow-up. In - * external mode the conversation history lives on the agent's machine (one - * long-lived session there), so expiry here only affects the overlay UI. - */ +/** One shared conversation, independent of the voice overlay or chat window. */ +export type AssistantTranscriptEntry = { role: "user" | "assistant"; text: string }; +export type AssistantSessionHistory = { + messages: LlmMessage[]; + transcript: AssistantTranscriptEntry[]; + /** Model identity for safe replay after Auto resolves differently on restart. */ + engine?: string; +}; /** Which engine answers utterances, plus what it needs to do so. */ export type AssistantBackendConfig = | { kind: "direct"; llm: ResolvedAssistantModel } | { kind: "external"; bridge: AssistantBridgeConfig }; -/** A new PTT after this much idle starts a fresh conversation. */ -const SESSION_IDLE_MS = 10 * 60 * 1000; - /** Trim history from the head once it grows past this many messages. */ const MAX_HISTORY_MESSAGES = 40; @@ -42,38 +36,61 @@ export class AssistantSession { private readonly directBackend = new DirectAssistantBackend(); private readonly messages: LlmMessage[] = []; private turnHandle: AssistantTurnHandle | null = null; - private lastActivityMs = Date.now(); + readonly transcript: AssistantTranscriptEntry[] = []; + status = ""; + private readonly listeners = new Set<() => void>(); + private turnGeneration = 0; // API-safe tool name -> canonical registry name. Provider APIs restrict tool // names, so dotted registry names are sanitized and mapped back on calls. private readonly toolNameMap = new Map(); constructor( - private readonly config: AssistantBackendConfig, + private config: AssistantBackendConfig, private readonly registry: ToolRegistry = toolRegistry, - ) {} - - matchesConfiguration(config: AssistantBackendConfig): boolean { - if (this.config.kind !== config.kind) return false; - if (this.config.kind === "direct" && config.kind === "direct") { - return ( - this.config.llm.provider === config.llm.provider && - this.config.llm.model === config.llm.model && - this.config.llm.apiKey === config.llm.apiKey - ); + history?: AssistantSessionHistory, + ) { + if (history) { + this.messages.push(...history.messages); + this.transcript.push(...history.transcript); + if (history.engine !== this.engine()) this.rebuildTextHistory(); } - if (this.config.kind === "external" && config.kind === "external") { - return ( - this.config.bridge.host === config.bridge.host && - this.config.bridge.port === config.bridge.port && - this.config.bridge.token === config.bridge.token - ); - } - return false; } - /** Whether this session has been idle long enough to retire. */ - isExpired(nowMs = Date.now()): boolean { - return nowMs - this.lastActivityMs > SESSION_IDLE_MS; + onChanged(listener: () => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + private changed(): void { + for (const listener of this.listeners) { + try { listener(); } catch (error) { console.warn("assistant history listener failed", error); } + } + } + + history(): AssistantSessionHistory { + return { messages: this.messages, transcript: this.transcript, engine: this.engine() }; + } + + configure(config: AssistantBackendConfig): void { + if (this.isTurnActive()) return; + // Opaque reasoning/tool provider items are only replayable on their + // original model. Preserve visible conversation when switching engines. + if (this.config.kind !== config.kind || + (this.config.kind === "direct" && config.kind === "direct" && + (this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) { + this.rebuildTextHistory(); + } + this.config = config; + } + + private engine(): string { + return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external"; + } + + private rebuildTextHistory(): void { + this.messages.splice(0, this.messages.length, ...this.transcript + .filter((entry) => entry.text.trim()) + .map((entry) => ({ role: entry.role, content: entry.text }))); } isTurnActive(): boolean { @@ -86,20 +103,44 @@ export class AssistantSession { callbacks.onError("The assistant is still working on the previous request"); return; } - this.lastActivityMs = Date.now(); + text = text.trim(); + if (!text) return; + const generation = ++this.turnGeneration; + const reply: AssistantTranscriptEntry = { role: "assistant", text: "" }; + this.transcript.push({ role: "user", text }, reply); + this.status = "Thinking..."; + // Install a sentinel before calling providers: some failures are synchronous. + this.turnHandle = { cancel: () => {} }; + this.changed(); const finish = () => { + ++this.turnGeneration; this.turnHandle = null; - this.lastActivityMs = Date.now(); + this.changed(); }; const wrappedCallbacks: AssistantTurnCallbacks = { - onTextDelta: callbacks.onTextDelta, - onToolActivity: callbacks.onToolActivity, + onTextDelta: (delta, full) => { + if (generation !== this.turnGeneration) return; + reply.text = full; + this.status = "Thinking..."; + this.changed(); + callbacks.onTextDelta(delta, full); + }, + onToolActivity: (label) => { + if (generation !== this.turnGeneration) return; + this.status = `→ ${label}`; + this.changed(); + callbacks.onToolActivity(label); + }, onTurnDone: (result) => { + if (generation !== this.turnGeneration) return; + this.status = ""; finish(); callbacks.onTurnDone(result); }, onError: (message) => { + if (generation !== this.turnGeneration) return; + this.status = message; finish(); callbacks.onError(message); }, @@ -108,7 +149,8 @@ export class AssistantSession { if (this.config.kind === "external") { // History and the agent loop live on the agent's machine; the phone just // streams this turn. The overlay keeps its own display state. - this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks); + const handle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks); + if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle; return; } @@ -123,24 +165,48 @@ export class AssistantSession { content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text, }); this.trimHistory(); - this.turnHandle = this.directBackend.runTurn({ + // Work on a turn-local array: cancelled tools may finish later, and must + // never append orphan results into a subsequent turn's context. + const turnMessages = this.messages.slice(); + const handle = this.directBackend.runTurn({ provider: llm.provider, apiKey: llm.apiKey, model: llm.model, effort: llm.effort, system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), - messages: this.messages, + messages: turnMessages, buildTools: () => this.buildToolDefinitions(), registry: this.registry, resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName, - callbacks: wrappedCallbacks, + callbacks: { + ...wrappedCallbacks, + onTurnDone: (result) => { + if (generation !== this.turnGeneration) return; + this.messages.splice(0, this.messages.length, ...turnMessages); + wrappedCallbacks.onTurnDone(result); + }, + onError: (message) => { + if (generation !== this.turnGeneration) return; + // Retain the visible partial answer, without incomplete tool pairs. + if (reply.text) this.messages.push({ role: "assistant", content: reply.text }); + wrappedCallbacks.onError(message); + }, + }, }); + if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle; } cancel(): void { - this.turnHandle?.cancel(); + if (!this.turnHandle) return; + ++this.turnGeneration; + this.turnHandle.cancel(); this.turnHandle = null; - this.lastActivityMs = Date.now(); + const reply = this.transcript[this.transcript.length - 1]; + if (reply?.role === "assistant" && reply.text) { + this.messages.push({ role: "assistant", content: reply.text }); + } + this.status = "Cancelled"; + this.changed(); } private buildToolDefinitions(): LlmToolDefinition[] { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index adb0c7ca..850f5daa 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -319,7 +319,7 @@ class DashboardController { toggle?: TextSettingsEditToggle, ) => this.startTextSettingsEdit(settings, title, onFinish, toggle), endTextSettingEdit: () => this.endTextSettingEdit(), - startVoiceCapture: () => this.startVoiceCapture(), + startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing), stopVoiceCapture: () => this.stopVoiceCapture(), startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), @@ -1874,11 +1874,14 @@ class DashboardController { * push-to-talk and the Transcribe app. Android mic permission is the consent * gate even though the audio source is the G2 mic over BLE. */ - private startVoiceCapture(endpointing = false): void { - this.beginVoiceCapture("ptt", endpointing); + private pttCaptureGeneration = 0; + + private startVoiceCapture(endpointing = false): Promise { + return this.beginVoiceCapture("ptt", endpointing); } private stopVoiceCapture(): void { + ++this.pttCaptureGeneration; voiceControlBridge.stopPushToTalk(); } @@ -1944,7 +1947,8 @@ class DashboardController { return granted; } - private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void { + private async beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): Promise { + const pttGeneration = kind === "ptt" ? ++this.pttCaptureGeneration : 0; // Preview mode captures from the phone mic (voiceCaptureOptions with a // null communicator); otherwise a live glasses session must be the source. const previewCapture = this.isPreviewDisplayActive(); @@ -1952,8 +1956,9 @@ class DashboardController { return; } const communicator = this.communicator; - void ensureVoicePermissions() + await ensureVoicePermissions() .then(() => { + if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return; if (previewCapture) { if (!this.isPreviewDisplayActive()) return; } else if (this.phase !== "connected" || this.communicator !== communicator) { diff --git a/app/graphics/icons.ts b/app/graphics/icons.ts index 2aeea17a..bee9d8cf 100644 --- a/app/graphics/icons.ts +++ b/app/graphics/icons.ts @@ -18,6 +18,8 @@ const ICON_STROKE_WIDTH = 2; // Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed // against upstream if an icon needs updating. export const ICON_SVGS = { + "message-circle": + '', "layout-grid": '', timer: diff --git a/app/ui/shell/in-process-window.ts b/app/ui/shell/in-process-window.ts index 646cb487..a3ec71d5 100644 --- a/app/ui/shell/in-process-window.ts +++ b/app/ui/shell/in-process-window.ts @@ -38,6 +38,12 @@ export type InProcessWindowOptions = { * has no menu of its own, and tap-then-hold opens the system menu instead. */ menuItems?: () => MenuItem[]; + /** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */ + holdToTalk?: boolean; + isVoiceCapturing?: () => boolean; + onSystemMenuOpened?: () => void; + onAppMenuOpened?: () => void; + setScreenOn?: ShellWindow["setScreenOn"]; /** Shared actions; requestRender is rebound to this window's render. */ actions: LayerActions; /** @@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces shell.openSystemMenu(options.windowId); return; } - stack.push(new WindowMenuLayer(options.title, items)); + options.onAppMenuOpened?.(); + stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk)); }; const window: ShellWindow = { @@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces // The window's own LayerStack decides per layer whether a swipe is // directional or falls back to click / double-click. acceptsDirectional: true, + holdToTalk: options.holdToTalk, + isVoiceCapturing: options.isVoiceCapturing, + setScreenOn: options.setScreenOn, heightMode, hasAppMenu: () => appMenuItems().length > 0, close: () => { @@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces // The shell opened its system menu over this window; close our own // context menu so the two never stack. Never forwarded to app layers. if (event.type === "system-menu-opened") { + options.onSystemMenuOpened?.(); if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) { requestRender(); } @@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces // submenus and app content alike. A plain long-press never arrives: // the shell keeps it for the system menu (no in-process window claims // it). - if (event.type === "short-then-long-press") { + if ((!options.holdToTalk && event.type === "short-then-long-press") || + (options.holdToTalk && event.type === "click" && stack.isAtBase())) { openWindowMenu(); await render(frameId); return; diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 332ff275..cd2595d3 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input" import { voiceActivity } from "./voice-activity"; import { AssistantLayer } from "./assistant"; import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session"; -import { resolveAssistantModel } from "../../assistant/models"; +import { resolveAssistantModel, type AssistantModel } from "../../assistant/models"; +import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations"; +import { getStringSetting, setStringSetting } from "../../native/settings-store"; import type { AssistantContext } from "../../assistant/types"; import { SingleNotificationLayer } from "../notifications"; import { @@ -91,6 +93,10 @@ export type ShellWindow = { * system menu; holding the press past the escape threshold still opens it. */ claimsLongPress?: () => boolean; + /** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */ + holdToTalk?: boolean; + /** A window owns microphone capture outside the shell voice dialog. */ + isVoiceCapturing?: () => boolean; /** * True when the window gives swipe-left / swipe-right (watch directional * input) a meaning; otherwise the shell forwards directionalFallback(event). @@ -298,7 +304,29 @@ class Shell { private readonly trayIcons = new Map(); private activeVoiceLayer: VoiceInputLayer | null = null; private activeKeyboardLayer: KeyboardInputLayer | null = null; - private assistantSession: AssistantSession | null = null; + private conversations: AssistantConversations | null = null; + private get assistantSession(): AssistantSession | null { + return this.conversations?.current().session ?? null; + } + + getAssistantConversations(): AssistantConversations { + if (!this.conversations) { + this.conversations = new AssistantConversations( + (model, reasoning) => this.resolveAssistantConfiguration(model, reasoning), + () => assistantModelSetting.get(), + (value) => setStringSetting("assistant.conversations", value), + getStringSetting("assistant.conversations", ""), + ); + } + return this.conversations; + } + + async prepareChatVoiceCapture(): Promise { + if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false; + const ready = (await this.config.prepareVoiceCapture?.()) ?? true; + return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer && + this.stack.isAtBase() && this.isWindowFocused("ai-chat"); + } private assistantLayer: AssistantLayer | null = null; private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>(); private readonly alertListeners = new Set<(text: string) => void>(); @@ -559,7 +587,7 @@ class Shell { // ring. An in-flight assistant turn suspends it for the same reason (a // tool loop can run for a while with no input); once the turn ends and // the Follow-up/Done menu is showing, the normal idle timeout resumes. - if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) { + if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) { this.lastInputAtMs = nowMs; return false; } @@ -645,6 +673,7 @@ class Shell { // Even AI app never launches, so the firmware does not power the display // for us either -- actions that need it wake the screen themselves. if (event.type === "wakeword") { + if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false }; const action = wakeWordActionSetting.get(); if (action === "off") { return { shell: false, window: false }; @@ -696,11 +725,11 @@ class Shell { if (!window) { return { shell: true, window: false }; } - if (!window.claimsLongPress?.()) { + if (!window.holdToTalk && !window.claimsLongPress?.()) { this.openEscapeMenu(); return { shell: true, window: false }; } - this.startEscapeMenuTimer(); + if (!window.holdToTalk) this.startEscapeMenuTimer(); if (this.focus === "sidebar") { this.focus = "window"; window.onFocus?.(this.lastInput); @@ -716,6 +745,10 @@ class Shell { // when it has none. if (event.type === "short-then-long-press") { const window = this.foregroundWindow(); + if (window?.holdToTalk) { + this.openEscapeMenu(); + return { shell: true, window: false }; + } // Over the open system menu it switches to the app's context menu: // close the system menu and deliver the gesture to the window as // usual. A window without a menu of its own would only ask for the @@ -1024,7 +1057,7 @@ class Shell { */ startKeyboardInput(): KeyboardInputSession | null { if (this.activeKeyboardLayer) return this.activeKeyboardLayer; - if (this.activeVoiceLayer) return null; + if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null; if (!this.screenOn) this.wake("sidebar"); const assistantLayer = this.assistantLayer; const assistantSession = this.assistantSession; @@ -1069,7 +1102,8 @@ class Shell { } isAssistantAvailable(): boolean { - return this.resolveAssistantConfiguration() !== null; + const current = this.getAssistantConversations().current(); + return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null; } /** Observe assistant turns (see AssistantActivityEvent). */ @@ -1107,20 +1141,13 @@ class Shell { } private ensureAssistantSession(): AssistantSession | null { - const config = this.resolveAssistantConfiguration(); - if (!config) return null; - if ( - !this.assistantSession || - this.assistantSession.isExpired() || - !this.assistantSession.matchesConfiguration(config) - ) { - this.assistantSession?.cancel(); - this.assistantSession = new AssistantSession(config); - } - return this.assistantSession; + return this.getAssistantConversations().ensureSession(); } - private resolveAssistantConfiguration(): AssistantBackendConfig | null { + private resolveAssistantConfiguration( + model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(), + reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default", + ): AssistantBackendConfig | null { if (assistantBackendSetting.get() === "external") { const host = assistantBridgeHostSetting.get().trim(); const token = assistantBridgeTokenSetting.get(); @@ -1128,10 +1155,11 @@ class Shell { const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790; return { kind: "external", bridge: { host, port, token } }; } - const llm = resolveAssistantModel(assistantModelSetting.get(), { + const llm = resolveAssistantModel(model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(), }); + if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning; return llm ? { kind: "direct", llm } : null; } @@ -1151,7 +1179,9 @@ class Shell { * Opens the assistant overlay if it isn't already up; a follow-up reuses the * existing session and overlay. */ - sendToAssistant(text: string): void { + sendToAssistant(text: string, showOverlay = true): void { + text = text.trim(); + if (!text) return; const session = this.ensureAssistantSession(); if (!session) { this.showAlert( @@ -1161,7 +1191,15 @@ class Shell { ); return; } + if (session.isTurnActive()) { + this.showAlert("The assistant is still working on the previous request"); + return; + } if (!this.screenOn) this.wake("sidebar"); + if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") { + this.runAssistantTurn(session, null, text); + return; + } let layer = this.assistantLayer; if (!layer) { const created = new AssistantLayer(this.config.actions, { @@ -1170,7 +1208,7 @@ class Shell { onClose: () => this.closeAssistantLayer(), onRemoved: () => { // Removed by any path (Done, or the screen sleeping mid-conversation): - // stop the turn and drop the reference so a later query starts clean. + // stop the turn and drop the overlay reference; keep shared history. this.assistantSession?.cancel(); if (this.assistantLayer === created) this.assistantLayer = null; this.emitAssistantActivity({ phase: "closed", text: "" }); @@ -1184,23 +1222,24 @@ class Shell { this.config.requestShellRender(); } - private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void { - layer.startTurn(); + private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void { + if (session.isTurnActive()) return; + layer?.startTurn(); let replySoFar = ""; this.emitAssistantActivity({ phase: "thinking", text: "" }); session.sendUtterance(text, this.buildAssistantContext(), { onTextDelta: (delta, textSoFar) => { replySoFar = textSoFar; - layer.onTextDelta(delta, textSoFar); + layer?.onTextDelta(delta, textSoFar); this.emitAssistantActivity({ phase: "streaming", text: textSoFar }); }, - onToolActivity: (label) => layer.onToolActivity(label), + onToolActivity: (label) => layer?.onToolActivity(label), onTurnDone: () => { - layer.onTurnDone(); + layer?.onTurnDone(); this.emitAssistantActivity({ phase: "done", text: replySoFar }); }, onError: (message) => { - layer.onError(message); + layer?.onError(message); this.emitAssistantActivity({ phase: "error", text: message }); }, }); @@ -1356,7 +1395,7 @@ class Shell { // when the app has one (otherwise it opens this very menu, not worth a // hint). const footer = foreground.hasAppMenu?.() - ? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]) + ? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])) : undefined; layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar()); layer.selectItem(initialSelection); diff --git a/app/ui/window-menu.ts b/app/ui/window-menu.ts index d0dd838f..f439bfb6 100644 --- a/app/ui/window-menu.ts +++ b/app/ui/window-menu.ts @@ -1,6 +1,6 @@ import { GrayImage } from "../graphics/image"; import { singlePlane, type Plane } from "../graphics/plane"; -import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures"; +import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures"; import { type Layer, LayerStack, noopLayerActions } from "./layers"; import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu"; import type { WorkerAppReply } from "./shell/worker-window"; @@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = { }; export class WindowMenuLayer extends MenuLayer { - constructor(title: string | null, items: MenuItem[]) { - super(title, items, WINDOW_MENU_LAYOUT); + constructor(title: string | null, items: MenuItem[], holdToTalk = false) { + super(title, items, holdToTalk + ? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) } + : WINDOW_MENU_LAYOUT); } }