Add "AI Chat" app.

This commit is contained in:
James Babcock committed 2026-09-09 01:15:19 -07:00
1 parent 78c56ffc4e
commit 0765e943fb
13 files changed
+616 -87

No files matched your search

+1
View File
@@ -5,6 +5,7 @@
* Transcribe app: Scrolling; automatic reconnection; insert newlines
* Notifications: Add notification filters
* Phone screen preview: Add rotation option; improve landscape layout
* Add "AI Chat" app (conversational-style use of the voice assistant with hold-to-talk)
## 0.6.5 - 2026-09-04
* Changed context menu gestures: — for system, ●— for app context menu.
+166
View File
@@ -0,0 +1,166 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
import type { AssistantTranscriptEntry } from "../../assistant/session";
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
import type { Layer, LayerContext } from "../../ui/layers";
import type { MenuItem } from "../../ui/menu";
import { WindowMenuLayer } from "../../ui/window-menu";
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
import { shell } from "../../ui/shell/shell";
import { VoiceDraft } from "./voice-draft";
class AiChatLayer implements Layer {
readonly acceptsDirectional = true;
private firstLine: number | null = null; // null follows the streaming tail
private maxFirstLine = 0;
private selectedId = "";
private requestRender = () => {};
private readonly wrappedEntries = new WeakMap<AssistantTranscriptEntry, {
font: UiFont; width: number; text: string; lines: string[];
}>();
readonly draft: VoiceDraft;
constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) {
this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(),
() => this.requestRender(), (text) => this.send(text));
}
start(render: () => void): () => void {
this.requestRender = render;
return this.conversations.onChanged(() => render());
}
send(text: string): void {
this.firstLine = null;
shell.sendToAssistant(text, false);
}
paint(ctx: LayerContext): GrayImage {
const size = ctx.stack.getBaseSize();
const image = new GrayImage(size.width, size.height);
const font = getDefaultSmallFont();
const step = font.lineHeight + 3;
const left = 12;
const width = size.width - 28;
const record = this.conversations.current();
if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; }
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
const title = this.conversations.title();
image.drawText(font, left, 6, truncateText(font, title, width), 180);
const label = assistantBackendSetting.get() === "external" ? "External agent" :
`${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`;
const footer = wrapText(font, `${label} ${gestureHints([
[GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"],
])}`, width);
const footerY = size.height - step * footer.length;
const footerTop = footerY - step * 3 - 12;
const historyTop = step + 16;
const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step));
const lines: { text: string; color: number }[] = [];
for (const entry of transcript) {
if (!entry.text) continue;
let wrapped = this.wrappedEntries.get(entry);
if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) {
wrapped = { font, width, text: entry.text,
lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) };
this.wrappedEntries.set(entry, wrapped);
}
for (const text of wrapped.lines) {
lines.push({ text, color: entry.role === "user" ? 175 : 245 });
}
lines.push({ text: "", color: 0 });
}
if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 },
{ text: "Voice and chat share this conversation.", color: 150 });
this.maxFirstLine = Math.max(0, lines.length - visible);
const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine);
for (let row = 0; row < visible; row++) {
const line = lines[first + row];
if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color);
}
if (this.maxFirstLine > 0) {
const track = visible * step;
const thumb = Math.max(8, Math.floor(track * visible / lines.length));
image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110);
}
image.fillRect(left, footerTop, width, 1, 65);
const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status;
if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160);
const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width);
pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220));
footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120));
return image;
}
handleInput(event: InputEvent): void {
switch (event.type) {
case "scroll-up": case "swipe-up":
this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return;
case "scroll-down": case "swipe-down": {
const next = (this.firstLine ?? this.maxFirstLine) + 3;
this.firstLine = next >= this.maxFirstLine ? null : next; return;
}
case "long-press":
if (!this.conversations.current().session?.isTurnActive()) {
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
void this.draft.start();
}
return;
case "long-press-release": this.draft.release(); return;
case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return;
}
}
menuItems(): MenuItem[] {
const record = this.conversations.current();
const busy = () => this.draft.active || Boolean(record.session?.isTurnActive());
const external = assistantBackendSetting.get() === "external";
const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true));
return [
...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []),
{ label: "New session", disabled: () => external || busy(), onSelect: (ctx) => {
if (this.conversations.create()) ctx.stack.clearToBase();
} },
{ label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions",
[...this.conversations.list()].reverse().map((item) => ({
label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`,
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
}))) },
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
ASSISTANT_MODEL_VALUES.map((model) => ({
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
disabled: () => !this.conversations.available(model, record.reasoning),
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
}))) },
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
return external || busy() || !llm?.effort;
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); },
}))) },
...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => {
record.session?.cancel(); ctx.stack.clearToBase();
} }] : []),
];
}
}
export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow {
const layer = new AiChatLayer(shell.getAssistantConversations(), options);
let unsubscribe = () => {};
const app = createInProcessWindow({
...options,
appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true,
baseLayer: layer, holdToTalk: true,
isVoiceCapturing: () => layer.draft.active,
onSystemMenuOpened: () => layer.draft.cancel(),
onAppMenuOpened: () => layer.draft.cancel(),
setScreenOn: (on) => { if (!on) layer.draft.cancel(); },
setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); },
menuItems: () => layer.menuItems(),
receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); },
onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); },
});
unsubscribe = layer.start(app.requestRender);
return app;
}
+10
View File
@@ -0,0 +1,10 @@
import type { AppDefinition } from "../app-definition";
import { createAiChatWindow } from "./ai-chat-app";
const aiChatApp: AppDefinition = {
appId: "ai-chat",
title: "AI Chat",
icon: "message-circle",
launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow),
};
export default aiChatApp;
+125
View File
@@ -0,0 +1,125 @@
import { voiceControlBridge } from "../../native/voice-control";
import { voiceActivity } from "../../ui/shell/voice-activity";
import type { LayerActions } from "../../ui/layers";
/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */
export class VoiceDraft {
phase: "idle" | "preparing" | "listening" | "finishing" = "idle";
text = "";
status = "";
private committed = "";
private live = "";
private generation = 0;
private started = false;
private ownsVoiceActivity = false;
private starting = false;
private unsubscribe: (() => void)[] = [];
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(
private readonly actions: Pick<LayerActions, "startVoiceCapture" | "stopVoiceCapture">,
private readonly prepare: () => Promise<boolean>,
private readonly changed: () => void,
private readonly send: (text: string) => void,
) {}
get active(): boolean { return this.phase !== "idle" || this.starting; }
async start(): Promise<void> {
if (this.active) return;
const generation = ++this.generation;
this.text = this.committed = this.live = "";
this.phase = "preparing";
this.status = "Preparing microphone...";
this.changed();
try {
const ready = await this.prepare();
if (generation !== this.generation) return;
if (!ready) { this.cancel("Microphone unavailable"); return; }
this.phase = "listening";
this.status = "Listening... release to send";
this.ownsVoiceActivity = true;
voiceActivity.setActive(true);
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
if (!this.active || generation !== this.generation) return;
if (event.isFinal) {
const text = event.text.trim();
if (text) this.committed = [this.committed, text].filter(Boolean).join(" ");
this.live = "";
} else this.live = event.text.trim();
this.text = [this.committed, this.live].filter(Boolean).join(" ");
if (event.isFinal && this.phase === "finishing") this.finish();
else this.changed();
}));
// onStatus immediately replays the previous capture's status.
let subscribed = false;
this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => {
if (!subscribed) return;
if (generation !== this.generation || !this.active) return;
if (/ignored|failed|error|denied/i.test(status)) this.cancel(status);
}));
subscribed = true;
if (generation !== this.generation) return;
this.starting = true;
this.changed();
await this.actions.startVoiceCapture();
this.starting = false;
if (generation !== this.generation) {
await this.actions.stopVoiceCapture();
this.changed();
return;
}
this.started = true;
if (this.isFinishing()) this.stopAndFinalize();
} catch (error) {
this.starting = false;
if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`);
}
}
private isFinishing(): boolean { return this.phase === "finishing"; }
release(): void {
if (this.phase === "preparing") { this.cancel(); return; }
if (this.phase !== "listening") return;
this.phase = "finishing";
this.status = "Finishing transcription...";
this.changed();
if (!this.starting) this.stopAndFinalize();
}
private stopAndFinalize(): void {
// Install fallback first: stop may synchronously deliver the final.
this.timer = setTimeout(() => this.finish(), 1500);
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
}
}
private finish(): void {
if (this.phase !== "finishing") return;
const text = this.text.trim();
this.cancel(text ? "" : "No speech detected");
if (text) this.send(text);
}
cancel(status = ""): void {
++this.generation;
this.phase = "idle";
this.status = status;
this.text = this.live = this.committed = "";
if (this.timer !== null) clearTimeout(this.timer);
this.timer = null;
for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {});
}
if (this.ownsVoiceActivity) {
this.ownsVoiceActivity = false;
voiceActivity.setActive(false);
}
this.changed();
}
}
+2
View File
@@ -1,4 +1,5 @@
import { type AppDefinition } from "./app-definition";
import aiChatApp from "./ai-chat";
import launcherApp from "./launcher";
import timerApp from "./timer";
import calculatorApp from "./calculator";
@@ -30,6 +31,7 @@ import settingsApp from "./settings";
*/
export const ALL_APPS: readonly AppDefinition[] = [
launcherApp,
aiChatApp,
timerApp,
calculatorApp,
terminalApp,
+95
View File
@@ -0,0 +1,95 @@
import { AssistantSession, type AssistantBackendConfig, type AssistantSessionHistory } from "./session";
import { ASSISTANT_MODEL_VALUES, type AssistantModel } from "./models";
export type ReasoningLevel = "default" | "low" | "medium" | "high";
export type Conversation = {
id: string;
model: AssistantModel;
reasoning: ReasoningLevel;
session: AssistantSession | null;
history?: AssistantSessionHistory;
};
type SavedConversation = Omit<Conversation, "session">;
type SavedConversations = { selectedId: string; conversations: SavedConversation[] };
/** Shared selection and history for every assistant entry point. No credentials are saved. */
export class AssistantConversations {
private records: Conversation[] = [];
private selectedId = "";
private readonly listeners = new Set<() => void>();
constructor(
private readonly resolve: (model: AssistantModel, reasoning: ReasoningLevel) => AssistantBackendConfig | null,
private readonly defaultModel: () => AssistantModel,
private readonly persist: (value: string) => void,
saved = "",
) {
try {
const data: SavedConversations = JSON.parse(saved);
if (Array.isArray(data.conversations)) {
this.records = data.conversations.filter((record) =>
typeof record.id === "string" && ASSISTANT_MODEL_VALUES.includes(record.model) &&
["default", "low", "medium", "high"].includes(record.reasoning) &&
Array.isArray(record.history?.messages) && Array.isArray(record.history?.transcript),
).map((record) => ({ ...record, session: null }));
this.selectedId = data.selectedId;
}
} catch { /* First launch or invalid saved state. */ }
if (!this.records.length) this.records.push(this.makeConversation());
if (!this.records.some((record) => record.id === this.selectedId)) this.selectedId = this.records[0]!.id;
}
current(): Conversation { return this.records.find((record) => record.id === this.selectedId)!; }
list(): readonly Conversation[] { return this.records; }
title(record = this.current()): string {
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
return transcript.find((entry) => entry.role === "user")?.text.replace(/\s+/g, " ").slice(0, 60) || "New session";
}
onChanged(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
private changed(save = true): void {
if (save) this.persist(JSON.stringify({ selectedId: this.selectedId, conversations: this.records.map(
({ session, ...record }) => ({ ...record, history: session?.history() ?? record.history ?? { messages: [], transcript: [] } }),
) }));
for (const listener of this.listeners) listener();
}
private makeConversation(): Conversation {
return { id: `${Date.now()}-${Math.random().toString(36).slice(2)}`, model: this.defaultModel(), reasoning: "default", session: null };
}
create(): boolean {
if (this.current().session?.isTurnActive()) return false;
const record = this.makeConversation();
this.records.push(record);
this.selectedId = record.id;
this.changed();
return true;
}
select(id: string): boolean {
if (this.current().session?.isTurnActive() || !this.records.some((record) => record.id === id)) return false;
this.selectedId = id;
this.changed();
return true;
}
configure(model: AssistantModel, reasoning: ReasoningLevel): boolean {
const record = this.current();
if (record.session?.isTurnActive() || !this.resolve(model, reasoning)) return false;
record.model = model;
record.reasoning = reasoning;
this.ensureSession();
this.changed();
return true;
}
available(model: AssistantModel, reasoning: ReasoningLevel): boolean { return this.resolve(model, reasoning) !== null; }
ensureSession(): AssistantSession | null {
const record = this.current();
const config = this.resolve(record.model, record.reasoning);
if (!config) return null;
if (!record.session) {
record.session = new AssistantSession(config, undefined, record.history);
record.session.onChanged(() => this.changed(!record.session!.isTurnActive()));
} else record.session.configure(config);
return record.session;
}
}
+5 -1
View File
@@ -50,6 +50,7 @@ export class DirectAssistantBackend {
const startMs = Date.now();
let cancelled = false;
let streamHandle: LlmStreamHandle | null = null;
let priorText = "";
const finishError = (message: string) => {
if (cancelled) return;
@@ -78,7 +79,7 @@ export class DirectAssistantBackend {
tools: options.buildTools(),
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar);
options.callbacks.onTextDelta(delta, priorText + textSoFar);
},
onDone: (result) => {
streamHandle = null;
@@ -87,6 +88,9 @@ export class DirectAssistantBackend {
finishError("The assistant declined this request");
return;
}
const fullText = priorText + result.text;
options.callbacks.onTextDelta("", fullText);
priorText = fullText ? `${fullText}\n\n` : "";
// Record what the model produced this iteration.
const assistantContent: LlmContentBlock[] = result.content.length
? result.content
+113 -47
View File
@@ -15,26 +15,20 @@ import type {
AssistantTurnHandle,
} from "./types";
/**
* One assistant conversation. Owns the message history and turn state and
* drives either the on-phone provider loop (direct) or the external agent
* bridge. The UI talks only to this surface, so AssistantLayer is
* backend-agnostic.
*
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
* one for a new wakeword and reuses the current one for a follow-up. In
* external mode the conversation history lives on the agent's machine (one
* long-lived session there), so expiry here only affects the overlay UI.
*/
/** One shared conversation, independent of the voice overlay or chat window. */
export type AssistantTranscriptEntry = { role: "user" | "assistant"; text: string };
export type AssistantSessionHistory = {
messages: LlmMessage[];
transcript: AssistantTranscriptEntry[];
/** Model identity for safe replay after Auto resolves differently on restart. */
engine?: string;
};
/** Which engine answers utterances, plus what it needs to do so. */
export type AssistantBackendConfig =
| { kind: "direct"; llm: ResolvedAssistantModel }
| { kind: "external"; bridge: AssistantBridgeConfig };
/** A new PTT after this much idle starts a fresh conversation. */
const SESSION_IDLE_MS = 10 * 60 * 1000;
/** Trim history from the head once it grows past this many messages. */
const MAX_HISTORY_MESSAGES = 40;
@@ -42,38 +36,61 @@ export class AssistantSession {
private readonly directBackend = new DirectAssistantBackend();
private readonly messages: LlmMessage[] = [];
private turnHandle: AssistantTurnHandle | null = null;
private lastActivityMs = Date.now();
readonly transcript: AssistantTranscriptEntry[] = [];
status = "";
private readonly listeners = new Set<() => void>();
private turnGeneration = 0;
// API-safe tool name -> canonical registry name. Provider APIs restrict tool
// names, so dotted registry names are sanitized and mapped back on calls.
private readonly toolNameMap = new Map<string, string>();
constructor(
private readonly config: AssistantBackendConfig,
private config: AssistantBackendConfig,
private readonly registry: ToolRegistry = toolRegistry,
) {}
matchesConfiguration(config: AssistantBackendConfig): boolean {
if (this.config.kind !== config.kind) return false;
if (this.config.kind === "direct" && config.kind === "direct") {
return (
this.config.llm.provider === config.llm.provider &&
this.config.llm.model === config.llm.model &&
this.config.llm.apiKey === config.llm.apiKey
);
history?: AssistantSessionHistory,
) {
if (history) {
this.messages.push(...history.messages);
this.transcript.push(...history.transcript);
if (history.engine !== this.engine()) this.rebuildTextHistory();
}
if (this.config.kind === "external" && config.kind === "external") {
return (
this.config.bridge.host === config.bridge.host &&
this.config.bridge.port === config.bridge.port &&
this.config.bridge.token === config.bridge.token
);
}
return false;
}
/** Whether this session has been idle long enough to retire. */
isExpired(nowMs = Date.now()): boolean {
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
onChanged(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
private changed(): void {
for (const listener of this.listeners) {
try { listener(); } catch (error) { console.warn("assistant history listener failed", error); }
}
}
history(): AssistantSessionHistory {
return { messages: this.messages, transcript: this.transcript, engine: this.engine() };
}
configure(config: AssistantBackendConfig): void {
if (this.isTurnActive()) return;
// Opaque reasoning/tool provider items are only replayable on their
// original model. Preserve visible conversation when switching engines.
if (this.config.kind !== config.kind ||
(this.config.kind === "direct" && config.kind === "direct" &&
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) {
this.rebuildTextHistory();
}
this.config = config;
}
private engine(): string {
return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external";
}
private rebuildTextHistory(): void {
this.messages.splice(0, this.messages.length, ...this.transcript
.filter((entry) => entry.text.trim())
.map((entry) => ({ role: entry.role, content: entry.text })));
}
isTurnActive(): boolean {
@@ -86,20 +103,44 @@ export class AssistantSession {
callbacks.onError("The assistant is still working on the previous request");
return;
}
this.lastActivityMs = Date.now();
text = text.trim();
if (!text) return;
const generation = ++this.turnGeneration;
const reply: AssistantTranscriptEntry = { role: "assistant", text: "" };
this.transcript.push({ role: "user", text }, reply);
this.status = "Thinking...";
// Install a sentinel before calling providers: some failures are synchronous.
this.turnHandle = { cancel: () => {} };
this.changed();
const finish = () => {
++this.turnGeneration;
this.turnHandle = null;
this.lastActivityMs = Date.now();
this.changed();
};
const wrappedCallbacks: AssistantTurnCallbacks = {
onTextDelta: callbacks.onTextDelta,
onToolActivity: callbacks.onToolActivity,
onTextDelta: (delta, full) => {
if (generation !== this.turnGeneration) return;
reply.text = full;
this.status = "Thinking...";
this.changed();
callbacks.onTextDelta(delta, full);
},
onToolActivity: (label) => {
if (generation !== this.turnGeneration) return;
this.status = `→ ${label}`;
this.changed();
callbacks.onToolActivity(label);
},
onTurnDone: (result) => {
if (generation !== this.turnGeneration) return;
this.status = "";
finish();
callbacks.onTurnDone(result);
},
onError: (message) => {
if (generation !== this.turnGeneration) return;
this.status = message;
finish();
callbacks.onError(message);
},
@@ -108,7 +149,8 @@ export class AssistantSession {
if (this.config.kind === "external") {
// History and the agent loop live on the agent's machine; the phone just
// streams this turn. The overlay keeps its own display state.
this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
const handle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
return;
}
@@ -123,24 +165,48 @@ export class AssistantSession {
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
});
this.trimHistory();
this.turnHandle = this.directBackend.runTurn({
// Work on a turn-local array: cancelled tools may finish later, and must
// never append orphan results into a subsequent turn's context.
const turnMessages = this.messages.slice();
const handle = this.directBackend.runTurn({
provider: llm.provider,
apiKey: llm.apiKey,
model: llm.model,
effort: llm.effort,
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
messages: this.messages,
messages: turnMessages,
buildTools: () => this.buildToolDefinitions(),
registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: wrappedCallbacks,
callbacks: {
...wrappedCallbacks,
onTurnDone: (result) => {
if (generation !== this.turnGeneration) return;
this.messages.splice(0, this.messages.length, ...turnMessages);
wrappedCallbacks.onTurnDone(result);
},
onError: (message) => {
if (generation !== this.turnGeneration) return;
// Retain the visible partial answer, without incomplete tool pairs.
if (reply.text) this.messages.push({ role: "assistant", content: reply.text });
wrappedCallbacks.onError(message);
},
},
});
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
}
cancel(): void {
this.turnHandle?.cancel();
if (!this.turnHandle) return;
++this.turnGeneration;
this.turnHandle.cancel();
this.turnHandle = null;
this.lastActivityMs = Date.now();
const reply = this.transcript[this.transcript.length - 1];
if (reply?.role === "assistant" && reply.text) {
this.messages.push({ role: "assistant", content: reply.text });
}
this.status = "Cancelled";
this.changed();
}
private buildToolDefinitions(): LlmToolDefinition[] {
+10 -5
View File
@@ -319,7 +319,7 @@ class DashboardController {
toggle?: TextSettingsEditToggle,
) => this.startTextSettingsEdit(settings, title, onFinish, toggle),
endTextSettingEdit: () => this.endTextSettingEdit(),
startVoiceCapture: () => this.startVoiceCapture(),
startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => this.stopVoiceCapture(),
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
@@ -1874,11 +1874,14 @@ class DashboardController {
* push-to-talk and the Transcribe app. Android mic permission is the consent
* gate even though the audio source is the G2 mic over BLE.
*/
private startVoiceCapture(endpointing = false): void {
this.beginVoiceCapture("ptt", endpointing);
private pttCaptureGeneration = 0;
private startVoiceCapture(endpointing = false): Promise<void> {
return this.beginVoiceCapture("ptt", endpointing);
}
private stopVoiceCapture(): void {
++this.pttCaptureGeneration;
voiceControlBridge.stopPushToTalk();
}
@@ -1944,7 +1947,8 @@ class DashboardController {
return granted;
}
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void {
private async beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): Promise<void> {
const pttGeneration = kind === "ptt" ? ++this.pttCaptureGeneration : 0;
// Preview mode captures from the phone mic (voiceCaptureOptions with a
// null communicator); otherwise a live glasses session must be the source.
const previewCapture = this.isPreviewDisplayActive();
@@ -1952,8 +1956,9 @@ class DashboardController {
return;
}
const communicator = this.communicator;
void ensureVoicePermissions()
await ensureVoicePermissions()
.then(() => {
if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return;
if (previewCapture) {
if (!this.isPreviewDisplayActive()) return;
} else if (this.phase !== "connected" || this.communicator !== communicator) {
+2
View File
@@ -18,6 +18,8 @@ const ICON_STROKE_WIDTH = 2;
// Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed
// against upstream if an icon needs updating.
export const ICON_SVGS = {
"message-circle":
'<svg viewBox="0 0 24 24" fill="none"><path d="M21 11.5a8.4 8.4 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.4 8.4 0 0 1-3.8-.9L3 21l1.9-5.7a8.4 8.4 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.4 8.4 0 0 1 3.8-.9h.5a8.5 8.5 0 0 1 8 8z"/></svg>',
"layout-grid":
'<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>',
timer:
+14 -2
View File
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
* has no menu of its own, and tap-then-hold opens the system menu instead.
*/
menuItems?: () => MenuItem[];
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
holdToTalk?: boolean;
isVoiceCapturing?: () => boolean;
onSystemMenuOpened?: () => void;
onAppMenuOpened?: () => void;
setScreenOn?: ShellWindow["setScreenOn"];
/** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions;
/**
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
shell.openSystemMenu(options.windowId);
return;
}
stack.push(new WindowMenuLayer(options.title, items));
options.onAppMenuOpened?.();
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
};
const window: ShellWindow = {
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// The window's own LayerStack decides per layer whether a swipe is
// directional or falls back to click / double-click.
acceptsDirectional: true,
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
setScreenOn: options.setScreenOn,
heightMode,
hasAppMenu: () => appMenuItems().length > 0,
close: () => {
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// The shell opened its system menu over this window; close our own
// context menu so the two never stack. Never forwarded to app layers.
if (event.type === "system-menu-opened") {
options.onSystemMenuOpened?.();
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
requestRender();
}
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// submenus and app content alike. A plain long-press never arrives:
// the shell keeps it for the system menu (no in-process window claims
// it).
if (event.type === "short-then-long-press") {
if ((!options.holdToTalk && event.type === "short-then-long-press") ||
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
openWindowMenu();
await render(frameId);
return;
+68 -29
View File
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
import { resolveAssistantModel } from "../../assistant/models";
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
@@ -91,6 +93,10 @@ export type ShellWindow = {
* system menu; holding the press past the escape threshold still opens it.
*/
claimsLongPress?: () => boolean;
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
@@ -298,7 +304,29 @@ class Shell {
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private conversations: AssistantConversations | null = null;
private get assistantSession(): AssistantSession | null {
return this.conversations?.current().session ?? null;
}
getAssistantConversations(): AssistantConversations {
if (!this.conversations) {
this.conversations = new AssistantConversations(
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
() => assistantModelSetting.get(),
(value) => setStringSetting("assistant.conversations", value),
getStringSetting("assistant.conversations", ""),
);
}
return this.conversations;
}
async prepareChatVoiceCapture(): Promise<boolean> {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
}
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>();
@@ -559,7 +587,7 @@ class Shell {
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -645,6 +673,7 @@ class Shell {
// Even AI app never launches, so the firmware does not power the display
// for us either -- actions that need it wake the screen themselves.
if (event.type === "wakeword") {
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
const action = wakeWordActionSetting.get();
if (action === "off") {
return { shell: false, window: false };
@@ -696,11 +725,11 @@ class Shell {
if (!window) {
return { shell: true, window: false };
}
if (!window.claimsLongPress?.()) {
if (!window.holdToTalk && !window.claimsLongPress?.()) {
this.openEscapeMenu();
return { shell: true, window: false };
}
this.startEscapeMenuTimer();
if (!window.holdToTalk) this.startEscapeMenuTimer();
if (this.focus === "sidebar") {
this.focus = "window";
window.onFocus?.(this.lastInput);
@@ -716,6 +745,10 @@ class Shell {
// when it has none.
if (event.type === "short-then-long-press") {
const window = this.foregroundWindow();
if (window?.holdToTalk) {
this.openEscapeMenu();
return { shell: true, window: false };
}
// Over the open system menu it switches to the app's context menu:
// close the system menu and deliver the gesture to the window as
// usual. A window without a menu of its own would only ask for the
@@ -1024,7 +1057,7 @@ class Shell {
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer) return null;
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
if (!this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
@@ -1069,7 +1102,8 @@ class Shell {
}
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
const current = this.getAssistantConversations().current();
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
}
/** Observe assistant turns (see AssistantActivityEvent). */
@@ -1107,20 +1141,13 @@ class Shell {
}
private ensureAssistantSession(): AssistantSession | null {
const config = this.resolveAssistantConfiguration();
if (!config) return null;
if (
!this.assistantSession ||
this.assistantSession.isExpired() ||
!this.assistantSession.matchesConfiguration(config)
) {
this.assistantSession?.cancel();
this.assistantSession = new AssistantSession(config);
}
return this.assistantSession;
return this.getAssistantConversations().ensureSession();
}
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
private resolveAssistantConfiguration(
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
): AssistantBackendConfig | null {
if (assistantBackendSetting.get() === "external") {
const host = assistantBridgeHostSetting.get().trim();
const token = assistantBridgeTokenSetting.get();
@@ -1128,10 +1155,11 @@ class Shell {
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
return { kind: "external", bridge: { host, port, token } };
}
const llm = resolveAssistantModel(assistantModelSetting.get(), {
const llm = resolveAssistantModel(model, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null;
}
@@ -1151,7 +1179,9 @@ class Shell {
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay.
*/
sendToAssistant(text: string): void {
sendToAssistant(text: string, showOverlay = true): void {
text = text.trim();
if (!text) return;
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert(
@@ -1161,7 +1191,15 @@ class Shell {
);
return;
}
if (session.isTurnActive()) {
this.showAlert("The assistant is still working on the previous request");
return;
}
if (!this.screenOn) this.wake("sidebar");
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
this.runAssistantTurn(session, null, text);
return;
}
let layer = this.assistantLayer;
if (!layer) {
const created = new AssistantLayer(this.config.actions, {
@@ -1170,7 +1208,7 @@ class Shell {
onClose: () => this.closeAssistantLayer(),
onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the reference so a later query starts clean.
// stop the turn and drop the overlay reference; keep shared history.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" });
@@ -1184,23 +1222,24 @@ class Shell {
this.config.requestShellRender();
}
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
if (session.isTurnActive()) return;
layer?.startTurn();
let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar;
layer.onTextDelta(delta, textSoFar);
layer?.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
},
onToolActivity: (label) => layer.onToolActivity(label),
onToolActivity: (label) => layer?.onToolActivity(label),
onTurnDone: () => {
layer.onTurnDone();
layer?.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar });
},
onError: (message) => {
layer.onError(message);
layer?.onError(message);
this.emitAssistantActivity({ phase: "error", text: message });
},
});
@@ -1356,7 +1395,7 @@ class Shell {
// when the app has one (otherwise it opens this very menu, not worth a
// hint).
const footer = foreground.hasAppMenu?.()
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
: undefined;
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
layer.selectItem(initialSelection);
+5 -3
View File
@@ -1,6 +1,6 @@
import { GrayImage } from "../graphics/image";
import { singlePlane, type Plane } from "../graphics/plane";
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
import { type Layer, LayerStack, noopLayerActions } from "./layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
import type { WorkerAppReply } from "./shell/worker-window";
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
};
export class WindowMenuLayer extends MenuLayer {
constructor(title: string | null, items: MenuItem[]) {
super(title, items, WINDOW_MENU_LAYOUT);
constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
super(title, items, holdToTalk
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
: WINDOW_MENU_LAYOUT);
}
}