Add "AI Chat" app.

This commit is contained in:
James Babcock committed 2026-09-09 01:15:19 -07:00
1 parent 78c56ffc4e
commit 0765e943fb
13 files changed
+616 -87

No files matched your search

+1
View File
@@ -5,6 +5,7 @@
* Transcribe app: Scrolling; automatic reconnection; insert newlines * Transcribe app: Scrolling; automatic reconnection; insert newlines
* Notifications: Add notification filters * Notifications: Add notification filters
* Phone screen preview: Add rotation option; improve landscape layout * Phone screen preview: Add rotation option; improve landscape layout
* Add "AI Chat" app (conversational-style use of the voice assistant with hold-to-talk)
## 0.6.5 - 2026-09-04 ## 0.6.5 - 2026-09-04
* Changed context menu gestures: — for system, ●— for app context menu. * Changed context menu gestures: — for system, ●— for app context menu.
+166
View File
@@ -0,0 +1,166 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
import type { AssistantTranscriptEntry } from "../../assistant/session";
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
import type { Layer, LayerContext } from "../../ui/layers";
import type { MenuItem } from "../../ui/menu";
import { WindowMenuLayer } from "../../ui/window-menu";
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
import { shell } from "../../ui/shell/shell";
import { VoiceDraft } from "./voice-draft";
class AiChatLayer implements Layer {
readonly acceptsDirectional = true;
private firstLine: number | null = null; // null follows the streaming tail
private maxFirstLine = 0;
private selectedId = "";
private requestRender = () => {};
private readonly wrappedEntries = new WeakMap<AssistantTranscriptEntry, {
font: UiFont; width: number; text: string; lines: string[];
}>();
readonly draft: VoiceDraft;
constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) {
this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(),
() => this.requestRender(), (text) => this.send(text));
}
start(render: () => void): () => void {
this.requestRender = render;
return this.conversations.onChanged(() => render());
}
send(text: string): void {
this.firstLine = null;
shell.sendToAssistant(text, false);
}
paint(ctx: LayerContext): GrayImage {
const size = ctx.stack.getBaseSize();
const image = new GrayImage(size.width, size.height);
const font = getDefaultSmallFont();
const step = font.lineHeight + 3;
const left = 12;
const width = size.width - 28;
const record = this.conversations.current();
if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; }
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
const title = this.conversations.title();
image.drawText(font, left, 6, truncateText(font, title, width), 180);
const label = assistantBackendSetting.get() === "external" ? "External agent" :
`${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`;
const footer = wrapText(font, `${label} ${gestureHints([
[GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"],
])}`, width);
const footerY = size.height - step * footer.length;
const footerTop = footerY - step * 3 - 12;
const historyTop = step + 16;
const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step));
const lines: { text: string; color: number }[] = [];
for (const entry of transcript) {
if (!entry.text) continue;
let wrapped = this.wrappedEntries.get(entry);
if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) {
wrapped = { font, width, text: entry.text,
lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) };
this.wrappedEntries.set(entry, wrapped);
}
for (const text of wrapped.lines) {
lines.push({ text, color: entry.role === "user" ? 175 : 245 });
}
lines.push({ text: "", color: 0 });
}
if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 },
{ text: "Voice and chat share this conversation.", color: 150 });
this.maxFirstLine = Math.max(0, lines.length - visible);
const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine);
for (let row = 0; row < visible; row++) {
const line = lines[first + row];
if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color);
}
if (this.maxFirstLine > 0) {
const track = visible * step;
const thumb = Math.max(8, Math.floor(track * visible / lines.length));
image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110);
}
image.fillRect(left, footerTop, width, 1, 65);
const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status;
if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160);
const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width);
pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220));
footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120));
return image;
}
handleInput(event: InputEvent): void {
switch (event.type) {
case "scroll-up": case "swipe-up":
this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return;
case "scroll-down": case "swipe-down": {
const next = (this.firstLine ?? this.maxFirstLine) + 3;
this.firstLine = next >= this.maxFirstLine ? null : next; return;
}
case "long-press":
if (!this.conversations.current().session?.isTurnActive()) {
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
void this.draft.start();
}
return;
case "long-press-release": this.draft.release(); return;
case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return;
}
}
menuItems(): MenuItem[] {
const record = this.conversations.current();
const busy = () => this.draft.active || Boolean(record.session?.isTurnActive());
const external = assistantBackendSetting.get() === "external";
const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true));
return [
...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []),
{ label: "New session", disabled: () => external || busy(), onSelect: (ctx) => {
if (this.conversations.create()) ctx.stack.clearToBase();
} },
{ label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions",
[...this.conversations.list()].reverse().map((item) => ({
label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`,
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
}))) },
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
ASSISTANT_MODEL_VALUES.map((model) => ({
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
disabled: () => !this.conversations.available(model, record.reasoning),
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
}))) },
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
return external || busy() || !llm?.effort;
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); },
}))) },
...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => {
record.session?.cancel(); ctx.stack.clearToBase();
} }] : []),
];
}
}
export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow {
const layer = new AiChatLayer(shell.getAssistantConversations(), options);
let unsubscribe = () => {};
const app = createInProcessWindow({
...options,
appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true,
baseLayer: layer, holdToTalk: true,
isVoiceCapturing: () => layer.draft.active,
onSystemMenuOpened: () => layer.draft.cancel(),
onAppMenuOpened: () => layer.draft.cancel(),
setScreenOn: (on) => { if (!on) layer.draft.cancel(); },
setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); },
menuItems: () => layer.menuItems(),
receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); },
onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); },
});
unsubscribe = layer.start(app.requestRender);
return app;
}
+10
View File
@@ -0,0 +1,10 @@
import type { AppDefinition } from "../app-definition";
import { createAiChatWindow } from "./ai-chat-app";
const aiChatApp: AppDefinition = {
appId: "ai-chat",
title: "AI Chat",
icon: "message-circle",
launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow),
};
export default aiChatApp;
+125
View File
@@ -0,0 +1,125 @@
import { voiceControlBridge } from "../../native/voice-control";
import { voiceActivity } from "../../ui/shell/voice-activity";
import type { LayerActions } from "../../ui/layers";
/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */
export class VoiceDraft {
phase: "idle" | "preparing" | "listening" | "finishing" = "idle";
text = "";
status = "";
private committed = "";
private live = "";
private generation = 0;
private started = false;
private ownsVoiceActivity = false;
private starting = false;
private unsubscribe: (() => void)[] = [];
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(
private readonly actions: Pick<LayerActions, "startVoiceCapture" | "stopVoiceCapture">,
private readonly prepare: () => Promise<boolean>,
private readonly changed: () => void,
private readonly send: (text: string) => void,
) {}
get active(): boolean { return this.phase !== "idle" || this.starting; }
async start(): Promise<void> {
if (this.active) return;
const generation = ++this.generation;
this.text = this.committed = this.live = "";
this.phase = "preparing";
this.status = "Preparing microphone...";
this.changed();
try {
const ready = await this.prepare();
if (generation !== this.generation) return;
if (!ready) { this.cancel("Microphone unavailable"); return; }
this.phase = "listening";
this.status = "Listening... release to send";
this.ownsVoiceActivity = true;
voiceActivity.setActive(true);
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
if (!this.active || generation !== this.generation) return;
if (event.isFinal) {
const text = event.text.trim();
if (text) this.committed = [this.committed, text].filter(Boolean).join(" ");
this.live = "";
} else this.live = event.text.trim();
this.text = [this.committed, this.live].filter(Boolean).join(" ");
if (event.isFinal && this.phase === "finishing") this.finish();
else this.changed();
}));
// onStatus immediately replays the previous capture's status.
let subscribed = false;
this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => {
if (!subscribed) return;
if (generation !== this.generation || !this.active) return;
if (/ignored|failed|error|denied/i.test(status)) this.cancel(status);
}));
subscribed = true;
if (generation !== this.generation) return;
this.starting = true;
this.changed();
await this.actions.startVoiceCapture();
this.starting = false;
if (generation !== this.generation) {
await this.actions.stopVoiceCapture();
this.changed();
return;
}
this.started = true;
if (this.isFinishing()) this.stopAndFinalize();
} catch (error) {
this.starting = false;
if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`);
}
}
private isFinishing(): boolean { return this.phase === "finishing"; }
release(): void {
if (this.phase === "preparing") { this.cancel(); return; }
if (this.phase !== "listening") return;
this.phase = "finishing";
this.status = "Finishing transcription...";
this.changed();
if (!this.starting) this.stopAndFinalize();
}
private stopAndFinalize(): void {
// Install fallback first: stop may synchronously deliver the final.
this.timer = setTimeout(() => this.finish(), 1500);
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
}
}
private finish(): void {
if (this.phase !== "finishing") return;
const text = this.text.trim();
this.cancel(text ? "" : "No speech detected");
if (text) this.send(text);
}
cancel(status = ""): void {
++this.generation;
this.phase = "idle";
this.status = status;
this.text = this.live = this.committed = "";
if (this.timer !== null) clearTimeout(this.timer);
this.timer = null;
for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {});
}
if (this.ownsVoiceActivity) {
this.ownsVoiceActivity = false;
voiceActivity.setActive(false);
}
this.changed();
}
}
+2
View File
@@ -1,4 +1,5 @@
import { type AppDefinition } from "./app-definition"; import { type AppDefinition } from "./app-definition";
import aiChatApp from "./ai-chat";
import launcherApp from "./launcher"; import launcherApp from "./launcher";
import timerApp from "./timer"; import timerApp from "./timer";
import calculatorApp from "./calculator"; import calculatorApp from "./calculator";
@@ -30,6 +31,7 @@ import settingsApp from "./settings";
*/ */
export const ALL_APPS: readonly AppDefinition[] = [ export const ALL_APPS: readonly AppDefinition[] = [
launcherApp, launcherApp,
aiChatApp,
timerApp, timerApp,
calculatorApp, calculatorApp,
terminalApp, terminalApp,
+95
View File
@@ -0,0 +1,95 @@
import { AssistantSession, type AssistantBackendConfig, type AssistantSessionHistory } from "./session";
import { ASSISTANT_MODEL_VALUES, type AssistantModel } from "./models";
export type ReasoningLevel = "default" | "low" | "medium" | "high";
export type Conversation = {
id: string;
model: AssistantModel;
reasoning: ReasoningLevel;
session: AssistantSession | null;
history?: AssistantSessionHistory;
};
type SavedConversation = Omit<Conversation, "session">;
type SavedConversations = { selectedId: string; conversations: SavedConversation[] };
/** Shared selection and history for every assistant entry point. No credentials are saved. */
export class AssistantConversations {
private records: Conversation[] = [];
private selectedId = "";
private readonly listeners = new Set<() => void>();
constructor(
private readonly resolve: (model: AssistantModel, reasoning: ReasoningLevel) => AssistantBackendConfig | null,
private readonly defaultModel: () => AssistantModel,
private readonly persist: (value: string) => void,
saved = "",
) {
try {
const data: SavedConversations = JSON.parse(saved);
if (Array.isArray(data.conversations)) {
this.records = data.conversations.filter((record) =>
typeof record.id === "string" && ASSISTANT_MODEL_VALUES.includes(record.model) &&
["default", "low", "medium", "high"].includes(record.reasoning) &&
Array.isArray(record.history?.messages) && Array.isArray(record.history?.transcript),
).map((record) => ({ ...record, session: null }));
this.selectedId = data.selectedId;
}
} catch { /* First launch or invalid saved state. */ }
if (!this.records.length) this.records.push(this.makeConversation());
if (!this.records.some((record) => record.id === this.selectedId)) this.selectedId = this.records[0]!.id;
}
current(): Conversation { return this.records.find((record) => record.id === this.selectedId)!; }
list(): readonly Conversation[] { return this.records; }
title(record = this.current()): string {
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
return transcript.find((entry) => entry.role === "user")?.text.replace(/\s+/g, " ").slice(0, 60) || "New session";
}
onChanged(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
private changed(save = true): void {
if (save) this.persist(JSON.stringify({ selectedId: this.selectedId, conversations: this.records.map(
({ session, ...record }) => ({ ...record, history: session?.history() ?? record.history ?? { messages: [], transcript: [] } }),
) }));
for (const listener of this.listeners) listener();
}
private makeConversation(): Conversation {
return { id: `${Date.now()}-${Math.random().toString(36).slice(2)}`, model: this.defaultModel(), reasoning: "default", session: null };
}
create(): boolean {
if (this.current().session?.isTurnActive()) return false;
const record = this.makeConversation();
this.records.push(record);
this.selectedId = record.id;
this.changed();
return true;
}
select(id: string): boolean {
if (this.current().session?.isTurnActive() || !this.records.some((record) => record.id === id)) return false;
this.selectedId = id;
this.changed();
return true;
}
configure(model: AssistantModel, reasoning: ReasoningLevel): boolean {
const record = this.current();
if (record.session?.isTurnActive() || !this.resolve(model, reasoning)) return false;
record.model = model;
record.reasoning = reasoning;
this.ensureSession();
this.changed();
return true;
}
available(model: AssistantModel, reasoning: ReasoningLevel): boolean { return this.resolve(model, reasoning) !== null; }
ensureSession(): AssistantSession | null {
const record = this.current();
const config = this.resolve(record.model, record.reasoning);
if (!config) return null;
if (!record.session) {
record.session = new AssistantSession(config, undefined, record.history);
record.session.onChanged(() => this.changed(!record.session!.isTurnActive()));
} else record.session.configure(config);
return record.session;
}
}
+5 -1
View File
@@ -50,6 +50,7 @@ export class DirectAssistantBackend {
const startMs = Date.now(); const startMs = Date.now();
let cancelled = false; let cancelled = false;
let streamHandle: LlmStreamHandle | null = null; let streamHandle: LlmStreamHandle | null = null;
let priorText = "";
const finishError = (message: string) => { const finishError = (message: string) => {
if (cancelled) return; if (cancelled) return;
@@ -78,7 +79,7 @@ export class DirectAssistantBackend {
tools: options.buildTools(), tools: options.buildTools(),
onTextDelta: (delta, textSoFar) => { onTextDelta: (delta, textSoFar) => {
if (cancelled) return; if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar); options.callbacks.onTextDelta(delta, priorText + textSoFar);
}, },
onDone: (result) => { onDone: (result) => {
streamHandle = null; streamHandle = null;
@@ -87,6 +88,9 @@ export class DirectAssistantBackend {
finishError("The assistant declined this request"); finishError("The assistant declined this request");
return; return;
} }
const fullText = priorText + result.text;
options.callbacks.onTextDelta("", fullText);
priorText = fullText ? `${fullText}\n\n` : "";
// Record what the model produced this iteration. // Record what the model produced this iteration.
const assistantContent: LlmContentBlock[] = result.content.length const assistantContent: LlmContentBlock[] = result.content.length
? result.content ? result.content
+113 -47
View File
@@ -15,26 +15,20 @@ import type {
AssistantTurnHandle, AssistantTurnHandle,
} from "./types"; } from "./types";
/** /** One shared conversation, independent of the voice overlay or chat window. */
* One assistant conversation. Owns the message history and turn state and export type AssistantTranscriptEntry = { role: "user" | "assistant"; text: string };
* drives either the on-phone provider loop (direct) or the external agent export type AssistantSessionHistory = {
* bridge. The UI talks only to this surface, so AssistantLayer is messages: LlmMessage[];
* backend-agnostic. transcript: AssistantTranscriptEntry[];
* /** Model identity for safe replay after Auto resolves differently on restart. */
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh engine?: string;
* one for a new wakeword and reuses the current one for a follow-up. In };
* external mode the conversation history lives on the agent's machine (one
* long-lived session there), so expiry here only affects the overlay UI.
*/
/** Which engine answers utterances, plus what it needs to do so. */ /** Which engine answers utterances, plus what it needs to do so. */
export type AssistantBackendConfig = export type AssistantBackendConfig =
| { kind: "direct"; llm: ResolvedAssistantModel } | { kind: "direct"; llm: ResolvedAssistantModel }
| { kind: "external"; bridge: AssistantBridgeConfig }; | { kind: "external"; bridge: AssistantBridgeConfig };
/** A new PTT after this much idle starts a fresh conversation. */
const SESSION_IDLE_MS = 10 * 60 * 1000;
/** Trim history from the head once it grows past this many messages. */ /** Trim history from the head once it grows past this many messages. */
const MAX_HISTORY_MESSAGES = 40; const MAX_HISTORY_MESSAGES = 40;
@@ -42,38 +36,61 @@ export class AssistantSession {
private readonly directBackend = new DirectAssistantBackend(); private readonly directBackend = new DirectAssistantBackend();
private readonly messages: LlmMessage[] = []; private readonly messages: LlmMessage[] = [];
private turnHandle: AssistantTurnHandle | null = null; private turnHandle: AssistantTurnHandle | null = null;
private lastActivityMs = Date.now(); readonly transcript: AssistantTranscriptEntry[] = [];
status = "";
private readonly listeners = new Set<() => void>();
private turnGeneration = 0;
// API-safe tool name -> canonical registry name. Provider APIs restrict tool // API-safe tool name -> canonical registry name. Provider APIs restrict tool
// names, so dotted registry names are sanitized and mapped back on calls. // names, so dotted registry names are sanitized and mapped back on calls.
private readonly toolNameMap = new Map<string, string>(); private readonly toolNameMap = new Map<string, string>();
constructor( constructor(
private readonly config: AssistantBackendConfig, private config: AssistantBackendConfig,
private readonly registry: ToolRegistry = toolRegistry, private readonly registry: ToolRegistry = toolRegistry,
) {} history?: AssistantSessionHistory,
) {
matchesConfiguration(config: AssistantBackendConfig): boolean { if (history) {
if (this.config.kind !== config.kind) return false; this.messages.push(...history.messages);
if (this.config.kind === "direct" && config.kind === "direct") { this.transcript.push(...history.transcript);
return ( if (history.engine !== this.engine()) this.rebuildTextHistory();
this.config.llm.provider === config.llm.provider &&
this.config.llm.model === config.llm.model &&
this.config.llm.apiKey === config.llm.apiKey
);
} }
if (this.config.kind === "external" && config.kind === "external") {
return (
this.config.bridge.host === config.bridge.host &&
this.config.bridge.port === config.bridge.port &&
this.config.bridge.token === config.bridge.token
);
}
return false;
} }
/** Whether this session has been idle long enough to retire. */ onChanged(listener: () => void): () => void {
isExpired(nowMs = Date.now()): boolean { this.listeners.add(listener);
return nowMs - this.lastActivityMs > SESSION_IDLE_MS; return () => this.listeners.delete(listener);
}
private changed(): void {
for (const listener of this.listeners) {
try { listener(); } catch (error) { console.warn("assistant history listener failed", error); }
}
}
history(): AssistantSessionHistory {
return { messages: this.messages, transcript: this.transcript, engine: this.engine() };
}
configure(config: AssistantBackendConfig): void {
if (this.isTurnActive()) return;
// Opaque reasoning/tool provider items are only replayable on their
// original model. Preserve visible conversation when switching engines.
if (this.config.kind !== config.kind ||
(this.config.kind === "direct" && config.kind === "direct" &&
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) {
this.rebuildTextHistory();
}
this.config = config;
}
private engine(): string {
return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external";
}
private rebuildTextHistory(): void {
this.messages.splice(0, this.messages.length, ...this.transcript
.filter((entry) => entry.text.trim())
.map((entry) => ({ role: entry.role, content: entry.text })));
} }
isTurnActive(): boolean { isTurnActive(): boolean {
@@ -86,20 +103,44 @@ export class AssistantSession {
callbacks.onError("The assistant is still working on the previous request"); callbacks.onError("The assistant is still working on the previous request");
return; return;
} }
this.lastActivityMs = Date.now(); text = text.trim();
if (!text) return;
const generation = ++this.turnGeneration;
const reply: AssistantTranscriptEntry = { role: "assistant", text: "" };
this.transcript.push({ role: "user", text }, reply);
this.status = "Thinking...";
// Install a sentinel before calling providers: some failures are synchronous.
this.turnHandle = { cancel: () => {} };
this.changed();
const finish = () => { const finish = () => {
++this.turnGeneration;
this.turnHandle = null; this.turnHandle = null;
this.lastActivityMs = Date.now(); this.changed();
}; };
const wrappedCallbacks: AssistantTurnCallbacks = { const wrappedCallbacks: AssistantTurnCallbacks = {
onTextDelta: callbacks.onTextDelta, onTextDelta: (delta, full) => {
onToolActivity: callbacks.onToolActivity, if (generation !== this.turnGeneration) return;
reply.text = full;
this.status = "Thinking...";
this.changed();
callbacks.onTextDelta(delta, full);
},
onToolActivity: (label) => {
if (generation !== this.turnGeneration) return;
this.status = `→ ${label}`;
this.changed();
callbacks.onToolActivity(label);
},
onTurnDone: (result) => { onTurnDone: (result) => {
if (generation !== this.turnGeneration) return;
this.status = "";
finish(); finish();
callbacks.onTurnDone(result); callbacks.onTurnDone(result);
}, },
onError: (message) => { onError: (message) => {
if (generation !== this.turnGeneration) return;
this.status = message;
finish(); finish();
callbacks.onError(message); callbacks.onError(message);
}, },
@@ -108,7 +149,8 @@ export class AssistantSession {
if (this.config.kind === "external") { if (this.config.kind === "external") {
// History and the agent loop live on the agent's machine; the phone just // History and the agent loop live on the agent's machine; the phone just
// streams this turn. The overlay keeps its own display state. // streams this turn. The overlay keeps its own display state.
this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks); const handle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
return; return;
} }
@@ -123,24 +165,48 @@ export class AssistantSession {
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text, content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
}); });
this.trimHistory(); this.trimHistory();
this.turnHandle = this.directBackend.runTurn({ // Work on a turn-local array: cancelled tools may finish later, and must
// never append orphan results into a subsequent turn's context.
const turnMessages = this.messages.slice();
const handle = this.directBackend.runTurn({
provider: llm.provider, provider: llm.provider,
apiKey: llm.apiKey, apiKey: llm.apiKey,
model: llm.model, model: llm.model,
effort: llm.effort, effort: llm.effort,
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
messages: this.messages, messages: turnMessages,
buildTools: () => this.buildToolDefinitions(), buildTools: () => this.buildToolDefinitions(),
registry: this.registry, registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName, resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: wrappedCallbacks, callbacks: {
...wrappedCallbacks,
onTurnDone: (result) => {
if (generation !== this.turnGeneration) return;
this.messages.splice(0, this.messages.length, ...turnMessages);
wrappedCallbacks.onTurnDone(result);
},
onError: (message) => {
if (generation !== this.turnGeneration) return;
// Retain the visible partial answer, without incomplete tool pairs.
if (reply.text) this.messages.push({ role: "assistant", content: reply.text });
wrappedCallbacks.onError(message);
},
},
}); });
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
} }
cancel(): void { cancel(): void {
this.turnHandle?.cancel(); if (!this.turnHandle) return;
++this.turnGeneration;
this.turnHandle.cancel();
this.turnHandle = null; this.turnHandle = null;
this.lastActivityMs = Date.now(); const reply = this.transcript[this.transcript.length - 1];
if (reply?.role === "assistant" && reply.text) {
this.messages.push({ role: "assistant", content: reply.text });
}
this.status = "Cancelled";
this.changed();
} }
private buildToolDefinitions(): LlmToolDefinition[] { private buildToolDefinitions(): LlmToolDefinition[] {
+10 -5
View File
@@ -319,7 +319,7 @@ class DashboardController {
toggle?: TextSettingsEditToggle, toggle?: TextSettingsEditToggle,
) => this.startTextSettingsEdit(settings, title, onFinish, toggle), ) => this.startTextSettingsEdit(settings, title, onFinish, toggle),
endTextSettingEdit: () => this.endTextSettingEdit(), endTextSettingEdit: () => this.endTextSettingEdit(),
startVoiceCapture: () => this.startVoiceCapture(), startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
stopVoiceCapture: () => this.stopVoiceCapture(), stopVoiceCapture: () => this.stopVoiceCapture(),
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
@@ -1874,11 +1874,14 @@ class DashboardController {
* push-to-talk and the Transcribe app. Android mic permission is the consent * push-to-talk and the Transcribe app. Android mic permission is the consent
* gate even though the audio source is the G2 mic over BLE. * gate even though the audio source is the G2 mic over BLE.
*/ */
private startVoiceCapture(endpointing = false): void { private pttCaptureGeneration = 0;
this.beginVoiceCapture("ptt", endpointing);
private startVoiceCapture(endpointing = false): Promise<void> {
return this.beginVoiceCapture("ptt", endpointing);
} }
private stopVoiceCapture(): void { private stopVoiceCapture(): void {
++this.pttCaptureGeneration;
voiceControlBridge.stopPushToTalk(); voiceControlBridge.stopPushToTalk();
} }
@@ -1944,7 +1947,8 @@ class DashboardController {
return granted; return granted;
} }
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void { private async beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): Promise<void> {
const pttGeneration = kind === "ptt" ? ++this.pttCaptureGeneration : 0;
// Preview mode captures from the phone mic (voiceCaptureOptions with a // Preview mode captures from the phone mic (voiceCaptureOptions with a
// null communicator); otherwise a live glasses session must be the source. // null communicator); otherwise a live glasses session must be the source.
const previewCapture = this.isPreviewDisplayActive(); const previewCapture = this.isPreviewDisplayActive();
@@ -1952,8 +1956,9 @@ class DashboardController {
return; return;
} }
const communicator = this.communicator; const communicator = this.communicator;
void ensureVoicePermissions() await ensureVoicePermissions()
.then(() => { .then(() => {
if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return;
if (previewCapture) { if (previewCapture) {
if (!this.isPreviewDisplayActive()) return; if (!this.isPreviewDisplayActive()) return;
} else if (this.phase !== "connected" || this.communicator !== communicator) { } else if (this.phase !== "connected" || this.communicator !== communicator) {
+2
View File
@@ -18,6 +18,8 @@ const ICON_STROKE_WIDTH = 2;
// Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed // Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed
// against upstream if an icon needs updating. // against upstream if an icon needs updating.
export const ICON_SVGS = { export const ICON_SVGS = {
"message-circle":
'<svg viewBox="0 0 24 24" fill="none"><path d="M21 11.5a8.4 8.4 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.4 8.4 0 0 1-3.8-.9L3 21l1.9-5.7a8.4 8.4 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.4 8.4 0 0 1 3.8-.9h.5a8.5 8.5 0 0 1 8 8z"/></svg>',
"layout-grid": "layout-grid":
'<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>', '<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>',
timer: timer:
+14 -2
View File
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
* has no menu of its own, and tap-then-hold opens the system menu instead. * has no menu of its own, and tap-then-hold opens the system menu instead.
*/ */
menuItems?: () => MenuItem[]; menuItems?: () => MenuItem[];
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
holdToTalk?: boolean;
isVoiceCapturing?: () => boolean;
onSystemMenuOpened?: () => void;
onAppMenuOpened?: () => void;
setScreenOn?: ShellWindow["setScreenOn"];
/** Shared actions; requestRender is rebound to this window's render. */ /** Shared actions; requestRender is rebound to this window's render. */
actions: LayerActions; actions: LayerActions;
/** /**
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
shell.openSystemMenu(options.windowId); shell.openSystemMenu(options.windowId);
return; return;
} }
stack.push(new WindowMenuLayer(options.title, items)); options.onAppMenuOpened?.();
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
}; };
const window: ShellWindow = { const window: ShellWindow = {
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// The window's own LayerStack decides per layer whether a swipe is // The window's own LayerStack decides per layer whether a swipe is
// directional or falls back to click / double-click. // directional or falls back to click / double-click.
acceptsDirectional: true, acceptsDirectional: true,
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
setScreenOn: options.setScreenOn,
heightMode, heightMode,
hasAppMenu: () => appMenuItems().length > 0, hasAppMenu: () => appMenuItems().length > 0,
close: () => { close: () => {
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// The shell opened its system menu over this window; close our own // The shell opened its system menu over this window; close our own
// context menu so the two never stack. Never forwarded to app layers. // context menu so the two never stack. Never forwarded to app layers.
if (event.type === "system-menu-opened") { if (event.type === "system-menu-opened") {
options.onSystemMenuOpened?.();
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) { if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
requestRender(); requestRender();
} }
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
// submenus and app content alike. A plain long-press never arrives: // submenus and app content alike. A plain long-press never arrives:
// the shell keeps it for the system menu (no in-process window claims // the shell keeps it for the system menu (no in-process window claims
// it). // it).
if (event.type === "short-then-long-press") { if ((!options.holdToTalk && event.type === "short-then-long-press") ||
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
openWindowMenu(); openWindowMenu();
await render(frameId); await render(frameId);
return; return;
+68 -29
View File
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
import { voiceActivity } from "./voice-activity"; import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant"; import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session"; import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
import { resolveAssistantModel } from "../../assistant/models"; import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import type { AssistantContext } from "../../assistant/types"; import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications"; import { SingleNotificationLayer } from "../notifications";
import { import {
@@ -91,6 +93,10 @@ export type ShellWindow = {
* system menu; holding the press past the escape threshold still opens it. * system menu; holding the press past the escape threshold still opens it.
*/ */
claimsLongPress?: () => boolean; claimsLongPress?: () => boolean;
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/** /**
* True when the window gives swipe-left / swipe-right (watch directional * True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event). * input) a meaning; otherwise the shell forwards directionalFallback(event).
@@ -298,7 +304,29 @@ class Shell {
private readonly trayIcons = new Map<string, GrayImage>(); private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null; private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null; private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null; private conversations: AssistantConversations | null = null;
private get assistantSession(): AssistantSession | null {
return this.conversations?.current().session ?? null;
}
getAssistantConversations(): AssistantConversations {
if (!this.conversations) {
this.conversations = new AssistantConversations(
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
() => assistantModelSetting.get(),
(value) => setStringSetting("assistant.conversations", value),
getStringSetting("assistant.conversations", ""),
);
}
return this.conversations;
}
async prepareChatVoiceCapture(): Promise<boolean> {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
}
private assistantLayer: AssistantLayer | null = null; private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>(); private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>(); private readonly alertListeners = new Set<(text: string) => void>();
@@ -559,7 +587,7 @@ class Shell {
// ring. An in-flight assistant turn suspends it for the same reason (a // ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and // tool loop can run for a while with no input); once the turn ends and
// the Follow-up/Done menu is showing, the normal idle timeout resumes. // the Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) { if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
this.lastInputAtMs = nowMs; this.lastInputAtMs = nowMs;
return false; return false;
} }
@@ -645,6 +673,7 @@ class Shell {
// Even AI app never launches, so the firmware does not power the display // Even AI app never launches, so the firmware does not power the display
// for us either -- actions that need it wake the screen themselves. // for us either -- actions that need it wake the screen themselves.
if (event.type === "wakeword") { if (event.type === "wakeword") {
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
const action = wakeWordActionSetting.get(); const action = wakeWordActionSetting.get();
if (action === "off") { if (action === "off") {
return { shell: false, window: false }; return { shell: false, window: false };
@@ -696,11 +725,11 @@ class Shell {
if (!window) { if (!window) {
return { shell: true, window: false }; return { shell: true, window: false };
} }
if (!window.claimsLongPress?.()) { if (!window.holdToTalk && !window.claimsLongPress?.()) {
this.openEscapeMenu(); this.openEscapeMenu();
return { shell: true, window: false }; return { shell: true, window: false };
} }
this.startEscapeMenuTimer(); if (!window.holdToTalk) this.startEscapeMenuTimer();
if (this.focus === "sidebar") { if (this.focus === "sidebar") {
this.focus = "window"; this.focus = "window";
window.onFocus?.(this.lastInput); window.onFocus?.(this.lastInput);
@@ -716,6 +745,10 @@ class Shell {
// when it has none. // when it has none.
if (event.type === "short-then-long-press") { if (event.type === "short-then-long-press") {
const window = this.foregroundWindow(); const window = this.foregroundWindow();
if (window?.holdToTalk) {
this.openEscapeMenu();
return { shell: true, window: false };
}
// Over the open system menu it switches to the app's context menu: // Over the open system menu it switches to the app's context menu:
// close the system menu and deliver the gesture to the window as // close the system menu and deliver the gesture to the window as
// usual. A window without a menu of its own would only ask for the // usual. A window without a menu of its own would only ask for the
@@ -1024,7 +1057,7 @@ class Shell {
*/ */
startKeyboardInput(): KeyboardInputSession | null { startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer; if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer) return null; if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
if (!this.screenOn) this.wake("sidebar"); if (!this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer; const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession; const assistantSession = this.assistantSession;
@@ -1069,7 +1102,8 @@ class Shell {
} }
isAssistantAvailable(): boolean { isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null; const current = this.getAssistantConversations().current();
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
} }
/** Observe assistant turns (see AssistantActivityEvent). */ /** Observe assistant turns (see AssistantActivityEvent). */
@@ -1107,20 +1141,13 @@ class Shell {
} }
private ensureAssistantSession(): AssistantSession | null { private ensureAssistantSession(): AssistantSession | null {
const config = this.resolveAssistantConfiguration(); return this.getAssistantConversations().ensureSession();
if (!config) return null;
if (
!this.assistantSession ||
this.assistantSession.isExpired() ||
!this.assistantSession.matchesConfiguration(config)
) {
this.assistantSession?.cancel();
this.assistantSession = new AssistantSession(config);
}
return this.assistantSession;
} }
private resolveAssistantConfiguration(): AssistantBackendConfig | null { private resolveAssistantConfiguration(
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
): AssistantBackendConfig | null {
if (assistantBackendSetting.get() === "external") { if (assistantBackendSetting.get() === "external") {
const host = assistantBridgeHostSetting.get().trim(); const host = assistantBridgeHostSetting.get().trim();
const token = assistantBridgeTokenSetting.get(); const token = assistantBridgeTokenSetting.get();
@@ -1128,10 +1155,11 @@ class Shell {
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790; const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
return { kind: "external", bridge: { host, port, token } }; return { kind: "external", bridge: { host, port, token } };
} }
const llm = resolveAssistantModel(assistantModelSetting.get(), { const llm = resolveAssistantModel(model, {
anthropic: anthropicApiKeySetting.get(), anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(), openai: openAiApiKeySetting.get(),
}); });
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null; return llm ? { kind: "direct", llm } : null;
} }
@@ -1151,7 +1179,9 @@ class Shell {
* Opens the assistant overlay if it isn't already up; a follow-up reuses the * Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay. * existing session and overlay.
*/ */
sendToAssistant(text: string): void { sendToAssistant(text: string, showOverlay = true): void {
text = text.trim();
if (!text) return;
const session = this.ensureAssistantSession(); const session = this.ensureAssistantSession();
if (!session) { if (!session) {
this.showAlert( this.showAlert(
@@ -1161,7 +1191,15 @@ class Shell {
); );
return; return;
} }
if (session.isTurnActive()) {
this.showAlert("The assistant is still working on the previous request");
return;
}
if (!this.screenOn) this.wake("sidebar"); if (!this.screenOn) this.wake("sidebar");
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
this.runAssistantTurn(session, null, text);
return;
}
let layer = this.assistantLayer; let layer = this.assistantLayer;
if (!layer) { if (!layer) {
const created = new AssistantLayer(this.config.actions, { const created = new AssistantLayer(this.config.actions, {
@@ -1170,7 +1208,7 @@ class Shell {
onClose: () => this.closeAssistantLayer(), onClose: () => this.closeAssistantLayer(),
onRemoved: () => { onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation): // Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the reference so a later query starts clean. // stop the turn and drop the overlay reference; keep shared history.
this.assistantSession?.cancel(); this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null; if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" }); this.emitAssistantActivity({ phase: "closed", text: "" });
@@ -1184,23 +1222,24 @@ class Shell {
this.config.requestShellRender(); this.config.requestShellRender();
} }
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void { private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
layer.startTurn(); if (session.isTurnActive()) return;
layer?.startTurn();
let replySoFar = ""; let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" }); this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), { session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => { onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar; replySoFar = textSoFar;
layer.onTextDelta(delta, textSoFar); layer?.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar }); this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
}, },
onToolActivity: (label) => layer.onToolActivity(label), onToolActivity: (label) => layer?.onToolActivity(label),
onTurnDone: () => { onTurnDone: () => {
layer.onTurnDone(); layer?.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar }); this.emitAssistantActivity({ phase: "done", text: replySoFar });
}, },
onError: (message) => { onError: (message) => {
layer.onError(message); layer?.onError(message);
this.emitAssistantActivity({ phase: "error", text: message }); this.emitAssistantActivity({ phase: "error", text: message });
}, },
}); });
@@ -1356,7 +1395,7 @@ class Shell {
// when the app has one (otherwise it opens this very menu, not worth a // when the app has one (otherwise it opens this very menu, not worth a
// hint). // hint).
const footer = foreground.hasAppMenu?.() const footer = foreground.hasAppMenu?.()
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]) ? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
: undefined; : undefined;
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar()); layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
layer.selectItem(initialSelection); layer.selectItem(initialSelection);
+5 -3
View File
@@ -1,6 +1,6 @@
import { GrayImage } from "../graphics/image"; import { GrayImage } from "../graphics/image";
import { singlePlane, type Plane } from "../graphics/plane"; import { singlePlane, type Plane } from "../graphics/plane";
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures"; import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
import { type Layer, LayerStack, noopLayerActions } from "./layers"; import { type Layer, LayerStack, noopLayerActions } from "./layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu"; import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
import type { WorkerAppReply } from "./shell/worker-window"; import type { WorkerAppReply } from "./shell/worker-window";
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
}; };
export class WindowMenuLayer extends MenuLayer { export class WindowMenuLayer extends MenuLayer {
constructor(title: string | null, items: MenuItem[]) { constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
super(title, items, WINDOW_MENU_LAYOUT); super(title, items, holdToTalk
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
: WINDOW_MENU_LAYOUT);
} }
} }