Add "AI Chat" app.

This commit is contained in:
James Babcock committed 2026-09-09 01:15:19 -07:00
1 parent 78c56ffc4e
commit 0765e943fb
13 files changed
+616 -87

No files matched your search

+68 -29
View File
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
import { voiceActivity } from "./voice-activity";
import { AssistantLayer } from "./assistant";
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
import { resolveAssistantModel } from "../../assistant/models";
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
@@ -91,6 +93,10 @@ export type ShellWindow = {
* system menu; holding the press past the escape threshold still opens it.
*/
claimsLongPress?: () => boolean;
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
holdToTalk?: boolean;
/** A window owns microphone capture outside the shell voice dialog. */
isVoiceCapturing?: () => boolean;
/**
* True when the window gives swipe-left / swipe-right (watch directional
* input) a meaning; otherwise the shell forwards directionalFallback(event).
@@ -298,7 +304,29 @@ class Shell {
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private activeKeyboardLayer: KeyboardInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private conversations: AssistantConversations | null = null;
private get assistantSession(): AssistantSession | null {
return this.conversations?.current().session ?? null;
}
getAssistantConversations(): AssistantConversations {
if (!this.conversations) {
this.conversations = new AssistantConversations(
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
() => assistantModelSetting.get(),
(value) => setStringSetting("assistant.conversations", value),
getStringSetting("assistant.conversations", ""),
);
}
return this.conversations;
}
async prepareChatVoiceCapture(): Promise<boolean> {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
}
private assistantLayer: AssistantLayer | null = null;
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
private readonly alertListeners = new Set<(text: string) => void>();
@@ -559,7 +587,7 @@ class Shell {
// ring. An in-flight assistant turn suspends it for the same reason (a
// tool loop can run for a while with no input); once the turn ends and
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -645,6 +673,7 @@ class Shell {
// Even AI app never launches, so the firmware does not power the display
// for us either -- actions that need it wake the screen themselves.
if (event.type === "wakeword") {
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
const action = wakeWordActionSetting.get();
if (action === "off") {
return { shell: false, window: false };
@@ -696,11 +725,11 @@ class Shell {
if (!window) {
return { shell: true, window: false };
}
if (!window.claimsLongPress?.()) {
if (!window.holdToTalk && !window.claimsLongPress?.()) {
this.openEscapeMenu();
return { shell: true, window: false };
}
this.startEscapeMenuTimer();
if (!window.holdToTalk) this.startEscapeMenuTimer();
if (this.focus === "sidebar") {
this.focus = "window";
window.onFocus?.(this.lastInput);
@@ -716,6 +745,10 @@ class Shell {
// when it has none.
if (event.type === "short-then-long-press") {
const window = this.foregroundWindow();
if (window?.holdToTalk) {
this.openEscapeMenu();
return { shell: true, window: false };
}
// Over the open system menu it switches to the app's context menu:
// close the system menu and deliver the gesture to the window as
// usual. A window without a menu of its own would only ask for the
@@ -1024,7 +1057,7 @@ class Shell {
*/
startKeyboardInput(): KeyboardInputSession | null {
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
if (this.activeVoiceLayer) return null;
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
if (!this.screenOn) this.wake("sidebar");
const assistantLayer = this.assistantLayer;
const assistantSession = this.assistantSession;
@@ -1069,7 +1102,8 @@ class Shell {
}
isAssistantAvailable(): boolean {
return this.resolveAssistantConfiguration() !== null;
const current = this.getAssistantConversations().current();
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
}
/** Observe assistant turns (see AssistantActivityEvent). */
@@ -1107,20 +1141,13 @@ class Shell {
}
private ensureAssistantSession(): AssistantSession | null {
const config = this.resolveAssistantConfiguration();
if (!config) return null;
if (
!this.assistantSession ||
this.assistantSession.isExpired() ||
!this.assistantSession.matchesConfiguration(config)
) {
this.assistantSession?.cancel();
this.assistantSession = new AssistantSession(config);
}
return this.assistantSession;
return this.getAssistantConversations().ensureSession();
}
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
private resolveAssistantConfiguration(
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
): AssistantBackendConfig | null {
if (assistantBackendSetting.get() === "external") {
const host = assistantBridgeHostSetting.get().trim();
const token = assistantBridgeTokenSetting.get();
@@ -1128,10 +1155,11 @@ class Shell {
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
return { kind: "external", bridge: { host, port, token } };
}
const llm = resolveAssistantModel(assistantModelSetting.get(), {
const llm = resolveAssistantModel(model, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null;
}
@@ -1151,7 +1179,9 @@ class Shell {
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay.
*/
sendToAssistant(text: string): void {
sendToAssistant(text: string, showOverlay = true): void {
text = text.trim();
if (!text) return;
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert(
@@ -1161,7 +1191,15 @@ class Shell {
);
return;
}
if (session.isTurnActive()) {
this.showAlert("The assistant is still working on the previous request");
return;
}
if (!this.screenOn) this.wake("sidebar");
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
this.runAssistantTurn(session, null, text);
return;
}
let layer = this.assistantLayer;
if (!layer) {
const created = new AssistantLayer(this.config.actions, {
@@ -1170,7 +1208,7 @@ class Shell {
onClose: () => this.closeAssistantLayer(),
onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the reference so a later query starts clean.
// stop the turn and drop the overlay reference; keep shared history.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
this.emitAssistantActivity({ phase: "closed", text: "" });
@@ -1184,23 +1222,24 @@ class Shell {
this.config.requestShellRender();
}
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
if (session.isTurnActive()) return;
layer?.startTurn();
let replySoFar = "";
this.emitAssistantActivity({ phase: "thinking", text: "" });
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => {
replySoFar = textSoFar;
layer.onTextDelta(delta, textSoFar);
layer?.onTextDelta(delta, textSoFar);
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
},
onToolActivity: (label) => layer.onToolActivity(label),
onToolActivity: (label) => layer?.onToolActivity(label),
onTurnDone: () => {
layer.onTurnDone();
layer?.onTurnDone();
this.emitAssistantActivity({ phase: "done", text: replySoFar });
},
onError: (message) => {
layer.onError(message);
layer?.onError(message);
this.emitAssistantActivity({ phase: "error", text: message });
},
});
@@ -1356,7 +1395,7 @@ class Shell {
// when the app has one (otherwise it opens this very menu, not worth a
// hint).
const footer = foreground.hasAppMenu?.()
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
: undefined;
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
layer.selectItem(initialSelection);