Add "AI Chat" app.
This commit is contained in:
1 parent
78c56ffc4e
commit
0765e943fb
13 files changed
+616
-87
No files matched your search
@@ -5,6 +5,7 @@
|
||||
* Transcribe app: Scrolling; automatic reconnection; insert newlines
|
||||
* Notifications: Add notification filters
|
||||
* Phone screen preview: Add rotation option; improve landscape layout
|
||||
* Add "AI Chat" app (conversational-style use of the voice assistant with hold-to-talk)
|
||||
|
||||
## 0.6.5 - 2026-09-04
|
||||
* Changed context menu gestures: — for system, ●— for app context menu.
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
|
||||
import type { AssistantTranscriptEntry } from "../../assistant/session";
|
||||
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext } from "../../ui/layers";
|
||||
import type { MenuItem } from "../../ui/menu";
|
||||
import { WindowMenuLayer } from "../../ui/window-menu";
|
||||
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { VoiceDraft } from "./voice-draft";
|
||||
|
||||
class AiChatLayer implements Layer {
|
||||
readonly acceptsDirectional = true;
|
||||
private firstLine: number | null = null; // null follows the streaming tail
|
||||
private maxFirstLine = 0;
|
||||
private selectedId = "";
|
||||
private requestRender = () => {};
|
||||
private readonly wrappedEntries = new WeakMap<AssistantTranscriptEntry, {
|
||||
font: UiFont; width: number; text: string; lines: string[];
|
||||
}>();
|
||||
readonly draft: VoiceDraft;
|
||||
|
||||
constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) {
|
||||
this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(),
|
||||
() => this.requestRender(), (text) => this.send(text));
|
||||
}
|
||||
start(render: () => void): () => void {
|
||||
this.requestRender = render;
|
||||
return this.conversations.onChanged(() => render());
|
||||
}
|
||||
send(text: string): void {
|
||||
this.firstLine = null;
|
||||
shell.sendToAssistant(text, false);
|
||||
}
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const size = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(size.width, size.height);
|
||||
const font = getDefaultSmallFont();
|
||||
const step = font.lineHeight + 3;
|
||||
const left = 12;
|
||||
const width = size.width - 28;
|
||||
const record = this.conversations.current();
|
||||
if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; }
|
||||
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
|
||||
const title = this.conversations.title();
|
||||
image.drawText(font, left, 6, truncateText(font, title, width), 180);
|
||||
const label = assistantBackendSetting.get() === "external" ? "External agent" :
|
||||
`${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`;
|
||||
const footer = wrapText(font, `${label} ${gestureHints([
|
||||
[GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"],
|
||||
])}`, width);
|
||||
const footerY = size.height - step * footer.length;
|
||||
const footerTop = footerY - step * 3 - 12;
|
||||
const historyTop = step + 16;
|
||||
const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step));
|
||||
const lines: { text: string; color: number }[] = [];
|
||||
for (const entry of transcript) {
|
||||
if (!entry.text) continue;
|
||||
let wrapped = this.wrappedEntries.get(entry);
|
||||
if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) {
|
||||
wrapped = { font, width, text: entry.text,
|
||||
lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) };
|
||||
this.wrappedEntries.set(entry, wrapped);
|
||||
}
|
||||
for (const text of wrapped.lines) {
|
||||
lines.push({ text, color: entry.role === "user" ? 175 : 245 });
|
||||
}
|
||||
lines.push({ text: "", color: 0 });
|
||||
}
|
||||
if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 },
|
||||
{ text: "Voice and chat share this conversation.", color: 150 });
|
||||
this.maxFirstLine = Math.max(0, lines.length - visible);
|
||||
const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine);
|
||||
for (let row = 0; row < visible; row++) {
|
||||
const line = lines[first + row];
|
||||
if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color);
|
||||
}
|
||||
if (this.maxFirstLine > 0) {
|
||||
const track = visible * step;
|
||||
const thumb = Math.max(8, Math.floor(track * visible / lines.length));
|
||||
image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110);
|
||||
}
|
||||
image.fillRect(left, footerTop, width, 1, 65);
|
||||
const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status;
|
||||
if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160);
|
||||
const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width);
|
||||
pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220));
|
||||
footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120));
|
||||
return image;
|
||||
}
|
||||
handleInput(event: InputEvent): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up": case "swipe-up":
|
||||
this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return;
|
||||
case "scroll-down": case "swipe-down": {
|
||||
const next = (this.firstLine ?? this.maxFirstLine) + 3;
|
||||
this.firstLine = next >= this.maxFirstLine ? null : next; return;
|
||||
}
|
||||
case "long-press":
|
||||
if (!this.conversations.current().session?.isTurnActive()) {
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
|
||||
void this.draft.start();
|
||||
}
|
||||
return;
|
||||
case "long-press-release": this.draft.release(); return;
|
||||
case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return;
|
||||
}
|
||||
}
|
||||
menuItems(): MenuItem[] {
|
||||
const record = this.conversations.current();
|
||||
const busy = () => this.draft.active || Boolean(record.session?.isTurnActive());
|
||||
const external = assistantBackendSetting.get() === "external";
|
||||
const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true));
|
||||
return [
|
||||
...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []),
|
||||
{ label: "New session", disabled: () => external || busy(), onSelect: (ctx) => {
|
||||
if (this.conversations.create()) ctx.stack.clearToBase();
|
||||
} },
|
||||
{ label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions",
|
||||
[...this.conversations.list()].reverse().map((item) => ({
|
||||
label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`,
|
||||
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
|
||||
ASSISTANT_MODEL_VALUES.map((model) => ({
|
||||
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
|
||||
disabled: () => !this.conversations.available(model, record.reasoning),
|
||||
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
|
||||
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
|
||||
return external || busy() || !llm?.effort;
|
||||
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
|
||||
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
|
||||
onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); },
|
||||
}))) },
|
||||
...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => {
|
||||
record.session?.cancel(); ctx.stack.clearToBase();
|
||||
} }] : []),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow {
|
||||
const layer = new AiChatLayer(shell.getAssistantConversations(), options);
|
||||
let unsubscribe = () => {};
|
||||
const app = createInProcessWindow({
|
||||
...options,
|
||||
appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true,
|
||||
baseLayer: layer, holdToTalk: true,
|
||||
isVoiceCapturing: () => layer.draft.active,
|
||||
onSystemMenuOpened: () => layer.draft.cancel(),
|
||||
onAppMenuOpened: () => layer.draft.cancel(),
|
||||
setScreenOn: (on) => { if (!on) layer.draft.cancel(); },
|
||||
setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); },
|
||||
menuItems: () => layer.menuItems(),
|
||||
receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); },
|
||||
onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); },
|
||||
});
|
||||
unsubscribe = layer.start(app.requestRender);
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { AppDefinition } from "../app-definition";
|
||||
import { createAiChatWindow } from "./ai-chat-app";
|
||||
|
||||
const aiChatApp: AppDefinition = {
|
||||
appId: "ai-chat",
|
||||
title: "AI Chat",
|
||||
icon: "message-circle",
|
||||
launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow),
|
||||
};
|
||||
export default aiChatApp;
|
||||
@@ -0,0 +1,125 @@
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceActivity } from "../../ui/shell/voice-activity";
|
||||
import type { LayerActions } from "../../ui/layers";
|
||||
|
||||
/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */
|
||||
export class VoiceDraft {
|
||||
phase: "idle" | "preparing" | "listening" | "finishing" = "idle";
|
||||
text = "";
|
||||
status = "";
|
||||
private committed = "";
|
||||
private live = "";
|
||||
private generation = 0;
|
||||
private started = false;
|
||||
private ownsVoiceActivity = false;
|
||||
private starting = false;
|
||||
private unsubscribe: (() => void)[] = [];
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: Pick<LayerActions, "startVoiceCapture" | "stopVoiceCapture">,
|
||||
private readonly prepare: () => Promise<boolean>,
|
||||
private readonly changed: () => void,
|
||||
private readonly send: (text: string) => void,
|
||||
) {}
|
||||
|
||||
get active(): boolean { return this.phase !== "idle" || this.starting; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.active) return;
|
||||
const generation = ++this.generation;
|
||||
this.text = this.committed = this.live = "";
|
||||
this.phase = "preparing";
|
||||
this.status = "Preparing microphone...";
|
||||
this.changed();
|
||||
try {
|
||||
const ready = await this.prepare();
|
||||
if (generation !== this.generation) return;
|
||||
if (!ready) { this.cancel("Microphone unavailable"); return; }
|
||||
this.phase = "listening";
|
||||
this.status = "Listening... release to send";
|
||||
this.ownsVoiceActivity = true;
|
||||
voiceActivity.setActive(true);
|
||||
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
|
||||
if (!this.active || generation !== this.generation) return;
|
||||
if (event.isFinal) {
|
||||
const text = event.text.trim();
|
||||
if (text) this.committed = [this.committed, text].filter(Boolean).join(" ");
|
||||
this.live = "";
|
||||
} else this.live = event.text.trim();
|
||||
this.text = [this.committed, this.live].filter(Boolean).join(" ");
|
||||
if (event.isFinal && this.phase === "finishing") this.finish();
|
||||
else this.changed();
|
||||
}));
|
||||
// onStatus immediately replays the previous capture's status.
|
||||
let subscribed = false;
|
||||
this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => {
|
||||
if (!subscribed) return;
|
||||
if (generation !== this.generation || !this.active) return;
|
||||
if (/ignored|failed|error|denied/i.test(status)) this.cancel(status);
|
||||
}));
|
||||
subscribed = true;
|
||||
if (generation !== this.generation) return;
|
||||
this.starting = true;
|
||||
this.changed();
|
||||
await this.actions.startVoiceCapture();
|
||||
this.starting = false;
|
||||
if (generation !== this.generation) {
|
||||
await this.actions.stopVoiceCapture();
|
||||
this.changed();
|
||||
return;
|
||||
}
|
||||
this.started = true;
|
||||
if (this.isFinishing()) this.stopAndFinalize();
|
||||
} catch (error) {
|
||||
this.starting = false;
|
||||
if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private isFinishing(): boolean { return this.phase === "finishing"; }
|
||||
|
||||
release(): void {
|
||||
if (this.phase === "preparing") { this.cancel(); return; }
|
||||
if (this.phase !== "listening") return;
|
||||
this.phase = "finishing";
|
||||
this.status = "Finishing transcription...";
|
||||
this.changed();
|
||||
if (!this.starting) this.stopAndFinalize();
|
||||
}
|
||||
|
||||
private stopAndFinalize(): void {
|
||||
// Install fallback first: stop may synchronously deliver the final.
|
||||
this.timer = setTimeout(() => this.finish(), 1500);
|
||||
if (this.started) {
|
||||
this.started = false;
|
||||
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
|
||||
}
|
||||
}
|
||||
|
||||
private finish(): void {
|
||||
if (this.phase !== "finishing") return;
|
||||
const text = this.text.trim();
|
||||
this.cancel(text ? "" : "No speech detected");
|
||||
if (text) this.send(text);
|
||||
}
|
||||
|
||||
cancel(status = ""): void {
|
||||
++this.generation;
|
||||
this.phase = "idle";
|
||||
this.status = status;
|
||||
this.text = this.live = this.committed = "";
|
||||
if (this.timer !== null) clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
|
||||
if (this.started) {
|
||||
this.started = false;
|
||||
void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {});
|
||||
}
|
||||
if (this.ownsVoiceActivity) {
|
||||
this.ownsVoiceActivity = false;
|
||||
voiceActivity.setActive(false);
|
||||
}
|
||||
this.changed();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type AppDefinition } from "./app-definition";
|
||||
import aiChatApp from "./ai-chat";
|
||||
import launcherApp from "./launcher";
|
||||
import timerApp from "./timer";
|
||||
import calculatorApp from "./calculator";
|
||||
@@ -30,6 +31,7 @@ import settingsApp from "./settings";
|
||||
*/
|
||||
export const ALL_APPS: readonly AppDefinition[] = [
|
||||
launcherApp,
|
||||
aiChatApp,
|
||||
timerApp,
|
||||
calculatorApp,
|
||||
terminalApp,
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { AssistantSession, type AssistantBackendConfig, type AssistantSessionHistory } from "./session";
|
||||
import { ASSISTANT_MODEL_VALUES, type AssistantModel } from "./models";
|
||||
|
||||
export type ReasoningLevel = "default" | "low" | "medium" | "high";
|
||||
export type Conversation = {
|
||||
id: string;
|
||||
model: AssistantModel;
|
||||
reasoning: ReasoningLevel;
|
||||
session: AssistantSession | null;
|
||||
history?: AssistantSessionHistory;
|
||||
};
|
||||
type SavedConversation = Omit<Conversation, "session">;
|
||||
type SavedConversations = { selectedId: string; conversations: SavedConversation[] };
|
||||
|
||||
/** Shared selection and history for every assistant entry point. No credentials are saved. */
|
||||
export class AssistantConversations {
|
||||
private records: Conversation[] = [];
|
||||
private selectedId = "";
|
||||
private readonly listeners = new Set<() => void>();
|
||||
|
||||
constructor(
|
||||
private readonly resolve: (model: AssistantModel, reasoning: ReasoningLevel) => AssistantBackendConfig | null,
|
||||
private readonly defaultModel: () => AssistantModel,
|
||||
private readonly persist: (value: string) => void,
|
||||
saved = "",
|
||||
) {
|
||||
try {
|
||||
const data: SavedConversations = JSON.parse(saved);
|
||||
if (Array.isArray(data.conversations)) {
|
||||
this.records = data.conversations.filter((record) =>
|
||||
typeof record.id === "string" && ASSISTANT_MODEL_VALUES.includes(record.model) &&
|
||||
["default", "low", "medium", "high"].includes(record.reasoning) &&
|
||||
Array.isArray(record.history?.messages) && Array.isArray(record.history?.transcript),
|
||||
).map((record) => ({ ...record, session: null }));
|
||||
this.selectedId = data.selectedId;
|
||||
}
|
||||
} catch { /* First launch or invalid saved state. */ }
|
||||
if (!this.records.length) this.records.push(this.makeConversation());
|
||||
if (!this.records.some((record) => record.id === this.selectedId)) this.selectedId = this.records[0]!.id;
|
||||
}
|
||||
|
||||
current(): Conversation { return this.records.find((record) => record.id === this.selectedId)!; }
|
||||
list(): readonly Conversation[] { return this.records; }
|
||||
title(record = this.current()): string {
|
||||
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
|
||||
return transcript.find((entry) => entry.role === "user")?.text.replace(/\s+/g, " ").slice(0, 60) || "New session";
|
||||
}
|
||||
onChanged(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
private changed(save = true): void {
|
||||
if (save) this.persist(JSON.stringify({ selectedId: this.selectedId, conversations: this.records.map(
|
||||
({ session, ...record }) => ({ ...record, history: session?.history() ?? record.history ?? { messages: [], transcript: [] } }),
|
||||
) }));
|
||||
for (const listener of this.listeners) listener();
|
||||
}
|
||||
private makeConversation(): Conversation {
|
||||
return { id: `${Date.now()}-${Math.random().toString(36).slice(2)}`, model: this.defaultModel(), reasoning: "default", session: null };
|
||||
}
|
||||
create(): boolean {
|
||||
if (this.current().session?.isTurnActive()) return false;
|
||||
const record = this.makeConversation();
|
||||
this.records.push(record);
|
||||
this.selectedId = record.id;
|
||||
this.changed();
|
||||
return true;
|
||||
}
|
||||
select(id: string): boolean {
|
||||
if (this.current().session?.isTurnActive() || !this.records.some((record) => record.id === id)) return false;
|
||||
this.selectedId = id;
|
||||
this.changed();
|
||||
return true;
|
||||
}
|
||||
configure(model: AssistantModel, reasoning: ReasoningLevel): boolean {
|
||||
const record = this.current();
|
||||
if (record.session?.isTurnActive() || !this.resolve(model, reasoning)) return false;
|
||||
record.model = model;
|
||||
record.reasoning = reasoning;
|
||||
this.ensureSession();
|
||||
this.changed();
|
||||
return true;
|
||||
}
|
||||
available(model: AssistantModel, reasoning: ReasoningLevel): boolean { return this.resolve(model, reasoning) !== null; }
|
||||
ensureSession(): AssistantSession | null {
|
||||
const record = this.current();
|
||||
const config = this.resolve(record.model, record.reasoning);
|
||||
if (!config) return null;
|
||||
if (!record.session) {
|
||||
record.session = new AssistantSession(config, undefined, record.history);
|
||||
record.session.onChanged(() => this.changed(!record.session!.isTurnActive()));
|
||||
} else record.session.configure(config);
|
||||
return record.session;
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,7 @@ export class DirectAssistantBackend {
|
||||
const startMs = Date.now();
|
||||
let cancelled = false;
|
||||
let streamHandle: LlmStreamHandle | null = null;
|
||||
let priorText = "";
|
||||
|
||||
const finishError = (message: string) => {
|
||||
if (cancelled) return;
|
||||
@@ -78,7 +79,7 @@ export class DirectAssistantBackend {
|
||||
tools: options.buildTools(),
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
if (cancelled) return;
|
||||
options.callbacks.onTextDelta(delta, textSoFar);
|
||||
options.callbacks.onTextDelta(delta, priorText + textSoFar);
|
||||
},
|
||||
onDone: (result) => {
|
||||
streamHandle = null;
|
||||
@@ -87,6 +88,9 @@ export class DirectAssistantBackend {
|
||||
finishError("The assistant declined this request");
|
||||
return;
|
||||
}
|
||||
const fullText = priorText + result.text;
|
||||
options.callbacks.onTextDelta("", fullText);
|
||||
priorText = fullText ? `${fullText}\n\n` : "";
|
||||
// Record what the model produced this iteration.
|
||||
const assistantContent: LlmContentBlock[] = result.content.length
|
||||
? result.content
|
||||
|
||||
+113
-47
@@ -15,26 +15,20 @@ import type {
|
||||
AssistantTurnHandle,
|
||||
} from "./types";
|
||||
|
||||
/**
|
||||
* One assistant conversation. Owns the message history and turn state and
|
||||
* drives either the on-phone provider loop (direct) or the external agent
|
||||
* bridge. The UI talks only to this surface, so AssistantLayer is
|
||||
* backend-agnostic.
|
||||
*
|
||||
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
|
||||
* one for a new wakeword and reuses the current one for a follow-up. In
|
||||
* external mode the conversation history lives on the agent's machine (one
|
||||
* long-lived session there), so expiry here only affects the overlay UI.
|
||||
*/
|
||||
/** One shared conversation, independent of the voice overlay or chat window. */
|
||||
export type AssistantTranscriptEntry = { role: "user" | "assistant"; text: string };
|
||||
export type AssistantSessionHistory = {
|
||||
messages: LlmMessage[];
|
||||
transcript: AssistantTranscriptEntry[];
|
||||
/** Model identity for safe replay after Auto resolves differently on restart. */
|
||||
engine?: string;
|
||||
};
|
||||
|
||||
/** Which engine answers utterances, plus what it needs to do so. */
|
||||
export type AssistantBackendConfig =
|
||||
| { kind: "direct"; llm: ResolvedAssistantModel }
|
||||
| { kind: "external"; bridge: AssistantBridgeConfig };
|
||||
|
||||
/** A new PTT after this much idle starts a fresh conversation. */
|
||||
const SESSION_IDLE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** Trim history from the head once it grows past this many messages. */
|
||||
const MAX_HISTORY_MESSAGES = 40;
|
||||
|
||||
@@ -42,38 +36,61 @@ export class AssistantSession {
|
||||
private readonly directBackend = new DirectAssistantBackend();
|
||||
private readonly messages: LlmMessage[] = [];
|
||||
private turnHandle: AssistantTurnHandle | null = null;
|
||||
private lastActivityMs = Date.now();
|
||||
readonly transcript: AssistantTranscriptEntry[] = [];
|
||||
status = "";
|
||||
private readonly listeners = new Set<() => void>();
|
||||
private turnGeneration = 0;
|
||||
// API-safe tool name -> canonical registry name. Provider APIs restrict tool
|
||||
// names, so dotted registry names are sanitized and mapped back on calls.
|
||||
private readonly toolNameMap = new Map<string, string>();
|
||||
|
||||
constructor(
|
||||
private readonly config: AssistantBackendConfig,
|
||||
private config: AssistantBackendConfig,
|
||||
private readonly registry: ToolRegistry = toolRegistry,
|
||||
) {}
|
||||
|
||||
matchesConfiguration(config: AssistantBackendConfig): boolean {
|
||||
if (this.config.kind !== config.kind) return false;
|
||||
if (this.config.kind === "direct" && config.kind === "direct") {
|
||||
return (
|
||||
this.config.llm.provider === config.llm.provider &&
|
||||
this.config.llm.model === config.llm.model &&
|
||||
this.config.llm.apiKey === config.llm.apiKey
|
||||
);
|
||||
history?: AssistantSessionHistory,
|
||||
) {
|
||||
if (history) {
|
||||
this.messages.push(...history.messages);
|
||||
this.transcript.push(...history.transcript);
|
||||
if (history.engine !== this.engine()) this.rebuildTextHistory();
|
||||
}
|
||||
if (this.config.kind === "external" && config.kind === "external") {
|
||||
return (
|
||||
this.config.bridge.host === config.bridge.host &&
|
||||
this.config.bridge.port === config.bridge.port &&
|
||||
this.config.bridge.token === config.bridge.token
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Whether this session has been idle long enough to retire. */
|
||||
isExpired(nowMs = Date.now()): boolean {
|
||||
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
|
||||
onChanged(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
private changed(): void {
|
||||
for (const listener of this.listeners) {
|
||||
try { listener(); } catch (error) { console.warn("assistant history listener failed", error); }
|
||||
}
|
||||
}
|
||||
|
||||
history(): AssistantSessionHistory {
|
||||
return { messages: this.messages, transcript: this.transcript, engine: this.engine() };
|
||||
}
|
||||
|
||||
configure(config: AssistantBackendConfig): void {
|
||||
if (this.isTurnActive()) return;
|
||||
// Opaque reasoning/tool provider items are only replayable on their
|
||||
// original model. Preserve visible conversation when switching engines.
|
||||
if (this.config.kind !== config.kind ||
|
||||
(this.config.kind === "direct" && config.kind === "direct" &&
|
||||
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) {
|
||||
this.rebuildTextHistory();
|
||||
}
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private engine(): string {
|
||||
return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external";
|
||||
}
|
||||
|
||||
private rebuildTextHistory(): void {
|
||||
this.messages.splice(0, this.messages.length, ...this.transcript
|
||||
.filter((entry) => entry.text.trim())
|
||||
.map((entry) => ({ role: entry.role, content: entry.text })));
|
||||
}
|
||||
|
||||
isTurnActive(): boolean {
|
||||
@@ -86,20 +103,44 @@ export class AssistantSession {
|
||||
callbacks.onError("The assistant is still working on the previous request");
|
||||
return;
|
||||
}
|
||||
this.lastActivityMs = Date.now();
|
||||
text = text.trim();
|
||||
if (!text) return;
|
||||
const generation = ++this.turnGeneration;
|
||||
const reply: AssistantTranscriptEntry = { role: "assistant", text: "" };
|
||||
this.transcript.push({ role: "user", text }, reply);
|
||||
this.status = "Thinking...";
|
||||
// Install a sentinel before calling providers: some failures are synchronous.
|
||||
this.turnHandle = { cancel: () => {} };
|
||||
this.changed();
|
||||
|
||||
const finish = () => {
|
||||
++this.turnGeneration;
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
this.changed();
|
||||
};
|
||||
const wrappedCallbacks: AssistantTurnCallbacks = {
|
||||
onTextDelta: callbacks.onTextDelta,
|
||||
onToolActivity: callbacks.onToolActivity,
|
||||
onTextDelta: (delta, full) => {
|
||||
if (generation !== this.turnGeneration) return;
|
||||
reply.text = full;
|
||||
this.status = "Thinking...";
|
||||
this.changed();
|
||||
callbacks.onTextDelta(delta, full);
|
||||
},
|
||||
onToolActivity: (label) => {
|
||||
if (generation !== this.turnGeneration) return;
|
||||
this.status = `→ ${label}`;
|
||||
this.changed();
|
||||
callbacks.onToolActivity(label);
|
||||
},
|
||||
onTurnDone: (result) => {
|
||||
if (generation !== this.turnGeneration) return;
|
||||
this.status = "";
|
||||
finish();
|
||||
callbacks.onTurnDone(result);
|
||||
},
|
||||
onError: (message) => {
|
||||
if (generation !== this.turnGeneration) return;
|
||||
this.status = message;
|
||||
finish();
|
||||
callbacks.onError(message);
|
||||
},
|
||||
@@ -108,7 +149,8 @@ export class AssistantSession {
|
||||
if (this.config.kind === "external") {
|
||||
// History and the agent loop live on the agent's machine; the phone just
|
||||
// streams this turn. The overlay keeps its own display state.
|
||||
this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
|
||||
const handle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
|
||||
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -123,24 +165,48 @@ export class AssistantSession {
|
||||
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
|
||||
});
|
||||
this.trimHistory();
|
||||
this.turnHandle = this.directBackend.runTurn({
|
||||
// Work on a turn-local array: cancelled tools may finish later, and must
|
||||
// never append orphan results into a subsequent turn's context.
|
||||
const turnMessages = this.messages.slice();
|
||||
const handle = this.directBackend.runTurn({
|
||||
provider: llm.provider,
|
||||
apiKey: llm.apiKey,
|
||||
model: llm.model,
|
||||
effort: llm.effort,
|
||||
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
||||
messages: this.messages,
|
||||
messages: turnMessages,
|
||||
buildTools: () => this.buildToolDefinitions(),
|
||||
registry: this.registry,
|
||||
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
||||
callbacks: wrappedCallbacks,
|
||||
callbacks: {
|
||||
...wrappedCallbacks,
|
||||
onTurnDone: (result) => {
|
||||
if (generation !== this.turnGeneration) return;
|
||||
this.messages.splice(0, this.messages.length, ...turnMessages);
|
||||
wrappedCallbacks.onTurnDone(result);
|
||||
},
|
||||
onError: (message) => {
|
||||
if (generation !== this.turnGeneration) return;
|
||||
// Retain the visible partial answer, without incomplete tool pairs.
|
||||
if (reply.text) this.messages.push({ role: "assistant", content: reply.text });
|
||||
wrappedCallbacks.onError(message);
|
||||
},
|
||||
},
|
||||
});
|
||||
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.turnHandle?.cancel();
|
||||
if (!this.turnHandle) return;
|
||||
++this.turnGeneration;
|
||||
this.turnHandle.cancel();
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
const reply = this.transcript[this.transcript.length - 1];
|
||||
if (reply?.role === "assistant" && reply.text) {
|
||||
this.messages.push({ role: "assistant", content: reply.text });
|
||||
}
|
||||
this.status = "Cancelled";
|
||||
this.changed();
|
||||
}
|
||||
|
||||
private buildToolDefinitions(): LlmToolDefinition[] {
|
||||
|
||||
@@ -319,7 +319,7 @@ class DashboardController {
|
||||
toggle?: TextSettingsEditToggle,
|
||||
) => this.startTextSettingsEdit(settings, title, onFinish, toggle),
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
startVoiceCapture: () => this.startVoiceCapture(),
|
||||
startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
|
||||
stopVoiceCapture: () => this.stopVoiceCapture(),
|
||||
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
|
||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||
@@ -1874,11 +1874,14 @@ class DashboardController {
|
||||
* push-to-talk and the Transcribe app. Android mic permission is the consent
|
||||
* gate even though the audio source is the G2 mic over BLE.
|
||||
*/
|
||||
private startVoiceCapture(endpointing = false): void {
|
||||
this.beginVoiceCapture("ptt", endpointing);
|
||||
private pttCaptureGeneration = 0;
|
||||
|
||||
private startVoiceCapture(endpointing = false): Promise<void> {
|
||||
return this.beginVoiceCapture("ptt", endpointing);
|
||||
}
|
||||
|
||||
private stopVoiceCapture(): void {
|
||||
++this.pttCaptureGeneration;
|
||||
voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
@@ -1944,7 +1947,8 @@ class DashboardController {
|
||||
return granted;
|
||||
}
|
||||
|
||||
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void {
|
||||
private async beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): Promise<void> {
|
||||
const pttGeneration = kind === "ptt" ? ++this.pttCaptureGeneration : 0;
|
||||
// Preview mode captures from the phone mic (voiceCaptureOptions with a
|
||||
// null communicator); otherwise a live glasses session must be the source.
|
||||
const previewCapture = this.isPreviewDisplayActive();
|
||||
@@ -1952,8 +1956,9 @@ class DashboardController {
|
||||
return;
|
||||
}
|
||||
const communicator = this.communicator;
|
||||
void ensureVoicePermissions()
|
||||
await ensureVoicePermissions()
|
||||
.then(() => {
|
||||
if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return;
|
||||
if (previewCapture) {
|
||||
if (!this.isPreviewDisplayActive()) return;
|
||||
} else if (this.phase !== "connected" || this.communicator !== communicator) {
|
||||
|
||||
@@ -18,6 +18,8 @@ const ICON_STROKE_WIDTH = 2;
|
||||
// Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed
|
||||
// against upstream if an icon needs updating.
|
||||
export const ICON_SVGS = {
|
||||
"message-circle":
|
||||
'<svg viewBox="0 0 24 24" fill="none"><path d="M21 11.5a8.4 8.4 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.4 8.4 0 0 1-3.8-.9L3 21l1.9-5.7a8.4 8.4 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.4 8.4 0 0 1 3.8-.9h.5a8.5 8.5 0 0 1 8 8z"/></svg>',
|
||||
"layout-grid":
|
||||
'<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>',
|
||||
timer:
|
||||
|
||||
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
|
||||
* has no menu of its own, and tap-then-hold opens the system menu instead.
|
||||
*/
|
||||
menuItems?: () => MenuItem[];
|
||||
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
|
||||
holdToTalk?: boolean;
|
||||
isVoiceCapturing?: () => boolean;
|
||||
onSystemMenuOpened?: () => void;
|
||||
onAppMenuOpened?: () => void;
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
actions: LayerActions;
|
||||
/**
|
||||
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
shell.openSystemMenu(options.windowId);
|
||||
return;
|
||||
}
|
||||
stack.push(new WindowMenuLayer(options.title, items));
|
||||
options.onAppMenuOpened?.();
|
||||
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
|
||||
};
|
||||
|
||||
const window: ShellWindow = {
|
||||
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// The window's own LayerStack decides per layer whether a swipe is
|
||||
// directional or falls back to click / double-click.
|
||||
acceptsDirectional: true,
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
close: () => {
|
||||
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// The shell opened its system menu over this window; close our own
|
||||
// context menu so the two never stack. Never forwarded to app layers.
|
||||
if (event.type === "system-menu-opened") {
|
||||
options.onSystemMenuOpened?.();
|
||||
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
|
||||
requestRender();
|
||||
}
|
||||
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// submenus and app content alike. A plain long-press never arrives:
|
||||
// the shell keeps it for the system menu (no in-process window claims
|
||||
// it).
|
||||
if (event.type === "short-then-long-press") {
|
||||
if ((!options.holdToTalk && event.type === "short-then-long-press") ||
|
||||
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
|
||||
openWindowMenu();
|
||||
await render(frameId);
|
||||
return;
|
||||
|
||||
+68
-29
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
|
||||
import { voiceActivity } from "./voice-activity";
|
||||
import { AssistantLayer } from "./assistant";
|
||||
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
|
||||
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
@@ -91,6 +93,10 @@ export type ShellWindow = {
|
||||
* system menu; holding the press past the escape threshold still opens it.
|
||||
*/
|
||||
claimsLongPress?: () => boolean;
|
||||
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
|
||||
holdToTalk?: boolean;
|
||||
/** A window owns microphone capture outside the shell voice dialog. */
|
||||
isVoiceCapturing?: () => boolean;
|
||||
/**
|
||||
* True when the window gives swipe-left / swipe-right (watch directional
|
||||
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
||||
@@ -298,7 +304,29 @@ class Shell {
|
||||
private readonly trayIcons = new Map<string, GrayImage>();
|
||||
private activeVoiceLayer: VoiceInputLayer | null = null;
|
||||
private activeKeyboardLayer: KeyboardInputLayer | null = null;
|
||||
private assistantSession: AssistantSession | null = null;
|
||||
private conversations: AssistantConversations | null = null;
|
||||
private get assistantSession(): AssistantSession | null {
|
||||
return this.conversations?.current().session ?? null;
|
||||
}
|
||||
|
||||
getAssistantConversations(): AssistantConversations {
|
||||
if (!this.conversations) {
|
||||
this.conversations = new AssistantConversations(
|
||||
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
|
||||
() => assistantModelSetting.get(),
|
||||
(value) => setStringSetting("assistant.conversations", value),
|
||||
getStringSetting("assistant.conversations", ""),
|
||||
);
|
||||
}
|
||||
return this.conversations;
|
||||
}
|
||||
|
||||
async prepareChatVoiceCapture(): Promise<boolean> {
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
|
||||
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
||||
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
|
||||
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
|
||||
}
|
||||
private assistantLayer: AssistantLayer | null = null;
|
||||
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
|
||||
private readonly alertListeners = new Set<(text: string) => void>();
|
||||
@@ -559,7 +587,7 @@ class Shell {
|
||||
// ring. An in-flight assistant turn suspends it for the same reason (a
|
||||
// tool loop can run for a while with no input); once the turn ends and
|
||||
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
@@ -645,6 +673,7 @@ class Shell {
|
||||
// Even AI app never launches, so the firmware does not power the display
|
||||
// for us either -- actions that need it wake the screen themselves.
|
||||
if (event.type === "wakeword") {
|
||||
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
|
||||
const action = wakeWordActionSetting.get();
|
||||
if (action === "off") {
|
||||
return { shell: false, window: false };
|
||||
@@ -696,11 +725,11 @@ class Shell {
|
||||
if (!window) {
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
if (!window.claimsLongPress?.()) {
|
||||
if (!window.holdToTalk && !window.claimsLongPress?.()) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
this.startEscapeMenuTimer();
|
||||
if (!window.holdToTalk) this.startEscapeMenuTimer();
|
||||
if (this.focus === "sidebar") {
|
||||
this.focus = "window";
|
||||
window.onFocus?.(this.lastInput);
|
||||
@@ -716,6 +745,10 @@ class Shell {
|
||||
// when it has none.
|
||||
if (event.type === "short-then-long-press") {
|
||||
const window = this.foregroundWindow();
|
||||
if (window?.holdToTalk) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
// Over the open system menu it switches to the app's context menu:
|
||||
// close the system menu and deliver the gesture to the window as
|
||||
// usual. A window without a menu of its own would only ask for the
|
||||
@@ -1024,7 +1057,7 @@ class Shell {
|
||||
*/
|
||||
startKeyboardInput(): KeyboardInputSession | null {
|
||||
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
|
||||
if (this.activeVoiceLayer) return null;
|
||||
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
const assistantLayer = this.assistantLayer;
|
||||
const assistantSession = this.assistantSession;
|
||||
@@ -1069,7 +1102,8 @@ class Shell {
|
||||
}
|
||||
|
||||
isAssistantAvailable(): boolean {
|
||||
return this.resolveAssistantConfiguration() !== null;
|
||||
const current = this.getAssistantConversations().current();
|
||||
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
|
||||
}
|
||||
|
||||
/** Observe assistant turns (see AssistantActivityEvent). */
|
||||
@@ -1107,20 +1141,13 @@ class Shell {
|
||||
}
|
||||
|
||||
private ensureAssistantSession(): AssistantSession | null {
|
||||
const config = this.resolveAssistantConfiguration();
|
||||
if (!config) return null;
|
||||
if (
|
||||
!this.assistantSession ||
|
||||
this.assistantSession.isExpired() ||
|
||||
!this.assistantSession.matchesConfiguration(config)
|
||||
) {
|
||||
this.assistantSession?.cancel();
|
||||
this.assistantSession = new AssistantSession(config);
|
||||
}
|
||||
return this.assistantSession;
|
||||
return this.getAssistantConversations().ensureSession();
|
||||
}
|
||||
|
||||
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
|
||||
private resolveAssistantConfiguration(
|
||||
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
|
||||
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
|
||||
): AssistantBackendConfig | null {
|
||||
if (assistantBackendSetting.get() === "external") {
|
||||
const host = assistantBridgeHostSetting.get().trim();
|
||||
const token = assistantBridgeTokenSetting.get();
|
||||
@@ -1128,10 +1155,11 @@ class Shell {
|
||||
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
||||
return { kind: "external", bridge: { host, port, token } };
|
||||
}
|
||||
const llm = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
const llm = resolveAssistantModel(model, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
});
|
||||
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
|
||||
return llm ? { kind: "direct", llm } : null;
|
||||
}
|
||||
|
||||
@@ -1151,7 +1179,9 @@ class Shell {
|
||||
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
||||
* existing session and overlay.
|
||||
*/
|
||||
sendToAssistant(text: string): void {
|
||||
sendToAssistant(text: string, showOverlay = true): void {
|
||||
text = text.trim();
|
||||
if (!text) return;
|
||||
const session = this.ensureAssistantSession();
|
||||
if (!session) {
|
||||
this.showAlert(
|
||||
@@ -1161,7 +1191,15 @@ class Shell {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (session.isTurnActive()) {
|
||||
this.showAlert("The assistant is still working on the previous request");
|
||||
return;
|
||||
}
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
|
||||
this.runAssistantTurn(session, null, text);
|
||||
return;
|
||||
}
|
||||
let layer = this.assistantLayer;
|
||||
if (!layer) {
|
||||
const created = new AssistantLayer(this.config.actions, {
|
||||
@@ -1170,7 +1208,7 @@ class Shell {
|
||||
onClose: () => this.closeAssistantLayer(),
|
||||
onRemoved: () => {
|
||||
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
||||
// stop the turn and drop the reference so a later query starts clean.
|
||||
// stop the turn and drop the overlay reference; keep shared history.
|
||||
this.assistantSession?.cancel();
|
||||
if (this.assistantLayer === created) this.assistantLayer = null;
|
||||
this.emitAssistantActivity({ phase: "closed", text: "" });
|
||||
@@ -1184,23 +1222,24 @@ class Shell {
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
|
||||
layer.startTurn();
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
|
||||
if (session.isTurnActive()) return;
|
||||
layer?.startTurn();
|
||||
let replySoFar = "";
|
||||
this.emitAssistantActivity({ phase: "thinking", text: "" });
|
||||
session.sendUtterance(text, this.buildAssistantContext(), {
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
replySoFar = textSoFar;
|
||||
layer.onTextDelta(delta, textSoFar);
|
||||
layer?.onTextDelta(delta, textSoFar);
|
||||
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
|
||||
},
|
||||
onToolActivity: (label) => layer.onToolActivity(label),
|
||||
onToolActivity: (label) => layer?.onToolActivity(label),
|
||||
onTurnDone: () => {
|
||||
layer.onTurnDone();
|
||||
layer?.onTurnDone();
|
||||
this.emitAssistantActivity({ phase: "done", text: replySoFar });
|
||||
},
|
||||
onError: (message) => {
|
||||
layer.onError(message);
|
||||
layer?.onError(message);
|
||||
this.emitAssistantActivity({ phase: "error", text: message });
|
||||
},
|
||||
});
|
||||
@@ -1356,7 +1395,7 @@ class Shell {
|
||||
// when the app has one (otherwise it opens this very menu, not worth a
|
||||
// hint).
|
||||
const footer = foreground.hasAppMenu?.()
|
||||
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
|
||||
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
|
||||
: undefined;
|
||||
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
|
||||
layer.selectItem(initialSelection);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { singlePlane, type Plane } from "../graphics/plane";
|
||||
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||
import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||
import { type Layer, LayerStack, noopLayerActions } from "./layers";
|
||||
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
|
||||
import type { WorkerAppReply } from "./shell/worker-window";
|
||||
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
|
||||
};
|
||||
|
||||
export class WindowMenuLayer extends MenuLayer {
|
||||
constructor(title: string | null, items: MenuItem[]) {
|
||||
super(title, items, WINDOW_MENU_LAYOUT);
|
||||
constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
|
||||
super(title, items, holdToTalk
|
||||
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
|
||||
: WINDOW_MENU_LAYOUT);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user