forked from Blendman974/faceclaw
Add "AI Chat" app.
This commit is contained in:
1 parent
78c56ffc4e
commit
0765e943fb
13 files changed
+616
-87
No files matched your search
@@ -5,6 +5,7 @@
|
|||||||
* Transcribe app: Scrolling; automatic reconnection; insert newlines
|
* Transcribe app: Scrolling; automatic reconnection; insert newlines
|
||||||
* Notifications: Add notification filters
|
* Notifications: Add notification filters
|
||||||
* Phone screen preview: Add rotation option; improve landscape layout
|
* Phone screen preview: Add rotation option; improve landscape layout
|
||||||
|
* Add "AI Chat" app (conversational-style use of the voice assistant with hold-to-talk)
|
||||||
|
|
||||||
## 0.6.5 - 2026-09-04
|
## 0.6.5 - 2026-09-04
|
||||||
* Changed context menu gestures: — for system, ●— for app context menu.
|
* Changed context menu gestures: — for system, ●— for app context menu.
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||||
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||||
|
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||||
|
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
|
||||||
|
import type { AssistantTranscriptEntry } from "../../assistant/session";
|
||||||
|
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
|
||||||
|
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||||
|
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||||
|
import type { Layer, LayerContext } from "../../ui/layers";
|
||||||
|
import type { MenuItem } from "../../ui/menu";
|
||||||
|
import { WindowMenuLayer } from "../../ui/window-menu";
|
||||||
|
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||||
|
import { shell } from "../../ui/shell/shell";
|
||||||
|
import { VoiceDraft } from "./voice-draft";
|
||||||
|
|
||||||
|
class AiChatLayer implements Layer {
|
||||||
|
readonly acceptsDirectional = true;
|
||||||
|
private firstLine: number | null = null; // null follows the streaming tail
|
||||||
|
private maxFirstLine = 0;
|
||||||
|
private selectedId = "";
|
||||||
|
private requestRender = () => {};
|
||||||
|
private readonly wrappedEntries = new WeakMap<AssistantTranscriptEntry, {
|
||||||
|
font: UiFont; width: number; text: string; lines: string[];
|
||||||
|
}>();
|
||||||
|
readonly draft: VoiceDraft;
|
||||||
|
|
||||||
|
constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) {
|
||||||
|
this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(),
|
||||||
|
() => this.requestRender(), (text) => this.send(text));
|
||||||
|
}
|
||||||
|
start(render: () => void): () => void {
|
||||||
|
this.requestRender = render;
|
||||||
|
return this.conversations.onChanged(() => render());
|
||||||
|
}
|
||||||
|
send(text: string): void {
|
||||||
|
this.firstLine = null;
|
||||||
|
shell.sendToAssistant(text, false);
|
||||||
|
}
|
||||||
|
paint(ctx: LayerContext): GrayImage {
|
||||||
|
const size = ctx.stack.getBaseSize();
|
||||||
|
const image = new GrayImage(size.width, size.height);
|
||||||
|
const font = getDefaultSmallFont();
|
||||||
|
const step = font.lineHeight + 3;
|
||||||
|
const left = 12;
|
||||||
|
const width = size.width - 28;
|
||||||
|
const record = this.conversations.current();
|
||||||
|
if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; }
|
||||||
|
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
|
||||||
|
const title = this.conversations.title();
|
||||||
|
image.drawText(font, left, 6, truncateText(font, title, width), 180);
|
||||||
|
const label = assistantBackendSetting.get() === "external" ? "External agent" :
|
||||||
|
`${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`;
|
||||||
|
const footer = wrapText(font, `${label} ${gestureHints([
|
||||||
|
[GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"],
|
||||||
|
])}`, width);
|
||||||
|
const footerY = size.height - step * footer.length;
|
||||||
|
const footerTop = footerY - step * 3 - 12;
|
||||||
|
const historyTop = step + 16;
|
||||||
|
const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step));
|
||||||
|
const lines: { text: string; color: number }[] = [];
|
||||||
|
for (const entry of transcript) {
|
||||||
|
if (!entry.text) continue;
|
||||||
|
let wrapped = this.wrappedEntries.get(entry);
|
||||||
|
if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) {
|
||||||
|
wrapped = { font, width, text: entry.text,
|
||||||
|
lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) };
|
||||||
|
this.wrappedEntries.set(entry, wrapped);
|
||||||
|
}
|
||||||
|
for (const text of wrapped.lines) {
|
||||||
|
lines.push({ text, color: entry.role === "user" ? 175 : 245 });
|
||||||
|
}
|
||||||
|
lines.push({ text: "", color: 0 });
|
||||||
|
}
|
||||||
|
if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 },
|
||||||
|
{ text: "Voice and chat share this conversation.", color: 150 });
|
||||||
|
this.maxFirstLine = Math.max(0, lines.length - visible);
|
||||||
|
const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine);
|
||||||
|
for (let row = 0; row < visible; row++) {
|
||||||
|
const line = lines[first + row];
|
||||||
|
if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color);
|
||||||
|
}
|
||||||
|
if (this.maxFirstLine > 0) {
|
||||||
|
const track = visible * step;
|
||||||
|
const thumb = Math.max(8, Math.floor(track * visible / lines.length));
|
||||||
|
image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110);
|
||||||
|
}
|
||||||
|
image.fillRect(left, footerTop, width, 1, 65);
|
||||||
|
const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status;
|
||||||
|
if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160);
|
||||||
|
const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width);
|
||||||
|
pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220));
|
||||||
|
footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120));
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
handleInput(event: InputEvent): void {
|
||||||
|
switch (event.type) {
|
||||||
|
case "scroll-up": case "swipe-up":
|
||||||
|
this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return;
|
||||||
|
case "scroll-down": case "swipe-down": {
|
||||||
|
const next = (this.firstLine ?? this.maxFirstLine) + 3;
|
||||||
|
this.firstLine = next >= this.maxFirstLine ? null : next; return;
|
||||||
|
}
|
||||||
|
case "long-press":
|
||||||
|
if (!this.conversations.current().session?.isTurnActive()) {
|
||||||
|
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
|
||||||
|
void this.draft.start();
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
case "long-press-release": this.draft.release(); return;
|
||||||
|
case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
menuItems(): MenuItem[] {
|
||||||
|
const record = this.conversations.current();
|
||||||
|
const busy = () => this.draft.active || Boolean(record.session?.isTurnActive());
|
||||||
|
const external = assistantBackendSetting.get() === "external";
|
||||||
|
const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true));
|
||||||
|
return [
|
||||||
|
...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []),
|
||||||
|
{ label: "New session", disabled: () => external || busy(), onSelect: (ctx) => {
|
||||||
|
if (this.conversations.create()) ctx.stack.clearToBase();
|
||||||
|
} },
|
||||||
|
{ label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions",
|
||||||
|
[...this.conversations.list()].reverse().map((item) => ({
|
||||||
|
label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`,
|
||||||
|
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
|
||||||
|
}))) },
|
||||||
|
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
|
||||||
|
ASSISTANT_MODEL_VALUES.map((model) => ({
|
||||||
|
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
|
||||||
|
disabled: () => !this.conversations.available(model, record.reasoning),
|
||||||
|
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
|
||||||
|
}))) },
|
||||||
|
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
|
||||||
|
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
|
||||||
|
return external || busy() || !llm?.effort;
|
||||||
|
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
|
||||||
|
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
|
||||||
|
onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); },
|
||||||
|
}))) },
|
||||||
|
...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => {
|
||||||
|
record.session?.cancel(); ctx.stack.clearToBase();
|
||||||
|
} }] : []),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow {
|
||||||
|
const layer = new AiChatLayer(shell.getAssistantConversations(), options);
|
||||||
|
let unsubscribe = () => {};
|
||||||
|
const app = createInProcessWindow({
|
||||||
|
...options,
|
||||||
|
appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true,
|
||||||
|
baseLayer: layer, holdToTalk: true,
|
||||||
|
isVoiceCapturing: () => layer.draft.active,
|
||||||
|
onSystemMenuOpened: () => layer.draft.cancel(),
|
||||||
|
onAppMenuOpened: () => layer.draft.cancel(),
|
||||||
|
setScreenOn: (on) => { if (!on) layer.draft.cancel(); },
|
||||||
|
setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); },
|
||||||
|
menuItems: () => layer.menuItems(),
|
||||||
|
receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); },
|
||||||
|
onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); },
|
||||||
|
});
|
||||||
|
unsubscribe = layer.start(app.requestRender);
|
||||||
|
return app;
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import type { AppDefinition } from "../app-definition";
|
||||||
|
import { createAiChatWindow } from "./ai-chat-app";
|
||||||
|
|
||||||
|
const aiChatApp: AppDefinition = {
|
||||||
|
appId: "ai-chat",
|
||||||
|
title: "AI Chat",
|
||||||
|
icon: "message-circle",
|
||||||
|
launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow),
|
||||||
|
};
|
||||||
|
export default aiChatApp;
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import { voiceControlBridge } from "../../native/voice-control";
|
||||||
|
import { voiceActivity } from "../../ui/shell/voice-activity";
|
||||||
|
import type { LayerActions } from "../../ui/layers";
|
||||||
|
|
||||||
|
/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */
|
||||||
|
export class VoiceDraft {
|
||||||
|
phase: "idle" | "preparing" | "listening" | "finishing" = "idle";
|
||||||
|
text = "";
|
||||||
|
status = "";
|
||||||
|
private committed = "";
|
||||||
|
private live = "";
|
||||||
|
private generation = 0;
|
||||||
|
private started = false;
|
||||||
|
private ownsVoiceActivity = false;
|
||||||
|
private starting = false;
|
||||||
|
private unsubscribe: (() => void)[] = [];
|
||||||
|
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly actions: Pick<LayerActions, "startVoiceCapture" | "stopVoiceCapture">,
|
||||||
|
private readonly prepare: () => Promise<boolean>,
|
||||||
|
private readonly changed: () => void,
|
||||||
|
private readonly send: (text: string) => void,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
get active(): boolean { return this.phase !== "idle" || this.starting; }
|
||||||
|
|
||||||
|
async start(): Promise<void> {
|
||||||
|
if (this.active) return;
|
||||||
|
const generation = ++this.generation;
|
||||||
|
this.text = this.committed = this.live = "";
|
||||||
|
this.phase = "preparing";
|
||||||
|
this.status = "Preparing microphone...";
|
||||||
|
this.changed();
|
||||||
|
try {
|
||||||
|
const ready = await this.prepare();
|
||||||
|
if (generation !== this.generation) return;
|
||||||
|
if (!ready) { this.cancel("Microphone unavailable"); return; }
|
||||||
|
this.phase = "listening";
|
||||||
|
this.status = "Listening... release to send";
|
||||||
|
this.ownsVoiceActivity = true;
|
||||||
|
voiceActivity.setActive(true);
|
||||||
|
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
|
||||||
|
if (!this.active || generation !== this.generation) return;
|
||||||
|
if (event.isFinal) {
|
||||||
|
const text = event.text.trim();
|
||||||
|
if (text) this.committed = [this.committed, text].filter(Boolean).join(" ");
|
||||||
|
this.live = "";
|
||||||
|
} else this.live = event.text.trim();
|
||||||
|
this.text = [this.committed, this.live].filter(Boolean).join(" ");
|
||||||
|
if (event.isFinal && this.phase === "finishing") this.finish();
|
||||||
|
else this.changed();
|
||||||
|
}));
|
||||||
|
// onStatus immediately replays the previous capture's status.
|
||||||
|
let subscribed = false;
|
||||||
|
this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => {
|
||||||
|
if (!subscribed) return;
|
||||||
|
if (generation !== this.generation || !this.active) return;
|
||||||
|
if (/ignored|failed|error|denied/i.test(status)) this.cancel(status);
|
||||||
|
}));
|
||||||
|
subscribed = true;
|
||||||
|
if (generation !== this.generation) return;
|
||||||
|
this.starting = true;
|
||||||
|
this.changed();
|
||||||
|
await this.actions.startVoiceCapture();
|
||||||
|
this.starting = false;
|
||||||
|
if (generation !== this.generation) {
|
||||||
|
await this.actions.stopVoiceCapture();
|
||||||
|
this.changed();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this.started = true;
|
||||||
|
if (this.isFinishing()) this.stopAndFinalize();
|
||||||
|
} catch (error) {
|
||||||
|
this.starting = false;
|
||||||
|
if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private isFinishing(): boolean { return this.phase === "finishing"; }
|
||||||
|
|
||||||
|
release(): void {
|
||||||
|
if (this.phase === "preparing") { this.cancel(); return; }
|
||||||
|
if (this.phase !== "listening") return;
|
||||||
|
this.phase = "finishing";
|
||||||
|
this.status = "Finishing transcription...";
|
||||||
|
this.changed();
|
||||||
|
if (!this.starting) this.stopAndFinalize();
|
||||||
|
}
|
||||||
|
|
||||||
|
private stopAndFinalize(): void {
|
||||||
|
// Install fallback first: stop may synchronously deliver the final.
|
||||||
|
this.timer = setTimeout(() => this.finish(), 1500);
|
||||||
|
if (this.started) {
|
||||||
|
this.started = false;
|
||||||
|
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private finish(): void {
|
||||||
|
if (this.phase !== "finishing") return;
|
||||||
|
const text = this.text.trim();
|
||||||
|
this.cancel(text ? "" : "No speech detected");
|
||||||
|
if (text) this.send(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(status = ""): void {
|
||||||
|
++this.generation;
|
||||||
|
this.phase = "idle";
|
||||||
|
this.status = status;
|
||||||
|
this.text = this.live = this.committed = "";
|
||||||
|
if (this.timer !== null) clearTimeout(this.timer);
|
||||||
|
this.timer = null;
|
||||||
|
for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
|
||||||
|
if (this.started) {
|
||||||
|
this.started = false;
|
||||||
|
void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {});
|
||||||
|
}
|
||||||
|
if (this.ownsVoiceActivity) {
|
||||||
|
this.ownsVoiceActivity = false;
|
||||||
|
voiceActivity.setActive(false);
|
||||||
|
}
|
||||||
|
this.changed();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { type AppDefinition } from "./app-definition";
|
import { type AppDefinition } from "./app-definition";
|
||||||
|
import aiChatApp from "./ai-chat";
|
||||||
import launcherApp from "./launcher";
|
import launcherApp from "./launcher";
|
||||||
import timerApp from "./timer";
|
import timerApp from "./timer";
|
||||||
import calculatorApp from "./calculator";
|
import calculatorApp from "./calculator";
|
||||||
@@ -30,6 +31,7 @@ import settingsApp from "./settings";
|
|||||||
*/
|
*/
|
||||||
export const ALL_APPS: readonly AppDefinition[] = [
|
export const ALL_APPS: readonly AppDefinition[] = [
|
||||||
launcherApp,
|
launcherApp,
|
||||||
|
aiChatApp,
|
||||||
timerApp,
|
timerApp,
|
||||||
calculatorApp,
|
calculatorApp,
|
||||||
terminalApp,
|
terminalApp,
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import { AssistantSession, type AssistantBackendConfig, type AssistantSessionHistory } from "./session";
|
||||||
|
import { ASSISTANT_MODEL_VALUES, type AssistantModel } from "./models";
|
||||||
|
|
||||||
|
export type ReasoningLevel = "default" | "low" | "medium" | "high";
|
||||||
|
export type Conversation = {
|
||||||
|
id: string;
|
||||||
|
model: AssistantModel;
|
||||||
|
reasoning: ReasoningLevel;
|
||||||
|
session: AssistantSession | null;
|
||||||
|
history?: AssistantSessionHistory;
|
||||||
|
};
|
||||||
|
type SavedConversation = Omit<Conversation, "session">;
|
||||||
|
type SavedConversations = { selectedId: string; conversations: SavedConversation[] };
|
||||||
|
|
||||||
|
/** Shared selection and history for every assistant entry point. No credentials are saved. */
|
||||||
|
export class AssistantConversations {
|
||||||
|
private records: Conversation[] = [];
|
||||||
|
private selectedId = "";
|
||||||
|
private readonly listeners = new Set<() => void>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly resolve: (model: AssistantModel, reasoning: ReasoningLevel) => AssistantBackendConfig | null,
|
||||||
|
private readonly defaultModel: () => AssistantModel,
|
||||||
|
private readonly persist: (value: string) => void,
|
||||||
|
saved = "",
|
||||||
|
) {
|
||||||
|
try {
|
||||||
|
const data: SavedConversations = JSON.parse(saved);
|
||||||
|
if (Array.isArray(data.conversations)) {
|
||||||
|
this.records = data.conversations.filter((record) =>
|
||||||
|
typeof record.id === "string" && ASSISTANT_MODEL_VALUES.includes(record.model) &&
|
||||||
|
["default", "low", "medium", "high"].includes(record.reasoning) &&
|
||||||
|
Array.isArray(record.history?.messages) && Array.isArray(record.history?.transcript),
|
||||||
|
).map((record) => ({ ...record, session: null }));
|
||||||
|
this.selectedId = data.selectedId;
|
||||||
|
}
|
||||||
|
} catch { /* First launch or invalid saved state. */ }
|
||||||
|
if (!this.records.length) this.records.push(this.makeConversation());
|
||||||
|
if (!this.records.some((record) => record.id === this.selectedId)) this.selectedId = this.records[0]!.id;
|
||||||
|
}
|
||||||
|
|
||||||
|
current(): Conversation { return this.records.find((record) => record.id === this.selectedId)!; }
|
||||||
|
list(): readonly Conversation[] { return this.records; }
|
||||||
|
title(record = this.current()): string {
|
||||||
|
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
|
||||||
|
return transcript.find((entry) => entry.role === "user")?.text.replace(/\s+/g, " ").slice(0, 60) || "New session";
|
||||||
|
}
|
||||||
|
onChanged(listener: () => void): () => void {
|
||||||
|
this.listeners.add(listener);
|
||||||
|
return () => this.listeners.delete(listener);
|
||||||
|
}
|
||||||
|
private changed(save = true): void {
|
||||||
|
if (save) this.persist(JSON.stringify({ selectedId: this.selectedId, conversations: this.records.map(
|
||||||
|
({ session, ...record }) => ({ ...record, history: session?.history() ?? record.history ?? { messages: [], transcript: [] } }),
|
||||||
|
) }));
|
||||||
|
for (const listener of this.listeners) listener();
|
||||||
|
}
|
||||||
|
private makeConversation(): Conversation {
|
||||||
|
return { id: `${Date.now()}-${Math.random().toString(36).slice(2)}`, model: this.defaultModel(), reasoning: "default", session: null };
|
||||||
|
}
|
||||||
|
create(): boolean {
|
||||||
|
if (this.current().session?.isTurnActive()) return false;
|
||||||
|
const record = this.makeConversation();
|
||||||
|
this.records.push(record);
|
||||||
|
this.selectedId = record.id;
|
||||||
|
this.changed();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
select(id: string): boolean {
|
||||||
|
if (this.current().session?.isTurnActive() || !this.records.some((record) => record.id === id)) return false;
|
||||||
|
this.selectedId = id;
|
||||||
|
this.changed();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
configure(model: AssistantModel, reasoning: ReasoningLevel): boolean {
|
||||||
|
const record = this.current();
|
||||||
|
if (record.session?.isTurnActive() || !this.resolve(model, reasoning)) return false;
|
||||||
|
record.model = model;
|
||||||
|
record.reasoning = reasoning;
|
||||||
|
this.ensureSession();
|
||||||
|
this.changed();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
available(model: AssistantModel, reasoning: ReasoningLevel): boolean { return this.resolve(model, reasoning) !== null; }
|
||||||
|
ensureSession(): AssistantSession | null {
|
||||||
|
const record = this.current();
|
||||||
|
const config = this.resolve(record.model, record.reasoning);
|
||||||
|
if (!config) return null;
|
||||||
|
if (!record.session) {
|
||||||
|
record.session = new AssistantSession(config, undefined, record.history);
|
||||||
|
record.session.onChanged(() => this.changed(!record.session!.isTurnActive()));
|
||||||
|
} else record.session.configure(config);
|
||||||
|
return record.session;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -50,6 +50,7 @@ export class DirectAssistantBackend {
|
|||||||
const startMs = Date.now();
|
const startMs = Date.now();
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
let streamHandle: LlmStreamHandle | null = null;
|
let streamHandle: LlmStreamHandle | null = null;
|
||||||
|
let priorText = "";
|
||||||
|
|
||||||
const finishError = (message: string) => {
|
const finishError = (message: string) => {
|
||||||
if (cancelled) return;
|
if (cancelled) return;
|
||||||
@@ -78,7 +79,7 @@ export class DirectAssistantBackend {
|
|||||||
tools: options.buildTools(),
|
tools: options.buildTools(),
|
||||||
onTextDelta: (delta, textSoFar) => {
|
onTextDelta: (delta, textSoFar) => {
|
||||||
if (cancelled) return;
|
if (cancelled) return;
|
||||||
options.callbacks.onTextDelta(delta, textSoFar);
|
options.callbacks.onTextDelta(delta, priorText + textSoFar);
|
||||||
},
|
},
|
||||||
onDone: (result) => {
|
onDone: (result) => {
|
||||||
streamHandle = null;
|
streamHandle = null;
|
||||||
@@ -87,6 +88,9 @@ export class DirectAssistantBackend {
|
|||||||
finishError("The assistant declined this request");
|
finishError("The assistant declined this request");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const fullText = priorText + result.text;
|
||||||
|
options.callbacks.onTextDelta("", fullText);
|
||||||
|
priorText = fullText ? `${fullText}\n\n` : "";
|
||||||
// Record what the model produced this iteration.
|
// Record what the model produced this iteration.
|
||||||
const assistantContent: LlmContentBlock[] = result.content.length
|
const assistantContent: LlmContentBlock[] = result.content.length
|
||||||
? result.content
|
? result.content
|
||||||
|
|||||||
+113
-47
@@ -15,26 +15,20 @@ import type {
|
|||||||
AssistantTurnHandle,
|
AssistantTurnHandle,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
/**
|
/** One shared conversation, independent of the voice overlay or chat window. */
|
||||||
* One assistant conversation. Owns the message history and turn state and
|
export type AssistantTranscriptEntry = { role: "user" | "assistant"; text: string };
|
||||||
* drives either the on-phone provider loop (direct) or the external agent
|
export type AssistantSessionHistory = {
|
||||||
* bridge. The UI talks only to this surface, so AssistantLayer is
|
messages: LlmMessage[];
|
||||||
* backend-agnostic.
|
transcript: AssistantTranscriptEntry[];
|
||||||
*
|
/** Model identity for safe replay after Auto resolves differently on restart. */
|
||||||
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
|
engine?: string;
|
||||||
* one for a new wakeword and reuses the current one for a follow-up. In
|
};
|
||||||
* external mode the conversation history lives on the agent's machine (one
|
|
||||||
* long-lived session there), so expiry here only affects the overlay UI.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/** Which engine answers utterances, plus what it needs to do so. */
|
/** Which engine answers utterances, plus what it needs to do so. */
|
||||||
export type AssistantBackendConfig =
|
export type AssistantBackendConfig =
|
||||||
| { kind: "direct"; llm: ResolvedAssistantModel }
|
| { kind: "direct"; llm: ResolvedAssistantModel }
|
||||||
| { kind: "external"; bridge: AssistantBridgeConfig };
|
| { kind: "external"; bridge: AssistantBridgeConfig };
|
||||||
|
|
||||||
/** A new PTT after this much idle starts a fresh conversation. */
|
|
||||||
const SESSION_IDLE_MS = 10 * 60 * 1000;
|
|
||||||
|
|
||||||
/** Trim history from the head once it grows past this many messages. */
|
/** Trim history from the head once it grows past this many messages. */
|
||||||
const MAX_HISTORY_MESSAGES = 40;
|
const MAX_HISTORY_MESSAGES = 40;
|
||||||
|
|
||||||
@@ -42,38 +36,61 @@ export class AssistantSession {
|
|||||||
private readonly directBackend = new DirectAssistantBackend();
|
private readonly directBackend = new DirectAssistantBackend();
|
||||||
private readonly messages: LlmMessage[] = [];
|
private readonly messages: LlmMessage[] = [];
|
||||||
private turnHandle: AssistantTurnHandle | null = null;
|
private turnHandle: AssistantTurnHandle | null = null;
|
||||||
private lastActivityMs = Date.now();
|
readonly transcript: AssistantTranscriptEntry[] = [];
|
||||||
|
status = "";
|
||||||
|
private readonly listeners = new Set<() => void>();
|
||||||
|
private turnGeneration = 0;
|
||||||
// API-safe tool name -> canonical registry name. Provider APIs restrict tool
|
// API-safe tool name -> canonical registry name. Provider APIs restrict tool
|
||||||
// names, so dotted registry names are sanitized and mapped back on calls.
|
// names, so dotted registry names are sanitized and mapped back on calls.
|
||||||
private readonly toolNameMap = new Map<string, string>();
|
private readonly toolNameMap = new Map<string, string>();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly config: AssistantBackendConfig,
|
private config: AssistantBackendConfig,
|
||||||
private readonly registry: ToolRegistry = toolRegistry,
|
private readonly registry: ToolRegistry = toolRegistry,
|
||||||
) {}
|
history?: AssistantSessionHistory,
|
||||||
|
) {
|
||||||
matchesConfiguration(config: AssistantBackendConfig): boolean {
|
if (history) {
|
||||||
if (this.config.kind !== config.kind) return false;
|
this.messages.push(...history.messages);
|
||||||
if (this.config.kind === "direct" && config.kind === "direct") {
|
this.transcript.push(...history.transcript);
|
||||||
return (
|
if (history.engine !== this.engine()) this.rebuildTextHistory();
|
||||||
this.config.llm.provider === config.llm.provider &&
|
|
||||||
this.config.llm.model === config.llm.model &&
|
|
||||||
this.config.llm.apiKey === config.llm.apiKey
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if (this.config.kind === "external" && config.kind === "external") {
|
|
||||||
return (
|
|
||||||
this.config.bridge.host === config.bridge.host &&
|
|
||||||
this.config.bridge.port === config.bridge.port &&
|
|
||||||
this.config.bridge.token === config.bridge.token
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether this session has been idle long enough to retire. */
|
onChanged(listener: () => void): () => void {
|
||||||
isExpired(nowMs = Date.now()): boolean {
|
this.listeners.add(listener);
|
||||||
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
|
return () => this.listeners.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
private changed(): void {
|
||||||
|
for (const listener of this.listeners) {
|
||||||
|
try { listener(); } catch (error) { console.warn("assistant history listener failed", error); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
history(): AssistantSessionHistory {
|
||||||
|
return { messages: this.messages, transcript: this.transcript, engine: this.engine() };
|
||||||
|
}
|
||||||
|
|
||||||
|
configure(config: AssistantBackendConfig): void {
|
||||||
|
if (this.isTurnActive()) return;
|
||||||
|
// Opaque reasoning/tool provider items are only replayable on their
|
||||||
|
// original model. Preserve visible conversation when switching engines.
|
||||||
|
if (this.config.kind !== config.kind ||
|
||||||
|
(this.config.kind === "direct" && config.kind === "direct" &&
|
||||||
|
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) {
|
||||||
|
this.rebuildTextHistory();
|
||||||
|
}
|
||||||
|
this.config = config;
|
||||||
|
}
|
||||||
|
|
||||||
|
private engine(): string {
|
||||||
|
return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external";
|
||||||
|
}
|
||||||
|
|
||||||
|
private rebuildTextHistory(): void {
|
||||||
|
this.messages.splice(0, this.messages.length, ...this.transcript
|
||||||
|
.filter((entry) => entry.text.trim())
|
||||||
|
.map((entry) => ({ role: entry.role, content: entry.text })));
|
||||||
}
|
}
|
||||||
|
|
||||||
isTurnActive(): boolean {
|
isTurnActive(): boolean {
|
||||||
@@ -86,20 +103,44 @@ export class AssistantSession {
|
|||||||
callbacks.onError("The assistant is still working on the previous request");
|
callbacks.onError("The assistant is still working on the previous request");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.lastActivityMs = Date.now();
|
text = text.trim();
|
||||||
|
if (!text) return;
|
||||||
|
const generation = ++this.turnGeneration;
|
||||||
|
const reply: AssistantTranscriptEntry = { role: "assistant", text: "" };
|
||||||
|
this.transcript.push({ role: "user", text }, reply);
|
||||||
|
this.status = "Thinking...";
|
||||||
|
// Install a sentinel before calling providers: some failures are synchronous.
|
||||||
|
this.turnHandle = { cancel: () => {} };
|
||||||
|
this.changed();
|
||||||
|
|
||||||
const finish = () => {
|
const finish = () => {
|
||||||
|
++this.turnGeneration;
|
||||||
this.turnHandle = null;
|
this.turnHandle = null;
|
||||||
this.lastActivityMs = Date.now();
|
this.changed();
|
||||||
};
|
};
|
||||||
const wrappedCallbacks: AssistantTurnCallbacks = {
|
const wrappedCallbacks: AssistantTurnCallbacks = {
|
||||||
onTextDelta: callbacks.onTextDelta,
|
onTextDelta: (delta, full) => {
|
||||||
onToolActivity: callbacks.onToolActivity,
|
if (generation !== this.turnGeneration) return;
|
||||||
|
reply.text = full;
|
||||||
|
this.status = "Thinking...";
|
||||||
|
this.changed();
|
||||||
|
callbacks.onTextDelta(delta, full);
|
||||||
|
},
|
||||||
|
onToolActivity: (label) => {
|
||||||
|
if (generation !== this.turnGeneration) return;
|
||||||
|
this.status = `→ ${label}`;
|
||||||
|
this.changed();
|
||||||
|
callbacks.onToolActivity(label);
|
||||||
|
},
|
||||||
onTurnDone: (result) => {
|
onTurnDone: (result) => {
|
||||||
|
if (generation !== this.turnGeneration) return;
|
||||||
|
this.status = "";
|
||||||
finish();
|
finish();
|
||||||
callbacks.onTurnDone(result);
|
callbacks.onTurnDone(result);
|
||||||
},
|
},
|
||||||
onError: (message) => {
|
onError: (message) => {
|
||||||
|
if (generation !== this.turnGeneration) return;
|
||||||
|
this.status = message;
|
||||||
finish();
|
finish();
|
||||||
callbacks.onError(message);
|
callbacks.onError(message);
|
||||||
},
|
},
|
||||||
@@ -108,7 +149,8 @@ export class AssistantSession {
|
|||||||
if (this.config.kind === "external") {
|
if (this.config.kind === "external") {
|
||||||
// History and the agent loop live on the agent's machine; the phone just
|
// History and the agent loop live on the agent's machine; the phone just
|
||||||
// streams this turn. The overlay keeps its own display state.
|
// streams this turn. The overlay keeps its own display state.
|
||||||
this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
|
const handle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
|
||||||
|
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,24 +165,48 @@ export class AssistantSession {
|
|||||||
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
|
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
|
||||||
});
|
});
|
||||||
this.trimHistory();
|
this.trimHistory();
|
||||||
this.turnHandle = this.directBackend.runTurn({
|
// Work on a turn-local array: cancelled tools may finish later, and must
|
||||||
|
// never append orphan results into a subsequent turn's context.
|
||||||
|
const turnMessages = this.messages.slice();
|
||||||
|
const handle = this.directBackend.runTurn({
|
||||||
provider: llm.provider,
|
provider: llm.provider,
|
||||||
apiKey: llm.apiKey,
|
apiKey: llm.apiKey,
|
||||||
model: llm.model,
|
model: llm.model,
|
||||||
effort: llm.effort,
|
effort: llm.effort,
|
||||||
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
||||||
messages: this.messages,
|
messages: turnMessages,
|
||||||
buildTools: () => this.buildToolDefinitions(),
|
buildTools: () => this.buildToolDefinitions(),
|
||||||
registry: this.registry,
|
registry: this.registry,
|
||||||
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
||||||
callbacks: wrappedCallbacks,
|
callbacks: {
|
||||||
|
...wrappedCallbacks,
|
||||||
|
onTurnDone: (result) => {
|
||||||
|
if (generation !== this.turnGeneration) return;
|
||||||
|
this.messages.splice(0, this.messages.length, ...turnMessages);
|
||||||
|
wrappedCallbacks.onTurnDone(result);
|
||||||
|
},
|
||||||
|
onError: (message) => {
|
||||||
|
if (generation !== this.turnGeneration) return;
|
||||||
|
// Retain the visible partial answer, without incomplete tool pairs.
|
||||||
|
if (reply.text) this.messages.push({ role: "assistant", content: reply.text });
|
||||||
|
wrappedCallbacks.onError(message);
|
||||||
|
},
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
if (generation === this.turnGeneration && this.turnHandle) this.turnHandle = handle;
|
||||||
}
|
}
|
||||||
|
|
||||||
cancel(): void {
|
cancel(): void {
|
||||||
this.turnHandle?.cancel();
|
if (!this.turnHandle) return;
|
||||||
|
++this.turnGeneration;
|
||||||
|
this.turnHandle.cancel();
|
||||||
this.turnHandle = null;
|
this.turnHandle = null;
|
||||||
this.lastActivityMs = Date.now();
|
const reply = this.transcript[this.transcript.length - 1];
|
||||||
|
if (reply?.role === "assistant" && reply.text) {
|
||||||
|
this.messages.push({ role: "assistant", content: reply.text });
|
||||||
|
}
|
||||||
|
this.status = "Cancelled";
|
||||||
|
this.changed();
|
||||||
}
|
}
|
||||||
|
|
||||||
private buildToolDefinitions(): LlmToolDefinition[] {
|
private buildToolDefinitions(): LlmToolDefinition[] {
|
||||||
|
|||||||
@@ -319,7 +319,7 @@ class DashboardController {
|
|||||||
toggle?: TextSettingsEditToggle,
|
toggle?: TextSettingsEditToggle,
|
||||||
) => this.startTextSettingsEdit(settings, title, onFinish, toggle),
|
) => this.startTextSettingsEdit(settings, title, onFinish, toggle),
|
||||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||||
startVoiceCapture: () => this.startVoiceCapture(),
|
startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing),
|
||||||
stopVoiceCapture: () => this.stopVoiceCapture(),
|
stopVoiceCapture: () => this.stopVoiceCapture(),
|
||||||
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
|
startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(),
|
||||||
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(),
|
||||||
@@ -1874,11 +1874,14 @@ class DashboardController {
|
|||||||
* push-to-talk and the Transcribe app. Android mic permission is the consent
|
* push-to-talk and the Transcribe app. Android mic permission is the consent
|
||||||
* gate even though the audio source is the G2 mic over BLE.
|
* gate even though the audio source is the G2 mic over BLE.
|
||||||
*/
|
*/
|
||||||
private startVoiceCapture(endpointing = false): void {
|
private pttCaptureGeneration = 0;
|
||||||
this.beginVoiceCapture("ptt", endpointing);
|
|
||||||
|
private startVoiceCapture(endpointing = false): Promise<void> {
|
||||||
|
return this.beginVoiceCapture("ptt", endpointing);
|
||||||
}
|
}
|
||||||
|
|
||||||
private stopVoiceCapture(): void {
|
private stopVoiceCapture(): void {
|
||||||
|
++this.pttCaptureGeneration;
|
||||||
voiceControlBridge.stopPushToTalk();
|
voiceControlBridge.stopPushToTalk();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1944,7 +1947,8 @@ class DashboardController {
|
|||||||
return granted;
|
return granted;
|
||||||
}
|
}
|
||||||
|
|
||||||
private beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): void {
|
private async beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): Promise<void> {
|
||||||
|
const pttGeneration = kind === "ptt" ? ++this.pttCaptureGeneration : 0;
|
||||||
// Preview mode captures from the phone mic (voiceCaptureOptions with a
|
// Preview mode captures from the phone mic (voiceCaptureOptions with a
|
||||||
// null communicator); otherwise a live glasses session must be the source.
|
// null communicator); otherwise a live glasses session must be the source.
|
||||||
const previewCapture = this.isPreviewDisplayActive();
|
const previewCapture = this.isPreviewDisplayActive();
|
||||||
@@ -1952,8 +1956,9 @@ class DashboardController {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const communicator = this.communicator;
|
const communicator = this.communicator;
|
||||||
void ensureVoicePermissions()
|
await ensureVoicePermissions()
|
||||||
.then(() => {
|
.then(() => {
|
||||||
|
if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return;
|
||||||
if (previewCapture) {
|
if (previewCapture) {
|
||||||
if (!this.isPreviewDisplayActive()) return;
|
if (!this.isPreviewDisplayActive()) return;
|
||||||
} else if (this.phase !== "connected" || this.communicator !== communicator) {
|
} else if (this.phase !== "connected" || this.communicator !== communicator) {
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ const ICON_STROKE_WIDTH = 2;
|
|||||||
// Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed
|
// Lucide icons (MIT/ISC licensed). Kept verbatim so they can be diffed
|
||||||
// against upstream if an icon needs updating.
|
// against upstream if an icon needs updating.
|
||||||
export const ICON_SVGS = {
|
export const ICON_SVGS = {
|
||||||
|
"message-circle":
|
||||||
|
'<svg viewBox="0 0 24 24" fill="none"><path d="M21 11.5a8.4 8.4 0 0 1-.9 3.8 8.5 8.5 0 0 1-7.6 4.7 8.4 8.4 0 0 1-3.8-.9L3 21l1.9-5.7a8.4 8.4 0 0 1-.9-3.8 8.5 8.5 0 0 1 4.7-7.6 8.4 8.4 0 0 1 3.8-.9h.5a8.5 8.5 0 0 1 8 8z"/></svg>',
|
||||||
"layout-grid":
|
"layout-grid":
|
||||||
'<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>',
|
'<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>',
|
||||||
timer:
|
timer:
|
||||||
|
|||||||
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
|
|||||||
* has no menu of its own, and tap-then-hold opens the system menu instead.
|
* has no menu of its own, and tap-then-hold opens the system menu instead.
|
||||||
*/
|
*/
|
||||||
menuItems?: () => MenuItem[];
|
menuItems?: () => MenuItem[];
|
||||||
|
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
|
||||||
|
holdToTalk?: boolean;
|
||||||
|
isVoiceCapturing?: () => boolean;
|
||||||
|
onSystemMenuOpened?: () => void;
|
||||||
|
onAppMenuOpened?: () => void;
|
||||||
|
setScreenOn?: ShellWindow["setScreenOn"];
|
||||||
/** Shared actions; requestRender is rebound to this window's render. */
|
/** Shared actions; requestRender is rebound to this window's render. */
|
||||||
actions: LayerActions;
|
actions: LayerActions;
|
||||||
/**
|
/**
|
||||||
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
|||||||
shell.openSystemMenu(options.windowId);
|
shell.openSystemMenu(options.windowId);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
stack.push(new WindowMenuLayer(options.title, items));
|
options.onAppMenuOpened?.();
|
||||||
|
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
|
||||||
};
|
};
|
||||||
|
|
||||||
const window: ShellWindow = {
|
const window: ShellWindow = {
|
||||||
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
|||||||
// The window's own LayerStack decides per layer whether a swipe is
|
// The window's own LayerStack decides per layer whether a swipe is
|
||||||
// directional or falls back to click / double-click.
|
// directional or falls back to click / double-click.
|
||||||
acceptsDirectional: true,
|
acceptsDirectional: true,
|
||||||
|
holdToTalk: options.holdToTalk,
|
||||||
|
isVoiceCapturing: options.isVoiceCapturing,
|
||||||
|
setScreenOn: options.setScreenOn,
|
||||||
heightMode,
|
heightMode,
|
||||||
hasAppMenu: () => appMenuItems().length > 0,
|
hasAppMenu: () => appMenuItems().length > 0,
|
||||||
close: () => {
|
close: () => {
|
||||||
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
|||||||
// The shell opened its system menu over this window; close our own
|
// The shell opened its system menu over this window; close our own
|
||||||
// context menu so the two never stack. Never forwarded to app layers.
|
// context menu so the two never stack. Never forwarded to app layers.
|
||||||
if (event.type === "system-menu-opened") {
|
if (event.type === "system-menu-opened") {
|
||||||
|
options.onSystemMenuOpened?.();
|
||||||
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
|
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
|
||||||
requestRender();
|
requestRender();
|
||||||
}
|
}
|
||||||
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
|||||||
// submenus and app content alike. A plain long-press never arrives:
|
// submenus and app content alike. A plain long-press never arrives:
|
||||||
// the shell keeps it for the system menu (no in-process window claims
|
// the shell keeps it for the system menu (no in-process window claims
|
||||||
// it).
|
// it).
|
||||||
if (event.type === "short-then-long-press") {
|
if ((!options.holdToTalk && event.type === "short-then-long-press") ||
|
||||||
|
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
|
||||||
openWindowMenu();
|
openWindowMenu();
|
||||||
await render(frameId);
|
await render(frameId);
|
||||||
return;
|
return;
|
||||||
|
|||||||
+68
-29
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
|
|||||||
import { voiceActivity } from "./voice-activity";
|
import { voiceActivity } from "./voice-activity";
|
||||||
import { AssistantLayer } from "./assistant";
|
import { AssistantLayer } from "./assistant";
|
||||||
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
||||||
import { resolveAssistantModel } from "../../assistant/models";
|
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
|
||||||
|
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
|
||||||
|
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||||
import type { AssistantContext } from "../../assistant/types";
|
import type { AssistantContext } from "../../assistant/types";
|
||||||
import { SingleNotificationLayer } from "../notifications";
|
import { SingleNotificationLayer } from "../notifications";
|
||||||
import {
|
import {
|
||||||
@@ -91,6 +93,10 @@ export type ShellWindow = {
|
|||||||
* system menu; holding the press past the escape threshold still opens it.
|
* system menu; holding the press past the escape threshold still opens it.
|
||||||
*/
|
*/
|
||||||
claimsLongPress?: () => boolean;
|
claimsLongPress?: () => boolean;
|
||||||
|
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
|
||||||
|
holdToTalk?: boolean;
|
||||||
|
/** A window owns microphone capture outside the shell voice dialog. */
|
||||||
|
isVoiceCapturing?: () => boolean;
|
||||||
/**
|
/**
|
||||||
* True when the window gives swipe-left / swipe-right (watch directional
|
* True when the window gives swipe-left / swipe-right (watch directional
|
||||||
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
||||||
@@ -298,7 +304,29 @@ class Shell {
|
|||||||
private readonly trayIcons = new Map<string, GrayImage>();
|
private readonly trayIcons = new Map<string, GrayImage>();
|
||||||
private activeVoiceLayer: VoiceInputLayer | null = null;
|
private activeVoiceLayer: VoiceInputLayer | null = null;
|
||||||
private activeKeyboardLayer: KeyboardInputLayer | null = null;
|
private activeKeyboardLayer: KeyboardInputLayer | null = null;
|
||||||
private assistantSession: AssistantSession | null = null;
|
private conversations: AssistantConversations | null = null;
|
||||||
|
private get assistantSession(): AssistantSession | null {
|
||||||
|
return this.conversations?.current().session ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
getAssistantConversations(): AssistantConversations {
|
||||||
|
if (!this.conversations) {
|
||||||
|
this.conversations = new AssistantConversations(
|
||||||
|
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
|
||||||
|
() => assistantModelSetting.get(),
|
||||||
|
(value) => setStringSetting("assistant.conversations", value),
|
||||||
|
getStringSetting("assistant.conversations", ""),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return this.conversations;
|
||||||
|
}
|
||||||
|
|
||||||
|
async prepareChatVoiceCapture(): Promise<boolean> {
|
||||||
|
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
|
||||||
|
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
||||||
|
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
|
||||||
|
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
|
||||||
|
}
|
||||||
private assistantLayer: AssistantLayer | null = null;
|
private assistantLayer: AssistantLayer | null = null;
|
||||||
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
|
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
|
||||||
private readonly alertListeners = new Set<(text: string) => void>();
|
private readonly alertListeners = new Set<(text: string) => void>();
|
||||||
@@ -559,7 +587,7 @@ class Shell {
|
|||||||
// ring. An in-flight assistant turn suspends it for the same reason (a
|
// ring. An in-flight assistant turn suspends it for the same reason (a
|
||||||
// tool loop can run for a while with no input); once the turn ends and
|
// tool loop can run for a while with no input); once the turn ends and
|
||||||
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
|
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
|
||||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
|
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
|
||||||
this.lastInputAtMs = nowMs;
|
this.lastInputAtMs = nowMs;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -645,6 +673,7 @@ class Shell {
|
|||||||
// Even AI app never launches, so the firmware does not power the display
|
// Even AI app never launches, so the firmware does not power the display
|
||||||
// for us either -- actions that need it wake the screen themselves.
|
// for us either -- actions that need it wake the screen themselves.
|
||||||
if (event.type === "wakeword") {
|
if (event.type === "wakeword") {
|
||||||
|
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
|
||||||
const action = wakeWordActionSetting.get();
|
const action = wakeWordActionSetting.get();
|
||||||
if (action === "off") {
|
if (action === "off") {
|
||||||
return { shell: false, window: false };
|
return { shell: false, window: false };
|
||||||
@@ -696,11 +725,11 @@ class Shell {
|
|||||||
if (!window) {
|
if (!window) {
|
||||||
return { shell: true, window: false };
|
return { shell: true, window: false };
|
||||||
}
|
}
|
||||||
if (!window.claimsLongPress?.()) {
|
if (!window.holdToTalk && !window.claimsLongPress?.()) {
|
||||||
this.openEscapeMenu();
|
this.openEscapeMenu();
|
||||||
return { shell: true, window: false };
|
return { shell: true, window: false };
|
||||||
}
|
}
|
||||||
this.startEscapeMenuTimer();
|
if (!window.holdToTalk) this.startEscapeMenuTimer();
|
||||||
if (this.focus === "sidebar") {
|
if (this.focus === "sidebar") {
|
||||||
this.focus = "window";
|
this.focus = "window";
|
||||||
window.onFocus?.(this.lastInput);
|
window.onFocus?.(this.lastInput);
|
||||||
@@ -716,6 +745,10 @@ class Shell {
|
|||||||
// when it has none.
|
// when it has none.
|
||||||
if (event.type === "short-then-long-press") {
|
if (event.type === "short-then-long-press") {
|
||||||
const window = this.foregroundWindow();
|
const window = this.foregroundWindow();
|
||||||
|
if (window?.holdToTalk) {
|
||||||
|
this.openEscapeMenu();
|
||||||
|
return { shell: true, window: false };
|
||||||
|
}
|
||||||
// Over the open system menu it switches to the app's context menu:
|
// Over the open system menu it switches to the app's context menu:
|
||||||
// close the system menu and deliver the gesture to the window as
|
// close the system menu and deliver the gesture to the window as
|
||||||
// usual. A window without a menu of its own would only ask for the
|
// usual. A window without a menu of its own would only ask for the
|
||||||
@@ -1024,7 +1057,7 @@ class Shell {
|
|||||||
*/
|
*/
|
||||||
startKeyboardInput(): KeyboardInputSession | null {
|
startKeyboardInput(): KeyboardInputSession | null {
|
||||||
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
|
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
|
||||||
if (this.activeVoiceLayer) return null;
|
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
|
||||||
if (!this.screenOn) this.wake("sidebar");
|
if (!this.screenOn) this.wake("sidebar");
|
||||||
const assistantLayer = this.assistantLayer;
|
const assistantLayer = this.assistantLayer;
|
||||||
const assistantSession = this.assistantSession;
|
const assistantSession = this.assistantSession;
|
||||||
@@ -1069,7 +1102,8 @@ class Shell {
|
|||||||
}
|
}
|
||||||
|
|
||||||
isAssistantAvailable(): boolean {
|
isAssistantAvailable(): boolean {
|
||||||
return this.resolveAssistantConfiguration() !== null;
|
const current = this.getAssistantConversations().current();
|
||||||
|
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Observe assistant turns (see AssistantActivityEvent). */
|
/** Observe assistant turns (see AssistantActivityEvent). */
|
||||||
@@ -1107,20 +1141,13 @@ class Shell {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ensureAssistantSession(): AssistantSession | null {
|
private ensureAssistantSession(): AssistantSession | null {
|
||||||
const config = this.resolveAssistantConfiguration();
|
return this.getAssistantConversations().ensureSession();
|
||||||
if (!config) return null;
|
|
||||||
if (
|
|
||||||
!this.assistantSession ||
|
|
||||||
this.assistantSession.isExpired() ||
|
|
||||||
!this.assistantSession.matchesConfiguration(config)
|
|
||||||
) {
|
|
||||||
this.assistantSession?.cancel();
|
|
||||||
this.assistantSession = new AssistantSession(config);
|
|
||||||
}
|
|
||||||
return this.assistantSession;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
|
private resolveAssistantConfiguration(
|
||||||
|
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
|
||||||
|
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
|
||||||
|
): AssistantBackendConfig | null {
|
||||||
if (assistantBackendSetting.get() === "external") {
|
if (assistantBackendSetting.get() === "external") {
|
||||||
const host = assistantBridgeHostSetting.get().trim();
|
const host = assistantBridgeHostSetting.get().trim();
|
||||||
const token = assistantBridgeTokenSetting.get();
|
const token = assistantBridgeTokenSetting.get();
|
||||||
@@ -1128,10 +1155,11 @@ class Shell {
|
|||||||
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
||||||
return { kind: "external", bridge: { host, port, token } };
|
return { kind: "external", bridge: { host, port, token } };
|
||||||
}
|
}
|
||||||
const llm = resolveAssistantModel(assistantModelSetting.get(), {
|
const llm = resolveAssistantModel(model, {
|
||||||
anthropic: anthropicApiKeySetting.get(),
|
anthropic: anthropicApiKeySetting.get(),
|
||||||
openai: openAiApiKeySetting.get(),
|
openai: openAiApiKeySetting.get(),
|
||||||
});
|
});
|
||||||
|
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
|
||||||
return llm ? { kind: "direct", llm } : null;
|
return llm ? { kind: "direct", llm } : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1151,7 +1179,9 @@ class Shell {
|
|||||||
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
||||||
* existing session and overlay.
|
* existing session and overlay.
|
||||||
*/
|
*/
|
||||||
sendToAssistant(text: string): void {
|
sendToAssistant(text: string, showOverlay = true): void {
|
||||||
|
text = text.trim();
|
||||||
|
if (!text) return;
|
||||||
const session = this.ensureAssistantSession();
|
const session = this.ensureAssistantSession();
|
||||||
if (!session) {
|
if (!session) {
|
||||||
this.showAlert(
|
this.showAlert(
|
||||||
@@ -1161,7 +1191,15 @@ class Shell {
|
|||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (session.isTurnActive()) {
|
||||||
|
this.showAlert("The assistant is still working on the previous request");
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!this.screenOn) this.wake("sidebar");
|
if (!this.screenOn) this.wake("sidebar");
|
||||||
|
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
|
||||||
|
this.runAssistantTurn(session, null, text);
|
||||||
|
return;
|
||||||
|
}
|
||||||
let layer = this.assistantLayer;
|
let layer = this.assistantLayer;
|
||||||
if (!layer) {
|
if (!layer) {
|
||||||
const created = new AssistantLayer(this.config.actions, {
|
const created = new AssistantLayer(this.config.actions, {
|
||||||
@@ -1170,7 +1208,7 @@ class Shell {
|
|||||||
onClose: () => this.closeAssistantLayer(),
|
onClose: () => this.closeAssistantLayer(),
|
||||||
onRemoved: () => {
|
onRemoved: () => {
|
||||||
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
||||||
// stop the turn and drop the reference so a later query starts clean.
|
// stop the turn and drop the overlay reference; keep shared history.
|
||||||
this.assistantSession?.cancel();
|
this.assistantSession?.cancel();
|
||||||
if (this.assistantLayer === created) this.assistantLayer = null;
|
if (this.assistantLayer === created) this.assistantLayer = null;
|
||||||
this.emitAssistantActivity({ phase: "closed", text: "" });
|
this.emitAssistantActivity({ phase: "closed", text: "" });
|
||||||
@@ -1184,23 +1222,24 @@ class Shell {
|
|||||||
this.config.requestShellRender();
|
this.config.requestShellRender();
|
||||||
}
|
}
|
||||||
|
|
||||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
|
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
|
||||||
layer.startTurn();
|
if (session.isTurnActive()) return;
|
||||||
|
layer?.startTurn();
|
||||||
let replySoFar = "";
|
let replySoFar = "";
|
||||||
this.emitAssistantActivity({ phase: "thinking", text: "" });
|
this.emitAssistantActivity({ phase: "thinking", text: "" });
|
||||||
session.sendUtterance(text, this.buildAssistantContext(), {
|
session.sendUtterance(text, this.buildAssistantContext(), {
|
||||||
onTextDelta: (delta, textSoFar) => {
|
onTextDelta: (delta, textSoFar) => {
|
||||||
replySoFar = textSoFar;
|
replySoFar = textSoFar;
|
||||||
layer.onTextDelta(delta, textSoFar);
|
layer?.onTextDelta(delta, textSoFar);
|
||||||
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
|
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
|
||||||
},
|
},
|
||||||
onToolActivity: (label) => layer.onToolActivity(label),
|
onToolActivity: (label) => layer?.onToolActivity(label),
|
||||||
onTurnDone: () => {
|
onTurnDone: () => {
|
||||||
layer.onTurnDone();
|
layer?.onTurnDone();
|
||||||
this.emitAssistantActivity({ phase: "done", text: replySoFar });
|
this.emitAssistantActivity({ phase: "done", text: replySoFar });
|
||||||
},
|
},
|
||||||
onError: (message) => {
|
onError: (message) => {
|
||||||
layer.onError(message);
|
layer?.onError(message);
|
||||||
this.emitAssistantActivity({ phase: "error", text: message });
|
this.emitAssistantActivity({ phase: "error", text: message });
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -1356,7 +1395,7 @@ class Shell {
|
|||||||
// when the app has one (otherwise it opens this very menu, not worth a
|
// when the app has one (otherwise it opens this very menu, not worth a
|
||||||
// hint).
|
// hint).
|
||||||
const footer = foreground.hasAppMenu?.()
|
const footer = foreground.hasAppMenu?.()
|
||||||
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
|
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
|
||||||
: undefined;
|
: undefined;
|
||||||
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
|
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
|
||||||
layer.selectItem(initialSelection);
|
layer.selectItem(initialSelection);
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { GrayImage } from "../graphics/image";
|
import { GrayImage } from "../graphics/image";
|
||||||
import { singlePlane, type Plane } from "../graphics/plane";
|
import { singlePlane, type Plane } from "../graphics/plane";
|
||||||
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||||
import { type Layer, LayerStack, noopLayerActions } from "./layers";
|
import { type Layer, LayerStack, noopLayerActions } from "./layers";
|
||||||
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
|
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
|
||||||
import type { WorkerAppReply } from "./shell/worker-window";
|
import type { WorkerAppReply } from "./shell/worker-window";
|
||||||
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export class WindowMenuLayer extends MenuLayer {
|
export class WindowMenuLayer extends MenuLayer {
|
||||||
constructor(title: string | null, items: MenuItem[]) {
|
constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
|
||||||
super(title, items, WINDOW_MENU_LAYOUT);
|
super(title, items, holdToTalk
|
||||||
|
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
|
||||||
|
: WINDOW_MENU_LAYOUT);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user