Add "AI Chat" app.

This commit is contained in:
James Babcock committed 2026-09-09 01:15:19 -07:00
1 parent 78c56ffc4e
commit 0765e943fb
13 files changed
+616 -87

No files matched your search

+166
View File
@@ -0,0 +1,166 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { truncateText, wrapText } from "../../graphics/textwrap";
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
import type { AssistantTranscriptEntry } from "../../assistant/session";
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
import type { Layer, LayerContext } from "../../ui/layers";
import type { MenuItem } from "../../ui/menu";
import { WindowMenuLayer } from "../../ui/window-menu";
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
import { shell } from "../../ui/shell/shell";
import { VoiceDraft } from "./voice-draft";
class AiChatLayer implements Layer {
readonly acceptsDirectional = true;
private firstLine: number | null = null; // null follows the streaming tail
private maxFirstLine = 0;
private selectedId = "";
private requestRender = () => {};
private readonly wrappedEntries = new WeakMap<AssistantTranscriptEntry, {
font: UiFont; width: number; text: string; lines: string[];
}>();
readonly draft: VoiceDraft;
constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) {
this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(),
() => this.requestRender(), (text) => this.send(text));
}
start(render: () => void): () => void {
this.requestRender = render;
return this.conversations.onChanged(() => render());
}
send(text: string): void {
this.firstLine = null;
shell.sendToAssistant(text, false);
}
paint(ctx: LayerContext): GrayImage {
const size = ctx.stack.getBaseSize();
const image = new GrayImage(size.width, size.height);
const font = getDefaultSmallFont();
const step = font.lineHeight + 3;
const left = 12;
const width = size.width - 28;
const record = this.conversations.current();
if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; }
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
const title = this.conversations.title();
image.drawText(font, left, 6, truncateText(font, title, width), 180);
const label = assistantBackendSetting.get() === "external" ? "External agent" :
`${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`;
const footer = wrapText(font, `${label} ${gestureHints([
[GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"],
])}`, width);
const footerY = size.height - step * footer.length;
const footerTop = footerY - step * 3 - 12;
const historyTop = step + 16;
const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step));
const lines: { text: string; color: number }[] = [];
for (const entry of transcript) {
if (!entry.text) continue;
let wrapped = this.wrappedEntries.get(entry);
if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) {
wrapped = { font, width, text: entry.text,
lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) };
this.wrappedEntries.set(entry, wrapped);
}
for (const text of wrapped.lines) {
lines.push({ text, color: entry.role === "user" ? 175 : 245 });
}
lines.push({ text: "", color: 0 });
}
if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 },
{ text: "Voice and chat share this conversation.", color: 150 });
this.maxFirstLine = Math.max(0, lines.length - visible);
const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine);
for (let row = 0; row < visible; row++) {
const line = lines[first + row];
if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color);
}
if (this.maxFirstLine > 0) {
const track = visible * step;
const thumb = Math.max(8, Math.floor(track * visible / lines.length));
image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110);
}
image.fillRect(left, footerTop, width, 1, 65);
const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status;
if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160);
const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width);
pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220));
footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120));
return image;
}
handleInput(event: InputEvent): void {
switch (event.type) {
case "scroll-up": case "swipe-up":
this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return;
case "scroll-down": case "swipe-down": {
const next = (this.firstLine ?? this.maxFirstLine) + 3;
this.firstLine = next >= this.maxFirstLine ? null : next; return;
}
case "long-press":
if (!this.conversations.current().session?.isTurnActive()) {
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
void this.draft.start();
}
return;
case "long-press-release": this.draft.release(); return;
case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return;
}
}
menuItems(): MenuItem[] {
const record = this.conversations.current();
const busy = () => this.draft.active || Boolean(record.session?.isTurnActive());
const external = assistantBackendSetting.get() === "external";
const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true));
return [
...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []),
{ label: "New session", disabled: () => external || busy(), onSelect: (ctx) => {
if (this.conversations.create()) ctx.stack.clearToBase();
} },
{ label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions",
[...this.conversations.list()].reverse().map((item) => ({
label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`,
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
}))) },
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
ASSISTANT_MODEL_VALUES.map((model) => ({
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
disabled: () => !this.conversations.available(model, record.reasoning),
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
}))) },
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
return external || busy() || !llm?.effort;
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); },
}))) },
...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => {
record.session?.cancel(); ctx.stack.clearToBase();
} }] : []),
];
}
}
export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow {
const layer = new AiChatLayer(shell.getAssistantConversations(), options);
let unsubscribe = () => {};
const app = createInProcessWindow({
...options,
appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true,
baseLayer: layer, holdToTalk: true,
isVoiceCapturing: () => layer.draft.active,
onSystemMenuOpened: () => layer.draft.cancel(),
onAppMenuOpened: () => layer.draft.cancel(),
setScreenOn: (on) => { if (!on) layer.draft.cancel(); },
setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); },
menuItems: () => layer.menuItems(),
receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); },
onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); },
});
unsubscribe = layer.start(app.requestRender);
return app;
}
+10
View File
@@ -0,0 +1,10 @@
import type { AppDefinition } from "../app-definition";
import { createAiChatWindow } from "./ai-chat-app";
const aiChatApp: AppDefinition = {
appId: "ai-chat",
title: "AI Chat",
icon: "message-circle",
launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow),
};
export default aiChatApp;
+125
View File
@@ -0,0 +1,125 @@
import { voiceControlBridge } from "../../native/voice-control";
import { voiceActivity } from "../../ui/shell/voice-activity";
import type { LayerActions } from "../../ui/layers";
/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */
export class VoiceDraft {
phase: "idle" | "preparing" | "listening" | "finishing" = "idle";
text = "";
status = "";
private committed = "";
private live = "";
private generation = 0;
private started = false;
private ownsVoiceActivity = false;
private starting = false;
private unsubscribe: (() => void)[] = [];
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(
private readonly actions: Pick<LayerActions, "startVoiceCapture" | "stopVoiceCapture">,
private readonly prepare: () => Promise<boolean>,
private readonly changed: () => void,
private readonly send: (text: string) => void,
) {}
get active(): boolean { return this.phase !== "idle" || this.starting; }
async start(): Promise<void> {
if (this.active) return;
const generation = ++this.generation;
this.text = this.committed = this.live = "";
this.phase = "preparing";
this.status = "Preparing microphone...";
this.changed();
try {
const ready = await this.prepare();
if (generation !== this.generation) return;
if (!ready) { this.cancel("Microphone unavailable"); return; }
this.phase = "listening";
this.status = "Listening... release to send";
this.ownsVoiceActivity = true;
voiceActivity.setActive(true);
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
if (!this.active || generation !== this.generation) return;
if (event.isFinal) {
const text = event.text.trim();
if (text) this.committed = [this.committed, text].filter(Boolean).join(" ");
this.live = "";
} else this.live = event.text.trim();
this.text = [this.committed, this.live].filter(Boolean).join(" ");
if (event.isFinal && this.phase === "finishing") this.finish();
else this.changed();
}));
// onStatus immediately replays the previous capture's status.
let subscribed = false;
this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => {
if (!subscribed) return;
if (generation !== this.generation || !this.active) return;
if (/ignored|failed|error|denied/i.test(status)) this.cancel(status);
}));
subscribed = true;
if (generation !== this.generation) return;
this.starting = true;
this.changed();
await this.actions.startVoiceCapture();
this.starting = false;
if (generation !== this.generation) {
await this.actions.stopVoiceCapture();
this.changed();
return;
}
this.started = true;
if (this.isFinishing()) this.stopAndFinalize();
} catch (error) {
this.starting = false;
if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`);
}
}
private isFinishing(): boolean { return this.phase === "finishing"; }
release(): void {
if (this.phase === "preparing") { this.cancel(); return; }
if (this.phase !== "listening") return;
this.phase = "finishing";
this.status = "Finishing transcription...";
this.changed();
if (!this.starting) this.stopAndFinalize();
}
private stopAndFinalize(): void {
// Install fallback first: stop may synchronously deliver the final.
this.timer = setTimeout(() => this.finish(), 1500);
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
}
}
private finish(): void {
if (this.phase !== "finishing") return;
const text = this.text.trim();
this.cancel(text ? "" : "No speech detected");
if (text) this.send(text);
}
cancel(status = ""): void {
++this.generation;
this.phase = "idle";
this.status = status;
this.text = this.live = this.committed = "";
if (this.timer !== null) clearTimeout(this.timer);
this.timer = null;
for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
if (this.started) {
this.started = false;
void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {});
}
if (this.ownsVoiceActivity) {
this.ownsVoiceActivity = false;
voiceActivity.setActive(false);
}
this.changed();
}
}
+2
View File
@@ -1,4 +1,5 @@
import { type AppDefinition } from "./app-definition";
import aiChatApp from "./ai-chat";
import launcherApp from "./launcher";
import timerApp from "./timer";
import calculatorApp from "./calculator";
@@ -30,6 +31,7 @@ import settingsApp from "./settings";
*/
export const ALL_APPS: readonly AppDefinition[] = [
launcherApp,
aiChatApp,
timerApp,
calculatorApp,
terminalApp,