forked from Blendman974/faceclaw
Add "AI Chat" app.
This commit is contained in:
1 parent
78c56ffc4e
commit
0765e943fb
13 files changed
+616
-87
No files matched your search
@@ -0,0 +1,166 @@
|
||||
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { ASSISTANT_MODEL_VALUES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
|
||||
import type { AssistantTranscriptEntry } from "../../assistant/session";
|
||||
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext } from "../../ui/layers";
|
||||
import type { MenuItem } from "../../ui/menu";
|
||||
import { WindowMenuLayer } from "../../ui/window-menu";
|
||||
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { VoiceDraft } from "./voice-draft";
|
||||
|
||||
class AiChatLayer implements Layer {
|
||||
readonly acceptsDirectional = true;
|
||||
private firstLine: number | null = null; // null follows the streaming tail
|
||||
private maxFirstLine = 0;
|
||||
private selectedId = "";
|
||||
private requestRender = () => {};
|
||||
private readonly wrappedEntries = new WeakMap<AssistantTranscriptEntry, {
|
||||
font: UiFont; width: number; text: string; lines: string[];
|
||||
}>();
|
||||
readonly draft: VoiceDraft;
|
||||
|
||||
constructor(private readonly conversations: AssistantConversations, options: InProcessAppOptions) {
|
||||
this.draft = new VoiceDraft(options.actions, () => shell.prepareChatVoiceCapture(),
|
||||
() => this.requestRender(), (text) => this.send(text));
|
||||
}
|
||||
start(render: () => void): () => void {
|
||||
this.requestRender = render;
|
||||
return this.conversations.onChanged(() => render());
|
||||
}
|
||||
send(text: string): void {
|
||||
this.firstLine = null;
|
||||
shell.sendToAssistant(text, false);
|
||||
}
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const size = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(size.width, size.height);
|
||||
const font = getDefaultSmallFont();
|
||||
const step = font.lineHeight + 3;
|
||||
const left = 12;
|
||||
const width = size.width - 28;
|
||||
const record = this.conversations.current();
|
||||
if (this.selectedId !== record.id) { this.selectedId = record.id; this.firstLine = null; }
|
||||
const transcript = record.session?.transcript ?? record.history?.transcript ?? [];
|
||||
const title = this.conversations.title();
|
||||
image.drawText(font, left, 6, truncateText(font, title, width), 180);
|
||||
const label = assistantBackendSetting.get() === "external" ? "External agent" :
|
||||
`${assistantModelLabel(record.model)} · ${record.reasoning === "default" ? "default reasoning" : record.reasoning}`;
|
||||
const footer = wrapText(font, `${label} ${gestureHints([
|
||||
[GESTURE_CLICK, "menu"], [GESTURE_DOUBLE_CLICK, "back"], [GESTURE_LONG_PRESS, "hold to talk"],
|
||||
])}`, width);
|
||||
const footerY = size.height - step * footer.length;
|
||||
const footerTop = footerY - step * 3 - 12;
|
||||
const historyTop = step + 16;
|
||||
const visible = Math.max(1, Math.floor((footerTop - historyTop - 8) / step));
|
||||
const lines: { text: string; color: number }[] = [];
|
||||
for (const entry of transcript) {
|
||||
if (!entry.text) continue;
|
||||
let wrapped = this.wrappedEntries.get(entry);
|
||||
if (!wrapped || wrapped.font !== font || wrapped.width !== width || wrapped.text !== entry.text) {
|
||||
wrapped = { font, width, text: entry.text,
|
||||
lines: wrapText(font, `${entry.role === "user" ? "You" : "AI"}: ${entry.text}`, width - 8) };
|
||||
this.wrappedEntries.set(entry, wrapped);
|
||||
}
|
||||
for (const text of wrapped.lines) {
|
||||
lines.push({ text, color: entry.role === "user" ? 175 : 245 });
|
||||
}
|
||||
lines.push({ text: "", color: 0 });
|
||||
}
|
||||
if (!lines.length) lines.push({ text: "Hold to talk to your assistant.", color: 175 },
|
||||
{ text: "Voice and chat share this conversation.", color: 150 });
|
||||
this.maxFirstLine = Math.max(0, lines.length - visible);
|
||||
const first = this.firstLine === null ? this.maxFirstLine : Math.min(this.firstLine, this.maxFirstLine);
|
||||
for (let row = 0; row < visible; row++) {
|
||||
const line = lines[first + row];
|
||||
if (line) image.drawText(font, left, historyTop + row * step, line.text, line.color);
|
||||
}
|
||||
if (this.maxFirstLine > 0) {
|
||||
const track = visible * step;
|
||||
const thumb = Math.max(8, Math.floor(track * visible / lines.length));
|
||||
image.fillRect(size.width - 5, historyTop + Math.floor((track - thumb) * first / this.maxFirstLine), 2, thumb, 110);
|
||||
}
|
||||
image.fillRect(left, footerTop, width, 1, 65);
|
||||
const state = this.draft.active ? this.draft.status : record.session?.status || this.draft.status;
|
||||
if (state) image.drawText(font, left, footerTop + 5, truncateText(font, state, width), 160);
|
||||
const pending = wrapText(font, this.draft.text || (this.draft.active ? "Listening..." : "Hold to speak; release to send"), width);
|
||||
pending.slice(-2).forEach((text, i) => image.drawText(font, left, footerTop + step * (i + 1) + 5, text, 220));
|
||||
footer.forEach((text, i) => image.drawText(font, left + width - font.measureText(text), footerY + i * step, text, 120));
|
||||
return image;
|
||||
}
|
||||
handleInput(event: InputEvent): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up": case "swipe-up":
|
||||
this.firstLine = Math.max(0, (this.firstLine ?? this.maxFirstLine) - 3); return;
|
||||
case "scroll-down": case "swipe-down": {
|
||||
const next = (this.firstLine ?? this.maxFirstLine) + 3;
|
||||
this.firstLine = next >= this.maxFirstLine ? null : next; return;
|
||||
}
|
||||
case "long-press":
|
||||
if (!this.conversations.current().session?.isTurnActive()) {
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert("Set an API key or download the on-phone model in Settings."); return; }
|
||||
void this.draft.start();
|
||||
}
|
||||
return;
|
||||
case "long-press-release": this.draft.release(); return;
|
||||
case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return;
|
||||
}
|
||||
}
|
||||
menuItems(): MenuItem[] {
|
||||
const record = this.conversations.current();
|
||||
const busy = () => this.draft.active || Boolean(record.session?.isTurnActive());
|
||||
const external = assistantBackendSetting.get() === "external";
|
||||
const submenu = (ctx: LayerContext, title: string, items: MenuItem[]) => ctx.stack.push(new WindowMenuLayer(title, items, true));
|
||||
return [
|
||||
...(external ? [{ label: "Sessions and model managed by bridge", disabled: true, onSelect: () => {} }] : []),
|
||||
{ label: "New session", disabled: () => external || busy(), onSelect: (ctx) => {
|
||||
if (this.conversations.create()) ctx.stack.clearToBase();
|
||||
} },
|
||||
{ label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions",
|
||||
[...this.conversations.list()].reverse().map((item) => ({
|
||||
label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`,
|
||||
onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model",
|
||||
ASSISTANT_MODEL_VALUES.map((model) => ({
|
||||
label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`,
|
||||
disabled: () => !this.conversations.available(model, record.reasoning),
|
||||
onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
|
||||
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
|
||||
return external || busy() || !llm?.effort;
|
||||
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
|
||||
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
|
||||
onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); },
|
||||
}))) },
|
||||
...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => {
|
||||
record.session?.cancel(); ctx.stack.clearToBase();
|
||||
} }] : []),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
export function createAiChatWindow(options: InProcessAppOptions): InProcessWindow {
|
||||
const layer = new AiChatLayer(shell.getAssistantConversations(), options);
|
||||
let unsubscribe = () => {};
|
||||
const app = createInProcessWindow({
|
||||
...options,
|
||||
appId: "ai-chat", windowId: "ai-chat", title: "AI Chat", iconLetter: "AI", icon: "message-circle", closeable: true,
|
||||
baseLayer: layer, holdToTalk: true,
|
||||
isVoiceCapturing: () => layer.draft.active,
|
||||
onSystemMenuOpened: () => layer.draft.cancel(),
|
||||
onAppMenuOpened: () => layer.draft.cancel(),
|
||||
setScreenOn: (on) => { if (!on) layer.draft.cancel(); },
|
||||
setSurfaceVisible: (visible) => { if (!visible) layer.draft.cancel(); options.setSurfaceVisible(visible); },
|
||||
menuItems: () => layer.menuItems(),
|
||||
receiveTextInput: (text) => { if (!layer.draft.active) layer.send(text); },
|
||||
onClosed: () => { unsubscribe(); layer.draft.cancel(); options.onClosed(); },
|
||||
});
|
||||
unsubscribe = layer.start(app.requestRender);
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import type { AppDefinition } from "../app-definition";
|
||||
import { createAiChatWindow } from "./ai-chat-app";
|
||||
|
||||
const aiChatApp: AppDefinition = {
|
||||
appId: "ai-chat",
|
||||
title: "AI Chat",
|
||||
icon: "message-circle",
|
||||
launch: (ctx) => ctx.launchInProcessApp("ai-chat", "window:ai-chat", createAiChatWindow),
|
||||
};
|
||||
export default aiChatApp;
|
||||
@@ -0,0 +1,125 @@
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceActivity } from "../../ui/shell/voice-activity";
|
||||
import type { LayerActions } from "../../ui/layers";
|
||||
|
||||
/** One hold/release cycle. Async startup and trailing cloud finals belong to this cycle only. */
|
||||
export class VoiceDraft {
|
||||
phase: "idle" | "preparing" | "listening" | "finishing" = "idle";
|
||||
text = "";
|
||||
status = "";
|
||||
private committed = "";
|
||||
private live = "";
|
||||
private generation = 0;
|
||||
private started = false;
|
||||
private ownsVoiceActivity = false;
|
||||
private starting = false;
|
||||
private unsubscribe: (() => void)[] = [];
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: Pick<LayerActions, "startVoiceCapture" | "stopVoiceCapture">,
|
||||
private readonly prepare: () => Promise<boolean>,
|
||||
private readonly changed: () => void,
|
||||
private readonly send: (text: string) => void,
|
||||
) {}
|
||||
|
||||
get active(): boolean { return this.phase !== "idle" || this.starting; }
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.active) return;
|
||||
const generation = ++this.generation;
|
||||
this.text = this.committed = this.live = "";
|
||||
this.phase = "preparing";
|
||||
this.status = "Preparing microphone...";
|
||||
this.changed();
|
||||
try {
|
||||
const ready = await this.prepare();
|
||||
if (generation !== this.generation) return;
|
||||
if (!ready) { this.cancel("Microphone unavailable"); return; }
|
||||
this.phase = "listening";
|
||||
this.status = "Listening... release to send";
|
||||
this.ownsVoiceActivity = true;
|
||||
voiceActivity.setActive(true);
|
||||
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
|
||||
if (!this.active || generation !== this.generation) return;
|
||||
if (event.isFinal) {
|
||||
const text = event.text.trim();
|
||||
if (text) this.committed = [this.committed, text].filter(Boolean).join(" ");
|
||||
this.live = "";
|
||||
} else this.live = event.text.trim();
|
||||
this.text = [this.committed, this.live].filter(Boolean).join(" ");
|
||||
if (event.isFinal && this.phase === "finishing") this.finish();
|
||||
else this.changed();
|
||||
}));
|
||||
// onStatus immediately replays the previous capture's status.
|
||||
let subscribed = false;
|
||||
this.unsubscribe.push(voiceControlBridge.onStatus(({ status }) => {
|
||||
if (!subscribed) return;
|
||||
if (generation !== this.generation || !this.active) return;
|
||||
if (/ignored|failed|error|denied/i.test(status)) this.cancel(status);
|
||||
}));
|
||||
subscribed = true;
|
||||
if (generation !== this.generation) return;
|
||||
this.starting = true;
|
||||
this.changed();
|
||||
await this.actions.startVoiceCapture();
|
||||
this.starting = false;
|
||||
if (generation !== this.generation) {
|
||||
await this.actions.stopVoiceCapture();
|
||||
this.changed();
|
||||
return;
|
||||
}
|
||||
this.started = true;
|
||||
if (this.isFinishing()) this.stopAndFinalize();
|
||||
} catch (error) {
|
||||
this.starting = false;
|
||||
if (generation === this.generation) this.cancel(`Microphone: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private isFinishing(): boolean { return this.phase === "finishing"; }
|
||||
|
||||
release(): void {
|
||||
if (this.phase === "preparing") { this.cancel(); return; }
|
||||
if (this.phase !== "listening") return;
|
||||
this.phase = "finishing";
|
||||
this.status = "Finishing transcription...";
|
||||
this.changed();
|
||||
if (!this.starting) this.stopAndFinalize();
|
||||
}
|
||||
|
||||
private stopAndFinalize(): void {
|
||||
// Install fallback first: stop may synchronously deliver the final.
|
||||
this.timer = setTimeout(() => this.finish(), 1500);
|
||||
if (this.started) {
|
||||
this.started = false;
|
||||
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
|
||||
}
|
||||
}
|
||||
|
||||
private finish(): void {
|
||||
if (this.phase !== "finishing") return;
|
||||
const text = this.text.trim();
|
||||
this.cancel(text ? "" : "No speech detected");
|
||||
if (text) this.send(text);
|
||||
}
|
||||
|
||||
cancel(status = ""): void {
|
||||
++this.generation;
|
||||
this.phase = "idle";
|
||||
this.status = status;
|
||||
this.text = this.live = this.committed = "";
|
||||
if (this.timer !== null) clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
|
||||
if (this.started) {
|
||||
this.started = false;
|
||||
void Promise.resolve(this.actions.stopVoiceCapture()).catch(() => {});
|
||||
}
|
||||
if (this.ownsVoiceActivity) {
|
||||
this.ownsVoiceActivity = false;
|
||||
voiceActivity.setActive(false);
|
||||
}
|
||||
this.changed();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type AppDefinition } from "./app-definition";
|
||||
import aiChatApp from "./ai-chat";
|
||||
import launcherApp from "./launcher";
|
||||
import timerApp from "./timer";
|
||||
import calculatorApp from "./calculator";
|
||||
@@ -30,6 +31,7 @@ import settingsApp from "./settings";
|
||||
*/
|
||||
export const ALL_APPS: readonly AppDefinition[] = [
|
||||
launcherApp,
|
||||
aiChatApp,
|
||||
timerApp,
|
||||
calculatorApp,
|
||||
terminalApp,
|
||||
|
||||
Reference in new issue
Block a user