forked from Blendman974/faceclaw
Add "AI Chat" app.
This commit is contained in:
1 parent
78c56ffc4e
commit
0765e943fb
13 files changed
+616
-87
No files matched your search
@@ -38,6 +38,12 @@ export type InProcessWindowOptions = {
|
||||
* has no menu of its own, and tap-then-hold opens the system menu instead.
|
||||
*/
|
||||
menuItems?: () => MenuItem[];
|
||||
/** Dedicated chat gestures: click menu, hold microphone, tap-hold system menu. */
|
||||
holdToTalk?: boolean;
|
||||
isVoiceCapturing?: () => boolean;
|
||||
onSystemMenuOpened?: () => void;
|
||||
onAppMenuOpened?: () => void;
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
actions: LayerActions;
|
||||
/**
|
||||
@@ -141,7 +147,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
shell.openSystemMenu(options.windowId);
|
||||
return;
|
||||
}
|
||||
stack.push(new WindowMenuLayer(options.title, items));
|
||||
options.onAppMenuOpened?.();
|
||||
stack.push(new WindowMenuLayer(options.title, items, options.holdToTalk));
|
||||
};
|
||||
|
||||
const window: ShellWindow = {
|
||||
@@ -153,6 +160,9 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// The window's own LayerStack decides per layer whether a swipe is
|
||||
// directional or falls back to click / double-click.
|
||||
acceptsDirectional: true,
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
close: () => {
|
||||
@@ -168,6 +178,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// The shell opened its system menu over this window; close our own
|
||||
// context menu so the two never stack. Never forwarded to app layers.
|
||||
if (event.type === "system-menu-opened") {
|
||||
options.onSystemMenuOpened?.();
|
||||
if (stack.popIfTop((layer) => layer instanceof WindowMenuLayer)) {
|
||||
requestRender();
|
||||
}
|
||||
@@ -178,7 +189,8 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
// submenus and app content alike. A plain long-press never arrives:
|
||||
// the shell keeps it for the system menu (no in-process window claims
|
||||
// it).
|
||||
if (event.type === "short-then-long-press") {
|
||||
if ((!options.holdToTalk && event.type === "short-then-long-press") ||
|
||||
(options.holdToTalk && event.type === "click" && stack.isAtBase())) {
|
||||
openWindowMenu();
|
||||
await render(frameId);
|
||||
return;
|
||||
|
||||
+68
-29
@@ -21,7 +21,9 @@ import { KeyboardInputLayer, type KeyboardInputSession } from "./keyboard-input"
|
||||
import { voiceActivity } from "./voice-activity";
|
||||
import { AssistantLayer } from "./assistant";
|
||||
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { resolveAssistantModel, type AssistantModel } from "../../assistant/models";
|
||||
import { AssistantConversations, type ReasoningLevel } from "../../assistant/conversations";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
@@ -91,6 +93,10 @@ export type ShellWindow = {
|
||||
* system menu; holding the press past the escape threshold still opens it.
|
||||
*/
|
||||
claimsLongPress?: () => boolean;
|
||||
/** Chat uses hold-to-talk and tap-then-hold for the system menu, without an escape timer. */
|
||||
holdToTalk?: boolean;
|
||||
/** A window owns microphone capture outside the shell voice dialog. */
|
||||
isVoiceCapturing?: () => boolean;
|
||||
/**
|
||||
* True when the window gives swipe-left / swipe-right (watch directional
|
||||
* input) a meaning; otherwise the shell forwards directionalFallback(event).
|
||||
@@ -298,7 +304,29 @@ class Shell {
|
||||
private readonly trayIcons = new Map<string, GrayImage>();
|
||||
private activeVoiceLayer: VoiceInputLayer | null = null;
|
||||
private activeKeyboardLayer: KeyboardInputLayer | null = null;
|
||||
private assistantSession: AssistantSession | null = null;
|
||||
private conversations: AssistantConversations | null = null;
|
||||
private get assistantSession(): AssistantSession | null {
|
||||
return this.conversations?.current().session ?? null;
|
||||
}
|
||||
|
||||
getAssistantConversations(): AssistantConversations {
|
||||
if (!this.conversations) {
|
||||
this.conversations = new AssistantConversations(
|
||||
(model, reasoning) => this.resolveAssistantConfiguration(model, reasoning),
|
||||
() => assistantModelSetting.get(),
|
||||
(value) => setStringSetting("assistant.conversations", value),
|
||||
getStringSetting("assistant.conversations", ""),
|
||||
);
|
||||
}
|
||||
return this.conversations;
|
||||
}
|
||||
|
||||
async prepareChatVoiceCapture(): Promise<boolean> {
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.voiceDialogPending) return false;
|
||||
const ready = (await this.config.prepareVoiceCapture?.()) ?? true;
|
||||
return ready && !this.activeVoiceLayer && !this.activeKeyboardLayer &&
|
||||
this.stack.isAtBase() && this.isWindowFocused("ai-chat");
|
||||
}
|
||||
private assistantLayer: AssistantLayer | null = null;
|
||||
private readonly assistantActivityListeners = new Set<(event: AssistantActivityEvent) => void>();
|
||||
private readonly alertListeners = new Set<(text: string) => void>();
|
||||
@@ -559,7 +587,7 @@ class Shell {
|
||||
// ring. An in-flight assistant turn suspends it for the same reason (a
|
||||
// tool loop can run for a while with no input); once the turn ends and
|
||||
// the Follow-up/Done menu is showing, the normal idle timeout resumes.
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive()) {
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
@@ -645,6 +673,7 @@ class Shell {
|
||||
// Even AI app never launches, so the firmware does not power the display
|
||||
// for us either -- actions that need it wake the screen themselves.
|
||||
if (event.type === "wakeword") {
|
||||
if (this.foregroundWindow()?.isVoiceCapturing?.()) return { shell: true, window: false };
|
||||
const action = wakeWordActionSetting.get();
|
||||
if (action === "off") {
|
||||
return { shell: false, window: false };
|
||||
@@ -696,11 +725,11 @@ class Shell {
|
||||
if (!window) {
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
if (!window.claimsLongPress?.()) {
|
||||
if (!window.holdToTalk && !window.claimsLongPress?.()) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
this.startEscapeMenuTimer();
|
||||
if (!window.holdToTalk) this.startEscapeMenuTimer();
|
||||
if (this.focus === "sidebar") {
|
||||
this.focus = "window";
|
||||
window.onFocus?.(this.lastInput);
|
||||
@@ -716,6 +745,10 @@ class Shell {
|
||||
// when it has none.
|
||||
if (event.type === "short-then-long-press") {
|
||||
const window = this.foregroundWindow();
|
||||
if (window?.holdToTalk) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
// Over the open system menu it switches to the app's context menu:
|
||||
// close the system menu and deliver the gesture to the window as
|
||||
// usual. A window without a menu of its own would only ask for the
|
||||
@@ -1024,7 +1057,7 @@ class Shell {
|
||||
*/
|
||||
startKeyboardInput(): KeyboardInputSession | null {
|
||||
if (this.activeKeyboardLayer) return this.activeKeyboardLayer;
|
||||
if (this.activeVoiceLayer) return null;
|
||||
if (this.activeVoiceLayer || this.foregroundWindow()?.isVoiceCapturing?.()) return null;
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
const assistantLayer = this.assistantLayer;
|
||||
const assistantSession = this.assistantSession;
|
||||
@@ -1069,7 +1102,8 @@ class Shell {
|
||||
}
|
||||
|
||||
isAssistantAvailable(): boolean {
|
||||
return this.resolveAssistantConfiguration() !== null;
|
||||
const current = this.getAssistantConversations().current();
|
||||
return this.resolveAssistantConfiguration(current.model, current.reasoning) !== null;
|
||||
}
|
||||
|
||||
/** Observe assistant turns (see AssistantActivityEvent). */
|
||||
@@ -1107,20 +1141,13 @@ class Shell {
|
||||
}
|
||||
|
||||
private ensureAssistantSession(): AssistantSession | null {
|
||||
const config = this.resolveAssistantConfiguration();
|
||||
if (!config) return null;
|
||||
if (
|
||||
!this.assistantSession ||
|
||||
this.assistantSession.isExpired() ||
|
||||
!this.assistantSession.matchesConfiguration(config)
|
||||
) {
|
||||
this.assistantSession?.cancel();
|
||||
this.assistantSession = new AssistantSession(config);
|
||||
}
|
||||
return this.assistantSession;
|
||||
return this.getAssistantConversations().ensureSession();
|
||||
}
|
||||
|
||||
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
|
||||
private resolveAssistantConfiguration(
|
||||
model: AssistantModel = this.conversations?.current().model ?? assistantModelSetting.get(),
|
||||
reasoning: ReasoningLevel = this.conversations?.current().reasoning ?? "default",
|
||||
): AssistantBackendConfig | null {
|
||||
if (assistantBackendSetting.get() === "external") {
|
||||
const host = assistantBridgeHostSetting.get().trim();
|
||||
const token = assistantBridgeTokenSetting.get();
|
||||
@@ -1128,10 +1155,11 @@ class Shell {
|
||||
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
||||
return { kind: "external", bridge: { host, port, token } };
|
||||
}
|
||||
const llm = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
const llm = resolveAssistantModel(model, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
});
|
||||
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
|
||||
return llm ? { kind: "direct", llm } : null;
|
||||
}
|
||||
|
||||
@@ -1151,7 +1179,9 @@ class Shell {
|
||||
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
||||
* existing session and overlay.
|
||||
*/
|
||||
sendToAssistant(text: string): void {
|
||||
sendToAssistant(text: string, showOverlay = true): void {
|
||||
text = text.trim();
|
||||
if (!text) return;
|
||||
const session = this.ensureAssistantSession();
|
||||
if (!session) {
|
||||
this.showAlert(
|
||||
@@ -1161,7 +1191,15 @@ class Shell {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (session.isTurnActive()) {
|
||||
this.showAlert("The assistant is still working on the previous request");
|
||||
return;
|
||||
}
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
if (!showOverlay || this.foregroundWindow()?.appId === "ai-chat") {
|
||||
this.runAssistantTurn(session, null, text);
|
||||
return;
|
||||
}
|
||||
let layer = this.assistantLayer;
|
||||
if (!layer) {
|
||||
const created = new AssistantLayer(this.config.actions, {
|
||||
@@ -1170,7 +1208,7 @@ class Shell {
|
||||
onClose: () => this.closeAssistantLayer(),
|
||||
onRemoved: () => {
|
||||
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
||||
// stop the turn and drop the reference so a later query starts clean.
|
||||
// stop the turn and drop the overlay reference; keep shared history.
|
||||
this.assistantSession?.cancel();
|
||||
if (this.assistantLayer === created) this.assistantLayer = null;
|
||||
this.emitAssistantActivity({ phase: "closed", text: "" });
|
||||
@@ -1184,23 +1222,24 @@ class Shell {
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
|
||||
layer.startTurn();
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer | null, text: string): void {
|
||||
if (session.isTurnActive()) return;
|
||||
layer?.startTurn();
|
||||
let replySoFar = "";
|
||||
this.emitAssistantActivity({ phase: "thinking", text: "" });
|
||||
session.sendUtterance(text, this.buildAssistantContext(), {
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
replySoFar = textSoFar;
|
||||
layer.onTextDelta(delta, textSoFar);
|
||||
layer?.onTextDelta(delta, textSoFar);
|
||||
this.emitAssistantActivity({ phase: "streaming", text: textSoFar });
|
||||
},
|
||||
onToolActivity: (label) => layer.onToolActivity(label),
|
||||
onToolActivity: (label) => layer?.onToolActivity(label),
|
||||
onTurnDone: () => {
|
||||
layer.onTurnDone();
|
||||
layer?.onTurnDone();
|
||||
this.emitAssistantActivity({ phase: "done", text: replySoFar });
|
||||
},
|
||||
onError: (message) => {
|
||||
layer.onError(message);
|
||||
layer?.onError(message);
|
||||
this.emitAssistantActivity({ phase: "error", text: message });
|
||||
},
|
||||
});
|
||||
@@ -1356,7 +1395,7 @@ class Shell {
|
||||
// when the app has one (otherwise it opens this very menu, not worth a
|
||||
// hint).
|
||||
const footer = foreground.hasAppMenu?.()
|
||||
? gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]])
|
||||
? (foreground.holdToTalk ? undefined : gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "app menu"]]))
|
||||
: undefined;
|
||||
layer = new ShellOverlayMenuLayer(items, footer, () => this.yieldFocusToSidebar());
|
||||
layer.selectItem(initialSelection);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { singlePlane, type Plane } from "../graphics/plane";
|
||||
import { GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||
import { GESTURE_LONG_PRESS, GESTURE_SHORT_THEN_LONG_PRESS, gestureHints, type InputEvent } from "./gestures";
|
||||
import { type Layer, LayerStack, noopLayerActions } from "./layers";
|
||||
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem, type MenuLayout } from "./menu";
|
||||
import type { WorkerAppReply } from "./shell/worker-window";
|
||||
@@ -27,8 +27,10 @@ export const WINDOW_MENU_LAYOUT: MenuLayout = {
|
||||
};
|
||||
|
||||
export class WindowMenuLayer extends MenuLayer {
|
||||
constructor(title: string | null, items: MenuItem[]) {
|
||||
super(title, items, WINDOW_MENU_LAYOUT);
|
||||
constructor(title: string | null, items: MenuItem[], holdToTalk = false) {
|
||||
super(title, items, holdToTalk
|
||||
? { ...WINDOW_MENU_LAYOUT, footer: gestureHints([[GESTURE_SHORT_THEN_LONG_PRESS, "system menu"]]) }
|
||||
: WINDOW_MENU_LAYOUT);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user