Add app tool registration for voice assistant; voice assistant can read and send to terminals

This commit is contained in:
James Babcock committed 2026-07-25 01:19:07 -07:00
1 parent 0aeb24ed00
commit fe205c0164
9 files changed
+533 -16

No files matched your search

+7 -2
View File
@@ -26,7 +26,12 @@ export type DirectTurnOptions = {
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: AnthropicMessage[];
tools: AnthropicToolDefinition[];
/**
* Build the tool list for the next request. Called once per loop iteration so
* the set reflects apps opening/closing and foreground changes mid-turn; it
* also refreshes the session's api-name map that resolveToolName reads.
*/
buildTools: () => AnthropicToolDefinition[];
registry: ToolRegistry;
/** Map an API tool name (as the model calls it) back to its registry name. */
resolveToolName: (apiName: string) => string;
@@ -62,7 +67,7 @@ export class DirectAnthropicBackend {
system: options.system,
effort: options.effort,
messages: options.messages,
tools: options.tools,
tools: options.buildTools(),
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar);
+1 -2
View File
@@ -60,7 +60,6 @@ export class AssistantSession {
this.messages.push({ role: "user", content: text });
this.trimHistory();
const tools = this.buildToolDefinitions();
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
const finish = () => {
@@ -74,7 +73,7 @@ export class AssistantSession {
effort: "low",
system,
messages: this.messages,
tools,
buildTools: () => this.buildToolDefinitions(),
registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: {
+62 -2
View File
@@ -70,14 +70,28 @@ export type CallToolOptions = {
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
/** How the shell reaches a window's tool handler and gates its foreground tools. */
export type AppToolProvider = {
windowId: string;
appId: string;
/** The window's declared specs, with UNPREFIXED names (registry adds the prefix). */
specs: ToolSpec[];
/** Invoke a tool on the window by its unprefixed name. */
invoke: (toolName: string, args: unknown) => Promise<ToolResult> | ToolResult;
/** Whether the window currently owns the foreground (gates `foreground` tools). */
isForeground: () => boolean;
};
export class ToolRegistry {
private readonly registrations = new Map<string, ToolRegistration>();
private readonly changeListeners = new Set<() => void>();
// windowId -> the canonical (prefixed) tool names it contributed, for bulk
// removal when the window closes or re-declares.
private readonly windowToolNames = new Map<string, string[]>();
/** Register (or replace) a tool. Names are unique across all tiers. */
register(registration: ToolRegistration): void {
const { name } = registration.spec;
this.registrations.set(name, registration);
this.addRegistration(registration);
this.fireToolsChanged();
}
@@ -95,6 +109,52 @@ export class ToolRegistry {
}
}
/**
* Declare (replacing any prior set) the tools contributed by one app window.
* Names are prefixed `app.<appId>.` so windows can't shadow system tools or
* each other. `open` tools stay live while the window exists; `foreground`
* tools are gated on `isForeground()` at both list and call time.
*/
setAppTools(provider: AppToolProvider): void {
this.clearWindowTools(provider.windowId, false);
const names: string[] = [];
for (const spec of provider.specs) {
if (spec.availability !== "open" && spec.availability !== "foreground") {
console.warn(`app tool ${spec.name} has non-app availability ${spec.availability}; skipping`);
continue;
}
const canonical = `app.${provider.appId}.${spec.name}`;
const toolName = spec.name;
this.addRegistration({
spec: { ...spec, name: canonical },
handler: (args) => provider.invoke(toolName, args),
isAvailable: spec.availability === "foreground" ? () => provider.isForeground() : undefined,
});
names.push(canonical);
}
this.windowToolNames.set(provider.windowId, names);
this.fireToolsChanged();
}
/** Remove all tools contributed by a window (its worker closed or the window did). */
removeAppTools(windowId: string): void {
this.clearWindowTools(windowId, true);
}
private clearWindowTools(windowId: string, fire: boolean): void {
const names = this.windowToolNames.get(windowId);
if (!names) return;
for (const name of names) {
this.registrations.delete(name);
}
this.windowToolNames.delete(windowId);
if (fire) this.fireToolsChanged();
}
private addRegistration(registration: ToolRegistration): void {
this.registrations.set(registration.spec.name, registration);
}
/** Specs for the tools callable right now, given the availability tiers. */
listTools(options: ListToolsOptions = {}): ToolSpec[] {
const specs: ToolSpec[] = [];
+85 -1
View File
@@ -1,6 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { windowIcon } from "./chrome-layer";
import { type IconName } from "../../graphics/icons";
import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/tool-registry";
import { shell, type ShellWindow } from "./shell";
/**
@@ -16,7 +17,9 @@ export type WorkerAppMessage =
| { type: "text-input"; windowId: string; text: string }
| { type: "render"; windowId: string; focused: boolean }
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
| { type: "screen"; on: boolean };
| { type: "screen"; on: boolean }
/** Assistant tool invocation aimed at a window; reply with tool-result. */
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
export type WorkerAppReply =
| { type: "yield-focus"; windowId: string }
@@ -59,6 +62,21 @@ export type WorkerAppReply =
*/
type: "set-tray-icon";
icon: { width: number; height: number; pixels: number[] } | null;
}
| {
/**
* Declare (replacing the prior set) the assistant tools this window
* contributes. Names are unprefixed; the registry adds `app.<appId>.`.
*/
type: "set-tools";
windowId: string;
tools: ToolSpec[];
}
| {
/** Result of a tool-call, matched to the request by callId. */
type: "tool-result";
callId: string;
result: ToolResult;
};
export type WorkerWindowSpec = {
@@ -92,8 +110,24 @@ export type WorkerAppHostOptions = {
* and manages compositor surfaces for the app's windows. The worker submits
* frames straight to the Java compositor, so no pixels cross this boundary.
*/
/** A tool-call awaiting its worker reply; also its own leak-safety timeout. */
type PendingToolCall = {
windowId: string;
resolve: (result: ToolResult) => void;
timer: ReturnType<typeof setTimeout>;
};
/**
* How long the host waits for a worker's tool-result before giving up. The
* registry applies its own (usually shorter) per-tool timeout; this is the
* backstop that also frees the pending-call entry if the worker never replies.
*/
const TOOL_CALL_HOST_TIMEOUT_MS = 15_000;
export class WorkerAppHost {
private readonly openWindows = new Set<string>();
private readonly pendingToolCalls = new Map<string, PendingToolCall>();
private nextCallSerial = 1;
constructor(private readonly options: WorkerAppHostOptions) {
options.worker.onmessage = (event: MessageEvent) => {
@@ -152,6 +186,27 @@ export class WorkerAppHost {
case "set-title":
// Titles are informational for now (sidebar shows icons only).
break;
case "set-tools":
// Only a window we actually have open may contribute tools.
if (this.openWindows.has(message.windowId)) {
toolRegistry.setAppTools({
windowId: message.windowId,
appId: this.options.appId,
specs: message.tools,
invoke: (toolName, args) => this.callWindowTool(message.windowId, toolName, args),
isForeground: () => shell.foregroundWindow()?.windowId === message.windowId,
});
}
break;
case "tool-result": {
const pending = this.pendingToolCalls.get(message.callId);
if (pending) {
clearTimeout(pending.timer);
this.pendingToolCalls.delete(message.callId);
pending.resolve(message.result);
}
break;
}
}
};
options.worker.onerror = (error) => {
@@ -181,6 +236,9 @@ export class WorkerAppHost {
closeable: true,
close: () => {
this.openWindows.delete(spec.windowId);
// Withdraw this window's tools and fail any in-flight calls to it.
toolRegistry.removeAppTools(spec.windowId);
this.failPendingToolCallsFor(spec.windowId);
this.post({ type: "close-window", windowId: spec.windowId });
this.options.removeSurface(surfaceId);
},
@@ -235,6 +293,32 @@ export class WorkerAppHost {
return window;
}
/**
* Post a tool-call to the worker and resolve when its tool-result comes back.
* Resolves with a tool error on timeout so a hung worker yields a tool error
* rather than a stuck assistant turn.
*/
private callWindowTool(windowId: string, toolName: string, args: unknown): Promise<ToolResult> {
return new Promise<ToolResult>((resolve) => {
const callId = `${this.options.appId}:${this.nextCallSerial++}`;
const timer = setTimeout(() => {
this.pendingToolCalls.delete(callId);
resolve({ ok: false, error: `App ${this.options.appId} did not respond to ${toolName}` });
}, TOOL_CALL_HOST_TIMEOUT_MS);
this.pendingToolCalls.set(callId, { windowId, resolve, timer });
this.post({ type: "tool-call", callId, windowId, name: toolName, args });
});
}
private failPendingToolCallsFor(windowId: string): void {
for (const [callId, pending] of this.pendingToolCalls) {
if (pending.windowId !== windowId) continue;
clearTimeout(pending.timer);
this.pendingToolCalls.delete(callId);
pending.resolve({ ok: false, error: "The target window was closed" });
}
}
private post(message: WorkerAppMessage): void {
this.options.worker.postMessage(message);
}
+123
View File
@@ -39,6 +39,7 @@ import {
} from "../ui/menu";
import { defaultWindowMenuItems, WindowMenu } from "../ui/window-menu";
import type { WorkerAppMessage, WorkerAppReply } from "../ui/shell/worker-window";
import type { ToolResult, ToolSpec } from "../assistant/tool-registry";
declare const global: any;
declare const com: any;
@@ -102,7 +103,43 @@ type TerminalWindow = HubWindow | ViewWindow;
const windows = new Map<string, TerminalWindow>();
const pendingViews = new Map<string, { socket: string; label: string }>();
// The view an assistant tool acts on when the terminal isn't foregrounded:
// the last view to be foregrounded or receive input.
let activeViewId: string | null = null;
let nextViewSerial = 1;
/**
* Assistant tools this app contributes, declared on the hub window (which the
* launcher opens and which persists for the app's life). All `open`-tier so
* "rerun the build" works while the terminal is backgrounded. send_input and
* read_screen act on the active view session (see resolveActiveView).
*/
const TERMINAL_TOOLS: ToolSpec[] = [
{
name: "list_sessions",
description: "List the live g2mirror terminal sessions the glasses can see.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
availability: "open",
},
{
name: "send_input",
description:
"Type a line into the active terminal session and submit it (as if typed and Enter pressed). Use to run a command in the terminal.",
inputSchema: {
type: "object",
properties: { text: { type: "string", description: "The line to type and run." } },
required: ["text"],
additionalProperties: false,
},
availability: "open",
},
{
name: "read_screen",
description: "Return the current visible contents of the active terminal session's screen.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
availability: "open",
},
];
let viewportWidth = 0;
let viewportHeight = 0;
let gridCols = 0;
@@ -161,6 +198,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
if (!window) break;
window.foreground = message.foreground;
window.focused = message.focused;
if (window.foreground && window.kind === "view") activeViewId = window.windowId;
if (window.foreground) renderAndSubmit(window, 0);
break;
}
@@ -172,6 +210,19 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
}
break;
case "tool-call": {
const callId = message.callId;
Promise.resolve(handleTerminalTool(message.name, message.args))
.then((result) => post({ type: "tool-result", callId, result }))
.catch((error) =>
post({
type: "tool-result",
callId,
result: { ok: false, error: String((error as Error)?.message ?? error) },
}),
);
break;
}
}
};
@@ -209,6 +260,8 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb
selectedIndex: 0,
scrollRow: 0,
});
// The hub carries the app's assistant tools; declare them once it exists.
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
if (!controlClient) {
startControlClient();
}
@@ -224,6 +277,7 @@ function closeWindow(windowId: string): void {
window.client.stop();
}
windows.delete(windowId);
if (activeViewId === windowId) activeViewId = null;
if (window.kind === "view") {
renderHubWindows();
}
@@ -414,6 +468,7 @@ function windowMenuItems(window: TerminalWindow): MenuItem[] {
}
function handleInput(window: TerminalWindow, event: DashboardInputEvent, frameId: number): void {
if (window.kind === "view") activeViewId = window.windowId;
// An open window menu owns all input (it closes itself via pop).
if (window.menu?.isOpen()) {
window.menu
@@ -743,3 +798,71 @@ function sessionLabel(session: G2MirrorSession): string {
const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/");
return hint || "session";
}
/** Dispatch an assistant tool-call (unprefixed name) to its handler. */
function handleTerminalTool(name: string, args: any): ToolResult {
switch (name) {
case "list_sessions":
return toolListSessions();
case "send_input":
return toolSendInput(args);
case "read_screen":
return toolReadScreen();
default:
return { ok: false, error: `Unknown terminal tool: ${name}` };
}
}
function toolListSessions(): ToolResult {
const sessions = controlState?.sessions ?? [];
if (!sessions.length) {
return { ok: true, content: "No live terminal sessions (run g2mirror <command> on the host)." };
}
const lines = sessions.map(
(session) => `- ${sessionLabel(session)}${viewWindowIdForSocket(session.socket) ? " [open]" : ""}`,
);
return { ok: true, content: lines.join("\n") };
}
function toolSendInput(args: any): ToolResult {
const text = String(args?.text ?? "");
if (!text) return { ok: false, error: "send_input requires non-empty text." };
const view = resolveActiveView();
if (!view) return { ok: false, error: "No terminal session is open to send input to." };
view.client.submitInput(text);
return { ok: true, content: `Sent to ${view.label}.` };
}
function toolReadScreen(): ToolResult {
const view = resolveActiveView();
if (!view) return { ok: false, error: "No terminal session is open." };
const length = view.emulator.bufferLength();
const start = Math.max(0, length - gridRows);
const lines: string[] = [];
for (let index = start; index < length; index++) {
lines.push(view.emulator.lineAt(index));
}
const text = lines.join("\n").replace(/\s+$/, "");
return { ok: true, content: text || "(screen is empty)" };
}
/**
* The terminal view an `open`-tier tool acts on: a foregrounded view if any,
* else the last view to be active, else the sole open view. Null when no view
* is open (the model gets a tool error it can relay).
*/
function resolveActiveView(): ViewWindow | null {
const views: ViewWindow[] = [];
let foreground: ViewWindow | null = null;
for (const window of windows.values()) {
if (window.kind !== "view") continue;
views.push(window);
if (window.foreground) foreground = window;
}
if (foreground) return foreground;
if (activeViewId) {
const window = windows.get(activeViewId);
if (window && window.kind === "view") return window;
}
return views.length === 1 ? views[0]! : null;
}