forked from Blendman974/faceclaw
Implement a basic voice assistant
This commit is contained in:
1 parent
d2c987d48a
commit
0aeb24ed00
14 files changed
+1923
-80
No files matched your search
@@ -0,0 +1,141 @@
|
||||
import {
|
||||
streamAnthropicMessage,
|
||||
type AnthropicContentBlock,
|
||||
type AnthropicMessage,
|
||||
type AnthropicStreamHandle,
|
||||
type AnthropicToolDefinition,
|
||||
} from "../native/anthropic";
|
||||
import type { ToolRegistry } from "./tool-registry";
|
||||
import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
/**
|
||||
* The on-phone agent loop: stream a reply, and while the model asks for tools,
|
||||
* execute them through the registry and feed the results back until it stops.
|
||||
* History lives in the AssistantSession; this backend appends the assistant
|
||||
* and tool-result messages to the array it is handed.
|
||||
*/
|
||||
|
||||
/** Safety caps so a tool-thrashing model can't loop or run forever. */
|
||||
const MAX_ITERATIONS = 8;
|
||||
const MAX_TURN_MS = 2 * 60 * 1000;
|
||||
|
||||
export type DirectTurnOptions = {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
effort?: "low" | "medium" | "high";
|
||||
system: string;
|
||||
/** Owned by the session; this loop appends assistant + tool_result messages. */
|
||||
messages: AnthropicMessage[];
|
||||
tools: AnthropicToolDefinition[];
|
||||
registry: ToolRegistry;
|
||||
/** Map an API tool name (as the model calls it) back to its registry name. */
|
||||
resolveToolName: (apiName: string) => string;
|
||||
callbacks: AssistantTurnCallbacks;
|
||||
};
|
||||
|
||||
export class DirectAnthropicBackend {
|
||||
runTurn(options: DirectTurnOptions): AssistantTurnHandle {
|
||||
const startMs = Date.now();
|
||||
let cancelled = false;
|
||||
let streamHandle: AnthropicStreamHandle | null = null;
|
||||
|
||||
const finishError = (message: string) => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
options.callbacks.onError(message);
|
||||
};
|
||||
|
||||
const runIteration = (iteration: number): void => {
|
||||
if (cancelled) return;
|
||||
if (iteration >= MAX_ITERATIONS) {
|
||||
finishError("Assistant stopped: too many tool steps");
|
||||
return;
|
||||
}
|
||||
if (Date.now() - startMs > MAX_TURN_MS) {
|
||||
finishError("Assistant stopped: turn took too long");
|
||||
return;
|
||||
}
|
||||
|
||||
streamHandle = streamAnthropicMessage({
|
||||
apiKey: options.apiKey,
|
||||
model: options.model,
|
||||
system: options.system,
|
||||
effort: options.effort,
|
||||
messages: options.messages,
|
||||
tools: options.tools,
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
if (cancelled) return;
|
||||
options.callbacks.onTextDelta(delta, textSoFar);
|
||||
},
|
||||
onDone: (result) => {
|
||||
streamHandle = null;
|
||||
if (cancelled) return;
|
||||
if (result.stopReason === "refusal") {
|
||||
finishError("The assistant declined this request");
|
||||
return;
|
||||
}
|
||||
// Record what the model produced this iteration.
|
||||
const assistantContent: AnthropicContentBlock[] = result.content.length
|
||||
? result.content
|
||||
: [{ type: "text", text: result.text }];
|
||||
options.messages.push({ role: "assistant", content: assistantContent });
|
||||
|
||||
const toolUses = assistantContent.filter(
|
||||
(block): block is Extract<AnthropicContentBlock, { type: "tool_use" }> =>
|
||||
block.type === "tool_use",
|
||||
);
|
||||
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
|
||||
options.callbacks.onTurnDone({ stopReason: result.stopReason });
|
||||
return;
|
||||
}
|
||||
void this.runToolCalls(toolUses, options)
|
||||
.then((toolResults) => {
|
||||
// Always record the results, even if cancelled mid-flight: an
|
||||
// assistant tool_use with no matching tool_result would make the
|
||||
// next turn's request invalid. Only the loop continuation is gated.
|
||||
options.messages.push({ role: "user", content: toolResults });
|
||||
if (cancelled) return;
|
||||
runIteration(iteration + 1);
|
||||
})
|
||||
.catch((error) => {
|
||||
finishError(`Tool execution failed: ${String((error as Error)?.message ?? error)}`);
|
||||
});
|
||||
},
|
||||
onError: (message) => {
|
||||
streamHandle = null;
|
||||
finishError(message);
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
runIteration(0);
|
||||
|
||||
return {
|
||||
cancel: () => {
|
||||
cancelled = true;
|
||||
streamHandle?.cancel();
|
||||
streamHandle = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async runToolCalls(
|
||||
toolUses: Array<Extract<AnthropicContentBlock, { type: "tool_use" }>>,
|
||||
options: DirectTurnOptions,
|
||||
): Promise<AnthropicContentBlock[]> {
|
||||
const results: AnthropicContentBlock[] = [];
|
||||
for (const toolUse of toolUses) {
|
||||
const canonicalName = options.resolveToolName(toolUse.name);
|
||||
options.callbacks.onToolActivity(canonicalName);
|
||||
const result = await options.registry.callTool(canonicalName, toolUse.input);
|
||||
const content = result.ok ? result.content ?? "" : result.error ?? "Tool error";
|
||||
results.push({
|
||||
type: "tool_result",
|
||||
tool_use_id: toolUse.id,
|
||||
content,
|
||||
is_error: !result.ok,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic";
|
||||
import { DirectAnthropicBackend } from "./direct-backend";
|
||||
import { toolRegistry, type ToolRegistry } from "./tool-registry";
|
||||
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
/**
|
||||
* One assistant conversation. Owns the message history and turn state and
|
||||
* drives a backend (direct Anthropic loop for phase 1). The UI talks only to
|
||||
* this surface, so swapping in the external-agent backend later doesn't touch
|
||||
* AssistantLayer.
|
||||
*
|
||||
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
|
||||
* one for a new wakeword and reuses the current one for a follow-up.
|
||||
*/
|
||||
|
||||
/** A new PTT after this much idle starts a fresh conversation. */
|
||||
const SESSION_IDLE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** Trim history from the head once it grows past this many messages. */
|
||||
const MAX_HISTORY_MESSAGES = 40;
|
||||
|
||||
const SYSTEM_PROMPT_BASE = [
|
||||
"You are the voice assistant built into a pair of Even Realities G2 smart glasses.",
|
||||
"Your reply is shown as text on a 576x288 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list.",
|
||||
"Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it.",
|
||||
"If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
|
||||
].join(" ");
|
||||
|
||||
export class AssistantSession {
|
||||
private readonly backend = new DirectAnthropicBackend();
|
||||
private readonly messages: AnthropicMessage[] = [];
|
||||
private turnHandle: AssistantTurnHandle | null = null;
|
||||
private lastActivityMs = Date.now();
|
||||
// API-safe tool name -> canonical registry name. The Anthropic API only
|
||||
// allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses.
|
||||
// get_state") are sanitized for the request and mapped back on tool calls.
|
||||
private readonly toolNameMap = new Map<string, string>();
|
||||
|
||||
constructor(
|
||||
private readonly apiKey: string,
|
||||
private readonly registry: ToolRegistry = toolRegistry,
|
||||
) {}
|
||||
|
||||
/** Whether this session has been idle long enough to retire. */
|
||||
isExpired(nowMs = Date.now()): boolean {
|
||||
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
|
||||
}
|
||||
|
||||
isTurnActive(): boolean {
|
||||
return this.turnHandle !== null;
|
||||
}
|
||||
|
||||
/** Begin a turn from a spoken utterance. Only one turn runs at a time. */
|
||||
sendUtterance(text: string, ctx: AssistantContext, callbacks: AssistantTurnCallbacks): void {
|
||||
if (this.turnHandle) {
|
||||
callbacks.onError("The assistant is still working on the previous request");
|
||||
return;
|
||||
}
|
||||
this.lastActivityMs = Date.now();
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
|
||||
const tools = this.buildToolDefinitions();
|
||||
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
|
||||
|
||||
const finish = () => {
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
};
|
||||
|
||||
this.turnHandle = this.backend.runTurn({
|
||||
apiKey: this.apiKey,
|
||||
model: DEFAULT_LLM_MODEL,
|
||||
effort: "low",
|
||||
system,
|
||||
messages: this.messages,
|
||||
tools,
|
||||
registry: this.registry,
|
||||
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
||||
callbacks: {
|
||||
onTextDelta: callbacks.onTextDelta,
|
||||
onToolActivity: callbacks.onToolActivity,
|
||||
onTurnDone: (result) => {
|
||||
finish();
|
||||
callbacks.onTurnDone(result);
|
||||
},
|
||||
onError: (message) => {
|
||||
finish();
|
||||
callbacks.onError(message);
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.turnHandle?.cancel();
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
}
|
||||
|
||||
private buildToolDefinitions(): AnthropicToolDefinition[] {
|
||||
this.toolNameMap.clear();
|
||||
return this.registry.listTools().map((spec) => {
|
||||
const apiName = this.toApiToolName(spec.name);
|
||||
this.toolNameMap.set(apiName, spec.name);
|
||||
return { name: apiName, description: spec.description, input_schema: spec.inputSchema };
|
||||
});
|
||||
}
|
||||
|
||||
/** Convert a canonical registry name to an Anthropic-legal tool name. */
|
||||
private toApiToolName(name: string): string {
|
||||
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
|
||||
// Disambiguate the rare case where two canonical names sanitize alike.
|
||||
if (this.toolNameMap.has(base)) {
|
||||
let i = 2;
|
||||
while (this.toolNameMap.has(`${base}_${i}`)) i++;
|
||||
base = `${base}_${i}`;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
private trimHistory(): void {
|
||||
if (this.messages.length <= MAX_HISTORY_MESSAGES) return;
|
||||
// Drop whole messages from the head. A leading orphan tool_result (whose
|
||||
// tool_use we trimmed) would be rejected by the API, so skip past any
|
||||
// tool_result-only user message left at the front.
|
||||
const overflow = this.messages.length - MAX_HISTORY_MESSAGES;
|
||||
this.messages.splice(0, overflow);
|
||||
while (this.messages.length && startsWithToolResult(this.messages[0]!)) {
|
||||
this.messages.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function startsWithToolResult(message: AnthropicMessage): boolean {
|
||||
return (
|
||||
Array.isArray(message.content) &&
|
||||
message.content.length > 0 &&
|
||||
message.content[0]!.type === "tool_result"
|
||||
);
|
||||
}
|
||||
|
||||
function describeContext(ctx: AssistantContext): string {
|
||||
const parts = [`Current time: ${ctx.localTime}.`];
|
||||
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
|
||||
if (ctx.foregroundApp) {
|
||||
const title = ctx.foregroundTitle ? ` ("${ctx.foregroundTitle}")` : "";
|
||||
parts.push(`The foreground app is ${ctx.foregroundApp}${title}.`);
|
||||
} else {
|
||||
parts.push("No app is in the foreground (the launcher is showing).");
|
||||
}
|
||||
if (ctx.headsetBattery !== null) {
|
||||
parts.push(`Glasses battery: ${ctx.headsetBattery}%.`);
|
||||
}
|
||||
return `Context: ${parts.join(" ")}`;
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import { readUpcomingEvents, type CalendarEvent } from "../native/calendar";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
|
||||
|
||||
/**
|
||||
* Registers the always-available system tools into the registry. Called once at
|
||||
* startup. These wrap shell state and the existing native bridges (calendar,
|
||||
* media, notifications); nothing here needs an app window to be open.
|
||||
*/
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
let registered = false;
|
||||
|
||||
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
|
||||
if (registered) return;
|
||||
registered = true;
|
||||
|
||||
// Keep a media listener warm so "what's playing" works without opening Music.
|
||||
void mediaControllerBridge.start();
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "glasses.get_state",
|
||||
description:
|
||||
"Get the current state of the glasses: display on/off, foreground app, headset battery, and the current local time.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
proactive: true,
|
||||
},
|
||||
() => ok(describeState()),
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "glasses.show_alert",
|
||||
description:
|
||||
"Show a short text popup on the glasses display. Use for a brief notice the user should see; keep it to a sentence or two.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { text: { type: "string", description: "The message to display." } },
|
||||
required: ["text"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
proactive: true,
|
||||
},
|
||||
(args) => {
|
||||
const text = String(args?.text ?? "").trim();
|
||||
if (!text) return err("show_alert requires non-empty text");
|
||||
shell.showAlert(text);
|
||||
return ok("Displayed.");
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "calendar.list_events",
|
||||
description:
|
||||
"List the user's upcoming calendar events. Returns events from now forward, ordered by start time.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
within_hours: {
|
||||
type: "number",
|
||||
description: "Only include events starting within this many hours (default 168 = one week).",
|
||||
},
|
||||
max_events: { type: "number", description: "Maximum number of events to return (default 10)." },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
(args) => {
|
||||
const withinHours = clampNumber(args?.within_hours, 1, 24 * 60, 168);
|
||||
const maxEvents = clampNumber(args?.max_events, 1, 50, 10);
|
||||
const events = readUpcomingEvents(maxEvents, withinHours * 60 * 60 * 1000);
|
||||
if (!events.length) return ok("No upcoming events in that window.");
|
||||
return ok(events.map(formatEvent).join("\n"));
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.now_playing",
|
||||
description: "What is currently playing (or paused) in the user's media app, if anything.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
proactive: true,
|
||||
},
|
||||
() => ok(describeNowPlaying()),
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.play_pause",
|
||||
description: "Toggle play/pause for the user's current media session.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canPlayPause) return err("No media session can be controlled right now.");
|
||||
await mediaControllerBridge.playPause();
|
||||
return ok("Toggled play/pause.");
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.next",
|
||||
description: "Skip to the next track in the user's current media session.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
|
||||
await mediaControllerBridge.skipNext();
|
||||
return ok("Skipped to the next track.");
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "notifications.list",
|
||||
description: "List the user's current Android notifications mirrored to the glasses.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { max: { type: "number", description: "Maximum notifications to return (default 10)." } },
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
(args) => {
|
||||
const max = clampNumber(args?.max, 1, 50, 10);
|
||||
const notifications = readActiveNotifications(max);
|
||||
if (!notifications.length) return ok("No current notifications.");
|
||||
return ok(notifications.map(formatNotification).join("\n"));
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "notifications.dismiss",
|
||||
description:
|
||||
"Dismiss a notification by its key (as returned by notifications.list). Returns whether it was dismissed.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { key: { type: "string", description: "The notification key from notifications.list." } },
|
||||
required: ["key"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
(args) => {
|
||||
const key = String(args?.key ?? "").trim();
|
||||
if (!key) return err("notifications.dismiss requires a key");
|
||||
return dismissNotification(key) ? ok("Dismissed.") : err("No notification with that key.");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function describeState(): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`Display: ${shell.isScreenOn() ? "on" : "off"}.`);
|
||||
const fg = shell.getForegroundApp();
|
||||
parts.push(fg ? `Foreground app: ${fg.appId} ("${fg.title}").` : "Foreground: launcher (no app open).");
|
||||
const battery = shell.getBatteryLevels();
|
||||
if (battery.headset !== null) {
|
||||
const charging = battery.headsetCharging ? ", charging" : "";
|
||||
parts.push(`Headset battery: ${battery.headset}%${charging}.`);
|
||||
}
|
||||
parts.push(`Local time: ${formatDateTime(new Date())}.`);
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
function describeNowPlaying(): string {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable.";
|
||||
if (state.playbackState === "playing" || state.playbackState === "paused") {
|
||||
const verb = state.playbackState === "playing" ? "Playing" : "Paused";
|
||||
const title = state.title || "(unknown title)";
|
||||
const artist = state.artist ? ` by ${state.artist}` : "";
|
||||
const app = state.appName ? ` in ${state.appName}` : "";
|
||||
return `${verb}: ${title}${artist}${app}.`;
|
||||
}
|
||||
return "Nothing is currently playing.";
|
||||
}
|
||||
|
||||
function formatEvent(event: CalendarEvent): string {
|
||||
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
|
||||
const location = event.location ? ` @ ${event.location}` : "";
|
||||
return `- ${event.title || "(untitled)"} — ${when}${location}`;
|
||||
}
|
||||
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
|
||||
const title = notification.title || "(no title)";
|
||||
const body = notification.text ? `: ${notification.text}` : "";
|
||||
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
|
||||
}
|
||||
|
||||
function formatDateTime(date: Date): string {
|
||||
return `${formatDate(date)}, ${formatTime(date)}`;
|
||||
}
|
||||
|
||||
function formatDate(date: Date): string {
|
||||
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
|
||||
}
|
||||
|
||||
function formatTime(date: Date): string {
|
||||
let hours = date.getHours();
|
||||
const minutes = date.getMinutes().toString().padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
}
|
||||
|
||||
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
|
||||
const num = typeof value === "number" ? value : Number(value);
|
||||
if (!Number.isFinite(num)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.round(num)));
|
||||
}
|
||||
|
||||
function ok(content: string): ToolResult {
|
||||
return { ok: true, content };
|
||||
}
|
||||
|
||||
function err(error: string): ToolResult {
|
||||
return { ok: false, error };
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* The assistant's tool registry: the single place that knows which tools exist,
|
||||
* whether each is currently callable, and how to invoke it. Both assistant
|
||||
* backends (direct Anthropic loop, external agent bridge) list and call tools
|
||||
* through here, so gating and timeouts are enforced in one spot rather than
|
||||
* trusted to a model or a remote agent.
|
||||
*
|
||||
* Availability tiers (see notes/voice-assistant-design.md):
|
||||
* - "always" system/shell tools; no app involved.
|
||||
* - "installed" app tool live whenever the app is installed (no window
|
||||
* needed) -- e.g. "start a 5 minute timer".
|
||||
* - "open" app tool live while the app has a window open, foreground
|
||||
* or backgrounded -- e.g. "next track".
|
||||
* - "foreground" app tool live only while the app owns the foreground window
|
||||
* -- e.g. "scroll down".
|
||||
*
|
||||
* Phase 1 only registers "always" tools; the tier machinery is here so the
|
||||
* app-tool phases slot in without reshaping the registry.
|
||||
*/
|
||||
|
||||
export type ToolAvailability = "always" | "installed" | "open" | "foreground";
|
||||
|
||||
export type ToolSpec = {
|
||||
/** Namespaced, e.g. "glasses.get_state" or "app.terminal.send_input". */
|
||||
name: string;
|
||||
description: string;
|
||||
/** JSON Schema for the tool's arguments. */
|
||||
inputSchema: object;
|
||||
availability: ToolAvailability;
|
||||
/** May be invoked outside an active voice turn (external mode proactive calls). */
|
||||
proactive?: boolean;
|
||||
/** Round-trip cap; the registry rejects with a tool error past it. Default 10s. */
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* A tool's outcome. Flat (not a discriminated union) on purpose: this project
|
||||
* builds with strictNullChecks off, so `ok: true|false` would widen to boolean
|
||||
* and defeat union narrowing. Read `content` when ok, `error` when not.
|
||||
*/
|
||||
export type ToolResult = {
|
||||
ok: boolean;
|
||||
/** Text payload back to the model (when ok). */
|
||||
content?: string;
|
||||
/** Error message back to the model (when not ok). */
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type ToolHandler = (args: any) => Promise<ToolResult> | ToolResult;
|
||||
|
||||
export type ToolRegistration = {
|
||||
spec: ToolSpec;
|
||||
handler: ToolHandler;
|
||||
/**
|
||||
* Whether the tool is currently live. Absent means always live (the "always"
|
||||
* tier). App-tool phases supply a predicate keyed on window/foreground state.
|
||||
*/
|
||||
isAvailable?: () => boolean;
|
||||
};
|
||||
|
||||
export type ListToolsOptions = {
|
||||
/** Only tools flagged proactive (for calls outside an active turn). */
|
||||
proactiveOnly?: boolean;
|
||||
};
|
||||
|
||||
export type CallToolOptions = {
|
||||
/** The call is happening outside an active voice turn; enforce proactive gating. */
|
||||
proactive?: boolean;
|
||||
};
|
||||
|
||||
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
|
||||
|
||||
export class ToolRegistry {
|
||||
private readonly registrations = new Map<string, ToolRegistration>();
|
||||
private readonly changeListeners = new Set<() => void>();
|
||||
|
||||
/** Register (or replace) a tool. Names are unique across all tiers. */
|
||||
register(registration: ToolRegistration): void {
|
||||
const { name } = registration.spec;
|
||||
this.registrations.set(name, registration);
|
||||
this.fireToolsChanged();
|
||||
}
|
||||
|
||||
/** Convenience for the always-live system tools. */
|
||||
registerSystemTool(
|
||||
spec: Omit<ToolSpec, "availability">,
|
||||
handler: ToolHandler,
|
||||
): void {
|
||||
this.register({ spec: { ...spec, availability: "always" }, handler });
|
||||
}
|
||||
|
||||
unregister(name: string): void {
|
||||
if (this.registrations.delete(name)) {
|
||||
this.fireToolsChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/** Specs for the tools callable right now, given the availability tiers. */
|
||||
listTools(options: ListToolsOptions = {}): ToolSpec[] {
|
||||
const specs: ToolSpec[] = [];
|
||||
for (const registration of this.registrations.values()) {
|
||||
if (!this.isLive(registration)) continue;
|
||||
if (options.proactiveOnly && !registration.spec.proactive) continue;
|
||||
specs.push(registration.spec);
|
||||
}
|
||||
return specs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke a tool by name. Never throws: a missing/unavailable tool, a
|
||||
* proactive-gating rejection, a handler exception, or a timeout all come back
|
||||
* as `{ ok: false }` so the agent loop can surface them as tool errors.
|
||||
*/
|
||||
async callTool(name: string, args: unknown, options: CallToolOptions = {}): Promise<ToolResult> {
|
||||
const registration = this.registrations.get(name);
|
||||
if (!registration) {
|
||||
return { ok: false, error: `Unknown tool: ${name}` };
|
||||
}
|
||||
if (!this.isLive(registration)) {
|
||||
return { ok: false, error: `Tool ${name} is not currently available` };
|
||||
}
|
||||
if (options.proactive && !registration.spec.proactive) {
|
||||
return { ok: false, error: `Tool ${name} cannot be called outside a conversation` };
|
||||
}
|
||||
const timeoutMs = registration.spec.timeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
|
||||
try {
|
||||
return await this.withTimeout(registration.handler(args), timeoutMs, name);
|
||||
} catch (error) {
|
||||
return { ok: false, error: `Tool ${name} failed: ${describeError(error)}` };
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to add/remove/availability changes (drives MCP list_changed). */
|
||||
onToolsChanged(listener: () => void): () => void {
|
||||
this.changeListeners.add(listener);
|
||||
return () => this.changeListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** App-tool phases call this after a foreground/window change re-gates tools. */
|
||||
fireToolsChanged(): void {
|
||||
for (const listener of this.changeListeners) {
|
||||
try {
|
||||
listener();
|
||||
} catch (error) {
|
||||
console.warn("onToolsChanged listener failed", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isLive(registration: ToolRegistration): boolean {
|
||||
return registration.isAvailable ? registration.isAvailable() : true;
|
||||
}
|
||||
|
||||
private withTimeout(
|
||||
result: Promise<ToolResult> | ToolResult,
|
||||
timeoutMs: number,
|
||||
name: string,
|
||||
): Promise<ToolResult> {
|
||||
if (!(result instanceof Promise)) return Promise.resolve(result);
|
||||
return new Promise<ToolResult>((resolve) => {
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve({ ok: false, error: `Tool ${name} timed out after ${timeoutMs}ms` });
|
||||
}, timeoutMs);
|
||||
result.then(
|
||||
(value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, error: `Tool ${name} failed: ${describeError(error)}` });
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function describeError(error: unknown): string {
|
||||
return String((error as Error)?.message ?? error);
|
||||
}
|
||||
|
||||
/** The process-wide registry; system tools register into it at startup. */
|
||||
export const toolRegistry = new ToolRegistry();
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer)
|
||||
* talks to the same session surface whether the turn runs on the phone
|
||||
* (direct Anthropic loop) or against the user's own agent (external bridge,
|
||||
* later phases).
|
||||
*/
|
||||
|
||||
/** Situational grounding injected per turn (system prompt in direct mode). */
|
||||
export type AssistantContext = {
|
||||
/** Foreground app id, or null when the sidebar/launcher is all that's up. */
|
||||
foregroundApp: string | null;
|
||||
foregroundTitle: string | null;
|
||||
screenOn: boolean;
|
||||
/** Human-readable local time, e.g. "Thu Jul 24, 3:15 PM". */
|
||||
localTime: string;
|
||||
/** Headset battery percent (0..100), or null if unknown. */
|
||||
headsetBattery: number | null;
|
||||
};
|
||||
|
||||
export type AssistantTurnCallbacks = {
|
||||
/** Streamed reply text (delta plus the full text so far). */
|
||||
onTextDelta: (delta: string, textSoFar: string) => void;
|
||||
/** A tool is being invoked; label is display-ready, e.g. "calendar.list_events". */
|
||||
onToolActivity: (label: string) => void;
|
||||
onTurnDone: (result: { stopReason: string | null }) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
/** A handle to an in-flight turn. */
|
||||
export type AssistantTurnHandle = {
|
||||
cancel(): void;
|
||||
};
|
||||
Reference in new issue
Block a user