Implement a basic voice assistant

This commit is contained in:
James Babcock committed 2026-07-25 00:44:04 -07:00
1 parent d2c987d48a
commit 0aeb24ed00
14 files changed
+1923 -80

No files matched your search

+141
View File
@@ -0,0 +1,141 @@
import {
streamAnthropicMessage,
type AnthropicContentBlock,
type AnthropicMessage,
type AnthropicStreamHandle,
type AnthropicToolDefinition,
} from "../native/anthropic";
import type { ToolRegistry } from "./tool-registry";
import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* The on-phone agent loop: stream a reply, and while the model asks for tools,
* execute them through the registry and feed the results back until it stops.
* History lives in the AssistantSession; this backend appends the assistant
* and tool-result messages to the array it is handed.
*/
/** Safety caps so a tool-thrashing model can't loop or run forever. */
const MAX_ITERATIONS = 8;
const MAX_TURN_MS = 2 * 60 * 1000;
export type DirectTurnOptions = {
apiKey: string;
model: string;
effort?: "low" | "medium" | "high";
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: AnthropicMessage[];
tools: AnthropicToolDefinition[];
registry: ToolRegistry;
/** Map an API tool name (as the model calls it) back to its registry name. */
resolveToolName: (apiName: string) => string;
callbacks: AssistantTurnCallbacks;
};
export class DirectAnthropicBackend {
runTurn(options: DirectTurnOptions): AssistantTurnHandle {
const startMs = Date.now();
let cancelled = false;
let streamHandle: AnthropicStreamHandle | null = null;
const finishError = (message: string) => {
if (cancelled) return;
cancelled = true;
options.callbacks.onError(message);
};
const runIteration = (iteration: number): void => {
if (cancelled) return;
if (iteration >= MAX_ITERATIONS) {
finishError("Assistant stopped: too many tool steps");
return;
}
if (Date.now() - startMs > MAX_TURN_MS) {
finishError("Assistant stopped: turn took too long");
return;
}
streamHandle = streamAnthropicMessage({
apiKey: options.apiKey,
model: options.model,
system: options.system,
effort: options.effort,
messages: options.messages,
tools: options.tools,
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar);
},
onDone: (result) => {
streamHandle = null;
if (cancelled) return;
if (result.stopReason === "refusal") {
finishError("The assistant declined this request");
return;
}
// Record what the model produced this iteration.
const assistantContent: AnthropicContentBlock[] = result.content.length
? result.content
: [{ type: "text", text: result.text }];
options.messages.push({ role: "assistant", content: assistantContent });
const toolUses = assistantContent.filter(
(block): block is Extract<AnthropicContentBlock, { type: "tool_use" }> =>
block.type === "tool_use",
);
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
options.callbacks.onTurnDone({ stopReason: result.stopReason });
return;
}
void this.runToolCalls(toolUses, options)
.then((toolResults) => {
// Always record the results, even if cancelled mid-flight: an
// assistant tool_use with no matching tool_result would make the
// next turn's request invalid. Only the loop continuation is gated.
options.messages.push({ role: "user", content: toolResults });
if (cancelled) return;
runIteration(iteration + 1);
})
.catch((error) => {
finishError(`Tool execution failed: ${String((error as Error)?.message ?? error)}`);
});
},
onError: (message) => {
streamHandle = null;
finishError(message);
},
});
};
runIteration(0);
return {
cancel: () => {
cancelled = true;
streamHandle?.cancel();
streamHandle = null;
},
};
}
private async runToolCalls(
toolUses: Array<Extract<AnthropicContentBlock, { type: "tool_use" }>>,
options: DirectTurnOptions,
): Promise<AnthropicContentBlock[]> {
const results: AnthropicContentBlock[] = [];
for (const toolUse of toolUses) {
const canonicalName = options.resolveToolName(toolUse.name);
options.callbacks.onToolActivity(canonicalName);
const result = await options.registry.callTool(canonicalName, toolUse.input);
const content = result.ok ? result.content ?? "" : result.error ?? "Tool error";
results.push({
type: "tool_result",
tool_use_id: toolUse.id,
content,
is_error: !result.ok,
});
}
return results;
}
}
+156
View File
@@ -0,0 +1,156 @@
import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic";
import { DirectAnthropicBackend } from "./direct-backend";
import { toolRegistry, type ToolRegistry } from "./tool-registry";
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* One assistant conversation. Owns the message history and turn state and
* drives a backend (direct Anthropic loop for phase 1). The UI talks only to
* this surface, so swapping in the external-agent backend later doesn't touch
* AssistantLayer.
*
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
* one for a new wakeword and reuses the current one for a follow-up.
*/
/** A new PTT after this much idle starts a fresh conversation. */
const SESSION_IDLE_MS = 10 * 60 * 1000;
/** Trim history from the head once it grows past this many messages. */
const MAX_HISTORY_MESSAGES = 40;
const SYSTEM_PROMPT_BASE = [
"You are the voice assistant built into a pair of Even Realities G2 smart glasses.",
"Your reply is shown as text on a 576x288 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list.",
"Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it.",
"If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
].join(" ");
export class AssistantSession {
private readonly backend = new DirectAnthropicBackend();
private readonly messages: AnthropicMessage[] = [];
private turnHandle: AssistantTurnHandle | null = null;
private lastActivityMs = Date.now();
// API-safe tool name -> canonical registry name. The Anthropic API only
// allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses.
// get_state") are sanitized for the request and mapped back on tool calls.
private readonly toolNameMap = new Map<string, string>();
constructor(
private readonly apiKey: string,
private readonly registry: ToolRegistry = toolRegistry,
) {}
/** Whether this session has been idle long enough to retire. */
isExpired(nowMs = Date.now()): boolean {
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
}
isTurnActive(): boolean {
return this.turnHandle !== null;
}
/** Begin a turn from a spoken utterance. Only one turn runs at a time. */
sendUtterance(text: string, ctx: AssistantContext, callbacks: AssistantTurnCallbacks): void {
if (this.turnHandle) {
callbacks.onError("The assistant is still working on the previous request");
return;
}
this.lastActivityMs = Date.now();
this.messages.push({ role: "user", content: text });
this.trimHistory();
const tools = this.buildToolDefinitions();
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
const finish = () => {
this.turnHandle = null;
this.lastActivityMs = Date.now();
};
this.turnHandle = this.backend.runTurn({
apiKey: this.apiKey,
model: DEFAULT_LLM_MODEL,
effort: "low",
system,
messages: this.messages,
tools,
registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: {
onTextDelta: callbacks.onTextDelta,
onToolActivity: callbacks.onToolActivity,
onTurnDone: (result) => {
finish();
callbacks.onTurnDone(result);
},
onError: (message) => {
finish();
callbacks.onError(message);
},
},
});
}
cancel(): void {
this.turnHandle?.cancel();
this.turnHandle = null;
this.lastActivityMs = Date.now();
}
private buildToolDefinitions(): AnthropicToolDefinition[] {
this.toolNameMap.clear();
return this.registry.listTools().map((spec) => {
const apiName = this.toApiToolName(spec.name);
this.toolNameMap.set(apiName, spec.name);
return { name: apiName, description: spec.description, input_schema: spec.inputSchema };
});
}
/** Convert a canonical registry name to an Anthropic-legal tool name. */
private toApiToolName(name: string): string {
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
// Disambiguate the rare case where two canonical names sanitize alike.
if (this.toolNameMap.has(base)) {
let i = 2;
while (this.toolNameMap.has(`${base}_${i}`)) i++;
base = `${base}_${i}`;
}
return base;
}
private trimHistory(): void {
if (this.messages.length <= MAX_HISTORY_MESSAGES) return;
// Drop whole messages from the head. A leading orphan tool_result (whose
// tool_use we trimmed) would be rejected by the API, so skip past any
// tool_result-only user message left at the front.
const overflow = this.messages.length - MAX_HISTORY_MESSAGES;
this.messages.splice(0, overflow);
while (this.messages.length && startsWithToolResult(this.messages[0]!)) {
this.messages.shift();
}
}
}
function startsWithToolResult(message: AnthropicMessage): boolean {
return (
Array.isArray(message.content) &&
message.content.length > 0 &&
message.content[0]!.type === "tool_result"
);
}
function describeContext(ctx: AssistantContext): string {
const parts = [`Current time: ${ctx.localTime}.`];
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
if (ctx.foregroundApp) {
const title = ctx.foregroundTitle ? ` ("${ctx.foregroundTitle}")` : "";
parts.push(`The foreground app is ${ctx.foregroundApp}${title}.`);
} else {
parts.push("No app is in the foreground (the launcher is showing).");
}
if (ctx.headsetBattery !== null) {
parts.push(`Glasses battery: ${ctx.headsetBattery}%.`);
}
return `Context: ${parts.join(" ")}`;
}
+227
View File
@@ -0,0 +1,227 @@
import { readUpcomingEvents, type CalendarEvent } from "../native/calendar";
import { mediaControllerBridge } from "../native/media-controller";
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
import { shell } from "../ui/shell/shell";
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
/**
* Registers the always-available system tools into the registry. Called once at
* startup. These wrap shell state and the existing native bridges (calendar,
* media, notifications); nothing here needs an app window to be open.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
let registered = false;
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
if (registered) return;
registered = true;
// Keep a media listener warm so "what's playing" works without opening Music.
void mediaControllerBridge.start();
registry.registerSystemTool(
{
name: "glasses.get_state",
description:
"Get the current state of the glasses: display on/off, foreground app, headset battery, and the current local time.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
proactive: true,
},
() => ok(describeState()),
);
registry.registerSystemTool(
{
name: "glasses.show_alert",
description:
"Show a short text popup on the glasses display. Use for a brief notice the user should see; keep it to a sentence or two.",
inputSchema: {
type: "object",
properties: { text: { type: "string", description: "The message to display." } },
required: ["text"],
additionalProperties: false,
},
proactive: true,
},
(args) => {
const text = String(args?.text ?? "").trim();
if (!text) return err("show_alert requires non-empty text");
shell.showAlert(text);
return ok("Displayed.");
},
);
registry.registerSystemTool(
{
name: "calendar.list_events",
description:
"List the user's upcoming calendar events. Returns events from now forward, ordered by start time.",
inputSchema: {
type: "object",
properties: {
within_hours: {
type: "number",
description: "Only include events starting within this many hours (default 168 = one week).",
},
max_events: { type: "number", description: "Maximum number of events to return (default 10)." },
},
additionalProperties: false,
},
},
(args) => {
const withinHours = clampNumber(args?.within_hours, 1, 24 * 60, 168);
const maxEvents = clampNumber(args?.max_events, 1, 50, 10);
const events = readUpcomingEvents(maxEvents, withinHours * 60 * 60 * 1000);
if (!events.length) return ok("No upcoming events in that window.");
return ok(events.map(formatEvent).join("\n"));
},
);
registry.registerSystemTool(
{
name: "media.now_playing",
description: "What is currently playing (or paused) in the user's media app, if anything.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
proactive: true,
},
() => ok(describeNowPlaying()),
);
registry.registerSystemTool(
{
name: "media.play_pause",
description: "Toggle play/pause for the user's current media session.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
},
async () => {
const state = mediaControllerBridge.snapshot();
if (!state.canPlayPause) return err("No media session can be controlled right now.");
await mediaControllerBridge.playPause();
return ok("Toggled play/pause.");
},
);
registry.registerSystemTool(
{
name: "media.next",
description: "Skip to the next track in the user's current media session.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
},
async () => {
const state = mediaControllerBridge.snapshot();
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
await mediaControllerBridge.skipNext();
return ok("Skipped to the next track.");
},
);
registry.registerSystemTool(
{
name: "notifications.list",
description: "List the user's current Android notifications mirrored to the glasses.",
inputSchema: {
type: "object",
properties: { max: { type: "number", description: "Maximum notifications to return (default 10)." } },
additionalProperties: false,
},
},
(args) => {
const max = clampNumber(args?.max, 1, 50, 10);
const notifications = readActiveNotifications(max);
if (!notifications.length) return ok("No current notifications.");
return ok(notifications.map(formatNotification).join("\n"));
},
);
registry.registerSystemTool(
{
name: "notifications.dismiss",
description:
"Dismiss a notification by its key (as returned by notifications.list). Returns whether it was dismissed.",
inputSchema: {
type: "object",
properties: { key: { type: "string", description: "The notification key from notifications.list." } },
required: ["key"],
additionalProperties: false,
},
},
(args) => {
const key = String(args?.key ?? "").trim();
if (!key) return err("notifications.dismiss requires a key");
return dismissNotification(key) ? ok("Dismissed.") : err("No notification with that key.");
},
);
}
function describeState(): string {
const parts: string[] = [];
parts.push(`Display: ${shell.isScreenOn() ? "on" : "off"}.`);
const fg = shell.getForegroundApp();
parts.push(fg ? `Foreground app: ${fg.appId} ("${fg.title}").` : "Foreground: launcher (no app open).");
const battery = shell.getBatteryLevels();
if (battery.headset !== null) {
const charging = battery.headsetCharging ? ", charging" : "";
parts.push(`Headset battery: ${battery.headset}%${charging}.`);
}
parts.push(`Local time: ${formatDateTime(new Date())}.`);
return parts.join(" ");
}
function describeNowPlaying(): string {
const state = mediaControllerBridge.snapshot();
if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable.";
if (state.playbackState === "playing" || state.playbackState === "paused") {
const verb = state.playbackState === "playing" ? "Playing" : "Paused";
const title = state.title || "(unknown title)";
const artist = state.artist ? ` by ${state.artist}` : "";
const app = state.appName ? ` in ${state.appName}` : "";
return `${verb}: ${title}${artist}${app}.`;
}
return "Nothing is currently playing.";
}
function formatEvent(event: CalendarEvent): string {
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
const location = event.location ? ` @ ${event.location}` : "";
return `- ${event.title || "(untitled)"} — ${when}${location}`;
}
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
const title = notification.title || "(no title)";
const body = notification.text ? `: ${notification.text}` : "";
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
}
function formatDateTime(date: Date): string {
return `${formatDate(date)}, ${formatTime(date)}`;
}
function formatDate(date: Date): string {
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
}
function formatTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
return `${hours}:${minutes} ${meridiem}`;
}
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
const num = typeof value === "number" ? value : Number(value);
if (!Number.isFinite(num)) return fallback;
return Math.min(max, Math.max(min, Math.round(num)));
}
function ok(content: string): ToolResult {
return { ok: true, content };
}
function err(error: string): ToolResult {
return { ok: false, error };
}
+190
View File
@@ -0,0 +1,190 @@
/**
* The assistant's tool registry: the single place that knows which tools exist,
* whether each is currently callable, and how to invoke it. Both assistant
* backends (direct Anthropic loop, external agent bridge) list and call tools
* through here, so gating and timeouts are enforced in one spot rather than
* trusted to a model or a remote agent.
*
* Availability tiers (see notes/voice-assistant-design.md):
* - "always" system/shell tools; no app involved.
* - "installed" app tool live whenever the app is installed (no window
* needed) -- e.g. "start a 5 minute timer".
* - "open" app tool live while the app has a window open, foreground
* or backgrounded -- e.g. "next track".
* - "foreground" app tool live only while the app owns the foreground window
* -- e.g. "scroll down".
*
* Phase 1 only registers "always" tools; the tier machinery is here so the
* app-tool phases slot in without reshaping the registry.
*/
export type ToolAvailability = "always" | "installed" | "open" | "foreground";
export type ToolSpec = {
/** Namespaced, e.g. "glasses.get_state" or "app.terminal.send_input". */
name: string;
description: string;
/** JSON Schema for the tool's arguments. */
inputSchema: object;
availability: ToolAvailability;
/** May be invoked outside an active voice turn (external mode proactive calls). */
proactive?: boolean;
/** Round-trip cap; the registry rejects with a tool error past it. Default 10s. */
timeoutMs?: number;
};
/**
* A tool's outcome. Flat (not a discriminated union) on purpose: this project
* builds with strictNullChecks off, so `ok: true|false` would widen to boolean
* and defeat union narrowing. Read `content` when ok, `error` when not.
*/
export type ToolResult = {
ok: boolean;
/** Text payload back to the model (when ok). */
content?: string;
/** Error message back to the model (when not ok). */
error?: string;
};
export type ToolHandler = (args: any) => Promise<ToolResult> | ToolResult;
export type ToolRegistration = {
spec: ToolSpec;
handler: ToolHandler;
/**
* Whether the tool is currently live. Absent means always live (the "always"
* tier). App-tool phases supply a predicate keyed on window/foreground state.
*/
isAvailable?: () => boolean;
};
export type ListToolsOptions = {
/** Only tools flagged proactive (for calls outside an active turn). */
proactiveOnly?: boolean;
};
export type CallToolOptions = {
/** The call is happening outside an active voice turn; enforce proactive gating. */
proactive?: boolean;
};
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
export class ToolRegistry {
private readonly registrations = new Map<string, ToolRegistration>();
private readonly changeListeners = new Set<() => void>();
/** Register (or replace) a tool. Names are unique across all tiers. */
register(registration: ToolRegistration): void {
const { name } = registration.spec;
this.registrations.set(name, registration);
this.fireToolsChanged();
}
/** Convenience for the always-live system tools. */
registerSystemTool(
spec: Omit<ToolSpec, "availability">,
handler: ToolHandler,
): void {
this.register({ spec: { ...spec, availability: "always" }, handler });
}
unregister(name: string): void {
if (this.registrations.delete(name)) {
this.fireToolsChanged();
}
}
/** Specs for the tools callable right now, given the availability tiers. */
listTools(options: ListToolsOptions = {}): ToolSpec[] {
const specs: ToolSpec[] = [];
for (const registration of this.registrations.values()) {
if (!this.isLive(registration)) continue;
if (options.proactiveOnly && !registration.spec.proactive) continue;
specs.push(registration.spec);
}
return specs;
}
/**
* Invoke a tool by name. Never throws: a missing/unavailable tool, a
* proactive-gating rejection, a handler exception, or a timeout all come back
* as `{ ok: false }` so the agent loop can surface them as tool errors.
*/
async callTool(name: string, args: unknown, options: CallToolOptions = {}): Promise<ToolResult> {
const registration = this.registrations.get(name);
if (!registration) {
return { ok: false, error: `Unknown tool: ${name}` };
}
if (!this.isLive(registration)) {
return { ok: false, error: `Tool ${name} is not currently available` };
}
if (options.proactive && !registration.spec.proactive) {
return { ok: false, error: `Tool ${name} cannot be called outside a conversation` };
}
const timeoutMs = registration.spec.timeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
try {
return await this.withTimeout(registration.handler(args), timeoutMs, name);
} catch (error) {
return { ok: false, error: `Tool ${name} failed: ${describeError(error)}` };
}
}
/** Subscribe to add/remove/availability changes (drives MCP list_changed). */
onToolsChanged(listener: () => void): () => void {
this.changeListeners.add(listener);
return () => this.changeListeners.delete(listener);
}
/** App-tool phases call this after a foreground/window change re-gates tools. */
fireToolsChanged(): void {
for (const listener of this.changeListeners) {
try {
listener();
} catch (error) {
console.warn("onToolsChanged listener failed", error);
}
}
}
private isLive(registration: ToolRegistration): boolean {
return registration.isAvailable ? registration.isAvailable() : true;
}
private withTimeout(
result: Promise<ToolResult> | ToolResult,
timeoutMs: number,
name: string,
): Promise<ToolResult> {
if (!(result instanceof Promise)) return Promise.resolve(result);
return new Promise<ToolResult>((resolve) => {
let settled = false;
const timer = setTimeout(() => {
if (settled) return;
settled = true;
resolve({ ok: false, error: `Tool ${name} timed out after ${timeoutMs}ms` });
}, timeoutMs);
result.then(
(value) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(value);
},
(error) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve({ ok: false, error: `Tool ${name} failed: ${describeError(error)}` });
},
);
});
}
}
function describeError(error: unknown): string {
return String((error as Error)?.message ?? error);
}
/** The process-wide registry; system tools register into it at startup. */
export const toolRegistry = new ToolRegistry();
+32
View File
@@ -0,0 +1,32 @@
/**
* Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer)
* talks to the same session surface whether the turn runs on the phone
* (direct Anthropic loop) or against the user's own agent (external bridge,
* later phases).
*/
/** Situational grounding injected per turn (system prompt in direct mode). */
export type AssistantContext = {
/** Foreground app id, or null when the sidebar/launcher is all that's up. */
foregroundApp: string | null;
foregroundTitle: string | null;
screenOn: boolean;
/** Human-readable local time, e.g. "Thu Jul 24, 3:15 PM". */
localTime: string;
/** Headset battery percent (0..100), or null if unknown. */
headsetBattery: number | null;
};
export type AssistantTurnCallbacks = {
/** Streamed reply text (delta plus the full text so far). */
onTextDelta: (delta: string, textSoFar: string) => void;
/** A tool is being invoked; label is display-ready, e.g. "calendar.list_events". */
onToolActivity: (label: string) => void;
onTurnDone: (result: { stopReason: string | null }) => void;
onError: (message: string) => void;
};
/** A handle to an in-flight turn. */
export type AssistantTurnHandle = {
cancel(): void;
};