Implement a basic voice assistant

This commit is contained in:
James Babcock committed 2026-07-25 00:44:04 -07:00
1 parent d2c987d48a
commit 0aeb24ed00
14 files changed
+1923 -80

No files matched your search

+141
View File
@@ -0,0 +1,141 @@
import {
streamAnthropicMessage,
type AnthropicContentBlock,
type AnthropicMessage,
type AnthropicStreamHandle,
type AnthropicToolDefinition,
} from "../native/anthropic";
import type { ToolRegistry } from "./tool-registry";
import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* The on-phone agent loop: stream a reply, and while the model asks for tools,
* execute them through the registry and feed the results back until it stops.
* History lives in the AssistantSession; this backend appends the assistant
* and tool-result messages to the array it is handed.
*/
/** Safety caps so a tool-thrashing model can't loop or run forever. */
const MAX_ITERATIONS = 8;
const MAX_TURN_MS = 2 * 60 * 1000;
export type DirectTurnOptions = {
apiKey: string;
model: string;
effort?: "low" | "medium" | "high";
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: AnthropicMessage[];
tools: AnthropicToolDefinition[];
registry: ToolRegistry;
/** Map an API tool name (as the model calls it) back to its registry name. */
resolveToolName: (apiName: string) => string;
callbacks: AssistantTurnCallbacks;
};
export class DirectAnthropicBackend {
runTurn(options: DirectTurnOptions): AssistantTurnHandle {
const startMs = Date.now();
let cancelled = false;
let streamHandle: AnthropicStreamHandle | null = null;
const finishError = (message: string) => {
if (cancelled) return;
cancelled = true;
options.callbacks.onError(message);
};
const runIteration = (iteration: number): void => {
if (cancelled) return;
if (iteration >= MAX_ITERATIONS) {
finishError("Assistant stopped: too many tool steps");
return;
}
if (Date.now() - startMs > MAX_TURN_MS) {
finishError("Assistant stopped: turn took too long");
return;
}
streamHandle = streamAnthropicMessage({
apiKey: options.apiKey,
model: options.model,
system: options.system,
effort: options.effort,
messages: options.messages,
tools: options.tools,
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar);
},
onDone: (result) => {
streamHandle = null;
if (cancelled) return;
if (result.stopReason === "refusal") {
finishError("The assistant declined this request");
return;
}
// Record what the model produced this iteration.
const assistantContent: AnthropicContentBlock[] = result.content.length
? result.content
: [{ type: "text", text: result.text }];
options.messages.push({ role: "assistant", content: assistantContent });
const toolUses = assistantContent.filter(
(block): block is Extract<AnthropicContentBlock, { type: "tool_use" }> =>
block.type === "tool_use",
);
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
options.callbacks.onTurnDone({ stopReason: result.stopReason });
return;
}
void this.runToolCalls(toolUses, options)
.then((toolResults) => {
// Always record the results, even if cancelled mid-flight: an
// assistant tool_use with no matching tool_result would make the
// next turn's request invalid. Only the loop continuation is gated.
options.messages.push({ role: "user", content: toolResults });
if (cancelled) return;
runIteration(iteration + 1);
})
.catch((error) => {
finishError(`Tool execution failed: ${String((error as Error)?.message ?? error)}`);
});
},
onError: (message) => {
streamHandle = null;
finishError(message);
},
});
};
runIteration(0);
return {
cancel: () => {
cancelled = true;
streamHandle?.cancel();
streamHandle = null;
},
};
}
private async runToolCalls(
toolUses: Array<Extract<AnthropicContentBlock, { type: "tool_use" }>>,
options: DirectTurnOptions,
): Promise<AnthropicContentBlock[]> {
const results: AnthropicContentBlock[] = [];
for (const toolUse of toolUses) {
const canonicalName = options.resolveToolName(toolUse.name);
options.callbacks.onToolActivity(canonicalName);
const result = await options.registry.callTool(canonicalName, toolUse.input);
const content = result.ok ? result.content ?? "" : result.error ?? "Tool error";
results.push({
type: "tool_result",
tool_use_id: toolUse.id,
content,
is_error: !result.ok,
});
}
return results;
}
}
+156
View File
@@ -0,0 +1,156 @@
import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic";
import { DirectAnthropicBackend } from "./direct-backend";
import { toolRegistry, type ToolRegistry } from "./tool-registry";
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* One assistant conversation. Owns the message history and turn state and
* drives a backend (direct Anthropic loop for phase 1). The UI talks only to
* this surface, so swapping in the external-agent backend later doesn't touch
* AssistantLayer.
*
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
* one for a new wakeword and reuses the current one for a follow-up.
*/
/** A new PTT after this much idle starts a fresh conversation. */
const SESSION_IDLE_MS = 10 * 60 * 1000;
/** Trim history from the head once it grows past this many messages. */
const MAX_HISTORY_MESSAGES = 40;
const SYSTEM_PROMPT_BASE = [
"You are the voice assistant built into a pair of Even Realities G2 smart glasses.",
"Your reply is shown as text on a 576x288 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list.",
"Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it.",
"If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
].join(" ");
export class AssistantSession {
private readonly backend = new DirectAnthropicBackend();
private readonly messages: AnthropicMessage[] = [];
private turnHandle: AssistantTurnHandle | null = null;
private lastActivityMs = Date.now();
// API-safe tool name -> canonical registry name. The Anthropic API only
// allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses.
// get_state") are sanitized for the request and mapped back on tool calls.
private readonly toolNameMap = new Map<string, string>();
constructor(
private readonly apiKey: string,
private readonly registry: ToolRegistry = toolRegistry,
) {}
/** Whether this session has been idle long enough to retire. */
isExpired(nowMs = Date.now()): boolean {
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
}
isTurnActive(): boolean {
return this.turnHandle !== null;
}
/** Begin a turn from a spoken utterance. Only one turn runs at a time. */
sendUtterance(text: string, ctx: AssistantContext, callbacks: AssistantTurnCallbacks): void {
if (this.turnHandle) {
callbacks.onError("The assistant is still working on the previous request");
return;
}
this.lastActivityMs = Date.now();
this.messages.push({ role: "user", content: text });
this.trimHistory();
const tools = this.buildToolDefinitions();
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
const finish = () => {
this.turnHandle = null;
this.lastActivityMs = Date.now();
};
this.turnHandle = this.backend.runTurn({
apiKey: this.apiKey,
model: DEFAULT_LLM_MODEL,
effort: "low",
system,
messages: this.messages,
tools,
registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: {
onTextDelta: callbacks.onTextDelta,
onToolActivity: callbacks.onToolActivity,
onTurnDone: (result) => {
finish();
callbacks.onTurnDone(result);
},
onError: (message) => {
finish();
callbacks.onError(message);
},
},
});
}
cancel(): void {
this.turnHandle?.cancel();
this.turnHandle = null;
this.lastActivityMs = Date.now();
}
private buildToolDefinitions(): AnthropicToolDefinition[] {
this.toolNameMap.clear();
return this.registry.listTools().map((spec) => {
const apiName = this.toApiToolName(spec.name);
this.toolNameMap.set(apiName, spec.name);
return { name: apiName, description: spec.description, input_schema: spec.inputSchema };
});
}
/** Convert a canonical registry name to an Anthropic-legal tool name. */
private toApiToolName(name: string): string {
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
// Disambiguate the rare case where two canonical names sanitize alike.
if (this.toolNameMap.has(base)) {
let i = 2;
while (this.toolNameMap.has(`${base}_${i}`)) i++;
base = `${base}_${i}`;
}
return base;
}
private trimHistory(): void {
if (this.messages.length <= MAX_HISTORY_MESSAGES) return;
// Drop whole messages from the head. A leading orphan tool_result (whose
// tool_use we trimmed) would be rejected by the API, so skip past any
// tool_result-only user message left at the front.
const overflow = this.messages.length - MAX_HISTORY_MESSAGES;
this.messages.splice(0, overflow);
while (this.messages.length && startsWithToolResult(this.messages[0]!)) {
this.messages.shift();
}
}
}
function startsWithToolResult(message: AnthropicMessage): boolean {
return (
Array.isArray(message.content) &&
message.content.length > 0 &&
message.content[0]!.type === "tool_result"
);
}
function describeContext(ctx: AssistantContext): string {
const parts = [`Current time: ${ctx.localTime}.`];
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
if (ctx.foregroundApp) {
const title = ctx.foregroundTitle ? ` ("${ctx.foregroundTitle}")` : "";
parts.push(`The foreground app is ${ctx.foregroundApp}${title}.`);
} else {
parts.push("No app is in the foreground (the launcher is showing).");
}
if (ctx.headsetBattery !== null) {
parts.push(`Glasses battery: ${ctx.headsetBattery}%.`);
}
return `Context: ${parts.join(" ")}`;
}
+227
View File
@@ -0,0 +1,227 @@
import { readUpcomingEvents, type CalendarEvent } from "../native/calendar";
import { mediaControllerBridge } from "../native/media-controller";
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
import { shell } from "../ui/shell/shell";
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
/**
* Registers the always-available system tools into the registry. Called once at
* startup. These wrap shell state and the existing native bridges (calendar,
* media, notifications); nothing here needs an app window to be open.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
let registered = false;
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
if (registered) return;
registered = true;
// Keep a media listener warm so "what's playing" works without opening Music.
void mediaControllerBridge.start();
registry.registerSystemTool(
{
name: "glasses.get_state",
description:
"Get the current state of the glasses: display on/off, foreground app, headset battery, and the current local time.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
proactive: true,
},
() => ok(describeState()),
);
registry.registerSystemTool(
{
name: "glasses.show_alert",
description:
"Show a short text popup on the glasses display. Use for a brief notice the user should see; keep it to a sentence or two.",
inputSchema: {
type: "object",
properties: { text: { type: "string", description: "The message to display." } },
required: ["text"],
additionalProperties: false,
},
proactive: true,
},
(args) => {
const text = String(args?.text ?? "").trim();
if (!text) return err("show_alert requires non-empty text");
shell.showAlert(text);
return ok("Displayed.");
},
);
registry.registerSystemTool(
{
name: "calendar.list_events",
description:
"List the user's upcoming calendar events. Returns events from now forward, ordered by start time.",
inputSchema: {
type: "object",
properties: {
within_hours: {
type: "number",
description: "Only include events starting within this many hours (default 168 = one week).",
},
max_events: { type: "number", description: "Maximum number of events to return (default 10)." },
},
additionalProperties: false,
},
},
(args) => {
const withinHours = clampNumber(args?.within_hours, 1, 24 * 60, 168);
const maxEvents = clampNumber(args?.max_events, 1, 50, 10);
const events = readUpcomingEvents(maxEvents, withinHours * 60 * 60 * 1000);
if (!events.length) return ok("No upcoming events in that window.");
return ok(events.map(formatEvent).join("\n"));
},
);
registry.registerSystemTool(
{
name: "media.now_playing",
description: "What is currently playing (or paused) in the user's media app, if anything.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
proactive: true,
},
() => ok(describeNowPlaying()),
);
registry.registerSystemTool(
{
name: "media.play_pause",
description: "Toggle play/pause for the user's current media session.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
},
async () => {
const state = mediaControllerBridge.snapshot();
if (!state.canPlayPause) return err("No media session can be controlled right now.");
await mediaControllerBridge.playPause();
return ok("Toggled play/pause.");
},
);
registry.registerSystemTool(
{
name: "media.next",
description: "Skip to the next track in the user's current media session.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
},
async () => {
const state = mediaControllerBridge.snapshot();
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
await mediaControllerBridge.skipNext();
return ok("Skipped to the next track.");
},
);
registry.registerSystemTool(
{
name: "notifications.list",
description: "List the user's current Android notifications mirrored to the glasses.",
inputSchema: {
type: "object",
properties: { max: { type: "number", description: "Maximum notifications to return (default 10)." } },
additionalProperties: false,
},
},
(args) => {
const max = clampNumber(args?.max, 1, 50, 10);
const notifications = readActiveNotifications(max);
if (!notifications.length) return ok("No current notifications.");
return ok(notifications.map(formatNotification).join("\n"));
},
);
registry.registerSystemTool(
{
name: "notifications.dismiss",
description:
"Dismiss a notification by its key (as returned by notifications.list). Returns whether it was dismissed.",
inputSchema: {
type: "object",
properties: { key: { type: "string", description: "The notification key from notifications.list." } },
required: ["key"],
additionalProperties: false,
},
},
(args) => {
const key = String(args?.key ?? "").trim();
if (!key) return err("notifications.dismiss requires a key");
return dismissNotification(key) ? ok("Dismissed.") : err("No notification with that key.");
},
);
}
function describeState(): string {
const parts: string[] = [];
parts.push(`Display: ${shell.isScreenOn() ? "on" : "off"}.`);
const fg = shell.getForegroundApp();
parts.push(fg ? `Foreground app: ${fg.appId} ("${fg.title}").` : "Foreground: launcher (no app open).");
const battery = shell.getBatteryLevels();
if (battery.headset !== null) {
const charging = battery.headsetCharging ? ", charging" : "";
parts.push(`Headset battery: ${battery.headset}%${charging}.`);
}
parts.push(`Local time: ${formatDateTime(new Date())}.`);
return parts.join(" ");
}
function describeNowPlaying(): string {
const state = mediaControllerBridge.snapshot();
if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable.";
if (state.playbackState === "playing" || state.playbackState === "paused") {
const verb = state.playbackState === "playing" ? "Playing" : "Paused";
const title = state.title || "(unknown title)";
const artist = state.artist ? ` by ${state.artist}` : "";
const app = state.appName ? ` in ${state.appName}` : "";
return `${verb}: ${title}${artist}${app}.`;
}
return "Nothing is currently playing.";
}
function formatEvent(event: CalendarEvent): string {
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
const location = event.location ? ` @ ${event.location}` : "";
return `- ${event.title || "(untitled)"} — ${when}${location}`;
}
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
const title = notification.title || "(no title)";
const body = notification.text ? `: ${notification.text}` : "";
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
}
function formatDateTime(date: Date): string {
return `${formatDate(date)}, ${formatTime(date)}`;
}
function formatDate(date: Date): string {
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
}
function formatTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
return `${hours}:${minutes} ${meridiem}`;
}
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
const num = typeof value === "number" ? value : Number(value);
if (!Number.isFinite(num)) return fallback;
return Math.min(max, Math.max(min, Math.round(num)));
}
function ok(content: string): ToolResult {
return { ok: true, content };
}
function err(error: string): ToolResult {
return { ok: false, error };
}
+190
View File
@@ -0,0 +1,190 @@
/**
* The assistant's tool registry: the single place that knows which tools exist,
* whether each is currently callable, and how to invoke it. Both assistant
* backends (direct Anthropic loop, external agent bridge) list and call tools
* through here, so gating and timeouts are enforced in one spot rather than
* trusted to a model or a remote agent.
*
* Availability tiers (see notes/voice-assistant-design.md):
* - "always" system/shell tools; no app involved.
* - "installed" app tool live whenever the app is installed (no window
* needed) -- e.g. "start a 5 minute timer".
* - "open" app tool live while the app has a window open, foreground
* or backgrounded -- e.g. "next track".
* - "foreground" app tool live only while the app owns the foreground window
* -- e.g. "scroll down".
*
* Phase 1 only registers "always" tools; the tier machinery is here so the
* app-tool phases slot in without reshaping the registry.
*/
export type ToolAvailability = "always" | "installed" | "open" | "foreground";
export type ToolSpec = {
/** Namespaced, e.g. "glasses.get_state" or "app.terminal.send_input". */
name: string;
description: string;
/** JSON Schema for the tool's arguments. */
inputSchema: object;
availability: ToolAvailability;
/** May be invoked outside an active voice turn (external mode proactive calls). */
proactive?: boolean;
/** Round-trip cap; the registry rejects with a tool error past it. Default 10s. */
timeoutMs?: number;
};
/**
* A tool's outcome. Flat (not a discriminated union) on purpose: this project
* builds with strictNullChecks off, so `ok: true|false` would widen to boolean
* and defeat union narrowing. Read `content` when ok, `error` when not.
*/
export type ToolResult = {
ok: boolean;
/** Text payload back to the model (when ok). */
content?: string;
/** Error message back to the model (when not ok). */
error?: string;
};
export type ToolHandler = (args: any) => Promise<ToolResult> | ToolResult;
export type ToolRegistration = {
spec: ToolSpec;
handler: ToolHandler;
/**
* Whether the tool is currently live. Absent means always live (the "always"
* tier). App-tool phases supply a predicate keyed on window/foreground state.
*/
isAvailable?: () => boolean;
};
export type ListToolsOptions = {
/** Only tools flagged proactive (for calls outside an active turn). */
proactiveOnly?: boolean;
};
export type CallToolOptions = {
/** The call is happening outside an active voice turn; enforce proactive gating. */
proactive?: boolean;
};
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
export class ToolRegistry {
private readonly registrations = new Map<string, ToolRegistration>();
private readonly changeListeners = new Set<() => void>();
/** Register (or replace) a tool. Names are unique across all tiers. */
register(registration: ToolRegistration): void {
const { name } = registration.spec;
this.registrations.set(name, registration);
this.fireToolsChanged();
}
/** Convenience for the always-live system tools. */
registerSystemTool(
spec: Omit<ToolSpec, "availability">,
handler: ToolHandler,
): void {
this.register({ spec: { ...spec, availability: "always" }, handler });
}
unregister(name: string): void {
if (this.registrations.delete(name)) {
this.fireToolsChanged();
}
}
/** Specs for the tools callable right now, given the availability tiers. */
listTools(options: ListToolsOptions = {}): ToolSpec[] {
const specs: ToolSpec[] = [];
for (const registration of this.registrations.values()) {
if (!this.isLive(registration)) continue;
if (options.proactiveOnly && !registration.spec.proactive) continue;
specs.push(registration.spec);
}
return specs;
}
/**
* Invoke a tool by name. Never throws: a missing/unavailable tool, a
* proactive-gating rejection, a handler exception, or a timeout all come back
* as `{ ok: false }` so the agent loop can surface them as tool errors.
*/
async callTool(name: string, args: unknown, options: CallToolOptions = {}): Promise<ToolResult> {
const registration = this.registrations.get(name);
if (!registration) {
return { ok: false, error: `Unknown tool: ${name}` };
}
if (!this.isLive(registration)) {
return { ok: false, error: `Tool ${name} is not currently available` };
}
if (options.proactive && !registration.spec.proactive) {
return { ok: false, error: `Tool ${name} cannot be called outside a conversation` };
}
const timeoutMs = registration.spec.timeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
try {
return await this.withTimeout(registration.handler(args), timeoutMs, name);
} catch (error) {
return { ok: false, error: `Tool ${name} failed: ${describeError(error)}` };
}
}
/** Subscribe to add/remove/availability changes (drives MCP list_changed). */
onToolsChanged(listener: () => void): () => void {
this.changeListeners.add(listener);
return () => this.changeListeners.delete(listener);
}
/** App-tool phases call this after a foreground/window change re-gates tools. */
fireToolsChanged(): void {
for (const listener of this.changeListeners) {
try {
listener();
} catch (error) {
console.warn("onToolsChanged listener failed", error);
}
}
}
private isLive(registration: ToolRegistration): boolean {
return registration.isAvailable ? registration.isAvailable() : true;
}
private withTimeout(
result: Promise<ToolResult> | ToolResult,
timeoutMs: number,
name: string,
): Promise<ToolResult> {
if (!(result instanceof Promise)) return Promise.resolve(result);
return new Promise<ToolResult>((resolve) => {
let settled = false;
const timer = setTimeout(() => {
if (settled) return;
settled = true;
resolve({ ok: false, error: `Tool ${name} timed out after ${timeoutMs}ms` });
}, timeoutMs);
result.then(
(value) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(value);
},
(error) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve({ ok: false, error: `Tool ${name} failed: ${describeError(error)}` });
},
);
});
}
}
function describeError(error: unknown): string {
return String((error as Error)?.message ?? error);
}
/** The process-wide registry; system tools register into it at startup. */
export const toolRegistry = new ToolRegistry();
+32
View File
@@ -0,0 +1,32 @@
/**
* Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer)
* talks to the same session surface whether the turn runs on the phone
* (direct Anthropic loop) or against the user's own agent (external bridge,
* later phases).
*/
/** Situational grounding injected per turn (system prompt in direct mode). */
export type AssistantContext = {
/** Foreground app id, or null when the sidebar/launcher is all that's up. */
foregroundApp: string | null;
foregroundTitle: string | null;
screenOn: boolean;
/** Human-readable local time, e.g. "Thu Jul 24, 3:15 PM". */
localTime: string;
/** Headset battery percent (0..100), or null if unknown. */
headsetBattery: number | null;
};
export type AssistantTurnCallbacks = {
/** Streamed reply text (delta plus the full text so far). */
onTextDelta: (delta: string, textSoFar: string) => void;
/** A tool is being invoked; label is display-ready, e.g. "calendar.list_events". */
onToolActivity: (label: string) => void;
onTurnDone: (result: { stopReason: string | null }) => void;
onError: (message: string) => void;
};
/** A handle to an in-flight turn. */
export type AssistantTurnHandle = {
cancel(): void;
};
+4
View File
@@ -25,6 +25,7 @@ import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
import { registerSystemTools } from "../assistant/system-tools";
import { WorkerAppHost } from "../ui/shell/worker-window";
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app";
import {
@@ -236,6 +237,9 @@ class DashboardController {
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
};
this.sharedActions = sharedActions;
// The always-available assistant tools (calendar, media, notifications,
// glasses state) register once at startup, independent of any connection.
registerSystemTools();
shell.configure({
actions: {
...sharedActions,
+37 -19
View File
@@ -1,5 +1,5 @@
// AUTO-GENERATED from g2flash/patches/cfw_patches.json — do not edit by hand.
// Regenerate by re-running the generator in the firmware onboarding change.
// Regenerate with scripts/update_deltas.sh.
// Turns the stock G2 2.2.6.10 image into the Faceclaw custom firmware.
export type FirmwarePatchOp = {
@@ -19,7 +19,7 @@ export type FirmwarePatchSet = {
export const CFW_PATCH_SET: FirmwarePatchSet = {
"base": "g2_2.2.6.10.bin",
"baseSha256": "f4dfb0b49ad3de3c2daf17f8a27a157c3dc98411d6a0d3ab2cfd0918f41b9afa",
"outputSha256": "162fa8aab6f41be5e314d51b56f3e8f3eb2143af93ee0518d1919d096701474d",
"outputSha256": "ba7e896e516d049e530d7aaed30e358b45f940e81a90f997a44465299151b452",
"patches": [
{
"offset": 1449549,
@@ -42,85 +42,103 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1447919,
"old": "7ef7fefd",
"new": "b8f28cfe",
"new": "b8f2f6fe",
"desc": "bl snapshot_side @ 0x4db968"
},
{
"offset": 1448931,
"old": "7ef704fc",
"new": "b8f292fc",
"new": "b8f2fcfc",
"desc": "bl snapshot_side @ 0x4dbd5c"
},
{
"offset": 1165461,
"old": "45f0cefd",
"new": "fdf23dfe",
"new": "fdf2a7fe",
"desc": "bl image_deferred (deferred consumer -> FIFO restore + worker, both lenses)"
},
{
"offset": 1186287,
"old": "d9f7d4ff",
"new": "f8f2d0fe",
"new": "f9f2b8fb",
"desc": "bl settings_send_wrapper (append caps field 100)"
},
{
"offset": 1183983,
"old": "f4f75aff",
"new": "f9f2b6fd",
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
},
{
"offset": 926945,
"old": "08f068fa",
"new": "38f3d9fa",
"desc": "bl faceclaw_display_start @ 0x45c65a (fail-open double-tap takeover)"
},
{
"offset": 927137,
"old": "08f008fa",
"new": "38f379fa",
"desc": "bl faceclaw_display_start @ 0x45c71a (fail-open double-tap takeover)"
},
{
"offset": 822553,
"old": "28f003f8",
"new": "51f39dfd",
"new": "52f39dfa",
"desc": "bl evenhub_longpress (replaces force-quit dialog)"
},
{
"offset": 823369,
"old": "1cf09bfb",
"new": "51f321fc",
"new": "52f321f9",
"desc": "bl ring_release (forward ring release-long-press)"
},
{
"offset": 1474163,
"old": "34d1",
"new": "34e0",
"desc": "even_ai_display_ctrl: skip Even AI app START (wakeword -> phone)"
"offset": 1474137,
"old": "7fb50600",
"new": "b3f271b9",
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
},
{
"offset": 4301227,
"old": "",
"new": "02fb01f044f6d341c0f24701084700bf084644f61751c0f24701084770b500284ed001684ef65a62ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf2de9f04f85b043f6fc76c2f200063568804628466ff3960044f6d344b0f1005fc0f2470406d128684ef65a61ccf2ff0188423bd04ff4fe70a0470546a8b30020002100bf6a1868540431b1f5fe7f50709070d070f6d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe003560b8f1000f18bf002d07d14af26950c0f2450005b0bde8f04f0047d8f80cb0bbf1000ff2d0d8f820a0baf1000fedd0a94659f8040f00287bd0686900287ad0686a002879d0686b002878d0686c002877d0686d002876d0686e002875d0686f002874d0d5f88400002872d0d5f89400002870d0d5f8a40000286ed0d5f8b40000286cd0d5f8b00029690490d5f8a0002a6b0390d5f8900000260290d5f880002f6f0190286ad5f860e0d5f850c02b6c884238bf014638bf01268a423cbf114602268b423cbf194603268c453cbf614604268e453cbf714605268f423cbf39460626019888423cbf01460726029888423cbf01460826039888423cbf014609260498884238bf0146d5f8c00038bf0a26884238bf0b2609eb0610406804f1440137018847002049f80700012085f8df0016e0002614e0012612e0022610e003260ee004260ce005260ae0062608e0072606e0082604e0092602e00a2600e00b265046a04700283ff451af0af0030c5fea9a034ff0000219d00496a2eb830e431c0bf101066ff0030700bff119cc78da19d4700c79043714714c79547189790eeb070404349171f0d1049e3a1dbcf1000f10d01bf80230bcf1010f83540ad0531c1bf80370bcf1020fc75403d0911c1bf80120425409eb06114860d5f8c400320149f80280421cc1f808a0c5f8c420c8600ce738464146fff7aebe2de9f04f81b043f6fc79c2f20009d9f80050044628466ff3960044f6d34816460f46b0f1005fc0f2470806d128684ef65a61ccf2ff0188423dd04ff4fe70c0470546002800f09580002000216a1868540431b1f5fe7f50709070d070f6d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe00c9f8005005f1040b4ff0000a4ff0000900bf4ff0ff30002200234ff0ff3105e000bf6032c02a03f1060332d0af187e68a64204d13e6986423cbf194630467e69a64204d13e6a86423cbf591c30467e6aa64204d13e6b86423cbf991c30467e6ba64204d13e6c86423cbfd91c30467e6ca64204d13e6d86423cbf191d30467e6da642ced13f6e87423cbf591d3846c8e700bf002910d40beb01104768090182684bf801a02046394600f013f8814608f1440138468847ace7484601b0bde8f08f204639463246c9f8005001b0bde8f04fffe72de9f041a4b0064640f6bb4043f6fc78c0f24e0014460d46c2f200088047002002af0390384600f01bf90df10c0c3046294622460123cdf800c000f0bff90446384600f015ffd8f8001005460a466ff39602b2f1005f06d108684ef65a62ccf2ff0290423fd044f6d341c0f247014ff4fe708847d0b30146002000228b1888540432b2f5fe7f58709870d870f6d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe00c8f80010c1f8d05002e00020c8f80000204624b0bde8f08100bf09295ad1f0b502eb030c352756248cf8107037278cf804404e248cf8117036278cf8064043248cf812707a278cf8074046248cf8177072274ff0a20e8cf8084057248cf819706c2702f803e04ff0060e8cf809402f248cf81a7065278cf801e04ff0450e8cf80a4032248cf81b7000278cf803e08cf805e08cf80b4020244ff0690e6d2567268cf81c7019271c338cf80c408cf80de08cf80e508cf80f608cf813408cf814e08cf815508cf816608cf818408cf80270bde8f04045f6153cc0f2470c60474df63040c2f234000078042818bf704780b582b04af26b1e4ff0000cc0f24d0e0020002100220923cdf804c0cdf800c0f04702b080bd00bff0b583b04df63043c2f234031b784ff6e704042bc0f2450420d144f2d043c2f207031b680546184616460f46a04703463946324628468bb11b68e02b0ed14af26b160027c0f24d060020002100220a2301970097b047012003b0f0bd04f1160303b0bde8f0401847f0b581b041f204054cf65561cef20005ccf2ac51c5f8ac1f4ef6fc51cef200010a6843f6fc7642f080720a6055f8041cc2f2000641f0010145f8041c316807460a466ff39602b2f1005f06d108684ef65a62ccf2ff02904240d044f6d341c0f247014ff4fe708847002870d001460020002200bf8b1888540432b2f5fe7f58709870d870f6d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe003160d1f8d800b8bb44f6342245f6e464c2f20702cff6f8741368204600bf16689e420ed116689e420bd116689e4208d116689e4203d1002800f10400efd19e421ad02b6816681068b0420ed11068b0420bd11068b04208d11068b04203d1002c04f10404efd1b04206d02868c01ac1f8d80001e0002030602868386001b0f0bd00bf2de9f04fdbb0002918bf002a04d15bb0bde8f04f00f00cbe0c7804f07f07052f00f0e580072f4dd0422ff0d0ddf89091b0f84080b0f842a0092f00f04e81082f40f058810746022a4ff0ff30c0f0b382002b00f0b082c3460bf10100032303eb500020f0030000fb0af07630824200f23c814d78fdb188460226b01c904200f2348108eb060318f806605b7856ea032300f02b811e18964200f2278108eb0001384614461a460023cdf80090fff7a8ff2246013de1d13846594652464b4600f01dfe77e243f6fc74c2f20004236818466ff39600b0f1005f06d118684ef65a67ccf2ff07b84243d00e4644f6d341c0f247014ff4fe70154688470346a8b30020002100bf5a1858540431b1f5fe7f50709070d070fLine truncated
"new": "02fb01f044f6d341c0f24701084700bf084644f61751c0f24701084770b500284ed001684ef65a62ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf2de9f04f83b043f6fc76c2f200063568824628466ff3960044f6d344b0f1005fc0f2470407d128684ef65a61ccf2ff01884200f0a4804ff40670a0470546002800f09c8000206ff021010022ab18a85443326f18b2f5067f01f1430158709870d870187158719871d871187258729872d872187358739873d873187458749874d874187558759875d875187658769876d876187758779877d87783f8200087f8430087f8440083f8230083f8240083f8250083f8260083f8270083f8280083f8290083f82a0083f82b0083f82c0083f82d0083f82e0083f82f0083f8300083f8310083f8320083f8330083f8340083f8350083f8360083f8370083f8380083f8390083f83a0083f83b0083f83c0083f83d0083f83e0083f83f0083f8400083f8410083f8420091d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe003560baf1000f18bf002d07d14af26950c0f2450003b0bde8f04f0047daf80cb0bbf1000ff2d0daf82080b8f1000fedd0a94659f8040f00287cd0686900287bd0686a00287ad0686b002879d0686c002878d0686d002877d0686e002876d0686f002875d0d5f88400002873d0d5f89400002871d0d5f8a40000286fd0d5f8b40000286dd0d5f8b00029690290d5f8a0002c6a0190d5f8900000260090286bd5f880e0d5f870c02b6e2f6d2a6c8c4238bf214638bf012688423cbf014602268a423cbf114603268f423cbf394604268b423cbf194605268c453cbf614606268e453cbf71460726009844f6d34488423cbf014608260198c0f2470488423cbf014609260298884238bf0146d5f8c00038bf0a26884238bf0b2609eb0610406804f1440137018847002049f80700012085f8df0016e0002614e0012612e0022610e003260ee004260ce005260ae0062608e0072606e0082604e0092602e00a2600e00b264046a04700283ff450af08f0030c5fea98034ff0000219d00296a2eb830e431c0bf101066ff0030700bff119cc78da19d4700c79043714714c79547189790eeb070404349171f0d1029e3a1dbcf1000f10d01bf80230bcf1010f83540ad0531c1bf80370bcf1020fc75403d0911c1bf80120425409eb06114860d5f8c400320149f802a0421cc1f80880c5f8c420c8600be738464146fff744be2de9f04f81b043f6fc79c2f20009d9f80050044628466ff3960044f6d34816460f46b0f1005fc0f2470807d128684ef65a61ccf2ff01884200f0a6804ff40670c0470546002800f0fc8000206ff02101002200bfab18a85443326f18b2f5067f01f1430158709870d870187158719871d871187258729872d872187358739873d873187458749874d874187558759875d875187658769876d876187758779877d87783f8200087f8430087f8440083f8230083f8240083f8250083f8260083f8270083f8280083f8290083f82a0083f82b0083f82c0083f82d0083f82e0083f82f0083f8300083f8310083f8320083f8330083f8340083f8350083f8360083f8370083f8380083f8390083f83a0083f83b0083f83c0083f83d0083f83e0083f83f0083f8400083f8410083f8420091d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe00c9f8005005f1040b4ff0000a4ff000094ff0ff30002200234ff0ff3105e000bf6032c02a03f1060332d0af187e68a64204d13e6986423cbf194630467e69a64204d13e6a86423cbf591c30467e6aa64204d13e6b86423cbf991c30467e6ba64204d13e6c86423cbfd91c30467e6ca64204d13e6d86423cbf191d30467e6da642ced13f6e87423cbf591d3846c8e700bf002910d40beb01104768090182684bf801a02046394600f013f8814608f1440138468847ace7484601b0bde8f08f204639463246c9f8005001b0bde8f04fffe72de9f041a4b0064640f6bb4043f6fc78c0f24e0014460d46c2f200088047002002af0390384600f049fd0df10c0c3046294622460123cdf800c000f055fe0446384601f07dfcd8f8001005460a466ff39602b2f1005f07d108684ef65a62ccf2ff02904200f0aa8044f6d341c0f247014ff406708847002800f0a380014600206ff02102002300bfcf18c85443338e18b3f5067f02f143027870b870f87038717871b871f87138727872b872f87238737873b873f87338747874b874f87438757875b875f87538767876b876f87638777877b877f87787f8200086f8430086f8440087f8230087f8240087f8250087f8260087f8270087f8280087f8290087f82a0087f82b0087f82c0087f82d0087f82e0087f82f0087f8300087f8310087f8320087f8330087f8340087f8350087f8360087f8370087f8380087f8390087f83a0087f83b0087f83c0087f83d0087f83e0087f83f0087f8400087f8410087f8420091d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe00c8f80010c1f8d05002e00020c8f80000204624b0bde8f08143f6fc70c2f20000016808466ff39600b0f1005f16d108684ef65a62ccf2ff0290420fd1d1f8fc0160b144f63422c2f207021268801a0028cebf01200020c1f8fc01704700207047f0b101684ef65a62ccf2ff02914218bf704790f8061299b110b544f62f344ff0000cc0f24604864680f806c20120002100220023aef804c2a446bde810406047704700bf2de9f04143f6fc77c2f200073f684ff000083c466ff39604b4f1005f07d13e684ef65a65ccf2ff05ae4208bfb84601280fd11c4615460f460646fff79bff2346044639462a463046002c18bfb8f1000f06d144f62f3cc0f2460cbde8f041604798f8060270b1d8f800020021002888f80612a8f804123ed049f2d941c0f24401884738e0b8f804020121013080b2012898bf0846a8f80402d8f8000249f2d944c0f2440488f8061268b94ff60370cff6ff70a4f594777844002142460023b847c8f8000240b1a047d8f80002a4f140024Line truncated
"desc": "append injected blobs to main-app payload"
},
{
"offset": 777711,
"old": "44c33500",
"new": "84fe3500",
"new": "380e3600",
"desc": "main-app subheader payload size (ps)"
},
{
"offset": 152,
"old": "c4c33500",
"new": "04ff3500",
"new": "b80e3600",
"desc": "main-app TOC entry size (ps + 128)"
},
{
"offset": 777831,
"old": "44c33504",
"new": "84fe3504",
"new": "380e3604",
"desc": "main-app preamble length (low 24 bits)"
},
{
"offset": 777835,
"old": "df8110c9",
"new": "6eddece4",
"new": "88ed0126",
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
},
{
"offset": 156,
"old": "c1be0fb1",
"new": "ed6b6bbd",
"new": "f1e9f8d8",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
},
{
"offset": 777715,
"old": "c1be0fb1",
"new": "ed6b6bbd",
"new": "f1e9f8d8",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
}
]
+86 -5
View File
@@ -13,14 +13,34 @@ const ANTHROPIC_VERSION = "2023-06-01";
/** Default model for voice continuations (and a sensible general default). */
export const DEFAULT_LLM_MODEL = "claude-sonnet-5";
/**
* A content block in an Anthropic message. Text-only flows (dictation refine)
* still pass a plain string for content; the assistant tool loop uses the
* block array to carry tool_use / tool_result alongside text.
*/
export type AnthropicContentBlock =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; input: any }
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean };
export type AnthropicMessage = {
role: "user" | "assistant";
content: string;
content: string | AnthropicContentBlock[];
};
/** An Anthropic tool definition (input_schema is snake_case as the API wants). */
export type AnthropicToolDefinition = {
name: string;
description: string;
input_schema: object;
};
export type AnthropicStreamResult = {
/** Concatenated text of all text blocks. */
text: string;
/** e.g. "end_turn", "max_tokens", "refusal"; null if the stream never said. */
/** The assembled assistant content blocks (text + any tool_use). */
content: AnthropicContentBlock[];
/** e.g. "end_turn", "tool_use", "max_tokens", "refusal"; null if never said. */
stopReason: string | null;
};
@@ -29,6 +49,7 @@ export type AnthropicStreamOptions = {
model?: string;
system?: string;
messages: AnthropicMessage[];
tools?: AnthropicToolDefinition[];
maxTokens?: number;
/** Thinking/latency tradeoff; voice flows want "low". Default: model default. */
effort?: "low" | "medium" | "high";
@@ -39,6 +60,11 @@ export type AnthropicStreamOptions = {
onError: (message: string) => void;
};
/** Accumulator for one streamed content block before it is finalized. */
type StreamingBlock =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; inputJson: string };
export type AnthropicStreamHandle = {
cancel(): void;
};
@@ -49,6 +75,8 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
let settled = false;
let text = "";
let stopReason: string | null = null;
// Content blocks, keyed by their SSE `index`, assembled as deltas arrive.
const blocks = new Map<number, StreamingBlock>();
const fail = (message: string) => {
if (settled) return;
@@ -58,7 +86,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
const done = () => {
if (settled) return;
settled = true;
options.onDone({ text, stopReason });
options.onDone({ text, content: finalizeBlocks(blocks), stopReason });
};
if (!global.isAndroid) {
@@ -78,6 +106,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
messages: options.messages,
};
if (options.system) body.system = options.system;
if (options.tools?.length) body.tools = options.tools;
if (options.effort) body.output_config = { effort: options.effort };
const listener = new com.faceclaw.app.FaceclawSseListener({
@@ -86,13 +115,41 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
const event = parseSseDataLine(String(line));
if (!event) return;
switch (event.type) {
case "content_block_delta":
case "content_block_start": {
const index = Number(event.index);
const block = event.content_block;
if (block?.type === "tool_use") {
blocks.set(index, {
type: "tool_use",
id: String(block.id ?? ""),
name: String(block.name ?? ""),
inputJson: "",
});
} else if (block?.type === "text") {
blocks.set(index, { type: "text", text: String(block.text ?? "") });
}
return;
}
case "content_block_delta": {
const index = Number(event.index);
if (event.delta?.type === "text_delta") {
const delta = String(event.delta.text ?? "");
text += delta;
const existing = blocks.get(index);
if (existing?.type === "text") {
existing.text += delta;
} else {
blocks.set(index, { type: "text", text: delta });
}
options.onTextDelta?.(delta, text);
} else if (event.delta?.type === "input_json_delta") {
const existing = blocks.get(index);
if (existing?.type === "tool_use") {
existing.inputJson += String(event.delta.partial_json ?? "");
}
}
return;
}
case "message_delta":
if (event.delta?.stop_reason) {
stopReason = String(event.delta.stop_reason);
@@ -105,7 +162,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
fail(`Anthropic: ${String(event.error?.message ?? event.error?.type ?? "stream error")}`);
return;
default:
// message_start, content_block_start/stop, ping — nothing to do.
// message_start, content_block_stop, ping — nothing to do.
return;
}
},
@@ -146,6 +203,30 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
};
}
/** Assemble the streamed blocks (in index order) into final content blocks. */
function finalizeBlocks(blocks: Map<number, StreamingBlock>): AnthropicContentBlock[] {
const content: AnthropicContentBlock[] = [];
for (const index of Array.from(blocks.keys()).sort((a, b) => a - b)) {
const block = blocks.get(index)!;
if (block.type === "text") {
content.push({ type: "text", text: block.text });
} else {
let input: any = {};
const trimmed = block.inputJson.trim();
if (trimmed) {
try {
input = JSON.parse(trimmed);
} catch {
// A tool_use whose JSON never completed (e.g. truncated stream); pass
// empty args and let the tool handler report what's missing.
}
}
content.push({ type: "tool_use", id: block.id, name: block.name, input });
}
}
return content;
}
/** Parse one SSE line; returns the JSON payload of a data: line, else null. */
function parseSseDataLine(line: string): any | null {
if (!line.startsWith("data:")) return null;
+190
View File
@@ -0,0 +1,190 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { GESTURE_DOUBLE_CLICK } from "../gestures";
import { drawSelectionHighlight } from "../menu";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
const DIALOG_X = 40;
const DIALOG_Y = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
const MENU_ROWS = 2;
/** thinking: turn running; done/error: finished, showing the Follow-up/Done menu. */
type AssistantPhase = "thinking" | "done" | "error";
export type AssistantLayerCallbacks = {
/** Record another utterance and continue this conversation. */
onFollowUp: () => void;
/** Abort the in-flight turn (double-click while thinking). */
onCancel: () => void;
/** The user closed the overlay (Done). */
onClose: () => void;
/** The layer left the stack by any path (Done, or the screen sleeping). */
onRemoved?: () => void;
};
/**
* The assistant overlay: streamed reply text with a status line for tool
* activity, and a Follow-up / Done menu once the turn ends. Double-click
* cancels an in-flight turn. The shell owns the AssistantSession and drives
* this layer's state through the on* methods as the turn streams.
*/
export class AssistantLayer implements Layer {
private phase: AssistantPhase = "thinking";
private replyText = "";
private status = "Thinking...";
private menuIndex = 0;
constructor(
private readonly actions: LayerActions,
private readonly callbacks: AssistantLayerCallbacks,
) {}
/** Shell: a new turn (initial utterance or follow-up) is starting. */
startTurn(): void {
this.phase = "thinking";
this.replyText = "";
this.status = "Thinking...";
this.menuIndex = 0;
this.actions.requestRender();
}
onTextDelta(_delta: string, textSoFar: string): void {
this.replyText = textSoFar;
if (this.phase === "thinking") this.status = "Thinking...";
this.actions.requestRender();
}
onToolActivity(label: string): void {
if (this.phase === "thinking") this.status = `→ ${label}`;
this.actions.requestRender();
}
onTurnDone(): void {
this.phase = "done";
this.status = this.replyText.trim() ? "" : "(no reply)";
this.menuIndex = 0;
this.actions.requestRender();
}
onError(message: string): void {
this.phase = "error";
this.status = message;
this.menuIndex = 0;
this.actions.requestRender();
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
// Solid dialog box (fill 1, matching the voice dialog: opaque after 4bpp
// quantization, but not the color-key transparent 0).
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, DIALOG_Y + 12, this.phase === "thinking" ? "Assistant ●" : "Assistant", 220);
if (this.status) {
const statusValue = this.phase === "error" ? 200 : 130;
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), statusValue);
}
const inMenu = this.phase !== "thinking";
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
const wrapped = wrapText(font, this.replyText, TEXT_MAX_WIDTH);
// Keep the tail visible as text streams past the bottom.
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (inMenu) {
const rows = ["Follow-up", "Done"];
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
}
image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200);
}
} else {
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110);
}
return image;
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
if (this.phase === "thinking") {
if (event.type === "double-click") {
this.callbacks.onCancel();
this.phase = "done";
this.status = "Cancelled";
this.menuIndex = 0;
this.actions.requestRender();
}
return;
}
// done / error: the Follow-up / Done menu.
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
this.callbacks.onFollowUp();
} else {
this.callbacks.onClose();
}
return;
case "double-click":
this.callbacks.onClose();
return;
default:
return;
}
}
onRemoved(): void {
this.callbacks.onRemoved?.();
}
}
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) lines.push(line);
return lines.length ? lines : [""];
}
function truncate(font: BdfFont, text: string, maxWidth: number): string {
if (font.measureText(text) <= maxWidth) return text;
let out = text;
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
out = out.slice(0, -1);
}
return `${out}...`;
}
+136 -42
View File
@@ -13,19 +13,47 @@ const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
const MENU_ROWS = 3;
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
/**
* The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or
* click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing"
* click-to-finish); "menu" is the send / Continue / Discard menu; "continuing"
* records a follow-up utterance (always click-to-finish); "refining" streams
* the LLM-merged text, then returns to "menu".
*/
type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
/**
* A destination the captured text can be sent to. The shell supplies these per
* entry point: "Send to Assistant" (hand off to the voice assistant) and/or
* "Type Into App" (deliver to the foreground window's text input).
*/
export type VoiceSendTarget = {
id: string;
label: string;
onSend: (text: string) => void;
};
export type VoiceInputLayerOptions = {
actions: LayerActions;
/** Post-removal cleanup (also fires when the screen turns off). */
onClosed: () => void;
/** Pop this layer off the shell stack. */
dismiss: () => void;
/** Ordered send destinations shown as the first menu rows. */
sendTargets: VoiceSendTarget[];
/** Which send target is highlighted by default (entry-point dependent). */
defaultTargetIndex?: number;
/** A single click ends the utterance (opened from a menu, no held button). */
finishOnClick?: boolean;
/** Wakeword flow: mic starts immediately and ends on detected silence. */
handsFree?: boolean;
/** Skip the menu: on capture end, send straight to the default target. */
autoSend?: boolean;
};
/**
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
* The mic runs while the button is held (long-press); releasing it stops the
@@ -58,17 +86,34 @@ export class VoiceInputLayer implements Layer {
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
private refineHandle: AnthropicStreamHandle | null = null;
private menuIndex = 0;
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
private pendingAutoSend = false;
private autoSendTimer: ReturnType<typeof setTimeout> | null = null;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private unsubscribeSpeechEnd: (() => void) | null = null;
constructor(
private readonly actions: LayerActions,
private readonly onClosed: () => void,
private readonly onSend: (text: string) => void,
private readonly finishOnClick = false,
private readonly handsFree = false,
) {}
private readonly actions: LayerActions;
private readonly onClosed: () => void;
private readonly dismiss: () => void;
private readonly sendTargets: VoiceSendTarget[];
private readonly defaultTargetIndex: number;
private readonly finishOnClick: boolean;
private readonly handsFree: boolean;
private readonly autoSend: boolean;
constructor(options: VoiceInputLayerOptions) {
this.actions = options.actions;
this.onClosed = options.onClosed;
this.dismiss = options.dismiss;
this.sendTargets = options.sendTargets;
this.finishOnClick = options.finishOnClick ?? false;
this.handsFree = options.handsFree ?? false;
this.autoSend = options.autoSend ?? false;
const defaultIndex = options.defaultTargetIndex ?? 0;
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
this.menuIndex = this.defaultTargetIndex;
}
startCapture(): void {
if (this.capturing) return;
@@ -113,13 +158,66 @@ export class VoiceInputLayer implements Layer {
this.capturing = false;
this.phase = "menu";
void this.actions.stopVoiceCapture();
if (this.status.startsWith("Listening")) {
if (this.autoSend && this.sendTargets.length) {
// Skip-confirmation (wakeword): send to the default target as soon as the
// transcript finalizes, or after a short wait for the trailing final.
this.pendingAutoSend = true;
this.status = "Sending...";
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
} else if (this.status.startsWith("Listening")) {
this.status = "Send, continue, or discard?";
}
this.actions.requestRender();
}
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
/** Fire the queued skip-confirmation send (or fall back to the menu). */
private performAutoSend(): void {
if (this.autoSendTimer !== null) {
clearTimeout(this.autoSendTimer);
this.autoSendTimer = null;
}
if (!this.pendingAutoSend) return;
this.pendingAutoSend = false;
const text = this.displayText().trim();
const target = this.sendTargets[this.defaultTargetIndex];
if (text && target) {
this.dismiss();
target.onSend(text);
} else {
// Nothing heard; fall back to the menu so the user can retry or discard.
this.status = "Send, continue, or discard?";
this.actions.requestRender();
}
}
/** The menu rows: one per send target, then Continue, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
rows.push({
label: target.label,
dim: !hasText,
onSelect: () => {
this.dismiss();
if (hasText) target.onSend(text);
},
});
}
rows.push({
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
},
});
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
return rows;
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
@@ -133,9 +231,9 @@ export class VoiceInputLayer implements Layer {
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
const inMenu = this.phase === "menu";
const hasText = this.displayText().trim().length > 0;
// Leave room for the three-row menu when it is showing.
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const rows = inMenu ? this.menuRows() : [];
// Reserve space for the actual number of rows this menu has.
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
@@ -147,14 +245,7 @@ export class VoiceInputLayer implements Layer {
}
if (inMenu) {
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean }> = [
// Dim "Send" when there is nothing to send.
{ label: "Send", dim: !hasText },
{ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey },
{ label: "Discard", dim: false },
];
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
const menuTop = DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
@@ -170,11 +261,11 @@ export class VoiceInputLayer implements Layer {
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
switch (this.phase) {
case "capturing":
if (event.type === "double-click") {
ctx.stack.pop();
this.dismiss();
} else if (this.clickEndsCapture && event.type === "click") {
this.endCapture();
}
@@ -192,37 +283,29 @@ export class VoiceInputLayer implements Layer {
}
return;
case "menu":
this.handleMenuInput(event, ctx);
this.handleMenuInput(event);
return;
}
}
private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void {
private handleMenuInput(event: DashboardInputEvent): void {
const rowCount = this.menuRows().length;
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
this.menuIndex = (this.menuIndex + 1) % rowCount;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
const text = this.displayText().trim();
if (text) this.onSend(text);
ctx.stack.pop();
} else if (this.menuIndex === 1) {
if (anthropicApiKeySetting.get().trim()) {
this.startContinuation();
}
// No key: the row is disabled; the click does nothing.
} else {
ctx.stack.pop();
}
case "click": {
const row = this.menuRows()[this.menuIndex];
row?.onSelect();
return;
}
case "double-click":
ctx.stack.pop();
this.dismiss();
return;
default:
return;
@@ -323,6 +406,11 @@ export class VoiceInputLayer implements Layer {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
}
if (this.autoSendTimer !== null) {
clearTimeout(this.autoSendTimer);
this.autoSendTimer = null;
}
this.pendingAutoSend = false;
this.refineHandle?.cancel();
this.refineHandle = null;
if (this.capturing) {
@@ -380,6 +468,12 @@ export class VoiceInputLayer implements Layer {
this.beginRefine();
return;
}
if (this.pendingAutoSend) {
// The utterance finalized; skip-confirmation can fire without waiting
// out the fallback timer.
this.performAutoSend();
return;
}
} else {
this.liveText = event.text.trim();
}
+7
View File
@@ -294,6 +294,13 @@ export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
defaultValue: false,
});
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
id: "assistant-skip-confirmation",
label: "Send to assistant without confirming",
storageKey: "assistant.skipConfirmationAfterWakeword",
defaultValue: false,
});
export const elevenLabsApiKeySetting = new ConfigSettingString({
id: "elevenlabs-api-key",
label: "ElevenLabs key",
+4
View File
@@ -3,6 +3,7 @@ import { getDashboardLogo } from "../../graphics/logo";
import { shell } from "../shell/shell";
import {
anthropicApiKeySetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
brightnessSetting,
elevenLabsApiKeySetting,
@@ -56,6 +57,9 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
// When on, a wakeword utterance goes straight to the assistant with no
// Send/Type menu step.
toggleSettingMenuItem(assistantSkipConfirmationSetting),
],
},
{
+291 -14
View File
@@ -1,11 +1,17 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/bdffont";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, LayerActions, LayerStack, noopLayerActions } from "../layers";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer } from "../apps/voice-input";
import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input";
import { AssistantLayer } from "../apps/assistant";
import { AssistantSession } from "../../assistant/session";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
anthropicApiKeySetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
onAnySettingChanged,
timeFormatSetting,
@@ -99,6 +105,64 @@ class ShellOverlayMenuLayer extends MenuLayer {
}
}
/** How long a show_alert popup stays before auto-dismissing. */
const ALERT_DISMISS_MS = 6000;
const ALERT_X = 40;
const ALERT_W = G2_LENS_WIDTH - 80;
const ALERT_Y = 96;
const ALERT_H = 96;
/**
* A brief text popup on the shell surface (the assistant's show_alert tool and
* other short notices). Auto-dismisses after a few seconds; a click or
* double-click dismisses it early.
*/
class ShellAlertLayer implements Layer {
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(private readonly text: string, private readonly onDismiss: () => void) {
this.timer = setTimeout(() => {
this.timer = null;
this.onDismiss();
}, ALERT_DISMISS_MS);
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const image = paintBelow();
const font = getDefaultSmallFont();
image.fillRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 1, 10);
image.drawRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 90, 10);
image.drawText(font, ALERT_X + 16, ALERT_Y + 12, "Assistant", 200);
image.drawTextWrapped({
font,
x: ALERT_X + 16,
y: ALERT_Y + 34,
width: ALERT_W - 32,
text: this.text,
value: 235,
});
return image;
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
if (event.type === "click" || event.type === "double-click") {
this.clearTimer();
this.onDismiss();
}
}
onRemoved(): void {
this.clearTimer();
}
private clearTimer(): void {
if (this.timer !== null) {
clearTimeout(this.timer);
this.timer = null;
}
}
}
class Shell {
private windows: ShellWindow[] = [];
private selectedIndex = 0;
@@ -111,6 +175,8 @@ class Shell {
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
@@ -236,6 +302,21 @@ class Shell {
return this.screenOn;
}
/** Current headset battery levels (for the assistant's get_state tool). */
getBatteryLevels(): ShellChromeState["battery"] {
return this.battery;
}
/**
* The foreground app, or null when only the launcher is showing. The launcher
* is a pinned window but counts as "no app" for the assistant's context.
*/
getForegroundApp(): { appId: string; title: string } | null {
const window = this.foregroundWindow();
if (!window || window.appId === "launcher") return null;
return { appId: window.appId, title: window.title };
}
noteUserActivity(nowMs = Date.now()): void {
this.lastInputAtMs = nowMs;
}
@@ -283,7 +364,10 @@ class Shell {
// An open voice dialog suspends the timeout: a long dictation or refine
// has no button presses, but the screen must stay on for it. Sliding
// lastInputAtMs forward also restarts the full timeout when it closes.
if (this.activeVoiceLayer) {
// An in-flight assistant turn suspends it for the same reason (a tool loop
// can run for a while with no input); once the turn ends and the
// Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -358,7 +442,13 @@ class Shell {
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake("sidebar");
if (action === "voice-input" && !this.activeVoiceLayer) {
this.openVoiceDialog(false, true);
if (this.assistantLayer) {
// The assistant overlay is up; a wakeword continues that conversation.
this.startAssistantFollowUp(true);
} else {
// Wakeword defaults the highlight to Send to Assistant.
this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" });
}
}
return { shell: wokeScreen || action === "voice-input", window: false };
}
@@ -489,25 +579,77 @@ class Shell {
next?.requestRender();
}
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
const layer = new VoiceInputLayer(
this.config.actions,
() => {
private openVoiceDialog(options: {
finishOnClick?: boolean;
handsFree?: boolean;
defaultTarget: "assistant" | "app";
}): void {
const targets = this.buildVoiceSendTargets();
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
if (defaultIndex < 0) defaultIndex = 0;
// Skip the menu only for a hands-free (wakeword) capture aimed at the
// assistant, when the user has opted into it.
const autoSend =
Boolean(options.handsFree) &&
options.defaultTarget === "assistant" &&
targets[defaultIndex]?.id === "assistant" &&
assistantSkipConfirmationSetting.get();
const layer = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeVoiceLayer === layer) {
this.activeVoiceLayer = null;
// The idle countdown restarts in full once voice input ends.
this.noteUserActivity();
}
},
(text) => this.sendTextToForegroundWindow(text),
finishOnClick,
handsFree,
);
dismiss: () => {
this.stack.popIfTop((top) => top === layer);
},
sendTargets: targets,
defaultTargetIndex: defaultIndex,
finishOnClick: options.finishOnClick ?? false,
handsFree: options.handsFree ?? false,
autoSend,
});
this.activeVoiceLayer = layer;
this.stack.push(layer);
layer.startCapture();
}
/**
* The send destinations offered by the voice dialog: the assistant (when an
* API key is configured) and/or typing into the foreground window (when it
* accepts text). Order fixes the menu row order.
*/
private buildVoiceSendTargets(): VoiceSendTarget[] {
const targets: VoiceSendTarget[] = [];
if (this.isAssistantAvailable()) {
targets.push({
id: "assistant",
label: "Send to Assistant",
onSend: (text) => this.sendToAssistant(text),
});
}
if (this.foregroundWindow()?.receiveTextInput) {
targets.push({
id: "app",
label: "Type Into App",
onSend: (text) => this.sendTextToForegroundWindow(text),
});
}
// Guarantee at least one destination so the dialog is never a dead end.
if (targets.length === 0) {
targets.push({
id: "app",
label: "Type Into App",
onSend: (text) => this.sendTextToForegroundWindow(text),
});
}
return targets;
}
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
sendTextToForegroundWindow(text: string): void {
this.foregroundWindow()?.receiveTextInput?.(text);
@@ -521,9 +663,130 @@ class Shell {
*/
startVoiceInput(): void {
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
// The transcript is aimed at the window whose menu requested it.
// The transcript is aimed at the window whose menu requested it; the menu
// entry point defaults the highlight to Type Into App.
this.focus = "window";
this.openVoiceDialog(true);
this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" });
this.config.requestShellRender();
}
private isAssistantAvailable(): boolean {
return anthropicApiKeySetting.get().trim().length > 0;
}
private ensureAssistantSession(): AssistantSession | null {
const apiKey = anthropicApiKeySetting.get().trim();
if (!apiKey) return null;
if (!this.assistantSession || this.assistantSession.isExpired()) {
this.assistantSession = new AssistantSession(apiKey);
}
return this.assistantSession;
}
private buildAssistantContext(): AssistantContext {
const foreground = this.getForegroundApp();
return {
foregroundApp: foreground?.appId ?? null,
foregroundTitle: foreground?.title ?? null,
screenOn: this.screenOn,
localTime: formatAssistantTime(new Date()),
headsetBattery: this.battery.headset,
};
}
/**
* Start (or continue) an assistant conversation from a finalized utterance.
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay.
*/
sendToAssistant(text: string): void {
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert("Set an Anthropic API key in Settings to use the assistant.");
return;
}
if (!this.screenOn) this.wake("sidebar");
let layer = this.assistantLayer;
if (!layer) {
const created = new AssistantLayer(this.config.actions, {
onFollowUp: () => this.startAssistantFollowUp(),
onCancel: () => this.assistantSession?.cancel(),
onClose: () => this.closeAssistantLayer(),
onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the reference so a later query starts clean.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
},
});
layer = created;
this.assistantLayer = created;
this.stack.push(created);
}
this.runAssistantTurn(session, layer, text);
this.config.requestShellRender();
}
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar),
onToolActivity: (label) => layer.onToolActivity(label),
onTurnDone: () => layer.onTurnDone(),
onError: (message) => layer.onError(message),
});
}
/**
* Record another utterance in the current assistant conversation. handsFree
* (from a wakeword over the overlay) starts the mic immediately; otherwise
* (the Follow-up menu button) a click ends the utterance.
*/
private startAssistantFollowUp(handsFree = false): void {
const layer = this.assistantLayer;
const session = this.assistantSession;
if (!layer || !session || this.activeVoiceLayer) return;
const voice = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeVoiceLayer === voice) {
this.activeVoiceLayer = null;
this.noteUserActivity();
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === voice);
},
sendTargets: [
{ id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) },
],
finishOnClick: !handsFree,
handsFree,
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
});
this.activeVoiceLayer = voice;
this.stack.push(voice);
voice.startCapture();
this.config.requestShellRender();
}
private closeAssistantLayer(): void {
// Popping fires the layer's onRemoved, which cancels the turn and clears
// this.assistantLayer.
const layer = this.assistantLayer;
if (layer) this.stack.popIfTop((top) => top === layer);
this.noteUserActivity();
this.config.requestShellRender();
}
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
showAlert(text: string): void {
if (!this.screenOn) this.wake("sidebar");
const layer = new ShellAlertLayer(text, () => {
this.stack.popIfTop((top) => top === layer);
this.config.requestShellRender();
});
this.stack.push(layer);
this.config.requestShellRender();
}
@@ -586,6 +849,20 @@ class Shell {
export const shell = new Shell();
const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
/** Human-readable local time for the assistant's per-turn context. */
function formatAssistantTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`;
return `${day}, ${hours}:${minutes} ${meridiem}`;
}
export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
if (event.kind === "sys-event") {
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
+422
View File
@@ -0,0 +1,422 @@
# Voice Assistant Architecture
Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability
ontology, misc fixes). **Phase 1 is implemented** (see the Implementation plan
below); phases 2-4 are still design-only.
## Goal
A default voice assistant reachable from anywhere in the shell: hold the
button, speak, and the utterance goes to an agent that can answer in text on
the lenses and act on the glasses through tools. The agent is pluggable:
- **Direct mode**: the phone runs the agent loop itself against the Anthropic
Messages API (tool-use loop, conversation state on-phone).
- **External mode**: the utterance is routed to the user's own long-running
agent (OpenClaw, Hermes, or anything similar running on their machine), and
that agent gets access to the glasses' tools while it works.
Tools come from two sources, with four availability tiers (see "Tool
availability ontology" below):
- **Glasses/system tools**: provided by the shell, always available (show
text, notifications, screen state, open app, ...).
- **App tools**: contributed by apps. Each app tool declares how much of the
app needs to exist for the tool to be live: merely **installed** ("start a
5 minute timer" must not require the Timer app to be open), **open**
("next track" needs a running Music window, foreground or not), or
**foreground** ("scroll down" only makes sense aimed at the foreground app).
## What we already have
- **PTT capture UI**: `VoiceInputLayer` (app/ui/apps/voice-input.ts) handles
push-to-talk, live transcripts, and a Send/Continue/Discard menu; `Shell`
opens it on long-press and delivers Send to the foreground window's
`receiveTextInput` (app/ui/shell/shell.ts).
- **Wakeword**: On-device classifier listens for the wakeword "Hey Even",
which sends a message to the phone which wakes the screen and starts voice
input.
- **STT**: `voiceControlBridge` with onboard Moonshine, Whisper, or ElevenLabs
cloud.
- **LLM client**: `streamAnthropicMessage` (app/native/anthropic.ts) — raw
SSE streaming over `FaceclawSseRequest`. Text-only today; no tool use.
- **Worker app protocol**: `WorkerAppMessage`/`WorkerAppReply`
(app/ui/shell/worker-window.ts) — small-JSON postMessage protocol between
shell and app workers; the natural place to add tool declarations.
- **Dial-out websocket precedent**: `G2MirrorClient` — JSON over a websocket
to a server on the user's machine (tailscale host/port/token settings).
The external-agent bridge should copy this shape.
## Architecture overview
```
main isolate (shell)
┌───────────────────────────────────────────────────────────┐
│ Shell ──(PTT)──> VoiceInputLayer ──> AssistantSession │
│ │ │
│ AssistantLayer (overlay UI) <──streaming──┤ │
│ │ │
│ AssistantBackend │
│ ┌────────┴─────────┐ │
│ │ Direct │ External│ │
│ └────┬────┴────┬────┘ │
│ │ │ │
│ ToolRegistry <────────────────────────┴─────────┘ │
│ ├─ system tools (in-process handlers) │
│ ├─ app "installed" tools (shell-side or launch-on-call) │
│ └─ app "open"/"foreground" tools (proxied to app worker) │
└──────────┬──────────────────────────────────────┬─────────┘
│ postMessage │
app worker (e.g. terminal) Anthropic API (SSE)
set-tools / tool-call / or agent bridge (WS)
tool-result on the user's machine
```
Everything assistant-side lives on the main isolate. Rationale: tool handlers
need shell state (window list, focus, notifications, settings), the agent
loop is I/O-bound not CPU-bound, and the overlay UI is a shell layer anyway.
If conversation rendering ever gets expensive it can move to a worker later
without changing the tool protocol, which is already async message passing.
### Core pieces
1. **`ToolRegistry`** (new, `app/assistant/tool-registry.ts`)
- Holds `ToolSpec`s and handlers; answers `listTools(scopeFilter)` and
`callTool(name, args) -> Promise<ToolResult>`.
- Emits `onToolsChanged` (needed for MCP `tools/list_changed` in external
mode, and consulted at each turn boundary in direct mode).
2. **`AssistantBackend`** (interface) with two implementations:
- `DirectAnthropicBackend`: tool-use loop over the Messages API.
- `ExternalAgentBackend`: websocket bridge to the user's agent.
3. **`AssistantSession`**: one conversation. Owns history/turn state,
exposes `sendUtterance(text, ctx)`, `cancel()`, and streaming callbacks
(`onTextDelta`, `onToolActivity`, `onTurnDone`, `onError`). Both backends
implement the same session surface so the UI is backend-agnostic.
4. **`AssistantLayer`**: shell overlay showing streamed reply text, a status
line for tool activity ("→ calendar.list_events"), and a small menu
(Follow-up / Done). Double-click cancels the in-flight turn.
## Entry points and routing
- **Wakeword** Saying "Hey Even" wakes the screen if necessary and opens the
dictation dialog. For one-off voice commands, this will be the main entry
point.
- **Context menu** The sidebar context menu and app context menu already have
a "Voice input" option, which opens the dictation dialog whose Send goes to
the foreground window. We'll split the Send option in this dialog into two
options, "Send to Assistant" and "Type Into App". If the entry point was via
context menu, "Type Into App" is highlighted by default; if the entry point
was the wakeword, "Send to Assistant" is highlighted by default instead.
(The capture UI is the existing `VoiceInputLayer` flow, with Send handing off
to `AssistantSession` and morphing into `AssistantLayer`).
## Tool model
```ts
type ToolAvailability =
| "always" // system tools; no app involved
| "installed" // app tool, live whenever the app is installed —
// the app need not have any window open
| "open" // app tool, live while the app has a window open,
// foreground or backgrounded
| "foreground"; // app tool, live only while the app owns the
// foreground window
type ToolSpec = {
name: string; // namespaced: "glasses.show_notification",
// "app.terminal.send_input"
description: string;
inputSchema: object; // JSON Schema
availability: ToolAvailability;
/** May be invoked by the external agent outside an active voice turn. */
proactive?: boolean;
timeoutMs?: number; // default 10s; registry enforces
};
type ToolResult =
| { ok: true; content: string } // text payload back to the model
| { ok: false; error: string };
```
### Tool availability ontology
Every tool sits in exactly one tier, and the tier answers two questions: *when
does the tool appear in `listTools()`*, and *who handles the call*.
| Tier | Live when | Handler | Examples |
|---|---|---|---|
| `always` | always | shell, in-process | `glasses.get_state`, `notifications.list` |
| `installed` | app installed (launcher entry exists) | shell-side handler registered with the app entry, or launch-the-app-then-proxy | `timer.set` ("start a 5 minute timer" must work with no Timer window), `music.play` (cold-start playback) |
| `open` | app has ≥1 open window, foreground or not | app worker (or in-process window) via tool-call proxy | `music.next_track`, `terminal.send_input` |
| `foreground` | app owns the foreground window | app worker via tool-call proxy | `teleprompt.scroll_down`, anything meaning "act on what I'm looking at" |
Mechanics per tier:
- **`installed`** tools cannot be declared by the worker (it may not be
running), so they are declared statically alongside the app's launcher
registration. Two handler flavors: (a) a shell-side handler function
registered with the entry — right for tools that don't actually need the
app UI (timers, starting playback); (b) *launch-on-call*: the shell opens
the app (as if launched from the launcher), waits for its `set-tools`
declaration, then proxies the call. Start with (a) only; (b) adds a
launch-latency/timeout dance that no phase-1/2 tool needs.
- **`open`** and **`foreground`** tools are declared by the running app via
`set-tools` (worker apps) or direct registration (in-process windows).
The registry filters `foreground` tools by whether the declaring app owns
the current foreground window at list time and again at call time —
a `foreground` tool called after a foreground swap fails with a normal tool
error rather than acting on a window the user is no longer looking at.
- A tier is a *floor*, not a routing rule: an `installed` tool stays listed
while the app is open or foreground. If an app wants different behavior when
its window exists (e.g. `timer.set` also shows a countdown in the open
Timer window), the shell-side handler checks for the window itself.
> **Flag:** with multiple windows per app (the worker protocol is
> per-window), tool names are namespaced per *app*, so two windows of one
> app declaring the same tool name would collide. Rule: `set-tools`
> replaces the app's toolset per window, and a call routes to the declaring
> window; if two windows declare the same name, last declaration wins and
> earlier ones are dropped with a logged warning. Revisit only if a real
> app needs per-window tools with the same name (none of the planned ones
> do).
### System tools (initial set)
| Tool | Notes |
|---|---|
| `glasses.show_alert` | short text popup on the lenses; proactive-capable |
| `glasses.list_windows` / `glasses.focus_window` / `glasses.open_app` | shell control |
| `glasses.get_state` | screen on/off, foreground app, battery levels, time |
| `glasses.read_screen` | text/description of the current foreground surface (start with window title + app-reported summary; pixel OCR is out of scope) |
| `calendar.list_events` | wraps FaceclawCalendarProvider |
| `media.play_pause` / `media.next` / `media.now_playing` | wraps FaceclawMediaController |
| `notifications.list` / `notifications.dismiss` | wraps notification listener |
| `timer.set` / `timer.cancel` | new, but trivial and high-value for an assistant |
Keep the first release to ~8–12 tools; every tool costs prompt tokens and
model attention in direct mode.
Note that `calendar.*`, `media.*`, and `timer.*` are conceptually
`installed`-tier app tools that happen to be implemented by in-process shell
code; listing them here just means the shell registers their handlers
directly. If any of them later becomes a real app, its tools move to the
app's launcher registration without the model-visible surface changing.
### App tools
Running apps declare `open`/`foreground` tools per window over the existing
worker protocol (`installed` tools are declared statically with the launcher
entry, per the ontology above):
```ts
// WorkerAppReply additions (worker -> shell)
| { type: "set-tools"; windowId: string; tools: ToolSpec[] } // replaces the set
| { type: "tool-result"; callId: string; result: ToolResult }
// WorkerAppMessage additions (shell -> worker)
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown }
```
Registry rules:
- Liveness follows the availability tier: `open` tools are removed when the
app's last window closes, `foreground` tools additionally drop out of
`listTools()` when the app loses the foreground. Either transition fires
`onToolsChanged`; in-flight calls to a closed window resolve as errors,
and calls to a `foreground` tool are re-checked at dispatch time.
- Names are auto-prefixed `app.<appId>.` by the registry so apps can't
shadow system tools or each other.
- The shell enforces `timeoutMs` on the postMessage round-trip; a hung
worker yields a tool error, not a hung turn (same philosophy as input
handling — the shell must survive a stuck app).
- In-process shell apps (calendar, music, ...) register handlers directly
with the registry; the worker protocol is just the remote flavor of the
same registration.
- **Mid-turn availability changes** (foreground swap, window close): direct
mode re-lists tools at each loop iteration (tool set is per-API-call
anyway); external mode sends MCP `notifications/tools/list_changed`. A
call to a tool that just vanished returns a normal tool error the model
can react to.
First app to wire up: the terminal —
`app.terminal.send_input` (type into the attached session; `open`,
proactive: false — "rerun the build" should work while the terminal is
backgrounded), `app.terminal.read_screen` (current grid contents; `open`),
`app.terminal.list_sessions` (`open`). This immediately enables "tell the
terminal to rerun the build" style commands and doubles as the reference
implementation — deliberately all `open`-tier; the first `foreground` tool
should come from an app where focus genuinely matters (teleprompt scrolling).
## Direct mode: agent loop on the phone
Extend `app/native/anthropic.ts`:
1. **Streaming tool-use support**: handle `content_block_start` with
`tool_use`, accumulate `input_json_delta`, surface complete tool_use
blocks alongside text deltas. (Today the client only reads `text_delta`.)
2. **`runAgentTurn` loop** (new, `app/assistant/direct-backend.ts`):
messages + `tools` from the registry → stream → if `stop_reason ==
"tool_use"`, execute each call via the registry, append `tool_result`
blocks, continue; until `end_turn`/`refusal` or a safety cap
(max 8 loop iterations, max ~2 min per turn).
3. **Session state**: history kept in memory per `AssistantSession`,
trimmed from the head past a token budget. Sessions idle-expire
(~10 min) so a new PTT starts fresh; "Follow-up" continues the session.
4. **System prompt**: glasses context — output renders on a 576×288
monochrome HUD, so answer in 1–3 short sentences, no markdown, no lists
unless asked; prefer acting via tools over describing; current time,
foreground app, and screen state are injected per turn.
5. Model `claude-sonnet-5`, `effort: "low"` (same latency posture as the
dictation refiner). Model/effort become settings later if needed.
## External mode: bridge to the user's agent
### Topology
The phone **dials out** (g2mirror pattern — phones don't accept inbound
connections reliably; tailscale host + port + token settings) to a small
**bridge server** co-located with the user's agent. One websocket, JSON
frames, three multiplexed channels:
```
{ v: 1, chan: "ctl" | "chat" | "mcp", ... }
ctl: hello {deviceName, token, capabilities}, hello-ack, ping/pong, error
chat: utterance {turnId, text, ctx} (phone -> agent)
text-delta {turnId, text} (agent -> phone)
tool-activity {turnId, label} (agent -> phone, optional status)
turn-done {turnId, stopReason} / turn-error {turnId, message}
cancel {turnId} (phone -> agent)
mcp: { msg: <raw MCP JSON-RPC frame> } (bidirectional)
```
On the `mcp` channel the **phone is an MCP server** (MCP roles come from the
initialize handshake, not from who dialed). It serves `tools/list` from the
ToolRegistry, `tools/call` into it, and emits `tools/list_changed` on
foreground changes. This means the agent-side integration needs zero
Faceclaw-specific tool code — the glasses appear as a normal MCP server.
### Agent-side adapters
The bridge server is a small standalone package (`faceclaw-agent-bridge`,
sibling of g2mirror) with two faces:
1. **MCP face**: re-exposes the phone's MCP channel as a localhost
streamable-HTTP MCP endpoint. The user registers
`http://localhost:<port>/mcp` with their agent (OpenClaw, Hermes, Claude
Code, anything MCP-capable) exactly like any other MCP server. When the
phone is disconnected, `tools/list` returns empty rather than erroring.
2. **Chat face**: pluggable per-agent adapter that forwards `utterance`
frames into the agent and streams replies back:
- **OpenClaw**: a plugin that injects the utterance as an inbound message
on a dedicated "glasses" channel/session and relays the streamed reply.
- **Hermes**: equivalent module against its session API.
- **Fallback adapter**: spawn a configured CLI (e.g. `claude -p --resume`)
per turn — lowest-fidelity but makes the bridge useful with no plugin.
Utterance `ctx` carries `{foregroundApp, screenOn, localTime}` so the remote
agent has the same situational grounding the direct loop gets in its system
prompt.
### Proactive agent actions
Because the MCP channel is live whenever the bridge is connected, the
external agent can call glasses tools **outside a voice turn** — e.g. push a
notification to the lenses when a long job finishes. Gating:
- Only tools marked `proactive: true` are callable outside an active turn
(registry-enforced; enforced on the phone, never trusted to the bridge).
Availability tiers apply to proactive calls unchanged: a proactive
`installed` tool (e.g. `timer.set`) works anytime, while a proactive call
to an `open`/`foreground` tool whose app isn't in the right state gets the
same tool error a mid-turn call would.
- Master setting `assistant.allowProactive` (default on — it's a marquee
feature — but visible and easy to turn off).
- Proactive display actions are rate-limited (e.g. 6/min) and never turn
the screen on unless the user enabled that specifically.
## Settings
New "Assistant" section in dashboard settings:
- `assistant.backend`: `anthropic` | `external` (default `anthropic` if an
API key is set, else `external` if a bridge host is set).
- `assistant.bridgeHost` / `assistant.bridgePort` / `assistant.bridgeToken`
(mirrors the terminal.* trio).
- `assistant.allowProactive`: boolean.
- `voice.skipConfirmationAfterWakeword`: Wakeword-triggered input sends to the
assistant immediately at the end of an utterance, rather than waiting for a
menu selection.
- Reuses existing `llm.anthropicApiKey` for direct mode.
## Security notes
- Bridge auth: bearer token in the `hello` frame, TLS optional because the
expected transport is tailscale (same stance as g2mirror); document that
plainly.
- The phone enforces all tool gating (proactive flags, rate limits,
timeouts). The bridge and agent are treated as honest-but-fallible.
- Tool results may contain private data (calendar, notifications); external
mode ships them to the user's own machine only — no third-party hop. In
direct mode they go to the Anthropic API like any other request content.
- App tools execute inside the app's worker with the app's existing
privileges; the registry adds no new capability beyond what the app could
already do.
## Implementation plan
**Phase 1 — direct assistant, system tools only.** ✅ Landed 2026-07-24
(builds + typechecks; not yet hardware-tested).
ToolRegistry (`app/assistant/tool-registry.ts`) with the full availability-tier
model but only `always` tools wired; 8 system tools (`glasses.get_state`,
`glasses.show_alert`, `calendar.list_events`, `media.now_playing`/`play_pause`/
`next`, `notifications.list`/`dismiss`) in `app/assistant/system-tools.ts`;
tool-use streaming in `anthropic.ts` (content-block assembly, `tools` param);
`DirectAnthropicBackend` + `AssistantSession`; `AssistantLayer` overlay
(`app/ui/apps/assistant.ts`); `VoiceInputLayer` reworked to route via a
`sendTargets` list ("Send to Assistant" / "Type Into App") with entry-point
default highlight and wakeword skip-confirmation auto-send;
`shell.showAlert` popup; `assistant.skipConfirmationAfterWakeword` setting.
*Deliverable: speak wakeword, ask "what's on my calendar", get an answer
on-lens.*
**Phase 2 — app tools.**
Worker protocol extension (set-tools / tool-call / tool-result),
availability-tier filtering in the registry (`open` on window open/close,
`foreground` on foreground change), `installed`-tier registration on launcher
entries (shell-side handlers only; launch-on-call deferred), terminal app
tools as the reference. *Deliverable: "run the build again" typed into the
terminal even while it's backgrounded.*
**Phase 3 — external bridge.**
Bridge wire protocol; ExternalAgentBackend (phone side, reusing the
FaceclawWebSocket plumbing); `faceclaw-agent-bridge` server with the MCP
face + fallback CLI chat adapter; settings; reconnect/offline behavior.
*Deliverable: utterances answered by the user's own agent, which can also
call glasses tools mid-task.*
**Phase 4 — OpenClaw/Hermes adapters + proactive polish.**
Native chat adapters for both agents; proactive gating + rate limits;
`tools/list_changed` on foreground swap; session persistence across
reconnects.
## Open questions (deferred, not blockers):
- Continuous conversation: after a turn ends, listen for a follow-up
utterance without requiring the wakeword/PTT again? There are some UI
design questions here; maybe multi-turn conversations take the form of an
Assistant app.
- Launch-on-call for `installed`-tier tools that genuinely need the app UI
(open the app, await `set-tools`, proxy the call) — deferred until a tool
needs it.
- `glasses.read_screen` fidelity beyond app-reported summaries? We probably
don't want to do OCR, but we might do something like extending the draw-text
draw call with an on-by-default option that also appends the drawn text to
a string that gets submitted along with the frame.
- Whether assistant conversations should be reviewable in the phone UI
(transcript log) — probably yes, cheap once sessions are objects. The
natural UI home for this would be inside an Assistant app window.