Add app tool registration for voice assistant; voice assistant can read and send to terminals

This commit is contained in:
James Babcock committed 2026-07-25 01:19:07 -07:00
1 parent 0aeb24ed00
commit fe205c0164
9 files changed
+533 -16

No files matched your search

+7 -2
View File
@@ -26,7 +26,12 @@ export type DirectTurnOptions = {
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: AnthropicMessage[];
tools: AnthropicToolDefinition[];
/**
* Build the tool list for the next request. Called once per loop iteration so
* the set reflects apps opening/closing and foreground changes mid-turn; it
* also refreshes the session's api-name map that resolveToolName reads.
*/
buildTools: () => AnthropicToolDefinition[];
registry: ToolRegistry;
/** Map an API tool name (as the model calls it) back to its registry name. */
resolveToolName: (apiName: string) => string;
@@ -62,7 +67,7 @@ export class DirectAnthropicBackend {
system: options.system,
effort: options.effort,
messages: options.messages,
tools: options.tools,
tools: options.buildTools(),
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar);
+1 -2
View File
@@ -60,7 +60,6 @@ export class AssistantSession {
this.messages.push({ role: "user", content: text });
this.trimHistory();
const tools = this.buildToolDefinitions();
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
const finish = () => {
@@ -74,7 +73,7 @@ export class AssistantSession {
effort: "low",
system,
messages: this.messages,
tools,
buildTools: () => this.buildToolDefinitions(),
registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: {
+62 -2
View File
@@ -70,14 +70,28 @@ export type CallToolOptions = {
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
/** How the shell reaches a window's tool handler and gates its foreground tools. */
export type AppToolProvider = {
windowId: string;
appId: string;
/** The window's declared specs, with UNPREFIXED names (registry adds the prefix). */
specs: ToolSpec[];
/** Invoke a tool on the window by its unprefixed name. */
invoke: (toolName: string, args: unknown) => Promise<ToolResult> | ToolResult;
/** Whether the window currently owns the foreground (gates `foreground` tools). */
isForeground: () => boolean;
};
export class ToolRegistry {
private readonly registrations = new Map<string, ToolRegistration>();
private readonly changeListeners = new Set<() => void>();
// windowId -> the canonical (prefixed) tool names it contributed, for bulk
// removal when the window closes or re-declares.
private readonly windowToolNames = new Map<string, string[]>();
/** Register (or replace) a tool. Names are unique across all tiers. */
register(registration: ToolRegistration): void {
const { name } = registration.spec;
this.registrations.set(name, registration);
this.addRegistration(registration);
this.fireToolsChanged();
}
@@ -95,6 +109,52 @@ export class ToolRegistry {
}
}
/**
* Declare (replacing any prior set) the tools contributed by one app window.
* Names are prefixed `app.<appId>.` so windows can't shadow system tools or
* each other. `open` tools stay live while the window exists; `foreground`
* tools are gated on `isForeground()` at both list and call time.
*/
setAppTools(provider: AppToolProvider): void {
this.clearWindowTools(provider.windowId, false);
const names: string[] = [];
for (const spec of provider.specs) {
if (spec.availability !== "open" && spec.availability !== "foreground") {
console.warn(`app tool ${spec.name} has non-app availability ${spec.availability}; skipping`);
continue;
}
const canonical = `app.${provider.appId}.${spec.name}`;
const toolName = spec.name;
this.addRegistration({
spec: { ...spec, name: canonical },
handler: (args) => provider.invoke(toolName, args),
isAvailable: spec.availability === "foreground" ? () => provider.isForeground() : undefined,
});
names.push(canonical);
}
this.windowToolNames.set(provider.windowId, names);
this.fireToolsChanged();
}
/** Remove all tools contributed by a window (its worker closed or the window did). */
removeAppTools(windowId: string): void {
this.clearWindowTools(windowId, true);
}
private clearWindowTools(windowId: string, fire: boolean): void {
const names = this.windowToolNames.get(windowId);
if (!names) return;
for (const name of names) {
this.registrations.delete(name);
}
this.windowToolNames.delete(windowId);
if (fire) this.fireToolsChanged();
}
private addRegistration(registration: ToolRegistration): void {
this.registrations.set(registration.spec.name, registration);
}
/** Specs for the tools callable right now, given the availability tiers. */
listTools(options: ListToolsOptions = {}): ToolSpec[] {
const specs: ToolSpec[] = [];
+85 -1
View File
@@ -1,6 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { windowIcon } from "./chrome-layer";
import { type IconName } from "../../graphics/icons";
import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/tool-registry";
import { shell, type ShellWindow } from "./shell";
/**
@@ -16,7 +17,9 @@ export type WorkerAppMessage =
| { type: "text-input"; windowId: string; text: string }
| { type: "render"; windowId: string; focused: boolean }
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
| { type: "screen"; on: boolean };
| { type: "screen"; on: boolean }
/** Assistant tool invocation aimed at a window; reply with tool-result. */
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
export type WorkerAppReply =
| { type: "yield-focus"; windowId: string }
@@ -59,6 +62,21 @@ export type WorkerAppReply =
*/
type: "set-tray-icon";
icon: { width: number; height: number; pixels: number[] } | null;
}
| {
/**
* Declare (replacing the prior set) the assistant tools this window
* contributes. Names are unprefixed; the registry adds `app.<appId>.`.
*/
type: "set-tools";
windowId: string;
tools: ToolSpec[];
}
| {
/** Result of a tool-call, matched to the request by callId. */
type: "tool-result";
callId: string;
result: ToolResult;
};
export type WorkerWindowSpec = {
@@ -92,8 +110,24 @@ export type WorkerAppHostOptions = {
* and manages compositor surfaces for the app's windows. The worker submits
* frames straight to the Java compositor, so no pixels cross this boundary.
*/
/** A tool-call awaiting its worker reply; also its own leak-safety timeout. */
type PendingToolCall = {
windowId: string;
resolve: (result: ToolResult) => void;
timer: ReturnType<typeof setTimeout>;
};
/**
* How long the host waits for a worker's tool-result before giving up. The
* registry applies its own (usually shorter) per-tool timeout; this is the
* backstop that also frees the pending-call entry if the worker never replies.
*/
const TOOL_CALL_HOST_TIMEOUT_MS = 15_000;
export class WorkerAppHost {
private readonly openWindows = new Set<string>();
private readonly pendingToolCalls = new Map<string, PendingToolCall>();
private nextCallSerial = 1;
constructor(private readonly options: WorkerAppHostOptions) {
options.worker.onmessage = (event: MessageEvent) => {
@@ -152,6 +186,27 @@ export class WorkerAppHost {
case "set-title":
// Titles are informational for now (sidebar shows icons only).
break;
case "set-tools":
// Only a window we actually have open may contribute tools.
if (this.openWindows.has(message.windowId)) {
toolRegistry.setAppTools({
windowId: message.windowId,
appId: this.options.appId,
specs: message.tools,
invoke: (toolName, args) => this.callWindowTool(message.windowId, toolName, args),
isForeground: () => shell.foregroundWindow()?.windowId === message.windowId,
});
}
break;
case "tool-result": {
const pending = this.pendingToolCalls.get(message.callId);
if (pending) {
clearTimeout(pending.timer);
this.pendingToolCalls.delete(message.callId);
pending.resolve(message.result);
}
break;
}
}
};
options.worker.onerror = (error) => {
@@ -181,6 +236,9 @@ export class WorkerAppHost {
closeable: true,
close: () => {
this.openWindows.delete(spec.windowId);
// Withdraw this window's tools and fail any in-flight calls to it.
toolRegistry.removeAppTools(spec.windowId);
this.failPendingToolCallsFor(spec.windowId);
this.post({ type: "close-window", windowId: spec.windowId });
this.options.removeSurface(surfaceId);
},
@@ -235,6 +293,32 @@ export class WorkerAppHost {
return window;
}
/**
* Post a tool-call to the worker and resolve when its tool-result comes back.
* Resolves with a tool error on timeout so a hung worker yields a tool error
* rather than a stuck assistant turn.
*/
private callWindowTool(windowId: string, toolName: string, args: unknown): Promise<ToolResult> {
return new Promise<ToolResult>((resolve) => {
const callId = `${this.options.appId}:${this.nextCallSerial++}`;
const timer = setTimeout(() => {
this.pendingToolCalls.delete(callId);
resolve({ ok: false, error: `App ${this.options.appId} did not respond to ${toolName}` });
}, TOOL_CALL_HOST_TIMEOUT_MS);
this.pendingToolCalls.set(callId, { windowId, resolve, timer });
this.post({ type: "tool-call", callId, windowId, name: toolName, args });
});
}
private failPendingToolCallsFor(windowId: string): void {
for (const [callId, pending] of this.pendingToolCalls) {
if (pending.windowId !== windowId) continue;
clearTimeout(pending.timer);
this.pendingToolCalls.delete(callId);
pending.resolve({ ok: false, error: "The target window was closed" });
}
}
private post(message: WorkerAppMessage): void {
this.options.worker.postMessage(message);
}
+123
View File
@@ -39,6 +39,7 @@ import {
} from "../ui/menu";
import { defaultWindowMenuItems, WindowMenu } from "../ui/window-menu";
import type { WorkerAppMessage, WorkerAppReply } from "../ui/shell/worker-window";
import type { ToolResult, ToolSpec } from "../assistant/tool-registry";
declare const global: any;
declare const com: any;
@@ -102,7 +103,43 @@ type TerminalWindow = HubWindow | ViewWindow;
const windows = new Map<string, TerminalWindow>();
const pendingViews = new Map<string, { socket: string; label: string }>();
// The view an assistant tool acts on when the terminal isn't foregrounded:
// the last view to be foregrounded or receive input.
let activeViewId: string | null = null;
let nextViewSerial = 1;
/**
* Assistant tools this app contributes, declared on the hub window (which the
* launcher opens and which persists for the app's life). All `open`-tier so
* "rerun the build" works while the terminal is backgrounded. send_input and
* read_screen act on the active view session (see resolveActiveView).
*/
const TERMINAL_TOOLS: ToolSpec[] = [
{
name: "list_sessions",
description: "List the live g2mirror terminal sessions the glasses can see.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
availability: "open",
},
{
name: "send_input",
description:
"Type a line into the active terminal session and submit it (as if typed and Enter pressed). Use to run a command in the terminal.",
inputSchema: {
type: "object",
properties: { text: { type: "string", description: "The line to type and run." } },
required: ["text"],
additionalProperties: false,
},
availability: "open",
},
{
name: "read_screen",
description: "Return the current visible contents of the active terminal session's screen.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
availability: "open",
},
];
let viewportWidth = 0;
let viewportHeight = 0;
let gridCols = 0;
@@ -161,6 +198,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
if (!window) break;
window.foreground = message.foreground;
window.focused = message.focused;
if (window.foreground && window.kind === "view") activeViewId = window.windowId;
if (window.foreground) renderAndSubmit(window, 0);
break;
}
@@ -172,6 +210,19 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
}
break;
case "tool-call": {
const callId = message.callId;
Promise.resolve(handleTerminalTool(message.name, message.args))
.then((result) => post({ type: "tool-result", callId, result }))
.catch((error) =>
post({
type: "tool-result",
callId,
result: { ok: false, error: String((error as Error)?.message ?? error) },
}),
);
break;
}
}
};
@@ -209,6 +260,8 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb
selectedIndex: 0,
scrollRow: 0,
});
// The hub carries the app's assistant tools; declare them once it exists.
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
if (!controlClient) {
startControlClient();
}
@@ -224,6 +277,7 @@ function closeWindow(windowId: string): void {
window.client.stop();
}
windows.delete(windowId);
if (activeViewId === windowId) activeViewId = null;
if (window.kind === "view") {
renderHubWindows();
}
@@ -414,6 +468,7 @@ function windowMenuItems(window: TerminalWindow): MenuItem[] {
}
function handleInput(window: TerminalWindow, event: DashboardInputEvent, frameId: number): void {
if (window.kind === "view") activeViewId = window.windowId;
// An open window menu owns all input (it closes itself via pop).
if (window.menu?.isOpen()) {
window.menu
@@ -743,3 +798,71 @@ function sessionLabel(session: G2MirrorSession): string {
const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/");
return hint || "session";
}
/** Dispatch an assistant tool-call (unprefixed name) to its handler. */
function handleTerminalTool(name: string, args: any): ToolResult {
switch (name) {
case "list_sessions":
return toolListSessions();
case "send_input":
return toolSendInput(args);
case "read_screen":
return toolReadScreen();
default:
return { ok: false, error: `Unknown terminal tool: ${name}` };
}
}
function toolListSessions(): ToolResult {
const sessions = controlState?.sessions ?? [];
if (!sessions.length) {
return { ok: true, content: "No live terminal sessions (run g2mirror <command> on the host)." };
}
const lines = sessions.map(
(session) => `- ${sessionLabel(session)}${viewWindowIdForSocket(session.socket) ? " [open]" : ""}`,
);
return { ok: true, content: lines.join("\n") };
}
function toolSendInput(args: any): ToolResult {
const text = String(args?.text ?? "");
if (!text) return { ok: false, error: "send_input requires non-empty text." };
const view = resolveActiveView();
if (!view) return { ok: false, error: "No terminal session is open to send input to." };
view.client.submitInput(text);
return { ok: true, content: `Sent to ${view.label}.` };
}
function toolReadScreen(): ToolResult {
const view = resolveActiveView();
if (!view) return { ok: false, error: "No terminal session is open." };
const length = view.emulator.bufferLength();
const start = Math.max(0, length - gridRows);
const lines: string[] = [];
for (let index = start; index < length; index++) {
lines.push(view.emulator.lineAt(index));
}
const text = lines.join("\n").replace(/\s+$/, "");
return { ok: true, content: text || "(screen is empty)" };
}
/**
* The terminal view an `open`-tier tool acts on: a foregrounded view if any,
* else the last view to be active, else the sole open view. Null when no view
* is open (the model gets a tool error it can relay).
*/
function resolveActiveView(): ViewWindow | null {
const views: ViewWindow[] = [];
let foreground: ViewWindow | null = null;
for (const window of windows.values()) {
if (window.kind !== "view") continue;
views.push(window);
if (window.foreground) foreground = window;
}
if (foreground) return foreground;
if (activeViewId) {
const window = windows.get(activeViewId);
if (window && window.kind === "view") return window;
}
return views.length === 1 ? views[0]! : null;
}
+87
View File
@@ -0,0 +1,87 @@
# Even-Analogous Apps
Ideally we'd like full feature parity with the stock Even app (or at least,
close enough that not-using the stock app doesn't feel like a sacrifice).
A rough list of things the stock app has that we currently don't:
## Quicklist
Quicklist is Even's version of a todo app, in the form of a dashboard card
that you add to and remove from with assistant voice commands. It's pretty
simple to build once a general voice assistant is in place, but I don't
actually like their design: a todo list really wants to be synchronized
with something external like Roam or Notion, so I think it'd be better to
build one of those integrations instead.
## Stocks, News
Even has a dashboard card which is a stock ticker. I think this is probably
misguided; while there do exist people who would want and benefit from a stock
tracker on their glasses, I do'm not sure there exist people who would want and
benefit from a stock tracker that isn't a professional-tier tool connected
to carefully chosen low-latency data sources, and neither I (nor Even) can
build that tool.
The News dashboard card is more broadly useful, but has a similar issue: a
generic news ticker isn't really useful without a much higher degree of
customization. Probably the right thing to build instead is more like an RSS
reader.
## Ring Connection Management
The stock Even app is responsible for pairing the glasses with the ring (ie,
sending the glasses a configuration messages telling it to connect to the
ring's MAC address). If you pair them in the stock app and then switch to a
custom app, they remain paired indefinitely (this configuration is stored in
flash), and our custom app can make use of the fact that they're paired.
We don't yet have support for connecting or disconnecting the glasses and ring
to/from each other. For latency reasons, it would be nice if we could unpair
the glasses from the ring and connect the phone to it directly; we don't yet
support that either.
## Ring Health Metrics
The R1 ring contains an oximeter-style LED sensor and an accelerometer, which
can be used for a step count, heart rate measurement, etc. I'm not sure
whether the sensor interpretation is done within the firmware (in which case
it would make sense to display them in the UI) or in the Even phone app (in
which case we'd be reimplementing all the interpretation algorithms, which
isn't worth it.
## Navigate
Even has a Navigate app, which is basically Google Maps for the specific use
case of "name a destination, get turn by turn directions to there". I believe
this is using Mapbox, which takes care of most of the heavy lifting (if you
have an API key). I don't think Even's version is very good (they ran into
trouble with image-upload performance preventing a good map display size), so
it's likely not too hard to do better. One caveat is the location permission;
Android normally only wants to prompt for location permission "while using the
app", but Android's idea of in use (phone screen unlocked and window focused)
doesn't match our idea of in use (G2 glasses on, phone could be locked). This
app also gets a lot better if we figure out how to use the device's
magnetometer (if it indeed has a working magnetometer).
## Distance/Viewport Configuration
During glasses onboarding, the stock firmware has you select a vertical offset
for the screen image. This accounts for glasses resting at different positions
on your face (the optics don't let you see the whole screen area), but cuts
the usable screen height from 480px to 288px. There is also a "distance"
config option in the stock Android app, which adjusts the horizontal position
of the image on the two lenses separately to create the impression of the
image being closer or farther; this cuts the usable screen width from 640px to
576px. Unlike the vertical size reduction, this 64px reduction is not
optically necessary.
## Even AI
## Translate and Conversate
## Other Configuration
Brightness, head-up trigger
# Other Apps
+13
View File
@@ -0,0 +1,13 @@
# Music: "Resume" action via media-key dispatch
Follow-up to the MediaBrowserService browsing support. When no media session
is active, `AudioManager.dispatchMediaKeyEvent(KEYCODE_MEDIA_PLAY)` (down+up
pair) is routed by the system to the *last* media app's manifest-declared
`MEDIA_BUTTON` receiver, even if that app's process is dead — the same
mechanism a Bluetooth headset play button uses. Most players respond by
resuming the last queue.
Idea: add a "Resume last" row to the Music app's no-active-session screen
(next to the browse entry). ~5 lines in `FaceclawMediaController` plus one UI
row; no per-app negotiation needed. Caveat: it's fire-and-forget — no feedback
if nothing handles it, so the UI should just wait for a session to appear.
+133
View File
@@ -0,0 +1,133 @@
# MentraOS app compatibility for Faceclaw — feasibility assessment (2026-07-23)
MentraOS: https://github.com/Mentra-Community/MentraOS (MIT), docs at https://docs.mentraglass.com.
## TL;DR
Technically feasible with moderate effort, but the timing is awkward: MentraOS is
**deprecating its entire current app API**. MentraOS 3.0 ships **Aug 3, 2026** and
cloud-SDK apps stop working on it (a "MentraOS Legacy" mode preserves them through
**October 2026**). The replacement — a **Miniapp SDK** where apps are phone-local JS
bundles running in a WebView — gets a public spec in **September 2026**, and that model
is a *much* better fit for Faceclaw than the cloud model. Recommendation: wait for the
miniapp spec and target that; only build a minimal cloud-protocol emulator if access to
the existing app corpus is wanted in the interim.
## MentraOS app model (current, "Cloud SDK" / TPA era)
- Apps are developer-hosted server processes (Node/Bun, `@mentra/sdk`). Chain:
glasses → phone (BLE) → MentraOS Cloud (WS) → app server (WS).
- Start flow: cloud POSTs a `session_request` webhook `{sessionId, userId, websocketUrl}`
to the app's registered URL; the app dials back to `websocketUrl` and sends
`tpa_connection_init {packageName, sessionId, apiKey}`; cloud acks with settings +
device capabilities. **The webhook carries the WS URL, so an unmodified `@mentra/sdk`
app can be pointed at any server** — that is the compat hook.
- Protocol: JSON over WebSocket (binary frames for audio). 28 app→cloud + 28 cloud→app
message types, defined in `cloud/packages/sdk/src/types/` (`message-types.ts`,
`streams.ts`, `layouts.ts`, `webhooks.ts`, `capabilities.ts`). The TS types are the spec.
- Subscriptions: `subscription_update` with a full list of stream types, some
parameterized (`transcription:en-US`, `touch_event:forward_swipe`). Events arrive in a
`data_stream` envelope.
- Display: 7 layout types — `text_wall`, `double_text_wall`, `dashboard_card`,
`reference_card`, `bitmap_view` (base64), `bitmap_animation`, `clear_view` — sent as
`display_event {view: 'main'|'dashboard', layout, durationMs?, forceDisplay?}`.
Cloud throttles displays to 1 per 300 ms. Dashboard = shared look-up surface.
- Streams: `transcription` (`{text, isFinal, ...}`), `translation`, `VAD`, `audio_chunk`
(PCM), `button_press`, `touch_event`, `head_position` (up/down), `glasses_battery_update`,
`phone_battery_update`, `location_update`, `phone_notification`(+dismissed),
`calendar_event`, plus camera/RTMP/LED families irrelevant to G2.
- Other: ElevenLabs TTS (`audio.speak`), per-app settings (schema in dev console, 9
setting types, live `settings_update`), webview UI with signed-token auth, cloud KV
storage, app-to-app messaging.
- Hardware abstraction: `Capabilities` flags per device. **Even Realities G2 is a
first-class device** (`capabilities/even-realities-g2.ts`): display 640×200 (their
rendering canvas, not the 576×288 EvenHub raster), maxTextLines 5, bitmap-capable,
capacitive swipe bar with TAP/DOUBLE_TAP/TRIPLE_TAP/PRESS_HOLD/SWIPE_UP/SWIPE_DOWN,
IMU, mic, no camera/speaker/LED.
- Licensing: MIT throughout; cloud is self-hostable in principle (Bun + MongoDB + S3 +
third-party STT/TTS keys) but heavy.
## Coming model (Miniapp SDK, MentraOS 3.0)
Miniapps are JS bundles (`index.html` + `miniapp.json`) running **on the phone** inside
the MentraOS host app's WebView ("Mentra Runtime"), packages `@mentra/miniapp` +
`@mentra/miniapp-cli`, sideloaded via QR. API is analogous but smaller:
`session.transcription`, `session.display.showTextWall`, `session.storage`. Example:
https://github.com/Mentra-Community/LiveCaptionsLocalSdkMiniapp. Public spec Sept 2026.
## Fit with Faceclaw
Faceclaw's worker-per-app architecture is unusually favorable: apps are already isolated
contexts speaking a small typed JSON vocabulary (`WorkerAppMessage`/`WorkerAppReply` in
`app/ui/shell/worker-window.ts`), with pixels out-of-band. The compat layer would be one
compiled-in "MentraOS host" app (a peer of `WorkerAppHost`) that presents each external
app as a `ShellWindow` and:
- **Rasterizes layouts** → trivial. All 7 layout types map onto `GrayImage` +
`drawTextWrapped`/`bitBlt` into the 540×260 `APP_VIEWPORT`. Bitmaps need scaling from
their 640×200/576×136 assumptions; monochrome already matches.
- **Maps input** → good. `DashboardInputEvent` (`click`/`double-click`/`long-press`/
`scroll-up`/`scroll-down`) covers their G2 button-event list almost 1:1 (TRIPLE_TAP
missing). Work item: input is currently routed only to the focused window; MentraOS
wants multi-subscriber fan-out (background apps), so a broadcast layer is needed.
- **Transcription** → the load-bearing stream, and Faceclaw already has it:
`voiceControlBridge.onTranscript` (`{text, isFinal}`, replace semantics — matches
MentraOS's `TranscriptionData` model well). Onboard Moonshine is English-only;
parameterized languages would fall back to ElevenLabs/Whisper cloud providers.
Needs multi-consumer fan-out and a policy for when the mic is open (MentraOS keeps it
effectively continuous; Faceclaw is PTT/wakeword-oriented — battery implications).
- **VAD** → sherpa endpointing exists. **audio_chunk** → PCM already surfaced on the
ElevenLabs path (`onPcm`).
- **Battery / phone notifications / calendar** → data already flows in Faceclaw
(top-bar battery, notification listener, calendar app); just needs re-emission as
streams.
- **head_position** → gap. Faceclaw exposes raw IMU only (no gesture classification);
the look-up dashboard gesture would need a small accelerometer classifier (cmd 19
stream exists, see imu-accelerometer memory).
- **TTS / audio out** → G2 has no speaker; MentraOS plays through the phone. Faceclaw
would need phone-side audio playback (new but simple Android work).
- **Settings** → MentraOS per-app settings map onto the shared `FaceclawSettings` store
with a `mentra.<packageName>.*` prefix; the 9 setting types map onto
`ConfigSetting` subclasses, mostly.
- **Not applicable on G2**: camera/photo/RTMP/LiveKit, LED, WiFi setup — can be
rejected via capabilities, which the protocol explicitly supports.
## The hard parts
1. **Who plays "cloud"?** Cloud-SDK apps dial out to a WS URL given in a webhook, so
Faceclaw must (a) run a WS **server** — none exists anywhere in the codebase today,
all networking is outbound okhttp client (`FaceclawWebSocket`, g2mirror pattern) —
reachable from wherever the app server runs (fine on LAN; NAT pain otherwise), and
(b) send the session webhook itself (simple outbound HTTP POST). A phone-hosted
mini-cloud with Java-WebSocket/NanoWSD listening on LAN is the sane MVP; self-hosting
their actual cloud (Bun+Mongo+Docker) is overkill.
2. **Session lifecycle**: connection init/ack, `subscription_update` bookkeeping,
`reconnect`/`reconnect_ack/rejected/deferred`, `app_stopped`, per-app permission
gating. This is bookkeeping-heavy but not deep.
3. **Display arbitration**: foreground vs `background`+`forceDisplay` apps, `durationMs`
auto-revert, the shared dashboard view. Faceclaw's shell has focus/foreground but no
equivalent of background-app display preemption or a look-up dashboard.
4. **App discovery/registration UX**: no dev console; needs a simple "add app by URL +
API key" settings flow (API key check can be a no-op or shared secret).
## Effort estimate
MVP covering the bulk of display-glasses apps (live captions, teleprompters, dashboards,
notification relays): ~10–15 message types + ~8 stream types + webhook sender + LAN WS
server + layout rasterizer + settings mapping. Roughly a few weeks of focused work, with
transcription-stream policy and the WS server being the riskiest pieces. Full-surface
emulation (webview auth, app messaging rooms, TTS, location tiers) is substantially more
and mostly not worth it.
## Recommendation
- **Don't invest in the cloud-TPA protocol as the long-term target** — the vendor is
killing it within ~3 months of now, and the ecosystem's apps will migrate to miniapps.
- **Target the Miniapp SDK when its spec lands (Sept 2026).** It's strictly easier for
Faceclaw: JS bundle in an Android WebView with an injected `session` bridge — no
server, no webhooks, no reconnect protocol — and Faceclaw already has every underlying
facility the bridge needs. The rasterizer/input/transcription plumbing built for it
would be shared with any interim TPA emulator anyway.
- If interim access to existing cloud-SDK apps matters, build the minimal LAN emulator
described above, scoped to text layouts + transcription + button streams, and treat it
as disposable.
+22 -9
View File
@@ -1,8 +1,8 @@
# Voice Assistant Architecture
Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability
ontology, misc fixes). **Phase 1 is implemented** (see the Implementation plan
below); phases 2-4 are still design-only.
ontology, misc fixes). **Phases 1 and 2 are implemented** (see the
Implementation plan below); phases 3-4 are still design-only.
## Goal
@@ -383,13 +383,26 @@ default highlight and wakeword skip-confirmation auto-send;
*Deliverable: speak wakeword, ask "what's on my calendar", get an answer
on-lens.*
**Phase 2 — app tools.**
Worker protocol extension (set-tools / tool-call / tool-result),
availability-tier filtering in the registry (`open` on window open/close,
`foreground` on foreground change), `installed`-tier registration on launcher
entries (shell-side handlers only; launch-on-call deferred), terminal app
tools as the reference. *Deliverable: "run the build again" typed into the
terminal even while it's backgrounded.*
**Phase 2 — app tools.** ✅ Landed 2026-07-25 (builds + typechecks; not yet
hardware-tested).
Worker protocol extended with `set-tools` / `tool-result` (WorkerAppReply) and
`tool-call` (WorkerAppMessage). `ToolRegistry.setAppTools`/`removeAppTools`
key tools by windowId, prefix names `app.<appId>.`, and gate `foreground`
tools on an `isForeground()` predicate (checked at list and dispatch time).
`WorkerAppHost` proxies calls to its worker (callId round-trip, 15s host
timeout backstop on top of the registry's per-tool timeout), registers a
window's tools on `set-tools`, and withdraws them + fails in-flight calls on
window close. Direct mode now re-lists tools each loop iteration.
Terminal is the reference: the hub window declares `send_input` /
`read_screen` / `list_sessions` (all `open`-tier); send_input/read_screen act
on the active view (foregrounded → last-active → sole view). *Deliverable:
"run the build again" typed into the terminal even while it's backgrounded.*
*Not yet done from the Phase 2 design:* `installed`-tier registration on
launcher entries (no `installed` app tool exists yet — timer stays deferred);
in-process-window tool registration (only worker apps wired); the per-window
same-name collision handling (the terminal sidesteps it by declaring all tools
on the single hub window).
**Phase 3 — external bridge.**
Bridge wire protocol; ExternalAgentBackend (phone side, reusing the