Add app tool registration for voice assistant; voice assistant can read and send to terminals
This commit is contained in:
1 parent
0aeb24ed00
commit
fe205c0164
9 files changed
+533
-16
No files matched your search
@@ -26,7 +26,12 @@ export type DirectTurnOptions = {
|
||||
system: string;
|
||||
/** Owned by the session; this loop appends assistant + tool_result messages. */
|
||||
messages: AnthropicMessage[];
|
||||
tools: AnthropicToolDefinition[];
|
||||
/**
|
||||
* Build the tool list for the next request. Called once per loop iteration so
|
||||
* the set reflects apps opening/closing and foreground changes mid-turn; it
|
||||
* also refreshes the session's api-name map that resolveToolName reads.
|
||||
*/
|
||||
buildTools: () => AnthropicToolDefinition[];
|
||||
registry: ToolRegistry;
|
||||
/** Map an API tool name (as the model calls it) back to its registry name. */
|
||||
resolveToolName: (apiName: string) => string;
|
||||
@@ -62,7 +67,7 @@ export class DirectAnthropicBackend {
|
||||
system: options.system,
|
||||
effort: options.effort,
|
||||
messages: options.messages,
|
||||
tools: options.tools,
|
||||
tools: options.buildTools(),
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
if (cancelled) return;
|
||||
options.callbacks.onTextDelta(delta, textSoFar);
|
||||
|
||||
@@ -60,7 +60,6 @@ export class AssistantSession {
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
|
||||
const tools = this.buildToolDefinitions();
|
||||
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
|
||||
|
||||
const finish = () => {
|
||||
@@ -74,7 +73,7 @@ export class AssistantSession {
|
||||
effort: "low",
|
||||
system,
|
||||
messages: this.messages,
|
||||
tools,
|
||||
buildTools: () => this.buildToolDefinitions(),
|
||||
registry: this.registry,
|
||||
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
||||
callbacks: {
|
||||
|
||||
@@ -70,14 +70,28 @@ export type CallToolOptions = {
|
||||
|
||||
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** How the shell reaches a window's tool handler and gates its foreground tools. */
|
||||
export type AppToolProvider = {
|
||||
windowId: string;
|
||||
appId: string;
|
||||
/** The window's declared specs, with UNPREFIXED names (registry adds the prefix). */
|
||||
specs: ToolSpec[];
|
||||
/** Invoke a tool on the window by its unprefixed name. */
|
||||
invoke: (toolName: string, args: unknown) => Promise<ToolResult> | ToolResult;
|
||||
/** Whether the window currently owns the foreground (gates `foreground` tools). */
|
||||
isForeground: () => boolean;
|
||||
};
|
||||
|
||||
export class ToolRegistry {
|
||||
private readonly registrations = new Map<string, ToolRegistration>();
|
||||
private readonly changeListeners = new Set<() => void>();
|
||||
// windowId -> the canonical (prefixed) tool names it contributed, for bulk
|
||||
// removal when the window closes or re-declares.
|
||||
private readonly windowToolNames = new Map<string, string[]>();
|
||||
|
||||
/** Register (or replace) a tool. Names are unique across all tiers. */
|
||||
register(registration: ToolRegistration): void {
|
||||
const { name } = registration.spec;
|
||||
this.registrations.set(name, registration);
|
||||
this.addRegistration(registration);
|
||||
this.fireToolsChanged();
|
||||
}
|
||||
|
||||
@@ -95,6 +109,52 @@ export class ToolRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Declare (replacing any prior set) the tools contributed by one app window.
|
||||
* Names are prefixed `app.<appId>.` so windows can't shadow system tools or
|
||||
* each other. `open` tools stay live while the window exists; `foreground`
|
||||
* tools are gated on `isForeground()` at both list and call time.
|
||||
*/
|
||||
setAppTools(provider: AppToolProvider): void {
|
||||
this.clearWindowTools(provider.windowId, false);
|
||||
const names: string[] = [];
|
||||
for (const spec of provider.specs) {
|
||||
if (spec.availability !== "open" && spec.availability !== "foreground") {
|
||||
console.warn(`app tool ${spec.name} has non-app availability ${spec.availability}; skipping`);
|
||||
continue;
|
||||
}
|
||||
const canonical = `app.${provider.appId}.${spec.name}`;
|
||||
const toolName = spec.name;
|
||||
this.addRegistration({
|
||||
spec: { ...spec, name: canonical },
|
||||
handler: (args) => provider.invoke(toolName, args),
|
||||
isAvailable: spec.availability === "foreground" ? () => provider.isForeground() : undefined,
|
||||
});
|
||||
names.push(canonical);
|
||||
}
|
||||
this.windowToolNames.set(provider.windowId, names);
|
||||
this.fireToolsChanged();
|
||||
}
|
||||
|
||||
/** Remove all tools contributed by a window (its worker closed or the window did). */
|
||||
removeAppTools(windowId: string): void {
|
||||
this.clearWindowTools(windowId, true);
|
||||
}
|
||||
|
||||
private clearWindowTools(windowId: string, fire: boolean): void {
|
||||
const names = this.windowToolNames.get(windowId);
|
||||
if (!names) return;
|
||||
for (const name of names) {
|
||||
this.registrations.delete(name);
|
||||
}
|
||||
this.windowToolNames.delete(windowId);
|
||||
if (fire) this.fireToolsChanged();
|
||||
}
|
||||
|
||||
private addRegistration(registration: ToolRegistration): void {
|
||||
this.registrations.set(registration.spec.name, registration);
|
||||
}
|
||||
|
||||
/** Specs for the tools callable right now, given the availability tiers. */
|
||||
listTools(options: ListToolsOptions = {}): ToolSpec[] {
|
||||
const specs: ToolSpec[] = [];
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { windowIcon } from "./chrome-layer";
|
||||
import { type IconName } from "../../graphics/icons";
|
||||
import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/tool-registry";
|
||||
import { shell, type ShellWindow } from "./shell";
|
||||
|
||||
/**
|
||||
@@ -16,7 +17,9 @@ export type WorkerAppMessage =
|
||||
| { type: "text-input"; windowId: string; text: string }
|
||||
| { type: "render"; windowId: string; focused: boolean }
|
||||
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
|
||||
| { type: "screen"; on: boolean };
|
||||
| { type: "screen"; on: boolean }
|
||||
/** Assistant tool invocation aimed at a window; reply with tool-result. */
|
||||
| { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown };
|
||||
|
||||
export type WorkerAppReply =
|
||||
| { type: "yield-focus"; windowId: string }
|
||||
@@ -59,6 +62,21 @@ export type WorkerAppReply =
|
||||
*/
|
||||
type: "set-tray-icon";
|
||||
icon: { width: number; height: number; pixels: number[] } | null;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
* Declare (replacing the prior set) the assistant tools this window
|
||||
* contributes. Names are unprefixed; the registry adds `app.<appId>.`.
|
||||
*/
|
||||
type: "set-tools";
|
||||
windowId: string;
|
||||
tools: ToolSpec[];
|
||||
}
|
||||
| {
|
||||
/** Result of a tool-call, matched to the request by callId. */
|
||||
type: "tool-result";
|
||||
callId: string;
|
||||
result: ToolResult;
|
||||
};
|
||||
|
||||
export type WorkerWindowSpec = {
|
||||
@@ -92,8 +110,24 @@ export type WorkerAppHostOptions = {
|
||||
* and manages compositor surfaces for the app's windows. The worker submits
|
||||
* frames straight to the Java compositor, so no pixels cross this boundary.
|
||||
*/
|
||||
/** A tool-call awaiting its worker reply; also its own leak-safety timeout. */
|
||||
type PendingToolCall = {
|
||||
windowId: string;
|
||||
resolve: (result: ToolResult) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
|
||||
/**
|
||||
* How long the host waits for a worker's tool-result before giving up. The
|
||||
* registry applies its own (usually shorter) per-tool timeout; this is the
|
||||
* backstop that also frees the pending-call entry if the worker never replies.
|
||||
*/
|
||||
const TOOL_CALL_HOST_TIMEOUT_MS = 15_000;
|
||||
|
||||
export class WorkerAppHost {
|
||||
private readonly openWindows = new Set<string>();
|
||||
private readonly pendingToolCalls = new Map<string, PendingToolCall>();
|
||||
private nextCallSerial = 1;
|
||||
|
||||
constructor(private readonly options: WorkerAppHostOptions) {
|
||||
options.worker.onmessage = (event: MessageEvent) => {
|
||||
@@ -152,6 +186,27 @@ export class WorkerAppHost {
|
||||
case "set-title":
|
||||
// Titles are informational for now (sidebar shows icons only).
|
||||
break;
|
||||
case "set-tools":
|
||||
// Only a window we actually have open may contribute tools.
|
||||
if (this.openWindows.has(message.windowId)) {
|
||||
toolRegistry.setAppTools({
|
||||
windowId: message.windowId,
|
||||
appId: this.options.appId,
|
||||
specs: message.tools,
|
||||
invoke: (toolName, args) => this.callWindowTool(message.windowId, toolName, args),
|
||||
isForeground: () => shell.foregroundWindow()?.windowId === message.windowId,
|
||||
});
|
||||
}
|
||||
break;
|
||||
case "tool-result": {
|
||||
const pending = this.pendingToolCalls.get(message.callId);
|
||||
if (pending) {
|
||||
clearTimeout(pending.timer);
|
||||
this.pendingToolCalls.delete(message.callId);
|
||||
pending.resolve(message.result);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
options.worker.onerror = (error) => {
|
||||
@@ -181,6 +236,9 @@ export class WorkerAppHost {
|
||||
closeable: true,
|
||||
close: () => {
|
||||
this.openWindows.delete(spec.windowId);
|
||||
// Withdraw this window's tools and fail any in-flight calls to it.
|
||||
toolRegistry.removeAppTools(spec.windowId);
|
||||
this.failPendingToolCallsFor(spec.windowId);
|
||||
this.post({ type: "close-window", windowId: spec.windowId });
|
||||
this.options.removeSurface(surfaceId);
|
||||
},
|
||||
@@ -235,6 +293,32 @@ export class WorkerAppHost {
|
||||
return window;
|
||||
}
|
||||
|
||||
/**
|
||||
* Post a tool-call to the worker and resolve when its tool-result comes back.
|
||||
* Resolves with a tool error on timeout so a hung worker yields a tool error
|
||||
* rather than a stuck assistant turn.
|
||||
*/
|
||||
private callWindowTool(windowId: string, toolName: string, args: unknown): Promise<ToolResult> {
|
||||
return new Promise<ToolResult>((resolve) => {
|
||||
const callId = `${this.options.appId}:${this.nextCallSerial++}`;
|
||||
const timer = setTimeout(() => {
|
||||
this.pendingToolCalls.delete(callId);
|
||||
resolve({ ok: false, error: `App ${this.options.appId} did not respond to ${toolName}` });
|
||||
}, TOOL_CALL_HOST_TIMEOUT_MS);
|
||||
this.pendingToolCalls.set(callId, { windowId, resolve, timer });
|
||||
this.post({ type: "tool-call", callId, windowId, name: toolName, args });
|
||||
});
|
||||
}
|
||||
|
||||
private failPendingToolCallsFor(windowId: string): void {
|
||||
for (const [callId, pending] of this.pendingToolCalls) {
|
||||
if (pending.windowId !== windowId) continue;
|
||||
clearTimeout(pending.timer);
|
||||
this.pendingToolCalls.delete(callId);
|
||||
pending.resolve({ ok: false, error: "The target window was closed" });
|
||||
}
|
||||
}
|
||||
|
||||
private post(message: WorkerAppMessage): void {
|
||||
this.options.worker.postMessage(message);
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
} from "../ui/menu";
|
||||
import { defaultWindowMenuItems, WindowMenu } from "../ui/window-menu";
|
||||
import type { WorkerAppMessage, WorkerAppReply } from "../ui/shell/worker-window";
|
||||
import type { ToolResult, ToolSpec } from "../assistant/tool-registry";
|
||||
|
||||
declare const global: any;
|
||||
declare const com: any;
|
||||
@@ -102,7 +103,43 @@ type TerminalWindow = HubWindow | ViewWindow;
|
||||
|
||||
const windows = new Map<string, TerminalWindow>();
|
||||
const pendingViews = new Map<string, { socket: string; label: string }>();
|
||||
// The view an assistant tool acts on when the terminal isn't foregrounded:
|
||||
// the last view to be foregrounded or receive input.
|
||||
let activeViewId: string | null = null;
|
||||
let nextViewSerial = 1;
|
||||
|
||||
/**
|
||||
* Assistant tools this app contributes, declared on the hub window (which the
|
||||
* launcher opens and which persists for the app's life). All `open`-tier so
|
||||
* "rerun the build" works while the terminal is backgrounded. send_input and
|
||||
* read_screen act on the active view session (see resolveActiveView).
|
||||
*/
|
||||
const TERMINAL_TOOLS: ToolSpec[] = [
|
||||
{
|
||||
name: "list_sessions",
|
||||
description: "List the live g2mirror terminal sessions the glasses can see.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
availability: "open",
|
||||
},
|
||||
{
|
||||
name: "send_input",
|
||||
description:
|
||||
"Type a line into the active terminal session and submit it (as if typed and Enter pressed). Use to run a command in the terminal.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { text: { type: "string", description: "The line to type and run." } },
|
||||
required: ["text"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
availability: "open",
|
||||
},
|
||||
{
|
||||
name: "read_screen",
|
||||
description: "Return the current visible contents of the active terminal session's screen.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
availability: "open",
|
||||
},
|
||||
];
|
||||
let viewportWidth = 0;
|
||||
let viewportHeight = 0;
|
||||
let gridCols = 0;
|
||||
@@ -161,6 +198,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
if (!window) break;
|
||||
window.foreground = message.foreground;
|
||||
window.focused = message.focused;
|
||||
if (window.foreground && window.kind === "view") activeViewId = window.windowId;
|
||||
if (window.foreground) renderAndSubmit(window, 0);
|
||||
break;
|
||||
}
|
||||
@@ -172,6 +210,19 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "tool-call": {
|
||||
const callId = message.callId;
|
||||
Promise.resolve(handleTerminalTool(message.name, message.args))
|
||||
.then((result) => post({ type: "tool-result", callId, result }))
|
||||
.catch((error) =>
|
||||
post({
|
||||
type: "tool-result",
|
||||
callId,
|
||||
result: { ok: false, error: String((error as Error)?.message ?? error) },
|
||||
}),
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -209,6 +260,8 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb
|
||||
selectedIndex: 0,
|
||||
scrollRow: 0,
|
||||
});
|
||||
// The hub carries the app's assistant tools; declare them once it exists.
|
||||
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
|
||||
if (!controlClient) {
|
||||
startControlClient();
|
||||
}
|
||||
@@ -224,6 +277,7 @@ function closeWindow(windowId: string): void {
|
||||
window.client.stop();
|
||||
}
|
||||
windows.delete(windowId);
|
||||
if (activeViewId === windowId) activeViewId = null;
|
||||
if (window.kind === "view") {
|
||||
renderHubWindows();
|
||||
}
|
||||
@@ -414,6 +468,7 @@ function windowMenuItems(window: TerminalWindow): MenuItem[] {
|
||||
}
|
||||
|
||||
function handleInput(window: TerminalWindow, event: DashboardInputEvent, frameId: number): void {
|
||||
if (window.kind === "view") activeViewId = window.windowId;
|
||||
// An open window menu owns all input (it closes itself via pop).
|
||||
if (window.menu?.isOpen()) {
|
||||
window.menu
|
||||
@@ -743,3 +798,71 @@ function sessionLabel(session: G2MirrorSession): string {
|
||||
const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/");
|
||||
return hint || "session";
|
||||
}
|
||||
|
||||
/** Dispatch an assistant tool-call (unprefixed name) to its handler. */
|
||||
function handleTerminalTool(name: string, args: any): ToolResult {
|
||||
switch (name) {
|
||||
case "list_sessions":
|
||||
return toolListSessions();
|
||||
case "send_input":
|
||||
return toolSendInput(args);
|
||||
case "read_screen":
|
||||
return toolReadScreen();
|
||||
default:
|
||||
return { ok: false, error: `Unknown terminal tool: ${name}` };
|
||||
}
|
||||
}
|
||||
|
||||
function toolListSessions(): ToolResult {
|
||||
const sessions = controlState?.sessions ?? [];
|
||||
if (!sessions.length) {
|
||||
return { ok: true, content: "No live terminal sessions (run g2mirror <command> on the host)." };
|
||||
}
|
||||
const lines = sessions.map(
|
||||
(session) => `- ${sessionLabel(session)}${viewWindowIdForSocket(session.socket) ? " [open]" : ""}`,
|
||||
);
|
||||
return { ok: true, content: lines.join("\n") };
|
||||
}
|
||||
|
||||
function toolSendInput(args: any): ToolResult {
|
||||
const text = String(args?.text ?? "");
|
||||
if (!text) return { ok: false, error: "send_input requires non-empty text." };
|
||||
const view = resolveActiveView();
|
||||
if (!view) return { ok: false, error: "No terminal session is open to send input to." };
|
||||
view.client.submitInput(text);
|
||||
return { ok: true, content: `Sent to ${view.label}.` };
|
||||
}
|
||||
|
||||
function toolReadScreen(): ToolResult {
|
||||
const view = resolveActiveView();
|
||||
if (!view) return { ok: false, error: "No terminal session is open." };
|
||||
const length = view.emulator.bufferLength();
|
||||
const start = Math.max(0, length - gridRows);
|
||||
const lines: string[] = [];
|
||||
for (let index = start; index < length; index++) {
|
||||
lines.push(view.emulator.lineAt(index));
|
||||
}
|
||||
const text = lines.join("\n").replace(/\s+$/, "");
|
||||
return { ok: true, content: text || "(screen is empty)" };
|
||||
}
|
||||
|
||||
/**
|
||||
* The terminal view an `open`-tier tool acts on: a foregrounded view if any,
|
||||
* else the last view to be active, else the sole open view. Null when no view
|
||||
* is open (the model gets a tool error it can relay).
|
||||
*/
|
||||
function resolveActiveView(): ViewWindow | null {
|
||||
const views: ViewWindow[] = [];
|
||||
let foreground: ViewWindow | null = null;
|
||||
for (const window of windows.values()) {
|
||||
if (window.kind !== "view") continue;
|
||||
views.push(window);
|
||||
if (window.foreground) foreground = window;
|
||||
}
|
||||
if (foreground) return foreground;
|
||||
if (activeViewId) {
|
||||
const window = windows.get(activeViewId);
|
||||
if (window && window.kind === "view") return window;
|
||||
}
|
||||
return views.length === 1 ? views[0]! : null;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
# Even-Analogous Apps
|
||||
|
||||
Ideally we'd like full feature parity with the stock Even app (or at least,
|
||||
close enough that not-using the stock app doesn't feel like a sacrifice).
|
||||
A rough list of things the stock app has that we currently don't:
|
||||
|
||||
## Quicklist
|
||||
|
||||
Quicklist is Even's version of a todo app, in the form of a dashboard card
|
||||
that you add to and remove from with assistant voice commands. It's pretty
|
||||
simple to build once a general voice assistant is in place, but I don't
|
||||
actually like their design: a todo list really wants to be synchronized
|
||||
with something external like Roam or Notion, so I think it'd be better to
|
||||
build one of those integrations instead.
|
||||
|
||||
## Stocks, News
|
||||
|
||||
Even has a dashboard card which is a stock ticker. I think this is probably
|
||||
misguided; while there do exist people who would want and benefit from a stock
|
||||
tracker on their glasses, I do'm not sure there exist people who would want and
|
||||
benefit from a stock tracker that isn't a professional-tier tool connected
|
||||
to carefully chosen low-latency data sources, and neither I (nor Even) can
|
||||
build that tool.
|
||||
|
||||
The News dashboard card is more broadly useful, but has a similar issue: a
|
||||
generic news ticker isn't really useful without a much higher degree of
|
||||
customization. Probably the right thing to build instead is more like an RSS
|
||||
reader.
|
||||
|
||||
## Ring Connection Management
|
||||
|
||||
The stock Even app is responsible for pairing the glasses with the ring (ie,
|
||||
sending the glasses a configuration messages telling it to connect to the
|
||||
ring's MAC address). If you pair them in the stock app and then switch to a
|
||||
custom app, they remain paired indefinitely (this configuration is stored in
|
||||
flash), and our custom app can make use of the fact that they're paired.
|
||||
|
||||
We don't yet have support for connecting or disconnecting the glasses and ring
|
||||
to/from each other. For latency reasons, it would be nice if we could unpair
|
||||
the glasses from the ring and connect the phone to it directly; we don't yet
|
||||
support that either.
|
||||
|
||||
## Ring Health Metrics
|
||||
|
||||
The R1 ring contains an oximeter-style LED sensor and an accelerometer, which
|
||||
can be used for a step count, heart rate measurement, etc. I'm not sure
|
||||
whether the sensor interpretation is done within the firmware (in which case
|
||||
it would make sense to display them in the UI) or in the Even phone app (in
|
||||
which case we'd be reimplementing all the interpretation algorithms, which
|
||||
isn't worth it.
|
||||
|
||||
## Navigate
|
||||
|
||||
Even has a Navigate app, which is basically Google Maps for the specific use
|
||||
case of "name a destination, get turn by turn directions to there". I believe
|
||||
this is using Mapbox, which takes care of most of the heavy lifting (if you
|
||||
have an API key). I don't think Even's version is very good (they ran into
|
||||
trouble with image-upload performance preventing a good map display size), so
|
||||
it's likely not too hard to do better. One caveat is the location permission;
|
||||
Android normally only wants to prompt for location permission "while using the
|
||||
app", but Android's idea of in use (phone screen unlocked and window focused)
|
||||
doesn't match our idea of in use (G2 glasses on, phone could be locked). This
|
||||
app also gets a lot better if we figure out how to use the device's
|
||||
magnetometer (if it indeed has a working magnetometer).
|
||||
|
||||
## Distance/Viewport Configuration
|
||||
|
||||
During glasses onboarding, the stock firmware has you select a vertical offset
|
||||
for the screen image. This accounts for glasses resting at different positions
|
||||
on your face (the optics don't let you see the whole screen area), but cuts
|
||||
the usable screen height from 480px to 288px. There is also a "distance"
|
||||
config option in the stock Android app, which adjusts the horizontal position
|
||||
of the image on the two lenses separately to create the impression of the
|
||||
image being closer or farther; this cuts the usable screen width from 640px to
|
||||
576px. Unlike the vertical size reduction, this 64px reduction is not
|
||||
optically necessary.
|
||||
|
||||
## Even AI
|
||||
## Translate and Conversate
|
||||
|
||||
## Other Configuration
|
||||
|
||||
Brightness, head-up trigger
|
||||
|
||||
|
||||
# Other Apps
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Music: "Resume" action via media-key dispatch
|
||||
|
||||
Follow-up to the MediaBrowserService browsing support. When no media session
|
||||
is active, `AudioManager.dispatchMediaKeyEvent(KEYCODE_MEDIA_PLAY)` (down+up
|
||||
pair) is routed by the system to the *last* media app's manifest-declared
|
||||
`MEDIA_BUTTON` receiver, even if that app's process is dead — the same
|
||||
mechanism a Bluetooth headset play button uses. Most players respond by
|
||||
resuming the last queue.
|
||||
|
||||
Idea: add a "Resume last" row to the Music app's no-active-session screen
|
||||
(next to the browse entry). ~5 lines in `FaceclawMediaController` plus one UI
|
||||
row; no per-app negotiation needed. Caveat: it's fire-and-forget — no feedback
|
||||
if nothing handles it, so the UI should just wait for a session to appear.
|
||||
@@ -0,0 +1,133 @@
|
||||
# MentraOS app compatibility for Faceclaw — feasibility assessment (2026-07-23)
|
||||
|
||||
MentraOS: https://github.com/Mentra-Community/MentraOS (MIT), docs at https://docs.mentraglass.com.
|
||||
|
||||
## TL;DR
|
||||
|
||||
Technically feasible with moderate effort, but the timing is awkward: MentraOS is
|
||||
**deprecating its entire current app API**. MentraOS 3.0 ships **Aug 3, 2026** and
|
||||
cloud-SDK apps stop working on it (a "MentraOS Legacy" mode preserves them through
|
||||
**October 2026**). The replacement — a **Miniapp SDK** where apps are phone-local JS
|
||||
bundles running in a WebView — gets a public spec in **September 2026**, and that model
|
||||
is a *much* better fit for Faceclaw than the cloud model. Recommendation: wait for the
|
||||
miniapp spec and target that; only build a minimal cloud-protocol emulator if access to
|
||||
the existing app corpus is wanted in the interim.
|
||||
|
||||
## MentraOS app model (current, "Cloud SDK" / TPA era)
|
||||
|
||||
- Apps are developer-hosted server processes (Node/Bun, `@mentra/sdk`). Chain:
|
||||
glasses → phone (BLE) → MentraOS Cloud (WS) → app server (WS).
|
||||
- Start flow: cloud POSTs a `session_request` webhook `{sessionId, userId, websocketUrl}`
|
||||
to the app's registered URL; the app dials back to `websocketUrl` and sends
|
||||
`tpa_connection_init {packageName, sessionId, apiKey}`; cloud acks with settings +
|
||||
device capabilities. **The webhook carries the WS URL, so an unmodified `@mentra/sdk`
|
||||
app can be pointed at any server** — that is the compat hook.
|
||||
- Protocol: JSON over WebSocket (binary frames for audio). 28 app→cloud + 28 cloud→app
|
||||
message types, defined in `cloud/packages/sdk/src/types/` (`message-types.ts`,
|
||||
`streams.ts`, `layouts.ts`, `webhooks.ts`, `capabilities.ts`). The TS types are the spec.
|
||||
- Subscriptions: `subscription_update` with a full list of stream types, some
|
||||
parameterized (`transcription:en-US`, `touch_event:forward_swipe`). Events arrive in a
|
||||
`data_stream` envelope.
|
||||
- Display: 7 layout types — `text_wall`, `double_text_wall`, `dashboard_card`,
|
||||
`reference_card`, `bitmap_view` (base64), `bitmap_animation`, `clear_view` — sent as
|
||||
`display_event {view: 'main'|'dashboard', layout, durationMs?, forceDisplay?}`.
|
||||
Cloud throttles displays to 1 per 300 ms. Dashboard = shared look-up surface.
|
||||
- Streams: `transcription` (`{text, isFinal, ...}`), `translation`, `VAD`, `audio_chunk`
|
||||
(PCM), `button_press`, `touch_event`, `head_position` (up/down), `glasses_battery_update`,
|
||||
`phone_battery_update`, `location_update`, `phone_notification`(+dismissed),
|
||||
`calendar_event`, plus camera/RTMP/LED families irrelevant to G2.
|
||||
- Other: ElevenLabs TTS (`audio.speak`), per-app settings (schema in dev console, 9
|
||||
setting types, live `settings_update`), webview UI with signed-token auth, cloud KV
|
||||
storage, app-to-app messaging.
|
||||
- Hardware abstraction: `Capabilities` flags per device. **Even Realities G2 is a
|
||||
first-class device** (`capabilities/even-realities-g2.ts`): display 640×200 (their
|
||||
rendering canvas, not the 576×288 EvenHub raster), maxTextLines 5, bitmap-capable,
|
||||
capacitive swipe bar with TAP/DOUBLE_TAP/TRIPLE_TAP/PRESS_HOLD/SWIPE_UP/SWIPE_DOWN,
|
||||
IMU, mic, no camera/speaker/LED.
|
||||
- Licensing: MIT throughout; cloud is self-hostable in principle (Bun + MongoDB + S3 +
|
||||
third-party STT/TTS keys) but heavy.
|
||||
|
||||
## Coming model (Miniapp SDK, MentraOS 3.0)
|
||||
|
||||
Miniapps are JS bundles (`index.html` + `miniapp.json`) running **on the phone** inside
|
||||
the MentraOS host app's WebView ("Mentra Runtime"), packages `@mentra/miniapp` +
|
||||
`@mentra/miniapp-cli`, sideloaded via QR. API is analogous but smaller:
|
||||
`session.transcription`, `session.display.showTextWall`, `session.storage`. Example:
|
||||
https://github.com/Mentra-Community/LiveCaptionsLocalSdkMiniapp. Public spec Sept 2026.
|
||||
|
||||
## Fit with Faceclaw
|
||||
|
||||
Faceclaw's worker-per-app architecture is unusually favorable: apps are already isolated
|
||||
contexts speaking a small typed JSON vocabulary (`WorkerAppMessage`/`WorkerAppReply` in
|
||||
`app/ui/shell/worker-window.ts`), with pixels out-of-band. The compat layer would be one
|
||||
compiled-in "MentraOS host" app (a peer of `WorkerAppHost`) that presents each external
|
||||
app as a `ShellWindow` and:
|
||||
|
||||
- **Rasterizes layouts** → trivial. All 7 layout types map onto `GrayImage` +
|
||||
`drawTextWrapped`/`bitBlt` into the 540×260 `APP_VIEWPORT`. Bitmaps need scaling from
|
||||
their 640×200/576×136 assumptions; monochrome already matches.
|
||||
- **Maps input** → good. `DashboardInputEvent` (`click`/`double-click`/`long-press`/
|
||||
`scroll-up`/`scroll-down`) covers their G2 button-event list almost 1:1 (TRIPLE_TAP
|
||||
missing). Work item: input is currently routed only to the focused window; MentraOS
|
||||
wants multi-subscriber fan-out (background apps), so a broadcast layer is needed.
|
||||
- **Transcription** → the load-bearing stream, and Faceclaw already has it:
|
||||
`voiceControlBridge.onTranscript` (`{text, isFinal}`, replace semantics — matches
|
||||
MentraOS's `TranscriptionData` model well). Onboard Moonshine is English-only;
|
||||
parameterized languages would fall back to ElevenLabs/Whisper cloud providers.
|
||||
Needs multi-consumer fan-out and a policy for when the mic is open (MentraOS keeps it
|
||||
effectively continuous; Faceclaw is PTT/wakeword-oriented — battery implications).
|
||||
- **VAD** → sherpa endpointing exists. **audio_chunk** → PCM already surfaced on the
|
||||
ElevenLabs path (`onPcm`).
|
||||
- **Battery / phone notifications / calendar** → data already flows in Faceclaw
|
||||
(top-bar battery, notification listener, calendar app); just needs re-emission as
|
||||
streams.
|
||||
- **head_position** → gap. Faceclaw exposes raw IMU only (no gesture classification);
|
||||
the look-up dashboard gesture would need a small accelerometer classifier (cmd 19
|
||||
stream exists, see imu-accelerometer memory).
|
||||
- **TTS / audio out** → G2 has no speaker; MentraOS plays through the phone. Faceclaw
|
||||
would need phone-side audio playback (new but simple Android work).
|
||||
- **Settings** → MentraOS per-app settings map onto the shared `FaceclawSettings` store
|
||||
with a `mentra.<packageName>.*` prefix; the 9 setting types map onto
|
||||
`ConfigSetting` subclasses, mostly.
|
||||
- **Not applicable on G2**: camera/photo/RTMP/LiveKit, LED, WiFi setup — can be
|
||||
rejected via capabilities, which the protocol explicitly supports.
|
||||
|
||||
## The hard parts
|
||||
|
||||
1. **Who plays "cloud"?** Cloud-SDK apps dial out to a WS URL given in a webhook, so
|
||||
Faceclaw must (a) run a WS **server** — none exists anywhere in the codebase today,
|
||||
all networking is outbound okhttp client (`FaceclawWebSocket`, g2mirror pattern) —
|
||||
reachable from wherever the app server runs (fine on LAN; NAT pain otherwise), and
|
||||
(b) send the session webhook itself (simple outbound HTTP POST). A phone-hosted
|
||||
mini-cloud with Java-WebSocket/NanoWSD listening on LAN is the sane MVP; self-hosting
|
||||
their actual cloud (Bun+Mongo+Docker) is overkill.
|
||||
2. **Session lifecycle**: connection init/ack, `subscription_update` bookkeeping,
|
||||
`reconnect`/`reconnect_ack/rejected/deferred`, `app_stopped`, per-app permission
|
||||
gating. This is bookkeeping-heavy but not deep.
|
||||
3. **Display arbitration**: foreground vs `background`+`forceDisplay` apps, `durationMs`
|
||||
auto-revert, the shared dashboard view. Faceclaw's shell has focus/foreground but no
|
||||
equivalent of background-app display preemption or a look-up dashboard.
|
||||
4. **App discovery/registration UX**: no dev console; needs a simple "add app by URL +
|
||||
API key" settings flow (API key check can be a no-op or shared secret).
|
||||
|
||||
## Effort estimate
|
||||
|
||||
MVP covering the bulk of display-glasses apps (live captions, teleprompters, dashboards,
|
||||
notification relays): ~10–15 message types + ~8 stream types + webhook sender + LAN WS
|
||||
server + layout rasterizer + settings mapping. Roughly a few weeks of focused work, with
|
||||
transcription-stream policy and the WS server being the riskiest pieces. Full-surface
|
||||
emulation (webview auth, app messaging rooms, TTS, location tiers) is substantially more
|
||||
and mostly not worth it.
|
||||
|
||||
## Recommendation
|
||||
|
||||
- **Don't invest in the cloud-TPA protocol as the long-term target** — the vendor is
|
||||
killing it within ~3 months of now, and the ecosystem's apps will migrate to miniapps.
|
||||
- **Target the Miniapp SDK when its spec lands (Sept 2026).** It's strictly easier for
|
||||
Faceclaw: JS bundle in an Android WebView with an injected `session` bridge — no
|
||||
server, no webhooks, no reconnect protocol — and Faceclaw already has every underlying
|
||||
facility the bridge needs. The rasterizer/input/transcription plumbing built for it
|
||||
would be shared with any interim TPA emulator anyway.
|
||||
- If interim access to existing cloud-SDK apps matters, build the minimal LAN emulator
|
||||
described above, scoped to text layouts + transcription + button streams, and treat it
|
||||
as disposable.
|
||||
@@ -1,8 +1,8 @@
|
||||
# Voice Assistant Architecture
|
||||
|
||||
Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability
|
||||
ontology, misc fixes). **Phase 1 is implemented** (see the Implementation plan
|
||||
below); phases 2-4 are still design-only.
|
||||
ontology, misc fixes). **Phases 1 and 2 are implemented** (see the
|
||||
Implementation plan below); phases 3-4 are still design-only.
|
||||
|
||||
## Goal
|
||||
|
||||
@@ -383,13 +383,26 @@ default highlight and wakeword skip-confirmation auto-send;
|
||||
*Deliverable: speak wakeword, ask "what's on my calendar", get an answer
|
||||
on-lens.*
|
||||
|
||||
**Phase 2 — app tools.**
|
||||
Worker protocol extension (set-tools / tool-call / tool-result),
|
||||
availability-tier filtering in the registry (`open` on window open/close,
|
||||
`foreground` on foreground change), `installed`-tier registration on launcher
|
||||
entries (shell-side handlers only; launch-on-call deferred), terminal app
|
||||
tools as the reference. *Deliverable: "run the build again" typed into the
|
||||
terminal even while it's backgrounded.*
|
||||
**Phase 2 — app tools.** ✅ Landed 2026-07-25 (builds + typechecks; not yet
|
||||
hardware-tested).
|
||||
Worker protocol extended with `set-tools` / `tool-result` (WorkerAppReply) and
|
||||
`tool-call` (WorkerAppMessage). `ToolRegistry.setAppTools`/`removeAppTools`
|
||||
key tools by windowId, prefix names `app.<appId>.`, and gate `foreground`
|
||||
tools on an `isForeground()` predicate (checked at list and dispatch time).
|
||||
`WorkerAppHost` proxies calls to its worker (callId round-trip, 15s host
|
||||
timeout backstop on top of the registry's per-tool timeout), registers a
|
||||
window's tools on `set-tools`, and withdraws them + fails in-flight calls on
|
||||
window close. Direct mode now re-lists tools each loop iteration.
|
||||
Terminal is the reference: the hub window declares `send_input` /
|
||||
`read_screen` / `list_sessions` (all `open`-tier); send_input/read_screen act
|
||||
on the active view (foregrounded → last-active → sole view). *Deliverable:
|
||||
"run the build again" typed into the terminal even while it's backgrounded.*
|
||||
|
||||
*Not yet done from the Phase 2 design:* `installed`-tier registration on
|
||||
launcher entries (no `installed` app tool exists yet — timer stays deferred);
|
||||
in-process-window tool registration (only worker apps wired); the per-window
|
||||
same-name collision handling (the terminal sidesteps it by declaring all tools
|
||||
on the single hub window).
|
||||
|
||||
**Phase 3 — external bridge.**
|
||||
Bridge wire protocol; ExternalAgentBackend (phone side, reusing the
|
||||
|
||||
Reference in new issue
Block a user