Implement a basic voice assistant

This commit is contained in:
James Babcock committed 2026-07-25 00:44:04 -07:00
1 parent d2c987d48a
commit 0aeb24ed00
14 files changed
+1923 -80

No files matched your search

+141
View File
@@ -0,0 +1,141 @@
import {
streamAnthropicMessage,
type AnthropicContentBlock,
type AnthropicMessage,
type AnthropicStreamHandle,
type AnthropicToolDefinition,
} from "../native/anthropic";
import type { ToolRegistry } from "./tool-registry";
import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* The on-phone agent loop: stream a reply, and while the model asks for tools,
* execute them through the registry and feed the results back until it stops.
* History lives in the AssistantSession; this backend appends the assistant
* and tool-result messages to the array it is handed.
*/
/** Safety caps so a tool-thrashing model can't loop or run forever. */
const MAX_ITERATIONS = 8;
const MAX_TURN_MS = 2 * 60 * 1000;
export type DirectTurnOptions = {
apiKey: string;
model: string;
effort?: "low" | "medium" | "high";
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: AnthropicMessage[];
tools: AnthropicToolDefinition[];
registry: ToolRegistry;
/** Map an API tool name (as the model calls it) back to its registry name. */
resolveToolName: (apiName: string) => string;
callbacks: AssistantTurnCallbacks;
};
export class DirectAnthropicBackend {
runTurn(options: DirectTurnOptions): AssistantTurnHandle {
const startMs = Date.now();
let cancelled = false;
let streamHandle: AnthropicStreamHandle | null = null;
const finishError = (message: string) => {
if (cancelled) return;
cancelled = true;
options.callbacks.onError(message);
};
const runIteration = (iteration: number): void => {
if (cancelled) return;
if (iteration >= MAX_ITERATIONS) {
finishError("Assistant stopped: too many tool steps");
return;
}
if (Date.now() - startMs > MAX_TURN_MS) {
finishError("Assistant stopped: turn took too long");
return;
}
streamHandle = streamAnthropicMessage({
apiKey: options.apiKey,
model: options.model,
system: options.system,
effort: options.effort,
messages: options.messages,
tools: options.tools,
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, textSoFar);
},
onDone: (result) => {
streamHandle = null;
if (cancelled) return;
if (result.stopReason === "refusal") {
finishError("The assistant declined this request");
return;
}
// Record what the model produced this iteration.
const assistantContent: AnthropicContentBlock[] = result.content.length
? result.content
: [{ type: "text", text: result.text }];
options.messages.push({ role: "assistant", content: assistantContent });
const toolUses = assistantContent.filter(
(block): block is Extract<AnthropicContentBlock, { type: "tool_use" }> =>
block.type === "tool_use",
);
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
options.callbacks.onTurnDone({ stopReason: result.stopReason });
return;
}
void this.runToolCalls(toolUses, options)
.then((toolResults) => {
// Always record the results, even if cancelled mid-flight: an
// assistant tool_use with no matching tool_result would make the
// next turn's request invalid. Only the loop continuation is gated.
options.messages.push({ role: "user", content: toolResults });
if (cancelled) return;
runIteration(iteration + 1);
})
.catch((error) => {
finishError(`Tool execution failed: ${String((error as Error)?.message ?? error)}`);
});
},
onError: (message) => {
streamHandle = null;
finishError(message);
},
});
};
runIteration(0);
return {
cancel: () => {
cancelled = true;
streamHandle?.cancel();
streamHandle = null;
},
};
}
private async runToolCalls(
toolUses: Array<Extract<AnthropicContentBlock, { type: "tool_use" }>>,
options: DirectTurnOptions,
): Promise<AnthropicContentBlock[]> {
const results: AnthropicContentBlock[] = [];
for (const toolUse of toolUses) {
const canonicalName = options.resolveToolName(toolUse.name);
options.callbacks.onToolActivity(canonicalName);
const result = await options.registry.callTool(canonicalName, toolUse.input);
const content = result.ok ? result.content ?? "" : result.error ?? "Tool error";
results.push({
type: "tool_result",
tool_use_id: toolUse.id,
content,
is_error: !result.ok,
});
}
return results;
}
}
+156
View File
@@ -0,0 +1,156 @@
import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic";
import { DirectAnthropicBackend } from "./direct-backend";
import { toolRegistry, type ToolRegistry } from "./tool-registry";
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* One assistant conversation. Owns the message history and turn state and
* drives a backend (direct Anthropic loop for phase 1). The UI talks only to
* this surface, so swapping in the external-agent backend later doesn't touch
* AssistantLayer.
*
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
* one for a new wakeword and reuses the current one for a follow-up.
*/
/** A new PTT after this much idle starts a fresh conversation. */
const SESSION_IDLE_MS = 10 * 60 * 1000;
/** Trim history from the head once it grows past this many messages. */
const MAX_HISTORY_MESSAGES = 40;
const SYSTEM_PROMPT_BASE = [
"You are the voice assistant built into a pair of Even Realities G2 smart glasses.",
"Your reply is shown as text on a 576x288 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list.",
"Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it.",
"If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
].join(" ");
export class AssistantSession {
private readonly backend = new DirectAnthropicBackend();
private readonly messages: AnthropicMessage[] = [];
private turnHandle: AssistantTurnHandle | null = null;
private lastActivityMs = Date.now();
// API-safe tool name -> canonical registry name. The Anthropic API only
// allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses.
// get_state") are sanitized for the request and mapped back on tool calls.
private readonly toolNameMap = new Map<string, string>();
constructor(
private readonly apiKey: string,
private readonly registry: ToolRegistry = toolRegistry,
) {}
/** Whether this session has been idle long enough to retire. */
isExpired(nowMs = Date.now()): boolean {
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
}
isTurnActive(): boolean {
return this.turnHandle !== null;
}
/** Begin a turn from a spoken utterance. Only one turn runs at a time. */
sendUtterance(text: string, ctx: AssistantContext, callbacks: AssistantTurnCallbacks): void {
if (this.turnHandle) {
callbacks.onError("The assistant is still working on the previous request");
return;
}
this.lastActivityMs = Date.now();
this.messages.push({ role: "user", content: text });
this.trimHistory();
const tools = this.buildToolDefinitions();
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
const finish = () => {
this.turnHandle = null;
this.lastActivityMs = Date.now();
};
this.turnHandle = this.backend.runTurn({
apiKey: this.apiKey,
model: DEFAULT_LLM_MODEL,
effort: "low",
system,
messages: this.messages,
tools,
registry: this.registry,
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
callbacks: {
onTextDelta: callbacks.onTextDelta,
onToolActivity: callbacks.onToolActivity,
onTurnDone: (result) => {
finish();
callbacks.onTurnDone(result);
},
onError: (message) => {
finish();
callbacks.onError(message);
},
},
});
}
cancel(): void {
this.turnHandle?.cancel();
this.turnHandle = null;
this.lastActivityMs = Date.now();
}
private buildToolDefinitions(): AnthropicToolDefinition[] {
this.toolNameMap.clear();
return this.registry.listTools().map((spec) => {
const apiName = this.toApiToolName(spec.name);
this.toolNameMap.set(apiName, spec.name);
return { name: apiName, description: spec.description, input_schema: spec.inputSchema };
});
}
/** Convert a canonical registry name to an Anthropic-legal tool name. */
private toApiToolName(name: string): string {
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
// Disambiguate the rare case where two canonical names sanitize alike.
if (this.toolNameMap.has(base)) {
let i = 2;
while (this.toolNameMap.has(`${base}_${i}`)) i++;
base = `${base}_${i}`;
}
return base;
}
private trimHistory(): void {
if (this.messages.length <= MAX_HISTORY_MESSAGES) return;
// Drop whole messages from the head. A leading orphan tool_result (whose
// tool_use we trimmed) would be rejected by the API, so skip past any
// tool_result-only user message left at the front.
const overflow = this.messages.length - MAX_HISTORY_MESSAGES;
this.messages.splice(0, overflow);
while (this.messages.length && startsWithToolResult(this.messages[0]!)) {
this.messages.shift();
}
}
}
function startsWithToolResult(message: AnthropicMessage): boolean {
return (
Array.isArray(message.content) &&
message.content.length > 0 &&
message.content[0]!.type === "tool_result"
);
}
function describeContext(ctx: AssistantContext): string {
const parts = [`Current time: ${ctx.localTime}.`];
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
if (ctx.foregroundApp) {
const title = ctx.foregroundTitle ? ` ("${ctx.foregroundTitle}")` : "";
parts.push(`The foreground app is ${ctx.foregroundApp}${title}.`);
} else {
parts.push("No app is in the foreground (the launcher is showing).");
}
if (ctx.headsetBattery !== null) {
parts.push(`Glasses battery: ${ctx.headsetBattery}%.`);
}
return `Context: ${parts.join(" ")}`;
}
+227
View File
@@ -0,0 +1,227 @@
import { readUpcomingEvents, type CalendarEvent } from "../native/calendar";
import { mediaControllerBridge } from "../native/media-controller";
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
import { shell } from "../ui/shell/shell";
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
/**
* Registers the always-available system tools into the registry. Called once at
* startup. These wrap shell state and the existing native bridges (calendar,
* media, notifications); nothing here needs an app window to be open.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
let registered = false;
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
if (registered) return;
registered = true;
// Keep a media listener warm so "what's playing" works without opening Music.
void mediaControllerBridge.start();
registry.registerSystemTool(
{
name: "glasses.get_state",
description:
"Get the current state of the glasses: display on/off, foreground app, headset battery, and the current local time.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
proactive: true,
},
() => ok(describeState()),
);
registry.registerSystemTool(
{
name: "glasses.show_alert",
description:
"Show a short text popup on the glasses display. Use for a brief notice the user should see; keep it to a sentence or two.",
inputSchema: {
type: "object",
properties: { text: { type: "string", description: "The message to display." } },
required: ["text"],
additionalProperties: false,
},
proactive: true,
},
(args) => {
const text = String(args?.text ?? "").trim();
if (!text) return err("show_alert requires non-empty text");
shell.showAlert(text);
return ok("Displayed.");
},
);
registry.registerSystemTool(
{
name: "calendar.list_events",
description:
"List the user's upcoming calendar events. Returns events from now forward, ordered by start time.",
inputSchema: {
type: "object",
properties: {
within_hours: {
type: "number",
description: "Only include events starting within this many hours (default 168 = one week).",
},
max_events: { type: "number", description: "Maximum number of events to return (default 10)." },
},
additionalProperties: false,
},
},
(args) => {
const withinHours = clampNumber(args?.within_hours, 1, 24 * 60, 168);
const maxEvents = clampNumber(args?.max_events, 1, 50, 10);
const events = readUpcomingEvents(maxEvents, withinHours * 60 * 60 * 1000);
if (!events.length) return ok("No upcoming events in that window.");
return ok(events.map(formatEvent).join("\n"));
},
);
registry.registerSystemTool(
{
name: "media.now_playing",
description: "What is currently playing (or paused) in the user's media app, if anything.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
proactive: true,
},
() => ok(describeNowPlaying()),
);
registry.registerSystemTool(
{
name: "media.play_pause",
description: "Toggle play/pause for the user's current media session.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
},
async () => {
const state = mediaControllerBridge.snapshot();
if (!state.canPlayPause) return err("No media session can be controlled right now.");
await mediaControllerBridge.playPause();
return ok("Toggled play/pause.");
},
);
registry.registerSystemTool(
{
name: "media.next",
description: "Skip to the next track in the user's current media session.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
},
async () => {
const state = mediaControllerBridge.snapshot();
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
await mediaControllerBridge.skipNext();
return ok("Skipped to the next track.");
},
);
registry.registerSystemTool(
{
name: "notifications.list",
description: "List the user's current Android notifications mirrored to the glasses.",
inputSchema: {
type: "object",
properties: { max: { type: "number", description: "Maximum notifications to return (default 10)." } },
additionalProperties: false,
},
},
(args) => {
const max = clampNumber(args?.max, 1, 50, 10);
const notifications = readActiveNotifications(max);
if (!notifications.length) return ok("No current notifications.");
return ok(notifications.map(formatNotification).join("\n"));
},
);
registry.registerSystemTool(
{
name: "notifications.dismiss",
description:
"Dismiss a notification by its key (as returned by notifications.list). Returns whether it was dismissed.",
inputSchema: {
type: "object",
properties: { key: { type: "string", description: "The notification key from notifications.list." } },
required: ["key"],
additionalProperties: false,
},
},
(args) => {
const key = String(args?.key ?? "").trim();
if (!key) return err("notifications.dismiss requires a key");
return dismissNotification(key) ? ok("Dismissed.") : err("No notification with that key.");
},
);
}
function describeState(): string {
const parts: string[] = [];
parts.push(`Display: ${shell.isScreenOn() ? "on" : "off"}.`);
const fg = shell.getForegroundApp();
parts.push(fg ? `Foreground app: ${fg.appId} ("${fg.title}").` : "Foreground: launcher (no app open).");
const battery = shell.getBatteryLevels();
if (battery.headset !== null) {
const charging = battery.headsetCharging ? ", charging" : "";
parts.push(`Headset battery: ${battery.headset}%${charging}.`);
}
parts.push(`Local time: ${formatDateTime(new Date())}.`);
return parts.join(" ");
}
function describeNowPlaying(): string {
const state = mediaControllerBridge.snapshot();
if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable.";
if (state.playbackState === "playing" || state.playbackState === "paused") {
const verb = state.playbackState === "playing" ? "Playing" : "Paused";
const title = state.title || "(unknown title)";
const artist = state.artist ? ` by ${state.artist}` : "";
const app = state.appName ? ` in ${state.appName}` : "";
return `${verb}: ${title}${artist}${app}.`;
}
return "Nothing is currently playing.";
}
function formatEvent(event: CalendarEvent): string {
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
const location = event.location ? ` @ ${event.location}` : "";
return `- ${event.title || "(untitled)"} — ${when}${location}`;
}
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
const title = notification.title || "(no title)";
const body = notification.text ? `: ${notification.text}` : "";
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
}
function formatDateTime(date: Date): string {
return `${formatDate(date)}, ${formatTime(date)}`;
}
function formatDate(date: Date): string {
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
}
function formatTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
return `${hours}:${minutes} ${meridiem}`;
}
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
const num = typeof value === "number" ? value : Number(value);
if (!Number.isFinite(num)) return fallback;
return Math.min(max, Math.max(min, Math.round(num)));
}
function ok(content: string): ToolResult {
return { ok: true, content };
}
function err(error: string): ToolResult {
return { ok: false, error };
}
+190
View File
@@ -0,0 +1,190 @@
/**
* The assistant's tool registry: the single place that knows which tools exist,
* whether each is currently callable, and how to invoke it. Both assistant
* backends (direct Anthropic loop, external agent bridge) list and call tools
* through here, so gating and timeouts are enforced in one spot rather than
* trusted to a model or a remote agent.
*
* Availability tiers (see notes/voice-assistant-design.md):
* - "always" system/shell tools; no app involved.
* - "installed" app tool live whenever the app is installed (no window
* needed) -- e.g. "start a 5 minute timer".
* - "open" app tool live while the app has a window open, foreground
* or backgrounded -- e.g. "next track".
* - "foreground" app tool live only while the app owns the foreground window
* -- e.g. "scroll down".
*
* Phase 1 only registers "always" tools; the tier machinery is here so the
* app-tool phases slot in without reshaping the registry.
*/
export type ToolAvailability = "always" | "installed" | "open" | "foreground";
export type ToolSpec = {
/** Namespaced, e.g. "glasses.get_state" or "app.terminal.send_input". */
name: string;
description: string;
/** JSON Schema for the tool's arguments. */
inputSchema: object;
availability: ToolAvailability;
/** May be invoked outside an active voice turn (external mode proactive calls). */
proactive?: boolean;
/** Round-trip cap; the registry rejects with a tool error past it. Default 10s. */
timeoutMs?: number;
};
/**
* A tool's outcome. Flat (not a discriminated union) on purpose: this project
* builds with strictNullChecks off, so `ok: true|false` would widen to boolean
* and defeat union narrowing. Read `content` when ok, `error` when not.
*/
export type ToolResult = {
ok: boolean;
/** Text payload back to the model (when ok). */
content?: string;
/** Error message back to the model (when not ok). */
error?: string;
};
export type ToolHandler = (args: any) => Promise<ToolResult> | ToolResult;
export type ToolRegistration = {
spec: ToolSpec;
handler: ToolHandler;
/**
* Whether the tool is currently live. Absent means always live (the "always"
* tier). App-tool phases supply a predicate keyed on window/foreground state.
*/
isAvailable?: () => boolean;
};
export type ListToolsOptions = {
/** Only tools flagged proactive (for calls outside an active turn). */
proactiveOnly?: boolean;
};
export type CallToolOptions = {
/** The call is happening outside an active voice turn; enforce proactive gating. */
proactive?: boolean;
};
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
export class ToolRegistry {
private readonly registrations = new Map<string, ToolRegistration>();
private readonly changeListeners = new Set<() => void>();
/** Register (or replace) a tool. Names are unique across all tiers. */
register(registration: ToolRegistration): void {
const { name } = registration.spec;
this.registrations.set(name, registration);
this.fireToolsChanged();
}
/** Convenience for the always-live system tools. */
registerSystemTool(
spec: Omit<ToolSpec, "availability">,
handler: ToolHandler,
): void {
this.register({ spec: { ...spec, availability: "always" }, handler });
}
unregister(name: string): void {
if (this.registrations.delete(name)) {
this.fireToolsChanged();
}
}
/** Specs for the tools callable right now, given the availability tiers. */
listTools(options: ListToolsOptions = {}): ToolSpec[] {
const specs: ToolSpec[] = [];
for (const registration of this.registrations.values()) {
if (!this.isLive(registration)) continue;
if (options.proactiveOnly && !registration.spec.proactive) continue;
specs.push(registration.spec);
}
return specs;
}
/**
* Invoke a tool by name. Never throws: a missing/unavailable tool, a
* proactive-gating rejection, a handler exception, or a timeout all come back
* as `{ ok: false }` so the agent loop can surface them as tool errors.
*/
async callTool(name: string, args: unknown, options: CallToolOptions = {}): Promise<ToolResult> {
const registration = this.registrations.get(name);
if (!registration) {
return { ok: false, error: `Unknown tool: ${name}` };
}
if (!this.isLive(registration)) {
return { ok: false, error: `Tool ${name} is not currently available` };
}
if (options.proactive && !registration.spec.proactive) {
return { ok: false, error: `Tool ${name} cannot be called outside a conversation` };
}
const timeoutMs = registration.spec.timeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
try {
return await this.withTimeout(registration.handler(args), timeoutMs, name);
} catch (error) {
return { ok: false, error: `Tool ${name} failed: ${describeError(error)}` };
}
}
/** Subscribe to add/remove/availability changes (drives MCP list_changed). */
onToolsChanged(listener: () => void): () => void {
this.changeListeners.add(listener);
return () => this.changeListeners.delete(listener);
}
/** App-tool phases call this after a foreground/window change re-gates tools. */
fireToolsChanged(): void {
for (const listener of this.changeListeners) {
try {
listener();
} catch (error) {
console.warn("onToolsChanged listener failed", error);
}
}
}
private isLive(registration: ToolRegistration): boolean {
return registration.isAvailable ? registration.isAvailable() : true;
}
private withTimeout(
result: Promise<ToolResult> | ToolResult,
timeoutMs: number,
name: string,
): Promise<ToolResult> {
if (!(result instanceof Promise)) return Promise.resolve(result);
return new Promise<ToolResult>((resolve) => {
let settled = false;
const timer = setTimeout(() => {
if (settled) return;
settled = true;
resolve({ ok: false, error: `Tool ${name} timed out after ${timeoutMs}ms` });
}, timeoutMs);
result.then(
(value) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(value);
},
(error) => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve({ ok: false, error: `Tool ${name} failed: ${describeError(error)}` });
},
);
});
}
}
function describeError(error: unknown): string {
return String((error as Error)?.message ?? error);
}
/** The process-wide registry; system tools register into it at startup. */
export const toolRegistry = new ToolRegistry();
+32
View File
@@ -0,0 +1,32 @@
/**
* Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer)
* talks to the same session surface whether the turn runs on the phone
* (direct Anthropic loop) or against the user's own agent (external bridge,
* later phases).
*/
/** Situational grounding injected per turn (system prompt in direct mode). */
export type AssistantContext = {
/** Foreground app id, or null when the sidebar/launcher is all that's up. */
foregroundApp: string | null;
foregroundTitle: string | null;
screenOn: boolean;
/** Human-readable local time, e.g. "Thu Jul 24, 3:15 PM". */
localTime: string;
/** Headset battery percent (0..100), or null if unknown. */
headsetBattery: number | null;
};
export type AssistantTurnCallbacks = {
/** Streamed reply text (delta plus the full text so far). */
onTextDelta: (delta: string, textSoFar: string) => void;
/** A tool is being invoked; label is display-ready, e.g. "calendar.list_events". */
onToolActivity: (label: string) => void;
onTurnDone: (result: { stopReason: string | null }) => void;
onError: (message: string) => void;
};
/** A handle to an in-flight turn. */
export type AssistantTurnHandle = {
cancel(): void;
};
+4
View File
@@ -25,6 +25,7 @@ import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
import { registerSystemTools } from "../assistant/system-tools";
import { WorkerAppHost } from "../ui/shell/worker-window";
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app";
import {
@@ -236,6 +237,9 @@ class DashboardController {
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
};
this.sharedActions = sharedActions;
// The always-available assistant tools (calendar, media, notifications,
// glasses state) register once at startup, independent of any connection.
registerSystemTools();
shell.configure({
actions: {
...sharedActions,
+37 -19
View File
@@ -1,5 +1,5 @@
// AUTO-GENERATED from g2flash/patches/cfw_patches.json — do not edit by hand.
// Regenerate by re-running the generator in the firmware onboarding change.
// Regenerate with scripts/update_deltas.sh.
// Turns the stock G2 2.2.6.10 image into the Faceclaw custom firmware.
export type FirmwarePatchOp = {
@@ -19,7 +19,7 @@ export type FirmwarePatchSet = {
export const CFW_PATCH_SET: FirmwarePatchSet = {
"base": "g2_2.2.6.10.bin",
"baseSha256": "f4dfb0b49ad3de3c2daf17f8a27a157c3dc98411d6a0d3ab2cfd0918f41b9afa",
"outputSha256": "162fa8aab6f41be5e314d51b56f3e8f3eb2143af93ee0518d1919d096701474d",
"outputSha256": "ba7e896e516d049e530d7aaed30e358b45f940e81a90f997a44465299151b452",
"patches": [
{
"offset": 1449549,
@@ -42,85 +42,103 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1447919,
"old": "7ef7fefd",
"new": "b8f28cfe",
"new": "b8f2f6fe",
"desc": "bl snapshot_side @ 0x4db968"
},
{
"offset": 1448931,
"old": "7ef704fc",
"new": "b8f292fc",
"new": "b8f2fcfc",
"desc": "bl snapshot_side @ 0x4dbd5c"
},
{
"offset": 1165461,
"old": "45f0cefd",
"new": "fdf23dfe",
"new": "fdf2a7fe",
"desc": "bl image_deferred (deferred consumer -> FIFO restore + worker, both lenses)"
},
{
"offset": 1186287,
"old": "d9f7d4ff",
"new": "f8f2d0fe",
"new": "f9f2b8fb",
"desc": "bl settings_send_wrapper (append caps field 100)"
},
{
"offset": 1183983,
"old": "f4f75aff",
"new": "f9f2b6fd",
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
},
{
"offset": 926945,
"old": "08f068fa",
"new": "38f3d9fa",
"desc": "bl faceclaw_display_start @ 0x45c65a (fail-open double-tap takeover)"
},
{
"offset": 927137,
"old": "08f008fa",
"new": "38f379fa",
"desc": "bl faceclaw_display_start @ 0x45c71a (fail-open double-tap takeover)"
},
{
"offset": 822553,
"old": "28f003f8",
"new": "51f39dfd",
"new": "52f39dfa",
"desc": "bl evenhub_longpress (replaces force-quit dialog)"
},
{
"offset": 823369,
"old": "1cf09bfb",
"new": "51f321fc",
"new": "52f321f9",
"desc": "bl ring_release (forward ring release-long-press)"
},
{
"offset": 1474163,
"old": "34d1",
"new": "34e0",
"desc": "even_ai_display_ctrl: skip Even AI app START (wakeword -> phone)"
"offset": 1474137,
"old": "7fb50600",
"new": "b3f271b9",
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
},
{
"offset": 4301227,
"old": "",
"new": "02fb01f044f6d341c0f24701084700bf084644f61751c0f24701084770b500284ed001684ef65a62ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf2de9f04f85b043f6fc76c2f200063568804628466ff3960044f6d344b0f1005fc0f2470406d128684ef65a61ccf2ff0188423bd04ff4fe70a0470546a8b30020002100bf6a1868540431b1f5fe7f50709070d070f6d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe003560b8f1000f18bf002d07d14af26950c0f2450005b0bde8f04f0047d8f80cb0bbf1000ff2d0d8f820a0baf1000fedd0a94659f8040f00287bd0686900287ad0686a002879d0686b002878d0686c002877d0686d002876d0686e002875d0686f002874d0d5f88400002872d0d5f89400002870d0d5f8a40000286ed0d5f8b40000286cd0d5f8b00029690490d5f8a0002a6b0390d5f8900000260290d5f880002f6f0190286ad5f860e0d5f850c02b6c884238bf014638bf01268a423cbf114602268b423cbf194603268c453cbf614604268e453cbf714605268f423cbf39460626019888423cbf01460726029888423cbf01460826039888423cbf014609260498884238bf0146d5f8c00038bf0a26884238bf0b2609eb0610406804f1440137018847002049f80700012085f8df0016e0002614e0012612e0022610e003260ee004260ce005260ae0062608e0072606e0082604e0092602e00a2600e00b265046a04700283ff451af0af0030c5fea9a034ff0000219d00496a2eb830e431c0bf101066ff0030700bff119cc78da19d4700c79043714714c79547189790eeb070404349171f0d1049e3a1dbcf1000f10d01bf80230bcf1010f83540ad0531c1bf80370bcf1020fc75403d0911c1bf80120425409eb06114860d5f8c400320149f80280421cc1f808a0c5f8c420c8600ce738464146fff7aebe2de9f04f81b043f6fc79c2f20009d9f80050044628466ff3960044f6d34816460f46b0f1005fc0f2470806d128684ef65a61ccf2ff0188423dd04ff4fe70c0470546002800f09580002000216a1868540431b1f5fe7f50709070d070f6d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe00c9f8005005f1040b4ff0000a4ff0000900bf4ff0ff30002200234ff0ff3105e000bf6032c02a03f1060332d0af187e68a64204d13e6986423cbf194630467e69a64204d13e6a86423cbf591c30467e6aa64204d13e6b86423cbf991c30467e6ba64204d13e6c86423cbfd91c30467e6ca64204d13e6d86423cbf191d30467e6da642ced13f6e87423cbf591d3846c8e700bf002910d40beb01104768090182684bf801a02046394600f013f8814608f1440138468847ace7484601b0bde8f08f204639463246c9f8005001b0bde8f04fffe72de9f041a4b0064640f6bb4043f6fc78c0f24e0014460d46c2f200088047002002af0390384600f01bf90df10c0c3046294622460123cdf800c000f0bff90446384600f015ffd8f8001005460a466ff39602b2f1005f06d108684ef65a62ccf2ff0290423fd044f6d341c0f247014ff4fe708847d0b30146002000228b1888540432b2f5fe7f58709870d870f6d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe00c8f80010c1f8d05002e00020c8f80000204624b0bde8f08100bf09295ad1f0b502eb030c352756248cf8107037278cf804404e248cf8117036278cf8064043248cf812707a278cf8074046248cf8177072274ff0a20e8cf8084057248cf819706c2702f803e04ff0060e8cf809402f248cf81a7065278cf801e04ff0450e8cf80a4032248cf81b7000278cf803e08cf805e08cf80b4020244ff0690e6d2567268cf81c7019271c338cf80c408cf80de08cf80e508cf80f608cf813408cf814e08cf815508cf816608cf818408cf80270bde8f04045f6153cc0f2470c60474df63040c2f234000078042818bf704780b582b04af26b1e4ff0000cc0f24d0e0020002100220923cdf804c0cdf800c0f04702b080bd00bff0b583b04df63043c2f234031b784ff6e704042bc0f2450420d144f2d043c2f207031b680546184616460f46a04703463946324628468bb11b68e02b0ed14af26b160027c0f24d060020002100220a2301970097b047012003b0f0bd04f1160303b0bde8f0401847f0b581b041f204054cf65561cef20005ccf2ac51c5f8ac1f4ef6fc51cef200010a6843f6fc7642f080720a6055f8041cc2f2000641f0010145f8041c316807460a466ff39602b2f1005f06d108684ef65a62ccf2ff02904240d044f6d341c0f247014ff4fe708847002870d001460020002200bf8b1888540432b2f5fe7f58709870d870f6d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe003160d1f8d800b8bb44f6342245f6e464c2f20702cff6f8741368204600bf16689e420ed116689e420bd116689e4208d116689e4203d1002800f10400efd19e421ad02b6816681068b0420ed11068b0420bd11068b04208d11068b04203d1002c04f10404efd1b04206d02868c01ac1f8d80001e0002030602868386001b0f0bd00bf2de9f04fdbb0002918bf002a04d15bb0bde8f04f00f00cbe0c7804f07f07052f00f0e580072f4dd0422ff0d0ddf89091b0f84080b0f842a0092f00f04e81082f40f058810746022a4ff0ff30c0f0b382002b00f0b082c3460bf10100032303eb500020f0030000fb0af07630824200f23c814d78fdb188460226b01c904200f2348108eb060318f806605b7856ea032300f02b811e18964200f2278108eb0001384614461a460023cdf80090fff7a8ff2246013de1d13846594652464b4600f01dfe77e243f6fc74c2f20004236818466ff39600b0f1005f06d118684ef65a67ccf2ff07b84243d00e4644f6d341c0f247014ff4fe70154688470346a8b30020002100bf5a1858540431b1f5fe7f50709070d070fLine truncated
"new": "02fb01f044f6d341c0f24701084700bf084644f61751c0f24701084770b500284ed001684ef65a62ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf2de9f04f83b043f6fc76c2f200063568824628466ff3960044f6d344b0f1005fc0f2470407d128684ef65a61ccf2ff01884200f0a4804ff40670a0470546002800f09c8000206ff021010022ab18a85443326f18b2f5067f01f1430158709870d870187158719871d871187258729872d872187358739873d873187458749874d874187558759875d875187658769876d876187758779877d87783f8200087f8430087f8440083f8230083f8240083f8250083f8260083f8270083f8280083f8290083f82a0083f82b0083f82c0083f82d0083f82e0083f82f0083f8300083f8310083f8320083f8330083f8340083f8350083f8360083f8370083f8380083f8390083f83a0083f83b0083f83c0083f83d0083f83e0083f83f0083f8400083f8410083f8420091d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe003560baf1000f18bf002d07d14af26950c0f2450003b0bde8f04f0047daf80cb0bbf1000ff2d0daf82080b8f1000fedd0a94659f8040f00287cd0686900287bd0686a00287ad0686b002879d0686c002878d0686d002877d0686e002876d0686f002875d0d5f88400002873d0d5f89400002871d0d5f8a40000286fd0d5f8b40000286dd0d5f8b00029690290d5f8a0002c6a0190d5f8900000260090286bd5f880e0d5f870c02b6e2f6d2a6c8c4238bf214638bf012688423cbf014602268a423cbf114603268f423cbf394604268b423cbf194605268c453cbf614606268e453cbf71460726009844f6d34488423cbf014608260198c0f2470488423cbf014609260298884238bf0146d5f8c00038bf0a26884238bf0b2609eb0610406804f1440137018847002049f80700012085f8df0016e0002614e0012612e0022610e003260ee004260ce005260ae0062608e0072606e0082604e0092602e00a2600e00b264046a04700283ff450af08f0030c5fea98034ff0000219d00296a2eb830e431c0bf101066ff0030700bff119cc78da19d4700c79043714714c79547189790eeb070404349171f0d1029e3a1dbcf1000f10d01bf80230bcf1010f83540ad0531c1bf80370bcf1020fc75403d0911c1bf80120425409eb06114860d5f8c400320149f802a0421cc1f80880c5f8c420c8600be738464146fff744be2de9f04f81b043f6fc79c2f20009d9f80050044628466ff3960044f6d34816460f46b0f1005fc0f2470807d128684ef65a61ccf2ff01884200f0a6804ff40670c0470546002800f0fc8000206ff02101002200bfab18a85443326f18b2f5067f01f1430158709870d870187158719871d871187258729872d872187358739873d873187458749874d874187558759875d875187658769876d876187758779877d87783f8200087f8430087f8440083f8230083f8240083f8250083f8260083f8270083f8280083f8290083f82a0083f82b0083f82c0083f82d0083f82e0083f82f0083f8300083f8310083f8320083f8330083f8340083f8350083f8360083f8370083f8380083f8390083f83a0083f83b0083f83c0083f83d0083f83e0083f83f0083f8400083f8410083f8420091d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe00c9f8005005f1040b4ff0000a4ff000094ff0ff30002200234ff0ff3105e000bf6032c02a03f1060332d0af187e68a64204d13e6986423cbf194630467e69a64204d13e6a86423cbf591c30467e6aa64204d13e6b86423cbf991c30467e6ba64204d13e6c86423cbfd91c30467e6ca64204d13e6d86423cbf191d30467e6da642ced13f6e87423cbf591d3846c8e700bf002910d40beb01104768090182684bf801a02046394600f013f8814608f1440138468847ace7484601b0bde8f08f204639463246c9f8005001b0bde8f04fffe72de9f041a4b0064640f6bb4043f6fc78c0f24e0014460d46c2f200088047002002af0390384600f049fd0df10c0c3046294622460123cdf800c000f055fe0446384601f07dfcd8f8001005460a466ff39602b2f1005f07d108684ef65a62ccf2ff02904200f0aa8044f6d341c0f247014ff406708847002800f0a380014600206ff02102002300bfcf18c85443338e18b3f5067f02f143027870b870f87038717871b871f87138727872b872f87238737873b873f87338747874b874f87438757875b875f87538767876b876f87638777877b877f87787f8200086f8430086f8440087f8230087f8240087f8250087f8260087f8270087f8280087f8290087f82a0087f82b0087f82c0087f82d0087f82e0087f82f0087f8300087f8310087f8320087f8330087f8340087f8350087f8360087f8370087f8380087f8390087f83a0087f83b0087f83c0087f83d0087f83e0087f83f0087f8400087f8410087f8420091d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe00c8f80010c1f8d05002e00020c8f80000204624b0bde8f08143f6fc70c2f20000016808466ff39600b0f1005f16d108684ef65a62ccf2ff0290420fd1d1f8fc0160b144f63422c2f207021268801a0028cebf01200020c1f8fc01704700207047f0b101684ef65a62ccf2ff02914218bf704790f8061299b110b544f62f344ff0000cc0f24604864680f806c20120002100220023aef804c2a446bde810406047704700bf2de9f04143f6fc77c2f200073f684ff000083c466ff39604b4f1005f07d13e684ef65a65ccf2ff05ae4208bfb84601280fd11c4615460f460646fff79bff2346044639462a463046002c18bfb8f1000f06d144f62f3cc0f2460cbde8f041604798f8060270b1d8f800020021002888f80612a8f804123ed049f2d941c0f24401884738e0b8f804020121013080b2012898bf0846a8f80402d8f8000249f2d944c0f2440488f8061268b94ff60370cff6ff70a4f594777844002142460023b847c8f8000240b1a047d8f80002a4f140024Line truncated
"desc": "append injected blobs to main-app payload"
},
{
"offset": 777711,
"old": "44c33500",
"new": "84fe3500",
"new": "380e3600",
"desc": "main-app subheader payload size (ps)"
},
{
"offset": 152,
"old": "c4c33500",
"new": "04ff3500",
"new": "b80e3600",
"desc": "main-app TOC entry size (ps + 128)"
},
{
"offset": 777831,
"old": "44c33504",
"new": "84fe3504",
"new": "380e3604",
"desc": "main-app preamble length (low 24 bits)"
},
{
"offset": 777835,
"old": "df8110c9",
"new": "6eddece4",
"new": "88ed0126",
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
},
{
"offset": 156,
"old": "c1be0fb1",
"new": "ed6b6bbd",
"new": "f1e9f8d8",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
},
{
"offset": 777715,
"old": "c1be0fb1",
"new": "ed6b6bbd",
"new": "f1e9f8d8",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
}
]
+86 -5
View File
@@ -13,14 +13,34 @@ const ANTHROPIC_VERSION = "2023-06-01";
/** Default model for voice continuations (and a sensible general default). */
export const DEFAULT_LLM_MODEL = "claude-sonnet-5";
/**
* A content block in an Anthropic message. Text-only flows (dictation refine)
* still pass a plain string for content; the assistant tool loop uses the
* block array to carry tool_use / tool_result alongside text.
*/
export type AnthropicContentBlock =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; input: any }
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean };
export type AnthropicMessage = {
role: "user" | "assistant";
content: string;
content: string | AnthropicContentBlock[];
};
/** An Anthropic tool definition (input_schema is snake_case as the API wants). */
export type AnthropicToolDefinition = {
name: string;
description: string;
input_schema: object;
};
export type AnthropicStreamResult = {
/** Concatenated text of all text blocks. */
text: string;
/** e.g. "end_turn", "max_tokens", "refusal"; null if the stream never said. */
/** The assembled assistant content blocks (text + any tool_use). */
content: AnthropicContentBlock[];
/** e.g. "end_turn", "tool_use", "max_tokens", "refusal"; null if never said. */
stopReason: string | null;
};
@@ -29,6 +49,7 @@ export type AnthropicStreamOptions = {
model?: string;
system?: string;
messages: AnthropicMessage[];
tools?: AnthropicToolDefinition[];
maxTokens?: number;
/** Thinking/latency tradeoff; voice flows want "low". Default: model default. */
effort?: "low" | "medium" | "high";
@@ -39,6 +60,11 @@ export type AnthropicStreamOptions = {
onError: (message: string) => void;
};
/** Accumulator for one streamed content block before it is finalized. */
type StreamingBlock =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; inputJson: string };
export type AnthropicStreamHandle = {
cancel(): void;
};
@@ -49,6 +75,8 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
let settled = false;
let text = "";
let stopReason: string | null = null;
// Content blocks, keyed by their SSE `index`, assembled as deltas arrive.
const blocks = new Map<number, StreamingBlock>();
const fail = (message: string) => {
if (settled) return;
@@ -58,7 +86,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
const done = () => {
if (settled) return;
settled = true;
options.onDone({ text, stopReason });
options.onDone({ text, content: finalizeBlocks(blocks), stopReason });
};
if (!global.isAndroid) {
@@ -78,6 +106,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
messages: options.messages,
};
if (options.system) body.system = options.system;
if (options.tools?.length) body.tools = options.tools;
if (options.effort) body.output_config = { effort: options.effort };
const listener = new com.faceclaw.app.FaceclawSseListener({
@@ -86,13 +115,41 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
const event = parseSseDataLine(String(line));
if (!event) return;
switch (event.type) {
case "content_block_delta":
case "content_block_start": {
const index = Number(event.index);
const block = event.content_block;
if (block?.type === "tool_use") {
blocks.set(index, {
type: "tool_use",
id: String(block.id ?? ""),
name: String(block.name ?? ""),
inputJson: "",
});
} else if (block?.type === "text") {
blocks.set(index, { type: "text", text: String(block.text ?? "") });
}
return;
}
case "content_block_delta": {
const index = Number(event.index);
if (event.delta?.type === "text_delta") {
const delta = String(event.delta.text ?? "");
text += delta;
const existing = blocks.get(index);
if (existing?.type === "text") {
existing.text += delta;
} else {
blocks.set(index, { type: "text", text: delta });
}
options.onTextDelta?.(delta, text);
} else if (event.delta?.type === "input_json_delta") {
const existing = blocks.get(index);
if (existing?.type === "tool_use") {
existing.inputJson += String(event.delta.partial_json ?? "");
}
}
return;
}
case "message_delta":
if (event.delta?.stop_reason) {
stopReason = String(event.delta.stop_reason);
@@ -105,7 +162,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
fail(`Anthropic: ${String(event.error?.message ?? event.error?.type ?? "stream error")}`);
return;
default:
// message_start, content_block_start/stop, ping — nothing to do.
// message_start, content_block_stop, ping — nothing to do.
return;
}
},
@@ -146,6 +203,30 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
};
}
/** Assemble the streamed blocks (in index order) into final content blocks. */
function finalizeBlocks(blocks: Map<number, StreamingBlock>): AnthropicContentBlock[] {
const content: AnthropicContentBlock[] = [];
for (const index of Array.from(blocks.keys()).sort((a, b) => a - b)) {
const block = blocks.get(index)!;
if (block.type === "text") {
content.push({ type: "text", text: block.text });
} else {
let input: any = {};
const trimmed = block.inputJson.trim();
if (trimmed) {
try {
input = JSON.parse(trimmed);
} catch {
// A tool_use whose JSON never completed (e.g. truncated stream); pass
// empty args and let the tool handler report what's missing.
}
}
content.push({ type: "tool_use", id: block.id, name: block.name, input });
}
}
return content;
}
/** Parse one SSE line; returns the JSON payload of a data: line, else null. */
function parseSseDataLine(line: string): any | null {
if (!line.startsWith("data:")) return null;
+190
View File
@@ -0,0 +1,190 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
import { GESTURE_DOUBLE_CLICK } from "../gestures";
import { drawSelectionHighlight } from "../menu";
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
const DIALOG_X = 40;
const DIALOG_Y = 40;
const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
const MENU_ROWS = 2;
/** thinking: turn running; done/error: finished, showing the Follow-up/Done menu. */
type AssistantPhase = "thinking" | "done" | "error";
export type AssistantLayerCallbacks = {
/** Record another utterance and continue this conversation. */
onFollowUp: () => void;
/** Abort the in-flight turn (double-click while thinking). */
onCancel: () => void;
/** The user closed the overlay (Done). */
onClose: () => void;
/** The layer left the stack by any path (Done, or the screen sleeping). */
onRemoved?: () => void;
};
/**
* The assistant overlay: streamed reply text with a status line for tool
* activity, and a Follow-up / Done menu once the turn ends. Double-click
* cancels an in-flight turn. The shell owns the AssistantSession and drives
* this layer's state through the on* methods as the turn streams.
*/
export class AssistantLayer implements Layer {
private phase: AssistantPhase = "thinking";
private replyText = "";
private status = "Thinking...";
private menuIndex = 0;
constructor(
private readonly actions: LayerActions,
private readonly callbacks: AssistantLayerCallbacks,
) {}
/** Shell: a new turn (initial utterance or follow-up) is starting. */
startTurn(): void {
this.phase = "thinking";
this.replyText = "";
this.status = "Thinking...";
this.menuIndex = 0;
this.actions.requestRender();
}
onTextDelta(_delta: string, textSoFar: string): void {
this.replyText = textSoFar;
if (this.phase === "thinking") this.status = "Thinking...";
this.actions.requestRender();
}
onToolActivity(label: string): void {
if (this.phase === "thinking") this.status = `→ ${label}`;
this.actions.requestRender();
}
onTurnDone(): void {
this.phase = "done";
this.status = this.replyText.trim() ? "" : "(no reply)";
this.menuIndex = 0;
this.actions.requestRender();
}
onError(message: string): void {
this.phase = "error";
this.status = message;
this.menuIndex = 0;
this.actions.requestRender();
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
// Solid dialog box (fill 1, matching the voice dialog: opaque after 4bpp
// quantization, but not the color-key transparent 0).
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10);
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
const left = DIALOG_X + 16;
image.drawText(font, left, DIALOG_Y + 12, this.phase === "thinking" ? "Assistant ●" : "Assistant", 220);
if (this.status) {
const statusValue = this.phase === "error" ? 200 : 130;
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), statusValue);
}
const inMenu = this.phase !== "thinking";
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
const wrapped = wrapText(font, this.replyText, TEXT_MAX_WIDTH);
// Keep the tail visible as text streams past the bottom.
const firstLine = Math.max(0, wrapped.length - maxLines);
for (let index = firstLine; index < wrapped.length; index++) {
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
}
if (inMenu) {
const rows = ["Follow-up", "Done"];
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
if (selected) {
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
}
image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200);
}
} else {
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110);
}
return image;
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
if (this.phase === "thinking") {
if (event.type === "double-click") {
this.callbacks.onCancel();
this.phase = "done";
this.status = "Cancelled";
this.menuIndex = 0;
this.actions.requestRender();
}
return;
}
// done / error: the Follow-up / Done menu.
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
this.callbacks.onFollowUp();
} else {
this.callbacks.onClose();
}
return;
case "double-click":
this.callbacks.onClose();
return;
default:
return;
}
}
onRemoved(): void {
this.callbacks.onRemoved?.();
}
}
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) lines.push(line);
return lines.length ? lines : [""];
}
function truncate(font: BdfFont, text: string, maxWidth: number): string {
if (font.measureText(text) <= maxWidth) return text;
let out = text;
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
out = out.slice(0, -1);
}
return `${out}...`;
}
+136 -42
View File
@@ -13,19 +13,47 @@ const DIALOG_W = G2_LENS_WIDTH - 80;
const DIALOG_H = G2_LENS_HEIGHT - 80;
const TEXT_MAX_WIDTH = DIALOG_W - 32;
const MENU_ROW_H = 20;
const MENU_ROWS = 3;
// After the mic stops, the provider's final transcript can trail in (cloud
// commit round-trip); wait this long for it before refining with what we have.
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
/**
* The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or
* click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing"
* click-to-finish); "menu" is the send / Continue / Discard menu; "continuing"
* records a follow-up utterance (always click-to-finish); "refining" streams
* the LLM-merged text, then returns to "menu".
*/
type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
/**
* A destination the captured text can be sent to. The shell supplies these per
* entry point: "Send to Assistant" (hand off to the voice assistant) and/or
* "Type Into App" (deliver to the foreground window's text input).
*/
export type VoiceSendTarget = {
id: string;
label: string;
onSend: (text: string) => void;
};
export type VoiceInputLayerOptions = {
actions: LayerActions;
/** Post-removal cleanup (also fires when the screen turns off). */
onClosed: () => void;
/** Pop this layer off the shell stack. */
dismiss: () => void;
/** Ordered send destinations shown as the first menu rows. */
sendTargets: VoiceSendTarget[];
/** Which send target is highlighted by default (entry-point dependent). */
defaultTargetIndex?: number;
/** A single click ends the utterance (opened from a menu, no held button). */
finishOnClick?: boolean;
/** Wakeword flow: mic starts immediately and ends on detected silence. */
handsFree?: boolean;
/** Skip the menu: on capture end, send straight to the default target. */
autoSend?: boolean;
};
/**
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
* The mic runs while the button is held (long-press); releasing it stops the
@@ -58,17 +86,34 @@ export class VoiceInputLayer implements Layer {
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
private refineHandle: AnthropicStreamHandle | null = null;
private menuIndex = 0;
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
private pendingAutoSend = false;
private autoSendTimer: ReturnType<typeof setTimeout> | null = null;
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private unsubscribeSpeechEnd: (() => void) | null = null;
constructor(
private readonly actions: LayerActions,
private readonly onClosed: () => void,
private readonly onSend: (text: string) => void,
private readonly finishOnClick = false,
private readonly handsFree = false,
) {}
private readonly actions: LayerActions;
private readonly onClosed: () => void;
private readonly dismiss: () => void;
private readonly sendTargets: VoiceSendTarget[];
private readonly defaultTargetIndex: number;
private readonly finishOnClick: boolean;
private readonly handsFree: boolean;
private readonly autoSend: boolean;
constructor(options: VoiceInputLayerOptions) {
this.actions = options.actions;
this.onClosed = options.onClosed;
this.dismiss = options.dismiss;
this.sendTargets = options.sendTargets;
this.finishOnClick = options.finishOnClick ?? false;
this.handsFree = options.handsFree ?? false;
this.autoSend = options.autoSend ?? false;
const defaultIndex = options.defaultTargetIndex ?? 0;
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
this.menuIndex = this.defaultTargetIndex;
}
startCapture(): void {
if (this.capturing) return;
@@ -113,13 +158,66 @@ export class VoiceInputLayer implements Layer {
this.capturing = false;
this.phase = "menu";
void this.actions.stopVoiceCapture();
if (this.status.startsWith("Listening")) {
if (this.autoSend && this.sendTargets.length) {
// Skip-confirmation (wakeword): send to the default target as soon as the
// transcript finalizes, or after a short wait for the trailing final.
this.pendingAutoSend = true;
this.status = "Sending...";
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
} else if (this.status.startsWith("Listening")) {
this.status = "Send, continue, or discard?";
}
this.actions.requestRender();
}
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
/** Fire the queued skip-confirmation send (or fall back to the menu). */
private performAutoSend(): void {
if (this.autoSendTimer !== null) {
clearTimeout(this.autoSendTimer);
this.autoSendTimer = null;
}
if (!this.pendingAutoSend) return;
this.pendingAutoSend = false;
const text = this.displayText().trim();
const target = this.sendTargets[this.defaultTargetIndex];
if (text && target) {
this.dismiss();
target.onSend(text);
} else {
// Nothing heard; fall back to the menu so the user can retry or discard.
this.status = "Send, continue, or discard?";
this.actions.requestRender();
}
}
/** The menu rows: one per send target, then Continue, then Discard. */
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
const text = this.displayText().trim();
const hasText = text.length > 0;
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
for (const target of this.sendTargets) {
rows.push({
label: target.label,
dim: !hasText,
onSelect: () => {
this.dismiss();
if (hasText) target.onSend(text);
},
});
}
rows.push({
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
dim: !hasLlmKey,
onSelect: () => {
if (hasLlmKey) this.startContinuation();
},
});
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
return rows;
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const font = getDefaultSmallFont();
const image = paintBelow();
@@ -133,9 +231,9 @@ export class VoiceInputLayer implements Layer {
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
const inMenu = this.phase === "menu";
const hasText = this.displayText().trim().length > 0;
// Leave room for the three-row menu when it is showing.
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const rows = inMenu ? this.menuRows() : [];
// Reserve space for the actual number of rows this menu has.
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
const textTop = DIALOG_Y + 56;
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
@@ -147,14 +245,7 @@ export class VoiceInputLayer implements Layer {
}
if (inMenu) {
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
const rows: Array<{ label: string; dim: boolean }> = [
// Dim "Send" when there is nothing to send.
{ label: "Send", dim: !hasText },
{ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey },
{ label: "Discard", dim: false },
];
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
const menuTop = DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 2;
for (let i = 0; i < rows.length; i++) {
const rowY = menuTop + i * MENU_ROW_H;
const selected = i === this.menuIndex;
@@ -170,11 +261,11 @@ export class VoiceInputLayer implements Layer {
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
switch (this.phase) {
case "capturing":
if (event.type === "double-click") {
ctx.stack.pop();
this.dismiss();
} else if (this.clickEndsCapture && event.type === "click") {
this.endCapture();
}
@@ -192,37 +283,29 @@ export class VoiceInputLayer implements Layer {
}
return;
case "menu":
this.handleMenuInput(event, ctx);
this.handleMenuInput(event);
return;
}
}
private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void {
private handleMenuInput(event: DashboardInputEvent): void {
const rowCount = this.menuRows().length;
switch (event.type) {
case "scroll-up":
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
this.actions.requestRender();
return;
case "scroll-down":
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
this.menuIndex = (this.menuIndex + 1) % rowCount;
this.actions.requestRender();
return;
case "click":
if (this.menuIndex === 0) {
const text = this.displayText().trim();
if (text) this.onSend(text);
ctx.stack.pop();
} else if (this.menuIndex === 1) {
if (anthropicApiKeySetting.get().trim()) {
this.startContinuation();
}
// No key: the row is disabled; the click does nothing.
} else {
ctx.stack.pop();
}
case "click": {
const row = this.menuRows()[this.menuIndex];
row?.onSelect();
return;
}
case "double-click":
ctx.stack.pop();
this.dismiss();
return;
default:
return;
@@ -323,6 +406,11 @@ export class VoiceInputLayer implements Layer {
clearTimeout(this.followupFinalizeTimer);
this.followupFinalizeTimer = null;
}
if (this.autoSendTimer !== null) {
clearTimeout(this.autoSendTimer);
this.autoSendTimer = null;
}
this.pendingAutoSend = false;
this.refineHandle?.cancel();
this.refineHandle = null;
if (this.capturing) {
@@ -380,6 +468,12 @@ export class VoiceInputLayer implements Layer {
this.beginRefine();
return;
}
if (this.pendingAutoSend) {
// The utterance finalized; skip-confirmation can fire without waiting
// out the fallback timer.
this.performAutoSend();
return;
}
} else {
this.liveText = event.text.trim();
}
+7
View File
@@ -294,6 +294,13 @@ export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
defaultValue: false,
});
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
id: "assistant-skip-confirmation",
label: "Send to assistant without confirming",
storageKey: "assistant.skipConfirmationAfterWakeword",
defaultValue: false,
});
export const elevenLabsApiKeySetting = new ConfigSettingString({
id: "elevenlabs-api-key",
label: "ElevenLabs key",
+4
View File
@@ -3,6 +3,7 @@ import { getDashboardLogo } from "../../graphics/logo";
import { shell } from "../shell/shell";
import {
anthropicApiKeySetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
brightnessSetting,
elevenLabsApiKeySetting,
@@ -56,6 +57,9 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
// When on, a wakeword utterance goes straight to the assistant with no
// Send/Type menu step.
toggleSettingMenuItem(assistantSkipConfirmationSetting),
],
},
{
+291 -14
View File
@@ -1,11 +1,17 @@
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/bdffont";
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
import type { RawInputEvent } from "../../native/faceclaw-communicator";
import { DashboardInputEvent, LayerActions, LayerStack, noopLayerActions } from "../layers";
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer } from "../apps/voice-input";
import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input";
import { AssistantLayer } from "../apps/assistant";
import { AssistantSession } from "../../assistant/session";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
anthropicApiKeySetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
onAnySettingChanged,
timeFormatSetting,
@@ -99,6 +105,64 @@ class ShellOverlayMenuLayer extends MenuLayer {
}
}
/** How long a show_alert popup stays before auto-dismissing. */
const ALERT_DISMISS_MS = 6000;
const ALERT_X = 40;
const ALERT_W = G2_LENS_WIDTH - 80;
const ALERT_Y = 96;
const ALERT_H = 96;
/**
* A brief text popup on the shell surface (the assistant's show_alert tool and
* other short notices). Auto-dismisses after a few seconds; a click or
* double-click dismisses it early.
*/
class ShellAlertLayer implements Layer {
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(private readonly text: string, private readonly onDismiss: () => void) {
this.timer = setTimeout(() => {
this.timer = null;
this.onDismiss();
}, ALERT_DISMISS_MS);
}
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
const image = paintBelow();
const font = getDefaultSmallFont();
image.fillRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 1, 10);
image.drawRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 90, 10);
image.drawText(font, ALERT_X + 16, ALERT_Y + 12, "Assistant", 200);
image.drawTextWrapped({
font,
x: ALERT_X + 16,
y: ALERT_Y + 34,
width: ALERT_W - 32,
text: this.text,
value: 235,
});
return image;
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
if (event.type === "click" || event.type === "double-click") {
this.clearTimer();
this.onDismiss();
}
}
onRemoved(): void {
this.clearTimer();
}
private clearTimer(): void {
if (this.timer !== null) {
clearTimeout(this.timer);
this.timer = null;
}
}
}
class Shell {
private windows: ShellWindow[] = [];
private selectedIndex = 0;
@@ -111,6 +175,8 @@ class Shell {
// notification icons and the battery indicators.
private readonly trayIcons = new Map<string, GrayImage>();
private activeVoiceLayer: VoiceInputLayer | null = null;
private assistantSession: AssistantSession | null = null;
private assistantLayer: AssistantLayer | null = null;
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
private readonly actions: LayerActions = { ...noopActions };
private config: ShellConfig = {
@@ -236,6 +302,21 @@ class Shell {
return this.screenOn;
}
/** Current headset battery levels (for the assistant's get_state tool). */
getBatteryLevels(): ShellChromeState["battery"] {
return this.battery;
}
/**
* The foreground app, or null when only the launcher is showing. The launcher
* is a pinned window but counts as "no app" for the assistant's context.
*/
getForegroundApp(): { appId: string; title: string } | null {
const window = this.foregroundWindow();
if (!window || window.appId === "launcher") return null;
return { appId: window.appId, title: window.title };
}
noteUserActivity(nowMs = Date.now()): void {
this.lastInputAtMs = nowMs;
}
@@ -283,7 +364,10 @@ class Shell {
// An open voice dialog suspends the timeout: a long dictation or refine
// has no button presses, but the screen must stay on for it. Sliding
// lastInputAtMs forward also restarts the full timeout when it closes.
if (this.activeVoiceLayer) {
// An in-flight assistant turn suspends it for the same reason (a tool loop
// can run for a while with no input); once the turn ends and the
// Follow-up/Done menu is showing, the normal idle timeout resumes.
if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
this.lastInputAtMs = nowMs;
return false;
}
@@ -358,7 +442,13 @@ class Shell {
this.lastInputAtMs = Date.now();
const wokeScreen = !this.screenOn && this.wake("sidebar");
if (action === "voice-input" && !this.activeVoiceLayer) {
this.openVoiceDialog(false, true);
if (this.assistantLayer) {
// The assistant overlay is up; a wakeword continues that conversation.
this.startAssistantFollowUp(true);
} else {
// Wakeword defaults the highlight to Send to Assistant.
this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" });
}
}
return { shell: wokeScreen || action === "voice-input", window: false };
}
@@ -489,25 +579,77 @@ class Shell {
next?.requestRender();
}
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
const layer = new VoiceInputLayer(
this.config.actions,
() => {
private openVoiceDialog(options: {
finishOnClick?: boolean;
handsFree?: boolean;
defaultTarget: "assistant" | "app";
}): void {
const targets = this.buildVoiceSendTargets();
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
if (defaultIndex < 0) defaultIndex = 0;
// Skip the menu only for a hands-free (wakeword) capture aimed at the
// assistant, when the user has opted into it.
const autoSend =
Boolean(options.handsFree) &&
options.defaultTarget === "assistant" &&
targets[defaultIndex]?.id === "assistant" &&
assistantSkipConfirmationSetting.get();
const layer = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeVoiceLayer === layer) {
this.activeVoiceLayer = null;
// The idle countdown restarts in full once voice input ends.
this.noteUserActivity();
}
},
(text) => this.sendTextToForegroundWindow(text),
finishOnClick,
handsFree,
);
dismiss: () => {
this.stack.popIfTop((top) => top === layer);
},
sendTargets: targets,
defaultTargetIndex: defaultIndex,
finishOnClick: options.finishOnClick ?? false,
handsFree: options.handsFree ?? false,
autoSend,
});
this.activeVoiceLayer = layer;
this.stack.push(layer);
layer.startCapture();
}
/**
* The send destinations offered by the voice dialog: the assistant (when an
* API key is configured) and/or typing into the foreground window (when it
* accepts text). Order fixes the menu row order.
*/
private buildVoiceSendTargets(): VoiceSendTarget[] {
const targets: VoiceSendTarget[] = [];
if (this.isAssistantAvailable()) {
targets.push({
id: "assistant",
label: "Send to Assistant",
onSend: (text) => this.sendToAssistant(text),
});
}
if (this.foregroundWindow()?.receiveTextInput) {
targets.push({
id: "app",
label: "Type Into App",
onSend: (text) => this.sendTextToForegroundWindow(text),
});
}
// Guarantee at least one destination so the dialog is never a dead end.
if (targets.length === 0) {
targets.push({
id: "app",
label: "Type Into App",
onSend: (text) => this.sendTextToForegroundWindow(text),
});
}
return targets;
}
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
sendTextToForegroundWindow(text: string): void {
this.foregroundWindow()?.receiveTextInput?.(text);
@@ -521,9 +663,130 @@ class Shell {
*/
startVoiceInput(): void {
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
// The transcript is aimed at the window whose menu requested it.
// The transcript is aimed at the window whose menu requested it; the menu
// entry point defaults the highlight to Type Into App.
this.focus = "window";
this.openVoiceDialog(true);
this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" });
this.config.requestShellRender();
}
private isAssistantAvailable(): boolean {
return anthropicApiKeySetting.get().trim().length > 0;
}
private ensureAssistantSession(): AssistantSession | null {
const apiKey = anthropicApiKeySetting.get().trim();
if (!apiKey) return null;
if (!this.assistantSession || this.assistantSession.isExpired()) {
this.assistantSession = new AssistantSession(apiKey);
}
return this.assistantSession;
}
private buildAssistantContext(): AssistantContext {
const foreground = this.getForegroundApp();
return {
foregroundApp: foreground?.appId ?? null,
foregroundTitle: foreground?.title ?? null,
screenOn: this.screenOn,
localTime: formatAssistantTime(new Date()),
headsetBattery: this.battery.headset,
};
}
/**
* Start (or continue) an assistant conversation from a finalized utterance.
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
* existing session and overlay.
*/
sendToAssistant(text: string): void {
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert("Set an Anthropic API key in Settings to use the assistant.");
return;
}
if (!this.screenOn) this.wake("sidebar");
let layer = this.assistantLayer;
if (!layer) {
const created = new AssistantLayer(this.config.actions, {
onFollowUp: () => this.startAssistantFollowUp(),
onCancel: () => this.assistantSession?.cancel(),
onClose: () => this.closeAssistantLayer(),
onRemoved: () => {
// Removed by any path (Done, or the screen sleeping mid-conversation):
// stop the turn and drop the reference so a later query starts clean.
this.assistantSession?.cancel();
if (this.assistantLayer === created) this.assistantLayer = null;
},
});
layer = created;
this.assistantLayer = created;
this.stack.push(created);
}
this.runAssistantTurn(session, layer, text);
this.config.requestShellRender();
}
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
layer.startTurn();
session.sendUtterance(text, this.buildAssistantContext(), {
onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar),
onToolActivity: (label) => layer.onToolActivity(label),
onTurnDone: () => layer.onTurnDone(),
onError: (message) => layer.onError(message),
});
}
/**
* Record another utterance in the current assistant conversation. handsFree
* (from a wakeword over the overlay) starts the mic immediately; otherwise
* (the Follow-up menu button) a click ends the utterance.
*/
private startAssistantFollowUp(handsFree = false): void {
const layer = this.assistantLayer;
const session = this.assistantSession;
if (!layer || !session || this.activeVoiceLayer) return;
const voice = new VoiceInputLayer({
actions: this.config.actions,
onClosed: () => {
if (this.activeVoiceLayer === voice) {
this.activeVoiceLayer = null;
this.noteUserActivity();
}
},
dismiss: () => {
this.stack.popIfTop((top) => top === voice);
},
sendTargets: [
{ id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) },
],
finishOnClick: !handsFree,
handsFree,
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
});
this.activeVoiceLayer = voice;
this.stack.push(voice);
voice.startCapture();
this.config.requestShellRender();
}
private closeAssistantLayer(): void {
// Popping fires the layer's onRemoved, which cancels the turn and clears
// this.assistantLayer.
const layer = this.assistantLayer;
if (layer) this.stack.popIfTop((top) => top === layer);
this.noteUserActivity();
this.config.requestShellRender();
}
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
showAlert(text: string): void {
if (!this.screenOn) this.wake("sidebar");
const layer = new ShellAlertLayer(text, () => {
this.stack.popIfTop((top) => top === layer);
this.config.requestShellRender();
});
this.stack.push(layer);
this.config.requestShellRender();
}
@@ -586,6 +849,20 @@ class Shell {
export const shell = new Shell();
const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
/** Human-readable local time for the assistant's per-turn context. */
function formatAssistantTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`;
return `${day}, ${hours}:${minutes} ${meridiem}`;
}
export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
if (event.kind === "sys-event") {
if (event.eventType === OsEventTypeList.CLICK_EVENT) {