forked from Blendman974/faceclaw
Implement a basic voice assistant
This commit is contained in:
1 parent
d2c987d48a
commit
0aeb24ed00
14 files changed
+1923
-80
No files matched your search
@@ -0,0 +1,141 @@
|
||||
import {
|
||||
streamAnthropicMessage,
|
||||
type AnthropicContentBlock,
|
||||
type AnthropicMessage,
|
||||
type AnthropicStreamHandle,
|
||||
type AnthropicToolDefinition,
|
||||
} from "../native/anthropic";
|
||||
import type { ToolRegistry } from "./tool-registry";
|
||||
import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
/**
|
||||
* The on-phone agent loop: stream a reply, and while the model asks for tools,
|
||||
* execute them through the registry and feed the results back until it stops.
|
||||
* History lives in the AssistantSession; this backend appends the assistant
|
||||
* and tool-result messages to the array it is handed.
|
||||
*/
|
||||
|
||||
/** Safety caps so a tool-thrashing model can't loop or run forever. */
|
||||
const MAX_ITERATIONS = 8;
|
||||
const MAX_TURN_MS = 2 * 60 * 1000;
|
||||
|
||||
export type DirectTurnOptions = {
|
||||
apiKey: string;
|
||||
model: string;
|
||||
effort?: "low" | "medium" | "high";
|
||||
system: string;
|
||||
/** Owned by the session; this loop appends assistant + tool_result messages. */
|
||||
messages: AnthropicMessage[];
|
||||
tools: AnthropicToolDefinition[];
|
||||
registry: ToolRegistry;
|
||||
/** Map an API tool name (as the model calls it) back to its registry name. */
|
||||
resolveToolName: (apiName: string) => string;
|
||||
callbacks: AssistantTurnCallbacks;
|
||||
};
|
||||
|
||||
export class DirectAnthropicBackend {
|
||||
runTurn(options: DirectTurnOptions): AssistantTurnHandle {
|
||||
const startMs = Date.now();
|
||||
let cancelled = false;
|
||||
let streamHandle: AnthropicStreamHandle | null = null;
|
||||
|
||||
const finishError = (message: string) => {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
options.callbacks.onError(message);
|
||||
};
|
||||
|
||||
const runIteration = (iteration: number): void => {
|
||||
if (cancelled) return;
|
||||
if (iteration >= MAX_ITERATIONS) {
|
||||
finishError("Assistant stopped: too many tool steps");
|
||||
return;
|
||||
}
|
||||
if (Date.now() - startMs > MAX_TURN_MS) {
|
||||
finishError("Assistant stopped: turn took too long");
|
||||
return;
|
||||
}
|
||||
|
||||
streamHandle = streamAnthropicMessage({
|
||||
apiKey: options.apiKey,
|
||||
model: options.model,
|
||||
system: options.system,
|
||||
effort: options.effort,
|
||||
messages: options.messages,
|
||||
tools: options.tools,
|
||||
onTextDelta: (delta, textSoFar) => {
|
||||
if (cancelled) return;
|
||||
options.callbacks.onTextDelta(delta, textSoFar);
|
||||
},
|
||||
onDone: (result) => {
|
||||
streamHandle = null;
|
||||
if (cancelled) return;
|
||||
if (result.stopReason === "refusal") {
|
||||
finishError("The assistant declined this request");
|
||||
return;
|
||||
}
|
||||
// Record what the model produced this iteration.
|
||||
const assistantContent: AnthropicContentBlock[] = result.content.length
|
||||
? result.content
|
||||
: [{ type: "text", text: result.text }];
|
||||
options.messages.push({ role: "assistant", content: assistantContent });
|
||||
|
||||
const toolUses = assistantContent.filter(
|
||||
(block): block is Extract<AnthropicContentBlock, { type: "tool_use" }> =>
|
||||
block.type === "tool_use",
|
||||
);
|
||||
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
|
||||
options.callbacks.onTurnDone({ stopReason: result.stopReason });
|
||||
return;
|
||||
}
|
||||
void this.runToolCalls(toolUses, options)
|
||||
.then((toolResults) => {
|
||||
// Always record the results, even if cancelled mid-flight: an
|
||||
// assistant tool_use with no matching tool_result would make the
|
||||
// next turn's request invalid. Only the loop continuation is gated.
|
||||
options.messages.push({ role: "user", content: toolResults });
|
||||
if (cancelled) return;
|
||||
runIteration(iteration + 1);
|
||||
})
|
||||
.catch((error) => {
|
||||
finishError(`Tool execution failed: ${String((error as Error)?.message ?? error)}`);
|
||||
});
|
||||
},
|
||||
onError: (message) => {
|
||||
streamHandle = null;
|
||||
finishError(message);
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
runIteration(0);
|
||||
|
||||
return {
|
||||
cancel: () => {
|
||||
cancelled = true;
|
||||
streamHandle?.cancel();
|
||||
streamHandle = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async runToolCalls(
|
||||
toolUses: Array<Extract<AnthropicContentBlock, { type: "tool_use" }>>,
|
||||
options: DirectTurnOptions,
|
||||
): Promise<AnthropicContentBlock[]> {
|
||||
const results: AnthropicContentBlock[] = [];
|
||||
for (const toolUse of toolUses) {
|
||||
const canonicalName = options.resolveToolName(toolUse.name);
|
||||
options.callbacks.onToolActivity(canonicalName);
|
||||
const result = await options.registry.callTool(canonicalName, toolUse.input);
|
||||
const content = result.ok ? result.content ?? "" : result.error ?? "Tool error";
|
||||
results.push({
|
||||
type: "tool_result",
|
||||
tool_use_id: toolUse.id,
|
||||
content,
|
||||
is_error: !result.ok,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic";
|
||||
import { DirectAnthropicBackend } from "./direct-backend";
|
||||
import { toolRegistry, type ToolRegistry } from "./tool-registry";
|
||||
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
/**
|
||||
* One assistant conversation. Owns the message history and turn state and
|
||||
* drives a backend (direct Anthropic loop for phase 1). The UI talks only to
|
||||
* this surface, so swapping in the external-agent backend later doesn't touch
|
||||
* AssistantLayer.
|
||||
*
|
||||
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
|
||||
* one for a new wakeword and reuses the current one for a follow-up.
|
||||
*/
|
||||
|
||||
/** A new PTT after this much idle starts a fresh conversation. */
|
||||
const SESSION_IDLE_MS = 10 * 60 * 1000;
|
||||
|
||||
/** Trim history from the head once it grows past this many messages. */
|
||||
const MAX_HISTORY_MESSAGES = 40;
|
||||
|
||||
const SYSTEM_PROMPT_BASE = [
|
||||
"You are the voice assistant built into a pair of Even Realities G2 smart glasses.",
|
||||
"Your reply is shown as text on a 576x288 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list.",
|
||||
"Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it.",
|
||||
"If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
|
||||
].join(" ");
|
||||
|
||||
export class AssistantSession {
|
||||
private readonly backend = new DirectAnthropicBackend();
|
||||
private readonly messages: AnthropicMessage[] = [];
|
||||
private turnHandle: AssistantTurnHandle | null = null;
|
||||
private lastActivityMs = Date.now();
|
||||
// API-safe tool name -> canonical registry name. The Anthropic API only
|
||||
// allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses.
|
||||
// get_state") are sanitized for the request and mapped back on tool calls.
|
||||
private readonly toolNameMap = new Map<string, string>();
|
||||
|
||||
constructor(
|
||||
private readonly apiKey: string,
|
||||
private readonly registry: ToolRegistry = toolRegistry,
|
||||
) {}
|
||||
|
||||
/** Whether this session has been idle long enough to retire. */
|
||||
isExpired(nowMs = Date.now()): boolean {
|
||||
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
|
||||
}
|
||||
|
||||
isTurnActive(): boolean {
|
||||
return this.turnHandle !== null;
|
||||
}
|
||||
|
||||
/** Begin a turn from a spoken utterance. Only one turn runs at a time. */
|
||||
sendUtterance(text: string, ctx: AssistantContext, callbacks: AssistantTurnCallbacks): void {
|
||||
if (this.turnHandle) {
|
||||
callbacks.onError("The assistant is still working on the previous request");
|
||||
return;
|
||||
}
|
||||
this.lastActivityMs = Date.now();
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
|
||||
const tools = this.buildToolDefinitions();
|
||||
const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
|
||||
|
||||
const finish = () => {
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
};
|
||||
|
||||
this.turnHandle = this.backend.runTurn({
|
||||
apiKey: this.apiKey,
|
||||
model: DEFAULT_LLM_MODEL,
|
||||
effort: "low",
|
||||
system,
|
||||
messages: this.messages,
|
||||
tools,
|
||||
registry: this.registry,
|
||||
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
||||
callbacks: {
|
||||
onTextDelta: callbacks.onTextDelta,
|
||||
onToolActivity: callbacks.onToolActivity,
|
||||
onTurnDone: (result) => {
|
||||
finish();
|
||||
callbacks.onTurnDone(result);
|
||||
},
|
||||
onError: (message) => {
|
||||
finish();
|
||||
callbacks.onError(message);
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.turnHandle?.cancel();
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
}
|
||||
|
||||
private buildToolDefinitions(): AnthropicToolDefinition[] {
|
||||
this.toolNameMap.clear();
|
||||
return this.registry.listTools().map((spec) => {
|
||||
const apiName = this.toApiToolName(spec.name);
|
||||
this.toolNameMap.set(apiName, spec.name);
|
||||
return { name: apiName, description: spec.description, input_schema: spec.inputSchema };
|
||||
});
|
||||
}
|
||||
|
||||
/** Convert a canonical registry name to an Anthropic-legal tool name. */
|
||||
private toApiToolName(name: string): string {
|
||||
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
|
||||
// Disambiguate the rare case where two canonical names sanitize alike.
|
||||
if (this.toolNameMap.has(base)) {
|
||||
let i = 2;
|
||||
while (this.toolNameMap.has(`${base}_${i}`)) i++;
|
||||
base = `${base}_${i}`;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
private trimHistory(): void {
|
||||
if (this.messages.length <= MAX_HISTORY_MESSAGES) return;
|
||||
// Drop whole messages from the head. A leading orphan tool_result (whose
|
||||
// tool_use we trimmed) would be rejected by the API, so skip past any
|
||||
// tool_result-only user message left at the front.
|
||||
const overflow = this.messages.length - MAX_HISTORY_MESSAGES;
|
||||
this.messages.splice(0, overflow);
|
||||
while (this.messages.length && startsWithToolResult(this.messages[0]!)) {
|
||||
this.messages.shift();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function startsWithToolResult(message: AnthropicMessage): boolean {
|
||||
return (
|
||||
Array.isArray(message.content) &&
|
||||
message.content.length > 0 &&
|
||||
message.content[0]!.type === "tool_result"
|
||||
);
|
||||
}
|
||||
|
||||
function describeContext(ctx: AssistantContext): string {
|
||||
const parts = [`Current time: ${ctx.localTime}.`];
|
||||
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
|
||||
if (ctx.foregroundApp) {
|
||||
const title = ctx.foregroundTitle ? ` ("${ctx.foregroundTitle}")` : "";
|
||||
parts.push(`The foreground app is ${ctx.foregroundApp}${title}.`);
|
||||
} else {
|
||||
parts.push("No app is in the foreground (the launcher is showing).");
|
||||
}
|
||||
if (ctx.headsetBattery !== null) {
|
||||
parts.push(`Glasses battery: ${ctx.headsetBattery}%.`);
|
||||
}
|
||||
return `Context: ${parts.join(" ")}`;
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import { readUpcomingEvents, type CalendarEvent } from "../native/calendar";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
|
||||
|
||||
/**
|
||||
* Registers the always-available system tools into the registry. Called once at
|
||||
* startup. These wrap shell state and the existing native bridges (calendar,
|
||||
* media, notifications); nothing here needs an app window to be open.
|
||||
*/
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
let registered = false;
|
||||
|
||||
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
|
||||
if (registered) return;
|
||||
registered = true;
|
||||
|
||||
// Keep a media listener warm so "what's playing" works without opening Music.
|
||||
void mediaControllerBridge.start();
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "glasses.get_state",
|
||||
description:
|
||||
"Get the current state of the glasses: display on/off, foreground app, headset battery, and the current local time.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
proactive: true,
|
||||
},
|
||||
() => ok(describeState()),
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "glasses.show_alert",
|
||||
description:
|
||||
"Show a short text popup on the glasses display. Use for a brief notice the user should see; keep it to a sentence or two.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { text: { type: "string", description: "The message to display." } },
|
||||
required: ["text"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
proactive: true,
|
||||
},
|
||||
(args) => {
|
||||
const text = String(args?.text ?? "").trim();
|
||||
if (!text) return err("show_alert requires non-empty text");
|
||||
shell.showAlert(text);
|
||||
return ok("Displayed.");
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "calendar.list_events",
|
||||
description:
|
||||
"List the user's upcoming calendar events. Returns events from now forward, ordered by start time.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
within_hours: {
|
||||
type: "number",
|
||||
description: "Only include events starting within this many hours (default 168 = one week).",
|
||||
},
|
||||
max_events: { type: "number", description: "Maximum number of events to return (default 10)." },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
(args) => {
|
||||
const withinHours = clampNumber(args?.within_hours, 1, 24 * 60, 168);
|
||||
const maxEvents = clampNumber(args?.max_events, 1, 50, 10);
|
||||
const events = readUpcomingEvents(maxEvents, withinHours * 60 * 60 * 1000);
|
||||
if (!events.length) return ok("No upcoming events in that window.");
|
||||
return ok(events.map(formatEvent).join("\n"));
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.now_playing",
|
||||
description: "What is currently playing (or paused) in the user's media app, if anything.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
proactive: true,
|
||||
},
|
||||
() => ok(describeNowPlaying()),
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.play_pause",
|
||||
description: "Toggle play/pause for the user's current media session.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canPlayPause) return err("No media session can be controlled right now.");
|
||||
await mediaControllerBridge.playPause();
|
||||
return ok("Toggled play/pause.");
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.next",
|
||||
description: "Skip to the next track in the user's current media session.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
|
||||
await mediaControllerBridge.skipNext();
|
||||
return ok("Skipped to the next track.");
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "notifications.list",
|
||||
description: "List the user's current Android notifications mirrored to the glasses.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { max: { type: "number", description: "Maximum notifications to return (default 10)." } },
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
(args) => {
|
||||
const max = clampNumber(args?.max, 1, 50, 10);
|
||||
const notifications = readActiveNotifications(max);
|
||||
if (!notifications.length) return ok("No current notifications.");
|
||||
return ok(notifications.map(formatNotification).join("\n"));
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "notifications.dismiss",
|
||||
description:
|
||||
"Dismiss a notification by its key (as returned by notifications.list). Returns whether it was dismissed.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: { key: { type: "string", description: "The notification key from notifications.list." } },
|
||||
required: ["key"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
(args) => {
|
||||
const key = String(args?.key ?? "").trim();
|
||||
if (!key) return err("notifications.dismiss requires a key");
|
||||
return dismissNotification(key) ? ok("Dismissed.") : err("No notification with that key.");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function describeState(): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`Display: ${shell.isScreenOn() ? "on" : "off"}.`);
|
||||
const fg = shell.getForegroundApp();
|
||||
parts.push(fg ? `Foreground app: ${fg.appId} ("${fg.title}").` : "Foreground: launcher (no app open).");
|
||||
const battery = shell.getBatteryLevels();
|
||||
if (battery.headset !== null) {
|
||||
const charging = battery.headsetCharging ? ", charging" : "";
|
||||
parts.push(`Headset battery: ${battery.headset}%${charging}.`);
|
||||
}
|
||||
parts.push(`Local time: ${formatDateTime(new Date())}.`);
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
function describeNowPlaying(): string {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable.";
|
||||
if (state.playbackState === "playing" || state.playbackState === "paused") {
|
||||
const verb = state.playbackState === "playing" ? "Playing" : "Paused";
|
||||
const title = state.title || "(unknown title)";
|
||||
const artist = state.artist ? ` by ${state.artist}` : "";
|
||||
const app = state.appName ? ` in ${state.appName}` : "";
|
||||
return `${verb}: ${title}${artist}${app}.`;
|
||||
}
|
||||
return "Nothing is currently playing.";
|
||||
}
|
||||
|
||||
function formatEvent(event: CalendarEvent): string {
|
||||
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
|
||||
const location = event.location ? ` @ ${event.location}` : "";
|
||||
return `- ${event.title || "(untitled)"} — ${when}${location}`;
|
||||
}
|
||||
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
|
||||
const title = notification.title || "(no title)";
|
||||
const body = notification.text ? `: ${notification.text}` : "";
|
||||
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
|
||||
}
|
||||
|
||||
function formatDateTime(date: Date): string {
|
||||
return `${formatDate(date)}, ${formatTime(date)}`;
|
||||
}
|
||||
|
||||
function formatDate(date: Date): string {
|
||||
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
|
||||
}
|
||||
|
||||
function formatTime(date: Date): string {
|
||||
let hours = date.getHours();
|
||||
const minutes = date.getMinutes().toString().padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
}
|
||||
|
||||
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
|
||||
const num = typeof value === "number" ? value : Number(value);
|
||||
if (!Number.isFinite(num)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.round(num)));
|
||||
}
|
||||
|
||||
function ok(content: string): ToolResult {
|
||||
return { ok: true, content };
|
||||
}
|
||||
|
||||
function err(error: string): ToolResult {
|
||||
return { ok: false, error };
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* The assistant's tool registry: the single place that knows which tools exist,
|
||||
* whether each is currently callable, and how to invoke it. Both assistant
|
||||
* backends (direct Anthropic loop, external agent bridge) list and call tools
|
||||
* through here, so gating and timeouts are enforced in one spot rather than
|
||||
* trusted to a model or a remote agent.
|
||||
*
|
||||
* Availability tiers (see notes/voice-assistant-design.md):
|
||||
* - "always" system/shell tools; no app involved.
|
||||
* - "installed" app tool live whenever the app is installed (no window
|
||||
* needed) -- e.g. "start a 5 minute timer".
|
||||
* - "open" app tool live while the app has a window open, foreground
|
||||
* or backgrounded -- e.g. "next track".
|
||||
* - "foreground" app tool live only while the app owns the foreground window
|
||||
* -- e.g. "scroll down".
|
||||
*
|
||||
* Phase 1 only registers "always" tools; the tier machinery is here so the
|
||||
* app-tool phases slot in without reshaping the registry.
|
||||
*/
|
||||
|
||||
export type ToolAvailability = "always" | "installed" | "open" | "foreground";
|
||||
|
||||
export type ToolSpec = {
|
||||
/** Namespaced, e.g. "glasses.get_state" or "app.terminal.send_input". */
|
||||
name: string;
|
||||
description: string;
|
||||
/** JSON Schema for the tool's arguments. */
|
||||
inputSchema: object;
|
||||
availability: ToolAvailability;
|
||||
/** May be invoked outside an active voice turn (external mode proactive calls). */
|
||||
proactive?: boolean;
|
||||
/** Round-trip cap; the registry rejects with a tool error past it. Default 10s. */
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* A tool's outcome. Flat (not a discriminated union) on purpose: this project
|
||||
* builds with strictNullChecks off, so `ok: true|false` would widen to boolean
|
||||
* and defeat union narrowing. Read `content` when ok, `error` when not.
|
||||
*/
|
||||
export type ToolResult = {
|
||||
ok: boolean;
|
||||
/** Text payload back to the model (when ok). */
|
||||
content?: string;
|
||||
/** Error message back to the model (when not ok). */
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type ToolHandler = (args: any) => Promise<ToolResult> | ToolResult;
|
||||
|
||||
export type ToolRegistration = {
|
||||
spec: ToolSpec;
|
||||
handler: ToolHandler;
|
||||
/**
|
||||
* Whether the tool is currently live. Absent means always live (the "always"
|
||||
* tier). App-tool phases supply a predicate keyed on window/foreground state.
|
||||
*/
|
||||
isAvailable?: () => boolean;
|
||||
};
|
||||
|
||||
export type ListToolsOptions = {
|
||||
/** Only tools flagged proactive (for calls outside an active turn). */
|
||||
proactiveOnly?: boolean;
|
||||
};
|
||||
|
||||
export type CallToolOptions = {
|
||||
/** The call is happening outside an active voice turn; enforce proactive gating. */
|
||||
proactive?: boolean;
|
||||
};
|
||||
|
||||
const DEFAULT_TOOL_TIMEOUT_MS = 10_000;
|
||||
|
||||
export class ToolRegistry {
|
||||
private readonly registrations = new Map<string, ToolRegistration>();
|
||||
private readonly changeListeners = new Set<() => void>();
|
||||
|
||||
/** Register (or replace) a tool. Names are unique across all tiers. */
|
||||
register(registration: ToolRegistration): void {
|
||||
const { name } = registration.spec;
|
||||
this.registrations.set(name, registration);
|
||||
this.fireToolsChanged();
|
||||
}
|
||||
|
||||
/** Convenience for the always-live system tools. */
|
||||
registerSystemTool(
|
||||
spec: Omit<ToolSpec, "availability">,
|
||||
handler: ToolHandler,
|
||||
): void {
|
||||
this.register({ spec: { ...spec, availability: "always" }, handler });
|
||||
}
|
||||
|
||||
unregister(name: string): void {
|
||||
if (this.registrations.delete(name)) {
|
||||
this.fireToolsChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/** Specs for the tools callable right now, given the availability tiers. */
|
||||
listTools(options: ListToolsOptions = {}): ToolSpec[] {
|
||||
const specs: ToolSpec[] = [];
|
||||
for (const registration of this.registrations.values()) {
|
||||
if (!this.isLive(registration)) continue;
|
||||
if (options.proactiveOnly && !registration.spec.proactive) continue;
|
||||
specs.push(registration.spec);
|
||||
}
|
||||
return specs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Invoke a tool by name. Never throws: a missing/unavailable tool, a
|
||||
* proactive-gating rejection, a handler exception, or a timeout all come back
|
||||
* as `{ ok: false }` so the agent loop can surface them as tool errors.
|
||||
*/
|
||||
async callTool(name: string, args: unknown, options: CallToolOptions = {}): Promise<ToolResult> {
|
||||
const registration = this.registrations.get(name);
|
||||
if (!registration) {
|
||||
return { ok: false, error: `Unknown tool: ${name}` };
|
||||
}
|
||||
if (!this.isLive(registration)) {
|
||||
return { ok: false, error: `Tool ${name} is not currently available` };
|
||||
}
|
||||
if (options.proactive && !registration.spec.proactive) {
|
||||
return { ok: false, error: `Tool ${name} cannot be called outside a conversation` };
|
||||
}
|
||||
const timeoutMs = registration.spec.timeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
|
||||
try {
|
||||
return await this.withTimeout(registration.handler(args), timeoutMs, name);
|
||||
} catch (error) {
|
||||
return { ok: false, error: `Tool ${name} failed: ${describeError(error)}` };
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to add/remove/availability changes (drives MCP list_changed). */
|
||||
onToolsChanged(listener: () => void): () => void {
|
||||
this.changeListeners.add(listener);
|
||||
return () => this.changeListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** App-tool phases call this after a foreground/window change re-gates tools. */
|
||||
fireToolsChanged(): void {
|
||||
for (const listener of this.changeListeners) {
|
||||
try {
|
||||
listener();
|
||||
} catch (error) {
|
||||
console.warn("onToolsChanged listener failed", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private isLive(registration: ToolRegistration): boolean {
|
||||
return registration.isAvailable ? registration.isAvailable() : true;
|
||||
}
|
||||
|
||||
private withTimeout(
|
||||
result: Promise<ToolResult> | ToolResult,
|
||||
timeoutMs: number,
|
||||
name: string,
|
||||
): Promise<ToolResult> {
|
||||
if (!(result instanceof Promise)) return Promise.resolve(result);
|
||||
return new Promise<ToolResult>((resolve) => {
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve({ ok: false, error: `Tool ${name} timed out after ${timeoutMs}ms` });
|
||||
}, timeoutMs);
|
||||
result.then(
|
||||
(value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, error: `Tool ${name} failed: ${describeError(error)}` });
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function describeError(error: unknown): string {
|
||||
return String((error as Error)?.message ?? error);
|
||||
}
|
||||
|
||||
/** The process-wide registry; system tools register into it at startup. */
|
||||
export const toolRegistry = new ToolRegistry();
|
||||
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer)
|
||||
* talks to the same session surface whether the turn runs on the phone
|
||||
* (direct Anthropic loop) or against the user's own agent (external bridge,
|
||||
* later phases).
|
||||
*/
|
||||
|
||||
/** Situational grounding injected per turn (system prompt in direct mode). */
|
||||
export type AssistantContext = {
|
||||
/** Foreground app id, or null when the sidebar/launcher is all that's up. */
|
||||
foregroundApp: string | null;
|
||||
foregroundTitle: string | null;
|
||||
screenOn: boolean;
|
||||
/** Human-readable local time, e.g. "Thu Jul 24, 3:15 PM". */
|
||||
localTime: string;
|
||||
/** Headset battery percent (0..100), or null if unknown. */
|
||||
headsetBattery: number | null;
|
||||
};
|
||||
|
||||
export type AssistantTurnCallbacks = {
|
||||
/** Streamed reply text (delta plus the full text so far). */
|
||||
onTextDelta: (delta: string, textSoFar: string) => void;
|
||||
/** A tool is being invoked; label is display-ready, e.g. "calendar.list_events". */
|
||||
onToolActivity: (label: string) => void;
|
||||
onTurnDone: (result: { stopReason: string | null }) => void;
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
/** A handle to an in-flight turn. */
|
||||
export type AssistantTurnHandle = {
|
||||
cancel(): void;
|
||||
};
|
||||
@@ -25,6 +25,7 @@ import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
|
||||
import { registerSystemTools } from "../assistant/system-tools";
|
||||
import { WorkerAppHost } from "../ui/shell/worker-window";
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from "../ui/shell/launcher-app";
|
||||
import {
|
||||
@@ -236,6 +237,9 @@ class DashboardController {
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
this.sharedActions = sharedActions;
|
||||
// The always-available assistant tools (calendar, media, notifications,
|
||||
// glasses state) register once at startup, independent of any connection.
|
||||
registerSystemTools();
|
||||
shell.configure({
|
||||
actions: {
|
||||
...sharedActions,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// AUTO-GENERATED from g2flash/patches/cfw_patches.json — do not edit by hand.
|
||||
// Regenerate by re-running the generator in the firmware onboarding change.
|
||||
// Regenerate with scripts/update_deltas.sh.
|
||||
// Turns the stock G2 2.2.6.10 image into the Faceclaw custom firmware.
|
||||
|
||||
export type FirmwarePatchOp = {
|
||||
@@ -19,7 +19,7 @@ export type FirmwarePatchSet = {
|
||||
export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"base": "g2_2.2.6.10.bin",
|
||||
"baseSha256": "f4dfb0b49ad3de3c2daf17f8a27a157c3dc98411d6a0d3ab2cfd0918f41b9afa",
|
||||
"outputSha256": "162fa8aab6f41be5e314d51b56f3e8f3eb2143af93ee0518d1919d096701474d",
|
||||
"outputSha256": "ba7e896e516d049e530d7aaed30e358b45f940e81a90f997a44465299151b452",
|
||||
"patches": [
|
||||
{
|
||||
"offset": 1449549,
|
||||
@@ -42,85 +42,103 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 1447919,
|
||||
"old": "7ef7fefd",
|
||||
"new": "b8f28cfe",
|
||||
"new": "b8f2f6fe",
|
||||
"desc": "bl snapshot_side @ 0x4db968"
|
||||
},
|
||||
{
|
||||
"offset": 1448931,
|
||||
"old": "7ef704fc",
|
||||
"new": "b8f292fc",
|
||||
"new": "b8f2fcfc",
|
||||
"desc": "bl snapshot_side @ 0x4dbd5c"
|
||||
},
|
||||
{
|
||||
"offset": 1165461,
|
||||
"old": "45f0cefd",
|
||||
"new": "fdf23dfe",
|
||||
"new": "fdf2a7fe",
|
||||
"desc": "bl image_deferred (deferred consumer -> FIFO restore + worker, both lenses)"
|
||||
},
|
||||
{
|
||||
"offset": 1186287,
|
||||
"old": "d9f7d4ff",
|
||||
"new": "f8f2d0fe",
|
||||
"new": "f9f2b8fb",
|
||||
"desc": "bl settings_send_wrapper (append caps field 100)"
|
||||
},
|
||||
{
|
||||
"offset": 1183983,
|
||||
"old": "f4f75aff",
|
||||
"new": "f9f2b6fd",
|
||||
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
|
||||
},
|
||||
{
|
||||
"offset": 926945,
|
||||
"old": "08f068fa",
|
||||
"new": "38f3d9fa",
|
||||
"desc": "bl faceclaw_display_start @ 0x45c65a (fail-open double-tap takeover)"
|
||||
},
|
||||
{
|
||||
"offset": 927137,
|
||||
"old": "08f008fa",
|
||||
"new": "38f379fa",
|
||||
"desc": "bl faceclaw_display_start @ 0x45c71a (fail-open double-tap takeover)"
|
||||
},
|
||||
{
|
||||
"offset": 822553,
|
||||
"old": "28f003f8",
|
||||
"new": "51f39dfd",
|
||||
"new": "52f39dfa",
|
||||
"desc": "bl evenhub_longpress (replaces force-quit dialog)"
|
||||
},
|
||||
{
|
||||
"offset": 823369,
|
||||
"old": "1cf09bfb",
|
||||
"new": "51f321fc",
|
||||
"new": "52f321f9",
|
||||
"desc": "bl ring_release (forward ring release-long-press)"
|
||||
},
|
||||
{
|
||||
"offset": 1474163,
|
||||
"old": "34d1",
|
||||
"new": "34e0",
|
||||
"desc": "even_ai_display_ctrl: skip Even AI app START (wakeword -> phone)"
|
||||
"offset": 1474137,
|
||||
"old": "7fb50600",
|
||||
"new": "b3f271b9",
|
||||
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
|
||||
},
|
||||
{
|
||||
"offset": 4301227,
|
||||
"old": "",
|
||||
"new": "02fb01f044f6d341c0f24701084700bf084644f61751c0f24701084770b500284ed001684ef65a62ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf2de9f04f85b043f6fc76c2f200063568804628466ff3960044f6d344b0f1005fc0f2470406d128684ef65a61ccf2ff0188423bd04ff4fe70a0470546a8b30020002100bf6a1868540431b1f5fe7f50709070d070f6d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe003560b8f1000f18bf002d07d14af26950c0f2450005b0bde8f04f0047d8f80cb0bbf1000ff2d0d8f820a0baf1000fedd0a94659f8040f00287bd0686900287ad0686a002879d0686b002878d0686c002877d0686d002876d0686e002875d0686f002874d0d5f88400002872d0d5f89400002870d0d5f8a40000286ed0d5f8b40000286cd0d5f8b00029690490d5f8a0002a6b0390d5f8900000260290d5f880002f6f0190286ad5f860e0d5f850c02b6c884238bf014638bf01268a423cbf114602268b423cbf194603268c453cbf614604268e453cbf714605268f423cbf39460626019888423cbf01460726029888423cbf01460826039888423cbf014609260498884238bf0146d5f8c00038bf0a26884238bf0b2609eb0610406804f1440137018847002049f80700012085f8df0016e0002614e0012612e0022610e003260ee004260ce005260ae0062608e0072606e0082604e0092602e00a2600e00b265046a04700283ff451af0af0030c5fea9a034ff0000219d00496a2eb830e431c0bf101066ff0030700bff119cc78da19d4700c79043714714c79547189790eeb070404349171f0d1049e3a1dbcf1000f10d01bf80230bcf1010f83540ad0531c1bf80370bcf1020fc75403d0911c1bf80120425409eb06114860d5f8c400320149f80280421cc1f808a0c5f8c420c8600ce738464146fff7aebe2de9f04f81b043f6fc79c2f20009d9f80050044628466ff3960044f6d34816460f46b0f1005fc0f2470806d128684ef65a61ccf2ff0188423dd04ff4fe70c0470546002800f09580002000216a1868540431b1f5fe7f50709070d070f6d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe00c9f8005005f1040b4ff0000a4ff0000900bf4ff0ff30002200234ff0ff3105e000bf6032c02a03f1060332d0af187e68a64204d13e6986423cbf194630467e69a64204d13e6a86423cbf591c30467e6aa64204d13e6b86423cbf991c30467e6ba64204d13e6c86423cbfd91c30467e6ca64204d13e6d86423cbf191d30467e6da642ced13f6e87423cbf591d3846c8e700bf002910d40beb01104768090182684bf801a02046394600f013f8814608f1440138468847ace7484601b0bde8f08f204639463246c9f8005001b0bde8f04fffe72de9f041a4b0064640f6bb4043f6fc78c0f24e0014460d46c2f200088047002002af0390384600f01bf90df10c0c3046294622460123cdf800c000f0bff90446384600f015ffd8f8001005460a466ff39602b2f1005f06d108684ef65a62ccf2ff0290423fd044f6d341c0f247014ff4fe708847d0b30146002000228b1888540432b2f5fe7f58709870d870f6d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe00c8f80010c1f8d05002e00020c8f80000204624b0bde8f08100bf09295ad1f0b502eb030c352756248cf8107037278cf804404e248cf8117036278cf8064043248cf812707a278cf8074046248cf8177072274ff0a20e8cf8084057248cf819706c2702f803e04ff0060e8cf809402f248cf81a7065278cf801e04ff0450e8cf80a4032248cf81b7000278cf803e08cf805e08cf80b4020244ff0690e6d2567268cf81c7019271c338cf80c408cf80de08cf80e508cf80f608cf813408cf814e08cf815508cf816608cf818408cf80270bde8f04045f6153cc0f2470c60474df63040c2f234000078042818bf704780b582b04af26b1e4ff0000cc0f24d0e0020002100220923cdf804c0cdf800c0f04702b080bd00bff0b583b04df63043c2f234031b784ff6e704042bc0f2450420d144f2d043c2f207031b680546184616460f46a04703463946324628468bb11b68e02b0ed14af26b160027c0f24d060020002100220a2301970097b047012003b0f0bd04f1160303b0bde8f0401847f0b581b041f204054cf65561cef20005ccf2ac51c5f8ac1f4ef6fc51cef200010a6843f6fc7642f080720a6055f8041cc2f2000641f0010145f8041c316807460a466ff39602b2f1005f06d108684ef65a62ccf2ff02904240d044f6d341c0f247014ff4fe708847002870d001460020002200bf8b1888540432b2f5fe7f58709870d870f6d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe003160d1f8d800b8bb44f6342245f6e464c2f20702cff6f8741368204600bf16689e420ed116689e420bd116689e4208d116689e4203d1002800f10400efd19e421ad02b6816681068b0420ed11068b0420bd11068b04208d11068b04203d1002c04f10404efd1b04206d02868c01ac1f8d80001e0002030602868386001b0f0bd00bf2de9f04fdbb0002918bf002a04d15bb0bde8f04f00f00cbe0c7804f07f07052f00f0e580072f4dd0422ff0d0ddf89091b0f84080b0f842a0092f00f04e81082f40f058810746022a4ff0ff30c0f0b382002b00f0b082c3460bf10100032303eb500020f0030000fb0af07630824200f23c814d78fdb188460226b01c904200f2348108eb060318f806605b7856ea032300f02b811e18964200f2278108eb0001384614461a460023cdf80090fff7a8ff2246013de1d13846594652464b4600f01dfe77e243f6fc74c2f20004236818466ff39600b0f1005f06d118684ef65a67ccf2ff07b84243d00e4644f6d341c0f247014ff4fe70154688470346a8b30020002100bf5a1858540431b1f5fe7f50709070d070fLine truncated
|
||||
"new": "02fb01f044f6d341c0f24701084700bf084644f61751c0f24701084770b500284ed001684ef65a62ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf2de9f04f83b043f6fc76c2f200063568824628466ff3960044f6d344b0f1005fc0f2470407d128684ef65a61ccf2ff01884200f0a4804ff40670a0470546002800f09c8000206ff021010022ab18a85443326f18b2f5067f01f1430158709870d870187158719871d871187258729872d872187358739873d873187458749874d874187558759875d875187658769876d876187758779877d87783f8200087f8430087f8440083f8230083f8240083f8250083f8260083f8270083f8280083f8290083f82a0083f82b0083f82c0083f82d0083f82e0083f82f0083f8300083f8310083f8320083f8330083f8340083f8350083f8360083f8370083f8380083f8390083f83a0083f83b0083f83c0083f83d0083f83e0083f83f0083f8400083f8410083f8420091d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe003560baf1000f18bf002d07d14af26950c0f2450003b0bde8f04f0047daf80cb0bbf1000ff2d0daf82080b8f1000fedd0a94659f8040f00287cd0686900287bd0686a00287ad0686b002879d0686c002878d0686d002877d0686e002876d0686f002875d0d5f88400002873d0d5f89400002871d0d5f8a40000286fd0d5f8b40000286dd0d5f8b00029690290d5f8a0002c6a0190d5f8900000260090286bd5f880e0d5f870c02b6e2f6d2a6c8c4238bf214638bf012688423cbf014602268a423cbf114603268f423cbf394604268b423cbf194605268c453cbf614606268e453cbf71460726009844f6d34488423cbf014608260198c0f2470488423cbf014609260298884238bf0146d5f8c00038bf0a26884238bf0b2609eb0610406804f1440137018847002049f80700012085f8df0016e0002614e0012612e0022610e003260ee004260ce005260ae0062608e0072606e0082604e0092602e00a2600e00b264046a04700283ff450af08f0030c5fea98034ff0000219d00296a2eb830e431c0bf101066ff0030700bff119cc78da19d4700c79043714714c79547189790eeb070404349171f0d1029e3a1dbcf1000f10d01bf80230bcf1010f83540ad0531c1bf80370bcf1020fc75403d0911c1bf80120425409eb06114860d5f8c400320149f802a0421cc1f80880c5f8c420c8600be738464146fff744be2de9f04f81b043f6fc79c2f20009d9f80050044628466ff3960044f6d34816460f46b0f1005fc0f2470807d128684ef65a61ccf2ff01884200f0a6804ff40670c0470546002800f0fc8000206ff02101002200bfab18a85443326f18b2f5067f01f1430158709870d870187158719871d871187258729872d872187358739873d873187458749874d874187558759875d875187658769876d876187758779877d87783f8200087f8430087f8440083f8230083f8240083f8250083f8260083f8270083f8280083f8290083f82a0083f82b0083f82c0083f82d0083f82e0083f82f0083f8300083f8310083f8320083f8330083f8340083f8350083f8360083f8370083f8380083f8390083f83a0083f83b0083f83c0083f83d0083f83e0083f83f0083f8400083f8410083f8420091d14ef65a60ccf2ff002860012085f8ce004ff0ff30a5f8e000a5f8e200a5f8e400a5f8e600a5f8e800a5f8ea00a5f8ec00a5f8ee00a5f8f000a5f8f200a5f8f400a5f8f600a5f8f800a5f8fa00a5f8fc00a5f8fe00c9f8005005f1040b4ff0000a4ff000094ff0ff30002200234ff0ff3105e000bf6032c02a03f1060332d0af187e68a64204d13e6986423cbf194630467e69a64204d13e6a86423cbf591c30467e6aa64204d13e6b86423cbf991c30467e6ba64204d13e6c86423cbfd91c30467e6ca64204d13e6d86423cbf191d30467e6da642ced13f6e87423cbf591d3846c8e700bf002910d40beb01104768090182684bf801a02046394600f013f8814608f1440138468847ace7484601b0bde8f08f204639463246c9f8005001b0bde8f04fffe72de9f041a4b0064640f6bb4043f6fc78c0f24e0014460d46c2f200088047002002af0390384600f049fd0df10c0c3046294622460123cdf800c000f055fe0446384601f07dfcd8f8001005460a466ff39602b2f1005f07d108684ef65a62ccf2ff02904200f0aa8044f6d341c0f247014ff406708847002800f0a380014600206ff02102002300bfcf18c85443338e18b3f5067f02f143027870b870f87038717871b871f87138727872b872f87238737873b873f87338747874b874f87438757875b875f87538767876b876f87638777877b877f87787f8200086f8430086f8440087f8230087f8240087f8250087f8260087f8270087f8280087f8290087f82a0087f82b0087f82c0087f82d0087f82e0087f82f0087f8300087f8310087f8320087f8330087f8340087f8350087f8360087f8370087f8380087f8390087f83a0087f83b0087f83c0087f83d0087f83e0087f83f0087f8400087f8410087f8420091d14ef65a60ccf2ff000860012081f8ce004ff0ff30a1f8e000a1f8e200a1f8e400a1f8e600a1f8e800a1f8ea00a1f8ec00a1f8ee00a1f8f000a1f8f200a1f8f400a1f8f600a1f8f800a1f8fa00a1f8fc00a1f8fe00c8f80010c1f8d05002e00020c8f80000204624b0bde8f08143f6fc70c2f20000016808466ff39600b0f1005f16d108684ef65a62ccf2ff0290420fd1d1f8fc0160b144f63422c2f207021268801a0028cebf01200020c1f8fc01704700207047f0b101684ef65a62ccf2ff02914218bf704790f8061299b110b544f62f344ff0000cc0f24604864680f806c20120002100220023aef804c2a446bde810406047704700bf2de9f04143f6fc77c2f200073f684ff000083c466ff39604b4f1005f07d13e684ef65a65ccf2ff05ae4208bfb84601280fd11c4615460f460646fff79bff2346044639462a463046002c18bfb8f1000f06d144f62f3cc0f2460cbde8f041604798f8060270b1d8f800020021002888f80612a8f804123ed049f2d941c0f24401884738e0b8f804020121013080b2012898bf0846a8f80402d8f8000249f2d944c0f2440488f8061268b94ff60370cff6ff70a4f594777844002142460023b847c8f8000240b1a047d8f80002a4f140024Line truncated
|
||||
"desc": "append injected blobs to main-app payload"
|
||||
},
|
||||
{
|
||||
"offset": 777711,
|
||||
"old": "44c33500",
|
||||
"new": "84fe3500",
|
||||
"new": "380e3600",
|
||||
"desc": "main-app subheader payload size (ps)"
|
||||
},
|
||||
{
|
||||
"offset": 152,
|
||||
"old": "c4c33500",
|
||||
"new": "04ff3500",
|
||||
"new": "b80e3600",
|
||||
"desc": "main-app TOC entry size (ps + 128)"
|
||||
},
|
||||
{
|
||||
"offset": 777831,
|
||||
"old": "44c33504",
|
||||
"new": "84fe3504",
|
||||
"new": "380e3604",
|
||||
"desc": "main-app preamble length (low 24 bits)"
|
||||
},
|
||||
{
|
||||
"offset": 777835,
|
||||
"old": "df8110c9",
|
||||
"new": "6eddece4",
|
||||
"new": "88ed0126",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
|
||||
},
|
||||
{
|
||||
"offset": 156,
|
||||
"old": "c1be0fb1",
|
||||
"new": "ed6b6bbd",
|
||||
"new": "f1e9f8d8",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
|
||||
},
|
||||
{
|
||||
"offset": 777715,
|
||||
"old": "c1be0fb1",
|
||||
"new": "ed6b6bbd",
|
||||
"new": "f1e9f8d8",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
|
||||
}
|
||||
]
|
||||
|
||||
+86
-5
@@ -13,14 +13,34 @@ const ANTHROPIC_VERSION = "2023-06-01";
|
||||
/** Default model for voice continuations (and a sensible general default). */
|
||||
export const DEFAULT_LLM_MODEL = "claude-sonnet-5";
|
||||
|
||||
/**
|
||||
* A content block in an Anthropic message. Text-only flows (dictation refine)
|
||||
* still pass a plain string for content; the assistant tool loop uses the
|
||||
* block array to carry tool_use / tool_result alongside text.
|
||||
*/
|
||||
export type AnthropicContentBlock =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "tool_use"; id: string; name: string; input: any }
|
||||
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean };
|
||||
|
||||
export type AnthropicMessage = {
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
content: string | AnthropicContentBlock[];
|
||||
};
|
||||
|
||||
/** An Anthropic tool definition (input_schema is snake_case as the API wants). */
|
||||
export type AnthropicToolDefinition = {
|
||||
name: string;
|
||||
description: string;
|
||||
input_schema: object;
|
||||
};
|
||||
|
||||
export type AnthropicStreamResult = {
|
||||
/** Concatenated text of all text blocks. */
|
||||
text: string;
|
||||
/** e.g. "end_turn", "max_tokens", "refusal"; null if the stream never said. */
|
||||
/** The assembled assistant content blocks (text + any tool_use). */
|
||||
content: AnthropicContentBlock[];
|
||||
/** e.g. "end_turn", "tool_use", "max_tokens", "refusal"; null if never said. */
|
||||
stopReason: string | null;
|
||||
};
|
||||
|
||||
@@ -29,6 +49,7 @@ export type AnthropicStreamOptions = {
|
||||
model?: string;
|
||||
system?: string;
|
||||
messages: AnthropicMessage[];
|
||||
tools?: AnthropicToolDefinition[];
|
||||
maxTokens?: number;
|
||||
/** Thinking/latency tradeoff; voice flows want "low". Default: model default. */
|
||||
effort?: "low" | "medium" | "high";
|
||||
@@ -39,6 +60,11 @@ export type AnthropicStreamOptions = {
|
||||
onError: (message: string) => void;
|
||||
};
|
||||
|
||||
/** Accumulator for one streamed content block before it is finalized. */
|
||||
type StreamingBlock =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "tool_use"; id: string; name: string; inputJson: string };
|
||||
|
||||
export type AnthropicStreamHandle = {
|
||||
cancel(): void;
|
||||
};
|
||||
@@ -49,6 +75,8 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
|
||||
let settled = false;
|
||||
let text = "";
|
||||
let stopReason: string | null = null;
|
||||
// Content blocks, keyed by their SSE `index`, assembled as deltas arrive.
|
||||
const blocks = new Map<number, StreamingBlock>();
|
||||
|
||||
const fail = (message: string) => {
|
||||
if (settled) return;
|
||||
@@ -58,7 +86,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
|
||||
const done = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
options.onDone({ text, stopReason });
|
||||
options.onDone({ text, content: finalizeBlocks(blocks), stopReason });
|
||||
};
|
||||
|
||||
if (!global.isAndroid) {
|
||||
@@ -78,6 +106,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
|
||||
messages: options.messages,
|
||||
};
|
||||
if (options.system) body.system = options.system;
|
||||
if (options.tools?.length) body.tools = options.tools;
|
||||
if (options.effort) body.output_config = { effort: options.effort };
|
||||
|
||||
const listener = new com.faceclaw.app.FaceclawSseListener({
|
||||
@@ -86,13 +115,41 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
|
||||
const event = parseSseDataLine(String(line));
|
||||
if (!event) return;
|
||||
switch (event.type) {
|
||||
case "content_block_delta":
|
||||
case "content_block_start": {
|
||||
const index = Number(event.index);
|
||||
const block = event.content_block;
|
||||
if (block?.type === "tool_use") {
|
||||
blocks.set(index, {
|
||||
type: "tool_use",
|
||||
id: String(block.id ?? ""),
|
||||
name: String(block.name ?? ""),
|
||||
inputJson: "",
|
||||
});
|
||||
} else if (block?.type === "text") {
|
||||
blocks.set(index, { type: "text", text: String(block.text ?? "") });
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "content_block_delta": {
|
||||
const index = Number(event.index);
|
||||
if (event.delta?.type === "text_delta") {
|
||||
const delta = String(event.delta.text ?? "");
|
||||
text += delta;
|
||||
const existing = blocks.get(index);
|
||||
if (existing?.type === "text") {
|
||||
existing.text += delta;
|
||||
} else {
|
||||
blocks.set(index, { type: "text", text: delta });
|
||||
}
|
||||
options.onTextDelta?.(delta, text);
|
||||
} else if (event.delta?.type === "input_json_delta") {
|
||||
const existing = blocks.get(index);
|
||||
if (existing?.type === "tool_use") {
|
||||
existing.inputJson += String(event.delta.partial_json ?? "");
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
case "message_delta":
|
||||
if (event.delta?.stop_reason) {
|
||||
stopReason = String(event.delta.stop_reason);
|
||||
@@ -105,7 +162,7 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
|
||||
fail(`Anthropic: ${String(event.error?.message ?? event.error?.type ?? "stream error")}`);
|
||||
return;
|
||||
default:
|
||||
// message_start, content_block_start/stop, ping — nothing to do.
|
||||
// message_start, content_block_stop, ping — nothing to do.
|
||||
return;
|
||||
}
|
||||
},
|
||||
@@ -146,6 +203,30 @@ export function streamAnthropicMessage(options: AnthropicStreamOptions): Anthrop
|
||||
};
|
||||
}
|
||||
|
||||
/** Assemble the streamed blocks (in index order) into final content blocks. */
|
||||
function finalizeBlocks(blocks: Map<number, StreamingBlock>): AnthropicContentBlock[] {
|
||||
const content: AnthropicContentBlock[] = [];
|
||||
for (const index of Array.from(blocks.keys()).sort((a, b) => a - b)) {
|
||||
const block = blocks.get(index)!;
|
||||
if (block.type === "text") {
|
||||
content.push({ type: "text", text: block.text });
|
||||
} else {
|
||||
let input: any = {};
|
||||
const trimmed = block.inputJson.trim();
|
||||
if (trimmed) {
|
||||
try {
|
||||
input = JSON.parse(trimmed);
|
||||
} catch {
|
||||
// A tool_use whose JSON never completed (e.g. truncated stream); pass
|
||||
// empty args and let the tool handler report what's missing.
|
||||
}
|
||||
}
|
||||
content.push({ type: "tool_use", id: block.id, name: block.name, input });
|
||||
}
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
/** Parse one SSE line; returns the JSON payload of a data: line, else null. */
|
||||
function parseSseDataLine(line: string): any | null {
|
||||
if (!line.startsWith("data:")) return null;
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont, type BdfFont } from "../../graphics/bdffont";
|
||||
import { GESTURE_DOUBLE_CLICK } from "../gestures";
|
||||
import { drawSelectionHighlight } from "../menu";
|
||||
import { Layer, type DashboardInputEvent, type LayerActions, type LayerContext } from "../layers";
|
||||
|
||||
const DIALOG_X = 40;
|
||||
const DIALOG_Y = 40;
|
||||
const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
const MENU_ROW_H = 20;
|
||||
const MENU_ROWS = 2;
|
||||
|
||||
/** thinking: turn running; done/error: finished, showing the Follow-up/Done menu. */
|
||||
type AssistantPhase = "thinking" | "done" | "error";
|
||||
|
||||
export type AssistantLayerCallbacks = {
|
||||
/** Record another utterance and continue this conversation. */
|
||||
onFollowUp: () => void;
|
||||
/** Abort the in-flight turn (double-click while thinking). */
|
||||
onCancel: () => void;
|
||||
/** The user closed the overlay (Done). */
|
||||
onClose: () => void;
|
||||
/** The layer left the stack by any path (Done, or the screen sleeping). */
|
||||
onRemoved?: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* The assistant overlay: streamed reply text with a status line for tool
|
||||
* activity, and a Follow-up / Done menu once the turn ends. Double-click
|
||||
* cancels an in-flight turn. The shell owns the AssistantSession and drives
|
||||
* this layer's state through the on* methods as the turn streams.
|
||||
*/
|
||||
export class AssistantLayer implements Layer {
|
||||
private phase: AssistantPhase = "thinking";
|
||||
private replyText = "";
|
||||
private status = "Thinking...";
|
||||
private menuIndex = 0;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly callbacks: AssistantLayerCallbacks,
|
||||
) {}
|
||||
|
||||
/** Shell: a new turn (initial utterance or follow-up) is starting. */
|
||||
startTurn(): void {
|
||||
this.phase = "thinking";
|
||||
this.replyText = "";
|
||||
this.status = "Thinking...";
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onTextDelta(_delta: string, textSoFar: string): void {
|
||||
this.replyText = textSoFar;
|
||||
if (this.phase === "thinking") this.status = "Thinking...";
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onToolActivity(label: string): void {
|
||||
if (this.phase === "thinking") this.status = `→ ${label}`;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onTurnDone(): void {
|
||||
this.phase = "done";
|
||||
this.status = this.replyText.trim() ? "" : "(no reply)";
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
onError(message: string): void {
|
||||
this.phase = "error";
|
||||
this.status = message;
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = paintBelow();
|
||||
|
||||
// Solid dialog box (fill 1, matching the voice dialog: opaque after 4bpp
|
||||
// quantization, but not the color-key transparent 0).
|
||||
image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 1, 10);
|
||||
image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_W, DIALOG_H, 90, 10);
|
||||
|
||||
const left = DIALOG_X + 16;
|
||||
image.drawText(font, left, DIALOG_Y + 12, this.phase === "thinking" ? "Assistant ●" : "Assistant", 220);
|
||||
if (this.status) {
|
||||
const statusValue = this.phase === "error" ? 200 : 130;
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), statusValue);
|
||||
}
|
||||
|
||||
const inMenu = this.phase !== "thinking";
|
||||
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const textTop = DIALOG_Y + 56;
|
||||
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
|
||||
|
||||
const wrapped = wrapText(font, this.replyText, TEXT_MAX_WIDTH);
|
||||
// Keep the tail visible as text streams past the bottom.
|
||||
const firstLine = Math.max(0, wrapped.length - maxLines);
|
||||
for (let index = firstLine; index < wrapped.length; index++) {
|
||||
image.drawText(font, left, textTop + (index - firstLine) * 16, wrapped[index]!, 235);
|
||||
}
|
||||
|
||||
if (inMenu) {
|
||||
const rows = ["Follow-up", "Done"];
|
||||
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const rowY = menuTop + i * MENU_ROW_H;
|
||||
const selected = i === this.menuIndex;
|
||||
if (selected) {
|
||||
drawSelectionHighlight(image, left - 4, rowY - 2, DIALOG_W - 24, MENU_ROW_H - 2, true, 6);
|
||||
}
|
||||
image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200);
|
||||
}
|
||||
} else {
|
||||
image.drawText(font, left, DIALOG_Y + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
if (this.phase === "thinking") {
|
||||
if (event.type === "double-click") {
|
||||
this.callbacks.onCancel();
|
||||
this.phase = "done";
|
||||
this.status = "Cancelled";
|
||||
this.menuIndex = 0;
|
||||
this.actions.requestRender();
|
||||
}
|
||||
return;
|
||||
}
|
||||
// done / error: the Follow-up / Done menu.
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "click":
|
||||
if (this.menuIndex === 0) {
|
||||
this.callbacks.onFollowUp();
|
||||
} else {
|
||||
this.callbacks.onClose();
|
||||
}
|
||||
return;
|
||||
case "double-click":
|
||||
this.callbacks.onClose();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.callbacks.onRemoved?.();
|
||||
}
|
||||
}
|
||||
|
||||
function wrapText(font: BdfFont, text: string, maxWidth: number): string[] {
|
||||
const words = text.split(/\s+/).filter(Boolean);
|
||||
const lines: string[] = [];
|
||||
let line = "";
|
||||
for (const word of words) {
|
||||
const candidate = line ? `${line} ${word}` : word;
|
||||
if (line && font.measureText(candidate) > maxWidth) {
|
||||
lines.push(line);
|
||||
line = word;
|
||||
} else {
|
||||
line = candidate;
|
||||
}
|
||||
}
|
||||
if (line) lines.push(line);
|
||||
return lines.length ? lines : [""];
|
||||
}
|
||||
|
||||
function truncate(font: BdfFont, text: string, maxWidth: number): string {
|
||||
if (font.measureText(text) <= maxWidth) return text;
|
||||
let out = text;
|
||||
while (out.length > 1 && font.measureText(`${out}...`) > maxWidth) {
|
||||
out = out.slice(0, -1);
|
||||
}
|
||||
return `${out}...`;
|
||||
}
|
||||
+136
-42
@@ -13,19 +13,47 @@ const DIALOG_W = G2_LENS_WIDTH - 80;
|
||||
const DIALOG_H = G2_LENS_HEIGHT - 80;
|
||||
const TEXT_MAX_WIDTH = DIALOG_W - 32;
|
||||
const MENU_ROW_H = 20;
|
||||
const MENU_ROWS = 3;
|
||||
// After the mic stops, the provider's final transcript can trail in (cloud
|
||||
// commit round-trip); wait this long for it before refining with what we have.
|
||||
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
|
||||
|
||||
/**
|
||||
* The dialog's lifecycle. "capturing" is the first utterance (push-to-talk or
|
||||
* click-to-finish); "menu" is the Send / Continue / Discard menu; "continuing"
|
||||
* click-to-finish); "menu" is the send / Continue / Discard menu; "continuing"
|
||||
* records a follow-up utterance (always click-to-finish); "refining" streams
|
||||
* the LLM-merged text, then returns to "menu".
|
||||
*/
|
||||
type VoicePhase = "capturing" | "menu" | "continuing" | "refining";
|
||||
|
||||
/**
|
||||
* A destination the captured text can be sent to. The shell supplies these per
|
||||
* entry point: "Send to Assistant" (hand off to the voice assistant) and/or
|
||||
* "Type Into App" (deliver to the foreground window's text input).
|
||||
*/
|
||||
export type VoiceSendTarget = {
|
||||
id: string;
|
||||
label: string;
|
||||
onSend: (text: string) => void;
|
||||
};
|
||||
|
||||
export type VoiceInputLayerOptions = {
|
||||
actions: LayerActions;
|
||||
/** Post-removal cleanup (also fires when the screen turns off). */
|
||||
onClosed: () => void;
|
||||
/** Pop this layer off the shell stack. */
|
||||
dismiss: () => void;
|
||||
/** Ordered send destinations shown as the first menu rows. */
|
||||
sendTargets: VoiceSendTarget[];
|
||||
/** Which send target is highlighted by default (entry-point dependent). */
|
||||
defaultTargetIndex?: number;
|
||||
/** A single click ends the utterance (opened from a menu, no held button). */
|
||||
finishOnClick?: boolean;
|
||||
/** Wakeword flow: mic starts immediately and ends on detected silence. */
|
||||
handsFree?: boolean;
|
||||
/** Skip the menu: on capture end, send straight to the default target. */
|
||||
autoSend?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Push-to-talk voice dialog, drawn on top of whatever is already on screen.
|
||||
* The mic runs while the button is held (long-press); releasing it stops the
|
||||
@@ -58,17 +86,34 @@ export class VoiceInputLayer implements Layer {
|
||||
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private refineHandle: AnthropicStreamHandle | null = null;
|
||||
private menuIndex = 0;
|
||||
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
|
||||
private pendingAutoSend = false;
|
||||
private autoSendTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeSpeechEnd: (() => void) | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly actions: LayerActions,
|
||||
private readonly onClosed: () => void,
|
||||
private readonly onSend: (text: string) => void,
|
||||
private readonly finishOnClick = false,
|
||||
private readonly handsFree = false,
|
||||
) {}
|
||||
private readonly actions: LayerActions;
|
||||
private readonly onClosed: () => void;
|
||||
private readonly dismiss: () => void;
|
||||
private readonly sendTargets: VoiceSendTarget[];
|
||||
private readonly defaultTargetIndex: number;
|
||||
private readonly finishOnClick: boolean;
|
||||
private readonly handsFree: boolean;
|
||||
private readonly autoSend: boolean;
|
||||
|
||||
constructor(options: VoiceInputLayerOptions) {
|
||||
this.actions = options.actions;
|
||||
this.onClosed = options.onClosed;
|
||||
this.dismiss = options.dismiss;
|
||||
this.sendTargets = options.sendTargets;
|
||||
this.finishOnClick = options.finishOnClick ?? false;
|
||||
this.handsFree = options.handsFree ?? false;
|
||||
this.autoSend = options.autoSend ?? false;
|
||||
const defaultIndex = options.defaultTargetIndex ?? 0;
|
||||
this.defaultTargetIndex = Math.min(Math.max(0, defaultIndex), Math.max(0, this.sendTargets.length - 1));
|
||||
this.menuIndex = this.defaultTargetIndex;
|
||||
}
|
||||
|
||||
startCapture(): void {
|
||||
if (this.capturing) return;
|
||||
@@ -113,13 +158,66 @@ export class VoiceInputLayer implements Layer {
|
||||
this.capturing = false;
|
||||
this.phase = "menu";
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.status.startsWith("Listening")) {
|
||||
if (this.autoSend && this.sendTargets.length) {
|
||||
// Skip-confirmation (wakeword): send to the default target as soon as the
|
||||
// transcript finalizes, or after a short wait for the trailing final.
|
||||
this.pendingAutoSend = true;
|
||||
this.status = "Sending...";
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
} else if (this.status.startsWith("Listening")) {
|
||||
this.status = "Send, continue, or discard?";
|
||||
}
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
/** Fire the queued skip-confirmation send (or fall back to the menu). */
|
||||
private performAutoSend(): void {
|
||||
if (this.autoSendTimer !== null) {
|
||||
clearTimeout(this.autoSendTimer);
|
||||
this.autoSendTimer = null;
|
||||
}
|
||||
if (!this.pendingAutoSend) return;
|
||||
this.pendingAutoSend = false;
|
||||
const text = this.displayText().trim();
|
||||
const target = this.sendTargets[this.defaultTargetIndex];
|
||||
if (text && target) {
|
||||
this.dismiss();
|
||||
target.onSend(text);
|
||||
} else {
|
||||
// Nothing heard; fall back to the menu so the user can retry or discard.
|
||||
this.status = "Send, continue, or discard?";
|
||||
this.actions.requestRender();
|
||||
}
|
||||
}
|
||||
|
||||
/** The menu rows: one per send target, then Continue, then Discard. */
|
||||
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
|
||||
const text = this.displayText().trim();
|
||||
const hasText = text.length > 0;
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
|
||||
for (const target of this.sendTargets) {
|
||||
rows.push({
|
||||
label: target.label,
|
||||
dim: !hasText,
|
||||
onSelect: () => {
|
||||
this.dismiss();
|
||||
if (hasText) target.onSend(text);
|
||||
},
|
||||
});
|
||||
}
|
||||
rows.push({
|
||||
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
dim: !hasLlmKey,
|
||||
onSelect: () => {
|
||||
if (hasLlmKey) this.startContinuation();
|
||||
},
|
||||
});
|
||||
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
|
||||
return rows;
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const image = paintBelow();
|
||||
|
||||
@@ -133,9 +231,9 @@ export class VoiceInputLayer implements Layer {
|
||||
image.drawText(font, left, DIALOG_Y + 30, truncate(font, this.status, TEXT_MAX_WIDTH), 130);
|
||||
|
||||
const inMenu = this.phase === "menu";
|
||||
const hasText = this.displayText().trim().length > 0;
|
||||
// Leave room for the three-row menu when it is showing.
|
||||
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const rows = inMenu ? this.menuRows() : [];
|
||||
// Reserve space for the actual number of rows this menu has.
|
||||
const textBottom = inMenu ? DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 8 : DIALOG_Y + DIALOG_H - 8;
|
||||
const textTop = DIALOG_Y + 56;
|
||||
const maxLines = Math.max(1, ((textBottom - textTop) / 16) | 0);
|
||||
|
||||
@@ -147,14 +245,7 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
if (inMenu) {
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean }> = [
|
||||
// Dim "Send" when there is nothing to send.
|
||||
{ label: "Send", dim: !hasText },
|
||||
{ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", dim: !hasLlmKey },
|
||||
{ label: "Discard", dim: false },
|
||||
];
|
||||
const menuTop = DIALOG_Y + DIALOG_H - MENU_ROWS * MENU_ROW_H - 2;
|
||||
const menuTop = DIALOG_Y + DIALOG_H - rows.length * MENU_ROW_H - 2;
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const rowY = menuTop + i * MENU_ROW_H;
|
||||
const selected = i === this.menuIndex;
|
||||
@@ -170,11 +261,11 @@ export class VoiceInputLayer implements Layer {
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
switch (this.phase) {
|
||||
case "capturing":
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
this.dismiss();
|
||||
} else if (this.clickEndsCapture && event.type === "click") {
|
||||
this.endCapture();
|
||||
}
|
||||
@@ -192,37 +283,29 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
return;
|
||||
case "menu":
|
||||
this.handleMenuInput(event, ctx);
|
||||
this.handleMenuInput(event);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private handleMenuInput(event: DashboardInputEvent, ctx: LayerContext): void {
|
||||
private handleMenuInput(event: DashboardInputEvent): void {
|
||||
const rowCount = this.menuRows().length;
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
this.menuIndex = (this.menuIndex + MENU_ROWS - 1) % MENU_ROWS;
|
||||
this.menuIndex = (this.menuIndex + rowCount - 1) % rowCount;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "scroll-down":
|
||||
this.menuIndex = (this.menuIndex + 1) % MENU_ROWS;
|
||||
this.menuIndex = (this.menuIndex + 1) % rowCount;
|
||||
this.actions.requestRender();
|
||||
return;
|
||||
case "click":
|
||||
if (this.menuIndex === 0) {
|
||||
const text = this.displayText().trim();
|
||||
if (text) this.onSend(text);
|
||||
ctx.stack.pop();
|
||||
} else if (this.menuIndex === 1) {
|
||||
if (anthropicApiKeySetting.get().trim()) {
|
||||
this.startContinuation();
|
||||
}
|
||||
// No key: the row is disabled; the click does nothing.
|
||||
} else {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
case "click": {
|
||||
const row = this.menuRows()[this.menuIndex];
|
||||
row?.onSelect();
|
||||
return;
|
||||
}
|
||||
case "double-click":
|
||||
ctx.stack.pop();
|
||||
this.dismiss();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
@@ -323,6 +406,11 @@ export class VoiceInputLayer implements Layer {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
}
|
||||
if (this.autoSendTimer !== null) {
|
||||
clearTimeout(this.autoSendTimer);
|
||||
this.autoSendTimer = null;
|
||||
}
|
||||
this.pendingAutoSend = false;
|
||||
this.refineHandle?.cancel();
|
||||
this.refineHandle = null;
|
||||
if (this.capturing) {
|
||||
@@ -380,6 +468,12 @@ export class VoiceInputLayer implements Layer {
|
||||
this.beginRefine();
|
||||
return;
|
||||
}
|
||||
if (this.pendingAutoSend) {
|
||||
// The utterance finalized; skip-confirmation can fire without waiting
|
||||
// out the fallback timer.
|
||||
this.performAutoSend();
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
this.liveText = event.text.trim();
|
||||
}
|
||||
|
||||
@@ -294,6 +294,13 @@ export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
|
||||
defaultValue: false,
|
||||
});
|
||||
|
||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
id: "assistant-skip-confirmation",
|
||||
label: "Send to assistant without confirming",
|
||||
storageKey: "assistant.skipConfirmationAfterWakeword",
|
||||
defaultValue: false,
|
||||
});
|
||||
|
||||
export const elevenLabsApiKeySetting = new ConfigSettingString({
|
||||
id: "elevenlabs-api-key",
|
||||
label: "ElevenLabs key",
|
||||
|
||||
@@ -3,6 +3,7 @@ import { getDashboardLogo } from "../../graphics/logo";
|
||||
import { shell } from "../shell/shell";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
brightnessSetting,
|
||||
elevenLabsApiKeySetting,
|
||||
@@ -56,6 +57,9 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
// When on, a wakeword utterance goes straight to the assistant with no
|
||||
// Send/Type menu step.
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
+291
-14
@@ -1,11 +1,17 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import { EvenAIStatus, EventSourceType, OsEventTypeList } from "../../g2/events";
|
||||
import type { RawInputEvent } from "../../native/faceclaw-communicator";
|
||||
import { DashboardInputEvent, LayerActions, LayerStack, noopLayerActions } from "../layers";
|
||||
import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
|
||||
import { MenuLayer, type MenuItem } from "../menu";
|
||||
import { VoiceInputLayer } from "../apps/voice-input";
|
||||
import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input";
|
||||
import { AssistantLayer } from "../apps/assistant";
|
||||
import { AssistantSession } from "../../assistant/session";
|
||||
import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
onAnySettingChanged,
|
||||
timeFormatSetting,
|
||||
@@ -99,6 +105,64 @@ class ShellOverlayMenuLayer extends MenuLayer {
|
||||
}
|
||||
}
|
||||
|
||||
/** How long a show_alert popup stays before auto-dismissing. */
|
||||
const ALERT_DISMISS_MS = 6000;
|
||||
const ALERT_X = 40;
|
||||
const ALERT_W = G2_LENS_WIDTH - 80;
|
||||
const ALERT_Y = 96;
|
||||
const ALERT_H = 96;
|
||||
|
||||
/**
|
||||
* A brief text popup on the shell surface (the assistant's show_alert tool and
|
||||
* other short notices). Auto-dismisses after a few seconds; a click or
|
||||
* double-click dismisses it early.
|
||||
*/
|
||||
class ShellAlertLayer implements Layer {
|
||||
private timer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(private readonly text: string, private readonly onDismiss: () => void) {
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null;
|
||||
this.onDismiss();
|
||||
}, ALERT_DISMISS_MS);
|
||||
}
|
||||
|
||||
paint(_ctx: LayerContext, paintBelow: () => GrayImage): GrayImage {
|
||||
const image = paintBelow();
|
||||
const font = getDefaultSmallFont();
|
||||
image.fillRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 1, 10);
|
||||
image.drawRoundedRect(ALERT_X, ALERT_Y, ALERT_W, ALERT_H, 90, 10);
|
||||
image.drawText(font, ALERT_X + 16, ALERT_Y + 12, "Assistant", 200);
|
||||
image.drawTextWrapped({
|
||||
font,
|
||||
x: ALERT_X + 16,
|
||||
y: ALERT_Y + 34,
|
||||
width: ALERT_W - 32,
|
||||
text: this.text,
|
||||
value: 235,
|
||||
});
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
if (event.type === "click" || event.type === "double-click") {
|
||||
this.clearTimer();
|
||||
this.onDismiss();
|
||||
}
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.clearTimer();
|
||||
}
|
||||
|
||||
private clearTimer(): void {
|
||||
if (this.timer !== null) {
|
||||
clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class Shell {
|
||||
private windows: ShellWindow[] = [];
|
||||
private selectedIndex = 0;
|
||||
@@ -111,6 +175,8 @@ class Shell {
|
||||
// notification icons and the battery indicators.
|
||||
private readonly trayIcons = new Map<string, GrayImage>();
|
||||
private activeVoiceLayer: VoiceInputLayer | null = null;
|
||||
private assistantSession: AssistantSession | null = null;
|
||||
private assistantLayer: AssistantLayer | null = null;
|
||||
private escapeMenuTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private readonly actions: LayerActions = { ...noopActions };
|
||||
private config: ShellConfig = {
|
||||
@@ -236,6 +302,21 @@ class Shell {
|
||||
return this.screenOn;
|
||||
}
|
||||
|
||||
/** Current headset battery levels (for the assistant's get_state tool). */
|
||||
getBatteryLevels(): ShellChromeState["battery"] {
|
||||
return this.battery;
|
||||
}
|
||||
|
||||
/**
|
||||
* The foreground app, or null when only the launcher is showing. The launcher
|
||||
* is a pinned window but counts as "no app" for the assistant's context.
|
||||
*/
|
||||
getForegroundApp(): { appId: string; title: string } | null {
|
||||
const window = this.foregroundWindow();
|
||||
if (!window || window.appId === "launcher") return null;
|
||||
return { appId: window.appId, title: window.title };
|
||||
}
|
||||
|
||||
noteUserActivity(nowMs = Date.now()): void {
|
||||
this.lastInputAtMs = nowMs;
|
||||
}
|
||||
@@ -283,7 +364,10 @@ class Shell {
|
||||
// An open voice dialog suspends the timeout: a long dictation or refine
|
||||
// has no button presses, but the screen must stay on for it. Sliding
|
||||
// lastInputAtMs forward also restarts the full timeout when it closes.
|
||||
if (this.activeVoiceLayer) {
|
||||
// An in-flight assistant turn suspends it for the same reason (a tool loop
|
||||
// can run for a while with no input); once the turn ends and the
|
||||
// Follow-up/Done menu is showing, the normal idle timeout resumes.
|
||||
if (this.activeVoiceLayer || this.assistantSession?.isTurnActive()) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
@@ -358,7 +442,13 @@ class Shell {
|
||||
this.lastInputAtMs = Date.now();
|
||||
const wokeScreen = !this.screenOn && this.wake("sidebar");
|
||||
if (action === "voice-input" && !this.activeVoiceLayer) {
|
||||
this.openVoiceDialog(false, true);
|
||||
if (this.assistantLayer) {
|
||||
// The assistant overlay is up; a wakeword continues that conversation.
|
||||
this.startAssistantFollowUp(true);
|
||||
} else {
|
||||
// Wakeword defaults the highlight to Send to Assistant.
|
||||
this.openVoiceDialog({ handsFree: true, defaultTarget: "assistant" });
|
||||
}
|
||||
}
|
||||
return { shell: wokeScreen || action === "voice-input", window: false };
|
||||
}
|
||||
@@ -489,25 +579,77 @@ class Shell {
|
||||
next?.requestRender();
|
||||
}
|
||||
|
||||
private openVoiceDialog(finishOnClick = false, handsFree = false): void {
|
||||
const layer = new VoiceInputLayer(
|
||||
this.config.actions,
|
||||
() => {
|
||||
private openVoiceDialog(options: {
|
||||
finishOnClick?: boolean;
|
||||
handsFree?: boolean;
|
||||
defaultTarget: "assistant" | "app";
|
||||
}): void {
|
||||
const targets = this.buildVoiceSendTargets();
|
||||
let defaultIndex = targets.findIndex((target) => target.id === options.defaultTarget);
|
||||
if (defaultIndex < 0) defaultIndex = 0;
|
||||
// Skip the menu only for a hands-free (wakeword) capture aimed at the
|
||||
// assistant, when the user has opted into it.
|
||||
const autoSend =
|
||||
Boolean(options.handsFree) &&
|
||||
options.defaultTarget === "assistant" &&
|
||||
targets[defaultIndex]?.id === "assistant" &&
|
||||
assistantSkipConfirmationSetting.get();
|
||||
|
||||
const layer = new VoiceInputLayer({
|
||||
actions: this.config.actions,
|
||||
onClosed: () => {
|
||||
if (this.activeVoiceLayer === layer) {
|
||||
this.activeVoiceLayer = null;
|
||||
// The idle countdown restarts in full once voice input ends.
|
||||
this.noteUserActivity();
|
||||
}
|
||||
},
|
||||
(text) => this.sendTextToForegroundWindow(text),
|
||||
finishOnClick,
|
||||
handsFree,
|
||||
);
|
||||
dismiss: () => {
|
||||
this.stack.popIfTop((top) => top === layer);
|
||||
},
|
||||
sendTargets: targets,
|
||||
defaultTargetIndex: defaultIndex,
|
||||
finishOnClick: options.finishOnClick ?? false,
|
||||
handsFree: options.handsFree ?? false,
|
||||
autoSend,
|
||||
});
|
||||
this.activeVoiceLayer = layer;
|
||||
this.stack.push(layer);
|
||||
layer.startCapture();
|
||||
}
|
||||
|
||||
/**
|
||||
* The send destinations offered by the voice dialog: the assistant (when an
|
||||
* API key is configured) and/or typing into the foreground window (when it
|
||||
* accepts text). Order fixes the menu row order.
|
||||
*/
|
||||
private buildVoiceSendTargets(): VoiceSendTarget[] {
|
||||
const targets: VoiceSendTarget[] = [];
|
||||
if (this.isAssistantAvailable()) {
|
||||
targets.push({
|
||||
id: "assistant",
|
||||
label: "Send to Assistant",
|
||||
onSend: (text) => this.sendToAssistant(text),
|
||||
});
|
||||
}
|
||||
if (this.foregroundWindow()?.receiveTextInput) {
|
||||
targets.push({
|
||||
id: "app",
|
||||
label: "Type Into App",
|
||||
onSend: (text) => this.sendTextToForegroundWindow(text),
|
||||
});
|
||||
}
|
||||
// Guarantee at least one destination so the dialog is never a dead end.
|
||||
if (targets.length === 0) {
|
||||
targets.push({
|
||||
id: "app",
|
||||
label: "Type Into App",
|
||||
onSend: (text) => this.sendTextToForegroundWindow(text),
|
||||
});
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
/** Deliver a text string to the foreground window (e.g. finalized voice input). */
|
||||
sendTextToForegroundWindow(text: string): void {
|
||||
this.foregroundWindow()?.receiveTextInput?.(text);
|
||||
@@ -521,9 +663,130 @@ class Shell {
|
||||
*/
|
||||
startVoiceInput(): void {
|
||||
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
||||
// The transcript is aimed at the window whose menu requested it.
|
||||
// The transcript is aimed at the window whose menu requested it; the menu
|
||||
// entry point defaults the highlight to Type Into App.
|
||||
this.focus = "window";
|
||||
this.openVoiceDialog(true);
|
||||
this.openVoiceDialog({ finishOnClick: true, defaultTarget: "app" });
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private isAssistantAvailable(): boolean {
|
||||
return anthropicApiKeySetting.get().trim().length > 0;
|
||||
}
|
||||
|
||||
private ensureAssistantSession(): AssistantSession | null {
|
||||
const apiKey = anthropicApiKeySetting.get().trim();
|
||||
if (!apiKey) return null;
|
||||
if (!this.assistantSession || this.assistantSession.isExpired()) {
|
||||
this.assistantSession = new AssistantSession(apiKey);
|
||||
}
|
||||
return this.assistantSession;
|
||||
}
|
||||
|
||||
private buildAssistantContext(): AssistantContext {
|
||||
const foreground = this.getForegroundApp();
|
||||
return {
|
||||
foregroundApp: foreground?.appId ?? null,
|
||||
foregroundTitle: foreground?.title ?? null,
|
||||
screenOn: this.screenOn,
|
||||
localTime: formatAssistantTime(new Date()),
|
||||
headsetBattery: this.battery.headset,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Start (or continue) an assistant conversation from a finalized utterance.
|
||||
* Opens the assistant overlay if it isn't already up; a follow-up reuses the
|
||||
* existing session and overlay.
|
||||
*/
|
||||
sendToAssistant(text: string): void {
|
||||
const session = this.ensureAssistantSession();
|
||||
if (!session) {
|
||||
this.showAlert("Set an Anthropic API key in Settings to use the assistant.");
|
||||
return;
|
||||
}
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
let layer = this.assistantLayer;
|
||||
if (!layer) {
|
||||
const created = new AssistantLayer(this.config.actions, {
|
||||
onFollowUp: () => this.startAssistantFollowUp(),
|
||||
onCancel: () => this.assistantSession?.cancel(),
|
||||
onClose: () => this.closeAssistantLayer(),
|
||||
onRemoved: () => {
|
||||
// Removed by any path (Done, or the screen sleeping mid-conversation):
|
||||
// stop the turn and drop the reference so a later query starts clean.
|
||||
this.assistantSession?.cancel();
|
||||
if (this.assistantLayer === created) this.assistantLayer = null;
|
||||
},
|
||||
});
|
||||
layer = created;
|
||||
this.assistantLayer = created;
|
||||
this.stack.push(created);
|
||||
}
|
||||
this.runAssistantTurn(session, layer, text);
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private runAssistantTurn(session: AssistantSession, layer: AssistantLayer, text: string): void {
|
||||
layer.startTurn();
|
||||
session.sendUtterance(text, this.buildAssistantContext(), {
|
||||
onTextDelta: (delta, textSoFar) => layer.onTextDelta(delta, textSoFar),
|
||||
onToolActivity: (label) => layer.onToolActivity(label),
|
||||
onTurnDone: () => layer.onTurnDone(),
|
||||
onError: (message) => layer.onError(message),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Record another utterance in the current assistant conversation. handsFree
|
||||
* (from a wakeword over the overlay) starts the mic immediately; otherwise
|
||||
* (the Follow-up menu button) a click ends the utterance.
|
||||
*/
|
||||
private startAssistantFollowUp(handsFree = false): void {
|
||||
const layer = this.assistantLayer;
|
||||
const session = this.assistantSession;
|
||||
if (!layer || !session || this.activeVoiceLayer) return;
|
||||
const voice = new VoiceInputLayer({
|
||||
actions: this.config.actions,
|
||||
onClosed: () => {
|
||||
if (this.activeVoiceLayer === voice) {
|
||||
this.activeVoiceLayer = null;
|
||||
this.noteUserActivity();
|
||||
}
|
||||
},
|
||||
dismiss: () => {
|
||||
this.stack.popIfTop((top) => top === voice);
|
||||
},
|
||||
sendTargets: [
|
||||
{ id: "assistant", label: "Send", onSend: (text) => this.runAssistantTurn(session, layer, text) },
|
||||
],
|
||||
finishOnClick: !handsFree,
|
||||
handsFree,
|
||||
autoSend: handsFree && assistantSkipConfirmationSetting.get(),
|
||||
});
|
||||
this.activeVoiceLayer = voice;
|
||||
this.stack.push(voice);
|
||||
voice.startCapture();
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
private closeAssistantLayer(): void {
|
||||
// Popping fires the layer's onRemoved, which cancels the turn and clears
|
||||
// this.assistantLayer.
|
||||
const layer = this.assistantLayer;
|
||||
if (layer) this.stack.popIfTop((top) => top === layer);
|
||||
this.noteUserActivity();
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
/** Show a brief text popup on the lenses (assistant show_alert / notices). */
|
||||
showAlert(text: string): void {
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
const layer = new ShellAlertLayer(text, () => {
|
||||
this.stack.popIfTop((top) => top === layer);
|
||||
this.config.requestShellRender();
|
||||
});
|
||||
this.stack.push(layer);
|
||||
this.config.requestShellRender();
|
||||
}
|
||||
|
||||
@@ -586,6 +849,20 @@ class Shell {
|
||||
|
||||
export const shell = new Shell();
|
||||
|
||||
const ASSISTANT_WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const ASSISTANT_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
/** Human-readable local time for the assistant's per-turn context. */
|
||||
function formatAssistantTime(date: Date): string {
|
||||
let hours = date.getHours();
|
||||
const minutes = date.getMinutes().toString().padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
const day = `${ASSISTANT_WEEKDAYS[date.getDay()]} ${ASSISTANT_MONTHS[date.getMonth()]} ${date.getDate()}`;
|
||||
return `${day}, ${hours}:${minutes} ${meridiem}`;
|
||||
}
|
||||
|
||||
export function rawInputEventToInputEvent(event: RawInputEvent): DashboardInputEvent {
|
||||
if (event.kind === "sys-event") {
|
||||
if (event.eventType === OsEventTypeList.CLICK_EVENT) {
|
||||
|
||||
Reference in new issue
Block a user