diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index da1b7ae2..dc15bd00 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -1,10 +1,13 @@ -import { - streamAnthropicMessage, - type AnthropicContentBlock, - type AnthropicMessage, - type AnthropicStreamHandle, - type AnthropicToolDefinition, -} from "../native/anthropic"; +import { streamAnthropicMessage } from "../native/anthropic"; +import { streamOpenAiResponse } from "../native/openai"; +import type { + LlmContentBlock, + LlmMessage, + LlmStreamHandle, + LlmStreamOptions, + LlmToolDefinition, +} from "./llm-protocol"; +import type { AssistantProvider } from "./models"; import type { ToolRegistry } from "./tool-registry"; import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types"; @@ -20,29 +23,30 @@ const MAX_ITERATIONS = 8; const MAX_TURN_MS = 2 * 60 * 1000; export type DirectTurnOptions = { + provider: AssistantProvider; apiKey: string; model: string; effort?: "low" | "medium" | "high"; system: string; /** Owned by the session; this loop appends assistant + tool_result messages. */ - messages: AnthropicMessage[]; + messages: LlmMessage[]; /** * Build the tool list for the next request. Called once per loop iteration so * the set reflects apps opening/closing and foreground changes mid-turn; it * also refreshes the session's api-name map that resolveToolName reads. */ - buildTools: () => AnthropicToolDefinition[]; + buildTools: () => LlmToolDefinition[]; registry: ToolRegistry; /** Map an API tool name (as the model calls it) back to its registry name. */ resolveToolName: (apiName: string) => string; callbacks: AssistantTurnCallbacks; }; -export class DirectAnthropicBackend { +export class DirectAssistantBackend { runTurn(options: DirectTurnOptions): AssistantTurnHandle { const startMs = Date.now(); let cancelled = false; - let streamHandle: AnthropicStreamHandle | null = null; + let streamHandle: LlmStreamHandle | null = null; const finishError = (message: string) => { if (cancelled) return; @@ -61,7 +65,7 @@ export class DirectAnthropicBackend { return; } - streamHandle = streamAnthropicMessage({ + const streamOptions: LlmStreamOptions = { apiKey: options.apiKey, model: options.model, system: options.system, @@ -80,13 +84,13 @@ export class DirectAnthropicBackend { return; } // Record what the model produced this iteration. - const assistantContent: AnthropicContentBlock[] = result.content.length + const assistantContent: LlmContentBlock[] = result.content.length ? result.content : [{ type: "text", text: result.text }]; options.messages.push({ role: "assistant", content: assistantContent }); const toolUses = assistantContent.filter( - (block): block is Extract => + (block): block is Extract => block.type === "tool_use", ); if (result.stopReason !== "tool_use" || toolUses.length === 0) { @@ -110,7 +114,10 @@ export class DirectAnthropicBackend { streamHandle = null; finishError(message); }, - }); + }; + streamHandle = options.provider === "openai" + ? streamOpenAiResponse(streamOptions) + : streamAnthropicMessage(streamOptions); }; runIteration(0); @@ -125,10 +132,10 @@ export class DirectAnthropicBackend { } private async runToolCalls( - toolUses: Array>, + toolUses: Array>, options: DirectTurnOptions, - ): Promise { - const results: AnthropicContentBlock[] = []; + ): Promise { + const results: LlmContentBlock[] = []; for (const toolUse of toolUses) { const canonicalName = options.resolveToolName(toolUse.name); options.callbacks.onToolActivity(canonicalName); diff --git a/app/assistant/llm-protocol.ts b/app/assistant/llm-protocol.ts new file mode 100644 index 00000000..e351c90f --- /dev/null +++ b/app/assistant/llm-protocol.ts @@ -0,0 +1,56 @@ +/** + * Provider-neutral shapes used by the direct assistant loop. Each native LLM + * client translates these messages and tools to its provider's wire format. + */ + +export type LlmContentBlock = + | { type: "text"; text: string } + | { + type: "tool_use"; + id: string; + name: string; + input: any; + /** Exact provider output item to replay during stateless continuation. */ + provider_item?: { provider: "openai"; item: any }; + } + | { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean } + /** Opaque state required to continue a provider's stateless conversation. */ + | { type: "provider_item"; provider: "openai"; item: any }; + +export type LlmMessage = { + role: "user" | "assistant"; + content: string | LlmContentBlock[]; +}; + +export type LlmToolDefinition = { + name: string; + description: string; + input_schema: object; +}; + +export type LlmStreamResult = { + /** Concatenated text of all text blocks. */ + text: string; + /** The assembled assistant content blocks (text + any tool calls). */ + content: LlmContentBlock[]; + /** Provider-normalized stop reason: end_turn, tool_use, refusal, etc. */ + stopReason: string | null; +}; + +export type LlmStreamOptions = { + apiKey: string; + model: string; + system?: string; + messages: LlmMessage[]; + tools?: LlmToolDefinition[]; + maxTokens?: number; + /** Thinking/latency tradeoff; voice flows want "low". */ + effort?: "low" | "medium" | "high"; + onTextDelta?: (delta: string, textSoFar: string) => void; + onDone: (result: LlmStreamResult) => void; + onError: (message: string) => void; +}; + +export type LlmStreamHandle = { + cancel(): void; +}; diff --git a/app/assistant/models.ts b/app/assistant/models.ts new file mode 100644 index 00000000..6184dad9 --- /dev/null +++ b/app/assistant/models.ts @@ -0,0 +1,88 @@ +export const ASSISTANT_MODEL_VALUES = [ + "auto", + "hauku", + "sonnet", + "opus", + "fable", + "luna", + "terra", + "sol", +] as const; + +export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number]; +export type AssistantProvider = "anthropic" | "openai"; + +export type AssistantApiKeys = { + anthropic: string; + openai: string; +}; + +export type ResolvedAssistantModel = { + selection: AssistantModel; + provider: AssistantProvider; + model: string; + apiKey: string; + effort?: "low" | "medium" | "high"; +}; + +type ModelDefinition = { + label: string; + provider: AssistantProvider; + model: string; + effort?: "low" | "medium" | "high"; +}; + +const MODEL_DEFINITIONS: Record, ModelDefinition> = { + // Keep the user-facing spelling requested for the picker while using + // Anthropic's canonical Haiku alias on the wire. + hauku: { label: "Hauku", provider: "anthropic", model: "claude-haiku-4-5" }, + sonnet: { label: "Sonnet", provider: "anthropic", model: "claude-sonnet-5", effort: "low" }, + opus: { label: "Opus", provider: "anthropic", model: "claude-opus-5", effort: "low" }, + fable: { label: "Fable", provider: "anthropic", model: "claude-fable-5", effort: "low" }, + luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" }, + terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" }, + sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" }, +}; + +export function assistantModelLabel(value: AssistantModel): string { + return value === "auto" ? "Auto" : MODEL_DEFINITIONS[value].label; +} + +export function assistantModelProvider(value: AssistantModel): AssistantProvider | null { + return value === "auto" ? null : MODEL_DEFINITIONS[value].provider; +} + +/** + * Resolve Auto and attach the credential needed by the chosen provider. + * Auto intentionally prefers Terra over Sonnet when both keys are available. + */ +export function resolveAssistantModel( + selection: AssistantModel, + keys: AssistantApiKeys, +): ResolvedAssistantModel | null { + const normalizedKeys: AssistantApiKeys = { + anthropic: keys.anthropic.trim(), + openai: keys.openai.trim(), + }; + let resolvedSelection = selection; + if (selection === "auto") { + if (normalizedKeys.openai) { + resolvedSelection = "terra"; + } else if (normalizedKeys.anthropic) { + resolvedSelection = "sonnet"; + } else { + return null; + } + } + + const definition = MODEL_DEFINITIONS[resolvedSelection as Exclude]; + const apiKey = normalizedKeys[definition.provider]; + if (!apiKey) return null; + return { + selection, + provider: definition.provider, + model: definition.model, + apiKey, + effort: definition.effort, + }; +} diff --git a/app/assistant/session.ts b/app/assistant/session.ts index 3a6f353c..dd554879 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -1,12 +1,13 @@ -import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic"; import { buildAssistantSystemPrompt } from "../prompts"; -import { DirectAnthropicBackend } from "./direct-backend"; +import { DirectAssistantBackend } from "./direct-backend"; +import type { LlmMessage, LlmToolDefinition } from "./llm-protocol"; +import type { ResolvedAssistantModel } from "./models"; import { toolRegistry, type ToolRegistry } from "./tool-registry"; import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types"; /** * One assistant conversation. Owns the message history and turn state and - * drives a backend (direct Anthropic loop for phase 1). The UI talks only to + * drives a direct provider backend. The UI talks only to * this surface, so swapping in the external-agent backend later doesn't touch * AssistantLayer. * @@ -21,20 +22,27 @@ const SESSION_IDLE_MS = 10 * 60 * 1000; const MAX_HISTORY_MESSAGES = 40; export class AssistantSession { - private readonly backend = new DirectAnthropicBackend(); - private readonly messages: AnthropicMessage[] = []; + private readonly backend = new DirectAssistantBackend(); + private readonly messages: LlmMessage[] = []; private turnHandle: AssistantTurnHandle | null = null; private lastActivityMs = Date.now(); - // API-safe tool name -> canonical registry name. The Anthropic API only - // allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses. - // get_state") are sanitized for the request and mapped back on tool calls. + // API-safe tool name -> canonical registry name. Provider APIs restrict tool + // names, so dotted registry names are sanitized and mapped back on calls. private readonly toolNameMap = new Map(); constructor( - private readonly apiKey: string, + private readonly llm: ResolvedAssistantModel, private readonly registry: ToolRegistry = toolRegistry, ) {} + matchesConfiguration(llm: ResolvedAssistantModel): boolean { + return ( + this.llm.provider === llm.provider && + this.llm.model === llm.model && + this.llm.apiKey === llm.apiKey + ); + } + /** Whether this session has been idle long enough to retire. */ isExpired(nowMs = Date.now()): boolean { return nowMs - this.lastActivityMs > SESSION_IDLE_MS; @@ -62,9 +70,10 @@ export class AssistantSession { }; this.turnHandle = this.backend.runTurn({ - apiKey: this.apiKey, - model: DEFAULT_LLM_MODEL, - effort: "low", + provider: this.llm.provider, + apiKey: this.llm.apiKey, + model: this.llm.model, + effort: this.llm.effort, system, messages: this.messages, buildTools: () => this.buildToolDefinitions(), @@ -91,7 +100,7 @@ export class AssistantSession { this.lastActivityMs = Date.now(); } - private buildToolDefinitions(): AnthropicToolDefinition[] { + private buildToolDefinitions(): LlmToolDefinition[] { this.toolNameMap.clear(); return this.registry.listTools().map((spec) => { const apiName = this.toApiToolName(spec.name); @@ -100,7 +109,7 @@ export class AssistantSession { }); } - /** Convert a canonical registry name to an Anthropic-legal tool name. */ + /** Convert a canonical registry name to a provider-legal tool name. */ private toApiToolName(name: string): string { let base = name.replace(/[^a-zA-Z0-9_-]/g, "_"); // Disambiguate the rare case where two canonical names sanitize alike. @@ -125,7 +134,7 @@ export class AssistantSession { } } -function startsWithToolResult(message: AnthropicMessage): boolean { +function startsWithToolResult(message: LlmMessage): boolean { return ( Array.isArray(message.content) && message.content.length > 0 && diff --git a/app/assistant/tool-registry.ts b/app/assistant/tool-registry.ts index 5f5494f5..8fd6dade 100644 --- a/app/assistant/tool-registry.ts +++ b/app/assistant/tool-registry.ts @@ -1,7 +1,7 @@ /** * The assistant's tool registry: the single place that knows which tools exist, * whether each is currently callable, and how to invoke it. Both assistant - * backends (direct Anthropic loop, external agent bridge) list and call tools + * backends (direct provider loop, external agent bridge) list and call tools * through here, so gating and timeouts are enforced in one spot rather than * trusted to a model or a remote agent. * diff --git a/app/assistant/types.ts b/app/assistant/types.ts index 10e90551..e310b2ff 100644 --- a/app/assistant/types.ts +++ b/app/assistant/types.ts @@ -1,7 +1,7 @@ /** * Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer) * talks to the same session surface whether the turn runs on the phone - * (direct Anthropic loop) or against the user's own agent (external bridge, + * (direct provider loop) or against the user's own agent (external bridge, * later phases). */ diff --git a/app/native/anthropic.ts b/app/native/anthropic.ts index bb861d3f..725f81fc 100644 --- a/app/native/anthropic.ts +++ b/app/native/anthropic.ts @@ -1,4 +1,12 @@ import { buildRefineUserMessage, REFINE_SYSTEM_PROMPT } from "../prompts"; +import type { + LlmContentBlock, + LlmMessage, + LlmStreamHandle, + LlmStreamOptions, + LlmStreamResult, + LlmToolDefinition, +} from "../assistant/llm-protocol"; declare const com: any; @@ -20,56 +28,20 @@ export const DEFAULT_LLM_MODEL = "claude-sonnet-5"; * still pass a plain string for content; the assistant tool loop uses the * block array to carry tool_use / tool_result alongside text. */ -export type AnthropicContentBlock = - | { type: "text"; text: string } - | { type: "tool_use"; id: string; name: string; input: any } - | { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean }; - -export type AnthropicMessage = { - role: "user" | "assistant"; - content: string | AnthropicContentBlock[]; -}; +export type AnthropicContentBlock = LlmContentBlock; +export type AnthropicMessage = LlmMessage; /** An Anthropic tool definition (input_schema is snake_case as the API wants). */ -export type AnthropicToolDefinition = { - name: string; - description: string; - input_schema: object; -}; - -export type AnthropicStreamResult = { - /** Concatenated text of all text blocks. */ - text: string; - /** The assembled assistant content blocks (text + any tool_use). */ - content: AnthropicContentBlock[]; - /** e.g. "end_turn", "tool_use", "max_tokens", "refusal"; null if never said. */ - stopReason: string | null; -}; - -export type AnthropicStreamOptions = { - apiKey: string; - model?: string; - system?: string; - messages: AnthropicMessage[]; - tools?: AnthropicToolDefinition[]; - maxTokens?: number; - /** Thinking/latency tradeoff; voice flows want "low". Default: model default. */ - effort?: "low" | "medium" | "high"; - /** Called with each incremental text delta as it streams in. */ - onTextDelta?: (delta: string, textSoFar: string) => void; - onDone: (result: AnthropicStreamResult) => void; - /** Called at most once, instead of onDone, with a user-displayable message. */ - onError: (message: string) => void; -}; +export type AnthropicToolDefinition = LlmToolDefinition; +export type AnthropicStreamResult = LlmStreamResult; +export type AnthropicStreamOptions = Omit & { model?: string }; /** Accumulator for one streamed content block before it is finalized. */ type StreamingBlock = | { type: "text"; text: string } | { type: "tool_use"; id: string; name: string; inputJson: string }; -export type AnthropicStreamHandle = { - cancel(): void; -}; +export type AnthropicStreamHandle = LlmStreamHandle; export function streamAnthropicMessage(options: AnthropicStreamOptions): AnthropicStreamHandle { const apiKey = options.apiKey.trim(); diff --git a/app/native/openai.ts b/app/native/openai.ts new file mode 100644 index 00000000..cb178329 --- /dev/null +++ b/app/native/openai.ts @@ -0,0 +1,333 @@ +import type { + LlmContentBlock, + LlmMessage, + LlmStreamHandle, + LlmStreamOptions, +} from "../assistant/llm-protocol"; + +declare const com: any; + +/** + * Minimal streaming client for the OpenAI Responses API. NativeScript cannot + * use the Node SDK, so this shares the okhttp/SSE bridge used by Anthropic. + */ + +const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"; + +type StreamingOutput = + | { type: "text"; text: string } + | { type: "tool_use"; id: string; name: string; inputJson: string; itemId: string; item: any } + | { type: "provider_item"; item: any }; + +export function streamOpenAiResponse(options: LlmStreamOptions): LlmStreamHandle { + const apiKey = options.apiKey.trim(); + let request: any = null; + let settled = false; + let text = ""; + let stopReason: string | null = null; + let refusal = false; + const outputs = new Map(); + + const fail = (message: string) => { + if (settled) return; + settled = true; + options.onError(message); + }; + const done = () => { + if (settled) return; + settled = true; + const content = finalizeOutputs(outputs); + const hasToolUse = content.some((block) => block.type === "tool_use"); + options.onDone({ + text, + content, + stopReason: refusal ? "refusal" : hasToolUse ? "tool_use" : stopReason ?? "end_turn", + }); + }; + + if (!global.isAndroid) { + setTimeout(() => fail("OpenAI API is only wired up on Android"), 0); + return { cancel: () => {} }; + } + if (!apiKey) { + setTimeout(() => fail("No OpenAI API key set"), 0); + return { cancel: () => {} }; + } + + const body: any = { + model: options.model, + input: buildOpenAiInput(options.messages), + stream: true, + store: false, + include: ["reasoning.encrypted_content"], + max_output_tokens: options.maxTokens ?? 8192, + }; + if (options.system) body.instructions = options.system; + if (options.tools?.length) { + body.tools = options.tools.map((tool) => ({ + type: "function", + name: tool.name, + description: tool.description, + parameters: tool.input_schema, + })); + } + if (options.effort) body.reasoning = { effort: options.effort }; + + const listener = new com.faceclaw.app.FaceclawSseListener({ + onLine: (line: string) => { + if (settled) return; + const event = parseSseDataLine(String(line)); + if (!event) return; + const outputIndex = Number(event.output_index); + switch (event.type) { + case "response.output_item.added": { + const item = event.item; + if (item?.type === "function_call") { + outputs.set(outputIndex, { + type: "tool_use", + id: String(item.call_id ?? item.id ?? ""), + name: String(item.name ?? ""), + inputJson: String(item.arguments ?? ""), + itemId: String(item.id ?? ""), + item, + }); + } else if (item?.type === "message") { + outputs.set(outputIndex, { type: "text", text: "" }); + } else if (item?.type === "reasoning") { + outputs.set(outputIndex, { type: "provider_item", item }); + } + return; + } + case "response.output_text.delta": { + const delta = String(event.delta ?? ""); + text += delta; + const output = outputs.get(outputIndex); + if (output?.type === "text") { + output.text += delta; + } else { + outputs.set(outputIndex, { type: "text", text: delta }); + } + options.onTextDelta?.(delta, text); + return; + } + case "response.function_call_arguments.delta": { + const output = findToolOutput(outputs, outputIndex, String(event.item_id ?? "")); + if (output) output.inputJson += String(event.delta ?? ""); + return; + } + case "response.output_item.done": { + const item = event.item; + if (item?.type === "function_call") { + outputs.set(outputIndex, { + type: "tool_use", + id: String(item.call_id ?? item.id ?? ""), + name: String(item.name ?? ""), + inputJson: String(item.arguments ?? ""), + itemId: String(item.id ?? ""), + item, + }); + } else if (item?.type === "message") { + const output = outputs.get(outputIndex); + const completedText = extractMessageText(item); + outputs.set(outputIndex, { + type: "text", + text: completedText || (output?.type === "text" ? output.text : ""), + }); + if (completedText && !text) text = completedText; + } else if (item?.type === "reasoning") { + outputs.set(outputIndex, { type: "provider_item", item }); + } + return; + } + case "response.refusal.delta": + case "response.refusal.done": + refusal = true; + return; + case "response.completed": + done(); + return; + case "response.incomplete": + stopReason = String(event.response?.incomplete_details?.reason ?? "incomplete"); + done(); + return; + case "response.failed": + fail(`OpenAI: ${String(event.response?.error?.message ?? "response failed")}`); + return; + case "error": + fail(`OpenAI: ${String(event.message ?? event.error?.message ?? "stream error")}`); + return; + default: + return; + } + }, + onHttpError: (code: number, errorBody: string) => { + fail(describeHttpError(Number(code), String(errorBody))); + }, + onComplete: () => { + done(); + }, + onFailure: (message: string) => { + fail(`OpenAI connection failed: ${String(message)}`); + }, + }); + + const headers = Array.create("java.lang.String", 2) as string[]; + headers[0] = "Authorization"; + headers[1] = `Bearer ${apiKey}`; + try { + request = new com.faceclaw.app.FaceclawSseRequest( + OPENAI_RESPONSES_URL, + JSON.stringify(body), + headers, + listener, + ); + } catch (error) { + setTimeout(() => fail(`OpenAI request failed: ${String((error as Error)?.message ?? error)}`), 0); + return { cancel: () => {} }; + } + + return { + cancel: () => { + settled = true; + try { + request?.cancel(); + } catch { + // ignore + } + }, + }; +} + +/** Convert neutral conversation history to Responses API input items. */ +function buildOpenAiInput(messages: LlmMessage[]): any[] { + const input: any[] = []; + for (const message of messages) { + if (typeof message.content === "string") { + input.push({ role: message.role, content: message.content }); + continue; + } + + let pendingText = ""; + const flushText = () => { + if (!pendingText) return; + input.push({ role: message.role, content: pendingText }); + pendingText = ""; + }; + for (const block of message.content) { + if (block.type === "text") { + pendingText += block.text; + } else if (block.type === "tool_use") { + flushText(); + if (block.provider_item?.provider === "openai") { + input.push(block.provider_item.item); + } else { + input.push({ + type: "function_call", + call_id: block.id, + name: block.name, + arguments: JSON.stringify(block.input ?? {}), + }); + } + } else if (block.type === "tool_result") { + flushText(); + input.push({ + type: "function_call_output", + call_id: block.tool_use_id, + output: block.is_error ? `Error: ${block.content}` : block.content, + }); + } else if (block.provider === "openai") { + flushText(); + input.push(block.item); + } + } + flushText(); + } + return input; +} + +function finalizeOutputs(outputs: Map): LlmContentBlock[] { + const content: LlmContentBlock[] = []; + for (const index of Array.from(outputs.keys()).sort((a, b) => a - b)) { + const output = outputs.get(index)!; + if (output.type === "text") { + if (output.text) content.push({ type: "text", text: output.text }); + continue; + } + if (output.type === "provider_item") { + content.push({ type: "provider_item", provider: "openai", item: output.item }); + continue; + } + let input: any = {}; + if (output.inputJson.trim()) { + try { + input = JSON.parse(output.inputJson); + } catch { + // Let the tool handler report any required arguments that are missing. + } + } + content.push({ + type: "tool_use", + id: output.id, + name: output.name, + input, + provider_item: { provider: "openai", item: output.item }, + }); + } + return content; +} + +function findToolOutput( + outputs: Map, + outputIndex: number, + itemId: string, +): Extract | null { + const indexed = outputs.get(outputIndex); + if (indexed?.type === "tool_use") return indexed; + for (const output of outputs.values()) { + if (output.type === "tool_use" && output.itemId === itemId) return output; + } + return null; +} + +function extractMessageText(item: any): string { + if (!Array.isArray(item?.content)) return ""; + return item.content + .filter((part: any) => part?.type === "output_text") + .map((part: any) => String(part.text ?? "")) + .join(""); +} + +function parseSseDataLine(line: string): any | null { + if (!line.startsWith("data:")) return null; + const payload = line.slice(5).trim(); + if (!payload || payload === "[DONE]") return null; + try { + return JSON.parse(payload); + } catch { + return null; + } +} + +function describeHttpError(code: number, body: string): string { + let detail = ""; + try { + detail = String(JSON.parse(body)?.error?.message ?? ""); + } catch { + // non-JSON error body + } + switch (code) { + case 401: + return "OpenAI: invalid API key"; + case 403: + return "OpenAI: API key lacks permission"; + case 429: + return "OpenAI: rate limited, try again shortly"; + case 500: + case 502: + case 503: + case 504: + return "OpenAI: service unavailable, try again shortly"; + default: + return `OpenAI: HTTP ${code}${detail ? ` (${detail})` : ""}`; + } +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 30ebedeb..62f8a241 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -13,7 +13,13 @@ import { type UiFontChoice, } from "~/graphics/bdffont"; import { wrapText } from "~/graphics/textwrap"; -import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from "./menu"; +import { + ASSISTANT_MODEL_VALUES, + assistantModelLabel, + assistantModelProvider, + type AssistantModel, +} from "~/assistant/models"; +import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; import { DashboardInputEvent, Layer, type LayerContext } from "./layers"; import { GrayImage } from "~/graphics/image"; @@ -109,15 +115,19 @@ export class ConfigSettingBoolean extends ConfigSet type ConfigSettingEnumOptions = ConfigSettingOptions & { values: readonly TValue[]; normalize?: (value: string | null | undefined) => TValue; + /** Dynamic availability check used by enum picker rows. */ + isDisabled?: (value: TValue) => boolean; }; export class ConfigSettingEnum extends ConfigSetting { readonly values: readonly TValue[]; private readonly normalizer: (value: string | null | undefined) => TValue; + private readonly disabledPredicate: (value: TValue) => boolean; constructor(options: ConfigSettingEnumOptions) { super(options); this.values = options.values; + this.disabledPredicate = options.isDisabled ?? (() => false); if (options.normalize) { this.normalizer = (value: string | null | undefined) => { const normalized = options.normalize(value) as TValue|undefined; @@ -139,9 +149,17 @@ export class ConfigSettingEnum({ + id: "assistant-model", + label: "Assistant model", + storageKey: "assistant.model", + defaultValue: "auto", + values: ASSISTANT_MODEL_VALUES, + formatValue: assistantModelLabel, + isDisabled: (value) => { + const provider = assistantModelProvider(value); + if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0; + if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0; + return false; + }, + description: "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set.", +}); + export const mapboxApiKeySetting = new ConfigSettingString({ id: "mapbox-api-key", label: "Mapbox token", @@ -494,7 +528,6 @@ function maskToken(token: string): string { type SettingsMenuOptions = { - style?: "cycle"|"submenu" onChange?: (ctx: LayerContext, newValue: T, oldValue: T) => void } @@ -502,36 +535,36 @@ export function enumSettingMenuItem, opts?: SettingsMenuOptions ): MenuItem { - const style = opts?.style ?? "cycle"; return { label: setting.label, description: setting.description, - onSelect: (ctx) => { - if (style === "cycle") { - const oldValue = setting.get(); - const newValue = setting.next(oldValue); - setting.set(newValue); - opts?.onChange?.(ctx, newValue, oldValue); - } else { - const submenu = new MenuLayer(setting.label, setting.values.map((value) => ({ - label: setting.displayValue(value), - onSelect: () => { - const oldValue = setting.get(); - setting.set(value); - opts?.onChange?.(ctx, value, oldValue); - ctx.stack.pop(); - }, - render: ({ image, x, y }) => { - const selected = setting.get() === value ? " *" : ""; - image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.displayValue(value)}${selected}`, 200); - } - }))); - ctx.stack.push(submenu); - } - }, - render: ({ image, x, y, width }) => { - drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, setting.label, setting.displayValue(setting.get())); - }, + onSelect: (ctx) => { + const current = setting.get(); + const items = setting.values.map((value): MenuItem => ({ + label: setting.displayValue(value), + disabled: () => setting.isDisabled(value), + onSelect: () => { + const oldValue = setting.get(); + setting.set(value); + opts?.onChange?.(ctx, value, oldValue); + ctx.stack.pop(); + }, + render: ({ image, x, y, disabled }) => { + const selected = setting.get() === value ? " *" : ""; + image.drawText( + getDefaultSmallFont(), + x, + y + 3, + `${setting.displayValue(value)}${selected}`, + disabled ? 70 : 200, + ); + }, + })); + openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current))); + }, + render: ({ image, x, y, width }) => { + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, setting.label, setting.displayValue(setting.get())); + }, }; } diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 85ada3cd..7ce6afcd 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -4,6 +4,7 @@ import { getDashboardLogo } from "../../graphics/logo"; import { shell } from "../shell/shell"; import { anthropicApiKeySetting, + assistantModelSetting, assistantSkipConfirmationSetting, batteryDisplayModeSetting, brightnessSetting, @@ -40,7 +41,7 @@ function settingsSections(): SettingsSection[] { items: [ // Auto (ambient sensor) or an exact level; pushed to the glasses by // the dashboard controller when changed and on each connect. - enumSettingMenuItem(brightnessSetting, { style: "submenu" }), + enumSettingMenuItem(brightnessSetting), enumSettingMenuItem(screenTimeoutSetting, { onChange: () => { shell.noteUserActivity(); @@ -59,6 +60,7 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), + enumSettingMenuItem(assistantModelSetting), // When on, a wakeword utterance goes straight to the assistant with no // Send/Type menu step. toggleSettingMenuItem(assistantSkipConfirmationSetting), diff --git a/app/ui/dashboard/settings-panel.ts b/app/ui/dashboard/settings-panel.ts index 93f72743..9744c36b 100644 --- a/app/ui/dashboard/settings-panel.ts +++ b/app/ui/dashboard/settings-panel.ts @@ -4,7 +4,7 @@ import { wrapText } from "../../graphics/textwrap"; import { clamp } from "../../util/numeric-util"; import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../gestures"; import { DashboardInputEvent, Layer, LayerContext } from "../layers"; -import { drawSelectionHighlight, MenuItem, MenuLayer, type MenuLayout } from "../menu"; +import { drawSelectionHighlight, isMenuItemDisabled, MenuItem, MenuLayer, openModalMenu } from "../menu"; import { shell } from "../shell/shell"; /** @@ -114,13 +114,24 @@ export class SettingsPanelLayer implements Layer { // A selection only appears once focus is in the right column; before // that the pane is a preview of what tapping would open. const selected = this.focus === "right" && i === this.rightIndex; + const disabled = isMenuItemDisabled(item); if (selected) { drawSelectionHighlight(image, rightX - 2, rowY, rightW + 2, ROW_H - 2, appFocused, 6); } if (item.render) { - item.render({ image, x: rightX + 8, y: rowY, width: rightW - 16, height: ROW_H - 3, selected, text: item.label, ctx }); + item.render({ + image, + x: rightX + 8, + y: rowY, + width: rightW - 16, + height: ROW_H - 3, + selected, + disabled, + text: item.label, + ctx, + }); } else { - image.drawText(font, rightX + 8, rowY + 4, item.label, selected ? 255 : 200); + image.drawText(font, rightX + 8, rowY + 4, item.label, disabled ? 70 : selected ? 255 : 200); } } if (descriptionLines.length) { @@ -174,7 +185,10 @@ export class SettingsPanelLayer implements Layer { if (items.length) this.rightIndex = (this.rightIndex + 1) % items.length; return; case "click": - if (items.length) await items[clamp(this.rightIndex, 0, items.length - 1)]!.onSelect(ctx, NO_MENU); + if (items.length) { + const item = items[clamp(this.rightIndex, 0, items.length - 1)]!; + if (!isMenuItemDisabled(item)) await item.onSelect(ctx, NO_MENU); + } return; case "double-click": this.focus = "left"; @@ -195,17 +209,7 @@ export class SettingsPanelLayer implements Layer { * MenuLayer (which draws the border and self-closes on double-click). */ export function openSettingsSubMenu(ctx: LayerContext, title: string, items: MenuItem[]): void { - const { width, height } = ctx.stack.getBaseSize(); - const menuW = Math.min(320, width - 40); - const menuH = 16 + PAD + items.length * ROW_H + PAD; - const layout: MenuLayout = { - x: ((width - menuW) / 2) | 0, - y: ((height - menuH) / 2) | 0, - width: menuW, - minHeight: menuH, - maxHeight: menuH, - }; - ctx.stack.push(new MenuLayer(title, items, layout)); + openModalMenu(ctx, title, items); } function clampScroll(selected: number, scroll: number, visible: number, count: number): number { diff --git a/app/ui/menu.ts b/app/ui/menu.ts index b442d504..4d39f29d 100644 --- a/app/ui/menu.ts +++ b/app/ui/menu.ts @@ -37,6 +37,7 @@ export type MenuItemRenderArgs = { width: number; height: number; selected: boolean; + disabled: boolean; text: string; ctx: LayerContext; }; @@ -45,6 +46,8 @@ export type MenuItem = { label: string; /** Extended description; the Settings panel shows it below the list while the item is selected. */ description?: string; + /** Disabled rows stay visible but render dimly and ignore clicks. */ + disabled?: boolean | (() => boolean); onSelect: (ctx: LayerContext, menu: MenuLayer) => Promise | void; render?: (args: MenuItemRenderArgs) => void; }; @@ -181,6 +184,12 @@ export class MenuLayer implements Layer { public readonly paintOverBase = false, ) {} + /** Start a newly opened picker on its current value. */ + selectItem(index: number): this { + this.selectedIndex = clamp(index, 0, Math.max(0, this.items.length - 1)); + return this; + } + paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { const font = getDefaultSmallFont(); const { x, y, width } = this.layout; @@ -216,6 +225,7 @@ export class MenuLayer implements Layer { const item = this.items[index]!; const rowY = bodyY + (index - this.scrollRow) * MENU_ROW_HEIGHT; const selected = index === this.selectedIndex; + const disabled = isMenuItemDisabled(item); if (selected) { drawSelectionHighlight( image, @@ -235,11 +245,12 @@ export class MenuLayer implements Layer { width: width - 44, height: MENU_ROW_HEIGHT - 3, selected, + disabled, text: item.label, ctx, }); } else { - image.drawText(font, x + 22, rowY + 3, item.label, selected ? 255 : 200); + image.drawText(font, x + 22, rowY + 3, item.label, disabled ? 70 : selected ? 255 : 200); } } @@ -276,7 +287,9 @@ export class MenuLayer implements Layer { ctx.stack.pop(); return; case "click": - await this.items[this.selectedIndex]!.onSelect(ctx, this); + if (!isMenuItemDisabled(this.items[this.selectedIndex]!)) { + await this.items[this.selectedIndex]!.onSelect(ctx, this); + } return; default: return; @@ -284,6 +297,31 @@ export class MenuLayer implements Layer { } } +/** Open a centered, bordered menu over the current layer. */ +export function openModalMenu( + ctx: LayerContext, + title: string, + items: MenuItem[], + initialSelectedIndex = 0, +): void { + const { width, height } = ctx.stack.getBaseSize(); + const menuWidth = Math.min(320, width - 40); + const naturalHeight = MENU_TITLE_HEIGHT + 2 * MENU_BODY_PADDING + items.length * MENU_ROW_HEIGHT; + const menuHeight = Math.min(naturalHeight, height - 40); + const layout: MenuLayout = { + x: ((width - menuWidth) / 2) | 0, + y: ((height - menuHeight) / 2) | 0, + width: menuWidth, + minHeight: menuHeight, + maxHeight: menuHeight, + }; + ctx.stack.push(new MenuLayer(title, items, layout).selectItem(initialSelectedIndex)); +} + +export function isMenuItemDisabled(item: MenuItem): boolean { + return typeof item.disabled === "function" ? item.disabled() : item.disabled === true; +} + export class TextPageLayer implements Layer { constructor( private readonly title: string, diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index c3e9de36..4ac79696 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -7,13 +7,16 @@ import { MenuLayer, type MenuItem } from "../menu"; import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input"; import { AssistantLayer } from "../apps/assistant"; import { AssistantSession } from "../../assistant/session"; +import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models"; import type { AssistantContext } from "../../assistant/types"; import { SingleNotificationLayer } from "../notifications"; import { anthropicApiKeySetting, + assistantModelSetting, assistantSkipConfirmationSetting, batteryDisplayModeSetting, onAnySettingChanged, + openAiApiKeySetting, timeFormatSetting, wakeWordActionSetting, } from "../dashboard-settings"; @@ -673,18 +676,30 @@ class Shell { } private isAssistantAvailable(): boolean { - return anthropicApiKeySetting.get().trim().length > 0; + return this.resolveAssistantConfiguration() !== null; } private ensureAssistantSession(): AssistantSession | null { - const apiKey = anthropicApiKeySetting.get().trim(); - if (!apiKey) return null; - if (!this.assistantSession || this.assistantSession.isExpired()) { - this.assistantSession = new AssistantSession(apiKey); + const llm = this.resolveAssistantConfiguration(); + if (!llm) return null; + if ( + !this.assistantSession || + this.assistantSession.isExpired() || + !this.assistantSession.matchesConfiguration(llm) + ) { + this.assistantSession?.cancel(); + this.assistantSession = new AssistantSession(llm); } return this.assistantSession; } + private resolveAssistantConfiguration(): ResolvedAssistantModel | null { + return resolveAssistantModel(assistantModelSetting.get(), { + anthropic: anthropicApiKeySetting.get(), + openai: openAiApiKeySetting.get(), + }); + } + private buildAssistantContext(): AssistantContext { const foreground = this.getForegroundApp(); return { @@ -704,7 +719,7 @@ class Shell { sendToAssistant(text: string): void { const session = this.ensureAssistantSession(); if (!session) { - this.showAlert("Set an Anthropic API key in Settings to use the assistant."); + this.showAlert("Set an API key for the selected assistant model in Settings."); return; } if (!this.screenOn) this.wake("sidebar");