diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index 753a89cf..a28b095a 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -1,4 +1,5 @@ import { streamAnthropicMessage } from "../native/anthropic"; +import { streamHermesChat } from "../native/hermes"; import { streamLocalQwen } from "../native/llama"; import { streamOpenAiResponse } from "../native/openai"; import type { @@ -30,6 +31,9 @@ export type DirectTurnOptions = { apiKey: string; model: string; effort?: "low" | "medium" | "high"; + /** Hermes: the API root, and the server-side session this conversation maps to. */ + baseUrl?: string; + sessionId?: string; system: string; /** Owned by the session; this loop appends assistant + tool_result messages. */ messages: LlmMessage[]; @@ -79,7 +83,8 @@ export class DirectAssistantBackend { system: options.system, effort: options.effort, messages: options.messages, - tools: options.buildTools(), + // Hermes runs its own tools; Faceclaw's are not offered to it. + tools: options.provider === "hermes" ? [] : options.buildTools(), onTextDelta: (delta, textSoFar) => { if (cancelled) return; options.callbacks.onTextDelta(delta, priorText + textSoFar); @@ -128,7 +133,9 @@ export class DirectAssistantBackend { }, }; streamHandle = - options.provider === "openai" + options.provider === "hermes" + ? streamHermesChat({ ...streamOptions, baseUrl: options.baseUrl ?? "", sessionId: options.sessionId ?? "" }) + : options.provider === "openai" ? streamOpenAiResponse(streamOptions) : options.provider === "local" ? streamLocalQwen(streamOptions) diff --git a/app/assistant/models.ts b/app/assistant/models.ts index 66d67670..567de0ad 100644 --- a/app/assistant/models.ts +++ b/app/assistant/models.ts @@ -10,6 +10,7 @@ export const ASSISTANT_MODEL_VALUES = [ "terra", "sol", "qwen", + "hermes", ] as const; export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number]; @@ -20,11 +21,14 @@ export function supportedAssistantModel(value: AssistantModel): AssistantModel { return global.isIOS && value === "qwen" ? "auto" : value; } -export type AssistantProvider = "anthropic" | "openai" | "local"; +export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes"; export type AssistantApiKeys = { anthropic: string; openai: string; + /** A Hermes Agent API server; only the voice assistant passes these. */ + hermesUrl?: string; + hermesKey?: string; }; export type ResolvedAssistantModel = { @@ -33,6 +37,8 @@ export type ResolvedAssistantModel = { model: string; apiKey: string; effort?: "low" | "medium" | "high"; + /** API root for providers that are not fixed services (Hermes). */ + baseUrl?: string; }; type ModelDefinition = { @@ -55,6 +61,8 @@ const MODEL_DEFINITIONS: Record, ModelDefinition // Runs on the phone itself (llama.cpp); needs the downloaded model file, // not an API key. The default for users who haven't set any key. qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id }, + // Your own Hermes Agent (its API server), with its own tools and memory. + hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" }, }; export function assistantModelLabel(value: AssistantModel): string { @@ -93,6 +101,11 @@ export function resolveAssistantModel( } const definition = MODEL_DEFINITIONS[resolvedSelection as Exclude]; + if (definition.provider === "hermes") { + const baseUrl = keys.hermesUrl?.trim() ?? ""; + if (!baseUrl) return null; + return { selection, provider: "hermes", model: definition.model, apiKey: keys.hermesKey?.trim() ?? "", baseUrl }; + } if (definition.provider === "local") { if (!isLocalModelReady()) return null; return { diff --git a/app/assistant/session.ts b/app/assistant/session.ts index 69a915fc..3065f4f5 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -32,6 +32,11 @@ export type AssistantBackendConfig = /** Trim history from the head once it grows past this many messages. */ const MAX_HISTORY_MESSAGES = 40; + +/** Hermes brings its own persona and tools; it only needs to know where the reply is shown. */ +const HERMES_SYSTEM_PROMPT = + "The user is talking to you through smart glasses with a small text display. Reply briefly in plain text, without Markdown."; + export class AssistantSession { private readonly directBackend = new DirectAssistantBackend(); private readonly messages: LlmMessage[] = []; @@ -43,6 +48,8 @@ export class AssistantSession { // API-safe tool name -> canonical registry name. Provider APIs restrict tool // names, so dotted registry names are sanitized and mapped back on calls. private readonly toolNameMap = new Map(); + // The Hermes server-side session for this conversation (Hermes keeps the history). + private readonly hermesSessionId = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; constructor( private config: AssistantBackendConfig, @@ -173,7 +180,10 @@ export class AssistantSession { apiKey: llm.apiKey, model: llm.model, effort: llm.effort, - system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), + baseUrl: llm.baseUrl, + sessionId: this.hermesSessionId, + system: llm.provider === "hermes" ? HERMES_SYSTEM_PROMPT + : isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), messages: turnMessages, buildTools: () => this.buildToolDefinitions(), registry: this.registry, diff --git a/app/native/hermes.ts b/app/native/hermes.ts new file mode 100644 index 00000000..dc45f834 --- /dev/null +++ b/app/native/hermes.ts @@ -0,0 +1,121 @@ +import type { LlmStreamHandle, LlmStreamOptions } from "../assistant/llm-protocol"; +import { openSseRequest, type SseListener } from "./sse"; + +/** + * Streaming client for a Hermes Agent gateway's OpenAI-compatible API server + * (POST {baseUrl}/chat/completions). Hermes is an agent with its own tools and + * memory, so only the latest user message is sent: the X-Hermes-Session-Id header + * keeps the conversation on the server. Faceclaw's own tools are not offered. + */ + +export type HermesStreamOptions = LlmStreamOptions & { + baseUrl: string; + sessionId: string; +}; + +/** The API root from what a user typed: adds /v1 to a bare host. Empty when unusable. */ +export function normalizeHermesUrl(input: string): string { + let url = input.trim().replace(/\/+$/, ""); + if (!/^https?:\/\/[^/\s]+/i.test(url)) return ""; + url = url.replace(/\/chat\/completions$/i, ""); + if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1"; + return url; +} + +export function streamHermesChat(options: HermesStreamOptions): LlmStreamHandle { + let request: { cancel(): void } | null = null; + let settled = false; + let text = ""; + let finishReason: string | null = null; + + const fail = (message: string) => { + if (settled) return; + settled = true; + request?.cancel(); + options.onError(message); + }; + const done = () => { + if (settled) return; + settled = true; + request?.cancel(); + options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: finishReason === "length" ? "max_tokens" : "end_turn" }); + }; + + const baseUrl = normalizeHermesUrl(options.baseUrl); + const last = [...options.messages].reverse().find((message) => message.role === "user"); + const prompt = typeof last?.content === "string" ? last.content : ""; + if (!baseUrl || !prompt) { + setTimeout(() => fail(baseUrl ? "Nothing to send to Hermes" : "No Hermes URL set"), 0); + return { cancel: () => { settled = true; } }; + } + + const messages = [ + ...(options.system ? [{ role: "system", content: options.system }] : []), + { role: "user", content: prompt }, + ]; + const body = JSON.stringify({ model: options.model, stream: true, messages }); + const headers: Record = { "Content-Type": "application/json", "X-Hermes-Session-Id": options.sessionId }; + if (options.apiKey.trim()) headers.Authorization = `Bearer ${options.apiKey.trim()}`; + + const listener: SseListener = { + onLine: (line: string) => { + if (settled || !line.startsWith("data:")) return; + const payload = line.slice(5).trim(); + if (payload === "[DONE]") { + done(); + return; + } + let event: any; + try { + event = JSON.parse(payload); + } catch { + return; + } + if (event?.error) { + fail(`Hermes: ${String(event.error.message ?? event.error)}`); + return; + } + const choice = event?.choices?.[0]; + // reasoning_content is the agent's thinking; only the answer goes to the glasses. + // Hermes opens its answer with blank lines after the reasoning; drop them. + const delta = text ? choice?.delta?.content : String(choice?.delta?.content ?? "").trimStart(); + if (typeof delta === "string" && delta) { + text += delta; + options.onTextDelta?.(delta, text); + } + if (choice?.finish_reason) finishReason = String(choice.finish_reason); + }, + onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))), + // Some servers close without [DONE] after the final chunk. + onComplete: () => (finishReason || text ? done() : fail("Hermes stream ended without a reply")), + onFailure: (message: string) => fail(`Hermes connection failed: ${String(message)}`), + }; + + try { + request = openSseRequest(`${baseUrl}/chat/completions`, body, headers, listener); + } catch (error) { + setTimeout(() => fail(`Hermes request failed: ${String((error as Error)?.message ?? error)}`), 0); + return { cancel: () => { settled = true; } }; + } + return { + cancel: () => { + settled = true; + try { + request?.cancel(); + } catch { + // ignore + } + }, + }; +} + +function describeHttpError(code: number, body: string): string { + if (code === 401 || code === 403) return "Hermes: the API server rejected the key"; + let detail = ""; + try { + detail = String(JSON.parse(body)?.error?.message ?? ""); + } catch { + // non-JSON error body + } + return `Hermes: HTTP ${code}${detail ? ` (${detail})` : ""}`; +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 083291fe..98017d20 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -894,6 +894,28 @@ export const assistantBackendSetting = new ConfigSettingEnum API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.", }); +export const assistantHermesUrlSetting = new ConfigSettingString({ + id: "assistant-hermes-url", + label: "Hermes URL", + storageKey: "assistant.hermesUrl", + defaultValue: "", + editorTitle: "Hermes API server (e.g. http://host:8642/v1)", + glassesEditTitle: "Edit Hermes URL", + formatValue: (value) => value || "(not set)", + description: "Address of a Hermes Agent gateway's API server, used when Hermes agent is the assistant model.", +}); + +export const assistantHermesKeySetting = new ConfigSettingString({ + id: "assistant-hermes-key", + label: "Hermes key", + storageKey: "assistant.hermesKey", + defaultValue: "", + editorTitle: "Hermes API server key", + glassesEditTitle: "Edit Hermes key", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), + description: "The API server's key (API_SERVER_KEY in Hermes's .env).", +}); + export const assistantBridgeHostSetting = new ConfigSettingString({ id: "assistant-bridge-host", label: "Bridge host", @@ -991,6 +1013,7 @@ export const assistantModelSetting = new ConfigSettingEnum({ if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0; if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0; if (provider === "local") return !isLocalModelReady(); + if (provider === "hermes") return assistantHermesUrlSetting.get().trim().length === 0; return false; }, description: diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 92833593..eecfba16 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -36,6 +36,8 @@ import { uiDepthSetting, assistantAllowProactiveSetting, assistantBackendSetting, + assistantHermesKeySetting, + assistantHermesUrlSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, @@ -134,6 +136,8 @@ function settingsSections(): SettingsSection[] { // iOS offers cloud APIs; Android also supports local models and the bridge. enumSettingMenuItem(assistantBackendSetting), enumSettingMenuItem(assistantModelSetting), + textSettingMenuItem(assistantHermesUrlSetting), + textSettingMenuItem(assistantHermesKeySetting), ...(!global.isIOS ? [localModelMenuItem()] : []), toggleSettingMenuItem(assistantSkipConfirmationSetting), // iOS has no local LLM or agent bridge yet. diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 001adcbb..59ad906b 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -31,6 +31,8 @@ import type { AssistantContext } from "../../assistant/types"; import { SingleNotificationLayer } from "../notifications"; import { anthropicApiKeySetting, + assistantHermesKeySetting, + assistantHermesUrlSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, @@ -1643,6 +1645,8 @@ class Shell { const llm = resolveAssistantModel(model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(), + hermesUrl: assistantHermesUrlSetting.get(), + hermesKey: assistantHermesKeySetting.get(), }); if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning; return llm ? { kind: "direct", llm } : null; diff --git a/notes/hermes-assistant.md b/notes/hermes-assistant.md new file mode 100644 index 00000000..91eff4cd --- /dev/null +++ b/notes/hermes-assistant.md @@ -0,0 +1,34 @@ +# Hermes agent as the assistant + +The "Hermes agent" assistant model sends what you say to a +[Hermes Agent](https://github.com/NousResearch/hermes-agent) gateway through its +OpenAI-compatible API server and streams the answer back to the glasses. Hermes +runs its own tools and keeps the conversation: each Faceclaw assistant +conversation maps to one Hermes session (`X-Hermes-Session-Id`), and only the +latest message is sent. Faceclaw's own glasses tools are not offered to Hermes. + +## Enable the API server in Hermes + +In the Hermes config, enable the `api_server` platform and bind it where the +phone can reach it, then set a key in Hermes's `.env` and restart the gateway: + + hermes config set platforms.api_server.enabled true + hermes config set platforms.api_server.extra.host
+ hermes config set platforms.api_server.extra.port 8642 + echo "API_SERVER_KEY=$(openssl rand -hex 24)" >> ~/.hermes/.env + +Check it with `curl -H "Authorization: Bearer " http://:8642/v1/models`. + +## Settings (Settings > Assistant) + +- **Assistant model**: Hermes agent. Disabled until the URL is set. +- **Hermes URL**: e.g. `http://host:8642/v1` (a bare host gets `/v1` added). +- **Hermes key**: the `API_SERVER_KEY` value. + +On iOS, App Transport Security blocks plain `http://` to IP addresses unless +`App_Resources/iOS/Info.plist` has an `NSExceptionDomains` entry for that host +(`NSExceptionAllowsInsecureHTTPLoads`); otherwise serve the API over https, for +example with `tailscale serve`. + +Actions that Hermes holds for approval (`approvals.mode manual`) cannot be +approved from the glasses; approve them from Hermes itself. diff --git a/tests/hermes-assistant.test.cjs b/tests/hermes-assistant.test.cjs new file mode 100644 index 00000000..114e000a --- /dev/null +++ b/tests/hermes-assistant.test.cjs @@ -0,0 +1,77 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +function hermes() { + const requests = []; + const load = loader({ setTimeout, clearTimeout, JSON, Object, String, Number, Error }, { + './sse': { openSseRequest: (url, body, headers, listener) => { + const request = { url, body: JSON.parse(body), headers, listener, cancelled: false, cancel() { this.cancelled = true; } }; + requests.push(request); + return request; + } }, + }); + return { ...load('app/native/hermes.ts'), requests }; +} + +function stream(api, extra = {}) { + const deltas = [], results = [], errors = []; + api.streamHermesChat({ + apiKey: 'k', model: 'hermes-agent', system: 'glasses', baseUrl: 'http://hermes-host:8642', sessionId: 'faceclaw-1', + messages: [{ role: 'user', content: 'Remember kiwi.' }, { role: 'assistant', content: 'OK' }, { role: 'user', content: 'Which word?' }], + onTextDelta: (delta) => deltas.push(delta), onDone: (r) => results.push(r), onError: (e) => errors.push(e), ...extra, + }); + return { deltas, results, errors, request: api.requests.at(-1) }; +} + +const chunk = (delta, finish = null) => `data: ${JSON.stringify({ choices: [{ index: 0, delta, finish_reason: finish }] })}`; + +test('Hermes gets only the latest user message, the session header and the key', () => { + const api = hermes(); + const { request } = stream(api); + assert.equal(request.url, 'http://hermes-host:8642/v1/chat/completions'); + assert.equal(request.headers['X-Hermes-Session-Id'], 'faceclaw-1'); + assert.equal(request.headers.Authorization, 'Bearer k'); + assert.equal(request.body.stream, true); + assert.deepEqual(request.body.messages.map((m) => [m.role, m.content]), [['system', 'glasses'], ['user', 'Which word?']]); +}); + +test('answer deltas stream, reasoning is not shown, [DONE] finishes the turn', () => { + const api = hermes(); + const s = stream(api); + s.request.listener.onLine(chunk({ role: 'assistant' })); + s.request.listener.onLine(chunk({ reasoning_content: 'The word was kiwi.' })); + s.request.listener.onLine(chunk({ content: '\n\n' })); + s.request.listener.onLine(chunk({ content: 'Ki' })); + s.request.listener.onLine(chunk({ content: 'wi' }, 'stop')); + s.request.listener.onLine('data: [DONE]'); + assert.deepEqual(s.deltas, ['Ki', 'wi']); + assert.deepEqual(s.results.map((r) => [r.text, r.stopReason]), [['Kiwi', 'end_turn']]); + assert.deepEqual(s.errors, []); +}); + +test('a rejected key and an empty stream are errors', () => { + const api = hermes(); + const rejected = stream(api); + rejected.request.listener.onHttpError(401, '{}'); + assert.deepEqual(rejected.errors, ['Hermes: the API server rejected the key']); + const empty = stream(api); + empty.request.listener.onComplete(); + assert.deepEqual(empty.errors, ['Hermes stream ended without a reply']); +}); + +test('URL normalization adds /v1 to a bare host and drops the endpoint path', () => { + const { normalizeHermesUrl } = hermes(); + assert.equal(normalizeHermesUrl('http://192.0.2.10:8642'), 'http://192.0.2.10:8642/v1'); + assert.equal(normalizeHermesUrl('http://h:8642/v1/chat/completions'), 'http://h:8642/v1'); + assert.equal(normalizeHermesUrl('h:8642'), ''); +}); + +test('the Hermes model resolves only with a URL, carrying the key and base URL', () => { + const load = loader({ global: { isIOS: true } }, { '../native/llama': { isLocalModelReady: () => false, LOCAL_MODEL: { id: 'q', label: 'Q' } } }); + const { resolveAssistantModel } = load('app/assistant/models.ts'); + assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null); + const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' }); + assert.deepEqual({ ...resolved }, { selection: 'hermes', provider: 'hermes', model: 'hermes-agent', apiKey: 'k', baseUrl: 'http://hermes-host:8642' }); + assert.equal(resolveAssistantModel('auto', { anthropic: '', openai: '', hermesUrl: 'http://hermes-host:8642' }), null, 'Auto never picks Hermes'); +});