From 4973fd2aea12269b8a9dc438dc58c54670b1f8be Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sat, 31 Oct 2026 17:48:45 -0700 Subject: [PATCH] Support Apple Intelligence as an on-device AI model for the voice assistant/etc --- App_Resources/iOS/build.xcconfig | 4 +- .../iOS/src/FaceclawSystemLanguageModel.swift | 694 ++++++++++++++++++ CHANGELOG | 1 + app/apps/calculator/calculator-llm.ts | 4 +- app/apps/microphones/conversation-qa.ts | 4 +- app/apps/microphones/name-verify.ts | 4 +- app/apps/microphones/speaker-insights.ts | 4 +- app/apps/translate/translate-llm.ts | 4 +- app/assistant/direct-backend.ts | 4 +- app/assistant/models.ts | 20 +- app/native/llama.ts | 10 +- app/native/local-llm-types.ts | 13 + app/native/local-llm.ios.ts | 383 ++++++++++ app/native/local-llm.ts | 28 + app/ui/dashboard-settings.ts | 8 +- app/ui/dashboard/settings-menus.ts | 31 +- scripts/apple-llm-harness/extract-tools.cjs | 52 ++ scripts/apple-llm-harness/main.swift | 50 ++ scripts/apple-llm-harness/run.sh | 13 + tests/hermes-assistant.test.cjs | 4 +- tests/ios-assistant.test.cjs | 8 +- tests/ios-local-llm.test.cjs | 248 +++++++ 22 files changed, 1551 insertions(+), 40 deletions(-) create mode 100644 App_Resources/iOS/src/FaceclawSystemLanguageModel.swift create mode 100644 app/native/local-llm-types.ts create mode 100644 app/native/local-llm.ios.ts create mode 100644 app/native/local-llm.ts create mode 100644 scripts/apple-llm-harness/extract-tools.cjs create mode 100644 scripts/apple-llm-harness/main.swift create mode 100755 scripts/apple-llm-harness/run.sh create mode 100644 tests/ios-local-llm.test.cjs diff --git a/App_Resources/iOS/build.xcconfig b/App_Resources/iOS/build.xcconfig index 13dda0ff..331a8be9 100644 --- a/App_Resources/iOS/build.xcconfig +++ b/App_Resources/iOS/build.xcconfig @@ -3,7 +3,9 @@ TARGETED_DEVICE_FAMILY = 1; CODE_SIGN_STYLE = Automatic; // Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team. -OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security +// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+, +// so it is weak-linked: the app still launches on older iOS and reports the model unavailable. +OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; diff --git a/App_Resources/iOS/src/FaceclawSystemLanguageModel.swift b/App_Resources/iOS/src/FaceclawSystemLanguageModel.swift new file mode 100644 index 00000000..ed9d3778 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawSystemLanguageModel.swift @@ -0,0 +1,694 @@ +import Foundation +#if canImport(FoundationModels) +import FoundationModels +#endif + +/// The on-phone LLM on iOS: Apple's Foundation Models framework (the Apple +/// Intelligence system model, iOS 26+, eligible hardware). It runs out of +/// process on the Neural Engine, so it costs the app no model download or +/// memory, and unlike Metal work it is allowed (within a system budget) while +/// Faceclaw is in the background driving the glasses. +/// +/// JavaScript (app/native/local-llm.ios.ts) drives the same provider-neutral +/// agent loop as the cloud models: each start() is one loop iteration. The +/// request is JSON: +/// +/// {"system":String, +/// "tools":[{"name","description","parameters":}], +/// "history":[{"type":"prompt"|"response","text"} | +/// {"type":"toolCalls","calls":[{"id","name","arguments":}]} | +/// {"type":"toolOutput","id","name","text"}], +/// "prompt":String, "maxTokens":Int, "permissive":Bool} +/// +/// and events arrive on the main queue through eventHandler: +/// +/// {"kind":"delta","text":String} streamed reply text +/// {"kind":"toolCalls","calls":[{"id","name","arguments":}]} +/// {"kind":"done"} +/// {"kind":"error","code":String,"message":String} +/// {"kind":"log","message":String} +/// +/// The framework runs tools itself, mid-response. When the model calls one, +/// the call is reported as "toolCalls" and the generation stays suspended +/// inside the tool until resolveToolCalls() hands back the results that +/// JavaScript's tool loop produced; the reply then continues on the same +/// instance. +/// +/// The model's context window is small (4096 tokens on iOS 26, shared by input +/// and reply) and the framework spends ~50 tokens on each tool before its +/// parameters, so the assistant's full tool list (30+ tools, ~5000 tokens) can't +/// be offered at once. When the tools don't fit their share of the window, a +/// quick schema-constrained routing request first asks the model which few +/// tools the request needs. History is then trimmed to fit before each start(). +/// Called from JavaScript, so on the main thread. +@MainActor +@objc(FaceclawSystemLanguageModel) +@objcMembers +public class FaceclawSystemLanguageModel: NSObject { + public var eventHandler: ((String) -> Void)? + // A LanguageModelEngine, typed loosely so the class itself loads before iOS 26. + private var engine: AnyObject? + + /// "available", or why not: "unsupportedOS", "deviceNotEligible", + /// "appleIntelligenceNotEnabled", "modelNotReady" (still downloading), "unavailable". + /// Nonisolated: worker isolates load the model-choice list too, off the main thread. + nonisolated public static func availability() -> String { + #if canImport(FoundationModels) + if #available(iOS 26.0, *) { + switch SystemLanguageModel.default.availability { + case .available: + return "available" + case .unavailable(let reason): + switch reason { + case .deviceNotEligible: return "deviceNotEligible" + case .appleIntelligenceNotEnabled: return "appleIntelligenceNotEnabled" + case .modelNotReady: return "modelNotReady" + @unknown default: return "unavailable" + } + } + } + #endif + return "unsupportedOS" + } + + /// The model's context window in tokens (input and output share it), or 0 when unavailable. + nonisolated public static func contextSize() -> Int { + #if canImport(FoundationModels) + if #available(iOS 26.0, *) { return SystemLanguageModel.default.contextSize } + #endif + return 0 + } + + public func start(_ requestJson: String) { + cancel() + #if canImport(FoundationModels) + if #available(iOS 26.0, *) { + guard let data = requestJson.data(using: .utf8), + let request = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else { + emit(["kind": "error", "code": "other", "message": "Invalid on-device model request"]) + return + } + let engine = LanguageModelEngine { [weak self] event in self?.emit(event) } + self.engine = engine + engine.start(request) + return + } + #endif + emit(["kind": "error", "code": "unavailable", "message": "The on-device model needs iOS 26 or later."]) + } + + /// `[{"id","content","isError"}]` for the calls of the last "toolCalls" event. + public func resolveToolCalls(_ resultsJson: String) { + #if canImport(FoundationModels) + if #available(iOS 26.0, *) { + let data = resultsJson.data(using: .utf8) ?? Data() + let results = ((try? JSONSerialization.jsonObject(with: data)) as? [[String: Any]]) ?? [] + (engine as? LanguageModelEngine)?.resolve(results) + } + #endif + } + + /// Abandon the generation (and any suspended tool calls) without further events. + public func cancel() { + #if canImport(FoundationModels) + if #available(iOS 26.0, *) { (engine as? LanguageModelEngine)?.cancel() } + #endif + engine = nil + } + + private func emit(_ event: [String: Any]) { + // JSONSerialization raises (not throws) on NaN and other invalid values. + guard JSONSerialization.isValidJSONObject(event), + let data = try? JSONSerialization.data(withJSONObject: event), + let json = String(data: data, encoding: .utf8) else { return } + eventHandler?(json) + } +} + +#if canImport(FoundationModels) + +/// A JavaScript-side tool: the framework calls it mid-response, and the +/// engine parks the call until JavaScript resolves it. +@available(iOS 26.0, *) +private struct BridgedTool: Tool { + typealias Arguments = GeneratedContent + typealias Output = String + + let name: String + let description: String + let parameters: GenerationSchema + let handler: @Sendable (String, String) async throws -> String + + func call(arguments: GeneratedContent) async throws -> String { + try await handler(name, arguments.jsonString) + } +} + +/// One tool from the request, convertible at two levels of verbosity. +@available(iOS 26.0, *) +private struct ToolSpec { + let name: String + let description: String + let parameters: [String: Any] +} + +@available(iOS 26.0, *) +@MainActor +final class LanguageModelEngine { + private struct PendingCall { + let id: String + let name: String + let arguments: String + let continuation: CheckedContinuation + } + + /// How long a call waits for any parallel calls before they're reported together. + private static let toolBatchDelayNs: UInt64 = 30_000_000 + /// Tokens kept free beyond the reply budget: the framework's own framing and rounding. + private static let contextMargin = 64 + /// Share of the context window tool definitions may take before tools are routed. + private static let toolBudgetFraction = 0.4 + /// Most tools a routing request picks. + private static let maxRoutedTools = 5 + /// Tools used in the latest exchanges stay offered, for follow-ups ("make it ten minutes"). + private static let keptRecentTools = 2 + + private let emit: ([String: Any]) -> Void + private var task: Task? + // Calls not yet reported, then reported calls awaiting results. + private var unreported: [PendingCall] = [] + private var awaiting: [String: PendingCall] = [:] + private var flushScheduled = false + private var closed = false + + init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit } + + func start(_ request: [String: Any]) { + task = Task { [weak self] in await self?.run(request) } + } + + func resolve(_ results: [[String: Any]]) { + for result in results { + guard let id = result["id"] as? String, let call = awaiting.removeValue(forKey: id) else { continue } + let content = (result["content"] as? String) ?? "" + let isError = (result["isError"] as? Bool) ?? false + // An error goes back to the model as text: a thrown error would end the whole response. + call.continuation.resume(returning: isError ? "Error: \(content)" : (content.isEmpty ? "(no output)" : content)) + } + // A call JavaScript didn't answer can't be left hanging. + for (_, call) in awaiting { + call.continuation.resume(returning: "Error: the tool returned no result") + } + awaiting.removeAll() + } + + func cancel() { + closed = true + task?.cancel() + task = nil + for call in unreported { call.continuation.resume(throwing: CancellationError()) } + for (_, call) in awaiting { call.continuation.resume(throwing: CancellationError()) } + unreported.removeAll() + awaiting.removeAll() + } + + /// Events stop once the session is cancelled, even from work still winding down. + private func send(_ event: [String: Any]) { + if !closed { emit(event) } + } + + private func run(_ request: [String: Any]) async { + let system = (request["system"] as? String) ?? "" + let prompt = (request["prompt"] as? String) ?? "" + let maxTokens = (request["maxTokens"] as? NSNumber)?.intValue ?? 512 + let permissive = (request["permissive"] as? Bool) ?? false + let toolSpecs: [ToolSpec] = ((request["tools"] as? [[String: Any]]) ?? []).compactMap { tool in + guard let name = tool["name"] as? String else { return nil } + return ToolSpec(name: name, description: (tool["description"] as? String) ?? "", + parameters: (tool["parameters"] as? [String: Any]) ?? [:]) + } + let history = (request["history"] as? [[String: Any]]) ?? [] + + // Transformations of user text (translate, summarize) get the permissive + // guardrails; they apply to plain string replies, which tool-free requests are. + let model = SystemLanguageModel(useCase: .general, + guardrails: permissive ? .permissiveContentTransformations : .default) + var offered = toolSpecs + if !toolSpecs.isEmpty { + switch await selectTools(model: model, specs: toolSpecs, history: history, prompt: prompt) { + case .tools(let specs): offered = specs + case .failed(let event): + send(event) + return + } + if closed { return } + } + let fitted = await fit(model: model, system: system, tools: offered, history: history, + prompt: prompt, maxTokens: maxTokens) + if closed { return } + + let handler: @Sendable (String, String) async throws -> String = { [weak self] name, arguments in + guard let self = self else { throw CancellationError() } + return try await self.toolInvoked(name: name, arguments: arguments) + } + let tools: [any Tool] = fitted.tools.map { + BridgedTool(name: $0.name, description: $0.description, parameters: $0.schema, handler: handler) + } + let session = LanguageModelSession(model: model, tools: tools, transcript: Transcript(entries: fitted.entries)) + let options = GenerationOptions(maximumResponseTokens: maxTokens) + var last = "" + do { + for try await snapshot in session.streamResponse(to: prompt, options: options) { + if closed { return } + let text = snapshot.content + // Snapshots are cumulative; one that doesn't extend the last is a new segment. + let delta = text.hasPrefix(last) ? String(text.dropFirst(last.count)) : text + last = text + if !delta.isEmpty { send(["kind": "delta", "text": delta]) } + } + send(["kind": "done"]) + } catch { + if closed || error is CancellationError { return } + send(describe(error)) + } + } + + // MARK: - Tool calls + + private func toolInvoked(name: String, arguments: String) async throws -> String { + if closed { throw CancellationError() } + return try await withCheckedThrowingContinuation { continuation in + if closed { + continuation.resume(throwing: CancellationError()) + return + } + let id = "fm_" + UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(16).lowercased() + unreported.append(PendingCall(id: id, name: name, arguments: arguments, continuation: continuation)) + scheduleFlush() + } + } + + /// Report calls in one event, after a moment for any parallel calls to arrive. + private func scheduleFlush() { + if flushScheduled { return } + flushScheduled = true + Task { [weak self] in + try? await Task.sleep(nanoseconds: LanguageModelEngine.toolBatchDelayNs) + self?.flush() + } + } + + private func flush() { + flushScheduled = false + if closed || unreported.isEmpty { return } + let calls = unreported + unreported.removeAll() + for call in calls { awaiting[call.id] = call } + send(["kind": "toolCalls", + "calls": calls.map { ["id": $0.id, "name": $0.name, "arguments": $0.arguments] }]) + } + + // MARK: - Tool routing + + private enum Selection { + case tools([ToolSpec]) + case failed([String: Any]) + } + + /// All tools when their definitions fit their share of the context window; + /// otherwise the few the model picks for this request, plus recently used ones. + private func selectTools(model: SystemLanguageModel, specs: [ToolSpec], history: [[String: Any]], + prompt: String) async -> Selection { + let budget = Int(Double(model.contextSize) * LanguageModelEngine.toolBudgetFraction) + let cost = await toolTokens(model, buildTools(specs, compact: false)) + if cost <= budget { return .tools(specs) } + + let names = specs.map { $0.name } + let catalog = specs.map { "- \($0.name): \(SchemaBuilder.shorten($0.description, sentences: 1, limit: 100))" } + .joined(separator: "\n") + let instructions = "You pick the tools a voice assistant needs to handle the user's latest request. " + + "Choose at most \(LanguageModelEngine.maxRoutedTools) tools from this list that the assistant should be able to use, " + + "most important first. Choose none for small talk or questions it can answer from general knowledge.\n\n" + catalog + var query = prompt + if let previous = lastPromptText(history) { + query = "Previous request: \(SchemaBuilder.shorten(previous, sentences: 3, limit: 300))\n\nLatest request: \(prompt)" + } + do { + let schema = try GenerationSchema(root: DynamicGenerationSchema(name: "ToolSelection", properties: [ + DynamicGenerationSchema.Property( + name: "tools", description: "Names of the tools needed, most important first.", + schema: DynamicGenerationSchema(arrayOf: DynamicGenerationSchema(type: String.self, guides: [.anyOf(names)]), + minimumElements: 0, + maximumElements: LanguageModelEngine.maxRoutedTools)), + ]), dependencies: []) + let session = LanguageModelSession(model: model, instructions: instructions) + let response = try await session.respond(to: query, schema: schema, + options: GenerationOptions(sampling: .greedy, maximumResponseTokens: 100)) + var chosen = Set(try response.content.value([String].self, forProperty: "tools")) + let recent = recentToolNames(history).prefix(LanguageModelEngine.keptRecentTools) + chosen.formUnion(recent) + let selected = specs.filter { chosen.contains($0.name) } + send(["kind": "log", "message": "routed \(specs.count) tools (\(cost) tokens) to: " + + (selected.isEmpty ? "none" : selected.map { $0.name }.joined(separator: ", "))]) + return .tools(selected) + } catch { + if let generationError = error as? LanguageModelSession.GenerationError { + switch generationError { + case .rateLimited, .concurrentRequests, .assetsUnavailable, .guardrailViolation, .refusal, + .unsupportedLanguageOrLocale: + // The real request would fail the same way. + return .failed(describe(error)) + default: + break + } + } + send(["kind": "log", "message": "tool routing failed (\(error.localizedDescription)); offering tools in order"]) + return .tools(specs) + } + } + + private func toolTokens(_ model: SystemLanguageModel, _ tools: [BuiltTool]) async -> Int { + let definitions = tools.map { + Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema) + } + let instructions = Transcript.Entry.instructions(Transcript.Instructions(segments: [], toolDefinitions: definitions)) + let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count } + return await tokenCount(model, [instructions], extraCharacters: schemaCharacters + 150 * tools.count) + } + + private func lastPromptText(_ history: [[String: Any]]) -> String? { + for item in history.reversed() where item["type"] as? String == "prompt" { + if let text = item["text"] as? String, !text.isEmpty { return text } + } + return nil + } + + /// Tool names from the newest tool calls back, without repeats. + private func recentToolNames(_ history: [[String: Any]]) -> [String] { + var names: [String] = [] + for item in history.reversed() where item["type"] as? String == "toolCalls" { + for call in (item["calls"] as? [[String: Any]]) ?? [] { + if let name = call["name"] as? String, !names.contains(name) { names.append(name) } + } + } + return names + } + + // MARK: - Context fitting + + private struct BuiltTool { + let name: String + let description: String + let schema: GenerationSchema + } + + private struct Fitted { + let tools: [BuiltTool] + let entries: [Transcript.Entry] + } + + /// Instructions + tool definitions + history + prompt must leave the reply + /// budget free. In order, until it fits: shorten tool descriptions, drop the + /// oldest exchanges, then drop the last-registered tools. + private func fit(model: SystemLanguageModel, system: String, tools specs: [ToolSpec], + history: [[String: Any]], prompt: String, maxTokens: Int) async -> Fitted { + let contextSize = model.contextSize + let reserve = min(maxTokens, contextSize / 3) + LanguageModelEngine.contextMargin + var tools = buildTools(specs, compact: false) + var past = transcriptEntries(history) + let promptEntry = Transcript.Entry.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: prompt))])) + + func entries() -> [Transcript.Entry] { + let definitions = tools.map { + Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema) + } + let instructions = Transcript.Instructions(segments: [.text(Transcript.TextSegment(content: system))], + toolDefinitions: definitions) + return [.instructions(instructions)] + past + } + func overBudget() async -> Bool { + // The tokenizer can't see tool schemas' size from a ToolDefinition; the fallback estimate adds them. + let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count } + let used = await tokenCount(model, entries() + [promptEntry], extraCharacters: schemaCharacters) + return used + reserve > contextSize + } + + if !tools.isEmpty, await overBudget() { + tools = buildTools(specs, compact: true) + send(["kind": "log", "message": "shortened tool descriptions to fit the context window"]) + } + var droppedEntries = 0 + while !past.isEmpty, await overBudget() { + // Drop the oldest exchange: through to the next user prompt. + past.removeFirst() + droppedEntries += 1 + while let first = past.first, !isPrompt(first) { + past.removeFirst() + droppedEntries += 1 + } + } + if droppedEntries > 0 { + send(["kind": "log", "message": "dropped \(droppedEntries) history entries to fit the context window"]) + } + var droppedTools: [String] = [] + while !tools.isEmpty, await overBudget() { + droppedTools.append(tools.removeLast().name) + } + if !droppedTools.isEmpty { + send(["kind": "log", "message": "dropped tools to fit the context window: \(droppedTools.joined(separator: ", "))"]) + } + return Fitted(tools: tools, entries: entries()) + } + + private func tokenCount(_ model: SystemLanguageModel, _ entries: [Transcript.Entry], extraCharacters: Int) async -> Int { + if #available(iOS 26.4, macOS 26.4, *) { + if let count = try? await model.tokenCount(for: entries) { return count } + } + // Before iOS 26.4 there's no tokenizer API: estimate conservatively. + var characters = extraCharacters + for entry in entries { characters += estimateCharacters(entry) + 16 } + return characters / 3 + } + + private func estimateCharacters(_ entry: Transcript.Entry) -> Int { + func segments(_ list: [Transcript.Segment]) -> Int { + list.reduce(0) { total, segment in + switch segment { + case .text(let text): return total + text.content.count + case .structure(let structure): return total + structure.content.jsonString.count + @unknown default: return total + } + } + } + switch entry { + case .instructions(let instructions): + var total = segments(instructions.segments) + for definition in instructions.toolDefinitions { + total += definition.name.count + definition.description.count + } + return total + case .prompt(let prompt): return segments(prompt.segments) + case .response(let response): return segments(response.segments) + case .toolCalls(let calls): return calls.reduce(0) { $0 + $1.toolName.count + $1.arguments.jsonString.count } + case .toolOutput(let output): return segments(output.segments) + @unknown default: return 0 + } + } + + private func isPrompt(_ entry: Transcript.Entry) -> Bool { + if case .prompt = entry { return true } + return false + } + + // MARK: - Request conversion + + private func transcriptEntries(_ history: [[String: Any]]) -> [Transcript.Entry] { + var entries: [Transcript.Entry] = [] + for item in history { + let text = (item["text"] as? String) ?? "" + switch item["type"] as? String { + case "prompt": + entries.append(.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: text))]))) + case "response": + entries.append(.response(Transcript.Response(assetIDs: [], segments: [.text(Transcript.TextSegment(content: text))]))) + case "toolCalls": + let calls: [Transcript.ToolCall] = ((item["calls"] as? [[String: Any]]) ?? []).compactMap { call in + guard let id = call["id"] as? String, let name = call["name"] as? String else { return nil } + let arguments = (try? GeneratedContent(json: (call["arguments"] as? String) ?? "{}")) + ?? GeneratedContent(properties: [:]) + return Transcript.ToolCall(id: id, toolName: name, arguments: arguments) + } + if !calls.isEmpty { entries.append(.toolCalls(Transcript.ToolCalls(calls))) } + case "toolOutput": + guard let id = item["id"] as? String else { continue } + entries.append(.toolOutput(Transcript.ToolOutput( + id: id, toolName: (item["name"] as? String) ?? "tool", + segments: [.text(Transcript.TextSegment(content: text))]))) + default: + continue + } + } + return entries + } + + private func buildTools(_ specs: [ToolSpec], compact: Bool) -> [BuiltTool] { + specs.map { spec in + var builder = SchemaBuilder(compact: compact) + let schema: GenerationSchema + do { + schema = try GenerationSchema(root: builder.object(spec.parameters, name: "\(spec.name)_arguments"), + dependencies: []) + } catch { + send(["kind": "log", "message": "tool \(spec.name): unusable parameter schema (\(error)); offering it without parameters"]) + schema = (try? GenerationSchema(root: DynamicGenerationSchema(name: "\(spec.name)_arguments", properties: []), + dependencies: [])) + ?? GenerationSchema(type: GeneratedContent.self, properties: []) + } + let description = compact ? SchemaBuilder.shorten(spec.description, sentences: 1, limit: 160) : spec.description + return BuiltTool(name: spec.name, description: description, schema: schema) + } + } + + // MARK: - Errors + + private func describe(_ error: Error) -> [String: Any] { + if let error = error as? LanguageModelSession.GenerationError { + switch error { + case .exceededContextWindowSize: + return failure("context", "That conversation is too long for the on-device model. Start a new conversation and try again.") + case .assetsUnavailable: + return failure("unavailable", "The Apple Intelligence model isn't available right now. Check that Apple Intelligence is on and finished downloading.") + case .guardrailViolation, .refusal: + return failure("refusal", "The on-device model declined this request.") + case .unsupportedLanguageOrLocale: + return failure("language", "The on-device model doesn't support this language.") + case .rateLimited: + return failure("rateLimited", "iOS is limiting the on-device model while Faceclaw is in the background. Try again in a moment, or with the phone unlocked.") + case .concurrentRequests: + return failure("busy", "The on-device model is busy with another request. Try again in a moment.") + case .decodingFailure, .unsupportedGuide: + return failure("other", "The on-device model produced an unreadable reply: \(error.localizedDescription)") + @unknown default: + return failure("other", "On-device model failed: \(error.localizedDescription)") + } + } + if let error = error as? LanguageModelSession.ToolCallError { + return failure("other", "On-device model tool call failed: \(error.underlyingError.localizedDescription)") + } + return failure("other", "On-device model failed: \(error.localizedDescription)") + } + + private func failure(_ code: String, _ message: String) -> [String: Any] { + ["kind": "error", "code": code, "message": message] + } +} + +/// JSON Schema (the subset our tool definitions use: objects of string, number, +/// integer, boolean, enum and array properties) to a DynamicGenerationSchema, so +/// the framework constrains tool arguments to valid values while decoding. +/// Anything it doesn't understand becomes a string rather than failing. +@available(iOS 26.0, *) +private struct SchemaBuilder { + let compact: Bool + private var names = Set() + + init(compact: Bool) { self.compact = compact } + + mutating func object(_ json: [String: Any], name: String) -> DynamicGenerationSchema { + let properties = (json["properties"] as? [String: Any]) ?? [:] + let required = Set((json["required"] as? [String]) ?? []) + // JSON objects lose declaration order; keep required properties first, then by name. + let keys = properties.keys.sorted { a, b in + let ra = required.contains(a), rb = required.contains(b) + return ra != rb ? ra : a < b + } + let members: [DynamicGenerationSchema.Property] = keys.compactMap { key in + guard let property = properties[key] as? [String: Any] else { return nil } + return DynamicGenerationSchema.Property( + name: key, + description: describe(property), + schema: value(property, name: "\(name)_\(key)"), + isOptional: !required.contains(key)) + } + return DynamicGenerationSchema(name: unique(name), description: nil, properties: members) + } + + private mutating func value(_ json: [String: Any], name: String) -> DynamicGenerationSchema { + if let choices = json["enum"] as? [Any], !choices.isEmpty { + let strings = choices.compactMap { $0 as? String } + if strings.count == choices.count { + // A guide renders as one compact JSON Schema enum; an anyOf schema lists + // every choice as its own subschema, at over twice the tokens. + return DynamicGenerationSchema(type: String.self, guides: [.anyOf(strings)]) + } + } + switch schemaType(json) { + case "integer": + var guides: [GenerationGuide] = [] + if let minimum = (json["minimum"] as? NSNumber)?.intValue { guides.append(.minimum(minimum)) } + if let maximum = (json["maximum"] as? NSNumber)?.intValue { guides.append(.maximum(maximum)) } + return DynamicGenerationSchema(type: Int.self, guides: guides) + case "number": + var guides: [GenerationGuide] = [] + if let minimum = (json["minimum"] as? NSNumber)?.doubleValue { guides.append(.minimum(minimum)) } + if let maximum = (json["maximum"] as? NSNumber)?.doubleValue { guides.append(.maximum(maximum)) } + return DynamicGenerationSchema(type: Double.self, guides: guides) + case "boolean": + return DynamicGenerationSchema(type: Bool.self) + case "array": + let items = (json["items"] as? [String: Any]) ?? ["type": "string"] + return DynamicGenerationSchema(arrayOf: value(items, name: "\(name)_item"), + minimumElements: (json["minItems"] as? NSNumber)?.intValue, + maximumElements: (json["maxItems"] as? NSNumber)?.intValue) + case "object": + return object(json, name: name) + default: + return DynamicGenerationSchema(type: String.self) + } + } + + private func schemaType(_ json: [String: Any]) -> String { + if let type = json["type"] as? String { return type } + if let types = json["type"] as? [String], let type = types.first(where: { $0 != "null" }) { return type } + return json["properties"] != nil ? "object" : "string" + } + + private func describe(_ json: [String: Any]) -> String? { + guard let text = json["description"] as? String, !text.isEmpty else { return nil } + return compact ? SchemaBuilder.shorten(text, sentences: 1, limit: 80) : text + } + + /// Schema names must be unique within one GenerationSchema. + private mutating func unique(_ base: String) -> String { + var name = base + var suffix = 2 + while names.contains(name) { + name = "\(base)_\(suffix)" + suffix += 1 + } + names.insert(name) + return name + } + + /// The first `sentences` sentences, cut to `limit` characters. + static func shorten(_ text: String, sentences: Int, limit: Int) -> String { + var result = "" + var count = 0 + var remainder = Substring(text) + while count < sentences, !remainder.isEmpty { + if let end = remainder.range(of: ". ") { + result += remainder[.. limit { result = String(result.prefix(limit - 1)) + "…" } + return result + } +} + +#endif diff --git a/CHANGELOG b/CHANGELOG index 43dd2244..2826f50d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,7 @@ * Ensure connection is re-set to 2M on reconnect * Add support for embedded icons and custom layouts in Android notifications * Port the Developer app's resource-usage monitor and various test pages to iOS +* iOS: Support Apple Intelligence as an on-device model for voice assistant/etc * iOS: Notifications are selectable via the task switcher * Android: Optionally sign in to Spotify's Web API for library browsing and other features diff --git a/app/apps/calculator/calculator-llm.ts b/app/apps/calculator/calculator-llm.ts index 2254c5cc..10d1b888 100644 --- a/app/apps/calculator/calculator-llm.ts +++ b/app/apps/calculator/calculator-llm.ts @@ -7,7 +7,7 @@ * surfacing a transport error. */ import { streamAnthropicMessage } from "../../native/anthropic"; -import { streamLocalQwen } from "../../native/llama"; +import { streamLocalModel } from "../../native/local-llm"; import { streamOpenAiResponse } from "../../native/openai"; import { resolveAssistantModel } from "../../assistant/models"; import { type LlmStreamOptions } from "../../assistant/llm-protocol"; @@ -41,7 +41,7 @@ export function rewriteSpokenProblem(spoken: string): Promise { streamOpenAiResponse(options); break; case "local": - streamLocalQwen(options); + streamLocalModel(options); break; default: streamAnthropicMessage(options); diff --git a/app/apps/microphones/conversation-qa.ts b/app/apps/microphones/conversation-qa.ts index c3300c2e..8f38088f 100644 --- a/app/apps/microphones/conversation-qa.ts +++ b/app/apps/microphones/conversation-qa.ts @@ -1,6 +1,6 @@ import { Utils } from "@nativescript/core"; -import { isLocalModelReady, streamLocalQwen } from "../../native/llama"; +import { isLocalModelReady, streamLocalModel } from "../../native/local-llm"; import type { LlmStreamHandle } from "../../assistant/llm-protocol"; import { parseConversationQuery } from "../../phone-ui/conversation-search"; @@ -245,7 +245,7 @@ export function askConversations( setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0); return { cancel: () => {} }; } - return streamLocalQwen({ + return streamLocalModel({ apiKey: "", model: "local", system: diff --git a/app/apps/microphones/name-verify.ts b/app/apps/microphones/name-verify.ts index 87acb77e..4cb01128 100644 --- a/app/apps/microphones/name-verify.ts +++ b/app/apps/microphones/name-verify.ts @@ -1,4 +1,4 @@ -import { isLocalModelReady, streamLocalQwen } from "../../native/llama"; +import { isLocalModelReady, streamLocalModel } from "../../native/local-llm"; import type { NameInference } from "./name-guess"; /** @@ -38,7 +38,7 @@ export function verifyIntroducedName( }; const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS); try { - streamLocalQwen({ + streamLocalModel({ apiKey: "", model: "local", system: diff --git a/app/apps/microphones/speaker-insights.ts b/app/apps/microphones/speaker-insights.ts index 0c9401ce..b4044645 100644 --- a/app/apps/microphones/speaker-insights.ts +++ b/app/apps/microphones/speaker-insights.ts @@ -1,6 +1,6 @@ import { Utils } from "@nativescript/core"; -import { isLocalModelReady, streamLocalQwen } from "../../native/llama"; +import { isLocalModelReady, streamLocalModel } from "../../native/local-llm"; import { speakerRegistry } from "./speakers"; declare const com: any; @@ -124,7 +124,7 @@ function generateForSpeaker( }; const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS); try { - streamLocalQwen({ + streamLocalModel({ apiKey: "", model: "local", system: diff --git a/app/apps/translate/translate-llm.ts b/app/apps/translate/translate-llm.ts index 70883340..37e992b2 100644 --- a/app/apps/translate/translate-llm.ts +++ b/app/apps/translate/translate-llm.ts @@ -4,7 +4,7 @@ * TranslationModel for how stretches are batched and given context. */ import { streamAnthropicMessage } from "../../native/anthropic"; -import { streamLocalQwen } from "../../native/llama"; +import { streamLocalModel } from "../../native/local-llm"; import { streamOpenAiResponse } from "../../native/openai"; import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models"; import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol"; @@ -55,7 +55,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl case "openai": return streamOpenAiResponse(options); case "local": - return streamLocalQwen(options); + return streamLocalModel(options); default: return streamAnthropicMessage(options); } diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index 20c6b2e6..c230dcf9 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -1,6 +1,6 @@ import { streamAnthropicMessage } from "../native/anthropic"; import { streamHermesChat } from "../native/hermes"; -import { streamLocalQwen } from "../native/llama"; +import { streamLocalModel } from "../native/local-llm"; import { streamOpenAiResponse } from "../native/openai"; import type { LlmContentBlock, @@ -146,7 +146,7 @@ export class DirectAssistantBackend { : options.provider === "openai" ? streamOpenAiResponse(streamOptions) : options.provider === "local" - ? streamLocalQwen(streamOptions) + ? streamLocalModel(streamOptions) : streamAnthropicMessage(streamOptions); }; diff --git a/app/assistant/models.ts b/app/assistant/models.ts index 567de0ad..750a9a79 100644 --- a/app/assistant/models.ts +++ b/app/assistant/models.ts @@ -1,4 +1,4 @@ -import { isLocalModelReady, LOCAL_MODEL } from "../native/llama"; +import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm"; export const ASSISTANT_MODEL_VALUES = [ "auto", @@ -13,12 +13,17 @@ export const ASSISTANT_MODEL_VALUES = [ "hermes", ] as const; +/** + * "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on + * Android, the Apple Intelligence system model on iOS (the ID predates iOS + * support and stays for saved settings and conversations). + */ export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number]; /** Keep saved model IDs portable, while offering only supported choices. */ export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] = - global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES; + isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen"); export function supportedAssistantModel(value: AssistantModel): AssistantModel { - return global.isIOS && value === "qwen" ? "auto" : value; + return value === "qwen" && !isLocalModelSupported() ? "auto" : value; } export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes"; @@ -58,8 +63,9 @@ const MODEL_DEFINITIONS: Record, ModelDefinition luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" }, terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" }, sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" }, - // Runs on the phone itself (llama.cpp); needs the downloaded model file, - // not an API key. The default for users who haven't set any key. + // Runs on the phone itself (Android: llama.cpp with the downloaded model + // file; iOS: Apple Intelligence), not an API key. The default for users who + // haven't set any key. qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id }, // Your own Hermes Agent (its API server), with its own tools and memory. hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" }, @@ -76,8 +82,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider /** * Resolve Auto and attach the credential needed by the chosen provider. * Auto intentionally prefers Terra over Sonnet when both keys are available, - * and falls back to the on-phone model when no key is set but the model file - * has been downloaded. + * and falls back to the on-phone model when no key is set but that model is + * ready (downloaded on Android; Apple Intelligence on and ready on iOS). */ export function resolveAssistantModel( selection: AssistantModel, diff --git a/app/native/llama.ts b/app/native/llama.ts index c60ad4bb..20225ae5 100644 --- a/app/native/llama.ts +++ b/app/native/llama.ts @@ -6,12 +6,14 @@ import type { LlmStreamOptions, } from "../assistant/llm-protocol"; import { buildQwenPrompt } from "../assistant/qwen-prompt"; +import type { LocalModelState } from "./local-llm-types"; declare const com: any; declare const java: any; /** - * On-phone LLM backend: model download management plus a streaming client + * Android's on-phone LLM backend (reached through local-llm.ts; iOS uses + * local-llm.ios.ts instead): model download management plus a streaming client * with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so * the direct assistant loop can treat "local" as just another provider. * Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI). @@ -38,11 +40,7 @@ const TOP_K = 20; /** Free the ~3GB of model + KV this long after the last generation. */ const IDLE_UNLOAD_MS = 2 * 60 * 1000; -export type LocalModelState = { - status: "absent" | "downloading" | "ready"; - bytesDownloaded: number; - totalBytes: number; -}; +export type { LocalModelState } from "./local-llm-types"; let runner: any = null; let downloader: any = null; diff --git a/app/native/local-llm-types.ts b/app/native/local-llm-types.ts new file mode 100644 index 00000000..43b1d820 --- /dev/null +++ b/app/native/local-llm-types.ts @@ -0,0 +1,13 @@ +/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */ + +export type LocalModelState = { + /** + * Android downloads its model: absent → downloading → ready. iOS uses the + * system's Apple Intelligence model: ready, or unavailable (see detail). + */ + status: "absent" | "downloading" | "ready" | "unavailable"; + bytesDownloaded: number; + totalBytes: number; + /** Why the model can't be used, when status is "unavailable" (short, for a settings row). */ + detail?: string; +}; diff --git a/app/native/local-llm.ios.ts b/app/native/local-llm.ios.ts new file mode 100644 index 00000000..321cd0c7 --- /dev/null +++ b/app/native/local-llm.ios.ts @@ -0,0 +1,383 @@ +import type { + LlmContentBlock, + LlmMessage, + LlmStreamHandle, + LlmStreamOptions, +} from "../assistant/llm-protocol"; +import type { LocalModelState } from "./local-llm-types"; +export type { LocalModelState } from "./local-llm-types"; + +declare const FaceclawSystemLanguageModel: any; + +/** + * The on-phone LLM on iOS: Apple's Foundation Models system model (Apple + * Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same + * surface as Android's local-llm.ts, so the direct assistant loop, Translate, + * Calculator and the Microphones helpers treat it as the "local" provider. + * + * Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the + * background (where Faceclaw usually is, driving the glasses with the phone + * locked), CPU-only inference of a 4B model is far too slow on a phone, and a + * ~3 GB in-process model makes the app the first thing jetsam kills. The + * system model runs out of process on the Neural Engine, needs no download, + * and decodes tool arguments under a schema constraint (like our GBNF grammar + * on Android). Its costs: a 4096-token context (the Swift side fits history + * and tool definitions into it), Apple's guardrails, a background request + * budget, and Apple Intelligence-capable hardware. + * + * Tool calls: the framework runs tools itself, mid-response. The Swift side + * parks each call and reports it; here that ends the loop iteration with + * stopReason "tool_use", the direct backend executes the tools as for any + * provider, and its next request (ending in the matching tool_result blocks) + * resumes the parked generation instead of starting a new one. + */ + +export const LOCAL_MODEL = { + id: "apple-foundation-model", + label: "Apple Intelligence", + sizeBytes: 0, +}; + +const DEFAULT_MAX_TOKENS = 512; +/** A generation parked in a tool call is abandoned if the tool loop never returns. */ +const PARKED_TIMEOUT_MS = 3 * 60 * 1000; +/** + * Only used when tool results arrive with no parked generation to resume (it + * timed out, or the app restarted mid-turn): the framework has no way to + * continue after a tool output without a new prompt. + */ +export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request."; + +type Availability = + | "available" + | "unsupportedOS" + | "deviceNotEligible" + | "appleIntelligenceNotEnabled" + | "modelNotReady" + | "unavailable"; + +const UNAVAILABLE_DETAIL: Record, string> = { + unsupportedOS: "needs iOS 26", + deviceNotEligible: "not on this iPhone", + appleIntelligenceNotEnabled: "Apple Intelligence off", + modelNotReady: "downloading", + unavailable: "unavailable", +}; + +const UNAVAILABLE_MESSAGE: Record, string> = { + unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.", + deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.", + appleIntelligenceNotEnabled: + "Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.", + modelNotReady: "Apple Intelligence is still downloading its model. Try again later.", + unavailable: "Apple Intelligence's on-device model is unavailable right now.", +}; + +export function localModelAvailability(): Availability { + try { + if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS"; + const value = String(FaceclawSystemLanguageModel.availability()); + return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable"; + } catch { + return "unsupportedOS"; + } +} + +/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */ +export function isLocalModelSupported(): boolean { + const availability = localModelAvailability(); + return availability !== "unsupportedOS" && availability !== "deviceNotEligible"; +} + +export function isLocalModelReady(): boolean { + return localModelAvailability() === "available"; +} + +export function localModelState(): LocalModelState { + const availability = localModelAvailability(); + if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 }; + return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] }; +} + +/** The user-facing reason the model can't run, or "" when it can. */ +export function localModelUnavailableMessage(): string { + const availability = localModelAvailability(); + return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability]; +} + +// iOS manages the system model itself: there is nothing to download, cancel, +// delete or unload, and no progress to watch. +export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void { + return () => {}; +} +export function startLocalModelDownload(): void {} +export function cancelLocalModelDownload(): void {} +export function deleteLocalModel(): void {} +export function unloadLocalModel(): void {} + +type HistoryEntry = + | { type: "prompt" | "response"; text: string } + | { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> } + | { type: "toolOutput"; id: string; name: string; text: string }; + +export type SystemModelRequest = { + system: string; + tools: Array<{ name: string; description: string; parameters: object }>; + history: HistoryEntry[]; + prompt: string; + maxTokens: number; + permissive: boolean; +}; + +type ToolResultBlock = Extract; + +/** + * Translate provider-neutral history into the Swift request: everything but + * the newest user prompt becomes Transcript entries, and that prompt is what + * the model answers. + */ +export function buildSystemModelRequest( + options: Pick, +): SystemModelRequest { + const messages = options.messages; + const last = messages[messages.length - 1]; + const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length; + const history: HistoryEntry[] = []; + const toolNames = new Map(); + for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) { + appendHistory(history, message, toolNames); + } + const tools = (options.tools ?? []).map((tool) => ({ + name: tool.name, + description: tool.description, + parameters: tool.input_schema, + })); + return { + system: options.system?.trim() || "You are a helpful assistant.", + tools, + history, + prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT, + maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS, + // Tool-free requests transform user text (translate, summarize, Q&A over transcripts). + permissive: tools.length === 0, + }; +} + +function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map): void { + if (message.role === "user") { + for (const result of toolResultsOf(message)) { + const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)"; + history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body }); + } + const text = textOf(message.content); + if (text) history.push({ type: "prompt", text }); + return; + } + const text = textOf(message.content); + if (text) history.push({ type: "response", text }); + if (typeof message.content === "string") return; + const calls: Array<{ id: string; name: string; arguments: string }> = []; + for (const block of message.content) { + if (block.type !== "tool_use") continue; + toolNames.set(block.id, block.name); + calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) }); + } + if (calls.length) history.push({ type: "toolCalls", calls }); +} + +function textOf(content: string | LlmContentBlock[]): string { + if (typeof content === "string") return content.trim(); + return content + .filter((block): block is Extract => block.type === "text") + .map((block) => block.text.trim()) + .filter(Boolean) + .join("\n"); +} + +function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] { + if (!message || message.role !== "user" || typeof message.content === "string") return []; + return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result"); +} + +function parseArguments(json: string): any { + try { + const value = JSON.parse(json); + return value && typeof value === "object" ? value : {}; + } catch { + return {}; + } +} + +type NativeEvent = { + kind: string; + text?: string; + calls?: Array<{ id: string; name: string; arguments: string }>; + code?: string; + message?: string; +}; + +/** Generations parked in tool calls, by call id. */ +const parked = new Map(); +/** Every unfinished generation, so its native object outlives the caller's handle. */ +const live = new Set(); + +/** + * One native generation, possibly spanning several loop iterations: each + * iteration ends at a batch of tool calls (or the end of the reply), and the + * next one resumes it with the results. + */ +class Generation { + private readonly native: any; + private options: LlmStreamOptions | null; + private iteration = 0; + private text = ""; + private callIds: string[] = []; + private expiry: ReturnType | null = null; + private closed = false; + + constructor(options: LlmStreamOptions) { + this.options = options; + this.native = FaceclawSystemLanguageModel.new(); + this.native.eventHandler = (json: string) => this.onEvent(String(json)); + live.add(this); + } + + start(request: SystemModelRequest): LlmStreamHandle { + this.native.start(JSON.stringify(request)); + return this.handle(); + } + + /** Whether these results answer this generation's parked calls. */ + awaits(results: ToolResultBlock[]): boolean { + return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id)); + } + + resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle { + this.unpark(); + this.options = options; + this.iteration++; + this.text = ""; + this.native.resolveToolCalls( + JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))), + ); + return this.handle(); + } + + cancel(): void { + if (this.closed) return; + this.close(); + try { + this.native.cancel(); + } catch { + // ignore + } + } + + private handle(): LlmStreamHandle { + const iteration = this.iteration; + // A stale handle (an earlier iteration's) still cancels a generation parked + // since then: its owner gave up on the turn. + return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } }; + } + + private close(): void { + this.closed = true; + this.unpark(); + this.options = null; + this.native.eventHandler = null; + live.delete(this); + } + + private park(ids: string[]): void { + this.callIds = ids; + for (const id of ids) parked.set(id, this); + this.expiry = setTimeout(() => { + console.warn("[local-llm] abandoning an on-device generation whose tool results never came back"); + this.cancel(); + }, PARKED_TIMEOUT_MS); + } + + private unpark(): void { + for (const id of this.callIds) { + if (parked.get(id) === this) parked.delete(id); + } + this.callIds = []; + if (this.expiry !== null) clearTimeout(this.expiry); + this.expiry = null; + } + + private onEvent(json: string): void { + if (this.closed) return; + let event: NativeEvent; + try { + event = JSON.parse(json); + } catch { + return; + } + if (event.kind === "log") { + console.log(`[local-llm] ${event.message ?? ""}`); + return; + } + const options = this.options; + if (!options) return; + switch (event.kind) { + case "delta": { + const delta = event.text ?? ""; + this.text += delta; + options.onTextDelta?.(delta, this.text); + return; + } + case "toolCalls": { + const calls = event.calls ?? []; + const text = this.text.trim(); + const content: LlmContentBlock[] = text ? [{ type: "text", text }] : []; + for (const call of calls) { + content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) }); + } + this.options = null; + this.park(calls.map((call) => call.id)); + options.onDone({ text, content, stopReason: "tool_use" }); + return; + } + case "done": { + this.close(); + const text = this.text.trim(); + options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" }); + return; + } + case "error": { + this.close(); + if (event.code === "refusal") { + options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" }); + } else { + options.onError(event.message || "The on-device model failed"); + } + return; + } + } + } +} + +/** + * Stream a reply from the system model. A request whose history ends in tool + * results for a parked generation resumes it; anything else starts afresh. + */ +export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle { + const availability = localModelAvailability(); + if (availability !== "available") { + setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0); + return { cancel: () => {} }; + } + const results = toolResultsOf(options.messages[options.messages.length - 1]); + const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean); + if (waiting?.awaits(results)) return waiting.resume(options, results); + let generation: Generation | null = null; + try { + generation = new Generation(options); + return generation.start(buildSystemModelRequest(options)); + } catch (error) { + generation?.cancel(); + setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0); + return { cancel: () => {} }; + } +} diff --git a/app/native/local-llm.ts b/app/native/local-llm.ts new file mode 100644 index 00000000..a308647e --- /dev/null +++ b/app/native/local-llm.ts @@ -0,0 +1,28 @@ +/** + * The on-phone LLM: one entry point for every feature that can run on the + * phone without an API key (the direct assistant's "local" provider, + * Translate, Calculator's rewrite fallback, the Microphones helpers). Each + * platform supplies a streaming client with the cloud clients' LlmStreamOptions + * contract, plus model status for Settings. + * + * - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts). + * - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing + * to download, available on Apple Intelligence devices running iOS 26+. + */ +export { + LOCAL_MODEL, + isLocalModelReady, + localModelState, + onLocalModelStateChanged, + startLocalModelDownload, + cancelLocalModelDownload, + deleteLocalModel, + unloadLocalModel, + streamLocalQwen as streamLocalModel, +} from "./llama"; +export type { LocalModelState } from "./local-llm-types"; + +/** Whether this phone can offer an on-phone model at all (Android: always, by download). */ +export function isLocalModelSupported(): boolean { + return true; +} diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 9f28f080..b8b697fb 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -16,7 +16,7 @@ import { assistantModelProvider, type AssistantModel, } from "~/assistant/models"; -import { isLocalModelReady } from "../native/llama"; +import { isLocalModelReady } from "../native/local-llm"; import { normalizeWhisperServerUrl } from "../native/whisper-server-stt"; import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; @@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({ export type AssistantBackendKind = "direct" | "external"; const assistantBackendLabels: Record = { - direct: global.isIOS ? "Cloud API" : "On-phone", + direct: "On-phone", external: "My own agent (bridge)", }; @@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum assistantBackendLabels[value] ?? value, description: - global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.", + global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.", }); export const assistantHermesUrlSetting = new ConfigSettingString({ @@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum({ return false; }, description: - global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.", + global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.", }); export const mapboxApiKeySetting = new ConfigSettingString({ diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index f4a6f6b6..9e36a1be 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -12,7 +12,7 @@ import { localModelState, onLocalModelStateChanged, startLocalModelDownload, -} from "../../native/llama"; +} from "../../native/local-llm"; import { ASR_MODELS, asrModelState, @@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] { { label: "Assistant", items: [ - // iOS offers cloud APIs; Android also supports local models and the bridge. + // iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge. enumSettingMenuItem(assistantBackendSetting), enumSettingMenuItem(assistantModelSetting), textSettingMenuItem(assistantHermesUrlSetting), textSettingMenuItem(assistantHermesKeySetting), - ...(!global.isIOS ? [localModelMenuItem()] : []), + global.isIOS ? systemModelMenuItem() : localModelMenuItem(), toggleSettingMenuItem(assistantSkipConfirmationSetting), - // iOS has no local LLM or agent bridge yet. + // iOS has no agent bridge yet. ...(!global.isIOS ? [ textSettingMenuItem(assistantBridgeHostSetting), textSettingMenuItem(assistantBridgePortSetting), @@ -399,6 +399,29 @@ function localModelStatusText(): string { return "not downloaded"; } +/** + * iOS: the on-phone model is the Apple Intelligence system model, which iOS + * installs and manages; the row only reports whether it can be used. + */ +function systemModelMenuItem(): MenuItem { + const statusText = (): string => { + const state = localModelState(); + return state.status === "ready" ? "ready" : state.detail ?? "unavailable"; + }; + return { + label: "On-phone model", + description: + "Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " + + "Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " + + "Faster and more private than the cloud models, but simpler, with a short memory. " + + "Used automatically when no API key is set.", + onSelect: () => {}, + render: ({ image, x, y, width }) => { + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText()); + }, + }; +} + /** Download/cancel/delete management for the on-phone assistant model. */ function localModelMenuItem(): MenuItem { return { diff --git a/scripts/apple-llm-harness/extract-tools.cjs b/scripts/apple-llm-harness/extract-tools.cjs new file mode 100644 index 00000000..849274bd --- /dev/null +++ b/scripts/apple-llm-harness/extract-tools.cjs @@ -0,0 +1,52 @@ +// Print the assistant's built-in tool definitions as JSON ([{name, description, parameters}]), +// pulled from the tool files' object literals via the TypeScript AST, with the dynamic +// parts (the app-id enum, NUMBER_ARG) filled in from a fixed list. An approximation of +// what ToolRegistry.listTools() offers with no app windows open. +// Usage: node scripts/apple-llm-harness/extract-tools.cjs > tools.json +const path = require('path'); +const fs = require('fs'); +const root = path.resolve(__dirname, '../..'); +const ts = require(path.join(root, 'node_modules/typescript')); + +const FILES = ['system-tools', 'timer-tools', 'window-tools', 'navigate-tools', 'roam-tools'] + .map(name => path.join(root, 'app/assistant', `${name}.ts`)); +const APP_IDS = fs.readdirSync(path.join(root, 'app/apps'), { withFileTypes: true }) + .filter(entry => entry.isDirectory() && !['launcher', 'onboarding', 'evenhub'].includes(entry.name)) + .map(entry => entry.name).sort(); +const NUMBER_ARG = (noun, listTool) => ({ + type: 'number', + description: `1-based ${noun} number, as shown by ${listTool}. May be omitted when only one ${noun} exists.`, +}); + +const out = []; +for (const file of FILES) { + const src = ts.createSourceFile(file, fs.readFileSync(file, 'utf8'), ts.ScriptTarget.ES2020, true); + const consts = {}; + src.forEachChild(function collect(node) { + if (ts.isVariableDeclaration(node) && node.initializer && ts.isStringLiteral(node.initializer)) { + consts[node.name.getText()] = node.initializer.text; + } + node.forEachChild(collect); + }); + const evaluate = node => { + const js = ts.transpileModule(`(${node.getText()})`, { compilerOptions: { target: ts.ScriptTarget.ES2020 } }).outputText; + const names = [...Object.keys(consts), 'MAX_TIMER_DURATION_MS', 'appIds', 'installedNote', 'NUMBER_ARG']; + return Function(...names, `return ${js}`)(...Object.values(consts), 86400000, APP_IDS, '', NUMBER_ARG); + }; + src.forEachChild(function visit(node) { + if (ts.isObjectLiteralExpression(node)) { + const keys = node.properties.map(p => p.name && p.name.getText()); + if (keys.includes('name') && keys.includes('description') && keys.includes('inputSchema')) { + try { + const spec = evaluate(node); + out.push({ name: String(spec.name).replace(/[^a-zA-Z0-9_-]/g, '_'), description: spec.description, parameters: spec.inputSchema }); + } catch (error) { + console.error(`skipped a tool in ${path.basename(file)}: ${error.message}`); + } + return; + } + } + node.forEachChild(visit); + }); +} +process.stdout.write(JSON.stringify(out, null, 1)); diff --git a/scripts/apple-llm-harness/main.swift b/scripts/apple-llm-harness/main.swift new file mode 100644 index 00000000..4e90bbf0 --- /dev/null +++ b/scripts/apple-llm-harness/main.swift @@ -0,0 +1,50 @@ +// macOS harness for App_Resources/iOS/src/FaceclawSystemLanguageModel.swift: runs one +// assistant request against the Apple Intelligence model the way local-llm.ios.ts does, +// answering every tool call with a canned result. See run.sh. +import Foundation + +@main struct Harness { + @MainActor static func main() async { + let args = CommandLine.arguments + guard args.count >= 3 else { + print("usage: harness [previous prompt]") + return + } + print("availability:", FaceclawSystemLanguageModel.availability(), + "context:", FaceclawSystemLanguageModel.contextSize()) + let tools = (try? JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: args[1])))) ?? [] + var history: [[String: Any]] = [] + if args.count > 3 { history = [["type": "prompt", "text": args[3]], ["type": "response", "text": "OK."]] } + let request: [String: Any] = [ + "system": "You are the voice assistant built into a pair of Even Realities G2 smart glasses. Your reply is shown as text on a 640x480 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list. Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it. If a tool fails or a capability is missing, say so briefly rather than inventing a result.", + "tools": tools, "history": history, "prompt": args[2], "maxTokens": 512, "permissive": false, + ] + let model = FaceclawSystemLanguageModel() + let started = Date() + var finished = false + var firstText: TimeInterval? + model.eventHandler = { json in + let elapsed = String(format: "%6.2fs", Date().timeIntervalSince(started)) + guard let event = (try? JSONSerialization.jsonObject(with: Data(json.utf8))) as? [String: Any] else { return } + switch event["kind"] as? String { + case "delta": + if firstText == nil { firstText = Date().timeIntervalSince(started); print("\(elapsed) first text") } + print(event["text"] as? String ?? "", terminator: "") + case "toolCalls": + let calls = event["calls"] as? [[String: Any]] ?? [] + for call in calls { print("\(elapsed) tool call: \(call["name"] ?? "") \(call["arguments"] ?? "")") } + let results = calls.map { ["id": $0["id"] ?? "", "content": "Done (simulated \($0["name"] ?? "tool") result).", "isError": false] } + let data = try! JSONSerialization.data(withJSONObject: results) + model.resolveToolCalls(String(data: data, encoding: .utf8)!) + case "done": + print("\n\(elapsed) done"); finished = true + default: + print("\(elapsed) \(json)") + if event["kind"] as? String == "error" { finished = true } + } + } + let data = try! JSONSerialization.data(withJSONObject: request) + model.start(String(data: data, encoding: .utf8)!) + while !finished && Date().timeIntervalSince(started) < 120 { try? await Task.sleep(nanoseconds: 50_000_000) } + } +} diff --git a/scripts/apple-llm-harness/run.sh b/scripts/apple-llm-harness/run.sh new file mode 100755 index 00000000..debc099d --- /dev/null +++ b/scripts/apple-llm-harness/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Run one assistant request against Apple's on-device model on this Mac (macOS 26.4+, +# Apple Intelligence on), through the same Swift bridge the iOS app uses, with the +# assistant's real tool list. Prints tool routing, tool calls and the streamed reply. +# scripts/apple-llm-harness/run.sh "set a timer for five minutes" ["previous prompt"] +set -euo pipefail +here="$(cd "$(dirname "$0")" && pwd)" +out="${TMPDIR:-/tmp}/faceclaw-apple-llm-harness" +mkdir -p "$out" +node "$here/extract-tools.cjs" > "$out/tools.json" +swiftc -swift-version 5 -parse-as-library -target "$(uname -m)-apple-macos26.4" \ + "$here/../../App_Resources/iOS/src/FaceclawSystemLanguageModel.swift" "$here/main.swift" -o "$out/harness" +"$out/harness" "$out/tools.json" "$@" diff --git a/tests/hermes-assistant.test.cjs b/tests/hermes-assistant.test.cjs index 476c3780..47242ca7 100644 --- a/tests/hermes-assistant.test.cjs +++ b/tests/hermes-assistant.test.cjs @@ -91,7 +91,7 @@ test('a restored conversation keeps its Hermes session, and Hermes turns get a l requests.push(request); return request; } }, - '../native/llama': { streamLocalQwen: () => assert.fail('no local model') }, + '../native/local-llm': { streamLocalModel: () => assert.fail('no local model') }, './bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: {} }, '../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' }, }); @@ -122,7 +122,7 @@ test('URL normalization adds /v1 to a bare host and drops the endpoint path', () }); test('the Hermes model resolves only with a URL, carrying the key and base URL', () => { - const load = loader({ global: { isIOS: true } }, { '../native/llama': { isLocalModelReady: () => false, LOCAL_MODEL: { id: 'q', label: 'Q' } } }); + const load = loader({ global: { isIOS: true } }, { '../native/local-llm': { isLocalModelReady: () => false, isLocalModelSupported: () => true, LOCAL_MODEL: { id: 'q', label: 'Q' } } }); const { resolveAssistantModel } = load('app/assistant/models.ts'); assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null); const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' }); diff --git a/tests/ios-assistant.test.cjs b/tests/ios-assistant.test.cjs index e4a965bb..6cb1ce99 100644 --- a/tests/ios-assistant.test.cjs +++ b/tests/ios-assistant.test.cjs @@ -12,7 +12,7 @@ function assistant(provider) { const request = { url, body: JSON.parse(body), headers, listener, cancelled: false }; requests.push(request); return { cancel: () => { request.cancelled = true; } }; } }, - '../native/llama': { streamLocalQwen: () => assert.fail('iOS must not invoke a local LLM') }, + '../native/local-llm': { streamLocalModel: () => assert.fail('cloud providers must not invoke the local LLM') }, './bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry }, '../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'test', buildAssistantSystemPrompt: () => 'test', describeAssistantContext: () => '' }, }); @@ -100,8 +100,8 @@ test('iOS SSE adapter forwards the Kotlin stream events once and stops after can assert.deepEqual(seen.at(-1), 'http:429:slow down'); }); -test('iOS only offers cloud assistant models and preserves old local-model conversations as Auto', () => { - const api = loader({ global: { isIOS: true } }, { '../native/llama': { LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false } })('app/assistant/models.ts'); +test('iOS without Apple Intelligence support only offers cloud assistant models and preserves old local-model conversations as Auto', () => { + const api = loader({ global: { isIOS: true } }, { '../native/local-llm': { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => false, isLocalModelSupported: () => false } })('app/assistant/models.ts'); assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), false); assert.equal(api.supportedAssistantModel('qwen'), 'auto'); assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai'); @@ -119,7 +119,7 @@ test('cancelling during a tool prevents subsequent tool side effects and a stall const calls = [], failures = []; const { DirectAssistantBackend } = loader({ setTimeout: fn => { deadline = fn; return 1; }, clearTimeout() {} }, { '../native/openai': { streamOpenAiResponse: options => { stream = options; return { cancel() {} }; } }, - '../native/anthropic': {}, '../native/llama': {}, + '../native/anthropic': {}, '../native/local-llm': {}, })('app/assistant/direct-backend.ts'); const backend = new DirectAssistantBackend(); const options = { provider: 'openai', messages: [], buildTools: () => [], resolveToolName: name => name, diff --git a/tests/ios-local-llm.test.cjs b/tests/ios-local-llm.test.cjs new file mode 100644 index 00000000..8e2c04d6 --- /dev/null +++ b/tests/ios-local-llm.test.cjs @@ -0,0 +1,248 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); +const flush = () => new Promise(resolve => setImmediate(resolve)); +// Values built inside the loader's VM context have that realm's prototypes. +const plain = value => JSON.parse(JSON.stringify(value)); + +/** A stand-in for FaceclawSystemLanguageModel.swift: records requests and lets tests emit its events. */ +function fakeNative(availability = 'available') { + const instances = []; + const cls = { + availability: () => availability, + new() { + const native = { + request: null, resolved: [], cancelled: false, eventHandler: null, + start(json) { this.request = JSON.parse(json); }, + resolveToolCalls(json) { this.resolved.push(JSON.parse(json)); }, + cancel() { this.cancelled = true; }, + emit(event) { if (this.eventHandler) this.eventHandler(JSON.stringify(event)); }, + }; + instances.push(native); + return native; + }, + }; + return { cls, instances, setAvailability(value) { availability = value; } }; +} + +function loadLocalLlm(native, extraGlobals = {}) { + const load = loader({ FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout, ...extraGlobals }, {}); + return load('app/native/local-llm.ios.ts'); +} + +function collector() { + const seen = { deltas: [], done: [], errors: [] }; + return { + seen, + callbacks: { + onTextDelta: (_delta, full) => seen.deltas.push(full), + onDone: result => seen.done.push(result), + onError: message => seen.errors.push(message), + }, + }; +} + +test('iOS local LLM converts provider-neutral history into transcript entries and a final prompt', () => { + const api = loadLocalLlm(fakeNative()); + const request = api.buildSystemModelRequest({ + system: 'Be brief.', + maxTokens: 200, + tools: [{ name: 'timer_set', description: 'Set a timer', input_schema: { type: 'object', properties: { seconds: { type: 'integer' } } } }], + messages: [ + { role: 'user', content: 'Set a timer' }, + { role: 'assistant', content: [{ type: 'text', text: 'On it.' }, { type: 'tool_use', id: 'fm_1', name: 'timer_set', input: { seconds: 60 } }] }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'Timer set' }] }, + { role: 'assistant', content: 'Done.' }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'gone', content: 'boom', is_error: true }, { type: 'text', text: 'And now?' }] }, + { role: 'assistant', content: [{ type: 'provider_item', provider: 'openai', item: {} }] }, + { role: 'user', content: 'What time is it?' }, + ], + }); + assert.equal(request.system, 'Be brief.'); + assert.equal(request.prompt, 'What time is it?'); + assert.equal(request.maxTokens, 200); + assert.equal(request.permissive, false); + assert.deepEqual(plain(request.tools), [{ name: 'timer_set', description: 'Set a timer', parameters: { type: 'object', properties: { seconds: { type: 'integer' } } } }]); + assert.deepEqual(plain(request.history), [ + { type: 'prompt', text: 'Set a timer' }, + { type: 'response', text: 'On it.' }, + { type: 'toolCalls', calls: [{ id: 'fm_1', name: 'timer_set', arguments: '{"seconds":60}' }] }, + { type: 'toolOutput', id: 'fm_1', name: 'timer_set', text: 'Timer set' }, + { type: 'response', text: 'Done.' }, + { type: 'toolOutput', id: 'gone', name: 'tool', text: 'Error: boom' }, + { type: 'prompt', text: 'And now?' }, + ]); + + // Tool-free requests (translate, Q&A) get the permissive guardrails; a + // trailing tool result with nothing to resume falls back to a continuation prompt. + const orphan = api.buildSystemModelRequest({ + messages: [{ role: 'user', content: 'x' }, { role: 'assistant', content: [{ type: 'tool_use', id: 'a', name: 't', input: {} }] }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'a', content: 'ok' }] }], + }); + assert.equal(orphan.permissive, true); + assert.equal(orphan.maxTokens, 512); + assert.equal(orphan.prompt, api.CONTINUE_PROMPT); + assert.deepEqual(plain(orphan.history.at(-1)), { type: 'toolOutput', id: 'a', name: 't', text: 'ok' }); +}); + +test('iOS local LLM streams text deltas, then ends the turn', () => { + const native = fakeNative(); + const api = loadLocalLlm(native); + const { seen, callbacks } = collector(); + api.streamLocalModel({ apiKey: '', model: 'm', system: 's', messages: [{ role: 'user', content: 'Hi' }], ...callbacks }); + const [instance] = native.instances; + assert.equal(instance.request.prompt, 'Hi'); + instance.emit({ kind: 'log', message: 'shortened tool descriptions' }); + instance.emit({ kind: 'delta', text: 'Hello' }); + instance.emit({ kind: 'delta', text: ' there' }); + instance.emit({ kind: 'done' }); + instance.emit({ kind: 'delta', text: 'late' }); + assert.deepEqual(plain(seen.deltas), ['Hello', 'Hello there']); + assert.equal(seen.done.length, 1); + assert.deepEqual(plain(seen.done[0]), { text: 'Hello there', content: [{ type: 'text', text: 'Hello there' }], stopReason: 'end_turn' }); + assert.equal(instance.eventHandler, null, 'finished generations drop their handler'); +}); + +test('iOS local LLM reports why the model is unavailable without touching native code', async () => { + const native = fakeNative('appleIntelligenceNotEnabled'); + const api = loadLocalLlm(native); + assert.equal(api.isLocalModelReady(), false); + assert.equal(api.isLocalModelSupported(), true); + assert.deepEqual(plain(api.localModelState()), { status: 'unavailable', bytesDownloaded: 0, totalBytes: 0, detail: 'Apple Intelligence off' }); + const { seen, callbacks } = collector(); + api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'Hi' }], ...callbacks }); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.match(seen.errors[0], /Turn on Apple Intelligence/); + assert.equal(native.instances.length, 0); + + native.setAvailability('deviceNotEligible'); + assert.equal(api.isLocalModelSupported(), false); + native.setAvailability('somethingNew'); + assert.equal(api.localModelState().detail, 'unavailable'); + const missing = loader({ setTimeout, clearTimeout }, {})('app/native/local-llm.ios.ts'); + assert.equal(missing.localModelAvailability(), 'unsupportedOS', 'pre-iOS-26 builds have no Swift class to call'); + native.setAvailability('available'); + assert.deepEqual(plain(api.localModelState()), { status: 'ready', bytesDownloaded: 0, totalBytes: 0 }); +}); + +test('iOS local LLM maps refusals to the refusal stop reason and other failures to errors', () => { + const native = fakeNative(); + const api = loadLocalLlm(native); + const first = collector(); + api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...first.callbacks }); + native.instances[0].emit({ kind: 'error', code: 'refusal', message: 'declined' }); + assert.equal(first.seen.done[0].stopReason, 'refusal'); + const second = collector(); + api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...second.callbacks }); + native.instances[1].emit({ kind: 'error', code: 'rateLimited', message: 'iOS is limiting the on-device model' }); + assert.deepEqual(plain(second.seen.errors), ['iOS is limiting the on-device model']); +}); + +test('iOS local LLM cancellation stops the native generation and silences later events', () => { + const native = fakeNative(); + const api = loadLocalLlm(native); + const { seen, callbacks } = collector(); + const handle = api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...callbacks }); + handle.cancel(); + native.instances[0].emit({ kind: 'delta', text: 'late' }); + native.instances[0].emit({ kind: 'done' }); + assert.equal(native.instances[0].cancelled, true); + assert.deepEqual(seen, { deltas: [], done: [], errors: [] }); +}); + +test('iOS assistant runs the on-device model through the shared tool loop, resuming the parked generation', async () => { + const native = fakeNative(); + const calls = [], deltas = [], errors = [], done = []; + const registry = { + listTools: () => [{ name: 'timer.set', description: 'Set a timer', inputSchema: { type: 'object', properties: { seconds: { type: 'integer' } }, required: ['seconds'] } }], + callTool: async (name, args) => { calls.push({ name, args }); return { ok: true, content: 'Timer set' }; }, + }; + const mocks = { + './bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry }, + '../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'base prompt', buildAssistantSystemPrompt: () => 'cloud prompt', describeAssistantContext: () => 'Context: now' }, + }; + const load = loader({ global: { isIOS: true, isAndroid: false }, FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout }, mocks); + mocks['../native/local-llm'] = load('app/native/local-llm.ios.ts'); + const { AssistantSession } = load('app/assistant/session.ts'); + const session = new AssistantSession({ kind: 'direct', llm: { selection: 'qwen', provider: 'local', model: 'apple-foundation-model', apiKey: '' } }, registry); + const send = text => session.sendUtterance(text, {}, { onTextDelta: (_d, full) => deltas.push(full), + onToolActivity() {}, onError: error => errors.push(error), onTurnDone: result => done.push(result) }); + + send('Set a one-minute timer'); + assert.equal(native.instances.length, 1); + const generation = native.instances[0]; + assert.equal(generation.request.system, 'base prompt', 'the local provider keeps the system prompt stable'); + assert.equal(generation.request.prompt, 'Set a one-minute timer\n\n(Context: now)'); + assert.deepEqual(plain(generation.request.tools.map(tool => tool.name)), ['timer_set']); + + generation.emit({ kind: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] }); + await flush(); + assert.deepEqual(plain(calls), [{ name: 'timer.set', args: { seconds: 60 } }]); + assert.equal(native.instances.length, 1, 'tool results resume the parked generation instead of starting another'); + assert.deepEqual(plain(generation.resolved), [[{ id: 'fm_abc', content: 'Timer set', isError: false }]]); + + generation.emit({ kind: 'delta', text: 'Your timer is running.' }); + generation.emit({ kind: 'done' }); + assert.equal(done.length, 1); + assert.equal(errors.length, 0); + assert.equal(deltas.at(-1), 'Your timer is running.'); + assert.equal(session.isTurnActive(), false); + + // The next turn starts a fresh generation that carries the whole exchange as transcript history. + send('Thanks'); + const next = native.instances[1]; + assert.deepEqual(plain(next.request.history), [ + { type: 'prompt', text: 'Set a one-minute timer\n\n(Context: now)' }, + { type: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] }, + { type: 'toolOutput', id: 'fm_abc', name: 'timer_set', text: 'Timer set' }, + { type: 'response', text: 'Your timer is running.' }, + ]); + session.cancel(); + assert.equal(next.cancelled, true); +}); + +test('a parked iOS generation survives unrelated requests and is abandoned if its results never arrive', () => { + const native = fakeNative(); + let expire = null; + const timers = { + setTimeout: (fn, ms) => { if (ms > 0) expire = fn; else fn(); return 1; }, + clearTimeout: () => { expire = null; }, + }; + const api = loadLocalLlm(native, timers); + const turn = collector(); + api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], tools: [], ...turn.callbacks }); + native.instances[0].emit({ kind: 'toolCalls', calls: [{ id: 'fm_1', name: 't', arguments: 'not json' }] }); + assert.deepEqual(plain(turn.seen.done[0].content), [{ type: 'tool_use', id: 'fm_1', name: 't', input: {} }]); + + // e.g. Translate asking the model something while the assistant's tool runs. + const other = collector(); + api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'translate this' }], ...other.callbacks }); + assert.equal(native.instances.length, 2); + assert.equal(native.instances[0].cancelled, false); + + assert.equal(typeof expire, 'function'); + expire(); + assert.equal(native.instances[0].cancelled, true); + // Results for the abandoned call now start a fresh generation with the fallback prompt. + api.streamLocalModel({ apiKey: '', model: 'm', ...turn.callbacks, messages: [ + { role: 'user', content: 'x' }, + { role: 'assistant', content: turn.seen.done[0].content }, + { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'ok' }] }, + ] }); + assert.equal(native.instances.length, 3); + assert.equal(native.instances[2].request.prompt, api.CONTINUE_PROMPT); +}); + +test('iOS offers the on-phone model where Apple Intelligence is supported', () => { + const local = { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => true, isLocalModelSupported: () => true }; + const api = loader({ global: { isIOS: true } }, { '../native/local-llm': local })('app/assistant/models.ts'); + assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), true); + assert.equal(api.supportedAssistantModel('qwen'), 'qwen'); + assert.equal(api.assistantModelLabel('qwen'), 'Apple Intelligence (on-phone)'); + const resolved = api.resolveAssistantModel('auto', { openai: '', anthropic: '' }); + assert.equal(resolved.provider, 'local'); + assert.equal(resolved.model, 'apple-foundation-model'); + assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai', 'API keys still win under Auto'); + local.isLocalModelReady = () => false; + assert.equal(api.resolveAssistantModel('qwen', { openai: '', anthropic: '' }), null); +});