forked from Blendman974/faceclaw
Support Apple Intelligence as an on-device AI model for the voice assistant/etc
This commit is contained in:
1 parent
3c3aa31542
commit
4973fd2aea
22 files changed
+1551
-40
No files matched your search
@@ -3,7 +3,9 @@ TARGETED_DEVICE_FAMILY = 1;
|
|||||||
CODE_SIGN_STYLE = Automatic;
|
CODE_SIGN_STYLE = Automatic;
|
||||||
|
|
||||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security
|
// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+,
|
||||||
|
// so it is weak-linked: the app still launches on older iOS and reports the model unavailable.
|
||||||
|
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels
|
||||||
|
|
||||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,694 @@
|
|||||||
|
import Foundation
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
import FoundationModels
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/// The on-phone LLM on iOS: Apple's Foundation Models framework (the Apple
|
||||||
|
/// Intelligence system model, iOS 26+, eligible hardware). It runs out of
|
||||||
|
/// process on the Neural Engine, so it costs the app no model download or
|
||||||
|
/// memory, and unlike Metal work it is allowed (within a system budget) while
|
||||||
|
/// Faceclaw is in the background driving the glasses.
|
||||||
|
///
|
||||||
|
/// JavaScript (app/native/local-llm.ios.ts) drives the same provider-neutral
|
||||||
|
/// agent loop as the cloud models: each start() is one loop iteration. The
|
||||||
|
/// request is JSON:
|
||||||
|
///
|
||||||
|
/// {"system":String,
|
||||||
|
/// "tools":[{"name","description","parameters":<JSON Schema>}],
|
||||||
|
/// "history":[{"type":"prompt"|"response","text"} |
|
||||||
|
/// {"type":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]} |
|
||||||
|
/// {"type":"toolOutput","id","name","text"}],
|
||||||
|
/// "prompt":String, "maxTokens":Int, "permissive":Bool}
|
||||||
|
///
|
||||||
|
/// and events arrive on the main queue through eventHandler:
|
||||||
|
///
|
||||||
|
/// {"kind":"delta","text":String} streamed reply text
|
||||||
|
/// {"kind":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]}
|
||||||
|
/// {"kind":"done"}
|
||||||
|
/// {"kind":"error","code":String,"message":String}
|
||||||
|
/// {"kind":"log","message":String}
|
||||||
|
///
|
||||||
|
/// The framework runs tools itself, mid-response. When the model calls one,
|
||||||
|
/// the call is reported as "toolCalls" and the generation stays suspended
|
||||||
|
/// inside the tool until resolveToolCalls() hands back the results that
|
||||||
|
/// JavaScript's tool loop produced; the reply then continues on the same
|
||||||
|
/// instance.
|
||||||
|
///
|
||||||
|
/// The model's context window is small (4096 tokens on iOS 26, shared by input
|
||||||
|
/// and reply) and the framework spends ~50 tokens on each tool before its
|
||||||
|
/// parameters, so the assistant's full tool list (30+ tools, ~5000 tokens) can't
|
||||||
|
/// be offered at once. When the tools don't fit their share of the window, a
|
||||||
|
/// quick schema-constrained routing request first asks the model which few
|
||||||
|
/// tools the request needs. History is then trimmed to fit before each start().
|
||||||
|
/// Called from JavaScript, so on the main thread.
|
||||||
|
@MainActor
|
||||||
|
@objc(FaceclawSystemLanguageModel)
|
||||||
|
@objcMembers
|
||||||
|
public class FaceclawSystemLanguageModel: NSObject {
|
||||||
|
public var eventHandler: ((String) -> Void)?
|
||||||
|
// A LanguageModelEngine, typed loosely so the class itself loads before iOS 26.
|
||||||
|
private var engine: AnyObject?
|
||||||
|
|
||||||
|
/// "available", or why not: "unsupportedOS", "deviceNotEligible",
|
||||||
|
/// "appleIntelligenceNotEnabled", "modelNotReady" (still downloading), "unavailable".
|
||||||
|
/// Nonisolated: worker isolates load the model-choice list too, off the main thread.
|
||||||
|
nonisolated public static func availability() -> String {
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
if #available(iOS 26.0, *) {
|
||||||
|
switch SystemLanguageModel.default.availability {
|
||||||
|
case .available:
|
||||||
|
return "available"
|
||||||
|
case .unavailable(let reason):
|
||||||
|
switch reason {
|
||||||
|
case .deviceNotEligible: return "deviceNotEligible"
|
||||||
|
case .appleIntelligenceNotEnabled: return "appleIntelligenceNotEnabled"
|
||||||
|
case .modelNotReady: return "modelNotReady"
|
||||||
|
@unknown default: return "unavailable"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
return "unsupportedOS"
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The model's context window in tokens (input and output share it), or 0 when unavailable.
|
||||||
|
nonisolated public static func contextSize() -> Int {
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
if #available(iOS 26.0, *) { return SystemLanguageModel.default.contextSize }
|
||||||
|
#endif
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
public func start(_ requestJson: String) {
|
||||||
|
cancel()
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
if #available(iOS 26.0, *) {
|
||||||
|
guard let data = requestJson.data(using: .utf8),
|
||||||
|
let request = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else {
|
||||||
|
emit(["kind": "error", "code": "other", "message": "Invalid on-device model request"])
|
||||||
|
return
|
||||||
|
}
|
||||||
|
let engine = LanguageModelEngine { [weak self] event in self?.emit(event) }
|
||||||
|
self.engine = engine
|
||||||
|
engine.start(request)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
emit(["kind": "error", "code": "unavailable", "message": "The on-device model needs iOS 26 or later."])
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `[{"id","content","isError"}]` for the calls of the last "toolCalls" event.
|
||||||
|
public func resolveToolCalls(_ resultsJson: String) {
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
if #available(iOS 26.0, *) {
|
||||||
|
let data = resultsJson.data(using: .utf8) ?? Data()
|
||||||
|
let results = ((try? JSONSerialization.jsonObject(with: data)) as? [[String: Any]]) ?? []
|
||||||
|
(engine as? LanguageModelEngine)?.resolve(results)
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Abandon the generation (and any suspended tool calls) without further events.
|
||||||
|
public func cancel() {
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
if #available(iOS 26.0, *) { (engine as? LanguageModelEngine)?.cancel() }
|
||||||
|
#endif
|
||||||
|
engine = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
private func emit(_ event: [String: Any]) {
|
||||||
|
// JSONSerialization raises (not throws) on NaN and other invalid values.
|
||||||
|
guard JSONSerialization.isValidJSONObject(event),
|
||||||
|
let data = try? JSONSerialization.data(withJSONObject: event),
|
||||||
|
let json = String(data: data, encoding: .utf8) else { return }
|
||||||
|
eventHandler?(json)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#if canImport(FoundationModels)
|
||||||
|
|
||||||
|
/// A JavaScript-side tool: the framework calls it mid-response, and the
|
||||||
|
/// engine parks the call until JavaScript resolves it.
|
||||||
|
@available(iOS 26.0, *)
|
||||||
|
private struct BridgedTool: Tool {
|
||||||
|
typealias Arguments = GeneratedContent
|
||||||
|
typealias Output = String
|
||||||
|
|
||||||
|
let name: String
|
||||||
|
let description: String
|
||||||
|
let parameters: GenerationSchema
|
||||||
|
let handler: @Sendable (String, String) async throws -> String
|
||||||
|
|
||||||
|
func call(arguments: GeneratedContent) async throws -> String {
|
||||||
|
try await handler(name, arguments.jsonString)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One tool from the request, convertible at two levels of verbosity.
|
||||||
|
@available(iOS 26.0, *)
|
||||||
|
private struct ToolSpec {
|
||||||
|
let name: String
|
||||||
|
let description: String
|
||||||
|
let parameters: [String: Any]
|
||||||
|
}
|
||||||
|
|
||||||
|
@available(iOS 26.0, *)
|
||||||
|
@MainActor
|
||||||
|
final class LanguageModelEngine {
|
||||||
|
private struct PendingCall {
|
||||||
|
let id: String
|
||||||
|
let name: String
|
||||||
|
let arguments: String
|
||||||
|
let continuation: CheckedContinuation<String, Error>
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long a call waits for any parallel calls before they're reported together.
|
||||||
|
private static let toolBatchDelayNs: UInt64 = 30_000_000
|
||||||
|
/// Tokens kept free beyond the reply budget: the framework's own framing and rounding.
|
||||||
|
private static let contextMargin = 64
|
||||||
|
/// Share of the context window tool definitions may take before tools are routed.
|
||||||
|
private static let toolBudgetFraction = 0.4
|
||||||
|
/// Most tools a routing request picks.
|
||||||
|
private static let maxRoutedTools = 5
|
||||||
|
/// Tools used in the latest exchanges stay offered, for follow-ups ("make it ten minutes").
|
||||||
|
private static let keptRecentTools = 2
|
||||||
|
|
||||||
|
private let emit: ([String: Any]) -> Void
|
||||||
|
private var task: Task<Void, Never>?
|
||||||
|
// Calls not yet reported, then reported calls awaiting results.
|
||||||
|
private var unreported: [PendingCall] = []
|
||||||
|
private var awaiting: [String: PendingCall] = [:]
|
||||||
|
private var flushScheduled = false
|
||||||
|
private var closed = false
|
||||||
|
|
||||||
|
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
|
||||||
|
|
||||||
|
func start(_ request: [String: Any]) {
|
||||||
|
task = Task { [weak self] in await self?.run(request) }
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolve(_ results: [[String: Any]]) {
|
||||||
|
for result in results {
|
||||||
|
guard let id = result["id"] as? String, let call = awaiting.removeValue(forKey: id) else { continue }
|
||||||
|
let content = (result["content"] as? String) ?? ""
|
||||||
|
let isError = (result["isError"] as? Bool) ?? false
|
||||||
|
// An error goes back to the model as text: a thrown error would end the whole response.
|
||||||
|
call.continuation.resume(returning: isError ? "Error: \(content)" : (content.isEmpty ? "(no output)" : content))
|
||||||
|
}
|
||||||
|
// A call JavaScript didn't answer can't be left hanging.
|
||||||
|
for (_, call) in awaiting {
|
||||||
|
call.continuation.resume(returning: "Error: the tool returned no result")
|
||||||
|
}
|
||||||
|
awaiting.removeAll()
|
||||||
|
}
|
||||||
|
|
||||||
|
func cancel() {
|
||||||
|
closed = true
|
||||||
|
task?.cancel()
|
||||||
|
task = nil
|
||||||
|
for call in unreported { call.continuation.resume(throwing: CancellationError()) }
|
||||||
|
for (_, call) in awaiting { call.continuation.resume(throwing: CancellationError()) }
|
||||||
|
unreported.removeAll()
|
||||||
|
awaiting.removeAll()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Events stop once the session is cancelled, even from work still winding down.
|
||||||
|
private func send(_ event: [String: Any]) {
|
||||||
|
if !closed { emit(event) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private func run(_ request: [String: Any]) async {
|
||||||
|
let system = (request["system"] as? String) ?? ""
|
||||||
|
let prompt = (request["prompt"] as? String) ?? ""
|
||||||
|
let maxTokens = (request["maxTokens"] as? NSNumber)?.intValue ?? 512
|
||||||
|
let permissive = (request["permissive"] as? Bool) ?? false
|
||||||
|
let toolSpecs: [ToolSpec] = ((request["tools"] as? [[String: Any]]) ?? []).compactMap { tool in
|
||||||
|
guard let name = tool["name"] as? String else { return nil }
|
||||||
|
return ToolSpec(name: name, description: (tool["description"] as? String) ?? "",
|
||||||
|
parameters: (tool["parameters"] as? [String: Any]) ?? [:])
|
||||||
|
}
|
||||||
|
let history = (request["history"] as? [[String: Any]]) ?? []
|
||||||
|
|
||||||
|
// Transformations of user text (translate, summarize) get the permissive
|
||||||
|
// guardrails; they apply to plain string replies, which tool-free requests are.
|
||||||
|
let model = SystemLanguageModel(useCase: .general,
|
||||||
|
guardrails: permissive ? .permissiveContentTransformations : .default)
|
||||||
|
var offered = toolSpecs
|
||||||
|
if !toolSpecs.isEmpty {
|
||||||
|
switch await selectTools(model: model, specs: toolSpecs, history: history, prompt: prompt) {
|
||||||
|
case .tools(let specs): offered = specs
|
||||||
|
case .failed(let event):
|
||||||
|
send(event)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if closed { return }
|
||||||
|
}
|
||||||
|
let fitted = await fit(model: model, system: system, tools: offered, history: history,
|
||||||
|
prompt: prompt, maxTokens: maxTokens)
|
||||||
|
if closed { return }
|
||||||
|
|
||||||
|
let handler: @Sendable (String, String) async throws -> String = { [weak self] name, arguments in
|
||||||
|
guard let self = self else { throw CancellationError() }
|
||||||
|
return try await self.toolInvoked(name: name, arguments: arguments)
|
||||||
|
}
|
||||||
|
let tools: [any Tool] = fitted.tools.map {
|
||||||
|
BridgedTool(name: $0.name, description: $0.description, parameters: $0.schema, handler: handler)
|
||||||
|
}
|
||||||
|
let session = LanguageModelSession(model: model, tools: tools, transcript: Transcript(entries: fitted.entries))
|
||||||
|
let options = GenerationOptions(maximumResponseTokens: maxTokens)
|
||||||
|
var last = ""
|
||||||
|
do {
|
||||||
|
for try await snapshot in session.streamResponse(to: prompt, options: options) {
|
||||||
|
if closed { return }
|
||||||
|
let text = snapshot.content
|
||||||
|
// Snapshots are cumulative; one that doesn't extend the last is a new segment.
|
||||||
|
let delta = text.hasPrefix(last) ? String(text.dropFirst(last.count)) : text
|
||||||
|
last = text
|
||||||
|
if !delta.isEmpty { send(["kind": "delta", "text": delta]) }
|
||||||
|
}
|
||||||
|
send(["kind": "done"])
|
||||||
|
} catch {
|
||||||
|
if closed || error is CancellationError { return }
|
||||||
|
send(describe(error))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Tool calls
|
||||||
|
|
||||||
|
private func toolInvoked(name: String, arguments: String) async throws -> String {
|
||||||
|
if closed { throw CancellationError() }
|
||||||
|
return try await withCheckedThrowingContinuation { continuation in
|
||||||
|
if closed {
|
||||||
|
continuation.resume(throwing: CancellationError())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
let id = "fm_" + UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(16).lowercased()
|
||||||
|
unreported.append(PendingCall(id: id, name: name, arguments: arguments, continuation: continuation))
|
||||||
|
scheduleFlush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Report calls in one event, after a moment for any parallel calls to arrive.
|
||||||
|
private func scheduleFlush() {
|
||||||
|
if flushScheduled { return }
|
||||||
|
flushScheduled = true
|
||||||
|
Task { [weak self] in
|
||||||
|
try? await Task.sleep(nanoseconds: LanguageModelEngine.toolBatchDelayNs)
|
||||||
|
self?.flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func flush() {
|
||||||
|
flushScheduled = false
|
||||||
|
if closed || unreported.isEmpty { return }
|
||||||
|
let calls = unreported
|
||||||
|
unreported.removeAll()
|
||||||
|
for call in calls { awaiting[call.id] = call }
|
||||||
|
send(["kind": "toolCalls",
|
||||||
|
"calls": calls.map { ["id": $0.id, "name": $0.name, "arguments": $0.arguments] }])
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Tool routing
|
||||||
|
|
||||||
|
private enum Selection {
|
||||||
|
case tools([ToolSpec])
|
||||||
|
case failed([String: Any])
|
||||||
|
}
|
||||||
|
|
||||||
|
/// All tools when their definitions fit their share of the context window;
|
||||||
|
/// otherwise the few the model picks for this request, plus recently used ones.
|
||||||
|
private func selectTools(model: SystemLanguageModel, specs: [ToolSpec], history: [[String: Any]],
|
||||||
|
prompt: String) async -> Selection {
|
||||||
|
let budget = Int(Double(model.contextSize) * LanguageModelEngine.toolBudgetFraction)
|
||||||
|
let cost = await toolTokens(model, buildTools(specs, compact: false))
|
||||||
|
if cost <= budget { return .tools(specs) }
|
||||||
|
|
||||||
|
let names = specs.map { $0.name }
|
||||||
|
let catalog = specs.map { "- \($0.name): \(SchemaBuilder.shorten($0.description, sentences: 1, limit: 100))" }
|
||||||
|
.joined(separator: "\n")
|
||||||
|
let instructions = "You pick the tools a voice assistant needs to handle the user's latest request. " +
|
||||||
|
"Choose at most \(LanguageModelEngine.maxRoutedTools) tools from this list that the assistant should be able to use, " +
|
||||||
|
"most important first. Choose none for small talk or questions it can answer from general knowledge.\n\n" + catalog
|
||||||
|
var query = prompt
|
||||||
|
if let previous = lastPromptText(history) {
|
||||||
|
query = "Previous request: \(SchemaBuilder.shorten(previous, sentences: 3, limit: 300))\n\nLatest request: \(prompt)"
|
||||||
|
}
|
||||||
|
do {
|
||||||
|
let schema = try GenerationSchema(root: DynamicGenerationSchema(name: "ToolSelection", properties: [
|
||||||
|
DynamicGenerationSchema.Property(
|
||||||
|
name: "tools", description: "Names of the tools needed, most important first.",
|
||||||
|
schema: DynamicGenerationSchema(arrayOf: DynamicGenerationSchema(type: String.self, guides: [.anyOf(names)]),
|
||||||
|
minimumElements: 0,
|
||||||
|
maximumElements: LanguageModelEngine.maxRoutedTools)),
|
||||||
|
]), dependencies: [])
|
||||||
|
let session = LanguageModelSession(model: model, instructions: instructions)
|
||||||
|
let response = try await session.respond(to: query, schema: schema,
|
||||||
|
options: GenerationOptions(sampling: .greedy, maximumResponseTokens: 100))
|
||||||
|
var chosen = Set(try response.content.value([String].self, forProperty: "tools"))
|
||||||
|
let recent = recentToolNames(history).prefix(LanguageModelEngine.keptRecentTools)
|
||||||
|
chosen.formUnion(recent)
|
||||||
|
let selected = specs.filter { chosen.contains($0.name) }
|
||||||
|
send(["kind": "log", "message": "routed \(specs.count) tools (\(cost) tokens) to: " +
|
||||||
|
(selected.isEmpty ? "none" : selected.map { $0.name }.joined(separator: ", "))])
|
||||||
|
return .tools(selected)
|
||||||
|
} catch {
|
||||||
|
if let generationError = error as? LanguageModelSession.GenerationError {
|
||||||
|
switch generationError {
|
||||||
|
case .rateLimited, .concurrentRequests, .assetsUnavailable, .guardrailViolation, .refusal,
|
||||||
|
.unsupportedLanguageOrLocale:
|
||||||
|
// The real request would fail the same way.
|
||||||
|
return .failed(describe(error))
|
||||||
|
default:
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
send(["kind": "log", "message": "tool routing failed (\(error.localizedDescription)); offering tools in order"])
|
||||||
|
return .tools(specs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func toolTokens(_ model: SystemLanguageModel, _ tools: [BuiltTool]) async -> Int {
|
||||||
|
let definitions = tools.map {
|
||||||
|
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
|
||||||
|
}
|
||||||
|
let instructions = Transcript.Entry.instructions(Transcript.Instructions(segments: [], toolDefinitions: definitions))
|
||||||
|
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
|
||||||
|
return await tokenCount(model, [instructions], extraCharacters: schemaCharacters + 150 * tools.count)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func lastPromptText(_ history: [[String: Any]]) -> String? {
|
||||||
|
for item in history.reversed() where item["type"] as? String == "prompt" {
|
||||||
|
if let text = item["text"] as? String, !text.isEmpty { return text }
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tool names from the newest tool calls back, without repeats.
|
||||||
|
private func recentToolNames(_ history: [[String: Any]]) -> [String] {
|
||||||
|
var names: [String] = []
|
||||||
|
for item in history.reversed() where item["type"] as? String == "toolCalls" {
|
||||||
|
for call in (item["calls"] as? [[String: Any]]) ?? [] {
|
||||||
|
if let name = call["name"] as? String, !names.contains(name) { names.append(name) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Context fitting
|
||||||
|
|
||||||
|
private struct BuiltTool {
|
||||||
|
let name: String
|
||||||
|
let description: String
|
||||||
|
let schema: GenerationSchema
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct Fitted {
|
||||||
|
let tools: [BuiltTool]
|
||||||
|
let entries: [Transcript.Entry]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Instructions + tool definitions + history + prompt must leave the reply
|
||||||
|
/// budget free. In order, until it fits: shorten tool descriptions, drop the
|
||||||
|
/// oldest exchanges, then drop the last-registered tools.
|
||||||
|
private func fit(model: SystemLanguageModel, system: String, tools specs: [ToolSpec],
|
||||||
|
history: [[String: Any]], prompt: String, maxTokens: Int) async -> Fitted {
|
||||||
|
let contextSize = model.contextSize
|
||||||
|
let reserve = min(maxTokens, contextSize / 3) + LanguageModelEngine.contextMargin
|
||||||
|
var tools = buildTools(specs, compact: false)
|
||||||
|
var past = transcriptEntries(history)
|
||||||
|
let promptEntry = Transcript.Entry.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: prompt))]))
|
||||||
|
|
||||||
|
func entries() -> [Transcript.Entry] {
|
||||||
|
let definitions = tools.map {
|
||||||
|
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
|
||||||
|
}
|
||||||
|
let instructions = Transcript.Instructions(segments: [.text(Transcript.TextSegment(content: system))],
|
||||||
|
toolDefinitions: definitions)
|
||||||
|
return [.instructions(instructions)] + past
|
||||||
|
}
|
||||||
|
func overBudget() async -> Bool {
|
||||||
|
// The tokenizer can't see tool schemas' size from a ToolDefinition; the fallback estimate adds them.
|
||||||
|
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
|
||||||
|
let used = await tokenCount(model, entries() + [promptEntry], extraCharacters: schemaCharacters)
|
||||||
|
return used + reserve > contextSize
|
||||||
|
}
|
||||||
|
|
||||||
|
if !tools.isEmpty, await overBudget() {
|
||||||
|
tools = buildTools(specs, compact: true)
|
||||||
|
send(["kind": "log", "message": "shortened tool descriptions to fit the context window"])
|
||||||
|
}
|
||||||
|
var droppedEntries = 0
|
||||||
|
while !past.isEmpty, await overBudget() {
|
||||||
|
// Drop the oldest exchange: through to the next user prompt.
|
||||||
|
past.removeFirst()
|
||||||
|
droppedEntries += 1
|
||||||
|
while let first = past.first, !isPrompt(first) {
|
||||||
|
past.removeFirst()
|
||||||
|
droppedEntries += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if droppedEntries > 0 {
|
||||||
|
send(["kind": "log", "message": "dropped \(droppedEntries) history entries to fit the context window"])
|
||||||
|
}
|
||||||
|
var droppedTools: [String] = []
|
||||||
|
while !tools.isEmpty, await overBudget() {
|
||||||
|
droppedTools.append(tools.removeLast().name)
|
||||||
|
}
|
||||||
|
if !droppedTools.isEmpty {
|
||||||
|
send(["kind": "log", "message": "dropped tools to fit the context window: \(droppedTools.joined(separator: ", "))"])
|
||||||
|
}
|
||||||
|
return Fitted(tools: tools, entries: entries())
|
||||||
|
}
|
||||||
|
|
||||||
|
private func tokenCount(_ model: SystemLanguageModel, _ entries: [Transcript.Entry], extraCharacters: Int) async -> Int {
|
||||||
|
if #available(iOS 26.4, macOS 26.4, *) {
|
||||||
|
if let count = try? await model.tokenCount(for: entries) { return count }
|
||||||
|
}
|
||||||
|
// Before iOS 26.4 there's no tokenizer API: estimate conservatively.
|
||||||
|
var characters = extraCharacters
|
||||||
|
for entry in entries { characters += estimateCharacters(entry) + 16 }
|
||||||
|
return characters / 3
|
||||||
|
}
|
||||||
|
|
||||||
|
private func estimateCharacters(_ entry: Transcript.Entry) -> Int {
|
||||||
|
func segments(_ list: [Transcript.Segment]) -> Int {
|
||||||
|
list.reduce(0) { total, segment in
|
||||||
|
switch segment {
|
||||||
|
case .text(let text): return total + text.content.count
|
||||||
|
case .structure(let structure): return total + structure.content.jsonString.count
|
||||||
|
@unknown default: return total
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch entry {
|
||||||
|
case .instructions(let instructions):
|
||||||
|
var total = segments(instructions.segments)
|
||||||
|
for definition in instructions.toolDefinitions {
|
||||||
|
total += definition.name.count + definition.description.count
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
case .prompt(let prompt): return segments(prompt.segments)
|
||||||
|
case .response(let response): return segments(response.segments)
|
||||||
|
case .toolCalls(let calls): return calls.reduce(0) { $0 + $1.toolName.count + $1.arguments.jsonString.count }
|
||||||
|
case .toolOutput(let output): return segments(output.segments)
|
||||||
|
@unknown default: return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func isPrompt(_ entry: Transcript.Entry) -> Bool {
|
||||||
|
if case .prompt = entry { return true }
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Request conversion
|
||||||
|
|
||||||
|
private func transcriptEntries(_ history: [[String: Any]]) -> [Transcript.Entry] {
|
||||||
|
var entries: [Transcript.Entry] = []
|
||||||
|
for item in history {
|
||||||
|
let text = (item["text"] as? String) ?? ""
|
||||||
|
switch item["type"] as? String {
|
||||||
|
case "prompt":
|
||||||
|
entries.append(.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: text))])))
|
||||||
|
case "response":
|
||||||
|
entries.append(.response(Transcript.Response(assetIDs: [], segments: [.text(Transcript.TextSegment(content: text))])))
|
||||||
|
case "toolCalls":
|
||||||
|
let calls: [Transcript.ToolCall] = ((item["calls"] as? [[String: Any]]) ?? []).compactMap { call in
|
||||||
|
guard let id = call["id"] as? String, let name = call["name"] as? String else { return nil }
|
||||||
|
let arguments = (try? GeneratedContent(json: (call["arguments"] as? String) ?? "{}"))
|
||||||
|
?? GeneratedContent(properties: [:])
|
||||||
|
return Transcript.ToolCall(id: id, toolName: name, arguments: arguments)
|
||||||
|
}
|
||||||
|
if !calls.isEmpty { entries.append(.toolCalls(Transcript.ToolCalls(calls))) }
|
||||||
|
case "toolOutput":
|
||||||
|
guard let id = item["id"] as? String else { continue }
|
||||||
|
entries.append(.toolOutput(Transcript.ToolOutput(
|
||||||
|
id: id, toolName: (item["name"] as? String) ?? "tool",
|
||||||
|
segments: [.text(Transcript.TextSegment(content: text))])))
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return entries
|
||||||
|
}
|
||||||
|
|
||||||
|
private func buildTools(_ specs: [ToolSpec], compact: Bool) -> [BuiltTool] {
|
||||||
|
specs.map { spec in
|
||||||
|
var builder = SchemaBuilder(compact: compact)
|
||||||
|
let schema: GenerationSchema
|
||||||
|
do {
|
||||||
|
schema = try GenerationSchema(root: builder.object(spec.parameters, name: "\(spec.name)_arguments"),
|
||||||
|
dependencies: [])
|
||||||
|
} catch {
|
||||||
|
send(["kind": "log", "message": "tool \(spec.name): unusable parameter schema (\(error)); offering it without parameters"])
|
||||||
|
schema = (try? GenerationSchema(root: DynamicGenerationSchema(name: "\(spec.name)_arguments", properties: []),
|
||||||
|
dependencies: []))
|
||||||
|
?? GenerationSchema(type: GeneratedContent.self, properties: [])
|
||||||
|
}
|
||||||
|
let description = compact ? SchemaBuilder.shorten(spec.description, sentences: 1, limit: 160) : spec.description
|
||||||
|
return BuiltTool(name: spec.name, description: description, schema: schema)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Errors
|
||||||
|
|
||||||
|
private func describe(_ error: Error) -> [String: Any] {
|
||||||
|
if let error = error as? LanguageModelSession.GenerationError {
|
||||||
|
switch error {
|
||||||
|
case .exceededContextWindowSize:
|
||||||
|
return failure("context", "That conversation is too long for the on-device model. Start a new conversation and try again.")
|
||||||
|
case .assetsUnavailable:
|
||||||
|
return failure("unavailable", "The Apple Intelligence model isn't available right now. Check that Apple Intelligence is on and finished downloading.")
|
||||||
|
case .guardrailViolation, .refusal:
|
||||||
|
return failure("refusal", "The on-device model declined this request.")
|
||||||
|
case .unsupportedLanguageOrLocale:
|
||||||
|
return failure("language", "The on-device model doesn't support this language.")
|
||||||
|
case .rateLimited:
|
||||||
|
return failure("rateLimited", "iOS is limiting the on-device model while Faceclaw is in the background. Try again in a moment, or with the phone unlocked.")
|
||||||
|
case .concurrentRequests:
|
||||||
|
return failure("busy", "The on-device model is busy with another request. Try again in a moment.")
|
||||||
|
case .decodingFailure, .unsupportedGuide:
|
||||||
|
return failure("other", "The on-device model produced an unreadable reply: \(error.localizedDescription)")
|
||||||
|
@unknown default:
|
||||||
|
return failure("other", "On-device model failed: \(error.localizedDescription)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let error = error as? LanguageModelSession.ToolCallError {
|
||||||
|
return failure("other", "On-device model tool call failed: \(error.underlyingError.localizedDescription)")
|
||||||
|
}
|
||||||
|
return failure("other", "On-device model failed: \(error.localizedDescription)")
|
||||||
|
}
|
||||||
|
|
||||||
|
private func failure(_ code: String, _ message: String) -> [String: Any] {
|
||||||
|
["kind": "error", "code": code, "message": message]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON Schema (the subset our tool definitions use: objects of string, number,
|
||||||
|
/// integer, boolean, enum and array properties) to a DynamicGenerationSchema, so
|
||||||
|
/// the framework constrains tool arguments to valid values while decoding.
|
||||||
|
/// Anything it doesn't understand becomes a string rather than failing.
|
||||||
|
@available(iOS 26.0, *)
|
||||||
|
private struct SchemaBuilder {
|
||||||
|
let compact: Bool
|
||||||
|
private var names = Set<String>()
|
||||||
|
|
||||||
|
init(compact: Bool) { self.compact = compact }
|
||||||
|
|
||||||
|
mutating func object(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
|
||||||
|
let properties = (json["properties"] as? [String: Any]) ?? [:]
|
||||||
|
let required = Set((json["required"] as? [String]) ?? [])
|
||||||
|
// JSON objects lose declaration order; keep required properties first, then by name.
|
||||||
|
let keys = properties.keys.sorted { a, b in
|
||||||
|
let ra = required.contains(a), rb = required.contains(b)
|
||||||
|
return ra != rb ? ra : a < b
|
||||||
|
}
|
||||||
|
let members: [DynamicGenerationSchema.Property] = keys.compactMap { key in
|
||||||
|
guard let property = properties[key] as? [String: Any] else { return nil }
|
||||||
|
return DynamicGenerationSchema.Property(
|
||||||
|
name: key,
|
||||||
|
description: describe(property),
|
||||||
|
schema: value(property, name: "\(name)_\(key)"),
|
||||||
|
isOptional: !required.contains(key))
|
||||||
|
}
|
||||||
|
return DynamicGenerationSchema(name: unique(name), description: nil, properties: members)
|
||||||
|
}
|
||||||
|
|
||||||
|
private mutating func value(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
|
||||||
|
if let choices = json["enum"] as? [Any], !choices.isEmpty {
|
||||||
|
let strings = choices.compactMap { $0 as? String }
|
||||||
|
if strings.count == choices.count {
|
||||||
|
// A guide renders as one compact JSON Schema enum; an anyOf schema lists
|
||||||
|
// every choice as its own subschema, at over twice the tokens.
|
||||||
|
return DynamicGenerationSchema(type: String.self, guides: [.anyOf(strings)])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch schemaType(json) {
|
||||||
|
case "integer":
|
||||||
|
var guides: [GenerationGuide<Int>] = []
|
||||||
|
if let minimum = (json["minimum"] as? NSNumber)?.intValue { guides.append(.minimum(minimum)) }
|
||||||
|
if let maximum = (json["maximum"] as? NSNumber)?.intValue { guides.append(.maximum(maximum)) }
|
||||||
|
return DynamicGenerationSchema(type: Int.self, guides: guides)
|
||||||
|
case "number":
|
||||||
|
var guides: [GenerationGuide<Double>] = []
|
||||||
|
if let minimum = (json["minimum"] as? NSNumber)?.doubleValue { guides.append(.minimum(minimum)) }
|
||||||
|
if let maximum = (json["maximum"] as? NSNumber)?.doubleValue { guides.append(.maximum(maximum)) }
|
||||||
|
return DynamicGenerationSchema(type: Double.self, guides: guides)
|
||||||
|
case "boolean":
|
||||||
|
return DynamicGenerationSchema(type: Bool.self)
|
||||||
|
case "array":
|
||||||
|
let items = (json["items"] as? [String: Any]) ?? ["type": "string"]
|
||||||
|
return DynamicGenerationSchema(arrayOf: value(items, name: "\(name)_item"),
|
||||||
|
minimumElements: (json["minItems"] as? NSNumber)?.intValue,
|
||||||
|
maximumElements: (json["maxItems"] as? NSNumber)?.intValue)
|
||||||
|
case "object":
|
||||||
|
return object(json, name: name)
|
||||||
|
default:
|
||||||
|
return DynamicGenerationSchema(type: String.self)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func schemaType(_ json: [String: Any]) -> String {
|
||||||
|
if let type = json["type"] as? String { return type }
|
||||||
|
if let types = json["type"] as? [String], let type = types.first(where: { $0 != "null" }) { return type }
|
||||||
|
return json["properties"] != nil ? "object" : "string"
|
||||||
|
}
|
||||||
|
|
||||||
|
private func describe(_ json: [String: Any]) -> String? {
|
||||||
|
guard let text = json["description"] as? String, !text.isEmpty else { return nil }
|
||||||
|
return compact ? SchemaBuilder.shorten(text, sentences: 1, limit: 80) : text
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Schema names must be unique within one GenerationSchema.
|
||||||
|
private mutating func unique(_ base: String) -> String {
|
||||||
|
var name = base
|
||||||
|
var suffix = 2
|
||||||
|
while names.contains(name) {
|
||||||
|
name = "\(base)_\(suffix)"
|
||||||
|
suffix += 1
|
||||||
|
}
|
||||||
|
names.insert(name)
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The first `sentences` sentences, cut to `limit` characters.
|
||||||
|
static func shorten(_ text: String, sentences: Int, limit: Int) -> String {
|
||||||
|
var result = ""
|
||||||
|
var count = 0
|
||||||
|
var remainder = Substring(text)
|
||||||
|
while count < sentences, !remainder.isEmpty {
|
||||||
|
if let end = remainder.range(of: ". ") {
|
||||||
|
result += remainder[..<end.lowerBound] + "."
|
||||||
|
remainder = remainder[end.upperBound...]
|
||||||
|
if count + 1 < sentences { result += " " }
|
||||||
|
} else {
|
||||||
|
result += remainder
|
||||||
|
remainder = ""
|
||||||
|
}
|
||||||
|
count += 1
|
||||||
|
}
|
||||||
|
if result.count > limit { result = String(result.prefix(limit - 1)) + "…" }
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -7,6 +7,7 @@
|
|||||||
* Ensure connection is re-set to 2M on reconnect
|
* Ensure connection is re-set to 2M on reconnect
|
||||||
* Add support for embedded icons and custom layouts in Android notifications
|
* Add support for embedded icons and custom layouts in Android notifications
|
||||||
* Port the Developer app's resource-usage monitor and various test pages to iOS
|
* Port the Developer app's resource-usage monitor and various test pages to iOS
|
||||||
|
* iOS: Support Apple Intelligence as an on-device model for voice assistant/etc
|
||||||
* iOS: Notifications are selectable via the task switcher
|
* iOS: Notifications are selectable via the task switcher
|
||||||
* Android: Optionally sign in to Spotify's Web API for library browsing and other features
|
* Android: Optionally sign in to Spotify's Web API for library browsing and other features
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
* surfacing a transport error.
|
* surfacing a transport error.
|
||||||
*/
|
*/
|
||||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||||
import { streamLocalQwen } from "../../native/llama";
|
import { streamLocalModel } from "../../native/local-llm";
|
||||||
import { streamOpenAiResponse } from "../../native/openai";
|
import { streamOpenAiResponse } from "../../native/openai";
|
||||||
import { resolveAssistantModel } from "../../assistant/models";
|
import { resolveAssistantModel } from "../../assistant/models";
|
||||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||||
@@ -41,7 +41,7 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
|||||||
streamOpenAiResponse(options);
|
streamOpenAiResponse(options);
|
||||||
break;
|
break;
|
||||||
case "local":
|
case "local":
|
||||||
streamLocalQwen(options);
|
streamLocalModel(options);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
streamAnthropicMessage(options);
|
streamAnthropicMessage(options);
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { Utils } from "@nativescript/core";
|
import { Utils } from "@nativescript/core";
|
||||||
|
|
||||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||||
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||||
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
||||||
|
|
||||||
@@ -245,7 +245,7 @@ export function askConversations(
|
|||||||
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
||||||
return { cancel: () => {} };
|
return { cancel: () => {} };
|
||||||
}
|
}
|
||||||
return streamLocalQwen({
|
return streamLocalModel({
|
||||||
apiKey: "",
|
apiKey: "",
|
||||||
model: "local",
|
model: "local",
|
||||||
system:
|
system:
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||||
import type { NameInference } from "./name-guess";
|
import type { NameInference } from "./name-guess";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
|
|||||||
};
|
};
|
||||||
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
||||||
try {
|
try {
|
||||||
streamLocalQwen({
|
streamLocalModel({
|
||||||
apiKey: "",
|
apiKey: "",
|
||||||
model: "local",
|
model: "local",
|
||||||
system:
|
system:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { Utils } from "@nativescript/core";
|
import { Utils } from "@nativescript/core";
|
||||||
|
|
||||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||||
import { speakerRegistry } from "./speakers";
|
import { speakerRegistry } from "./speakers";
|
||||||
|
|
||||||
declare const com: any;
|
declare const com: any;
|
||||||
@@ -124,7 +124,7 @@ function generateForSpeaker(
|
|||||||
};
|
};
|
||||||
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
||||||
try {
|
try {
|
||||||
streamLocalQwen({
|
streamLocalModel({
|
||||||
apiKey: "",
|
apiKey: "",
|
||||||
model: "local",
|
model: "local",
|
||||||
system:
|
system:
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
* TranslationModel for how stretches are batched and given context.
|
* TranslationModel for how stretches are batched and given context.
|
||||||
*/
|
*/
|
||||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||||
import { streamLocalQwen } from "../../native/llama";
|
import { streamLocalModel } from "../../native/local-llm";
|
||||||
import { streamOpenAiResponse } from "../../native/openai";
|
import { streamOpenAiResponse } from "../../native/openai";
|
||||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||||
@@ -55,7 +55,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
|||||||
case "openai":
|
case "openai":
|
||||||
return streamOpenAiResponse(options);
|
return streamOpenAiResponse(options);
|
||||||
case "local":
|
case "local":
|
||||||
return streamLocalQwen(options);
|
return streamLocalModel(options);
|
||||||
default:
|
default:
|
||||||
return streamAnthropicMessage(options);
|
return streamAnthropicMessage(options);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { streamAnthropicMessage } from "../native/anthropic";
|
import { streamAnthropicMessage } from "../native/anthropic";
|
||||||
import { streamHermesChat } from "../native/hermes";
|
import { streamHermesChat } from "../native/hermes";
|
||||||
import { streamLocalQwen } from "../native/llama";
|
import { streamLocalModel } from "../native/local-llm";
|
||||||
import { streamOpenAiResponse } from "../native/openai";
|
import { streamOpenAiResponse } from "../native/openai";
|
||||||
import type {
|
import type {
|
||||||
LlmContentBlock,
|
LlmContentBlock,
|
||||||
@@ -146,7 +146,7 @@ export class DirectAssistantBackend {
|
|||||||
: options.provider === "openai"
|
: options.provider === "openai"
|
||||||
? streamOpenAiResponse(streamOptions)
|
? streamOpenAiResponse(streamOptions)
|
||||||
: options.provider === "local"
|
: options.provider === "local"
|
||||||
? streamLocalQwen(streamOptions)
|
? streamLocalModel(streamOptions)
|
||||||
: streamAnthropicMessage(streamOptions);
|
: streamAnthropicMessage(streamOptions);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
+13
-7
@@ -1,4 +1,4 @@
|
|||||||
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
|
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
|
||||||
|
|
||||||
export const ASSISTANT_MODEL_VALUES = [
|
export const ASSISTANT_MODEL_VALUES = [
|
||||||
"auto",
|
"auto",
|
||||||
@@ -13,12 +13,17 @@ export const ASSISTANT_MODEL_VALUES = [
|
|||||||
"hermes",
|
"hermes",
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
|
||||||
|
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
|
||||||
|
* support and stays for saved settings and conversations).
|
||||||
|
*/
|
||||||
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
|
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
|
||||||
/** Keep saved model IDs portable, while offering only supported choices. */
|
/** Keep saved model IDs portable, while offering only supported choices. */
|
||||||
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
|
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
|
||||||
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
|
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
|
||||||
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
|
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
|
||||||
return global.isIOS && value === "qwen" ? "auto" : value;
|
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
|
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
|
||||||
@@ -58,8 +63,9 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
|
|||||||
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
|
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
|
||||||
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
|
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
|
||||||
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
|
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
|
||||||
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
|
// Runs on the phone itself (Android: llama.cpp with the downloaded model
|
||||||
// not an API key. The default for users who haven't set any key.
|
// file; iOS: Apple Intelligence), not an API key. The default for users who
|
||||||
|
// haven't set any key.
|
||||||
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
|
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
|
||||||
// Your own Hermes Agent (its API server), with its own tools and memory.
|
// Your own Hermes Agent (its API server), with its own tools and memory.
|
||||||
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
|
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
|
||||||
@@ -76,8 +82,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
|
|||||||
/**
|
/**
|
||||||
* Resolve Auto and attach the credential needed by the chosen provider.
|
* Resolve Auto and attach the credential needed by the chosen provider.
|
||||||
* Auto intentionally prefers Terra over Sonnet when both keys are available,
|
* Auto intentionally prefers Terra over Sonnet when both keys are available,
|
||||||
* and falls back to the on-phone model when no key is set but the model file
|
* and falls back to the on-phone model when no key is set but that model is
|
||||||
* has been downloaded.
|
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
|
||||||
*/
|
*/
|
||||||
export function resolveAssistantModel(
|
export function resolveAssistantModel(
|
||||||
selection: AssistantModel,
|
selection: AssistantModel,
|
||||||
|
|||||||
+4
-6
@@ -6,12 +6,14 @@ import type {
|
|||||||
LlmStreamOptions,
|
LlmStreamOptions,
|
||||||
} from "../assistant/llm-protocol";
|
} from "../assistant/llm-protocol";
|
||||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||||
|
import type { LocalModelState } from "./local-llm-types";
|
||||||
|
|
||||||
declare const com: any;
|
declare const com: any;
|
||||||
declare const java: any;
|
declare const java: any;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* On-phone LLM backend: model download management plus a streaming client
|
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
|
||||||
|
* local-llm.ios.ts instead): model download management plus a streaming client
|
||||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||||
* the direct assistant loop can treat "local" as just another provider.
|
* the direct assistant loop can treat "local" as just another provider.
|
||||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||||
@@ -38,11 +40,7 @@ const TOP_K = 20;
|
|||||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||||
|
|
||||||
export type LocalModelState = {
|
export type { LocalModelState } from "./local-llm-types";
|
||||||
status: "absent" | "downloading" | "ready";
|
|
||||||
bytesDownloaded: number;
|
|
||||||
totalBytes: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
let runner: any = null;
|
let runner: any = null;
|
||||||
let downloader: any = null;
|
let downloader: any = null;
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
|
||||||
|
|
||||||
|
export type LocalModelState = {
|
||||||
|
/**
|
||||||
|
* Android downloads its model: absent → downloading → ready. iOS uses the
|
||||||
|
* system's Apple Intelligence model: ready, or unavailable (see detail).
|
||||||
|
*/
|
||||||
|
status: "absent" | "downloading" | "ready" | "unavailable";
|
||||||
|
bytesDownloaded: number;
|
||||||
|
totalBytes: number;
|
||||||
|
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
|
||||||
|
detail?: string;
|
||||||
|
};
|
||||||
@@ -0,0 +1,383 @@
|
|||||||
|
import type {
|
||||||
|
LlmContentBlock,
|
||||||
|
LlmMessage,
|
||||||
|
LlmStreamHandle,
|
||||||
|
LlmStreamOptions,
|
||||||
|
} from "../assistant/llm-protocol";
|
||||||
|
import type { LocalModelState } from "./local-llm-types";
|
||||||
|
export type { LocalModelState } from "./local-llm-types";
|
||||||
|
|
||||||
|
declare const FaceclawSystemLanguageModel: any;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
|
||||||
|
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
|
||||||
|
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
|
||||||
|
* Calculator and the Microphones helpers treat it as the "local" provider.
|
||||||
|
*
|
||||||
|
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
|
||||||
|
* background (where Faceclaw usually is, driving the glasses with the phone
|
||||||
|
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
|
||||||
|
* ~3 GB in-process model makes the app the first thing jetsam kills. The
|
||||||
|
* system model runs out of process on the Neural Engine, needs no download,
|
||||||
|
* and decodes tool arguments under a schema constraint (like our GBNF grammar
|
||||||
|
* on Android). Its costs: a 4096-token context (the Swift side fits history
|
||||||
|
* and tool definitions into it), Apple's guardrails, a background request
|
||||||
|
* budget, and Apple Intelligence-capable hardware.
|
||||||
|
*
|
||||||
|
* Tool calls: the framework runs tools itself, mid-response. The Swift side
|
||||||
|
* parks each call and reports it; here that ends the loop iteration with
|
||||||
|
* stopReason "tool_use", the direct backend executes the tools as for any
|
||||||
|
* provider, and its next request (ending in the matching tool_result blocks)
|
||||||
|
* resumes the parked generation instead of starting a new one.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const LOCAL_MODEL = {
|
||||||
|
id: "apple-foundation-model",
|
||||||
|
label: "Apple Intelligence",
|
||||||
|
sizeBytes: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_MAX_TOKENS = 512;
|
||||||
|
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
|
||||||
|
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
|
||||||
|
/**
|
||||||
|
* Only used when tool results arrive with no parked generation to resume (it
|
||||||
|
* timed out, or the app restarted mid-turn): the framework has no way to
|
||||||
|
* continue after a tool output without a new prompt.
|
||||||
|
*/
|
||||||
|
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
|
||||||
|
|
||||||
|
type Availability =
|
||||||
|
| "available"
|
||||||
|
| "unsupportedOS"
|
||||||
|
| "deviceNotEligible"
|
||||||
|
| "appleIntelligenceNotEnabled"
|
||||||
|
| "modelNotReady"
|
||||||
|
| "unavailable";
|
||||||
|
|
||||||
|
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
|
||||||
|
unsupportedOS: "needs iOS 26",
|
||||||
|
deviceNotEligible: "not on this iPhone",
|
||||||
|
appleIntelligenceNotEnabled: "Apple Intelligence off",
|
||||||
|
modelNotReady: "downloading",
|
||||||
|
unavailable: "unavailable",
|
||||||
|
};
|
||||||
|
|
||||||
|
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
|
||||||
|
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
|
||||||
|
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
|
||||||
|
appleIntelligenceNotEnabled:
|
||||||
|
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
|
||||||
|
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
|
||||||
|
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
|
||||||
|
};
|
||||||
|
|
||||||
|
export function localModelAvailability(): Availability {
|
||||||
|
try {
|
||||||
|
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
|
||||||
|
const value = String(FaceclawSystemLanguageModel.availability());
|
||||||
|
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
|
||||||
|
} catch {
|
||||||
|
return "unsupportedOS";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
|
||||||
|
export function isLocalModelSupported(): boolean {
|
||||||
|
const availability = localModelAvailability();
|
||||||
|
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isLocalModelReady(): boolean {
|
||||||
|
return localModelAvailability() === "available";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function localModelState(): LocalModelState {
|
||||||
|
const availability = localModelAvailability();
|
||||||
|
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
|
||||||
|
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The user-facing reason the model can't run, or "" when it can. */
|
||||||
|
export function localModelUnavailableMessage(): string {
|
||||||
|
const availability = localModelAvailability();
|
||||||
|
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
|
||||||
|
}
|
||||||
|
|
||||||
|
// iOS manages the system model itself: there is nothing to download, cancel,
|
||||||
|
// delete or unload, and no progress to watch.
|
||||||
|
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
|
||||||
|
return () => {};
|
||||||
|
}
|
||||||
|
export function startLocalModelDownload(): void {}
|
||||||
|
export function cancelLocalModelDownload(): void {}
|
||||||
|
export function deleteLocalModel(): void {}
|
||||||
|
export function unloadLocalModel(): void {}
|
||||||
|
|
||||||
|
type HistoryEntry =
|
||||||
|
| { type: "prompt" | "response"; text: string }
|
||||||
|
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
||||||
|
| { type: "toolOutput"; id: string; name: string; text: string };
|
||||||
|
|
||||||
|
export type SystemModelRequest = {
|
||||||
|
system: string;
|
||||||
|
tools: Array<{ name: string; description: string; parameters: object }>;
|
||||||
|
history: HistoryEntry[];
|
||||||
|
prompt: string;
|
||||||
|
maxTokens: number;
|
||||||
|
permissive: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Translate provider-neutral history into the Swift request: everything but
|
||||||
|
* the newest user prompt becomes Transcript entries, and that prompt is what
|
||||||
|
* the model answers.
|
||||||
|
*/
|
||||||
|
export function buildSystemModelRequest(
|
||||||
|
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
|
||||||
|
): SystemModelRequest {
|
||||||
|
const messages = options.messages;
|
||||||
|
const last = messages[messages.length - 1];
|
||||||
|
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
|
||||||
|
const history: HistoryEntry[] = [];
|
||||||
|
const toolNames = new Map<string, string>();
|
||||||
|
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
|
||||||
|
appendHistory(history, message, toolNames);
|
||||||
|
}
|
||||||
|
const tools = (options.tools ?? []).map((tool) => ({
|
||||||
|
name: tool.name,
|
||||||
|
description: tool.description,
|
||||||
|
parameters: tool.input_schema,
|
||||||
|
}));
|
||||||
|
return {
|
||||||
|
system: options.system?.trim() || "You are a helpful assistant.",
|
||||||
|
tools,
|
||||||
|
history,
|
||||||
|
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
|
||||||
|
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||||
|
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
|
||||||
|
permissive: tools.length === 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
|
||||||
|
if (message.role === "user") {
|
||||||
|
for (const result of toolResultsOf(message)) {
|
||||||
|
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
|
||||||
|
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
|
||||||
|
}
|
||||||
|
const text = textOf(message.content);
|
||||||
|
if (text) history.push({ type: "prompt", text });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const text = textOf(message.content);
|
||||||
|
if (text) history.push({ type: "response", text });
|
||||||
|
if (typeof message.content === "string") return;
|
||||||
|
const calls: Array<{ id: string; name: string; arguments: string }> = [];
|
||||||
|
for (const block of message.content) {
|
||||||
|
if (block.type !== "tool_use") continue;
|
||||||
|
toolNames.set(block.id, block.name);
|
||||||
|
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
|
||||||
|
}
|
||||||
|
if (calls.length) history.push({ type: "toolCalls", calls });
|
||||||
|
}
|
||||||
|
|
||||||
|
function textOf(content: string | LlmContentBlock[]): string {
|
||||||
|
if (typeof content === "string") return content.trim();
|
||||||
|
return content
|
||||||
|
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
|
||||||
|
.map((block) => block.text.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
|
||||||
|
if (!message || message.role !== "user" || typeof message.content === "string") return [];
|
||||||
|
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseArguments(json: string): any {
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(json);
|
||||||
|
return value && typeof value === "object" ? value : {};
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type NativeEvent = {
|
||||||
|
kind: string;
|
||||||
|
text?: string;
|
||||||
|
calls?: Array<{ id: string; name: string; arguments: string }>;
|
||||||
|
code?: string;
|
||||||
|
message?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Generations parked in tool calls, by call id. */
|
||||||
|
const parked = new Map<string, Generation>();
|
||||||
|
/** Every unfinished generation, so its native object outlives the caller's handle. */
|
||||||
|
const live = new Set<Generation>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One native generation, possibly spanning several loop iterations: each
|
||||||
|
* iteration ends at a batch of tool calls (or the end of the reply), and the
|
||||||
|
* next one resumes it with the results.
|
||||||
|
*/
|
||||||
|
class Generation {
|
||||||
|
private readonly native: any;
|
||||||
|
private options: LlmStreamOptions | null;
|
||||||
|
private iteration = 0;
|
||||||
|
private text = "";
|
||||||
|
private callIds: string[] = [];
|
||||||
|
private expiry: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
private closed = false;
|
||||||
|
|
||||||
|
constructor(options: LlmStreamOptions) {
|
||||||
|
this.options = options;
|
||||||
|
this.native = FaceclawSystemLanguageModel.new();
|
||||||
|
this.native.eventHandler = (json: string) => this.onEvent(String(json));
|
||||||
|
live.add(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
start(request: SystemModelRequest): LlmStreamHandle {
|
||||||
|
this.native.start(JSON.stringify(request));
|
||||||
|
return this.handle();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether these results answer this generation's parked calls. */
|
||||||
|
awaits(results: ToolResultBlock[]): boolean {
|
||||||
|
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
|
||||||
|
}
|
||||||
|
|
||||||
|
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
|
||||||
|
this.unpark();
|
||||||
|
this.options = options;
|
||||||
|
this.iteration++;
|
||||||
|
this.text = "";
|
||||||
|
this.native.resolveToolCalls(
|
||||||
|
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
|
||||||
|
);
|
||||||
|
return this.handle();
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel(): void {
|
||||||
|
if (this.closed) return;
|
||||||
|
this.close();
|
||||||
|
try {
|
||||||
|
this.native.cancel();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private handle(): LlmStreamHandle {
|
||||||
|
const iteration = this.iteration;
|
||||||
|
// A stale handle (an earlier iteration's) still cancels a generation parked
|
||||||
|
// since then: its owner gave up on the turn.
|
||||||
|
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
|
||||||
|
}
|
||||||
|
|
||||||
|
private close(): void {
|
||||||
|
this.closed = true;
|
||||||
|
this.unpark();
|
||||||
|
this.options = null;
|
||||||
|
this.native.eventHandler = null;
|
||||||
|
live.delete(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
private park(ids: string[]): void {
|
||||||
|
this.callIds = ids;
|
||||||
|
for (const id of ids) parked.set(id, this);
|
||||||
|
this.expiry = setTimeout(() => {
|
||||||
|
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
|
||||||
|
this.cancel();
|
||||||
|
}, PARKED_TIMEOUT_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private unpark(): void {
|
||||||
|
for (const id of this.callIds) {
|
||||||
|
if (parked.get(id) === this) parked.delete(id);
|
||||||
|
}
|
||||||
|
this.callIds = [];
|
||||||
|
if (this.expiry !== null) clearTimeout(this.expiry);
|
||||||
|
this.expiry = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private onEvent(json: string): void {
|
||||||
|
if (this.closed) return;
|
||||||
|
let event: NativeEvent;
|
||||||
|
try {
|
||||||
|
event = JSON.parse(json);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.kind === "log") {
|
||||||
|
console.log(`[local-llm] ${event.message ?? ""}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const options = this.options;
|
||||||
|
if (!options) return;
|
||||||
|
switch (event.kind) {
|
||||||
|
case "delta": {
|
||||||
|
const delta = event.text ?? "";
|
||||||
|
this.text += delta;
|
||||||
|
options.onTextDelta?.(delta, this.text);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case "toolCalls": {
|
||||||
|
const calls = event.calls ?? [];
|
||||||
|
const text = this.text.trim();
|
||||||
|
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
|
||||||
|
for (const call of calls) {
|
||||||
|
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
|
||||||
|
}
|
||||||
|
this.options = null;
|
||||||
|
this.park(calls.map((call) => call.id));
|
||||||
|
options.onDone({ text, content, stopReason: "tool_use" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case "done": {
|
||||||
|
this.close();
|
||||||
|
const text = this.text.trim();
|
||||||
|
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
case "error": {
|
||||||
|
this.close();
|
||||||
|
if (event.code === "refusal") {
|
||||||
|
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
|
||||||
|
} else {
|
||||||
|
options.onError(event.message || "The on-device model failed");
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stream a reply from the system model. A request whose history ends in tool
|
||||||
|
* results for a parked generation resumes it; anything else starts afresh.
|
||||||
|
*/
|
||||||
|
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
|
||||||
|
const availability = localModelAvailability();
|
||||||
|
if (availability !== "available") {
|
||||||
|
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
|
||||||
|
return { cancel: () => {} };
|
||||||
|
}
|
||||||
|
const results = toolResultsOf(options.messages[options.messages.length - 1]);
|
||||||
|
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
|
||||||
|
if (waiting?.awaits(results)) return waiting.resume(options, results);
|
||||||
|
let generation: Generation | null = null;
|
||||||
|
try {
|
||||||
|
generation = new Generation(options);
|
||||||
|
return generation.start(buildSystemModelRequest(options));
|
||||||
|
} catch (error) {
|
||||||
|
generation?.cancel();
|
||||||
|
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||||
|
return { cancel: () => {} };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
/**
|
||||||
|
* The on-phone LLM: one entry point for every feature that can run on the
|
||||||
|
* phone without an API key (the direct assistant's "local" provider,
|
||||||
|
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
|
||||||
|
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
|
||||||
|
* contract, plus model status for Settings.
|
||||||
|
*
|
||||||
|
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
|
||||||
|
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
|
||||||
|
* to download, available on Apple Intelligence devices running iOS 26+.
|
||||||
|
*/
|
||||||
|
export {
|
||||||
|
LOCAL_MODEL,
|
||||||
|
isLocalModelReady,
|
||||||
|
localModelState,
|
||||||
|
onLocalModelStateChanged,
|
||||||
|
startLocalModelDownload,
|
||||||
|
cancelLocalModelDownload,
|
||||||
|
deleteLocalModel,
|
||||||
|
unloadLocalModel,
|
||||||
|
streamLocalQwen as streamLocalModel,
|
||||||
|
} from "./llama";
|
||||||
|
export type { LocalModelState } from "./local-llm-types";
|
||||||
|
|
||||||
|
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
|
||||||
|
export function isLocalModelSupported(): boolean {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -16,7 +16,7 @@ import {
|
|||||||
assistantModelProvider,
|
assistantModelProvider,
|
||||||
type AssistantModel,
|
type AssistantModel,
|
||||||
} from "~/assistant/models";
|
} from "~/assistant/models";
|
||||||
import { isLocalModelReady } from "../native/llama";
|
import { isLocalModelReady } from "../native/local-llm";
|
||||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||||
@@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
|||||||
export type AssistantBackendKind = "direct" | "external";
|
export type AssistantBackendKind = "direct" | "external";
|
||||||
|
|
||||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||||
direct: global.isIOS ? "Cloud API" : "On-phone",
|
direct: "On-phone",
|
||||||
external: "My own agent (bridge)",
|
external: "My own agent (bridge)",
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
|
|||||||
values: global.isIOS ? ["direct"] : ["direct", "external"],
|
values: global.isIOS ? ["direct"] : ["direct", "external"],
|
||||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||||
description:
|
description:
|
||||||
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const assistantHermesUrlSetting = new ConfigSettingString({
|
export const assistantHermesUrlSetting = new ConfigSettingString({
|
||||||
@@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
|
|||||||
return false;
|
return false;
|
||||||
},
|
},
|
||||||
description:
|
description:
|
||||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const mapboxApiKeySetting = new ConfigSettingString({
|
export const mapboxApiKeySetting = new ConfigSettingString({
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import {
|
|||||||
localModelState,
|
localModelState,
|
||||||
onLocalModelStateChanged,
|
onLocalModelStateChanged,
|
||||||
startLocalModelDownload,
|
startLocalModelDownload,
|
||||||
} from "../../native/llama";
|
} from "../../native/local-llm";
|
||||||
import {
|
import {
|
||||||
ASR_MODELS,
|
ASR_MODELS,
|
||||||
asrModelState,
|
asrModelState,
|
||||||
@@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] {
|
|||||||
{
|
{
|
||||||
label: "Assistant",
|
label: "Assistant",
|
||||||
items: [
|
items: [
|
||||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
|
||||||
enumSettingMenuItem(assistantBackendSetting),
|
enumSettingMenuItem(assistantBackendSetting),
|
||||||
enumSettingMenuItem(assistantModelSetting),
|
enumSettingMenuItem(assistantModelSetting),
|
||||||
textSettingMenuItem(assistantHermesUrlSetting),
|
textSettingMenuItem(assistantHermesUrlSetting),
|
||||||
textSettingMenuItem(assistantHermesKeySetting),
|
textSettingMenuItem(assistantHermesKeySetting),
|
||||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
|
||||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||||
// iOS has no local LLM or agent bridge yet.
|
// iOS has no agent bridge yet.
|
||||||
...(!global.isIOS ? [
|
...(!global.isIOS ? [
|
||||||
textSettingMenuItem(assistantBridgeHostSetting),
|
textSettingMenuItem(assistantBridgeHostSetting),
|
||||||
textSettingMenuItem(assistantBridgePortSetting),
|
textSettingMenuItem(assistantBridgePortSetting),
|
||||||
@@ -399,6 +399,29 @@ function localModelStatusText(): string {
|
|||||||
return "not downloaded";
|
return "not downloaded";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
|
||||||
|
* installs and manages; the row only reports whether it can be used.
|
||||||
|
*/
|
||||||
|
function systemModelMenuItem(): MenuItem {
|
||||||
|
const statusText = (): string => {
|
||||||
|
const state = localModelState();
|
||||||
|
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
label: "On-phone model",
|
||||||
|
description:
|
||||||
|
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
|
||||||
|
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
|
||||||
|
"Faster and more private than the cloud models, but simpler, with a short memory. " +
|
||||||
|
"Used automatically when no API key is set.",
|
||||||
|
onSelect: () => {},
|
||||||
|
render: ({ image, x, y, width }) => {
|
||||||
|
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/** Download/cancel/delete management for the on-phone assistant model. */
|
/** Download/cancel/delete management for the on-phone assistant model. */
|
||||||
function localModelMenuItem(): MenuItem {
|
function localModelMenuItem(): MenuItem {
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
// Print the assistant's built-in tool definitions as JSON ([{name, description, parameters}]),
|
||||||
|
// pulled from the tool files' object literals via the TypeScript AST, with the dynamic
|
||||||
|
// parts (the app-id enum, NUMBER_ARG) filled in from a fixed list. An approximation of
|
||||||
|
// what ToolRegistry.listTools() offers with no app windows open.
|
||||||
|
// Usage: node scripts/apple-llm-harness/extract-tools.cjs > tools.json
|
||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs');
|
||||||
|
const root = path.resolve(__dirname, '../..');
|
||||||
|
const ts = require(path.join(root, 'node_modules/typescript'));
|
||||||
|
|
||||||
|
const FILES = ['system-tools', 'timer-tools', 'window-tools', 'navigate-tools', 'roam-tools']
|
||||||
|
.map(name => path.join(root, 'app/assistant', `${name}.ts`));
|
||||||
|
const APP_IDS = fs.readdirSync(path.join(root, 'app/apps'), { withFileTypes: true })
|
||||||
|
.filter(entry => entry.isDirectory() && !['launcher', 'onboarding', 'evenhub'].includes(entry.name))
|
||||||
|
.map(entry => entry.name).sort();
|
||||||
|
const NUMBER_ARG = (noun, listTool) => ({
|
||||||
|
type: 'number',
|
||||||
|
description: `1-based ${noun} number, as shown by ${listTool}. May be omitted when only one ${noun} exists.`,
|
||||||
|
});
|
||||||
|
|
||||||
|
const out = [];
|
||||||
|
for (const file of FILES) {
|
||||||
|
const src = ts.createSourceFile(file, fs.readFileSync(file, 'utf8'), ts.ScriptTarget.ES2020, true);
|
||||||
|
const consts = {};
|
||||||
|
src.forEachChild(function collect(node) {
|
||||||
|
if (ts.isVariableDeclaration(node) && node.initializer && ts.isStringLiteral(node.initializer)) {
|
||||||
|
consts[node.name.getText()] = node.initializer.text;
|
||||||
|
}
|
||||||
|
node.forEachChild(collect);
|
||||||
|
});
|
||||||
|
const evaluate = node => {
|
||||||
|
const js = ts.transpileModule(`(${node.getText()})`, { compilerOptions: { target: ts.ScriptTarget.ES2020 } }).outputText;
|
||||||
|
const names = [...Object.keys(consts), 'MAX_TIMER_DURATION_MS', 'appIds', 'installedNote', 'NUMBER_ARG'];
|
||||||
|
return Function(...names, `return ${js}`)(...Object.values(consts), 86400000, APP_IDS, '', NUMBER_ARG);
|
||||||
|
};
|
||||||
|
src.forEachChild(function visit(node) {
|
||||||
|
if (ts.isObjectLiteralExpression(node)) {
|
||||||
|
const keys = node.properties.map(p => p.name && p.name.getText());
|
||||||
|
if (keys.includes('name') && keys.includes('description') && keys.includes('inputSchema')) {
|
||||||
|
try {
|
||||||
|
const spec = evaluate(node);
|
||||||
|
out.push({ name: String(spec.name).replace(/[^a-zA-Z0-9_-]/g, '_'), description: spec.description, parameters: spec.inputSchema });
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`skipped a tool in ${path.basename(file)}: ${error.message}`);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
node.forEachChild(visit);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
process.stdout.write(JSON.stringify(out, null, 1));
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
// macOS harness for App_Resources/iOS/src/FaceclawSystemLanguageModel.swift: runs one
|
||||||
|
// assistant request against the Apple Intelligence model the way local-llm.ios.ts does,
|
||||||
|
// answering every tool call with a canned result. See run.sh.
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
@main struct Harness {
|
||||||
|
@MainActor static func main() async {
|
||||||
|
let args = CommandLine.arguments
|
||||||
|
guard args.count >= 3 else {
|
||||||
|
print("usage: harness <tools.json> <prompt> [previous prompt]")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
print("availability:", FaceclawSystemLanguageModel.availability(),
|
||||||
|
"context:", FaceclawSystemLanguageModel.contextSize())
|
||||||
|
let tools = (try? JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: args[1])))) ?? []
|
||||||
|
var history: [[String: Any]] = []
|
||||||
|
if args.count > 3 { history = [["type": "prompt", "text": args[3]], ["type": "response", "text": "OK."]] }
|
||||||
|
let request: [String: Any] = [
|
||||||
|
"system": "You are the voice assistant built into a pair of Even Realities G2 smart glasses. Your reply is shown as text on a 640x480 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list. Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it. If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
|
||||||
|
"tools": tools, "history": history, "prompt": args[2], "maxTokens": 512, "permissive": false,
|
||||||
|
]
|
||||||
|
let model = FaceclawSystemLanguageModel()
|
||||||
|
let started = Date()
|
||||||
|
var finished = false
|
||||||
|
var firstText: TimeInterval?
|
||||||
|
model.eventHandler = { json in
|
||||||
|
let elapsed = String(format: "%6.2fs", Date().timeIntervalSince(started))
|
||||||
|
guard let event = (try? JSONSerialization.jsonObject(with: Data(json.utf8))) as? [String: Any] else { return }
|
||||||
|
switch event["kind"] as? String {
|
||||||
|
case "delta":
|
||||||
|
if firstText == nil { firstText = Date().timeIntervalSince(started); print("\(elapsed) first text") }
|
||||||
|
print(event["text"] as? String ?? "", terminator: "")
|
||||||
|
case "toolCalls":
|
||||||
|
let calls = event["calls"] as? [[String: Any]] ?? []
|
||||||
|
for call in calls { print("\(elapsed) tool call: \(call["name"] ?? "") \(call["arguments"] ?? "")") }
|
||||||
|
let results = calls.map { ["id": $0["id"] ?? "", "content": "Done (simulated \($0["name"] ?? "tool") result).", "isError": false] }
|
||||||
|
let data = try! JSONSerialization.data(withJSONObject: results)
|
||||||
|
model.resolveToolCalls(String(data: data, encoding: .utf8)!)
|
||||||
|
case "done":
|
||||||
|
print("\n\(elapsed) done"); finished = true
|
||||||
|
default:
|
||||||
|
print("\(elapsed) \(json)")
|
||||||
|
if event["kind"] as? String == "error" { finished = true }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let data = try! JSONSerialization.data(withJSONObject: request)
|
||||||
|
model.start(String(data: data, encoding: .utf8)!)
|
||||||
|
while !finished && Date().timeIntervalSince(started) < 120 { try? await Task.sleep(nanoseconds: 50_000_000) }
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Run one assistant request against Apple's on-device model on this Mac (macOS 26.4+,
|
||||||
|
# Apple Intelligence on), through the same Swift bridge the iOS app uses, with the
|
||||||
|
# assistant's real tool list. Prints tool routing, tool calls and the streamed reply.
|
||||||
|
# scripts/apple-llm-harness/run.sh "set a timer for five minutes" ["previous prompt"]
|
||||||
|
set -euo pipefail
|
||||||
|
here="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
out="${TMPDIR:-/tmp}/faceclaw-apple-llm-harness"
|
||||||
|
mkdir -p "$out"
|
||||||
|
node "$here/extract-tools.cjs" > "$out/tools.json"
|
||||||
|
swiftc -swift-version 5 -parse-as-library -target "$(uname -m)-apple-macos26.4" \
|
||||||
|
"$here/../../App_Resources/iOS/src/FaceclawSystemLanguageModel.swift" "$here/main.swift" -o "$out/harness"
|
||||||
|
"$out/harness" "$out/tools.json" "$@"
|
||||||
@@ -91,7 +91,7 @@ test('a restored conversation keeps its Hermes session, and Hermes turns get a l
|
|||||||
requests.push(request);
|
requests.push(request);
|
||||||
return request;
|
return request;
|
||||||
} },
|
} },
|
||||||
'../native/llama': { streamLocalQwen: () => assert.fail('no local model') },
|
'../native/local-llm': { streamLocalModel: () => assert.fail('no local model') },
|
||||||
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: {} },
|
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: {} },
|
||||||
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' },
|
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' },
|
||||||
});
|
});
|
||||||
@@ -122,7 +122,7 @@ test('URL normalization adds /v1 to a bare host and drops the endpoint path', ()
|
|||||||
});
|
});
|
||||||
|
|
||||||
test('the Hermes model resolves only with a URL, carrying the key and base URL', () => {
|
test('the Hermes model resolves only with a URL, carrying the key and base URL', () => {
|
||||||
const load = loader({ global: { isIOS: true } }, { '../native/llama': { isLocalModelReady: () => false, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
|
const load = loader({ global: { isIOS: true } }, { '../native/local-llm': { isLocalModelReady: () => false, isLocalModelSupported: () => true, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
|
||||||
const { resolveAssistantModel } = load('app/assistant/models.ts');
|
const { resolveAssistantModel } = load('app/assistant/models.ts');
|
||||||
assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null);
|
assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null);
|
||||||
const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' });
|
const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' });
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ function assistant(provider) {
|
|||||||
const request = { url, body: JSON.parse(body), headers, listener, cancelled: false };
|
const request = { url, body: JSON.parse(body), headers, listener, cancelled: false };
|
||||||
requests.push(request); return { cancel: () => { request.cancelled = true; } };
|
requests.push(request); return { cancel: () => { request.cancelled = true; } };
|
||||||
} },
|
} },
|
||||||
'../native/llama': { streamLocalQwen: () => assert.fail('iOS must not invoke a local LLM') },
|
'../native/local-llm': { streamLocalModel: () => assert.fail('cloud providers must not invoke the local LLM') },
|
||||||
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
|
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
|
||||||
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'test', buildAssistantSystemPrompt: () => 'test', describeAssistantContext: () => '' },
|
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'test', buildAssistantSystemPrompt: () => 'test', describeAssistantContext: () => '' },
|
||||||
});
|
});
|
||||||
@@ -100,8 +100,8 @@ test('iOS SSE adapter forwards the Kotlin stream events once and stops after can
|
|||||||
assert.deepEqual(seen.at(-1), 'http:429:slow down');
|
assert.deepEqual(seen.at(-1), 'http:429:slow down');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('iOS only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
|
test('iOS without Apple Intelligence support only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
|
||||||
const api = loader({ global: { isIOS: true } }, { '../native/llama': { LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false } })('app/assistant/models.ts');
|
const api = loader({ global: { isIOS: true } }, { '../native/local-llm': { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => false, isLocalModelSupported: () => false } })('app/assistant/models.ts');
|
||||||
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), false);
|
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), false);
|
||||||
assert.equal(api.supportedAssistantModel('qwen'), 'auto');
|
assert.equal(api.supportedAssistantModel('qwen'), 'auto');
|
||||||
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai');
|
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai');
|
||||||
@@ -119,7 +119,7 @@ test('cancelling during a tool prevents subsequent tool side effects and a stall
|
|||||||
const calls = [], failures = [];
|
const calls = [], failures = [];
|
||||||
const { DirectAssistantBackend } = loader({ setTimeout: fn => { deadline = fn; return 1; }, clearTimeout() {} }, {
|
const { DirectAssistantBackend } = loader({ setTimeout: fn => { deadline = fn; return 1; }, clearTimeout() {} }, {
|
||||||
'../native/openai': { streamOpenAiResponse: options => { stream = options; return { cancel() {} }; } },
|
'../native/openai': { streamOpenAiResponse: options => { stream = options; return { cancel() {} }; } },
|
||||||
'../native/anthropic': {}, '../native/llama': {},
|
'../native/anthropic': {}, '../native/local-llm': {},
|
||||||
})('app/assistant/direct-backend.ts');
|
})('app/assistant/direct-backend.ts');
|
||||||
const backend = new DirectAssistantBackend();
|
const backend = new DirectAssistantBackend();
|
||||||
const options = { provider: 'openai', messages: [], buildTools: () => [], resolveToolName: name => name,
|
const options = { provider: 'openai', messages: [], buildTools: () => [], resolveToolName: name => name,
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
const test = require('node:test');
|
||||||
|
const assert = require('node:assert/strict');
|
||||||
|
const { loader } = require('./helpers/load-typescript.cjs');
|
||||||
|
const flush = () => new Promise(resolve => setImmediate(resolve));
|
||||||
|
// Values built inside the loader's VM context have that realm's prototypes.
|
||||||
|
const plain = value => JSON.parse(JSON.stringify(value));
|
||||||
|
|
||||||
|
/** A stand-in for FaceclawSystemLanguageModel.swift: records requests and lets tests emit its events. */
|
||||||
|
function fakeNative(availability = 'available') {
|
||||||
|
const instances = [];
|
||||||
|
const cls = {
|
||||||
|
availability: () => availability,
|
||||||
|
new() {
|
||||||
|
const native = {
|
||||||
|
request: null, resolved: [], cancelled: false, eventHandler: null,
|
||||||
|
start(json) { this.request = JSON.parse(json); },
|
||||||
|
resolveToolCalls(json) { this.resolved.push(JSON.parse(json)); },
|
||||||
|
cancel() { this.cancelled = true; },
|
||||||
|
emit(event) { if (this.eventHandler) this.eventHandler(JSON.stringify(event)); },
|
||||||
|
};
|
||||||
|
instances.push(native);
|
||||||
|
return native;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return { cls, instances, setAvailability(value) { availability = value; } };
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadLocalLlm(native, extraGlobals = {}) {
|
||||||
|
const load = loader({ FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout, ...extraGlobals }, {});
|
||||||
|
return load('app/native/local-llm.ios.ts');
|
||||||
|
}
|
||||||
|
|
||||||
|
function collector() {
|
||||||
|
const seen = { deltas: [], done: [], errors: [] };
|
||||||
|
return {
|
||||||
|
seen,
|
||||||
|
callbacks: {
|
||||||
|
onTextDelta: (_delta, full) => seen.deltas.push(full),
|
||||||
|
onDone: result => seen.done.push(result),
|
||||||
|
onError: message => seen.errors.push(message),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('iOS local LLM converts provider-neutral history into transcript entries and a final prompt', () => {
|
||||||
|
const api = loadLocalLlm(fakeNative());
|
||||||
|
const request = api.buildSystemModelRequest({
|
||||||
|
system: 'Be brief.',
|
||||||
|
maxTokens: 200,
|
||||||
|
tools: [{ name: 'timer_set', description: 'Set a timer', input_schema: { type: 'object', properties: { seconds: { type: 'integer' } } } }],
|
||||||
|
messages: [
|
||||||
|
{ role: 'user', content: 'Set a timer' },
|
||||||
|
{ role: 'assistant', content: [{ type: 'text', text: 'On it.' }, { type: 'tool_use', id: 'fm_1', name: 'timer_set', input: { seconds: 60 } }] },
|
||||||
|
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'Timer set' }] },
|
||||||
|
{ role: 'assistant', content: 'Done.' },
|
||||||
|
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'gone', content: 'boom', is_error: true }, { type: 'text', text: 'And now?' }] },
|
||||||
|
{ role: 'assistant', content: [{ type: 'provider_item', provider: 'openai', item: {} }] },
|
||||||
|
{ role: 'user', content: 'What time is it?' },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
assert.equal(request.system, 'Be brief.');
|
||||||
|
assert.equal(request.prompt, 'What time is it?');
|
||||||
|
assert.equal(request.maxTokens, 200);
|
||||||
|
assert.equal(request.permissive, false);
|
||||||
|
assert.deepEqual(plain(request.tools), [{ name: 'timer_set', description: 'Set a timer', parameters: { type: 'object', properties: { seconds: { type: 'integer' } } } }]);
|
||||||
|
assert.deepEqual(plain(request.history), [
|
||||||
|
{ type: 'prompt', text: 'Set a timer' },
|
||||||
|
{ type: 'response', text: 'On it.' },
|
||||||
|
{ type: 'toolCalls', calls: [{ id: 'fm_1', name: 'timer_set', arguments: '{"seconds":60}' }] },
|
||||||
|
{ type: 'toolOutput', id: 'fm_1', name: 'timer_set', text: 'Timer set' },
|
||||||
|
{ type: 'response', text: 'Done.' },
|
||||||
|
{ type: 'toolOutput', id: 'gone', name: 'tool', text: 'Error: boom' },
|
||||||
|
{ type: 'prompt', text: 'And now?' },
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Tool-free requests (translate, Q&A) get the permissive guardrails; a
|
||||||
|
// trailing tool result with nothing to resume falls back to a continuation prompt.
|
||||||
|
const orphan = api.buildSystemModelRequest({
|
||||||
|
messages: [{ role: 'user', content: 'x' }, { role: 'assistant', content: [{ type: 'tool_use', id: 'a', name: 't', input: {} }] },
|
||||||
|
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'a', content: 'ok' }] }],
|
||||||
|
});
|
||||||
|
assert.equal(orphan.permissive, true);
|
||||||
|
assert.equal(orphan.maxTokens, 512);
|
||||||
|
assert.equal(orphan.prompt, api.CONTINUE_PROMPT);
|
||||||
|
assert.deepEqual(plain(orphan.history.at(-1)), { type: 'toolOutput', id: 'a', name: 't', text: 'ok' });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS local LLM streams text deltas, then ends the turn', () => {
|
||||||
|
const native = fakeNative();
|
||||||
|
const api = loadLocalLlm(native);
|
||||||
|
const { seen, callbacks } = collector();
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', system: 's', messages: [{ role: 'user', content: 'Hi' }], ...callbacks });
|
||||||
|
const [instance] = native.instances;
|
||||||
|
assert.equal(instance.request.prompt, 'Hi');
|
||||||
|
instance.emit({ kind: 'log', message: 'shortened tool descriptions' });
|
||||||
|
instance.emit({ kind: 'delta', text: 'Hello' });
|
||||||
|
instance.emit({ kind: 'delta', text: ' there' });
|
||||||
|
instance.emit({ kind: 'done' });
|
||||||
|
instance.emit({ kind: 'delta', text: 'late' });
|
||||||
|
assert.deepEqual(plain(seen.deltas), ['Hello', 'Hello there']);
|
||||||
|
assert.equal(seen.done.length, 1);
|
||||||
|
assert.deepEqual(plain(seen.done[0]), { text: 'Hello there', content: [{ type: 'text', text: 'Hello there' }], stopReason: 'end_turn' });
|
||||||
|
assert.equal(instance.eventHandler, null, 'finished generations drop their handler');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS local LLM reports why the model is unavailable without touching native code', async () => {
|
||||||
|
const native = fakeNative('appleIntelligenceNotEnabled');
|
||||||
|
const api = loadLocalLlm(native);
|
||||||
|
assert.equal(api.isLocalModelReady(), false);
|
||||||
|
assert.equal(api.isLocalModelSupported(), true);
|
||||||
|
assert.deepEqual(plain(api.localModelState()), { status: 'unavailable', bytesDownloaded: 0, totalBytes: 0, detail: 'Apple Intelligence off' });
|
||||||
|
const { seen, callbacks } = collector();
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'Hi' }], ...callbacks });
|
||||||
|
await new Promise(resolve => setTimeout(resolve, 0));
|
||||||
|
assert.match(seen.errors[0], /Turn on Apple Intelligence/);
|
||||||
|
assert.equal(native.instances.length, 0);
|
||||||
|
|
||||||
|
native.setAvailability('deviceNotEligible');
|
||||||
|
assert.equal(api.isLocalModelSupported(), false);
|
||||||
|
native.setAvailability('somethingNew');
|
||||||
|
assert.equal(api.localModelState().detail, 'unavailable');
|
||||||
|
const missing = loader({ setTimeout, clearTimeout }, {})('app/native/local-llm.ios.ts');
|
||||||
|
assert.equal(missing.localModelAvailability(), 'unsupportedOS', 'pre-iOS-26 builds have no Swift class to call');
|
||||||
|
native.setAvailability('available');
|
||||||
|
assert.deepEqual(plain(api.localModelState()), { status: 'ready', bytesDownloaded: 0, totalBytes: 0 });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS local LLM maps refusals to the refusal stop reason and other failures to errors', () => {
|
||||||
|
const native = fakeNative();
|
||||||
|
const api = loadLocalLlm(native);
|
||||||
|
const first = collector();
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...first.callbacks });
|
||||||
|
native.instances[0].emit({ kind: 'error', code: 'refusal', message: 'declined' });
|
||||||
|
assert.equal(first.seen.done[0].stopReason, 'refusal');
|
||||||
|
const second = collector();
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...second.callbacks });
|
||||||
|
native.instances[1].emit({ kind: 'error', code: 'rateLimited', message: 'iOS is limiting the on-device model' });
|
||||||
|
assert.deepEqual(plain(second.seen.errors), ['iOS is limiting the on-device model']);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS local LLM cancellation stops the native generation and silences later events', () => {
|
||||||
|
const native = fakeNative();
|
||||||
|
const api = loadLocalLlm(native);
|
||||||
|
const { seen, callbacks } = collector();
|
||||||
|
const handle = api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...callbacks });
|
||||||
|
handle.cancel();
|
||||||
|
native.instances[0].emit({ kind: 'delta', text: 'late' });
|
||||||
|
native.instances[0].emit({ kind: 'done' });
|
||||||
|
assert.equal(native.instances[0].cancelled, true);
|
||||||
|
assert.deepEqual(seen, { deltas: [], done: [], errors: [] });
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS assistant runs the on-device model through the shared tool loop, resuming the parked generation', async () => {
|
||||||
|
const native = fakeNative();
|
||||||
|
const calls = [], deltas = [], errors = [], done = [];
|
||||||
|
const registry = {
|
||||||
|
listTools: () => [{ name: 'timer.set', description: 'Set a timer', inputSchema: { type: 'object', properties: { seconds: { type: 'integer' } }, required: ['seconds'] } }],
|
||||||
|
callTool: async (name, args) => { calls.push({ name, args }); return { ok: true, content: 'Timer set' }; },
|
||||||
|
};
|
||||||
|
const mocks = {
|
||||||
|
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
|
||||||
|
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'base prompt', buildAssistantSystemPrompt: () => 'cloud prompt', describeAssistantContext: () => 'Context: now' },
|
||||||
|
};
|
||||||
|
const load = loader({ global: { isIOS: true, isAndroid: false }, FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout }, mocks);
|
||||||
|
mocks['../native/local-llm'] = load('app/native/local-llm.ios.ts');
|
||||||
|
const { AssistantSession } = load('app/assistant/session.ts');
|
||||||
|
const session = new AssistantSession({ kind: 'direct', llm: { selection: 'qwen', provider: 'local', model: 'apple-foundation-model', apiKey: '' } }, registry);
|
||||||
|
const send = text => session.sendUtterance(text, {}, { onTextDelta: (_d, full) => deltas.push(full),
|
||||||
|
onToolActivity() {}, onError: error => errors.push(error), onTurnDone: result => done.push(result) });
|
||||||
|
|
||||||
|
send('Set a one-minute timer');
|
||||||
|
assert.equal(native.instances.length, 1);
|
||||||
|
const generation = native.instances[0];
|
||||||
|
assert.equal(generation.request.system, 'base prompt', 'the local provider keeps the system prompt stable');
|
||||||
|
assert.equal(generation.request.prompt, 'Set a one-minute timer\n\n(Context: now)');
|
||||||
|
assert.deepEqual(plain(generation.request.tools.map(tool => tool.name)), ['timer_set']);
|
||||||
|
|
||||||
|
generation.emit({ kind: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] });
|
||||||
|
await flush();
|
||||||
|
assert.deepEqual(plain(calls), [{ name: 'timer.set', args: { seconds: 60 } }]);
|
||||||
|
assert.equal(native.instances.length, 1, 'tool results resume the parked generation instead of starting another');
|
||||||
|
assert.deepEqual(plain(generation.resolved), [[{ id: 'fm_abc', content: 'Timer set', isError: false }]]);
|
||||||
|
|
||||||
|
generation.emit({ kind: 'delta', text: 'Your timer is running.' });
|
||||||
|
generation.emit({ kind: 'done' });
|
||||||
|
assert.equal(done.length, 1);
|
||||||
|
assert.equal(errors.length, 0);
|
||||||
|
assert.equal(deltas.at(-1), 'Your timer is running.');
|
||||||
|
assert.equal(session.isTurnActive(), false);
|
||||||
|
|
||||||
|
// The next turn starts a fresh generation that carries the whole exchange as transcript history.
|
||||||
|
send('Thanks');
|
||||||
|
const next = native.instances[1];
|
||||||
|
assert.deepEqual(plain(next.request.history), [
|
||||||
|
{ type: 'prompt', text: 'Set a one-minute timer\n\n(Context: now)' },
|
||||||
|
{ type: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] },
|
||||||
|
{ type: 'toolOutput', id: 'fm_abc', name: 'timer_set', text: 'Timer set' },
|
||||||
|
{ type: 'response', text: 'Your timer is running.' },
|
||||||
|
]);
|
||||||
|
session.cancel();
|
||||||
|
assert.equal(next.cancelled, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a parked iOS generation survives unrelated requests and is abandoned if its results never arrive', () => {
|
||||||
|
const native = fakeNative();
|
||||||
|
let expire = null;
|
||||||
|
const timers = {
|
||||||
|
setTimeout: (fn, ms) => { if (ms > 0) expire = fn; else fn(); return 1; },
|
||||||
|
clearTimeout: () => { expire = null; },
|
||||||
|
};
|
||||||
|
const api = loadLocalLlm(native, timers);
|
||||||
|
const turn = collector();
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], tools: [], ...turn.callbacks });
|
||||||
|
native.instances[0].emit({ kind: 'toolCalls', calls: [{ id: 'fm_1', name: 't', arguments: 'not json' }] });
|
||||||
|
assert.deepEqual(plain(turn.seen.done[0].content), [{ type: 'tool_use', id: 'fm_1', name: 't', input: {} }]);
|
||||||
|
|
||||||
|
// e.g. Translate asking the model something while the assistant's tool runs.
|
||||||
|
const other = collector();
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'translate this' }], ...other.callbacks });
|
||||||
|
assert.equal(native.instances.length, 2);
|
||||||
|
assert.equal(native.instances[0].cancelled, false);
|
||||||
|
|
||||||
|
assert.equal(typeof expire, 'function');
|
||||||
|
expire();
|
||||||
|
assert.equal(native.instances[0].cancelled, true);
|
||||||
|
// Results for the abandoned call now start a fresh generation with the fallback prompt.
|
||||||
|
api.streamLocalModel({ apiKey: '', model: 'm', ...turn.callbacks, messages: [
|
||||||
|
{ role: 'user', content: 'x' },
|
||||||
|
{ role: 'assistant', content: turn.seen.done[0].content },
|
||||||
|
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'ok' }] },
|
||||||
|
] });
|
||||||
|
assert.equal(native.instances.length, 3);
|
||||||
|
assert.equal(native.instances[2].request.prompt, api.CONTINUE_PROMPT);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('iOS offers the on-phone model where Apple Intelligence is supported', () => {
|
||||||
|
const local = { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => true, isLocalModelSupported: () => true };
|
||||||
|
const api = loader({ global: { isIOS: true } }, { '../native/local-llm': local })('app/assistant/models.ts');
|
||||||
|
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), true);
|
||||||
|
assert.equal(api.supportedAssistantModel('qwen'), 'qwen');
|
||||||
|
assert.equal(api.assistantModelLabel('qwen'), 'Apple Intelligence (on-phone)');
|
||||||
|
const resolved = api.resolveAssistantModel('auto', { openai: '', anthropic: '' });
|
||||||
|
assert.equal(resolved.provider, 'local');
|
||||||
|
assert.equal(resolved.model, 'apple-foundation-model');
|
||||||
|
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai', 'API keys still win under Auto');
|
||||||
|
local.isLocalModelReady = () => false;
|
||||||
|
assert.equal(api.resolveAssistantModel('qwen', { openai: '', anthropic: '' }), null);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user