Support Apple Intelligence as an on-device AI model for the voice assistant/etc
This commit is contained in:
1 parent
3c3aa31542
commit
4973fd2aea
22 files changed
+1551
-40
No files matched your search
@@ -3,7 +3,9 @@ TARGETED_DEVICE_FAMILY = 1;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
|
||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security
|
||||
// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+,
|
||||
// so it is weak-linked: the app still launches on older iOS and reports the model unavailable.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels
|
||||
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
|
||||
|
||||
@@ -0,0 +1,694 @@
|
||||
import Foundation
|
||||
#if canImport(FoundationModels)
|
||||
import FoundationModels
|
||||
#endif
|
||||
|
||||
/// The on-phone LLM on iOS: Apple's Foundation Models framework (the Apple
|
||||
/// Intelligence system model, iOS 26+, eligible hardware). It runs out of
|
||||
/// process on the Neural Engine, so it costs the app no model download or
|
||||
/// memory, and unlike Metal work it is allowed (within a system budget) while
|
||||
/// Faceclaw is in the background driving the glasses.
|
||||
///
|
||||
/// JavaScript (app/native/local-llm.ios.ts) drives the same provider-neutral
|
||||
/// agent loop as the cloud models: each start() is one loop iteration. The
|
||||
/// request is JSON:
|
||||
///
|
||||
/// {"system":String,
|
||||
/// "tools":[{"name","description","parameters":<JSON Schema>}],
|
||||
/// "history":[{"type":"prompt"|"response","text"} |
|
||||
/// {"type":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]} |
|
||||
/// {"type":"toolOutput","id","name","text"}],
|
||||
/// "prompt":String, "maxTokens":Int, "permissive":Bool}
|
||||
///
|
||||
/// and events arrive on the main queue through eventHandler:
|
||||
///
|
||||
/// {"kind":"delta","text":String} streamed reply text
|
||||
/// {"kind":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]}
|
||||
/// {"kind":"done"}
|
||||
/// {"kind":"error","code":String,"message":String}
|
||||
/// {"kind":"log","message":String}
|
||||
///
|
||||
/// The framework runs tools itself, mid-response. When the model calls one,
|
||||
/// the call is reported as "toolCalls" and the generation stays suspended
|
||||
/// inside the tool until resolveToolCalls() hands back the results that
|
||||
/// JavaScript's tool loop produced; the reply then continues on the same
|
||||
/// instance.
|
||||
///
|
||||
/// The model's context window is small (4096 tokens on iOS 26, shared by input
|
||||
/// and reply) and the framework spends ~50 tokens on each tool before its
|
||||
/// parameters, so the assistant's full tool list (30+ tools, ~5000 tokens) can't
|
||||
/// be offered at once. When the tools don't fit their share of the window, a
|
||||
/// quick schema-constrained routing request first asks the model which few
|
||||
/// tools the request needs. History is then trimmed to fit before each start().
|
||||
/// Called from JavaScript, so on the main thread.
|
||||
@MainActor
|
||||
@objc(FaceclawSystemLanguageModel)
|
||||
@objcMembers
|
||||
public class FaceclawSystemLanguageModel: NSObject {
|
||||
public var eventHandler: ((String) -> Void)?
|
||||
// A LanguageModelEngine, typed loosely so the class itself loads before iOS 26.
|
||||
private var engine: AnyObject?
|
||||
|
||||
/// "available", or why not: "unsupportedOS", "deviceNotEligible",
|
||||
/// "appleIntelligenceNotEnabled", "modelNotReady" (still downloading), "unavailable".
|
||||
/// Nonisolated: worker isolates load the model-choice list too, off the main thread.
|
||||
nonisolated public static func availability() -> String {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) {
|
||||
switch SystemLanguageModel.default.availability {
|
||||
case .available:
|
||||
return "available"
|
||||
case .unavailable(let reason):
|
||||
switch reason {
|
||||
case .deviceNotEligible: return "deviceNotEligible"
|
||||
case .appleIntelligenceNotEnabled: return "appleIntelligenceNotEnabled"
|
||||
case .modelNotReady: return "modelNotReady"
|
||||
@unknown default: return "unavailable"
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return "unsupportedOS"
|
||||
}
|
||||
|
||||
/// The model's context window in tokens (input and output share it), or 0 when unavailable.
|
||||
nonisolated public static func contextSize() -> Int {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) { return SystemLanguageModel.default.contextSize }
|
||||
#endif
|
||||
return 0
|
||||
}
|
||||
|
||||
public func start(_ requestJson: String) {
|
||||
cancel()
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) {
|
||||
guard let data = requestJson.data(using: .utf8),
|
||||
let request = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else {
|
||||
emit(["kind": "error", "code": "other", "message": "Invalid on-device model request"])
|
||||
return
|
||||
}
|
||||
let engine = LanguageModelEngine { [weak self] event in self?.emit(event) }
|
||||
self.engine = engine
|
||||
engine.start(request)
|
||||
return
|
||||
}
|
||||
#endif
|
||||
emit(["kind": "error", "code": "unavailable", "message": "The on-device model needs iOS 26 or later."])
|
||||
}
|
||||
|
||||
/// `[{"id","content","isError"}]` for the calls of the last "toolCalls" event.
|
||||
public func resolveToolCalls(_ resultsJson: String) {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) {
|
||||
let data = resultsJson.data(using: .utf8) ?? Data()
|
||||
let results = ((try? JSONSerialization.jsonObject(with: data)) as? [[String: Any]]) ?? []
|
||||
(engine as? LanguageModelEngine)?.resolve(results)
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// Abandon the generation (and any suspended tool calls) without further events.
|
||||
public func cancel() {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) { (engine as? LanguageModelEngine)?.cancel() }
|
||||
#endif
|
||||
engine = nil
|
||||
}
|
||||
|
||||
private func emit(_ event: [String: Any]) {
|
||||
// JSONSerialization raises (not throws) on NaN and other invalid values.
|
||||
guard JSONSerialization.isValidJSONObject(event),
|
||||
let data = try? JSONSerialization.data(withJSONObject: event),
|
||||
let json = String(data: data, encoding: .utf8) else { return }
|
||||
eventHandler?(json)
|
||||
}
|
||||
}
|
||||
|
||||
#if canImport(FoundationModels)
|
||||
|
||||
/// A JavaScript-side tool: the framework calls it mid-response, and the
|
||||
/// engine parks the call until JavaScript resolves it.
|
||||
@available(iOS 26.0, *)
|
||||
private struct BridgedTool: Tool {
|
||||
typealias Arguments = GeneratedContent
|
||||
typealias Output = String
|
||||
|
||||
let name: String
|
||||
let description: String
|
||||
let parameters: GenerationSchema
|
||||
let handler: @Sendable (String, String) async throws -> String
|
||||
|
||||
func call(arguments: GeneratedContent) async throws -> String {
|
||||
try await handler(name, arguments.jsonString)
|
||||
}
|
||||
}
|
||||
|
||||
/// One tool from the request, convertible at two levels of verbosity.
|
||||
@available(iOS 26.0, *)
|
||||
private struct ToolSpec {
|
||||
let name: String
|
||||
let description: String
|
||||
let parameters: [String: Any]
|
||||
}
|
||||
|
||||
@available(iOS 26.0, *)
|
||||
@MainActor
|
||||
final class LanguageModelEngine {
|
||||
private struct PendingCall {
|
||||
let id: String
|
||||
let name: String
|
||||
let arguments: String
|
||||
let continuation: CheckedContinuation<String, Error>
|
||||
}
|
||||
|
||||
/// How long a call waits for any parallel calls before they're reported together.
|
||||
private static let toolBatchDelayNs: UInt64 = 30_000_000
|
||||
/// Tokens kept free beyond the reply budget: the framework's own framing and rounding.
|
||||
private static let contextMargin = 64
|
||||
/// Share of the context window tool definitions may take before tools are routed.
|
||||
private static let toolBudgetFraction = 0.4
|
||||
/// Most tools a routing request picks.
|
||||
private static let maxRoutedTools = 5
|
||||
/// Tools used in the latest exchanges stay offered, for follow-ups ("make it ten minutes").
|
||||
private static let keptRecentTools = 2
|
||||
|
||||
private let emit: ([String: Any]) -> Void
|
||||
private var task: Task<Void, Never>?
|
||||
// Calls not yet reported, then reported calls awaiting results.
|
||||
private var unreported: [PendingCall] = []
|
||||
private var awaiting: [String: PendingCall] = [:]
|
||||
private var flushScheduled = false
|
||||
private var closed = false
|
||||
|
||||
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
|
||||
|
||||
func start(_ request: [String: Any]) {
|
||||
task = Task { [weak self] in await self?.run(request) }
|
||||
}
|
||||
|
||||
func resolve(_ results: [[String: Any]]) {
|
||||
for result in results {
|
||||
guard let id = result["id"] as? String, let call = awaiting.removeValue(forKey: id) else { continue }
|
||||
let content = (result["content"] as? String) ?? ""
|
||||
let isError = (result["isError"] as? Bool) ?? false
|
||||
// An error goes back to the model as text: a thrown error would end the whole response.
|
||||
call.continuation.resume(returning: isError ? "Error: \(content)" : (content.isEmpty ? "(no output)" : content))
|
||||
}
|
||||
// A call JavaScript didn't answer can't be left hanging.
|
||||
for (_, call) in awaiting {
|
||||
call.continuation.resume(returning: "Error: the tool returned no result")
|
||||
}
|
||||
awaiting.removeAll()
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
closed = true
|
||||
task?.cancel()
|
||||
task = nil
|
||||
for call in unreported { call.continuation.resume(throwing: CancellationError()) }
|
||||
for (_, call) in awaiting { call.continuation.resume(throwing: CancellationError()) }
|
||||
unreported.removeAll()
|
||||
awaiting.removeAll()
|
||||
}
|
||||
|
||||
/// Events stop once the session is cancelled, even from work still winding down.
|
||||
private func send(_ event: [String: Any]) {
|
||||
if !closed { emit(event) }
|
||||
}
|
||||
|
||||
private func run(_ request: [String: Any]) async {
|
||||
let system = (request["system"] as? String) ?? ""
|
||||
let prompt = (request["prompt"] as? String) ?? ""
|
||||
let maxTokens = (request["maxTokens"] as? NSNumber)?.intValue ?? 512
|
||||
let permissive = (request["permissive"] as? Bool) ?? false
|
||||
let toolSpecs: [ToolSpec] = ((request["tools"] as? [[String: Any]]) ?? []).compactMap { tool in
|
||||
guard let name = tool["name"] as? String else { return nil }
|
||||
return ToolSpec(name: name, description: (tool["description"] as? String) ?? "",
|
||||
parameters: (tool["parameters"] as? [String: Any]) ?? [:])
|
||||
}
|
||||
let history = (request["history"] as? [[String: Any]]) ?? []
|
||||
|
||||
// Transformations of user text (translate, summarize) get the permissive
|
||||
// guardrails; they apply to plain string replies, which tool-free requests are.
|
||||
let model = SystemLanguageModel(useCase: .general,
|
||||
guardrails: permissive ? .permissiveContentTransformations : .default)
|
||||
var offered = toolSpecs
|
||||
if !toolSpecs.isEmpty {
|
||||
switch await selectTools(model: model, specs: toolSpecs, history: history, prompt: prompt) {
|
||||
case .tools(let specs): offered = specs
|
||||
case .failed(let event):
|
||||
send(event)
|
||||
return
|
||||
}
|
||||
if closed { return }
|
||||
}
|
||||
let fitted = await fit(model: model, system: system, tools: offered, history: history,
|
||||
prompt: prompt, maxTokens: maxTokens)
|
||||
if closed { return }
|
||||
|
||||
let handler: @Sendable (String, String) async throws -> String = { [weak self] name, arguments in
|
||||
guard let self = self else { throw CancellationError() }
|
||||
return try await self.toolInvoked(name: name, arguments: arguments)
|
||||
}
|
||||
let tools: [any Tool] = fitted.tools.map {
|
||||
BridgedTool(name: $0.name, description: $0.description, parameters: $0.schema, handler: handler)
|
||||
}
|
||||
let session = LanguageModelSession(model: model, tools: tools, transcript: Transcript(entries: fitted.entries))
|
||||
let options = GenerationOptions(maximumResponseTokens: maxTokens)
|
||||
var last = ""
|
||||
do {
|
||||
for try await snapshot in session.streamResponse(to: prompt, options: options) {
|
||||
if closed { return }
|
||||
let text = snapshot.content
|
||||
// Snapshots are cumulative; one that doesn't extend the last is a new segment.
|
||||
let delta = text.hasPrefix(last) ? String(text.dropFirst(last.count)) : text
|
||||
last = text
|
||||
if !delta.isEmpty { send(["kind": "delta", "text": delta]) }
|
||||
}
|
||||
send(["kind": "done"])
|
||||
} catch {
|
||||
if closed || error is CancellationError { return }
|
||||
send(describe(error))
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Tool calls
|
||||
|
||||
private func toolInvoked(name: String, arguments: String) async throws -> String {
|
||||
if closed { throw CancellationError() }
|
||||
return try await withCheckedThrowingContinuation { continuation in
|
||||
if closed {
|
||||
continuation.resume(throwing: CancellationError())
|
||||
return
|
||||
}
|
||||
let id = "fm_" + UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(16).lowercased()
|
||||
unreported.append(PendingCall(id: id, name: name, arguments: arguments, continuation: continuation))
|
||||
scheduleFlush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Report calls in one event, after a moment for any parallel calls to arrive.
|
||||
private func scheduleFlush() {
|
||||
if flushScheduled { return }
|
||||
flushScheduled = true
|
||||
Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: LanguageModelEngine.toolBatchDelayNs)
|
||||
self?.flush()
|
||||
}
|
||||
}
|
||||
|
||||
private func flush() {
|
||||
flushScheduled = false
|
||||
if closed || unreported.isEmpty { return }
|
||||
let calls = unreported
|
||||
unreported.removeAll()
|
||||
for call in calls { awaiting[call.id] = call }
|
||||
send(["kind": "toolCalls",
|
||||
"calls": calls.map { ["id": $0.id, "name": $0.name, "arguments": $0.arguments] }])
|
||||
}
|
||||
|
||||
// MARK: - Tool routing
|
||||
|
||||
private enum Selection {
|
||||
case tools([ToolSpec])
|
||||
case failed([String: Any])
|
||||
}
|
||||
|
||||
/// All tools when their definitions fit their share of the context window;
|
||||
/// otherwise the few the model picks for this request, plus recently used ones.
|
||||
private func selectTools(model: SystemLanguageModel, specs: [ToolSpec], history: [[String: Any]],
|
||||
prompt: String) async -> Selection {
|
||||
let budget = Int(Double(model.contextSize) * LanguageModelEngine.toolBudgetFraction)
|
||||
let cost = await toolTokens(model, buildTools(specs, compact: false))
|
||||
if cost <= budget { return .tools(specs) }
|
||||
|
||||
let names = specs.map { $0.name }
|
||||
let catalog = specs.map { "- \($0.name): \(SchemaBuilder.shorten($0.description, sentences: 1, limit: 100))" }
|
||||
.joined(separator: "\n")
|
||||
let instructions = "You pick the tools a voice assistant needs to handle the user's latest request. " +
|
||||
"Choose at most \(LanguageModelEngine.maxRoutedTools) tools from this list that the assistant should be able to use, " +
|
||||
"most important first. Choose none for small talk or questions it can answer from general knowledge.\n\n" + catalog
|
||||
var query = prompt
|
||||
if let previous = lastPromptText(history) {
|
||||
query = "Previous request: \(SchemaBuilder.shorten(previous, sentences: 3, limit: 300))\n\nLatest request: \(prompt)"
|
||||
}
|
||||
do {
|
||||
let schema = try GenerationSchema(root: DynamicGenerationSchema(name: "ToolSelection", properties: [
|
||||
DynamicGenerationSchema.Property(
|
||||
name: "tools", description: "Names of the tools needed, most important first.",
|
||||
schema: DynamicGenerationSchema(arrayOf: DynamicGenerationSchema(type: String.self, guides: [.anyOf(names)]),
|
||||
minimumElements: 0,
|
||||
maximumElements: LanguageModelEngine.maxRoutedTools)),
|
||||
]), dependencies: [])
|
||||
let session = LanguageModelSession(model: model, instructions: instructions)
|
||||
let response = try await session.respond(to: query, schema: schema,
|
||||
options: GenerationOptions(sampling: .greedy, maximumResponseTokens: 100))
|
||||
var chosen = Set(try response.content.value([String].self, forProperty: "tools"))
|
||||
let recent = recentToolNames(history).prefix(LanguageModelEngine.keptRecentTools)
|
||||
chosen.formUnion(recent)
|
||||
let selected = specs.filter { chosen.contains($0.name) }
|
||||
send(["kind": "log", "message": "routed \(specs.count) tools (\(cost) tokens) to: " +
|
||||
(selected.isEmpty ? "none" : selected.map { $0.name }.joined(separator: ", "))])
|
||||
return .tools(selected)
|
||||
} catch {
|
||||
if let generationError = error as? LanguageModelSession.GenerationError {
|
||||
switch generationError {
|
||||
case .rateLimited, .concurrentRequests, .assetsUnavailable, .guardrailViolation, .refusal,
|
||||
.unsupportedLanguageOrLocale:
|
||||
// The real request would fail the same way.
|
||||
return .failed(describe(error))
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
send(["kind": "log", "message": "tool routing failed (\(error.localizedDescription)); offering tools in order"])
|
||||
return .tools(specs)
|
||||
}
|
||||
}
|
||||
|
||||
private func toolTokens(_ model: SystemLanguageModel, _ tools: [BuiltTool]) async -> Int {
|
||||
let definitions = tools.map {
|
||||
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
|
||||
}
|
||||
let instructions = Transcript.Entry.instructions(Transcript.Instructions(segments: [], toolDefinitions: definitions))
|
||||
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
|
||||
return await tokenCount(model, [instructions], extraCharacters: schemaCharacters + 150 * tools.count)
|
||||
}
|
||||
|
||||
private func lastPromptText(_ history: [[String: Any]]) -> String? {
|
||||
for item in history.reversed() where item["type"] as? String == "prompt" {
|
||||
if let text = item["text"] as? String, !text.isEmpty { return text }
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// Tool names from the newest tool calls back, without repeats.
|
||||
private func recentToolNames(_ history: [[String: Any]]) -> [String] {
|
||||
var names: [String] = []
|
||||
for item in history.reversed() where item["type"] as? String == "toolCalls" {
|
||||
for call in (item["calls"] as? [[String: Any]]) ?? [] {
|
||||
if let name = call["name"] as? String, !names.contains(name) { names.append(name) }
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// MARK: - Context fitting
|
||||
|
||||
private struct BuiltTool {
|
||||
let name: String
|
||||
let description: String
|
||||
let schema: GenerationSchema
|
||||
}
|
||||
|
||||
private struct Fitted {
|
||||
let tools: [BuiltTool]
|
||||
let entries: [Transcript.Entry]
|
||||
}
|
||||
|
||||
/// Instructions + tool definitions + history + prompt must leave the reply
|
||||
/// budget free. In order, until it fits: shorten tool descriptions, drop the
|
||||
/// oldest exchanges, then drop the last-registered tools.
|
||||
private func fit(model: SystemLanguageModel, system: String, tools specs: [ToolSpec],
|
||||
history: [[String: Any]], prompt: String, maxTokens: Int) async -> Fitted {
|
||||
let contextSize = model.contextSize
|
||||
let reserve = min(maxTokens, contextSize / 3) + LanguageModelEngine.contextMargin
|
||||
var tools = buildTools(specs, compact: false)
|
||||
var past = transcriptEntries(history)
|
||||
let promptEntry = Transcript.Entry.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: prompt))]))
|
||||
|
||||
func entries() -> [Transcript.Entry] {
|
||||
let definitions = tools.map {
|
||||
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
|
||||
}
|
||||
let instructions = Transcript.Instructions(segments: [.text(Transcript.TextSegment(content: system))],
|
||||
toolDefinitions: definitions)
|
||||
return [.instructions(instructions)] + past
|
||||
}
|
||||
func overBudget() async -> Bool {
|
||||
// The tokenizer can't see tool schemas' size from a ToolDefinition; the fallback estimate adds them.
|
||||
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
|
||||
let used = await tokenCount(model, entries() + [promptEntry], extraCharacters: schemaCharacters)
|
||||
return used + reserve > contextSize
|
||||
}
|
||||
|
||||
if !tools.isEmpty, await overBudget() {
|
||||
tools = buildTools(specs, compact: true)
|
||||
send(["kind": "log", "message": "shortened tool descriptions to fit the context window"])
|
||||
}
|
||||
var droppedEntries = 0
|
||||
while !past.isEmpty, await overBudget() {
|
||||
// Drop the oldest exchange: through to the next user prompt.
|
||||
past.removeFirst()
|
||||
droppedEntries += 1
|
||||
while let first = past.first, !isPrompt(first) {
|
||||
past.removeFirst()
|
||||
droppedEntries += 1
|
||||
}
|
||||
}
|
||||
if droppedEntries > 0 {
|
||||
send(["kind": "log", "message": "dropped \(droppedEntries) history entries to fit the context window"])
|
||||
}
|
||||
var droppedTools: [String] = []
|
||||
while !tools.isEmpty, await overBudget() {
|
||||
droppedTools.append(tools.removeLast().name)
|
||||
}
|
||||
if !droppedTools.isEmpty {
|
||||
send(["kind": "log", "message": "dropped tools to fit the context window: \(droppedTools.joined(separator: ", "))"])
|
||||
}
|
||||
return Fitted(tools: tools, entries: entries())
|
||||
}
|
||||
|
||||
private func tokenCount(_ model: SystemLanguageModel, _ entries: [Transcript.Entry], extraCharacters: Int) async -> Int {
|
||||
if #available(iOS 26.4, macOS 26.4, *) {
|
||||
if let count = try? await model.tokenCount(for: entries) { return count }
|
||||
}
|
||||
// Before iOS 26.4 there's no tokenizer API: estimate conservatively.
|
||||
var characters = extraCharacters
|
||||
for entry in entries { characters += estimateCharacters(entry) + 16 }
|
||||
return characters / 3
|
||||
}
|
||||
|
||||
private func estimateCharacters(_ entry: Transcript.Entry) -> Int {
|
||||
func segments(_ list: [Transcript.Segment]) -> Int {
|
||||
list.reduce(0) { total, segment in
|
||||
switch segment {
|
||||
case .text(let text): return total + text.content.count
|
||||
case .structure(let structure): return total + structure.content.jsonString.count
|
||||
@unknown default: return total
|
||||
}
|
||||
}
|
||||
}
|
||||
switch entry {
|
||||
case .instructions(let instructions):
|
||||
var total = segments(instructions.segments)
|
||||
for definition in instructions.toolDefinitions {
|
||||
total += definition.name.count + definition.description.count
|
||||
}
|
||||
return total
|
||||
case .prompt(let prompt): return segments(prompt.segments)
|
||||
case .response(let response): return segments(response.segments)
|
||||
case .toolCalls(let calls): return calls.reduce(0) { $0 + $1.toolName.count + $1.arguments.jsonString.count }
|
||||
case .toolOutput(let output): return segments(output.segments)
|
||||
@unknown default: return 0
|
||||
}
|
||||
}
|
||||
|
||||
private func isPrompt(_ entry: Transcript.Entry) -> Bool {
|
||||
if case .prompt = entry { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
// MARK: - Request conversion
|
||||
|
||||
private func transcriptEntries(_ history: [[String: Any]]) -> [Transcript.Entry] {
|
||||
var entries: [Transcript.Entry] = []
|
||||
for item in history {
|
||||
let text = (item["text"] as? String) ?? ""
|
||||
switch item["type"] as? String {
|
||||
case "prompt":
|
||||
entries.append(.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: text))])))
|
||||
case "response":
|
||||
entries.append(.response(Transcript.Response(assetIDs: [], segments: [.text(Transcript.TextSegment(content: text))])))
|
||||
case "toolCalls":
|
||||
let calls: [Transcript.ToolCall] = ((item["calls"] as? [[String: Any]]) ?? []).compactMap { call in
|
||||
guard let id = call["id"] as? String, let name = call["name"] as? String else { return nil }
|
||||
let arguments = (try? GeneratedContent(json: (call["arguments"] as? String) ?? "{}"))
|
||||
?? GeneratedContent(properties: [:])
|
||||
return Transcript.ToolCall(id: id, toolName: name, arguments: arguments)
|
||||
}
|
||||
if !calls.isEmpty { entries.append(.toolCalls(Transcript.ToolCalls(calls))) }
|
||||
case "toolOutput":
|
||||
guard let id = item["id"] as? String else { continue }
|
||||
entries.append(.toolOutput(Transcript.ToolOutput(
|
||||
id: id, toolName: (item["name"] as? String) ?? "tool",
|
||||
segments: [.text(Transcript.TextSegment(content: text))])))
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
private func buildTools(_ specs: [ToolSpec], compact: Bool) -> [BuiltTool] {
|
||||
specs.map { spec in
|
||||
var builder = SchemaBuilder(compact: compact)
|
||||
let schema: GenerationSchema
|
||||
do {
|
||||
schema = try GenerationSchema(root: builder.object(spec.parameters, name: "\(spec.name)_arguments"),
|
||||
dependencies: [])
|
||||
} catch {
|
||||
send(["kind": "log", "message": "tool \(spec.name): unusable parameter schema (\(error)); offering it without parameters"])
|
||||
schema = (try? GenerationSchema(root: DynamicGenerationSchema(name: "\(spec.name)_arguments", properties: []),
|
||||
dependencies: []))
|
||||
?? GenerationSchema(type: GeneratedContent.self, properties: [])
|
||||
}
|
||||
let description = compact ? SchemaBuilder.shorten(spec.description, sentences: 1, limit: 160) : spec.description
|
||||
return BuiltTool(name: spec.name, description: description, schema: schema)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Errors
|
||||
|
||||
private func describe(_ error: Error) -> [String: Any] {
|
||||
if let error = error as? LanguageModelSession.GenerationError {
|
||||
switch error {
|
||||
case .exceededContextWindowSize:
|
||||
return failure("context", "That conversation is too long for the on-device model. Start a new conversation and try again.")
|
||||
case .assetsUnavailable:
|
||||
return failure("unavailable", "The Apple Intelligence model isn't available right now. Check that Apple Intelligence is on and finished downloading.")
|
||||
case .guardrailViolation, .refusal:
|
||||
return failure("refusal", "The on-device model declined this request.")
|
||||
case .unsupportedLanguageOrLocale:
|
||||
return failure("language", "The on-device model doesn't support this language.")
|
||||
case .rateLimited:
|
||||
return failure("rateLimited", "iOS is limiting the on-device model while Faceclaw is in the background. Try again in a moment, or with the phone unlocked.")
|
||||
case .concurrentRequests:
|
||||
return failure("busy", "The on-device model is busy with another request. Try again in a moment.")
|
||||
case .decodingFailure, .unsupportedGuide:
|
||||
return failure("other", "The on-device model produced an unreadable reply: \(error.localizedDescription)")
|
||||
@unknown default:
|
||||
return failure("other", "On-device model failed: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
if let error = error as? LanguageModelSession.ToolCallError {
|
||||
return failure("other", "On-device model tool call failed: \(error.underlyingError.localizedDescription)")
|
||||
}
|
||||
return failure("other", "On-device model failed: \(error.localizedDescription)")
|
||||
}
|
||||
|
||||
private func failure(_ code: String, _ message: String) -> [String: Any] {
|
||||
["kind": "error", "code": code, "message": message]
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON Schema (the subset our tool definitions use: objects of string, number,
|
||||
/// integer, boolean, enum and array properties) to a DynamicGenerationSchema, so
|
||||
/// the framework constrains tool arguments to valid values while decoding.
|
||||
/// Anything it doesn't understand becomes a string rather than failing.
|
||||
@available(iOS 26.0, *)
|
||||
private struct SchemaBuilder {
|
||||
let compact: Bool
|
||||
private var names = Set<String>()
|
||||
|
||||
init(compact: Bool) { self.compact = compact }
|
||||
|
||||
mutating func object(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
|
||||
let properties = (json["properties"] as? [String: Any]) ?? [:]
|
||||
let required = Set((json["required"] as? [String]) ?? [])
|
||||
// JSON objects lose declaration order; keep required properties first, then by name.
|
||||
let keys = properties.keys.sorted { a, b in
|
||||
let ra = required.contains(a), rb = required.contains(b)
|
||||
return ra != rb ? ra : a < b
|
||||
}
|
||||
let members: [DynamicGenerationSchema.Property] = keys.compactMap { key in
|
||||
guard let property = properties[key] as? [String: Any] else { return nil }
|
||||
return DynamicGenerationSchema.Property(
|
||||
name: key,
|
||||
description: describe(property),
|
||||
schema: value(property, name: "\(name)_\(key)"),
|
||||
isOptional: !required.contains(key))
|
||||
}
|
||||
return DynamicGenerationSchema(name: unique(name), description: nil, properties: members)
|
||||
}
|
||||
|
||||
private mutating func value(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
|
||||
if let choices = json["enum"] as? [Any], !choices.isEmpty {
|
||||
let strings = choices.compactMap { $0 as? String }
|
||||
if strings.count == choices.count {
|
||||
// A guide renders as one compact JSON Schema enum; an anyOf schema lists
|
||||
// every choice as its own subschema, at over twice the tokens.
|
||||
return DynamicGenerationSchema(type: String.self, guides: [.anyOf(strings)])
|
||||
}
|
||||
}
|
||||
switch schemaType(json) {
|
||||
case "integer":
|
||||
var guides: [GenerationGuide<Int>] = []
|
||||
if let minimum = (json["minimum"] as? NSNumber)?.intValue { guides.append(.minimum(minimum)) }
|
||||
if let maximum = (json["maximum"] as? NSNumber)?.intValue { guides.append(.maximum(maximum)) }
|
||||
return DynamicGenerationSchema(type: Int.self, guides: guides)
|
||||
case "number":
|
||||
var guides: [GenerationGuide<Double>] = []
|
||||
if let minimum = (json["minimum"] as? NSNumber)?.doubleValue { guides.append(.minimum(minimum)) }
|
||||
if let maximum = (json["maximum"] as? NSNumber)?.doubleValue { guides.append(.maximum(maximum)) }
|
||||
return DynamicGenerationSchema(type: Double.self, guides: guides)
|
||||
case "boolean":
|
||||
return DynamicGenerationSchema(type: Bool.self)
|
||||
case "array":
|
||||
let items = (json["items"] as? [String: Any]) ?? ["type": "string"]
|
||||
return DynamicGenerationSchema(arrayOf: value(items, name: "\(name)_item"),
|
||||
minimumElements: (json["minItems"] as? NSNumber)?.intValue,
|
||||
maximumElements: (json["maxItems"] as? NSNumber)?.intValue)
|
||||
case "object":
|
||||
return object(json, name: name)
|
||||
default:
|
||||
return DynamicGenerationSchema(type: String.self)
|
||||
}
|
||||
}
|
||||
|
||||
private func schemaType(_ json: [String: Any]) -> String {
|
||||
if let type = json["type"] as? String { return type }
|
||||
if let types = json["type"] as? [String], let type = types.first(where: { $0 != "null" }) { return type }
|
||||
return json["properties"] != nil ? "object" : "string"
|
||||
}
|
||||
|
||||
private func describe(_ json: [String: Any]) -> String? {
|
||||
guard let text = json["description"] as? String, !text.isEmpty else { return nil }
|
||||
return compact ? SchemaBuilder.shorten(text, sentences: 1, limit: 80) : text
|
||||
}
|
||||
|
||||
/// Schema names must be unique within one GenerationSchema.
|
||||
private mutating func unique(_ base: String) -> String {
|
||||
var name = base
|
||||
var suffix = 2
|
||||
while names.contains(name) {
|
||||
name = "\(base)_\(suffix)"
|
||||
suffix += 1
|
||||
}
|
||||
names.insert(name)
|
||||
return name
|
||||
}
|
||||
|
||||
/// The first `sentences` sentences, cut to `limit` characters.
|
||||
static func shorten(_ text: String, sentences: Int, limit: Int) -> String {
|
||||
var result = ""
|
||||
var count = 0
|
||||
var remainder = Substring(text)
|
||||
while count < sentences, !remainder.isEmpty {
|
||||
if let end = remainder.range(of: ". ") {
|
||||
result += remainder[..<end.lowerBound] + "."
|
||||
remainder = remainder[end.upperBound...]
|
||||
if count + 1 < sentences { result += " " }
|
||||
} else {
|
||||
result += remainder
|
||||
remainder = ""
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
if result.count > limit { result = String(result.prefix(limit - 1)) + "…" }
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -7,6 +7,7 @@
|
||||
* Ensure connection is re-set to 2M on reconnect
|
||||
* Add support for embedded icons and custom layouts in Android notifications
|
||||
* Port the Developer app's resource-usage monitor and various test pages to iOS
|
||||
* iOS: Support Apple Intelligence as an on-device model for voice assistant/etc
|
||||
* iOS: Notifications are selectable via the task switcher
|
||||
* Android: Optionally sign in to Spotify's Web API for library browsing and other features
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* surfacing a transport error.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
@@ -41,7 +41,7 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
streamOpenAiResponse(options);
|
||||
break;
|
||||
case "local":
|
||||
streamLocalQwen(options);
|
||||
streamLocalModel(options);
|
||||
break;
|
||||
default:
|
||||
streamAnthropicMessage(options);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
||||
|
||||
@@ -245,7 +245,7 @@ export function askConversations(
|
||||
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
return streamLocalQwen({
|
||||
return streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { NameInference } from "./name-guess";
|
||||
|
||||
/**
|
||||
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import { speakerRegistry } from "./speakers";
|
||||
|
||||
declare const com: any;
|
||||
@@ -124,7 +124,7 @@ function generateForSpeaker(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* TranslationModel for how stretches are batched and given context.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
@@ -55,7 +55,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
case "openai":
|
||||
return streamOpenAiResponse(options);
|
||||
case "local":
|
||||
return streamLocalQwen(options);
|
||||
return streamLocalModel(options);
|
||||
default:
|
||||
return streamAnthropicMessage(options);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { streamAnthropicMessage } from "../native/anthropic";
|
||||
import { streamHermesChat } from "../native/hermes";
|
||||
import { streamLocalQwen } from "../native/llama";
|
||||
import { streamLocalModel } from "../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../native/openai";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
@@ -146,7 +146,7 @@ export class DirectAssistantBackend {
|
||||
: options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: options.provider === "local"
|
||||
? streamLocalQwen(streamOptions)
|
||||
? streamLocalModel(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
};
|
||||
|
||||
|
||||
+13
-7
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
|
||||
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
|
||||
|
||||
export const ASSISTANT_MODEL_VALUES = [
|
||||
"auto",
|
||||
@@ -13,12 +13,17 @@ export const ASSISTANT_MODEL_VALUES = [
|
||||
"hermes",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
|
||||
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
|
||||
* support and stays for saved settings and conversations).
|
||||
*/
|
||||
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
|
||||
/** Keep saved model IDs portable, while offering only supported choices. */
|
||||
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
|
||||
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
|
||||
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
|
||||
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
|
||||
return global.isIOS && value === "qwen" ? "auto" : value;
|
||||
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
|
||||
}
|
||||
|
||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
|
||||
@@ -58,8 +63,9 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
|
||||
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
|
||||
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
|
||||
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
|
||||
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
|
||||
// not an API key. The default for users who haven't set any key.
|
||||
// Runs on the phone itself (Android: llama.cpp with the downloaded model
|
||||
// file; iOS: Apple Intelligence), not an API key. The default for users who
|
||||
// haven't set any key.
|
||||
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
|
||||
// Your own Hermes Agent (its API server), with its own tools and memory.
|
||||
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
|
||||
@@ -76,8 +82,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
|
||||
/**
|
||||
* Resolve Auto and attach the credential needed by the chosen provider.
|
||||
* Auto intentionally prefers Terra over Sonnet when both keys are available,
|
||||
* and falls back to the on-phone model when no key is set but the model file
|
||||
* has been downloaded.
|
||||
* and falls back to the on-phone model when no key is set but that model is
|
||||
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
|
||||
*/
|
||||
export function resolveAssistantModel(
|
||||
selection: AssistantModel,
|
||||
|
||||
+4
-6
@@ -6,12 +6,14 @@ import type {
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* On-phone LLM backend: model download management plus a streaming client
|
||||
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
|
||||
* local-llm.ios.ts instead): model download management plus a streaming client
|
||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||
* the direct assistant loop can treat "local" as just another provider.
|
||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||
@@ -38,11 +40,7 @@ const TOP_K = 20;
|
||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||
|
||||
export type LocalModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
let runner: any = null;
|
||||
let downloader: any = null;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
|
||||
|
||||
export type LocalModelState = {
|
||||
/**
|
||||
* Android downloads its model: absent → downloading → ready. iOS uses the
|
||||
* system's Apple Intelligence model: ready, or unavailable (see detail).
|
||||
*/
|
||||
status: "absent" | "downloading" | "ready" | "unavailable";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
|
||||
detail?: string;
|
||||
};
|
||||
@@ -0,0 +1,383 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const FaceclawSystemLanguageModel: any;
|
||||
|
||||
/**
|
||||
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
|
||||
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
|
||||
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
|
||||
* Calculator and the Microphones helpers treat it as the "local" provider.
|
||||
*
|
||||
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
|
||||
* background (where Faceclaw usually is, driving the glasses with the phone
|
||||
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
|
||||
* ~3 GB in-process model makes the app the first thing jetsam kills. The
|
||||
* system model runs out of process on the Neural Engine, needs no download,
|
||||
* and decodes tool arguments under a schema constraint (like our GBNF grammar
|
||||
* on Android). Its costs: a 4096-token context (the Swift side fits history
|
||||
* and tool definitions into it), Apple's guardrails, a background request
|
||||
* budget, and Apple Intelligence-capable hardware.
|
||||
*
|
||||
* Tool calls: the framework runs tools itself, mid-response. The Swift side
|
||||
* parks each call and reports it; here that ends the loop iteration with
|
||||
* stopReason "tool_use", the direct backend executes the tools as for any
|
||||
* provider, and its next request (ending in the matching tool_result blocks)
|
||||
* resumes the parked generation instead of starting a new one.
|
||||
*/
|
||||
|
||||
export const LOCAL_MODEL = {
|
||||
id: "apple-foundation-model",
|
||||
label: "Apple Intelligence",
|
||||
sizeBytes: 0,
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_TOKENS = 512;
|
||||
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
|
||||
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
/**
|
||||
* Only used when tool results arrive with no parked generation to resume (it
|
||||
* timed out, or the app restarted mid-turn): the framework has no way to
|
||||
* continue after a tool output without a new prompt.
|
||||
*/
|
||||
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
|
||||
|
||||
type Availability =
|
||||
| "available"
|
||||
| "unsupportedOS"
|
||||
| "deviceNotEligible"
|
||||
| "appleIntelligenceNotEnabled"
|
||||
| "modelNotReady"
|
||||
| "unavailable";
|
||||
|
||||
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "needs iOS 26",
|
||||
deviceNotEligible: "not on this iPhone",
|
||||
appleIntelligenceNotEnabled: "Apple Intelligence off",
|
||||
modelNotReady: "downloading",
|
||||
unavailable: "unavailable",
|
||||
};
|
||||
|
||||
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
|
||||
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
|
||||
appleIntelligenceNotEnabled:
|
||||
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
|
||||
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
|
||||
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
|
||||
};
|
||||
|
||||
export function localModelAvailability(): Availability {
|
||||
try {
|
||||
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
|
||||
const value = String(FaceclawSystemLanguageModel.availability());
|
||||
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
|
||||
} catch {
|
||||
return "unsupportedOS";
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
const availability = localModelAvailability();
|
||||
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
|
||||
}
|
||||
|
||||
export function isLocalModelReady(): boolean {
|
||||
return localModelAvailability() === "available";
|
||||
}
|
||||
|
||||
export function localModelState(): LocalModelState {
|
||||
const availability = localModelAvailability();
|
||||
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
|
||||
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
|
||||
}
|
||||
|
||||
/** The user-facing reason the model can't run, or "" when it can. */
|
||||
export function localModelUnavailableMessage(): string {
|
||||
const availability = localModelAvailability();
|
||||
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
|
||||
}
|
||||
|
||||
// iOS manages the system model itself: there is nothing to download, cancel,
|
||||
// delete or unload, and no progress to watch.
|
||||
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
|
||||
return () => {};
|
||||
}
|
||||
export function startLocalModelDownload(): void {}
|
||||
export function cancelLocalModelDownload(): void {}
|
||||
export function deleteLocalModel(): void {}
|
||||
export function unloadLocalModel(): void {}
|
||||
|
||||
type HistoryEntry =
|
||||
| { type: "prompt" | "response"; text: string }
|
||||
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
||||
| { type: "toolOutput"; id: string; name: string; text: string };
|
||||
|
||||
export type SystemModelRequest = {
|
||||
system: string;
|
||||
tools: Array<{ name: string; description: string; parameters: object }>;
|
||||
history: HistoryEntry[];
|
||||
prompt: string;
|
||||
maxTokens: number;
|
||||
permissive: boolean;
|
||||
};
|
||||
|
||||
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
|
||||
|
||||
/**
|
||||
* Translate provider-neutral history into the Swift request: everything but
|
||||
* the newest user prompt becomes Transcript entries, and that prompt is what
|
||||
* the model answers.
|
||||
*/
|
||||
export function buildSystemModelRequest(
|
||||
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
|
||||
): SystemModelRequest {
|
||||
const messages = options.messages;
|
||||
const last = messages[messages.length - 1];
|
||||
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
|
||||
const history: HistoryEntry[] = [];
|
||||
const toolNames = new Map<string, string>();
|
||||
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
|
||||
appendHistory(history, message, toolNames);
|
||||
}
|
||||
const tools = (options.tools ?? []).map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
}));
|
||||
return {
|
||||
system: options.system?.trim() || "You are a helpful assistant.",
|
||||
tools,
|
||||
history,
|
||||
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
|
||||
permissive: tools.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
|
||||
if (message.role === "user") {
|
||||
for (const result of toolResultsOf(message)) {
|
||||
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
|
||||
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "prompt", text });
|
||||
return;
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "response", text });
|
||||
if (typeof message.content === "string") return;
|
||||
const calls: Array<{ id: string; name: string; arguments: string }> = [];
|
||||
for (const block of message.content) {
|
||||
if (block.type !== "tool_use") continue;
|
||||
toolNames.set(block.id, block.name);
|
||||
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
|
||||
}
|
||||
if (calls.length) history.push({ type: "toolCalls", calls });
|
||||
}
|
||||
|
||||
function textOf(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content.trim();
|
||||
return content
|
||||
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
|
||||
.map((block) => block.text.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
|
||||
if (!message || message.role !== "user" || typeof message.content === "string") return [];
|
||||
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
|
||||
}
|
||||
|
||||
function parseArguments(json: string): any {
|
||||
try {
|
||||
const value = JSON.parse(json);
|
||||
return value && typeof value === "object" ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
type NativeEvent = {
|
||||
kind: string;
|
||||
text?: string;
|
||||
calls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
code?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** Generations parked in tool calls, by call id. */
|
||||
const parked = new Map<string, Generation>();
|
||||
/** Every unfinished generation, so its native object outlives the caller's handle. */
|
||||
const live = new Set<Generation>();
|
||||
|
||||
/**
|
||||
* One native generation, possibly spanning several loop iterations: each
|
||||
* iteration ends at a batch of tool calls (or the end of the reply), and the
|
||||
* next one resumes it with the results.
|
||||
*/
|
||||
class Generation {
|
||||
private readonly native: any;
|
||||
private options: LlmStreamOptions | null;
|
||||
private iteration = 0;
|
||||
private text = "";
|
||||
private callIds: string[] = [];
|
||||
private expiry: ReturnType<typeof setTimeout> | null = null;
|
||||
private closed = false;
|
||||
|
||||
constructor(options: LlmStreamOptions) {
|
||||
this.options = options;
|
||||
this.native = FaceclawSystemLanguageModel.new();
|
||||
this.native.eventHandler = (json: string) => this.onEvent(String(json));
|
||||
live.add(this);
|
||||
}
|
||||
|
||||
start(request: SystemModelRequest): LlmStreamHandle {
|
||||
this.native.start(JSON.stringify(request));
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
/** Whether these results answer this generation's parked calls. */
|
||||
awaits(results: ToolResultBlock[]): boolean {
|
||||
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
|
||||
}
|
||||
|
||||
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
|
||||
this.unpark();
|
||||
this.options = options;
|
||||
this.iteration++;
|
||||
this.text = "";
|
||||
this.native.resolveToolCalls(
|
||||
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
|
||||
);
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (this.closed) return;
|
||||
this.close();
|
||||
try {
|
||||
this.native.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private handle(): LlmStreamHandle {
|
||||
const iteration = this.iteration;
|
||||
// A stale handle (an earlier iteration's) still cancels a generation parked
|
||||
// since then: its owner gave up on the turn.
|
||||
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
|
||||
}
|
||||
|
||||
private close(): void {
|
||||
this.closed = true;
|
||||
this.unpark();
|
||||
this.options = null;
|
||||
this.native.eventHandler = null;
|
||||
live.delete(this);
|
||||
}
|
||||
|
||||
private park(ids: string[]): void {
|
||||
this.callIds = ids;
|
||||
for (const id of ids) parked.set(id, this);
|
||||
this.expiry = setTimeout(() => {
|
||||
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
|
||||
this.cancel();
|
||||
}, PARKED_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private unpark(): void {
|
||||
for (const id of this.callIds) {
|
||||
if (parked.get(id) === this) parked.delete(id);
|
||||
}
|
||||
this.callIds = [];
|
||||
if (this.expiry !== null) clearTimeout(this.expiry);
|
||||
this.expiry = null;
|
||||
}
|
||||
|
||||
private onEvent(json: string): void {
|
||||
if (this.closed) return;
|
||||
let event: NativeEvent;
|
||||
try {
|
||||
event = JSON.parse(json);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.kind === "log") {
|
||||
console.log(`[local-llm] ${event.message ?? ""}`);
|
||||
return;
|
||||
}
|
||||
const options = this.options;
|
||||
if (!options) return;
|
||||
switch (event.kind) {
|
||||
case "delta": {
|
||||
const delta = event.text ?? "";
|
||||
this.text += delta;
|
||||
options.onTextDelta?.(delta, this.text);
|
||||
return;
|
||||
}
|
||||
case "toolCalls": {
|
||||
const calls = event.calls ?? [];
|
||||
const text = this.text.trim();
|
||||
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
|
||||
for (const call of calls) {
|
||||
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
|
||||
}
|
||||
this.options = null;
|
||||
this.park(calls.map((call) => call.id));
|
||||
options.onDone({ text, content, stopReason: "tool_use" });
|
||||
return;
|
||||
}
|
||||
case "done": {
|
||||
this.close();
|
||||
const text = this.text.trim();
|
||||
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
|
||||
return;
|
||||
}
|
||||
case "error": {
|
||||
this.close();
|
||||
if (event.code === "refusal") {
|
||||
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
|
||||
} else {
|
||||
options.onError(event.message || "The on-device model failed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a reply from the system model. A request whose history ends in tool
|
||||
* results for a parked generation resumes it; anything else starts afresh.
|
||||
*/
|
||||
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
|
||||
const availability = localModelAvailability();
|
||||
if (availability !== "available") {
|
||||
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
const results = toolResultsOf(options.messages[options.messages.length - 1]);
|
||||
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
|
||||
if (waiting?.awaits(results)) return waiting.resume(options, results);
|
||||
let generation: Generation | null = null;
|
||||
try {
|
||||
generation = new Generation(options);
|
||||
return generation.start(buildSystemModelRequest(options));
|
||||
} catch (error) {
|
||||
generation?.cancel();
|
||||
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* The on-phone LLM: one entry point for every feature that can run on the
|
||||
* phone without an API key (the direct assistant's "local" provider,
|
||||
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
|
||||
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
|
||||
* contract, plus model status for Settings.
|
||||
*
|
||||
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
|
||||
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
|
||||
* to download, available on Apple Intelligence devices running iOS 26+.
|
||||
*/
|
||||
export {
|
||||
LOCAL_MODEL,
|
||||
isLocalModelReady,
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
unloadLocalModel,
|
||||
streamLocalQwen as streamLocalModel,
|
||||
} from "./llama";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
return true;
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
assistantModelProvider,
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
@@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
export type AssistantBackendKind = "direct" | "external";
|
||||
|
||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||
direct: global.isIOS ? "Cloud API" : "On-phone",
|
||||
direct: "On-phone",
|
||||
external: "My own agent (bridge)",
|
||||
};
|
||||
|
||||
@@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
|
||||
values: global.isIOS ? ["direct"] : ["direct", "external"],
|
||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||
description:
|
||||
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
});
|
||||
|
||||
export const assistantHermesUrlSetting = new ConfigSettingString({
|
||||
@@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
|
||||
return false;
|
||||
},
|
||||
description:
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
});
|
||||
|
||||
export const mapboxApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
} from "../../native/local-llm";
|
||||
import {
|
||||
ASR_MODELS,
|
||||
asrModelState,
|
||||
@@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] {
|
||||
{
|
||||
label: "Assistant",
|
||||
items: [
|
||||
// iOS offers cloud APIs; Android also supports local models and the bridge.
|
||||
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
textSettingMenuItem(assistantHermesUrlSetting),
|
||||
textSettingMenuItem(assistantHermesKeySetting),
|
||||
...(!global.isIOS ? [localModelMenuItem()] : []),
|
||||
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
// iOS has no local LLM or agent bridge yet.
|
||||
// iOS has no agent bridge yet.
|
||||
...(!global.isIOS ? [
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
@@ -399,6 +399,29 @@ function localModelStatusText(): string {
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/**
|
||||
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
|
||||
* installs and manages; the row only reports whether it can be used.
|
||||
*/
|
||||
function systemModelMenuItem(): MenuItem {
|
||||
const statusText = (): string => {
|
||||
const state = localModelState();
|
||||
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
|
||||
};
|
||||
return {
|
||||
label: "On-phone model",
|
||||
description:
|
||||
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
|
||||
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
|
||||
"Faster and more private than the cloud models, but simpler, with a short memory. " +
|
||||
"Used automatically when no API key is set.",
|
||||
onSelect: () => {},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-phone assistant model. */
|
||||
function localModelMenuItem(): MenuItem {
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
// Print the assistant's built-in tool definitions as JSON ([{name, description, parameters}]),
|
||||
// pulled from the tool files' object literals via the TypeScript AST, with the dynamic
|
||||
// parts (the app-id enum, NUMBER_ARG) filled in from a fixed list. An approximation of
|
||||
// what ToolRegistry.listTools() offers with no app windows open.
|
||||
// Usage: node scripts/apple-llm-harness/extract-tools.cjs > tools.json
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const ts = require(path.join(root, 'node_modules/typescript'));
|
||||
|
||||
const FILES = ['system-tools', 'timer-tools', 'window-tools', 'navigate-tools', 'roam-tools']
|
||||
.map(name => path.join(root, 'app/assistant', `${name}.ts`));
|
||||
const APP_IDS = fs.readdirSync(path.join(root, 'app/apps'), { withFileTypes: true })
|
||||
.filter(entry => entry.isDirectory() && !['launcher', 'onboarding', 'evenhub'].includes(entry.name))
|
||||
.map(entry => entry.name).sort();
|
||||
const NUMBER_ARG = (noun, listTool) => ({
|
||||
type: 'number',
|
||||
description: `1-based ${noun} number, as shown by ${listTool}. May be omitted when only one ${noun} exists.`,
|
||||
});
|
||||
|
||||
const out = [];
|
||||
for (const file of FILES) {
|
||||
const src = ts.createSourceFile(file, fs.readFileSync(file, 'utf8'), ts.ScriptTarget.ES2020, true);
|
||||
const consts = {};
|
||||
src.forEachChild(function collect(node) {
|
||||
if (ts.isVariableDeclaration(node) && node.initializer && ts.isStringLiteral(node.initializer)) {
|
||||
consts[node.name.getText()] = node.initializer.text;
|
||||
}
|
||||
node.forEachChild(collect);
|
||||
});
|
||||
const evaluate = node => {
|
||||
const js = ts.transpileModule(`(${node.getText()})`, { compilerOptions: { target: ts.ScriptTarget.ES2020 } }).outputText;
|
||||
const names = [...Object.keys(consts), 'MAX_TIMER_DURATION_MS', 'appIds', 'installedNote', 'NUMBER_ARG'];
|
||||
return Function(...names, `return ${js}`)(...Object.values(consts), 86400000, APP_IDS, '', NUMBER_ARG);
|
||||
};
|
||||
src.forEachChild(function visit(node) {
|
||||
if (ts.isObjectLiteralExpression(node)) {
|
||||
const keys = node.properties.map(p => p.name && p.name.getText());
|
||||
if (keys.includes('name') && keys.includes('description') && keys.includes('inputSchema')) {
|
||||
try {
|
||||
const spec = evaluate(node);
|
||||
out.push({ name: String(spec.name).replace(/[^a-zA-Z0-9_-]/g, '_'), description: spec.description, parameters: spec.inputSchema });
|
||||
} catch (error) {
|
||||
console.error(`skipped a tool in ${path.basename(file)}: ${error.message}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
node.forEachChild(visit);
|
||||
});
|
||||
}
|
||||
process.stdout.write(JSON.stringify(out, null, 1));
|
||||
@@ -0,0 +1,50 @@
|
||||
// macOS harness for App_Resources/iOS/src/FaceclawSystemLanguageModel.swift: runs one
|
||||
// assistant request against the Apple Intelligence model the way local-llm.ios.ts does,
|
||||
// answering every tool call with a canned result. See run.sh.
|
||||
import Foundation
|
||||
|
||||
@main struct Harness {
|
||||
@MainActor static func main() async {
|
||||
let args = CommandLine.arguments
|
||||
guard args.count >= 3 else {
|
||||
print("usage: harness <tools.json> <prompt> [previous prompt]")
|
||||
return
|
||||
}
|
||||
print("availability:", FaceclawSystemLanguageModel.availability(),
|
||||
"context:", FaceclawSystemLanguageModel.contextSize())
|
||||
let tools = (try? JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: args[1])))) ?? []
|
||||
var history: [[String: Any]] = []
|
||||
if args.count > 3 { history = [["type": "prompt", "text": args[3]], ["type": "response", "text": "OK."]] }
|
||||
let request: [String: Any] = [
|
||||
"system": "You are the voice assistant built into a pair of Even Realities G2 smart glasses. Your reply is shown as text on a 640x480 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list. Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it. If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
|
||||
"tools": tools, "history": history, "prompt": args[2], "maxTokens": 512, "permissive": false,
|
||||
]
|
||||
let model = FaceclawSystemLanguageModel()
|
||||
let started = Date()
|
||||
var finished = false
|
||||
var firstText: TimeInterval?
|
||||
model.eventHandler = { json in
|
||||
let elapsed = String(format: "%6.2fs", Date().timeIntervalSince(started))
|
||||
guard let event = (try? JSONSerialization.jsonObject(with: Data(json.utf8))) as? [String: Any] else { return }
|
||||
switch event["kind"] as? String {
|
||||
case "delta":
|
||||
if firstText == nil { firstText = Date().timeIntervalSince(started); print("\(elapsed) first text") }
|
||||
print(event["text"] as? String ?? "", terminator: "")
|
||||
case "toolCalls":
|
||||
let calls = event["calls"] as? [[String: Any]] ?? []
|
||||
for call in calls { print("\(elapsed) tool call: \(call["name"] ?? "") \(call["arguments"] ?? "")") }
|
||||
let results = calls.map { ["id": $0["id"] ?? "", "content": "Done (simulated \($0["name"] ?? "tool") result).", "isError": false] }
|
||||
let data = try! JSONSerialization.data(withJSONObject: results)
|
||||
model.resolveToolCalls(String(data: data, encoding: .utf8)!)
|
||||
case "done":
|
||||
print("\n\(elapsed) done"); finished = true
|
||||
default:
|
||||
print("\(elapsed) \(json)")
|
||||
if event["kind"] as? String == "error" { finished = true }
|
||||
}
|
||||
}
|
||||
let data = try! JSONSerialization.data(withJSONObject: request)
|
||||
model.start(String(data: data, encoding: .utf8)!)
|
||||
while !finished && Date().timeIntervalSince(started) < 120 { try? await Task.sleep(nanoseconds: 50_000_000) }
|
||||
}
|
||||
}
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Run one assistant request against Apple's on-device model on this Mac (macOS 26.4+,
|
||||
# Apple Intelligence on), through the same Swift bridge the iOS app uses, with the
|
||||
# assistant's real tool list. Prints tool routing, tool calls and the streamed reply.
|
||||
# scripts/apple-llm-harness/run.sh "set a timer for five minutes" ["previous prompt"]
|
||||
set -euo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
out="${TMPDIR:-/tmp}/faceclaw-apple-llm-harness"
|
||||
mkdir -p "$out"
|
||||
node "$here/extract-tools.cjs" > "$out/tools.json"
|
||||
swiftc -swift-version 5 -parse-as-library -target "$(uname -m)-apple-macos26.4" \
|
||||
"$here/../../App_Resources/iOS/src/FaceclawSystemLanguageModel.swift" "$here/main.swift" -o "$out/harness"
|
||||
"$out/harness" "$out/tools.json" "$@"
|
||||
@@ -91,7 +91,7 @@ test('a restored conversation keeps its Hermes session, and Hermes turns get a l
|
||||
requests.push(request);
|
||||
return request;
|
||||
} },
|
||||
'../native/llama': { streamLocalQwen: () => assert.fail('no local model') },
|
||||
'../native/local-llm': { streamLocalModel: () => assert.fail('no local model') },
|
||||
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: {} },
|
||||
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' },
|
||||
});
|
||||
@@ -122,7 +122,7 @@ test('URL normalization adds /v1 to a bare host and drops the endpoint path', ()
|
||||
});
|
||||
|
||||
test('the Hermes model resolves only with a URL, carrying the key and base URL', () => {
|
||||
const load = loader({ global: { isIOS: true } }, { '../native/llama': { isLocalModelReady: () => false, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
|
||||
const load = loader({ global: { isIOS: true } }, { '../native/local-llm': { isLocalModelReady: () => false, isLocalModelSupported: () => true, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
|
||||
const { resolveAssistantModel } = load('app/assistant/models.ts');
|
||||
assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null);
|
||||
const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' });
|
||||
|
||||
@@ -12,7 +12,7 @@ function assistant(provider) {
|
||||
const request = { url, body: JSON.parse(body), headers, listener, cancelled: false };
|
||||
requests.push(request); return { cancel: () => { request.cancelled = true; } };
|
||||
} },
|
||||
'../native/llama': { streamLocalQwen: () => assert.fail('iOS must not invoke a local LLM') },
|
||||
'../native/local-llm': { streamLocalModel: () => assert.fail('cloud providers must not invoke the local LLM') },
|
||||
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
|
||||
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'test', buildAssistantSystemPrompt: () => 'test', describeAssistantContext: () => '' },
|
||||
});
|
||||
@@ -100,8 +100,8 @@ test('iOS SSE adapter forwards the Kotlin stream events once and stops after can
|
||||
assert.deepEqual(seen.at(-1), 'http:429:slow down');
|
||||
});
|
||||
|
||||
test('iOS only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
|
||||
const api = loader({ global: { isIOS: true } }, { '../native/llama': { LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false } })('app/assistant/models.ts');
|
||||
test('iOS without Apple Intelligence support only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
|
||||
const api = loader({ global: { isIOS: true } }, { '../native/local-llm': { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => false, isLocalModelSupported: () => false } })('app/assistant/models.ts');
|
||||
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), false);
|
||||
assert.equal(api.supportedAssistantModel('qwen'), 'auto');
|
||||
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai');
|
||||
@@ -119,7 +119,7 @@ test('cancelling during a tool prevents subsequent tool side effects and a stall
|
||||
const calls = [], failures = [];
|
||||
const { DirectAssistantBackend } = loader({ setTimeout: fn => { deadline = fn; return 1; }, clearTimeout() {} }, {
|
||||
'../native/openai': { streamOpenAiResponse: options => { stream = options; return { cancel() {} }; } },
|
||||
'../native/anthropic': {}, '../native/llama': {},
|
||||
'../native/anthropic': {}, '../native/local-llm': {},
|
||||
})('app/assistant/direct-backend.ts');
|
||||
const backend = new DirectAssistantBackend();
|
||||
const options = { provider: 'openai', messages: [], buildTools: () => [], resolveToolName: name => name,
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const flush = () => new Promise(resolve => setImmediate(resolve));
|
||||
// Values built inside the loader's VM context have that realm's prototypes.
|
||||
const plain = value => JSON.parse(JSON.stringify(value));
|
||||
|
||||
/** A stand-in for FaceclawSystemLanguageModel.swift: records requests and lets tests emit its events. */
|
||||
function fakeNative(availability = 'available') {
|
||||
const instances = [];
|
||||
const cls = {
|
||||
availability: () => availability,
|
||||
new() {
|
||||
const native = {
|
||||
request: null, resolved: [], cancelled: false, eventHandler: null,
|
||||
start(json) { this.request = JSON.parse(json); },
|
||||
resolveToolCalls(json) { this.resolved.push(JSON.parse(json)); },
|
||||
cancel() { this.cancelled = true; },
|
||||
emit(event) { if (this.eventHandler) this.eventHandler(JSON.stringify(event)); },
|
||||
};
|
||||
instances.push(native);
|
||||
return native;
|
||||
},
|
||||
};
|
||||
return { cls, instances, setAvailability(value) { availability = value; } };
|
||||
}
|
||||
|
||||
function loadLocalLlm(native, extraGlobals = {}) {
|
||||
const load = loader({ FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout, ...extraGlobals }, {});
|
||||
return load('app/native/local-llm.ios.ts');
|
||||
}
|
||||
|
||||
function collector() {
|
||||
const seen = { deltas: [], done: [], errors: [] };
|
||||
return {
|
||||
seen,
|
||||
callbacks: {
|
||||
onTextDelta: (_delta, full) => seen.deltas.push(full),
|
||||
onDone: result => seen.done.push(result),
|
||||
onError: message => seen.errors.push(message),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('iOS local LLM converts provider-neutral history into transcript entries and a final prompt', () => {
|
||||
const api = loadLocalLlm(fakeNative());
|
||||
const request = api.buildSystemModelRequest({
|
||||
system: 'Be brief.',
|
||||
maxTokens: 200,
|
||||
tools: [{ name: 'timer_set', description: 'Set a timer', input_schema: { type: 'object', properties: { seconds: { type: 'integer' } } } }],
|
||||
messages: [
|
||||
{ role: 'user', content: 'Set a timer' },
|
||||
{ role: 'assistant', content: [{ type: 'text', text: 'On it.' }, { type: 'tool_use', id: 'fm_1', name: 'timer_set', input: { seconds: 60 } }] },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'Timer set' }] },
|
||||
{ role: 'assistant', content: 'Done.' },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'gone', content: 'boom', is_error: true }, { type: 'text', text: 'And now?' }] },
|
||||
{ role: 'assistant', content: [{ type: 'provider_item', provider: 'openai', item: {} }] },
|
||||
{ role: 'user', content: 'What time is it?' },
|
||||
],
|
||||
});
|
||||
assert.equal(request.system, 'Be brief.');
|
||||
assert.equal(request.prompt, 'What time is it?');
|
||||
assert.equal(request.maxTokens, 200);
|
||||
assert.equal(request.permissive, false);
|
||||
assert.deepEqual(plain(request.tools), [{ name: 'timer_set', description: 'Set a timer', parameters: { type: 'object', properties: { seconds: { type: 'integer' } } } }]);
|
||||
assert.deepEqual(plain(request.history), [
|
||||
{ type: 'prompt', text: 'Set a timer' },
|
||||
{ type: 'response', text: 'On it.' },
|
||||
{ type: 'toolCalls', calls: [{ id: 'fm_1', name: 'timer_set', arguments: '{"seconds":60}' }] },
|
||||
{ type: 'toolOutput', id: 'fm_1', name: 'timer_set', text: 'Timer set' },
|
||||
{ type: 'response', text: 'Done.' },
|
||||
{ type: 'toolOutput', id: 'gone', name: 'tool', text: 'Error: boom' },
|
||||
{ type: 'prompt', text: 'And now?' },
|
||||
]);
|
||||
|
||||
// Tool-free requests (translate, Q&A) get the permissive guardrails; a
|
||||
// trailing tool result with nothing to resume falls back to a continuation prompt.
|
||||
const orphan = api.buildSystemModelRequest({
|
||||
messages: [{ role: 'user', content: 'x' }, { role: 'assistant', content: [{ type: 'tool_use', id: 'a', name: 't', input: {} }] },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'a', content: 'ok' }] }],
|
||||
});
|
||||
assert.equal(orphan.permissive, true);
|
||||
assert.equal(orphan.maxTokens, 512);
|
||||
assert.equal(orphan.prompt, api.CONTINUE_PROMPT);
|
||||
assert.deepEqual(plain(orphan.history.at(-1)), { type: 'toolOutput', id: 'a', name: 't', text: 'ok' });
|
||||
});
|
||||
|
||||
test('iOS local LLM streams text deltas, then ends the turn', () => {
|
||||
const native = fakeNative();
|
||||
const api = loadLocalLlm(native);
|
||||
const { seen, callbacks } = collector();
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', system: 's', messages: [{ role: 'user', content: 'Hi' }], ...callbacks });
|
||||
const [instance] = native.instances;
|
||||
assert.equal(instance.request.prompt, 'Hi');
|
||||
instance.emit({ kind: 'log', message: 'shortened tool descriptions' });
|
||||
instance.emit({ kind: 'delta', text: 'Hello' });
|
||||
instance.emit({ kind: 'delta', text: ' there' });
|
||||
instance.emit({ kind: 'done' });
|
||||
instance.emit({ kind: 'delta', text: 'late' });
|
||||
assert.deepEqual(plain(seen.deltas), ['Hello', 'Hello there']);
|
||||
assert.equal(seen.done.length, 1);
|
||||
assert.deepEqual(plain(seen.done[0]), { text: 'Hello there', content: [{ type: 'text', text: 'Hello there' }], stopReason: 'end_turn' });
|
||||
assert.equal(instance.eventHandler, null, 'finished generations drop their handler');
|
||||
});
|
||||
|
||||
test('iOS local LLM reports why the model is unavailable without touching native code', async () => {
|
||||
const native = fakeNative('appleIntelligenceNotEnabled');
|
||||
const api = loadLocalLlm(native);
|
||||
assert.equal(api.isLocalModelReady(), false);
|
||||
assert.equal(api.isLocalModelSupported(), true);
|
||||
assert.deepEqual(plain(api.localModelState()), { status: 'unavailable', bytesDownloaded: 0, totalBytes: 0, detail: 'Apple Intelligence off' });
|
||||
const { seen, callbacks } = collector();
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'Hi' }], ...callbacks });
|
||||
await new Promise(resolve => setTimeout(resolve, 0));
|
||||
assert.match(seen.errors[0], /Turn on Apple Intelligence/);
|
||||
assert.equal(native.instances.length, 0);
|
||||
|
||||
native.setAvailability('deviceNotEligible');
|
||||
assert.equal(api.isLocalModelSupported(), false);
|
||||
native.setAvailability('somethingNew');
|
||||
assert.equal(api.localModelState().detail, 'unavailable');
|
||||
const missing = loader({ setTimeout, clearTimeout }, {})('app/native/local-llm.ios.ts');
|
||||
assert.equal(missing.localModelAvailability(), 'unsupportedOS', 'pre-iOS-26 builds have no Swift class to call');
|
||||
native.setAvailability('available');
|
||||
assert.deepEqual(plain(api.localModelState()), { status: 'ready', bytesDownloaded: 0, totalBytes: 0 });
|
||||
});
|
||||
|
||||
test('iOS local LLM maps refusals to the refusal stop reason and other failures to errors', () => {
|
||||
const native = fakeNative();
|
||||
const api = loadLocalLlm(native);
|
||||
const first = collector();
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...first.callbacks });
|
||||
native.instances[0].emit({ kind: 'error', code: 'refusal', message: 'declined' });
|
||||
assert.equal(first.seen.done[0].stopReason, 'refusal');
|
||||
const second = collector();
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...second.callbacks });
|
||||
native.instances[1].emit({ kind: 'error', code: 'rateLimited', message: 'iOS is limiting the on-device model' });
|
||||
assert.deepEqual(plain(second.seen.errors), ['iOS is limiting the on-device model']);
|
||||
});
|
||||
|
||||
test('iOS local LLM cancellation stops the native generation and silences later events', () => {
|
||||
const native = fakeNative();
|
||||
const api = loadLocalLlm(native);
|
||||
const { seen, callbacks } = collector();
|
||||
const handle = api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...callbacks });
|
||||
handle.cancel();
|
||||
native.instances[0].emit({ kind: 'delta', text: 'late' });
|
||||
native.instances[0].emit({ kind: 'done' });
|
||||
assert.equal(native.instances[0].cancelled, true);
|
||||
assert.deepEqual(seen, { deltas: [], done: [], errors: [] });
|
||||
});
|
||||
|
||||
test('iOS assistant runs the on-device model through the shared tool loop, resuming the parked generation', async () => {
|
||||
const native = fakeNative();
|
||||
const calls = [], deltas = [], errors = [], done = [];
|
||||
const registry = {
|
||||
listTools: () => [{ name: 'timer.set', description: 'Set a timer', inputSchema: { type: 'object', properties: { seconds: { type: 'integer' } }, required: ['seconds'] } }],
|
||||
callTool: async (name, args) => { calls.push({ name, args }); return { ok: true, content: 'Timer set' }; },
|
||||
};
|
||||
const mocks = {
|
||||
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
|
||||
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'base prompt', buildAssistantSystemPrompt: () => 'cloud prompt', describeAssistantContext: () => 'Context: now' },
|
||||
};
|
||||
const load = loader({ global: { isIOS: true, isAndroid: false }, FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout }, mocks);
|
||||
mocks['../native/local-llm'] = load('app/native/local-llm.ios.ts');
|
||||
const { AssistantSession } = load('app/assistant/session.ts');
|
||||
const session = new AssistantSession({ kind: 'direct', llm: { selection: 'qwen', provider: 'local', model: 'apple-foundation-model', apiKey: '' } }, registry);
|
||||
const send = text => session.sendUtterance(text, {}, { onTextDelta: (_d, full) => deltas.push(full),
|
||||
onToolActivity() {}, onError: error => errors.push(error), onTurnDone: result => done.push(result) });
|
||||
|
||||
send('Set a one-minute timer');
|
||||
assert.equal(native.instances.length, 1);
|
||||
const generation = native.instances[0];
|
||||
assert.equal(generation.request.system, 'base prompt', 'the local provider keeps the system prompt stable');
|
||||
assert.equal(generation.request.prompt, 'Set a one-minute timer\n\n(Context: now)');
|
||||
assert.deepEqual(plain(generation.request.tools.map(tool => tool.name)), ['timer_set']);
|
||||
|
||||
generation.emit({ kind: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] });
|
||||
await flush();
|
||||
assert.deepEqual(plain(calls), [{ name: 'timer.set', args: { seconds: 60 } }]);
|
||||
assert.equal(native.instances.length, 1, 'tool results resume the parked generation instead of starting another');
|
||||
assert.deepEqual(plain(generation.resolved), [[{ id: 'fm_abc', content: 'Timer set', isError: false }]]);
|
||||
|
||||
generation.emit({ kind: 'delta', text: 'Your timer is running.' });
|
||||
generation.emit({ kind: 'done' });
|
||||
assert.equal(done.length, 1);
|
||||
assert.equal(errors.length, 0);
|
||||
assert.equal(deltas.at(-1), 'Your timer is running.');
|
||||
assert.equal(session.isTurnActive(), false);
|
||||
|
||||
// The next turn starts a fresh generation that carries the whole exchange as transcript history.
|
||||
send('Thanks');
|
||||
const next = native.instances[1];
|
||||
assert.deepEqual(plain(next.request.history), [
|
||||
{ type: 'prompt', text: 'Set a one-minute timer\n\n(Context: now)' },
|
||||
{ type: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] },
|
||||
{ type: 'toolOutput', id: 'fm_abc', name: 'timer_set', text: 'Timer set' },
|
||||
{ type: 'response', text: 'Your timer is running.' },
|
||||
]);
|
||||
session.cancel();
|
||||
assert.equal(next.cancelled, true);
|
||||
});
|
||||
|
||||
test('a parked iOS generation survives unrelated requests and is abandoned if its results never arrive', () => {
|
||||
const native = fakeNative();
|
||||
let expire = null;
|
||||
const timers = {
|
||||
setTimeout: (fn, ms) => { if (ms > 0) expire = fn; else fn(); return 1; },
|
||||
clearTimeout: () => { expire = null; },
|
||||
};
|
||||
const api = loadLocalLlm(native, timers);
|
||||
const turn = collector();
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], tools: [], ...turn.callbacks });
|
||||
native.instances[0].emit({ kind: 'toolCalls', calls: [{ id: 'fm_1', name: 't', arguments: 'not json' }] });
|
||||
assert.deepEqual(plain(turn.seen.done[0].content), [{ type: 'tool_use', id: 'fm_1', name: 't', input: {} }]);
|
||||
|
||||
// e.g. Translate asking the model something while the assistant's tool runs.
|
||||
const other = collector();
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'translate this' }], ...other.callbacks });
|
||||
assert.equal(native.instances.length, 2);
|
||||
assert.equal(native.instances[0].cancelled, false);
|
||||
|
||||
assert.equal(typeof expire, 'function');
|
||||
expire();
|
||||
assert.equal(native.instances[0].cancelled, true);
|
||||
// Results for the abandoned call now start a fresh generation with the fallback prompt.
|
||||
api.streamLocalModel({ apiKey: '', model: 'm', ...turn.callbacks, messages: [
|
||||
{ role: 'user', content: 'x' },
|
||||
{ role: 'assistant', content: turn.seen.done[0].content },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'ok' }] },
|
||||
] });
|
||||
assert.equal(native.instances.length, 3);
|
||||
assert.equal(native.instances[2].request.prompt, api.CONTINUE_PROMPT);
|
||||
});
|
||||
|
||||
test('iOS offers the on-phone model where Apple Intelligence is supported', () => {
|
||||
const local = { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => true, isLocalModelSupported: () => true };
|
||||
const api = loader({ global: { isIOS: true } }, { '../native/local-llm': local })('app/assistant/models.ts');
|
||||
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), true);
|
||||
assert.equal(api.supportedAssistantModel('qwen'), 'qwen');
|
||||
assert.equal(api.assistantModelLabel('qwen'), 'Apple Intelligence (on-phone)');
|
||||
const resolved = api.resolveAssistantModel('auto', { openai: '', anthropic: '' });
|
||||
assert.equal(resolved.provider, 'local');
|
||||
assert.equal(resolved.model, 'apple-foundation-model');
|
||||
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai', 'API keys still win under Auto');
|
||||
local.isLocalModelReady = () => false;
|
||||
assert.equal(api.resolveAssistantModel('qwen', { openai: '', anthropic: '' }), null);
|
||||
});
|
||||
Reference in new issue
Block a user