forked from Blendman974/faceclaw
Add support for a local model (Qwen3)
This commit is contained in:
1 parent
6103c00ca3
commit
6f45888627
16 files changed
+1532
-15
No files matched your search
@@ -149,6 +149,102 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
}
|
||||
}
|
||||
|
||||
// On-phone assistant model: build llama.cpp (pinned release) + our JNI shim
|
||||
// into libfaceclaw_llama.so. CPU-only, arm64, armv8.2 dotprod+fp16 baseline
|
||||
// (Tensor G2 floor; no i8mm). The first build compiles llama.cpp (~2-4 min);
|
||||
// afterwards the cached .so short-circuits everything.
|
||||
def faceclawLlamaCppTag = "b10333"
|
||||
|
||||
def faceclawFindSdkCmakeDir = {
|
||||
def sdkRoot = System.getenv("ANDROID_HOME") ?: System.getenv("ANDROID_SDK_ROOT")
|
||||
if (sdkRoot == null) return null
|
||||
def cmakeRoot = file("$sdkRoot/cmake")
|
||||
if (!cmakeRoot.exists()) return null
|
||||
def installed = cmakeRoot.listFiles()?.findAll { it.isDirectory() }?.sort { a, b -> b.name <=> a.name }
|
||||
return (installed && !installed.isEmpty()) ? installed[0] : null
|
||||
}
|
||||
|
||||
tasks.register("prepareFaceclawLlama") {
|
||||
def llamaSo = file("$faceclawJniLibsDir/libfaceclaw_llama.so")
|
||||
inputs.files(
|
||||
file("$projectDir/src/main/native/llama/faceclaw_llama.cpp"),
|
||||
file("$projectDir/src/main/native/llama/CMakeLists.txt")
|
||||
)
|
||||
outputs.files(llamaSo)
|
||||
|
||||
doLast {
|
||||
def llamaBuildDir = file("$faceclawNativeBuildDir/llama-build")
|
||||
def builtSo = file("$llamaBuildDir/libfaceclaw_llama.so")
|
||||
|
||||
def downloadsDir = file("$faceclawNativeBuildDir/downloads")
|
||||
def llamaArchive = file("$downloadsDir/llama.cpp-${faceclawLlamaCppTag}.tar.gz")
|
||||
faceclawDownload(
|
||||
"https://github.com/ggml-org/llama.cpp/archive/refs/tags/${faceclawLlamaCppTag}.tar.gz",
|
||||
llamaArchive
|
||||
)
|
||||
def llamaSourceDir = file("$faceclawNativeBuildDir/llama.cpp-${faceclawLlamaCppTag}")
|
||||
if (!file("$llamaSourceDir/CMakeLists.txt").exists()) {
|
||||
copy {
|
||||
from tarTree(resources.gzip(llamaArchive))
|
||||
into faceclawNativeBuildDir
|
||||
}
|
||||
}
|
||||
|
||||
def ndkRoot = faceclawFindNdkRoot()
|
||||
if (ndkRoot == null || !ndkRoot.exists()) {
|
||||
throw new GradleException("Android NDK is required to build libfaceclaw_llama.so")
|
||||
}
|
||||
def cmakeDir = faceclawFindSdkCmakeDir()
|
||||
if (cmakeDir == null) {
|
||||
throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so")
|
||||
}
|
||||
def cmake = file("$cmakeDir/bin/cmake")
|
||||
def ninja = file("$cmakeDir/bin/ninja")
|
||||
def march = "-march=armv8.2-a+dotprod+fp16"
|
||||
|
||||
// Configure once; afterwards the ninja no-op build keeps this cheap while
|
||||
// still picking up JNI source edits.
|
||||
if (!file("$llamaBuildDir/build.ninja").exists()) {
|
||||
exec {
|
||||
commandLine([
|
||||
cmake.absolutePath,
|
||||
"-S", file("$projectDir/src/main/native/llama").absolutePath,
|
||||
"-B", llamaBuildDir.absolutePath,
|
||||
"-G", "Ninja",
|
||||
"-DCMAKE_MAKE_PROGRAM=${ninja.absolutePath}",
|
||||
"-DCMAKE_TOOLCHAIN_FILE=${file("$ndkRoot/build/cmake/android.toolchain.cmake").absolutePath}",
|
||||
"-DANDROID_ABI=arm64-v8a",
|
||||
"-DANDROID_PLATFORM=android-24",
|
||||
"-DCMAKE_BUILD_TYPE=Release",
|
||||
"-DLLAMA_CPP_SOURCE_DIR=${llamaSourceDir.absolutePath}",
|
||||
"-DCMAKE_C_FLAGS=$march",
|
||||
"-DCMAKE_CXX_FLAGS=$march",
|
||||
])
|
||||
}
|
||||
}
|
||||
exec {
|
||||
commandLine([cmake.absolutePath, "--build", llamaBuildDir.absolutePath, "-j", "8"])
|
||||
}
|
||||
|
||||
faceclawJniLibsDir.mkdirs()
|
||||
copy {
|
||||
from builtSo
|
||||
into faceclawJniLibsDir
|
||||
}
|
||||
// The static-linked lib carries llama.cpp's full debug info (~50MB);
|
||||
// strip it so the APK only pays for the code (~8MB).
|
||||
def ndkRootForStrip = faceclawFindNdkRoot()
|
||||
if (ndkRootForStrip != null) {
|
||||
def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip")
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
android {
|
||||
compileSdkVersion 35
|
||||
buildToolsVersion "35"
|
||||
@@ -179,4 +275,5 @@ android {
|
||||
|
||||
tasks.matching { it.name == "preBuild" }.configureEach {
|
||||
dependsOn tasks.named("prepareFaceclawNativeLibs")
|
||||
dependsOn tasks.named("prepareFaceclawLlama")
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/**
|
||||
* Callbacks for one on-phone LLM generation. Like the other bridge listeners,
|
||||
* implementations are typically created on the NativeScript JS thread and
|
||||
* FaceclawLlamaRunner posts these callbacks back to that thread's Looper.
|
||||
*/
|
||||
public interface FaceclawLlamaListener {
|
||||
/** A piece of generated text (may be a partial word). */
|
||||
void onToken(String piece);
|
||||
|
||||
/** Generation finished: "stop" (EOG), "length", or "cancelled". */
|
||||
void onDone(String stopReason);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
/**
|
||||
* On-phone LLM inference for the voice assistant (llama.cpp over JNI).
|
||||
*
|
||||
* All model work (load, generate, free) runs on a single background executor
|
||||
* thread so calls are naturally serialized; listener callbacks are posted to
|
||||
* the Looper of the thread that constructed this object (the JS thread), the
|
||||
* same contract as FaceclawSseRequest. cancel() may be called from any thread
|
||||
* and interrupts the in-flight generation at the next token boundary.
|
||||
*
|
||||
* The model stays loaded between generations (turn iterations reuse the KV
|
||||
* cache prefix); the TypeScript side calls unload() on an idle timer to give
|
||||
* the ~3GB back.
|
||||
*/
|
||||
public class FaceclawLlamaRunner {
|
||||
private static final String TAG = "FaceclawLlamaRunner";
|
||||
|
||||
static {
|
||||
System.loadLibrary("faceclaw_llama");
|
||||
}
|
||||
|
||||
private final Handler callbackHandler;
|
||||
private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "FaceclawLlama");
|
||||
thread.setPriority(Thread.NORM_PRIORITY);
|
||||
return thread;
|
||||
});
|
||||
|
||||
// Mutated only on the executor thread; volatile so cancel() can read it.
|
||||
private volatile long handle;
|
||||
private String loadedPath;
|
||||
|
||||
public FaceclawLlamaRunner() {
|
||||
Looper looper = Looper.myLooper();
|
||||
callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
|
||||
}
|
||||
|
||||
public boolean isModelLoaded() {
|
||||
return handle != 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a completion, loading (or swapping) the model first if needed.
|
||||
* grammar may be null/empty for unconstrained generation.
|
||||
*/
|
||||
public void generate(final String modelPath, final int nCtx, final int nThreads,
|
||||
final String prompt, final String grammar, final int maxTokens,
|
||||
final float temperature, final float topP, final int topK,
|
||||
final FaceclawLlamaListener listener) {
|
||||
executor.execute(() -> {
|
||||
try {
|
||||
if (handle != 0 && !modelPath.equals(loadedPath)) {
|
||||
nativeFree(handle);
|
||||
handle = 0;
|
||||
loadedPath = null;
|
||||
}
|
||||
if (handle == 0) {
|
||||
long loaded = nativeLoadModel(modelPath, nCtx, nThreads);
|
||||
if (loaded == 0) {
|
||||
post(() -> listener.onError("Could not load the on-phone model"));
|
||||
return;
|
||||
}
|
||||
handle = loaded;
|
||||
loadedPath = modelPath;
|
||||
}
|
||||
nativeGenerate(handle, prompt, grammar == null ? "" : grammar,
|
||||
maxTokens, temperature, topP, topK, new FaceclawLlamaListener() {
|
||||
@Override public void onToken(final String piece) {
|
||||
post(() -> listener.onToken(piece));
|
||||
}
|
||||
@Override public void onDone(final String stopReason) {
|
||||
post(() -> listener.onDone(stopReason));
|
||||
}
|
||||
@Override public void onError(final String message) {
|
||||
post(() -> listener.onError(message));
|
||||
}
|
||||
});
|
||||
} catch (Throwable t) {
|
||||
Log.e(TAG, "generation failed", t);
|
||||
final String message = String.valueOf(t);
|
||||
post(() -> listener.onError("On-phone model failed: " + message));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Interrupt the current generation (its listener gets onDone("cancelled")). */
|
||||
public void cancel() {
|
||||
long current = handle;
|
||||
if (current != 0) {
|
||||
nativeCancel(current);
|
||||
}
|
||||
}
|
||||
|
||||
/** Free the model and its KV cache. Safe to call when nothing is loaded. */
|
||||
public void unload() {
|
||||
executor.execute(() -> {
|
||||
if (handle != 0) {
|
||||
nativeFree(handle);
|
||||
handle = 0;
|
||||
loadedPath = null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void post(Runnable runnable) {
|
||||
callbackHandler.post(runnable);
|
||||
}
|
||||
|
||||
private static native long nativeLoadModel(String path, int nCtx, int nThreads);
|
||||
private static native void nativeGenerate(long handle, String prompt, String grammar,
|
||||
int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener);
|
||||
private static native void nativeCancel(long handle);
|
||||
private static native void nativeFree(long handle);
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import okhttp3.Call;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.ResponseBody;
|
||||
|
||||
/**
|
||||
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
|
||||
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
|
||||
* restarts, verifies a pinned sha256 (hashed incrementally, including the
|
||||
* resumed prefix), then renames into place. Listener callbacks are posted to
|
||||
* the constructing thread's Looper, like the other Faceclaw bridges.
|
||||
*/
|
||||
public class FaceclawModelDownloader {
|
||||
private static final String TAG = "FaceclawModelDl";
|
||||
private static final long PROGRESS_INTERVAL_MS = 500;
|
||||
|
||||
private static volatile OkHttpClient sharedClient;
|
||||
|
||||
private final Handler callbackHandler;
|
||||
private final String url;
|
||||
private final File destFile;
|
||||
private final File partFile;
|
||||
private final String expectedSha256;
|
||||
private final long expectedTotalBytes;
|
||||
private final FaceclawModelDownloaderListener listener;
|
||||
|
||||
private volatile boolean cancelled;
|
||||
private volatile Call call;
|
||||
private Thread worker;
|
||||
|
||||
public FaceclawModelDownloader(String url, String destPath, String expectedSha256,
|
||||
long expectedTotalBytes, FaceclawModelDownloaderListener listener) {
|
||||
if (listener == null) throw new IllegalArgumentException("listener is required");
|
||||
Looper looper = Looper.myLooper();
|
||||
this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
|
||||
this.url = url;
|
||||
this.destFile = new File(destPath);
|
||||
this.partFile = new File(destPath + ".part");
|
||||
this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase();
|
||||
this.expectedTotalBytes = expectedTotalBytes;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
private static OkHttpClient getClient() {
|
||||
if (sharedClient == null) {
|
||||
synchronized (FaceclawModelDownloader.class) {
|
||||
if (sharedClient == null) {
|
||||
sharedClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
return sharedClient;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
worker = new Thread(this::run, "FaceclawModelDl");
|
||||
worker.start();
|
||||
}
|
||||
|
||||
/** Stops the download but keeps the .part file so a later start() resumes. */
|
||||
public void cancel() {
|
||||
cancelled = true;
|
||||
Call current = call;
|
||||
if (current != null) current.cancel();
|
||||
}
|
||||
|
||||
private void run() {
|
||||
try {
|
||||
if (destFile.exists() && destFile.length() > 0) {
|
||||
post(() -> listener.onDone(destFile.getAbsolutePath()));
|
||||
return;
|
||||
}
|
||||
destFile.getParentFile().mkdirs();
|
||||
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
long offset = 0;
|
||||
if (partFile.exists()) {
|
||||
offset = hashExistingPrefix(digest);
|
||||
if (cancelled) return;
|
||||
}
|
||||
|
||||
Request.Builder builder = new Request.Builder().url(url);
|
||||
if (offset > 0) builder.header("Range", "bytes=" + offset + "-");
|
||||
call = getClient().newCall(builder.build());
|
||||
|
||||
try (Response response = call.execute()) {
|
||||
ResponseBody body = response.body();
|
||||
if (!response.isSuccessful() || body == null) {
|
||||
postError("Model download failed: HTTP " + response.code());
|
||||
return;
|
||||
}
|
||||
boolean resumed = response.code() == 206;
|
||||
if (offset > 0 && !resumed) {
|
||||
// Server ignored the Range header; start over.
|
||||
offset = 0;
|
||||
digest = MessageDigest.getInstance("SHA-256");
|
||||
}
|
||||
long total = expectedTotalBytes > 0 ? expectedTotalBytes
|
||||
: (body.contentLength() > 0 ? offset + body.contentLength() : -1);
|
||||
|
||||
try (FileOutputStream out = new FileOutputStream(partFile, resumed);
|
||||
InputStream in = body.byteStream()) {
|
||||
byte[] buffer = new byte[1 << 16];
|
||||
long done = offset;
|
||||
long lastProgressAt = 0;
|
||||
int read;
|
||||
while ((read = in.read(buffer)) != -1) {
|
||||
if (cancelled) return;
|
||||
out.write(buffer, 0, read);
|
||||
digest.update(buffer, 0, read);
|
||||
done += read;
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
|
||||
lastProgressAt = now;
|
||||
final long doneFinal = done;
|
||||
final long totalFinal = total;
|
||||
post(() -> listener.onProgress(doneFinal, totalFinal));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
|
||||
postError("Model download ended early (" + partFile.length() + " of "
|
||||
+ expectedTotalBytes + " bytes); try again to resume");
|
||||
return;
|
||||
}
|
||||
String actualSha = toHex(digest.digest());
|
||||
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
|
||||
partFile.delete();
|
||||
postError("Model download was corrupted (checksum mismatch); download it again");
|
||||
return;
|
||||
}
|
||||
if (!partFile.renameTo(destFile)) {
|
||||
postError("Could not move the downloaded model into place");
|
||||
return;
|
||||
}
|
||||
post(() -> listener.onDone(destFile.getAbsolutePath()));
|
||||
} catch (Exception e) {
|
||||
if (cancelled) return;
|
||||
Log.w(TAG, "download failed", e);
|
||||
postError("Model download failed: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
|
||||
private long hashExistingPrefix(MessageDigest digest) throws IOException {
|
||||
long hashed = 0;
|
||||
try (FileInputStream in = new FileInputStream(partFile)) {
|
||||
byte[] buffer = new byte[1 << 16];
|
||||
int read;
|
||||
while ((read = in.read(buffer)) != -1) {
|
||||
if (cancelled) return hashed;
|
||||
digest.update(buffer, 0, read);
|
||||
hashed += read;
|
||||
}
|
||||
}
|
||||
return hashed;
|
||||
}
|
||||
|
||||
private static String toHex(byte[] bytes) {
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) sb.append(String.format("%02x", b));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void post(Runnable runnable) {
|
||||
callbackHandler.post(runnable);
|
||||
}
|
||||
|
||||
private void postError(final String message) {
|
||||
post(() -> listener.onError(message));
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/** Progress callbacks for FaceclawModelDownloader, posted to the creating thread's Looper. */
|
||||
public interface FaceclawModelDownloaderListener {
|
||||
void onProgress(long bytesDownloaded, long totalBytes);
|
||||
|
||||
/** The file is fully downloaded, verified, and moved to its final path. */
|
||||
void onDone(String path);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
cmake_minimum_required(VERSION 3.22)
|
||||
project(faceclaw_llama CXX C)
|
||||
|
||||
# llama.cpp source tree location is injected by the gradle task (it downloads
|
||||
# and extracts a pinned release tag before configuring this project).
|
||||
if(NOT DEFINED LLAMA_CPP_SOURCE_DIR)
|
||||
message(FATAL_ERROR "Pass -DLLAMA_CPP_SOURCE_DIR=<path to llama.cpp checkout>")
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
|
||||
# Static llama/ggml folded into our single JNI .so. CPU only: Vulkan on Mali
|
||||
# and mixed Adreno drivers is not worth the instability for short assistant
|
||||
# turns. The -march baseline lives in the gradle invocation (armv8.2-a with
|
||||
# dotprod+fp16: the floor for Tensor G2's X1/A78/A55, so no i8mm).
|
||||
set(BUILD_SHARED_LIBS OFF)
|
||||
set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
||||
set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
||||
set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
|
||||
set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE)
|
||||
set(LLAMA_CURL OFF CACHE BOOL "" FORCE)
|
||||
set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
|
||||
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
|
||||
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
|
||||
|
||||
add_subdirectory(${LLAMA_CPP_SOURCE_DIR} llama.cpp-build EXCLUDE_FROM_ALL)
|
||||
|
||||
add_library(faceclaw_llama SHARED faceclaw_llama.cpp)
|
||||
target_link_libraries(faceclaw_llama PRIVATE llama android log)
|
||||
@@ -0,0 +1,391 @@
|
||||
// JNI bridge for the on-phone assistant model (llama.cpp, CPU inference).
|
||||
// Java surface: FaceclawLlamaRunner's nativeLoadModel / nativeGenerate /
|
||||
// nativeCancel / nativeFree. One model+context per handle; the Java side
|
||||
// serializes all calls except nativeCancel onto a single executor thread,
|
||||
// so only the cancel flag needs to be thread-safe.
|
||||
|
||||
#include <jni.h>
|
||||
#include <android/log.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "ggml-cpu.h"
|
||||
#include "llama.h"
|
||||
|
||||
#define TAG "FaceclawLlama"
|
||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
|
||||
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
|
||||
|
||||
namespace {
|
||||
|
||||
struct FcLlama {
|
||||
llama_model * model = nullptr;
|
||||
llama_context * ctx = nullptr;
|
||||
ggml_threadpool * threadpool = nullptr;
|
||||
const llama_vocab * vocab = nullptr;
|
||||
int n_ctx = 0;
|
||||
// Tokens currently materialized in the KV cache (prompt + generated),
|
||||
// used to skip re-decoding the common prefix on the next call. Within a
|
||||
// turn the next prompt extends the previous one, so this usually reduces
|
||||
// prefill to just the new tool results.
|
||||
std::vector<llama_token> cache_tokens;
|
||||
std::atomic<bool> cancel{false};
|
||||
};
|
||||
|
||||
void android_llama_log(ggml_log_level level, const char * text, void * /*user*/) {
|
||||
if (level < GGML_LOG_LEVEL_WARN) return; // llama.cpp is chatty at INFO during load
|
||||
__android_log_write(level >= GGML_LOG_LEVEL_ERROR ? ANDROID_LOG_ERROR : ANDROID_LOG_WARN, TAG, text);
|
||||
}
|
||||
|
||||
void ensure_backend_init() {
|
||||
static std::once_flag once;
|
||||
std::call_once(once, [] {
|
||||
llama_log_set(android_llama_log, nullptr);
|
||||
llama_backend_init();
|
||||
});
|
||||
}
|
||||
|
||||
int64_t now_ms() {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count();
|
||||
}
|
||||
|
||||
// Big-core detection for big.LITTLE phones. Without explicit affinity the
|
||||
// scheduler is free to put our compute threads on the little cores (in-order
|
||||
// A55s on Tensor G2), which slows prefill by 5-10x; pin to the cores whose
|
||||
// max frequency is within 75% of the fastest core's (on a Pixel 7/7a this
|
||||
// selects the 2x X1 + 2x A78 and excludes the 4x A55).
|
||||
std::vector<int> detect_fast_cpus() {
|
||||
std::vector<long> freqs;
|
||||
for (int cpu = 0; cpu < GGML_MAX_N_THREADS; cpu++) {
|
||||
char path[128];
|
||||
snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", cpu);
|
||||
FILE * f = fopen(path, "r");
|
||||
if (f == nullptr) break;
|
||||
long freq = 0;
|
||||
if (fscanf(f, "%ld", &freq) != 1) freq = 0;
|
||||
fclose(f);
|
||||
freqs.push_back(freq);
|
||||
}
|
||||
std::vector<int> fast;
|
||||
long max_freq = 0;
|
||||
for (long f : freqs) max_freq = std::max(max_freq, f);
|
||||
if (max_freq <= 0) return fast;
|
||||
for (size_t cpu = 0; cpu < freqs.size(); cpu++) {
|
||||
if (freqs[cpu] * 4 >= max_freq * 3) fast.push_back((int) cpu);
|
||||
}
|
||||
// A uniform-frequency CPU has no little cores to avoid.
|
||||
if (fast.size() == freqs.size()) fast.clear();
|
||||
return fast;
|
||||
}
|
||||
|
||||
std::string jstring_to_utf8(JNIEnv * env, jstring value) {
|
||||
if (value == nullptr) return "";
|
||||
const char * chars = env->GetStringUTFChars(value, nullptr);
|
||||
std::string out = chars ? chars : "";
|
||||
env->ReleaseStringUTFChars(value, chars);
|
||||
return out;
|
||||
}
|
||||
|
||||
// Callback shims into the Java listener (an in-process wrapper object that
|
||||
// posts to the JS thread; these calls happen on the executor thread).
|
||||
struct Listener {
|
||||
JNIEnv * env;
|
||||
jobject obj;
|
||||
jmethodID on_token;
|
||||
jmethodID on_done;
|
||||
jmethodID on_error;
|
||||
|
||||
static Listener resolve(JNIEnv * env, jobject listener) {
|
||||
jclass cls = env->GetObjectClass(listener);
|
||||
return Listener{
|
||||
env,
|
||||
listener,
|
||||
env->GetMethodID(cls, "onToken", "(Ljava/lang/String;)V"),
|
||||
env->GetMethodID(cls, "onDone", "(Ljava/lang/String;)V"),
|
||||
env->GetMethodID(cls, "onError", "(Ljava/lang/String;)V"),
|
||||
};
|
||||
}
|
||||
|
||||
void token(const std::string & piece) const { call(on_token, piece); }
|
||||
void done(const char * reason) const { call(on_done, reason); }
|
||||
void error(const std::string & message) const { call(on_error, message); }
|
||||
|
||||
private:
|
||||
void call(jmethodID method, const std::string & text) const {
|
||||
jstring jtext = env->NewStringUTF(text.c_str());
|
||||
env->CallVoidMethod(obj, method, jtext);
|
||||
env->DeleteLocalRef(jtext);
|
||||
if (env->ExceptionCheck()) {
|
||||
env->ExceptionClear();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// NewStringUTF aborts on invalid (modified-)UTF-8, and token pieces can end
|
||||
// mid-codepoint, so hold back any incomplete trailing UTF-8 sequence.
|
||||
size_t utf8_complete_prefix(const std::string & data) {
|
||||
size_t len = data.size();
|
||||
size_t i = len;
|
||||
// Find the start byte of the final codepoint (at most 3 bytes back).
|
||||
while (i > 0 && len - i < 4 && (static_cast<unsigned char>(data[i - 1]) & 0xC0) == 0x80) i--;
|
||||
if (i == 0) return len; // malformed; let it through rather than stall
|
||||
unsigned char start = static_cast<unsigned char>(data[i - 1]);
|
||||
size_t expected;
|
||||
if (start < 0x80) expected = 1;
|
||||
else if ((start & 0xE0) == 0xC0) expected = 2;
|
||||
else if ((start & 0xF0) == 0xE0) expected = 3;
|
||||
else if ((start & 0xF8) == 0xF0) expected = 4;
|
||||
else return len; // malformed start byte; pass through
|
||||
size_t have = len - (i - 1);
|
||||
return have >= expected ? len : i - 1;
|
||||
}
|
||||
|
||||
std::vector<llama_token> tokenize(const llama_vocab * vocab, const std::string & text, bool add_special) {
|
||||
int n = -llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), nullptr, 0, add_special, true);
|
||||
if (n <= 0) return {};
|
||||
std::vector<llama_token> tokens(n);
|
||||
int written = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), n, add_special, true);
|
||||
if (written < 0) return {};
|
||||
tokens.resize(written);
|
||||
return tokens;
|
||||
}
|
||||
|
||||
std::string token_piece(const llama_vocab * vocab, llama_token token) {
|
||||
char buf[256];
|
||||
int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true);
|
||||
if (n <= 0) return "";
|
||||
return std::string(buf, n);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
extern "C" JNIEXPORT jlong JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeLoadModel(
|
||||
JNIEnv * env, jclass, jstring jpath, jint n_ctx, jint n_threads) {
|
||||
ensure_backend_init();
|
||||
const std::string path = jstring_to_utf8(env, jpath);
|
||||
const int64_t t_start = now_ms();
|
||||
|
||||
// Cap threads to the big-core count and pin them there (see
|
||||
// detect_fast_cpus): little cores would otherwise both drag the barrier
|
||||
// and invite the scheduler to keep us off the fast cores entirely.
|
||||
const std::vector<int> fast_cpus = detect_fast_cpus();
|
||||
int threads = (int) n_threads;
|
||||
if (!fast_cpus.empty()) {
|
||||
threads = std::min(threads, (int) fast_cpus.size());
|
||||
}
|
||||
|
||||
llama_model_params mparams = llama_model_default_params();
|
||||
mparams.n_gpu_layers = 0;
|
||||
llama_model * model = llama_model_load_from_file(path.c_str(), mparams);
|
||||
if (model == nullptr) {
|
||||
LOGW("model load failed: %s", path.c_str());
|
||||
return 0;
|
||||
}
|
||||
const int64_t t_model = now_ms();
|
||||
|
||||
llama_context_params cparams = llama_context_default_params();
|
||||
cparams.n_ctx = (uint32_t) n_ctx;
|
||||
cparams.n_batch = 512;
|
||||
cparams.n_threads = threads;
|
||||
cparams.n_threads_batch = threads;
|
||||
llama_context * ctx = llama_init_from_model(model, cparams);
|
||||
if (ctx == nullptr) {
|
||||
LOGW("context init failed");
|
||||
llama_model_free(model);
|
||||
return 0;
|
||||
}
|
||||
|
||||
ggml_threadpool * threadpool = nullptr;
|
||||
if (!fast_cpus.empty()) {
|
||||
ggml_threadpool_params tpp = ggml_threadpool_params_default(threads);
|
||||
for (size_t i = 0; i < fast_cpus.size() && fast_cpus[i] < GGML_MAX_N_THREADS; i++) {
|
||||
tpp.cpumask[fast_cpus[i]] = true;
|
||||
}
|
||||
tpp.strict_cpu = true;
|
||||
threadpool = ggml_threadpool_new(&tpp);
|
||||
if (threadpool != nullptr) {
|
||||
llama_attach_threadpool(ctx, threadpool, threadpool);
|
||||
std::string cpus;
|
||||
for (int cpu : fast_cpus) cpus += std::to_string(cpu) + " ";
|
||||
LOGI("pinned %d threads to fast cpus: %s", threads, cpus.c_str());
|
||||
} else {
|
||||
LOGW("threadpool creation failed; using default scheduling");
|
||||
}
|
||||
}
|
||||
|
||||
auto * handle = new FcLlama();
|
||||
handle->model = model;
|
||||
handle->ctx = ctx;
|
||||
handle->threadpool = threadpool;
|
||||
handle->vocab = llama_model_get_vocab(model);
|
||||
handle->n_ctx = (int) llama_n_ctx(ctx);
|
||||
LOGI("model loaded: %s (n_ctx=%d, threads=%d, model_load_ms=%lld, ctx_init_ms=%lld)",
|
||||
path.c_str(), handle->n_ctx, threads,
|
||||
(long long) (t_model - t_start), (long long) (now_ms() - t_model));
|
||||
return reinterpret_cast<jlong>(handle);
|
||||
}
|
||||
|
||||
extern "C" JNIEXPORT void JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeFree(JNIEnv *, jclass, jlong jhandle) {
|
||||
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
|
||||
if (handle == nullptr) return;
|
||||
llama_free(handle->ctx);
|
||||
llama_model_free(handle->model);
|
||||
if (handle->threadpool != nullptr) {
|
||||
ggml_threadpool_free(handle->threadpool);
|
||||
}
|
||||
delete handle;
|
||||
LOGI("model freed");
|
||||
}
|
||||
|
||||
extern "C" JNIEXPORT void JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeCancel(JNIEnv *, jclass, jlong jhandle) {
|
||||
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
|
||||
if (handle != nullptr) handle->cancel.store(true);
|
||||
}
|
||||
|
||||
extern "C" JNIEXPORT void JNICALL
|
||||
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeGenerate(
|
||||
JNIEnv * env, jclass, jlong jhandle, jstring jprompt, jstring jgrammar,
|
||||
jint max_tokens, jfloat temperature, jfloat top_p, jint top_k,
|
||||
jobject jlistener) {
|
||||
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
|
||||
Listener listener = Listener::resolve(env, jlistener);
|
||||
if (handle == nullptr) {
|
||||
listener.error("Model is not loaded");
|
||||
return;
|
||||
}
|
||||
handle->cancel.store(false);
|
||||
|
||||
const std::string prompt = jstring_to_utf8(env, jprompt);
|
||||
const std::string grammar = jstring_to_utf8(env, jgrammar);
|
||||
|
||||
std::vector<llama_token> tokens = tokenize(handle->vocab, prompt, true);
|
||||
if (tokens.empty()) {
|
||||
listener.error("Prompt tokenization failed");
|
||||
return;
|
||||
}
|
||||
if ((int) tokens.size() + 16 > handle->n_ctx) {
|
||||
listener.error("Conversation too long for the on-phone model");
|
||||
return;
|
||||
}
|
||||
const int max_gen = std::min((int) max_tokens, handle->n_ctx - (int) tokens.size());
|
||||
|
||||
// Reuse the KV prefix shared with the previous call; drop the rest. Keep
|
||||
// at least the final prompt token out of the reused prefix so this decode
|
||||
// produces logits for it.
|
||||
size_t common = 0;
|
||||
while (common < tokens.size() && common < handle->cache_tokens.size() &&
|
||||
tokens[common] == handle->cache_tokens[common]) {
|
||||
common++;
|
||||
}
|
||||
if (common == tokens.size()) common = tokens.size() - 1;
|
||||
llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) common, -1);
|
||||
handle->cache_tokens.assign(tokens.begin(), tokens.begin() + common);
|
||||
|
||||
// Sampler chain per generation: Qwen3 instruct posture (temp 0.7, top-p
|
||||
// 0.8, top-k 20) plus an optional lazy grammar that activates at the
|
||||
// <tool_call> special token and forces a schema-valid call.
|
||||
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
|
||||
if (!grammar.empty()) {
|
||||
llama_sampler * grammar_sampler = nullptr;
|
||||
std::vector<llama_token> trigger = tokenize(handle->vocab, "<tool_call>", false);
|
||||
if (trigger.size() == 1) {
|
||||
grammar_sampler = llama_sampler_init_grammar_lazy_patterns(
|
||||
handle->vocab, grammar.c_str(), "root", nullptr, 0, trigger.data(), 1);
|
||||
} else {
|
||||
static const char * patterns[] = {"(<tool_call>[\\s\\S]*)"};
|
||||
grammar_sampler = llama_sampler_init_grammar_lazy_patterns(
|
||||
handle->vocab, grammar.c_str(), "root", patterns, 1, nullptr, 0);
|
||||
}
|
||||
if (grammar_sampler != nullptr) {
|
||||
llama_sampler_chain_add(chain, grammar_sampler);
|
||||
} else {
|
||||
LOGW("tool grammar failed to parse; generating unconstrained");
|
||||
}
|
||||
}
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_top_k(top_k));
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_top_p(top_p, 1));
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_temp(temperature));
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_dist(LLAMA_DEFAULT_SEED));
|
||||
|
||||
const auto fail = [&](const std::string & message) {
|
||||
llama_sampler_free(chain);
|
||||
// The KV state past the reused prefix is now unknown; drop it so the
|
||||
// next call starts from a consistent cache.
|
||||
llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) handle->cache_tokens.size(), -1);
|
||||
listener.error(message);
|
||||
};
|
||||
|
||||
// Prefill in n_batch chunks.
|
||||
const int64_t t_prefill_start = now_ms();
|
||||
const int n_batch = (int) llama_n_batch(handle->ctx);
|
||||
for (size_t i = common; i < tokens.size(); i += n_batch) {
|
||||
if (handle->cancel.load()) {
|
||||
llama_sampler_free(chain);
|
||||
listener.done("cancelled");
|
||||
return;
|
||||
}
|
||||
const int chunk = (int) std::min((size_t) n_batch, tokens.size() - i);
|
||||
if (llama_decode(handle->ctx, llama_batch_get_one(tokens.data() + i, chunk)) != 0) {
|
||||
fail("Prompt evaluation failed");
|
||||
return;
|
||||
}
|
||||
handle->cache_tokens.insert(handle->cache_tokens.end(), tokens.begin() + i, tokens.begin() + i + chunk);
|
||||
}
|
||||
|
||||
// Decode loop: sample, stream, feed back.
|
||||
const int64_t t_decode_start = now_ms();
|
||||
const size_t n_prefilled = tokens.size() - common;
|
||||
int n_decoded = 0;
|
||||
std::string pending_utf8;
|
||||
const char * stop_reason = "length";
|
||||
for (int i = 0; i < max_gen; i++) {
|
||||
if (handle->cancel.load()) {
|
||||
stop_reason = "cancelled";
|
||||
break;
|
||||
}
|
||||
llama_token token = llama_sampler_sample(chain, handle->ctx, -1);
|
||||
if (llama_vocab_is_eog(handle->vocab, token)) {
|
||||
stop_reason = "stop";
|
||||
break;
|
||||
}
|
||||
pending_utf8 += token_piece(handle->vocab, token);
|
||||
size_t emit = utf8_complete_prefix(pending_utf8);
|
||||
if (emit > 0) {
|
||||
listener.token(pending_utf8.substr(0, emit));
|
||||
pending_utf8.erase(0, emit);
|
||||
}
|
||||
if (llama_decode(handle->ctx, llama_batch_get_one(&token, 1)) != 0) {
|
||||
fail("Token evaluation failed");
|
||||
return;
|
||||
}
|
||||
handle->cache_tokens.push_back(token);
|
||||
n_decoded++;
|
||||
}
|
||||
|
||||
const int64_t t_end = now_ms();
|
||||
const int64_t prefill_ms = t_decode_start - t_prefill_start;
|
||||
const int64_t decode_ms = t_end - t_decode_start;
|
||||
LOGI("timing: prompt=%zu reused=%zu prefill=%zu tok in %lld ms (%.1f tok/s), decode=%d tok in %lld ms (%.1f tok/s)",
|
||||
tokens.size(), common, n_prefilled, (long long) prefill_ms,
|
||||
prefill_ms > 0 ? 1000.0 * (double) n_prefilled / (double) prefill_ms : 0.0,
|
||||
n_decoded, (long long) decode_ms,
|
||||
decode_ms > 0 ? 1000.0 * (double) n_decoded / (double) decode_ms : 0.0);
|
||||
|
||||
if (!pending_utf8.empty()) {
|
||||
size_t emit = utf8_complete_prefix(pending_utf8);
|
||||
if (emit > 0) listener.token(pending_utf8.substr(0, emit));
|
||||
}
|
||||
llama_sampler_free(chain);
|
||||
listener.done(stop_reason);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { streamAnthropicMessage } from "../native/anthropic";
|
||||
import { streamLocalQwen } from "../native/llama";
|
||||
import { streamOpenAiResponse } from "../native/openai";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
@@ -20,6 +21,8 @@ import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
/** Safety caps so a tool-thrashing model can't loop or run forever. */
|
||||
const MAX_ITERATIONS = 8;
|
||||
/** The on-phone model is slower per step and unreliable on long tool chains. */
|
||||
const MAX_ITERATIONS_LOCAL = 4;
|
||||
const MAX_TURN_MS = 2 * 60 * 1000;
|
||||
|
||||
export type DirectTurnOptions = {
|
||||
@@ -54,9 +57,10 @@ export class DirectAssistantBackend {
|
||||
options.callbacks.onError(message);
|
||||
};
|
||||
|
||||
const maxIterations = options.provider === "local" ? MAX_ITERATIONS_LOCAL : MAX_ITERATIONS;
|
||||
const runIteration = (iteration: number): void => {
|
||||
if (cancelled) return;
|
||||
if (iteration >= MAX_ITERATIONS) {
|
||||
if (iteration >= maxIterations) {
|
||||
finishError("Assistant stopped: too many tool steps");
|
||||
return;
|
||||
}
|
||||
@@ -115,9 +119,12 @@ export class DirectAssistantBackend {
|
||||
finishError(message);
|
||||
},
|
||||
};
|
||||
streamHandle = options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
streamHandle =
|
||||
options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: options.provider === "local"
|
||||
? streamLocalQwen(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
};
|
||||
|
||||
runIteration(0);
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import type { LlmToolDefinition } from "./llm-protocol";
|
||||
|
||||
/**
|
||||
* Generate a GBNF grammar (llama.cpp grammar dialect) that constrains a
|
||||
* Qwen-style tool call to be well-formed: one <tool_call> block whose JSON
|
||||
* names a real tool and whose arguments match that tool's schema. Used as a
|
||||
* *lazy* grammar — sampling is unconstrained until the model emits
|
||||
* <tool_call>, so plain text answers are unaffected, and once the block
|
||||
* closes the grammar is complete (the model can only end the message, which
|
||||
* serializes multi-call turns into one call per loop iteration — deliberate:
|
||||
* small models are far more reliable one call at a time).
|
||||
*
|
||||
* Our tool schemas are flat objects of string/number/boolean/enum properties.
|
||||
* Anything fancier degrades to a generic JSON value rule rather than failing.
|
||||
*/
|
||||
|
||||
type JsonSchema = {
|
||||
type?: string;
|
||||
enum?: unknown[];
|
||||
properties?: Record<string, JsonSchema>;
|
||||
required?: string[];
|
||||
description?: string;
|
||||
};
|
||||
|
||||
/** Longest run of optional properties we'll enumerate subsets for. */
|
||||
const MAX_SUBSET_PROPS = 5;
|
||||
|
||||
export function buildToolCallGrammar(tools: LlmToolDefinition[]): string | null {
|
||||
if (!tools.length) return null;
|
||||
const used = new Set<string>();
|
||||
const toolRules: string[] = [];
|
||||
for (let i = 0; i < tools.length; i++) {
|
||||
const args = argsRule(tools[i]!.input_schema as JsonSchema, used);
|
||||
toolRules.push(
|
||||
`tc${i} ::= "<tool_call>" nl "{" sp ${jsonKey("name")} sp ":" sp "\\"${gbnfEscape(
|
||||
tools[i]!.name,
|
||||
)}\\"" sp "," sp ${jsonKey("arguments")} sp ":" sp ${args} sp "}" nl "</tool_call>"`,
|
||||
);
|
||||
}
|
||||
used.add("sp");
|
||||
used.add("nl");
|
||||
|
||||
const lines = [`root ::= ${toolRules.map((_, i) => `tc${i}`).join(" | ")}`, ...toolRules];
|
||||
for (const [name, rule] of SHARED_RULES) {
|
||||
if (used.has(name)) lines.push(rule);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Build the arguments-object expression for one tool schema. */
|
||||
function argsRule(schema: JsonSchema | undefined, used: Set<string>): string {
|
||||
const properties = schema?.properties ?? {};
|
||||
const names = Object.keys(properties);
|
||||
if (!names.length) return `"{" sp "}"`;
|
||||
const required = new Set(schema?.required ?? []);
|
||||
const optionalCount = names.filter((n) => !required.has(n)).length;
|
||||
if (names.length > MAX_SUBSET_PROPS && optionalCount > 0) {
|
||||
used.add("jobj");
|
||||
return "jobj";
|
||||
}
|
||||
|
||||
// Enumerate every subset of properties that includes all required ones,
|
||||
// keeping declaration order. Small N keeps this tiny (<= 2^5 alternatives).
|
||||
const members = names.map((name) => {
|
||||
const value = valueExpr(properties[name]!, used);
|
||||
return `${jsonKey(name)} sp ":" sp ${value}`;
|
||||
});
|
||||
const alternatives: string[] = [];
|
||||
for (let mask = 0; mask < 1 << names.length; mask++) {
|
||||
const chosen: string[] = [];
|
||||
let valid = true;
|
||||
for (let j = 0; j < names.length; j++) {
|
||||
const included = (mask & (1 << j)) !== 0;
|
||||
if (included) chosen.push(members[j]!);
|
||||
else if (required.has(names[j]!)) valid = false;
|
||||
}
|
||||
if (!valid) continue;
|
||||
if (!chosen.length) {
|
||||
alternatives.push(`"{" sp "}"`);
|
||||
} else {
|
||||
alternatives.push(`"{" sp ${chosen.join(` sp "," sp `)} sp "}"`);
|
||||
}
|
||||
}
|
||||
return alternatives.length === 1 ? alternatives[0]! : `(${alternatives.join(" | ")})`;
|
||||
}
|
||||
|
||||
/** Grammar expression for one property value. */
|
||||
function valueExpr(schema: JsonSchema, used: Set<string>): string {
|
||||
if (Array.isArray(schema.enum) && schema.enum.length) {
|
||||
if (schema.enum.every((v) => typeof v === "string")) {
|
||||
return `(${schema.enum.map((v) => `"\\"${gbnfEscape(String(v))}\\""`).join(" | ")})`;
|
||||
}
|
||||
if (schema.enum.every((v) => typeof v === "number" || typeof v === "boolean")) {
|
||||
return `(${schema.enum.map((v) => `"${String(v)}"`).join(" | ")})`;
|
||||
}
|
||||
}
|
||||
switch (schema.type) {
|
||||
case "string":
|
||||
used.add("jstring");
|
||||
used.add("jchar");
|
||||
used.add("jhex");
|
||||
return "jstring";
|
||||
case "number":
|
||||
used.add("jnumber");
|
||||
return "jnumber";
|
||||
case "integer":
|
||||
used.add("jint");
|
||||
return "jint";
|
||||
case "boolean":
|
||||
used.add("jbool");
|
||||
return "jbool";
|
||||
default:
|
||||
for (const name of ["jvalue", "jstring", "jchar", "jhex", "jnumber", "jbool", "jarray", "jobj", "jmember"]) {
|
||||
used.add(name);
|
||||
}
|
||||
return "jvalue";
|
||||
}
|
||||
}
|
||||
|
||||
/** A JSON object key as a GBNF literal: "name" -> "\"name\"". */
|
||||
function jsonKey(name: string): string {
|
||||
return `"\\"${gbnfEscape(name)}\\""`;
|
||||
}
|
||||
|
||||
/** Escape for the inside of a GBNF double-quoted literal. */
|
||||
function gbnfEscape(text: string): string {
|
||||
return text.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
|
||||
}
|
||||
|
||||
// Terminal rules, emitted only when referenced. jchar mirrors llama.cpp's
|
||||
// json.gbnf string character rule.
|
||||
const SHARED_RULES: Array<[string, string]> = [
|
||||
["sp", `sp ::= " "?`],
|
||||
["nl", `nl ::= "\\n"`],
|
||||
["jstring", `jstring ::= "\\"" jchar* "\\""`],
|
||||
["jchar", `jchar ::= [^"\\\\\\x7F\\x00-\\x1F] | "\\\\" (["\\\\bfnrt] | "u" jhex jhex jhex jhex)`],
|
||||
["jhex", `jhex ::= [0-9a-fA-F]`],
|
||||
["jnumber", `jnumber ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? (("e" | "E") ("-" | "+")? [0-9]+)?`],
|
||||
["jint", `jint ::= "-"? ("0" | [1-9] [0-9]*)`],
|
||||
["jbool", `jbool ::= "true" | "false"`],
|
||||
["jvalue", `jvalue ::= jstring | jnumber | jbool | "null" | jarray | jobj`],
|
||||
["jarray", `jarray ::= "[" sp (jvalue (sp "," sp jvalue)*)? sp "]"`],
|
||||
["jobj", `jobj ::= "{" sp (jmember (sp "," sp jmember)*)? sp "}"`],
|
||||
["jmember", `jmember ::= jstring sp ":" sp jvalue`],
|
||||
];
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { LlmContentBlock, LlmMessage, LlmToolDefinition } from "./llm-protocol";
|
||||
|
||||
/**
|
||||
* Render our provider-neutral message history into a Qwen3 (ChatML) prompt
|
||||
* string, matching the model's chat template closely enough that its
|
||||
* tool-calling priors apply: Hermes-style <tools> declarations in the system
|
||||
* block, <tool_call> JSON blocks in assistant turns, and tool results wrapped
|
||||
* in <tool_response> inside a user turn. The caller appends nothing — the
|
||||
* returned string ends with the assistant generation header.
|
||||
*/
|
||||
|
||||
const IM_START = "<|im_start|>";
|
||||
const IM_END = "<|im_end|>";
|
||||
|
||||
export function buildQwenPrompt(
|
||||
system: string | undefined,
|
||||
messages: LlmMessage[],
|
||||
tools: LlmToolDefinition[] | undefined,
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`${IM_START}system\n${systemBlock(system, tools)}${IM_END}\n`);
|
||||
for (const message of messages) {
|
||||
if (message.role === "assistant") {
|
||||
parts.push(`${IM_START}assistant\n${assistantBlock(message.content)}${IM_END}\n`);
|
||||
} else {
|
||||
parts.push(`${IM_START}user\n${userBlock(message.content)}${IM_END}\n`);
|
||||
}
|
||||
}
|
||||
parts.push(`${IM_START}assistant\n`);
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function systemBlock(system: string | undefined, tools: LlmToolDefinition[] | undefined): string {
|
||||
const base = system?.trim() ?? "You are a helpful assistant.";
|
||||
if (!tools?.length) return base;
|
||||
const declarations = tools
|
||||
.map((tool) =>
|
||||
JSON.stringify({
|
||||
type: "function",
|
||||
function: { name: tool.name, description: tool.description, parameters: tool.input_schema },
|
||||
}),
|
||||
)
|
||||
.join("\n");
|
||||
return (
|
||||
`${base}\n\n# Tools\n\n` +
|
||||
`You may call one or more functions to assist with the user query.\n\n` +
|
||||
`You are provided with function signatures within <tools></tools> XML tags:\n` +
|
||||
`<tools>\n${declarations}\n</tools>\n\n` +
|
||||
`For each function call, return a json object with function name and arguments ` +
|
||||
`within <tool_call></tool_call> XML tags:\n` +
|
||||
`<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>`
|
||||
);
|
||||
}
|
||||
|
||||
function assistantBlock(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content;
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (block.type === "text") {
|
||||
if (block.text.trim()) parts.push(block.text.trim());
|
||||
} else if (block.type === "tool_use") {
|
||||
parts.push(`<tool_call>\n${JSON.stringify({ name: block.name, arguments: block.input ?? {} })}\n</tool_call>`);
|
||||
}
|
||||
// provider_item blocks are OpenAI-specific continuation state; skip.
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
function userBlock(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content;
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (block.type === "text") {
|
||||
if (block.text.trim()) parts.push(block.text.trim());
|
||||
} else if (block.type === "tool_result") {
|
||||
const body = block.is_error ? `Error: ${block.content}` : block.content || "(no output)";
|
||||
parts.push(`<tool_response>\n${body}\n</tool_response>`);
|
||||
}
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
@@ -1,4 +1,8 @@
|
||||
import { buildAssistantSystemPrompt } from "../prompts";
|
||||
import {
|
||||
ASSISTANT_SYSTEM_PROMPT_BASE,
|
||||
buildAssistantSystemPrompt,
|
||||
describeAssistantContext,
|
||||
} from "../prompts";
|
||||
import { assistantBridge } from "./bridge-client";
|
||||
import { DirectAssistantBackend } from "./direct-backend";
|
||||
import type { LlmMessage, LlmToolDefinition } from "./llm-protocol";
|
||||
@@ -108,15 +112,23 @@ export class AssistantSession {
|
||||
return;
|
||||
}
|
||||
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
const llm = this.config.llm;
|
||||
// Local provider: keep the system prompt (and thus the tool declarations
|
||||
// that follow it in the rendered prompt) byte-stable so the on-phone
|
||||
// model's KV prefix cache survives across turns; the volatile context
|
||||
// (time, foreground app, battery) rides along with the utterance instead.
|
||||
const isLocal = llm.provider === "local";
|
||||
this.messages.push({
|
||||
role: "user",
|
||||
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
|
||||
});
|
||||
this.trimHistory();
|
||||
this.turnHandle = this.directBackend.runTurn({
|
||||
provider: llm.provider,
|
||||
apiKey: llm.apiKey,
|
||||
model: llm.model,
|
||||
effort: llm.effort,
|
||||
system: buildAssistantSystemPrompt(ctx),
|
||||
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
||||
messages: this.messages,
|
||||
buildTools: () => this.buildToolDefinitions(),
|
||||
registry: this.registry,
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { buildToolCallGrammar } from "../assistant/gbnf";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* On-phone LLM backend: model download management plus a streaming client
|
||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||
* the direct assistant loop can treat "local" as just another provider.
|
||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||
*/
|
||||
|
||||
export const LOCAL_MODEL = {
|
||||
id: "qwen3-4b-instruct-2507",
|
||||
label: "Qwen3 4B",
|
||||
fileName: "Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
|
||||
url: "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_M.gguf?download=true",
|
||||
sha256: "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597",
|
||||
sizeBytes: 2497281120,
|
||||
};
|
||||
|
||||
// Bounded so KV cache stays ~0.5GB (Qwen3-4B KV is ~144KB/token at f16);
|
||||
// the assistant trims history well before this in practice.
|
||||
const N_CTX = 4096;
|
||||
const N_THREADS = 4;
|
||||
const DEFAULT_MAX_TOKENS = 512;
|
||||
// Qwen3-4B-Instruct-2507 model-card sampling posture.
|
||||
const TEMPERATURE = 0.7;
|
||||
const TOP_P = 0.8;
|
||||
const TOP_K = 20;
|
||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||
|
||||
export type LocalModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
|
||||
let runner: any = null;
|
||||
let downloader: any = null;
|
||||
let downloadedBytes = 0;
|
||||
let unloadTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let toolUseCounter = 0;
|
||||
const stateListeners = new Set<(state: LocalModelState) => void>();
|
||||
|
||||
function modelPath(): string {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
const dir = context.getExternalFilesDir("llm") ?? context.getFilesDir();
|
||||
return `${dir.getAbsolutePath()}/${LOCAL_MODEL.fileName}`;
|
||||
}
|
||||
|
||||
export function isLocalModelReady(): boolean {
|
||||
if (!global.isAndroid) return false;
|
||||
try {
|
||||
const file = new java.io.File(modelPath());
|
||||
return file.exists() && file.length() > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function localModelState(): LocalModelState {
|
||||
if (downloader) {
|
||||
return { status: "downloading", bytesDownloaded: downloadedBytes, totalBytes: LOCAL_MODEL.sizeBytes };
|
||||
}
|
||||
return {
|
||||
status: isLocalModelReady() ? "ready" : "absent",
|
||||
bytesDownloaded: 0,
|
||||
totalBytes: LOCAL_MODEL.sizeBytes,
|
||||
};
|
||||
}
|
||||
|
||||
export function onLocalModelStateChanged(listener: (state: LocalModelState) => void): () => void {
|
||||
stateListeners.add(listener);
|
||||
return () => stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
function notifyStateChanged(): void {
|
||||
const state = localModelState();
|
||||
for (const listener of stateListeners) listener(state);
|
||||
}
|
||||
|
||||
export function startLocalModelDownload(): void {
|
||||
if (!global.isAndroid || downloader || isLocalModelReady()) return;
|
||||
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
|
||||
onProgress: (bytes: number, _total: number) => {
|
||||
downloadedBytes = Number(bytes);
|
||||
notifyStateChanged();
|
||||
},
|
||||
onDone: () => {
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
},
|
||||
onError: (message: string) => {
|
||||
console.error(`Local model download failed: ${message}`);
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
},
|
||||
});
|
||||
downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
LOCAL_MODEL.url,
|
||||
modelPath(),
|
||||
LOCAL_MODEL.sha256,
|
||||
LOCAL_MODEL.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloadedBytes = 0;
|
||||
downloader.start();
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
/** Stops the download; already-fetched bytes are kept and resumed next time. */
|
||||
export function cancelLocalModelDownload(): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloader = null;
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
export function deleteLocalModel(): void {
|
||||
if (!global.isAndroid) return;
|
||||
cancelLocalModelDownload();
|
||||
unloadLocalModel();
|
||||
try {
|
||||
new java.io.File(modelPath()).delete();
|
||||
new java.io.File(`${modelPath()}.part`).delete();
|
||||
} catch (error) {
|
||||
console.error(`Local model delete failed: ${String(error)}`);
|
||||
}
|
||||
notifyStateChanged();
|
||||
}
|
||||
|
||||
export function unloadLocalModel(): void {
|
||||
if (unloadTimer) {
|
||||
clearTimeout(unloadTimer);
|
||||
unloadTimer = null;
|
||||
}
|
||||
runner?.unload();
|
||||
}
|
||||
|
||||
function scheduleIdleUnload(): void {
|
||||
if (unloadTimer) clearTimeout(unloadTimer);
|
||||
unloadTimer = setTimeout(() => {
|
||||
unloadTimer = null;
|
||||
runner?.unload();
|
||||
}, IDLE_UNLOAD_MS);
|
||||
}
|
||||
|
||||
function ensureRunner(): any {
|
||||
if (!runner) {
|
||||
runner = new com.faceclaw.app.FaceclawLlamaRunner();
|
||||
}
|
||||
return runner;
|
||||
}
|
||||
|
||||
const TOOL_CALL_OPEN = "<tool_call>";
|
||||
const TOOL_CALL_CLOSE = "</tool_call>";
|
||||
|
||||
/**
|
||||
* Stream a completion from the on-phone model. Tool calls arrive inline as
|
||||
* <tool_call> JSON blocks; they are hidden from the text stream and surfaced
|
||||
* as tool_use content blocks in the final result, mirroring the cloud
|
||||
* providers' contract (stopReason "tool_use" when the model called a tool).
|
||||
*/
|
||||
export function streamLocalQwen(options: LlmStreamOptions): LlmStreamHandle {
|
||||
let settled = false;
|
||||
let raw = "";
|
||||
let emitted = "";
|
||||
|
||||
const fail = (message: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
scheduleIdleUnload();
|
||||
options.onError(message);
|
||||
};
|
||||
|
||||
if (!global.isAndroid) {
|
||||
setTimeout(() => fail("The on-phone model is only available on Android"), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
if (!isLocalModelReady()) {
|
||||
setTimeout(() => fail("The on-phone model isn't downloaded yet (see Settings)"), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
if (unloadTimer) {
|
||||
clearTimeout(unloadTimer);
|
||||
unloadTimer = null;
|
||||
}
|
||||
|
||||
const prompt = buildQwenPrompt(options.system, options.messages, options.tools);
|
||||
const grammar = options.tools?.length ? buildToolCallGrammar(options.tools) : null;
|
||||
|
||||
const listener = new com.faceclaw.app.FaceclawLlamaListener({
|
||||
onToken: (piece: string) => {
|
||||
if (settled) return;
|
||||
raw += String(piece);
|
||||
const visible = visibleText(raw);
|
||||
if (visible.length > emitted.length) {
|
||||
const delta = visible.slice(emitted.length);
|
||||
emitted = visible;
|
||||
options.onTextDelta?.(delta, emitted);
|
||||
}
|
||||
},
|
||||
onDone: (nativeStopReason: string) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
scheduleIdleUnload();
|
||||
const { text, content, toolCount } = assembleContent(raw);
|
||||
const stopReason =
|
||||
toolCount > 0 ? "tool_use" : String(nativeStopReason) === "length" ? "max_tokens" : "end_turn";
|
||||
options.onDone({ text, content, stopReason });
|
||||
},
|
||||
onError: (message: string) => fail(String(message)),
|
||||
});
|
||||
|
||||
try {
|
||||
ensureRunner().generate(
|
||||
modelPath(),
|
||||
N_CTX,
|
||||
N_THREADS,
|
||||
prompt,
|
||||
grammar ?? "",
|
||||
options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
TEMPERATURE,
|
||||
TOP_P,
|
||||
TOP_K,
|
||||
listener,
|
||||
);
|
||||
} catch (error) {
|
||||
setTimeout(() => fail(`On-phone model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
|
||||
return {
|
||||
cancel: () => {
|
||||
settled = true;
|
||||
scheduleIdleUnload();
|
||||
try {
|
||||
runner?.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The streamable portion of the raw output: text outside completed
|
||||
* <tool_call> blocks, holding back any trailing partial marker so a tool
|
||||
* call never leaks into the visible stream. Monotonic under appends.
|
||||
*/
|
||||
function visibleText(raw: string): string {
|
||||
let out = "";
|
||||
let i = 0;
|
||||
for (;;) {
|
||||
const start = raw.indexOf(TOOL_CALL_OPEN, i);
|
||||
if (start === -1) {
|
||||
const tail = raw.slice(i);
|
||||
out += tail.slice(0, tail.length - partialSuffixLength(tail, TOOL_CALL_OPEN));
|
||||
return out;
|
||||
}
|
||||
out += raw.slice(i, start);
|
||||
const end = raw.indexOf(TOOL_CALL_CLOSE, start);
|
||||
if (end === -1) return out; // inside an unfinished tool call
|
||||
i = end + TOOL_CALL_CLOSE.length;
|
||||
}
|
||||
}
|
||||
|
||||
/** Length of the longest proper prefix of marker that text ends with. */
|
||||
function partialSuffixLength(text: string, marker: string): number {
|
||||
const max = Math.min(text.length, marker.length - 1);
|
||||
for (let k = max; k > 0; k--) {
|
||||
if (text.endsWith(marker.slice(0, k))) return k;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Split the finished raw output into text + tool_use content blocks. */
|
||||
function assembleContent(raw: string): { text: string; content: LlmContentBlock[]; toolCount: number } {
|
||||
const content: LlmContentBlock[] = [];
|
||||
let text = "";
|
||||
let toolCount = 0;
|
||||
let i = 0;
|
||||
for (;;) {
|
||||
const start = raw.indexOf(TOOL_CALL_OPEN, i);
|
||||
if (start === -1) {
|
||||
text += raw.slice(i);
|
||||
break;
|
||||
}
|
||||
text += raw.slice(i, start);
|
||||
const end = raw.indexOf(TOOL_CALL_CLOSE, start);
|
||||
if (end === -1) break; // truncated call (length stop); drop it
|
||||
const body = raw.slice(start + TOOL_CALL_OPEN.length, end).trim();
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
if (parsed && typeof parsed.name === "string") {
|
||||
toolCount++;
|
||||
content.push({
|
||||
type: "tool_use",
|
||||
id: `local_${Date.now()}_${++toolUseCounter}`,
|
||||
name: parsed.name,
|
||||
input: parsed.arguments ?? {},
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
console.error(`Local model emitted unparseable tool call: ${body.slice(0, 200)}`);
|
||||
}
|
||||
i = end + TOOL_CALL_CLOSE.length;
|
||||
}
|
||||
text = text.trim();
|
||||
if (text) content.unshift({ type: "text", text });
|
||||
return { text, content, toolCount };
|
||||
}
|
||||
+9
-2
@@ -15,10 +15,17 @@ export const ASSISTANT_SYSTEM_PROMPT_BASE = [
|
||||
|
||||
/** Full assistant system prompt: the base plus the current device context. */
|
||||
export function buildAssistantSystemPrompt(ctx: AssistantContext): string {
|
||||
return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
|
||||
return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeAssistantContext(ctx)}`;
|
||||
}
|
||||
|
||||
function describeContext(ctx: AssistantContext): string {
|
||||
/**
|
||||
* The volatile device context on its own. The local provider appends this to
|
||||
* the user message instead of the system prompt: the system prompt + tool
|
||||
* declarations are the bulk of the prompt, and keeping them byte-stable lets
|
||||
* the on-phone model reuse its KV-cache prefix across turns instead of
|
||||
* re-prefilling ~2-3k tokens because the clock changed.
|
||||
*/
|
||||
export function describeAssistantContext(ctx: AssistantContext): string {
|
||||
const parts = [`Current time: ${ctx.localTime}.`];
|
||||
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
|
||||
if (ctx.foregroundApp) {
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
assistantModelProvider,
|
||||
type AssistantModel,
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/llama";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
import { DashboardInputEvent, Layer, type LayerContext } from "./layers";
|
||||
import { GrayImage } from "~/graphics/image";
|
||||
@@ -357,7 +358,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
export type AssistantBackendKind = "direct" | "external";
|
||||
|
||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||
direct: "On-phone (API key)",
|
||||
direct: "On-phone",
|
||||
external: "My own agent (bridge)",
|
||||
};
|
||||
|
||||
@@ -369,7 +370,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
|
||||
values: ["direct", "external"],
|
||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||
description:
|
||||
"Who answers assistant queries: an LLM called directly from the phone (needs an API key), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
"Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
});
|
||||
|
||||
export const assistantBridgeHostSetting = new ConfigSettingString({
|
||||
@@ -468,9 +469,11 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
|
||||
const provider = assistantModelProvider(value);
|
||||
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
|
||||
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
|
||||
if (provider === "local") return !isLocalModelReady();
|
||||
return false;
|
||||
},
|
||||
description: "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set.",
|
||||
description:
|
||||
"Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
|
||||
});
|
||||
|
||||
export const mapboxApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -3,8 +3,17 @@ import { getDefaultSmallFont } from "../../graphics/bdffont";
|
||||
import type { GrayImage } from "../../graphics/image";
|
||||
import { getDashboardLogo } from "../../graphics/logo";
|
||||
import { wrapText } from "../../graphics/textwrap";
|
||||
import {
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
LOCAL_MODEL,
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
} from "../../native/llama";
|
||||
import { TextViewerLayer } from "../../apps/files/text-viewer";
|
||||
import type { MenuItem } from "../menu";
|
||||
import type { LayerContext } from "../layers";
|
||||
import { drawRightValueMenuItem, openModalMenu, type MenuItem } from "../menu";
|
||||
import { shell } from "../shell/shell";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
@@ -88,6 +97,7 @@ function settingsSections(): SettingsSection[] {
|
||||
// On-phone LLM loop vs the user's own agent via the bridge plugin.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
localModelMenuItem(),
|
||||
// When on, a wakeword utterance goes straight to the assistant with no
|
||||
// Send/Type menu step.
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
@@ -161,6 +171,77 @@ function settingsSections(): SettingsSection[] {
|
||||
];
|
||||
}
|
||||
|
||||
const LOCAL_MODEL_GB = `${(LOCAL_MODEL.sizeBytes / 1e9).toFixed(1)}GB`;
|
||||
|
||||
// While a download is running, re-render on progress updates so the row's
|
||||
// percentage stays live; the watch tears itself down when the download ends.
|
||||
let localModelRenderUnsub: (() => void) | null = null;
|
||||
|
||||
function watchLocalModelDownload(ctx: LayerContext): void {
|
||||
localModelRenderUnsub?.();
|
||||
localModelRenderUnsub = onLocalModelStateChanged((state) => {
|
||||
ctx.actions.requestRender();
|
||||
if (state.status !== "downloading") {
|
||||
localModelRenderUnsub?.();
|
||||
localModelRenderUnsub = null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function localModelStatusText(): string {
|
||||
const state = localModelState();
|
||||
if (state.status === "ready") return "downloaded";
|
||||
if (state.status === "downloading") {
|
||||
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
|
||||
return `${pct}% of ${LOCAL_MODEL_GB}`;
|
||||
}
|
||||
return "not downloaded";
|
||||
}
|
||||
|
||||
/** Download/cancel/delete management for the on-phone assistant model. */
|
||||
function localModelMenuItem(): MenuItem {
|
||||
return {
|
||||
label: "On-phone model",
|
||||
description:
|
||||
`${LOCAL_MODEL.label} (${LOCAL_MODEL_GB} download over Wi-Fi recommended). ` +
|
||||
"Answers assistant queries on the phone itself, with no API key or cloud service. " +
|
||||
"Slower and simpler than the cloud models, but free and private. " +
|
||||
"Used automatically when no API key is set. An interrupted download resumes where it left off.",
|
||||
onSelect: (ctx) => {
|
||||
const state = localModelState();
|
||||
const action: MenuItem =
|
||||
state.status === "downloading"
|
||||
? {
|
||||
label: "Cancel download",
|
||||
onSelect: (innerCtx) => {
|
||||
cancelLocalModelDownload();
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
: state.status === "ready"
|
||||
? {
|
||||
label: "Delete model",
|
||||
onSelect: (innerCtx) => {
|
||||
deleteLocalModel();
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
}
|
||||
: {
|
||||
label: `Download (${LOCAL_MODEL_GB})`,
|
||||
onSelect: (innerCtx) => {
|
||||
startLocalModelDownload();
|
||||
watchLocalModelDownload(innerCtx);
|
||||
innerCtx.stack.pop();
|
||||
},
|
||||
};
|
||||
openModalMenu(ctx, "On-phone model", [action], 0);
|
||||
},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", localModelStatusText());
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** A row that opens one of the project docs (copied into the bundle under
|
||||
* about/ by webpack.config.js) in the paged text viewer. */
|
||||
function bundledDocMenuItem(fileName: string, label: string): MenuItem {
|
||||
|
||||
@@ -801,7 +801,7 @@ class Shell {
|
||||
this.showAlert(
|
||||
assistantBackendSetting.get() === "external"
|
||||
? "Configure the agent bridge host and token in Settings."
|
||||
: "Set an API key for the selected assistant model in Settings.",
|
||||
: "Set an API key or download the on-phone model in Settings.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user