Add support for a local model (Qwen3)

This commit is contained in:
James Babcock committed 2026-08-09 23:52:21 -07:00
1 parent 6103c00ca3
commit 6f45888627
16 files changed
+1532 -15

No files matched your search

+97
View File
@@ -149,6 +149,102 @@ tasks.register("prepareFaceclawNativeLibs") {
}
}
// On-phone assistant model: build llama.cpp (pinned release) + our JNI shim
// into libfaceclaw_llama.so. CPU-only, arm64, armv8.2 dotprod+fp16 baseline
// (Tensor G2 floor; no i8mm). The first build compiles llama.cpp (~2-4 min);
// afterwards the cached .so short-circuits everything.
def faceclawLlamaCppTag = "b10333"
def faceclawFindSdkCmakeDir = {
def sdkRoot = System.getenv("ANDROID_HOME") ?: System.getenv("ANDROID_SDK_ROOT")
if (sdkRoot == null) return null
def cmakeRoot = file("$sdkRoot/cmake")
if (!cmakeRoot.exists()) return null
def installed = cmakeRoot.listFiles()?.findAll { it.isDirectory() }?.sort { a, b -> b.name <=> a.name }
return (installed && !installed.isEmpty()) ? installed[0] : null
}
tasks.register("prepareFaceclawLlama") {
def llamaSo = file("$faceclawJniLibsDir/libfaceclaw_llama.so")
inputs.files(
file("$projectDir/src/main/native/llama/faceclaw_llama.cpp"),
file("$projectDir/src/main/native/llama/CMakeLists.txt")
)
outputs.files(llamaSo)
doLast {
def llamaBuildDir = file("$faceclawNativeBuildDir/llama-build")
def builtSo = file("$llamaBuildDir/libfaceclaw_llama.so")
def downloadsDir = file("$faceclawNativeBuildDir/downloads")
def llamaArchive = file("$downloadsDir/llama.cpp-${faceclawLlamaCppTag}.tar.gz")
faceclawDownload(
"https://github.com/ggml-org/llama.cpp/archive/refs/tags/${faceclawLlamaCppTag}.tar.gz",
llamaArchive
)
def llamaSourceDir = file("$faceclawNativeBuildDir/llama.cpp-${faceclawLlamaCppTag}")
if (!file("$llamaSourceDir/CMakeLists.txt").exists()) {
copy {
from tarTree(resources.gzip(llamaArchive))
into faceclawNativeBuildDir
}
}
def ndkRoot = faceclawFindNdkRoot()
if (ndkRoot == null || !ndkRoot.exists()) {
throw new GradleException("Android NDK is required to build libfaceclaw_llama.so")
}
def cmakeDir = faceclawFindSdkCmakeDir()
if (cmakeDir == null) {
throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so")
}
def cmake = file("$cmakeDir/bin/cmake")
def ninja = file("$cmakeDir/bin/ninja")
def march = "-march=armv8.2-a+dotprod+fp16"
// Configure once; afterwards the ninja no-op build keeps this cheap while
// still picking up JNI source edits.
if (!file("$llamaBuildDir/build.ninja").exists()) {
exec {
commandLine([
cmake.absolutePath,
"-S", file("$projectDir/src/main/native/llama").absolutePath,
"-B", llamaBuildDir.absolutePath,
"-G", "Ninja",
"-DCMAKE_MAKE_PROGRAM=${ninja.absolutePath}",
"-DCMAKE_TOOLCHAIN_FILE=${file("$ndkRoot/build/cmake/android.toolchain.cmake").absolutePath}",
"-DANDROID_ABI=arm64-v8a",
"-DANDROID_PLATFORM=android-24",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLAMA_CPP_SOURCE_DIR=${llamaSourceDir.absolutePath}",
"-DCMAKE_C_FLAGS=$march",
"-DCMAKE_CXX_FLAGS=$march",
])
}
}
exec {
commandLine([cmake.absolutePath, "--build", llamaBuildDir.absolutePath, "-j", "8"])
}
faceclawJniLibsDir.mkdirs()
copy {
from builtSo
into faceclawJniLibsDir
}
// The static-linked lib carries llama.cpp's full debug info (~50MB);
// strip it so the APK only pays for the code (~8MB).
def ndkRootForStrip = faceclawFindNdkRoot()
if (ndkRootForStrip != null) {
def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip")
if (llvmStrip.exists()) {
exec {
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
}
}
}
}
}
android {
compileSdkVersion 35
buildToolsVersion "35"
@@ -179,4 +275,5 @@ android {
tasks.matching { it.name == "preBuild" }.configureEach {
dependsOn tasks.named("prepareFaceclawNativeLibs")
dependsOn tasks.named("prepareFaceclawLlama")
}
@@ -0,0 +1,16 @@
package com.faceclaw.app;
/**
* Callbacks for one on-phone LLM generation. Like the other bridge listeners,
* implementations are typically created on the NativeScript JS thread and
* FaceclawLlamaRunner posts these callbacks back to that thread's Looper.
*/
public interface FaceclawLlamaListener {
/** A piece of generated text (may be a partial word). */
void onToken(String piece);
/** Generation finished: "stop" (EOG), "length", or "cancelled". */
void onDone(String stopReason);
void onError(String message);
}
@@ -0,0 +1,122 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* On-phone LLM inference for the voice assistant (llama.cpp over JNI).
*
* All model work (load, generate, free) runs on a single background executor
* thread so calls are naturally serialized; listener callbacks are posted to
* the Looper of the thread that constructed this object (the JS thread), the
* same contract as FaceclawSseRequest. cancel() may be called from any thread
* and interrupts the in-flight generation at the next token boundary.
*
* The model stays loaded between generations (turn iterations reuse the KV
* cache prefix); the TypeScript side calls unload() on an idle timer to give
* the ~3GB back.
*/
public class FaceclawLlamaRunner {
private static final String TAG = "FaceclawLlamaRunner";
static {
System.loadLibrary("faceclaw_llama");
}
private final Handler callbackHandler;
private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> {
Thread thread = new Thread(runnable, "FaceclawLlama");
thread.setPriority(Thread.NORM_PRIORITY);
return thread;
});
// Mutated only on the executor thread; volatile so cancel() can read it.
private volatile long handle;
private String loadedPath;
public FaceclawLlamaRunner() {
Looper looper = Looper.myLooper();
callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
}
public boolean isModelLoaded() {
return handle != 0;
}
/**
* Generate a completion, loading (or swapping) the model first if needed.
* grammar may be null/empty for unconstrained generation.
*/
public void generate(final String modelPath, final int nCtx, final int nThreads,
final String prompt, final String grammar, final int maxTokens,
final float temperature, final float topP, final int topK,
final FaceclawLlamaListener listener) {
executor.execute(() -> {
try {
if (handle != 0 && !modelPath.equals(loadedPath)) {
nativeFree(handle);
handle = 0;
loadedPath = null;
}
if (handle == 0) {
long loaded = nativeLoadModel(modelPath, nCtx, nThreads);
if (loaded == 0) {
post(() -> listener.onError("Could not load the on-phone model"));
return;
}
handle = loaded;
loadedPath = modelPath;
}
nativeGenerate(handle, prompt, grammar == null ? "" : grammar,
maxTokens, temperature, topP, topK, new FaceclawLlamaListener() {
@Override public void onToken(final String piece) {
post(() -> listener.onToken(piece));
}
@Override public void onDone(final String stopReason) {
post(() -> listener.onDone(stopReason));
}
@Override public void onError(final String message) {
post(() -> listener.onError(message));
}
});
} catch (Throwable t) {
Log.e(TAG, "generation failed", t);
final String message = String.valueOf(t);
post(() -> listener.onError("On-phone model failed: " + message));
}
});
}
/** Interrupt the current generation (its listener gets onDone("cancelled")). */
public void cancel() {
long current = handle;
if (current != 0) {
nativeCancel(current);
}
}
/** Free the model and its KV cache. Safe to call when nothing is loaded. */
public void unload() {
executor.execute(() -> {
if (handle != 0) {
nativeFree(handle);
handle = 0;
loadedPath = null;
}
});
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
private static native long nativeLoadModel(String path, int nCtx, int nThreads);
private static native void nativeGenerate(long handle, String prompt, String grammar,
int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener);
private static native void nativeCancel(long handle);
private static native void nativeFree(long handle);
}
@@ -0,0 +1,192 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.concurrent.TimeUnit;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.ResponseBody;
/**
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
* restarts, verifies a pinned sha256 (hashed incrementally, including the
* resumed prefix), then renames into place. Listener callbacks are posted to
* the constructing thread's Looper, like the other Faceclaw bridges.
*/
public class FaceclawModelDownloader {
private static final String TAG = "FaceclawModelDl";
private static final long PROGRESS_INTERVAL_MS = 500;
private static volatile OkHttpClient sharedClient;
private final Handler callbackHandler;
private final String url;
private final File destFile;
private final File partFile;
private final String expectedSha256;
private final long expectedTotalBytes;
private final FaceclawModelDownloaderListener listener;
private volatile boolean cancelled;
private volatile Call call;
private Thread worker;
public FaceclawModelDownloader(String url, String destPath, String expectedSha256,
long expectedTotalBytes, FaceclawModelDownloaderListener listener) {
if (listener == null) throw new IllegalArgumentException("listener is required");
Looper looper = Looper.myLooper();
this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
this.url = url;
this.destFile = new File(destPath);
this.partFile = new File(destPath + ".part");
this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase();
this.expectedTotalBytes = expectedTotalBytes;
this.listener = listener;
}
private static OkHttpClient getClient() {
if (sharedClient == null) {
synchronized (FaceclawModelDownloader.class) {
if (sharedClient == null) {
sharedClient = new OkHttpClient.Builder()
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.build();
}
}
}
return sharedClient;
}
public void start() {
worker = new Thread(this::run, "FaceclawModelDl");
worker.start();
}
/** Stops the download but keeps the .part file so a later start() resumes. */
public void cancel() {
cancelled = true;
Call current = call;
if (current != null) current.cancel();
}
private void run() {
try {
if (destFile.exists() && destFile.length() > 0) {
post(() -> listener.onDone(destFile.getAbsolutePath()));
return;
}
destFile.getParentFile().mkdirs();
MessageDigest digest = MessageDigest.getInstance("SHA-256");
long offset = 0;
if (partFile.exists()) {
offset = hashExistingPrefix(digest);
if (cancelled) return;
}
Request.Builder builder = new Request.Builder().url(url);
if (offset > 0) builder.header("Range", "bytes=" + offset + "-");
call = getClient().newCall(builder.build());
try (Response response = call.execute()) {
ResponseBody body = response.body();
if (!response.isSuccessful() || body == null) {
postError("Model download failed: HTTP " + response.code());
return;
}
boolean resumed = response.code() == 206;
if (offset > 0 && !resumed) {
// Server ignored the Range header; start over.
offset = 0;
digest = MessageDigest.getInstance("SHA-256");
}
long total = expectedTotalBytes > 0 ? expectedTotalBytes
: (body.contentLength() > 0 ? offset + body.contentLength() : -1);
try (FileOutputStream out = new FileOutputStream(partFile, resumed);
InputStream in = body.byteStream()) {
byte[] buffer = new byte[1 << 16];
long done = offset;
long lastProgressAt = 0;
int read;
while ((read = in.read(buffer)) != -1) {
if (cancelled) return;
out.write(buffer, 0, read);
digest.update(buffer, 0, read);
done += read;
long now = System.currentTimeMillis();
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
lastProgressAt = now;
final long doneFinal = done;
final long totalFinal = total;
post(() -> listener.onProgress(doneFinal, totalFinal));
}
}
}
}
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
postError("Model download ended early (" + partFile.length() + " of "
+ expectedTotalBytes + " bytes); try again to resume");
return;
}
String actualSha = toHex(digest.digest());
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
partFile.delete();
postError("Model download was corrupted (checksum mismatch); download it again");
return;
}
if (!partFile.renameTo(destFile)) {
postError("Could not move the downloaded model into place");
return;
}
post(() -> listener.onDone(destFile.getAbsolutePath()));
} catch (Exception e) {
if (cancelled) return;
Log.w(TAG, "download failed", e);
postError("Model download failed: " + e);
}
}
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
private long hashExistingPrefix(MessageDigest digest) throws IOException {
long hashed = 0;
try (FileInputStream in = new FileInputStream(partFile)) {
byte[] buffer = new byte[1 << 16];
int read;
while ((read = in.read(buffer)) != -1) {
if (cancelled) return hashed;
digest.update(buffer, 0, read);
hashed += read;
}
}
return hashed;
}
private static String toHex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) sb.append(String.format("%02x", b));
return sb.toString();
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
private void postError(final String message) {
post(() -> listener.onError(message));
}
}
@@ -0,0 +1,11 @@
package com.faceclaw.app;
/** Progress callbacks for FaceclawModelDownloader, posted to the creating thread's Looper. */
public interface FaceclawModelDownloaderListener {
void onProgress(long bytesDownloaded, long totalBytes);
/** The file is fully downloaded, verified, and moved to its final path. */
void onDone(String path);
void onError(String message);
}
@@ -0,0 +1,31 @@
cmake_minimum_required(VERSION 3.22)
project(faceclaw_llama CXX C)
# llama.cpp source tree location is injected by the gradle task (it downloads
# and extracts a pinned release tag before configuring this project).
if(NOT DEFINED LLAMA_CPP_SOURCE_DIR)
message(FATAL_ERROR "Pass -DLLAMA_CPP_SOURCE_DIR=<path to llama.cpp checkout>")
endif()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
# Static llama/ggml folded into our single JNI .so. CPU only: Vulkan on Mali
# and mixed Adreno drivers is not worth the instability for short assistant
# turns. The -march baseline lives in the gradle invocation (armv8.2-a with
# dotprod+fp16: the floor for Tensor G2's X1/A78/A55, so no i8mm).
set(BUILD_SHARED_LIBS OFF)
set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE)
set(LLAMA_CURL OFF CACHE BOOL "" FORCE)
set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
add_subdirectory(${LLAMA_CPP_SOURCE_DIR} llama.cpp-build EXCLUDE_FROM_ALL)
add_library(faceclaw_llama SHARED faceclaw_llama.cpp)
target_link_libraries(faceclaw_llama PRIVATE llama android log)
@@ -0,0 +1,391 @@
// JNI bridge for the on-phone assistant model (llama.cpp, CPU inference).
// Java surface: FaceclawLlamaRunner's nativeLoadModel / nativeGenerate /
// nativeCancel / nativeFree. One model+context per handle; the Java side
// serializes all calls except nativeCancel onto a single executor thread,
// so only the cancel flag needs to be thread-safe.
#include <jni.h>
#include <android/log.h>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstring>
#include <mutex>
#include <string>
#include <vector>
#include "ggml-cpu.h"
#include "llama.h"
#define TAG "FaceclawLlama"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
namespace {
struct FcLlama {
llama_model * model = nullptr;
llama_context * ctx = nullptr;
ggml_threadpool * threadpool = nullptr;
const llama_vocab * vocab = nullptr;
int n_ctx = 0;
// Tokens currently materialized in the KV cache (prompt + generated),
// used to skip re-decoding the common prefix on the next call. Within a
// turn the next prompt extends the previous one, so this usually reduces
// prefill to just the new tool results.
std::vector<llama_token> cache_tokens;
std::atomic<bool> cancel{false};
};
void android_llama_log(ggml_log_level level, const char * text, void * /*user*/) {
if (level < GGML_LOG_LEVEL_WARN) return; // llama.cpp is chatty at INFO during load
__android_log_write(level >= GGML_LOG_LEVEL_ERROR ? ANDROID_LOG_ERROR : ANDROID_LOG_WARN, TAG, text);
}
void ensure_backend_init() {
static std::once_flag once;
std::call_once(once, [] {
llama_log_set(android_llama_log, nullptr);
llama_backend_init();
});
}
int64_t now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
}
// Big-core detection for big.LITTLE phones. Without explicit affinity the
// scheduler is free to put our compute threads on the little cores (in-order
// A55s on Tensor G2), which slows prefill by 5-10x; pin to the cores whose
// max frequency is within 75% of the fastest core's (on a Pixel 7/7a this
// selects the 2x X1 + 2x A78 and excludes the 4x A55).
std::vector<int> detect_fast_cpus() {
std::vector<long> freqs;
for (int cpu = 0; cpu < GGML_MAX_N_THREADS; cpu++) {
char path[128];
snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", cpu);
FILE * f = fopen(path, "r");
if (f == nullptr) break;
long freq = 0;
if (fscanf(f, "%ld", &freq) != 1) freq = 0;
fclose(f);
freqs.push_back(freq);
}
std::vector<int> fast;
long max_freq = 0;
for (long f : freqs) max_freq = std::max(max_freq, f);
if (max_freq <= 0) return fast;
for (size_t cpu = 0; cpu < freqs.size(); cpu++) {
if (freqs[cpu] * 4 >= max_freq * 3) fast.push_back((int) cpu);
}
// A uniform-frequency CPU has no little cores to avoid.
if (fast.size() == freqs.size()) fast.clear();
return fast;
}
std::string jstring_to_utf8(JNIEnv * env, jstring value) {
if (value == nullptr) return "";
const char * chars = env->GetStringUTFChars(value, nullptr);
std::string out = chars ? chars : "";
env->ReleaseStringUTFChars(value, chars);
return out;
}
// Callback shims into the Java listener (an in-process wrapper object that
// posts to the JS thread; these calls happen on the executor thread).
struct Listener {
JNIEnv * env;
jobject obj;
jmethodID on_token;
jmethodID on_done;
jmethodID on_error;
static Listener resolve(JNIEnv * env, jobject listener) {
jclass cls = env->GetObjectClass(listener);
return Listener{
env,
listener,
env->GetMethodID(cls, "onToken", "(Ljava/lang/String;)V"),
env->GetMethodID(cls, "onDone", "(Ljava/lang/String;)V"),
env->GetMethodID(cls, "onError", "(Ljava/lang/String;)V"),
};
}
void token(const std::string & piece) const { call(on_token, piece); }
void done(const char * reason) const { call(on_done, reason); }
void error(const std::string & message) const { call(on_error, message); }
private:
void call(jmethodID method, const std::string & text) const {
jstring jtext = env->NewStringUTF(text.c_str());
env->CallVoidMethod(obj, method, jtext);
env->DeleteLocalRef(jtext);
if (env->ExceptionCheck()) {
env->ExceptionClear();
}
}
};
// NewStringUTF aborts on invalid (modified-)UTF-8, and token pieces can end
// mid-codepoint, so hold back any incomplete trailing UTF-8 sequence.
size_t utf8_complete_prefix(const std::string & data) {
size_t len = data.size();
size_t i = len;
// Find the start byte of the final codepoint (at most 3 bytes back).
while (i > 0 && len - i < 4 && (static_cast<unsigned char>(data[i - 1]) & 0xC0) == 0x80) i--;
if (i == 0) return len; // malformed; let it through rather than stall
unsigned char start = static_cast<unsigned char>(data[i - 1]);
size_t expected;
if (start < 0x80) expected = 1;
else if ((start & 0xE0) == 0xC0) expected = 2;
else if ((start & 0xF0) == 0xE0) expected = 3;
else if ((start & 0xF8) == 0xF0) expected = 4;
else return len; // malformed start byte; pass through
size_t have = len - (i - 1);
return have >= expected ? len : i - 1;
}
std::vector<llama_token> tokenize(const llama_vocab * vocab, const std::string & text, bool add_special) {
int n = -llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), nullptr, 0, add_special, true);
if (n <= 0) return {};
std::vector<llama_token> tokens(n);
int written = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), n, add_special, true);
if (written < 0) return {};
tokens.resize(written);
return tokens;
}
std::string token_piece(const llama_vocab * vocab, llama_token token) {
char buf[256];
int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true);
if (n <= 0) return "";
return std::string(buf, n);
}
} // namespace
extern "C" JNIEXPORT jlong JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeLoadModel(
JNIEnv * env, jclass, jstring jpath, jint n_ctx, jint n_threads) {
ensure_backend_init();
const std::string path = jstring_to_utf8(env, jpath);
const int64_t t_start = now_ms();
// Cap threads to the big-core count and pin them there (see
// detect_fast_cpus): little cores would otherwise both drag the barrier
// and invite the scheduler to keep us off the fast cores entirely.
const std::vector<int> fast_cpus = detect_fast_cpus();
int threads = (int) n_threads;
if (!fast_cpus.empty()) {
threads = std::min(threads, (int) fast_cpus.size());
}
llama_model_params mparams = llama_model_default_params();
mparams.n_gpu_layers = 0;
llama_model * model = llama_model_load_from_file(path.c_str(), mparams);
if (model == nullptr) {
LOGW("model load failed: %s", path.c_str());
return 0;
}
const int64_t t_model = now_ms();
llama_context_params cparams = llama_context_default_params();
cparams.n_ctx = (uint32_t) n_ctx;
cparams.n_batch = 512;
cparams.n_threads = threads;
cparams.n_threads_batch = threads;
llama_context * ctx = llama_init_from_model(model, cparams);
if (ctx == nullptr) {
LOGW("context init failed");
llama_model_free(model);
return 0;
}
ggml_threadpool * threadpool = nullptr;
if (!fast_cpus.empty()) {
ggml_threadpool_params tpp = ggml_threadpool_params_default(threads);
for (size_t i = 0; i < fast_cpus.size() && fast_cpus[i] < GGML_MAX_N_THREADS; i++) {
tpp.cpumask[fast_cpus[i]] = true;
}
tpp.strict_cpu = true;
threadpool = ggml_threadpool_new(&tpp);
if (threadpool != nullptr) {
llama_attach_threadpool(ctx, threadpool, threadpool);
std::string cpus;
for (int cpu : fast_cpus) cpus += std::to_string(cpu) + " ";
LOGI("pinned %d threads to fast cpus: %s", threads, cpus.c_str());
} else {
LOGW("threadpool creation failed; using default scheduling");
}
}
auto * handle = new FcLlama();
handle->model = model;
handle->ctx = ctx;
handle->threadpool = threadpool;
handle->vocab = llama_model_get_vocab(model);
handle->n_ctx = (int) llama_n_ctx(ctx);
LOGI("model loaded: %s (n_ctx=%d, threads=%d, model_load_ms=%lld, ctx_init_ms=%lld)",
path.c_str(), handle->n_ctx, threads,
(long long) (t_model - t_start), (long long) (now_ms() - t_model));
return reinterpret_cast<jlong>(handle);
}
extern "C" JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeFree(JNIEnv *, jclass, jlong jhandle) {
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
if (handle == nullptr) return;
llama_free(handle->ctx);
llama_model_free(handle->model);
if (handle->threadpool != nullptr) {
ggml_threadpool_free(handle->threadpool);
}
delete handle;
LOGI("model freed");
}
extern "C" JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeCancel(JNIEnv *, jclass, jlong jhandle) {
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
if (handle != nullptr) handle->cancel.store(true);
}
extern "C" JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeGenerate(
JNIEnv * env, jclass, jlong jhandle, jstring jprompt, jstring jgrammar,
jint max_tokens, jfloat temperature, jfloat top_p, jint top_k,
jobject jlistener) {
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
Listener listener = Listener::resolve(env, jlistener);
if (handle == nullptr) {
listener.error("Model is not loaded");
return;
}
handle->cancel.store(false);
const std::string prompt = jstring_to_utf8(env, jprompt);
const std::string grammar = jstring_to_utf8(env, jgrammar);
std::vector<llama_token> tokens = tokenize(handle->vocab, prompt, true);
if (tokens.empty()) {
listener.error("Prompt tokenization failed");
return;
}
if ((int) tokens.size() + 16 > handle->n_ctx) {
listener.error("Conversation too long for the on-phone model");
return;
}
const int max_gen = std::min((int) max_tokens, handle->n_ctx - (int) tokens.size());
// Reuse the KV prefix shared with the previous call; drop the rest. Keep
// at least the final prompt token out of the reused prefix so this decode
// produces logits for it.
size_t common = 0;
while (common < tokens.size() && common < handle->cache_tokens.size() &&
tokens[common] == handle->cache_tokens[common]) {
common++;
}
if (common == tokens.size()) common = tokens.size() - 1;
llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) common, -1);
handle->cache_tokens.assign(tokens.begin(), tokens.begin() + common);
// Sampler chain per generation: Qwen3 instruct posture (temp 0.7, top-p
// 0.8, top-k 20) plus an optional lazy grammar that activates at the
// <tool_call> special token and forces a schema-valid call.
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
if (!grammar.empty()) {
llama_sampler * grammar_sampler = nullptr;
std::vector<llama_token> trigger = tokenize(handle->vocab, "<tool_call>", false);
if (trigger.size() == 1) {
grammar_sampler = llama_sampler_init_grammar_lazy_patterns(
handle->vocab, grammar.c_str(), "root", nullptr, 0, trigger.data(), 1);
} else {
static const char * patterns[] = {"(<tool_call>[\\s\\S]*)"};
grammar_sampler = llama_sampler_init_grammar_lazy_patterns(
handle->vocab, grammar.c_str(), "root", patterns, 1, nullptr, 0);
}
if (grammar_sampler != nullptr) {
llama_sampler_chain_add(chain, grammar_sampler);
} else {
LOGW("tool grammar failed to parse; generating unconstrained");
}
}
llama_sampler_chain_add(chain, llama_sampler_init_top_k(top_k));
llama_sampler_chain_add(chain, llama_sampler_init_top_p(top_p, 1));
llama_sampler_chain_add(chain, llama_sampler_init_temp(temperature));
llama_sampler_chain_add(chain, llama_sampler_init_dist(LLAMA_DEFAULT_SEED));
const auto fail = [&](const std::string & message) {
llama_sampler_free(chain);
// The KV state past the reused prefix is now unknown; drop it so the
// next call starts from a consistent cache.
llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) handle->cache_tokens.size(), -1);
listener.error(message);
};
// Prefill in n_batch chunks.
const int64_t t_prefill_start = now_ms();
const int n_batch = (int) llama_n_batch(handle->ctx);
for (size_t i = common; i < tokens.size(); i += n_batch) {
if (handle->cancel.load()) {
llama_sampler_free(chain);
listener.done("cancelled");
return;
}
const int chunk = (int) std::min((size_t) n_batch, tokens.size() - i);
if (llama_decode(handle->ctx, llama_batch_get_one(tokens.data() + i, chunk)) != 0) {
fail("Prompt evaluation failed");
return;
}
handle->cache_tokens.insert(handle->cache_tokens.end(), tokens.begin() + i, tokens.begin() + i + chunk);
}
// Decode loop: sample, stream, feed back.
const int64_t t_decode_start = now_ms();
const size_t n_prefilled = tokens.size() - common;
int n_decoded = 0;
std::string pending_utf8;
const char * stop_reason = "length";
for (int i = 0; i < max_gen; i++) {
if (handle->cancel.load()) {
stop_reason = "cancelled";
break;
}
llama_token token = llama_sampler_sample(chain, handle->ctx, -1);
if (llama_vocab_is_eog(handle->vocab, token)) {
stop_reason = "stop";
break;
}
pending_utf8 += token_piece(handle->vocab, token);
size_t emit = utf8_complete_prefix(pending_utf8);
if (emit > 0) {
listener.token(pending_utf8.substr(0, emit));
pending_utf8.erase(0, emit);
}
if (llama_decode(handle->ctx, llama_batch_get_one(&token, 1)) != 0) {
fail("Token evaluation failed");
return;
}
handle->cache_tokens.push_back(token);
n_decoded++;
}
const int64_t t_end = now_ms();
const int64_t prefill_ms = t_decode_start - t_prefill_start;
const int64_t decode_ms = t_end - t_decode_start;
LOGI("timing: prompt=%zu reused=%zu prefill=%zu tok in %lld ms (%.1f tok/s), decode=%d tok in %lld ms (%.1f tok/s)",
tokens.size(), common, n_prefilled, (long long) prefill_ms,
prefill_ms > 0 ? 1000.0 * (double) n_prefilled / (double) prefill_ms : 0.0,
n_decoded, (long long) decode_ms,
decode_ms > 0 ? 1000.0 * (double) n_decoded / (double) decode_ms : 0.0);
if (!pending_utf8.empty()) {
size_t emit = utf8_complete_prefix(pending_utf8);
if (emit > 0) listener.token(pending_utf8.substr(0, emit));
}
llama_sampler_free(chain);
listener.done(stop_reason);
}
+11 -4
View File
@@ -1,4 +1,5 @@
import { streamAnthropicMessage } from "../native/anthropic";
import { streamLocalQwen } from "../native/llama";
import { streamOpenAiResponse } from "../native/openai";
import type {
LlmContentBlock,
@@ -20,6 +21,8 @@ import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/** Safety caps so a tool-thrashing model can't loop or run forever. */
const MAX_ITERATIONS = 8;
/** The on-phone model is slower per step and unreliable on long tool chains. */
const MAX_ITERATIONS_LOCAL = 4;
const MAX_TURN_MS = 2 * 60 * 1000;
export type DirectTurnOptions = {
@@ -54,9 +57,10 @@ export class DirectAssistantBackend {
options.callbacks.onError(message);
};
const maxIterations = options.provider === "local" ? MAX_ITERATIONS_LOCAL : MAX_ITERATIONS;
const runIteration = (iteration: number): void => {
if (cancelled) return;
if (iteration >= MAX_ITERATIONS) {
if (iteration >= maxIterations) {
finishError("Assistant stopped: too many tool steps");
return;
}
@@ -115,9 +119,12 @@ export class DirectAssistantBackend {
finishError(message);
},
};
streamHandle = options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: streamAnthropicMessage(streamOptions);
streamHandle =
options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: options.provider === "local"
? streamLocalQwen(streamOptions)
: streamAnthropicMessage(streamOptions);
};
runIteration(0);
+145
View File
@@ -0,0 +1,145 @@
import type { LlmToolDefinition } from "./llm-protocol";
/**
* Generate a GBNF grammar (llama.cpp grammar dialect) that constrains a
* Qwen-style tool call to be well-formed: one <tool_call> block whose JSON
* names a real tool and whose arguments match that tool's schema. Used as a
* *lazy* grammar — sampling is unconstrained until the model emits
* <tool_call>, so plain text answers are unaffected, and once the block
* closes the grammar is complete (the model can only end the message, which
* serializes multi-call turns into one call per loop iteration — deliberate:
* small models are far more reliable one call at a time).
*
* Our tool schemas are flat objects of string/number/boolean/enum properties.
* Anything fancier degrades to a generic JSON value rule rather than failing.
*/
type JsonSchema = {
type?: string;
enum?: unknown[];
properties?: Record<string, JsonSchema>;
required?: string[];
description?: string;
};
/** Longest run of optional properties we'll enumerate subsets for. */
const MAX_SUBSET_PROPS = 5;
export function buildToolCallGrammar(tools: LlmToolDefinition[]): string | null {
if (!tools.length) return null;
const used = new Set<string>();
const toolRules: string[] = [];
for (let i = 0; i < tools.length; i++) {
const args = argsRule(tools[i]!.input_schema as JsonSchema, used);
toolRules.push(
`tc${i} ::= "<tool_call>" nl "{" sp ${jsonKey("name")} sp ":" sp "\\"${gbnfEscape(
tools[i]!.name,
)}\\"" sp "," sp ${jsonKey("arguments")} sp ":" sp ${args} sp "}" nl "</tool_call>"`,
);
}
used.add("sp");
used.add("nl");
const lines = [`root ::= ${toolRules.map((_, i) => `tc${i}`).join(" | ")}`, ...toolRules];
for (const [name, rule] of SHARED_RULES) {
if (used.has(name)) lines.push(rule);
}
return lines.join("\n");
}
/** Build the arguments-object expression for one tool schema. */
function argsRule(schema: JsonSchema | undefined, used: Set<string>): string {
const properties = schema?.properties ?? {};
const names = Object.keys(properties);
if (!names.length) return `"{" sp "}"`;
const required = new Set(schema?.required ?? []);
const optionalCount = names.filter((n) => !required.has(n)).length;
if (names.length > MAX_SUBSET_PROPS && optionalCount > 0) {
used.add("jobj");
return "jobj";
}
// Enumerate every subset of properties that includes all required ones,
// keeping declaration order. Small N keeps this tiny (<= 2^5 alternatives).
const members = names.map((name) => {
const value = valueExpr(properties[name]!, used);
return `${jsonKey(name)} sp ":" sp ${value}`;
});
const alternatives: string[] = [];
for (let mask = 0; mask < 1 << names.length; mask++) {
const chosen: string[] = [];
let valid = true;
for (let j = 0; j < names.length; j++) {
const included = (mask & (1 << j)) !== 0;
if (included) chosen.push(members[j]!);
else if (required.has(names[j]!)) valid = false;
}
if (!valid) continue;
if (!chosen.length) {
alternatives.push(`"{" sp "}"`);
} else {
alternatives.push(`"{" sp ${chosen.join(` sp "," sp `)} sp "}"`);
}
}
return alternatives.length === 1 ? alternatives[0]! : `(${alternatives.join(" | ")})`;
}
/** Grammar expression for one property value. */
function valueExpr(schema: JsonSchema, used: Set<string>): string {
if (Array.isArray(schema.enum) && schema.enum.length) {
if (schema.enum.every((v) => typeof v === "string")) {
return `(${schema.enum.map((v) => `"\\"${gbnfEscape(String(v))}\\""`).join(" | ")})`;
}
if (schema.enum.every((v) => typeof v === "number" || typeof v === "boolean")) {
return `(${schema.enum.map((v) => `"${String(v)}"`).join(" | ")})`;
}
}
switch (schema.type) {
case "string":
used.add("jstring");
used.add("jchar");
used.add("jhex");
return "jstring";
case "number":
used.add("jnumber");
return "jnumber";
case "integer":
used.add("jint");
return "jint";
case "boolean":
used.add("jbool");
return "jbool";
default:
for (const name of ["jvalue", "jstring", "jchar", "jhex", "jnumber", "jbool", "jarray", "jobj", "jmember"]) {
used.add(name);
}
return "jvalue";
}
}
/** A JSON object key as a GBNF literal: "name" -> "\"name\"". */
function jsonKey(name: string): string {
return `"\\"${gbnfEscape(name)}\\""`;
}
/** Escape for the inside of a GBNF double-quoted literal. */
function gbnfEscape(text: string): string {
return text.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
}
// Terminal rules, emitted only when referenced. jchar mirrors llama.cpp's
// json.gbnf string character rule.
const SHARED_RULES: Array<[string, string]> = [
["sp", `sp ::= " "?`],
["nl", `nl ::= "\\n"`],
["jstring", `jstring ::= "\\"" jchar* "\\""`],
["jchar", `jchar ::= [^"\\\\\\x7F\\x00-\\x1F] | "\\\\" (["\\\\bfnrt] | "u" jhex jhex jhex jhex)`],
["jhex", `jhex ::= [0-9a-fA-F]`],
["jnumber", `jnumber ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? (("e" | "E") ("-" | "+")? [0-9]+)?`],
["jint", `jint ::= "-"? ("0" | [1-9] [0-9]*)`],
["jbool", `jbool ::= "true" | "false"`],
["jvalue", `jvalue ::= jstring | jnumber | jbool | "null" | jarray | jobj`],
["jarray", `jarray ::= "[" sp (jvalue (sp "," sp jvalue)*)? sp "]"`],
["jobj", `jobj ::= "{" sp (jmember (sp "," sp jmember)*)? sp "}"`],
["jmember", `jmember ::= jstring sp ":" sp jvalue`],
];
+81
View File
@@ -0,0 +1,81 @@
import type { LlmContentBlock, LlmMessage, LlmToolDefinition } from "./llm-protocol";
/**
* Render our provider-neutral message history into a Qwen3 (ChatML) prompt
* string, matching the model's chat template closely enough that its
* tool-calling priors apply: Hermes-style <tools> declarations in the system
* block, <tool_call> JSON blocks in assistant turns, and tool results wrapped
* in <tool_response> inside a user turn. The caller appends nothing — the
* returned string ends with the assistant generation header.
*/
const IM_START = "<|im_start|>";
const IM_END = "<|im_end|>";
export function buildQwenPrompt(
system: string | undefined,
messages: LlmMessage[],
tools: LlmToolDefinition[] | undefined,
): string {
const parts: string[] = [];
parts.push(`${IM_START}system\n${systemBlock(system, tools)}${IM_END}\n`);
for (const message of messages) {
if (message.role === "assistant") {
parts.push(`${IM_START}assistant\n${assistantBlock(message.content)}${IM_END}\n`);
} else {
parts.push(`${IM_START}user\n${userBlock(message.content)}${IM_END}\n`);
}
}
parts.push(`${IM_START}assistant\n`);
return parts.join("");
}
function systemBlock(system: string | undefined, tools: LlmToolDefinition[] | undefined): string {
const base = system?.trim() ?? "You are a helpful assistant.";
if (!tools?.length) return base;
const declarations = tools
.map((tool) =>
JSON.stringify({
type: "function",
function: { name: tool.name, description: tool.description, parameters: tool.input_schema },
}),
)
.join("\n");
return (
`${base}\n\n# Tools\n\n` +
`You may call one or more functions to assist with the user query.\n\n` +
`You are provided with function signatures within <tools></tools> XML tags:\n` +
`<tools>\n${declarations}\n</tools>\n\n` +
`For each function call, return a json object with function name and arguments ` +
`within <tool_call></tool_call> XML tags:\n` +
`<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>`
);
}
function assistantBlock(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content;
const parts: string[] = [];
for (const block of content) {
if (block.type === "text") {
if (block.text.trim()) parts.push(block.text.trim());
} else if (block.type === "tool_use") {
parts.push(`<tool_call>\n${JSON.stringify({ name: block.name, arguments: block.input ?? {} })}\n</tool_call>`);
}
// provider_item blocks are OpenAI-specific continuation state; skip.
}
return parts.join("\n");
}
function userBlock(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content;
const parts: string[] = [];
for (const block of content) {
if (block.type === "text") {
if (block.text.trim()) parts.push(block.text.trim());
} else if (block.type === "tool_result") {
const body = block.is_error ? `Error: ${block.content}` : block.content || "(no output)";
parts.push(`<tool_response>\n${body}\n</tool_response>`);
}
}
return parts.join("\n");
}
+16 -4
View File
@@ -1,4 +1,8 @@
import { buildAssistantSystemPrompt } from "../prompts";
import {
ASSISTANT_SYSTEM_PROMPT_BASE,
buildAssistantSystemPrompt,
describeAssistantContext,
} from "../prompts";
import { assistantBridge } from "./bridge-client";
import { DirectAssistantBackend } from "./direct-backend";
import type { LlmMessage, LlmToolDefinition } from "./llm-protocol";
@@ -108,15 +112,23 @@ export class AssistantSession {
return;
}
this.messages.push({ role: "user", content: text });
this.trimHistory();
const llm = this.config.llm;
// Local provider: keep the system prompt (and thus the tool declarations
// that follow it in the rendered prompt) byte-stable so the on-phone
// model's KV prefix cache survives across turns; the volatile context
// (time, foreground app, battery) rides along with the utterance instead.
const isLocal = llm.provider === "local";
this.messages.push({
role: "user",
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
});
this.trimHistory();
this.turnHandle = this.directBackend.runTurn({
provider: llm.provider,
apiKey: llm.apiKey,
model: llm.model,
effort: llm.effort,
system: buildAssistantSystemPrompt(ctx),
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
messages: this.messages,
buildTools: () => this.buildToolDefinitions(),
registry: this.registry,
+321
View File
@@ -0,0 +1,321 @@
import { Utils } from "@nativescript/core";
import { buildToolCallGrammar } from "../assistant/gbnf";
import type {
LlmContentBlock,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { buildQwenPrompt } from "../assistant/qwen-prompt";
declare const com: any;
declare const java: any;
/**
* On-phone LLM backend: model download management plus a streaming client
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
* the direct assistant loop can treat "local" as just another provider.
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
*/
export const LOCAL_MODEL = {
id: "qwen3-4b-instruct-2507",
label: "Qwen3 4B",
fileName: "Qwen3-4B-Instruct-2507-Q4_K_M.gguf",
url: "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_M.gguf?download=true",
sha256: "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597",
sizeBytes: 2497281120,
};
// Bounded so KV cache stays ~0.5GB (Qwen3-4B KV is ~144KB/token at f16);
// the assistant trims history well before this in practice.
const N_CTX = 4096;
const N_THREADS = 4;
const DEFAULT_MAX_TOKENS = 512;
// Qwen3-4B-Instruct-2507 model-card sampling posture.
const TEMPERATURE = 0.7;
const TOP_P = 0.8;
const TOP_K = 20;
/** Free the ~3GB of model + KV this long after the last generation. */
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
export type LocalModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
let runner: any = null;
let downloader: any = null;
let downloadedBytes = 0;
let unloadTimer: ReturnType<typeof setTimeout> | null = null;
let toolUseCounter = 0;
const stateListeners = new Set<(state: LocalModelState) => void>();
function modelPath(): string {
const context = Utils.android.getApplicationContext();
const dir = context.getExternalFilesDir("llm") ?? context.getFilesDir();
return `${dir.getAbsolutePath()}/${LOCAL_MODEL.fileName}`;
}
export function isLocalModelReady(): boolean {
if (!global.isAndroid) return false;
try {
const file = new java.io.File(modelPath());
return file.exists() && file.length() > 0;
} catch {
return false;
}
}
export function localModelState(): LocalModelState {
if (downloader) {
return { status: "downloading", bytesDownloaded: downloadedBytes, totalBytes: LOCAL_MODEL.sizeBytes };
}
return {
status: isLocalModelReady() ? "ready" : "absent",
bytesDownloaded: 0,
totalBytes: LOCAL_MODEL.sizeBytes,
};
}
export function onLocalModelStateChanged(listener: (state: LocalModelState) => void): () => void {
stateListeners.add(listener);
return () => stateListeners.delete(listener);
}
function notifyStateChanged(): void {
const state = localModelState();
for (const listener of stateListeners) listener(state);
}
export function startLocalModelDownload(): void {
if (!global.isAndroid || downloader || isLocalModelReady()) return;
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
onProgress: (bytes: number, _total: number) => {
downloadedBytes = Number(bytes);
notifyStateChanged();
},
onDone: () => {
downloader = null;
notifyStateChanged();
},
onError: (message: string) => {
console.error(`Local model download failed: ${message}`);
downloader = null;
notifyStateChanged();
},
});
downloader = new com.faceclaw.app.FaceclawModelDownloader(
LOCAL_MODEL.url,
modelPath(),
LOCAL_MODEL.sha256,
LOCAL_MODEL.sizeBytes,
listener,
);
downloadedBytes = 0;
downloader.start();
notifyStateChanged();
}
/** Stops the download; already-fetched bytes are kept and resumed next time. */
export function cancelLocalModelDownload(): void {
if (!downloader) return;
downloader.cancel();
downloader = null;
notifyStateChanged();
}
export function deleteLocalModel(): void {
if (!global.isAndroid) return;
cancelLocalModelDownload();
unloadLocalModel();
try {
new java.io.File(modelPath()).delete();
new java.io.File(`${modelPath()}.part`).delete();
} catch (error) {
console.error(`Local model delete failed: ${String(error)}`);
}
notifyStateChanged();
}
export function unloadLocalModel(): void {
if (unloadTimer) {
clearTimeout(unloadTimer);
unloadTimer = null;
}
runner?.unload();
}
function scheduleIdleUnload(): void {
if (unloadTimer) clearTimeout(unloadTimer);
unloadTimer = setTimeout(() => {
unloadTimer = null;
runner?.unload();
}, IDLE_UNLOAD_MS);
}
function ensureRunner(): any {
if (!runner) {
runner = new com.faceclaw.app.FaceclawLlamaRunner();
}
return runner;
}
const TOOL_CALL_OPEN = "<tool_call>";
const TOOL_CALL_CLOSE = "</tool_call>";
/**
* Stream a completion from the on-phone model. Tool calls arrive inline as
* <tool_call> JSON blocks; they are hidden from the text stream and surfaced
* as tool_use content blocks in the final result, mirroring the cloud
* providers' contract (stopReason "tool_use" when the model called a tool).
*/
export function streamLocalQwen(options: LlmStreamOptions): LlmStreamHandle {
let settled = false;
let raw = "";
let emitted = "";
const fail = (message: string) => {
if (settled) return;
settled = true;
scheduleIdleUnload();
options.onError(message);
};
if (!global.isAndroid) {
setTimeout(() => fail("The on-phone model is only available on Android"), 0);
return { cancel: () => {} };
}
if (!isLocalModelReady()) {
setTimeout(() => fail("The on-phone model isn't downloaded yet (see Settings)"), 0);
return { cancel: () => {} };
}
if (unloadTimer) {
clearTimeout(unloadTimer);
unloadTimer = null;
}
const prompt = buildQwenPrompt(options.system, options.messages, options.tools);
const grammar = options.tools?.length ? buildToolCallGrammar(options.tools) : null;
const listener = new com.faceclaw.app.FaceclawLlamaListener({
onToken: (piece: string) => {
if (settled) return;
raw += String(piece);
const visible = visibleText(raw);
if (visible.length > emitted.length) {
const delta = visible.slice(emitted.length);
emitted = visible;
options.onTextDelta?.(delta, emitted);
}
},
onDone: (nativeStopReason: string) => {
if (settled) return;
settled = true;
scheduleIdleUnload();
const { text, content, toolCount } = assembleContent(raw);
const stopReason =
toolCount > 0 ? "tool_use" : String(nativeStopReason) === "length" ? "max_tokens" : "end_turn";
options.onDone({ text, content, stopReason });
},
onError: (message: string) => fail(String(message)),
});
try {
ensureRunner().generate(
modelPath(),
N_CTX,
N_THREADS,
prompt,
grammar ?? "",
options.maxTokens ?? DEFAULT_MAX_TOKENS,
TEMPERATURE,
TOP_P,
TOP_K,
listener,
);
} catch (error) {
setTimeout(() => fail(`On-phone model failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
}
return {
cancel: () => {
settled = true;
scheduleIdleUnload();
try {
runner?.cancel();
} catch {
// ignore
}
},
};
}
/**
* The streamable portion of the raw output: text outside completed
* <tool_call> blocks, holding back any trailing partial marker so a tool
* call never leaks into the visible stream. Monotonic under appends.
*/
function visibleText(raw: string): string {
let out = "";
let i = 0;
for (;;) {
const start = raw.indexOf(TOOL_CALL_OPEN, i);
if (start === -1) {
const tail = raw.slice(i);
out += tail.slice(0, tail.length - partialSuffixLength(tail, TOOL_CALL_OPEN));
return out;
}
out += raw.slice(i, start);
const end = raw.indexOf(TOOL_CALL_CLOSE, start);
if (end === -1) return out; // inside an unfinished tool call
i = end + TOOL_CALL_CLOSE.length;
}
}
/** Length of the longest proper prefix of marker that text ends with. */
function partialSuffixLength(text: string, marker: string): number {
const max = Math.min(text.length, marker.length - 1);
for (let k = max; k > 0; k--) {
if (text.endsWith(marker.slice(0, k))) return k;
}
return 0;
}
/** Split the finished raw output into text + tool_use content blocks. */
function assembleContent(raw: string): { text: string; content: LlmContentBlock[]; toolCount: number } {
const content: LlmContentBlock[] = [];
let text = "";
let toolCount = 0;
let i = 0;
for (;;) {
const start = raw.indexOf(TOOL_CALL_OPEN, i);
if (start === -1) {
text += raw.slice(i);
break;
}
text += raw.slice(i, start);
const end = raw.indexOf(TOOL_CALL_CLOSE, start);
if (end === -1) break; // truncated call (length stop); drop it
const body = raw.slice(start + TOOL_CALL_OPEN.length, end).trim();
try {
const parsed = JSON.parse(body);
if (parsed && typeof parsed.name === "string") {
toolCount++;
content.push({
type: "tool_use",
id: `local_${Date.now()}_${++toolUseCounter}`,
name: parsed.name,
input: parsed.arguments ?? {},
});
}
} catch {
console.error(`Local model emitted unparseable tool call: ${body.slice(0, 200)}`);
}
i = end + TOOL_CALL_CLOSE.length;
}
text = text.trim();
if (text) content.unshift({ type: "text", text });
return { text, content, toolCount };
}
+9 -2
View File
@@ -15,10 +15,17 @@ export const ASSISTANT_SYSTEM_PROMPT_BASE = [
/** Full assistant system prompt: the base plus the current device context. */
export function buildAssistantSystemPrompt(ctx: AssistantContext): string {
return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`;
return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeAssistantContext(ctx)}`;
}
function describeContext(ctx: AssistantContext): string {
/**
* The volatile device context on its own. The local provider appends this to
* the user message instead of the system prompt: the system prompt + tool
* declarations are the bulk of the prompt, and keeping them byte-stable lets
* the on-phone model reuse its KV-cache prefix across turns instead of
* re-prefilling ~2-3k tokens because the clock changed.
*/
export function describeAssistantContext(ctx: AssistantContext): string {
const parts = [`Current time: ${ctx.localTime}.`];
parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`);
if (ctx.foregroundApp) {
+6 -3
View File
@@ -19,6 +19,7 @@ import {
assistantModelProvider,
type AssistantModel,
} from "~/assistant/models";
import { isLocalModelReady } from "../native/llama";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
import { DashboardInputEvent, Layer, type LayerContext } from "./layers";
import { GrayImage } from "~/graphics/image";
@@ -357,7 +358,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
export type AssistantBackendKind = "direct" | "external";
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
direct: "On-phone (API key)",
direct: "On-phone",
external: "My own agent (bridge)",
};
@@ -369,7 +370,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
values: ["direct", "external"],
formatValue: (value) => assistantBackendLabels[value] ?? value,
description:
"Who answers assistant queries: an LLM called directly from the phone (needs an API key), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
"Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantBridgeHostSetting = new ConfigSettingString({
@@ -468,9 +469,11 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
const provider = assistantModelProvider(value);
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
if (provider === "local") return !isLocalModelReady();
return false;
},
description: "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set.",
description:
"Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
+82 -1
View File
@@ -3,8 +3,17 @@ import { getDefaultSmallFont } from "../../graphics/bdffont";
import type { GrayImage } from "../../graphics/image";
import { getDashboardLogo } from "../../graphics/logo";
import { wrapText } from "../../graphics/textwrap";
import {
cancelLocalModelDownload,
deleteLocalModel,
LOCAL_MODEL,
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
} from "../../native/llama";
import { TextViewerLayer } from "../../apps/files/text-viewer";
import type { MenuItem } from "../menu";
import type { LayerContext } from "../layers";
import { drawRightValueMenuItem, openModalMenu, type MenuItem } from "../menu";
import { shell } from "../shell/shell";
import {
anthropicApiKeySetting,
@@ -88,6 +97,7 @@ function settingsSections(): SettingsSection[] {
// On-phone LLM loop vs the user's own agent via the bridge plugin.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
localModelMenuItem(),
// When on, a wakeword utterance goes straight to the assistant with no
// Send/Type menu step.
toggleSettingMenuItem(assistantSkipConfirmationSetting),
@@ -161,6 +171,77 @@ function settingsSections(): SettingsSection[] {
];
}
const LOCAL_MODEL_GB = `${(LOCAL_MODEL.sizeBytes / 1e9).toFixed(1)}GB`;
// While a download is running, re-render on progress updates so the row's
// percentage stays live; the watch tears itself down when the download ends.
let localModelRenderUnsub: (() => void) | null = null;
function watchLocalModelDownload(ctx: LayerContext): void {
localModelRenderUnsub?.();
localModelRenderUnsub = onLocalModelStateChanged((state) => {
ctx.actions.requestRender();
if (state.status !== "downloading") {
localModelRenderUnsub?.();
localModelRenderUnsub = null;
}
});
}
function localModelStatusText(): string {
const state = localModelState();
if (state.status === "ready") return "downloaded";
if (state.status === "downloading") {
const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0;
return `${pct}% of ${LOCAL_MODEL_GB}`;
}
return "not downloaded";
}
/** Download/cancel/delete management for the on-phone assistant model. */
function localModelMenuItem(): MenuItem {
return {
label: "On-phone model",
description:
`${LOCAL_MODEL.label} (${LOCAL_MODEL_GB} download over Wi-Fi recommended). ` +
"Answers assistant queries on the phone itself, with no API key or cloud service. " +
"Slower and simpler than the cloud models, but free and private. " +
"Used automatically when no API key is set. An interrupted download resumes where it left off.",
onSelect: (ctx) => {
const state = localModelState();
const action: MenuItem =
state.status === "downloading"
? {
label: "Cancel download",
onSelect: (innerCtx) => {
cancelLocalModelDownload();
innerCtx.stack.pop();
},
}
: state.status === "ready"
? {
label: "Delete model",
onSelect: (innerCtx) => {
deleteLocalModel();
innerCtx.stack.pop();
},
}
: {
label: `Download (${LOCAL_MODEL_GB})`,
onSelect: (innerCtx) => {
startLocalModelDownload();
watchLocalModelDownload(innerCtx);
innerCtx.stack.pop();
},
};
openModalMenu(ctx, "On-phone model", [action], 0);
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", localModelStatusText());
},
};
}
/** A row that opens one of the project docs (copied into the bundle under
* about/ by webpack.config.js) in the paged text viewer. */
function bundledDocMenuItem(fileName: string, label: string): MenuItem {
+1 -1
View File
@@ -801,7 +801,7 @@ class Shell {
this.showAlert(
assistantBackendSetting.get() === "external"
? "Configure the agent bridge host and token in Settings."
: "Set an API key for the selected assistant model in Settings.",
: "Set an API key or download the on-phone model in Settings.",
);
return;
}