diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index d7622600..6044f239 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -149,6 +149,102 @@ tasks.register("prepareFaceclawNativeLibs") { } } +// On-phone assistant model: build llama.cpp (pinned release) + our JNI shim +// into libfaceclaw_llama.so. CPU-only, arm64, armv8.2 dotprod+fp16 baseline +// (Tensor G2 floor; no i8mm). The first build compiles llama.cpp (~2-4 min); +// afterwards the cached .so short-circuits everything. +def faceclawLlamaCppTag = "b10333" + +def faceclawFindSdkCmakeDir = { + def sdkRoot = System.getenv("ANDROID_HOME") ?: System.getenv("ANDROID_SDK_ROOT") + if (sdkRoot == null) return null + def cmakeRoot = file("$sdkRoot/cmake") + if (!cmakeRoot.exists()) return null + def installed = cmakeRoot.listFiles()?.findAll { it.isDirectory() }?.sort { a, b -> b.name <=> a.name } + return (installed && !installed.isEmpty()) ? installed[0] : null +} + +tasks.register("prepareFaceclawLlama") { + def llamaSo = file("$faceclawJniLibsDir/libfaceclaw_llama.so") + inputs.files( + file("$projectDir/src/main/native/llama/faceclaw_llama.cpp"), + file("$projectDir/src/main/native/llama/CMakeLists.txt") + ) + outputs.files(llamaSo) + + doLast { + def llamaBuildDir = file("$faceclawNativeBuildDir/llama-build") + def builtSo = file("$llamaBuildDir/libfaceclaw_llama.so") + + def downloadsDir = file("$faceclawNativeBuildDir/downloads") + def llamaArchive = file("$downloadsDir/llama.cpp-${faceclawLlamaCppTag}.tar.gz") + faceclawDownload( + "https://github.com/ggml-org/llama.cpp/archive/refs/tags/${faceclawLlamaCppTag}.tar.gz", + llamaArchive + ) + def llamaSourceDir = file("$faceclawNativeBuildDir/llama.cpp-${faceclawLlamaCppTag}") + if (!file("$llamaSourceDir/CMakeLists.txt").exists()) { + copy { + from tarTree(resources.gzip(llamaArchive)) + into faceclawNativeBuildDir + } + } + + def ndkRoot = faceclawFindNdkRoot() + if (ndkRoot == null || !ndkRoot.exists()) { + throw new GradleException("Android NDK is required to build libfaceclaw_llama.so") + } + def cmakeDir = faceclawFindSdkCmakeDir() + if (cmakeDir == null) { + throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so") + } + def cmake = file("$cmakeDir/bin/cmake") + def ninja = file("$cmakeDir/bin/ninja") + def march = "-march=armv8.2-a+dotprod+fp16" + + // Configure once; afterwards the ninja no-op build keeps this cheap while + // still picking up JNI source edits. + if (!file("$llamaBuildDir/build.ninja").exists()) { + exec { + commandLine([ + cmake.absolutePath, + "-S", file("$projectDir/src/main/native/llama").absolutePath, + "-B", llamaBuildDir.absolutePath, + "-G", "Ninja", + "-DCMAKE_MAKE_PROGRAM=${ninja.absolutePath}", + "-DCMAKE_TOOLCHAIN_FILE=${file("$ndkRoot/build/cmake/android.toolchain.cmake").absolutePath}", + "-DANDROID_ABI=arm64-v8a", + "-DANDROID_PLATFORM=android-24", + "-DCMAKE_BUILD_TYPE=Release", + "-DLLAMA_CPP_SOURCE_DIR=${llamaSourceDir.absolutePath}", + "-DCMAKE_C_FLAGS=$march", + "-DCMAKE_CXX_FLAGS=$march", + ]) + } + } + exec { + commandLine([cmake.absolutePath, "--build", llamaBuildDir.absolutePath, "-j", "8"]) + } + + faceclawJniLibsDir.mkdirs() + copy { + from builtSo + into faceclawJniLibsDir + } + // The static-linked lib carries llama.cpp's full debug info (~50MB); + // strip it so the APK only pays for the code (~8MB). + def ndkRootForStrip = faceclawFindNdkRoot() + if (ndkRootForStrip != null) { + def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip") + if (llvmStrip.exists()) { + exec { + commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath]) + } + } + } + } +} + android { compileSdkVersion 35 buildToolsVersion "35" @@ -179,4 +275,5 @@ android { tasks.matching { it.name == "preBuild" }.configureEach { dependsOn tasks.named("prepareFaceclawNativeLibs") + dependsOn tasks.named("prepareFaceclawLlama") } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaListener.java new file mode 100644 index 00000000..f5419b84 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaListener.java @@ -0,0 +1,16 @@ +package com.faceclaw.app; + +/** + * Callbacks for one on-phone LLM generation. Like the other bridge listeners, + * implementations are typically created on the NativeScript JS thread and + * FaceclawLlamaRunner posts these callbacks back to that thread's Looper. + */ +public interface FaceclawLlamaListener { + /** A piece of generated text (may be a partial word). */ + void onToken(String piece); + + /** Generation finished: "stop" (EOG), "length", or "cancelled". */ + void onDone(String stopReason); + + void onError(String message); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java new file mode 100644 index 00000000..17e98834 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLlamaRunner.java @@ -0,0 +1,122 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * On-phone LLM inference for the voice assistant (llama.cpp over JNI). + * + * All model work (load, generate, free) runs on a single background executor + * thread so calls are naturally serialized; listener callbacks are posted to + * the Looper of the thread that constructed this object (the JS thread), the + * same contract as FaceclawSseRequest. cancel() may be called from any thread + * and interrupts the in-flight generation at the next token boundary. + * + * The model stays loaded between generations (turn iterations reuse the KV + * cache prefix); the TypeScript side calls unload() on an idle timer to give + * the ~3GB back. + */ +public class FaceclawLlamaRunner { + private static final String TAG = "FaceclawLlamaRunner"; + + static { + System.loadLibrary("faceclaw_llama"); + } + + private final Handler callbackHandler; + private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "FaceclawLlama"); + thread.setPriority(Thread.NORM_PRIORITY); + return thread; + }); + + // Mutated only on the executor thread; volatile so cancel() can read it. + private volatile long handle; + private String loadedPath; + + public FaceclawLlamaRunner() { + Looper looper = Looper.myLooper(); + callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); + } + + public boolean isModelLoaded() { + return handle != 0; + } + + /** + * Generate a completion, loading (or swapping) the model first if needed. + * grammar may be null/empty for unconstrained generation. + */ + public void generate(final String modelPath, final int nCtx, final int nThreads, + final String prompt, final String grammar, final int maxTokens, + final float temperature, final float topP, final int topK, + final FaceclawLlamaListener listener) { + executor.execute(() -> { + try { + if (handle != 0 && !modelPath.equals(loadedPath)) { + nativeFree(handle); + handle = 0; + loadedPath = null; + } + if (handle == 0) { + long loaded = nativeLoadModel(modelPath, nCtx, nThreads); + if (loaded == 0) { + post(() -> listener.onError("Could not load the on-phone model")); + return; + } + handle = loaded; + loadedPath = modelPath; + } + nativeGenerate(handle, prompt, grammar == null ? "" : grammar, + maxTokens, temperature, topP, topK, new FaceclawLlamaListener() { + @Override public void onToken(final String piece) { + post(() -> listener.onToken(piece)); + } + @Override public void onDone(final String stopReason) { + post(() -> listener.onDone(stopReason)); + } + @Override public void onError(final String message) { + post(() -> listener.onError(message)); + } + }); + } catch (Throwable t) { + Log.e(TAG, "generation failed", t); + final String message = String.valueOf(t); + post(() -> listener.onError("On-phone model failed: " + message)); + } + }); + } + + /** Interrupt the current generation (its listener gets onDone("cancelled")). */ + public void cancel() { + long current = handle; + if (current != 0) { + nativeCancel(current); + } + } + + /** Free the model and its KV cache. Safe to call when nothing is loaded. */ + public void unload() { + executor.execute(() -> { + if (handle != 0) { + nativeFree(handle); + handle = 0; + loadedPath = null; + } + }); + } + + private void post(Runnable runnable) { + callbackHandler.post(runnable); + } + + private static native long nativeLoadModel(String path, int nCtx, int nThreads); + private static native void nativeGenerate(long handle, String prompt, String grammar, + int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener); + private static native void nativeCancel(long handle); + private static native void nativeFree(long handle); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java new file mode 100644 index 00000000..64b93219 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java @@ -0,0 +1,192 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.util.concurrent.TimeUnit; + +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; + +/** + * Large-file downloader for on-phone model weights (multi-GB, from Hugging + * Face). Downloads to ".part" with HTTP Range resume across app + * restarts, verifies a pinned sha256 (hashed incrementally, including the + * resumed prefix), then renames into place. Listener callbacks are posted to + * the constructing thread's Looper, like the other Faceclaw bridges. + */ +public class FaceclawModelDownloader { + private static final String TAG = "FaceclawModelDl"; + private static final long PROGRESS_INTERVAL_MS = 500; + + private static volatile OkHttpClient sharedClient; + + private final Handler callbackHandler; + private final String url; + private final File destFile; + private final File partFile; + private final String expectedSha256; + private final long expectedTotalBytes; + private final FaceclawModelDownloaderListener listener; + + private volatile boolean cancelled; + private volatile Call call; + private Thread worker; + + public FaceclawModelDownloader(String url, String destPath, String expectedSha256, + long expectedTotalBytes, FaceclawModelDownloaderListener listener) { + if (listener == null) throw new IllegalArgumentException("listener is required"); + Looper looper = Looper.myLooper(); + this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper()); + this.url = url; + this.destFile = new File(destPath); + this.partFile = new File(destPath + ".part"); + this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase(); + this.expectedTotalBytes = expectedTotalBytes; + this.listener = listener; + } + + private static OkHttpClient getClient() { + if (sharedClient == null) { + synchronized (FaceclawModelDownloader.class) { + if (sharedClient == null) { + sharedClient = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .build(); + } + } + } + return sharedClient; + } + + public void start() { + worker = new Thread(this::run, "FaceclawModelDl"); + worker.start(); + } + + /** Stops the download but keeps the .part file so a later start() resumes. */ + public void cancel() { + cancelled = true; + Call current = call; + if (current != null) current.cancel(); + } + + private void run() { + try { + if (destFile.exists() && destFile.length() > 0) { + post(() -> listener.onDone(destFile.getAbsolutePath())); + return; + } + destFile.getParentFile().mkdirs(); + + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + long offset = 0; + if (partFile.exists()) { + offset = hashExistingPrefix(digest); + if (cancelled) return; + } + + Request.Builder builder = new Request.Builder().url(url); + if (offset > 0) builder.header("Range", "bytes=" + offset + "-"); + call = getClient().newCall(builder.build()); + + try (Response response = call.execute()) { + ResponseBody body = response.body(); + if (!response.isSuccessful() || body == null) { + postError("Model download failed: HTTP " + response.code()); + return; + } + boolean resumed = response.code() == 206; + if (offset > 0 && !resumed) { + // Server ignored the Range header; start over. + offset = 0; + digest = MessageDigest.getInstance("SHA-256"); + } + long total = expectedTotalBytes > 0 ? expectedTotalBytes + : (body.contentLength() > 0 ? offset + body.contentLength() : -1); + + try (FileOutputStream out = new FileOutputStream(partFile, resumed); + InputStream in = body.byteStream()) { + byte[] buffer = new byte[1 << 16]; + long done = offset; + long lastProgressAt = 0; + int read; + while ((read = in.read(buffer)) != -1) { + if (cancelled) return; + out.write(buffer, 0, read); + digest.update(buffer, 0, read); + done += read; + long now = System.currentTimeMillis(); + if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) { + lastProgressAt = now; + final long doneFinal = done; + final long totalFinal = total; + post(() -> listener.onProgress(doneFinal, totalFinal)); + } + } + } + } + + if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) { + postError("Model download ended early (" + partFile.length() + " of " + + expectedTotalBytes + " bytes); try again to resume"); + return; + } + String actualSha = toHex(digest.digest()); + if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) { + partFile.delete(); + postError("Model download was corrupted (checksum mismatch); download it again"); + return; + } + if (!partFile.renameTo(destFile)) { + postError("Could not move the downloaded model into place"); + return; + } + post(() -> listener.onDone(destFile.getAbsolutePath())); + } catch (Exception e) { + if (cancelled) return; + Log.w(TAG, "download failed", e); + postError("Model download failed: " + e); + } + } + + /** Hash the already-downloaded prefix so the final digest covers the whole file. */ + private long hashExistingPrefix(MessageDigest digest) throws IOException { + long hashed = 0; + try (FileInputStream in = new FileInputStream(partFile)) { + byte[] buffer = new byte[1 << 16]; + int read; + while ((read = in.read(buffer)) != -1) { + if (cancelled) return hashed; + digest.update(buffer, 0, read); + hashed += read; + } + } + return hashed; + } + + private static String toHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) sb.append(String.format("%02x", b)); + return sb.toString(); + } + + private void post(Runnable runnable) { + callbackHandler.post(runnable); + } + + private void postError(final String message) { + post(() -> listener.onError(message)); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloaderListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloaderListener.java new file mode 100644 index 00000000..90bf9380 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloaderListener.java @@ -0,0 +1,11 @@ +package com.faceclaw.app; + +/** Progress callbacks for FaceclawModelDownloader, posted to the creating thread's Looper. */ +public interface FaceclawModelDownloaderListener { + void onProgress(long bytesDownloaded, long totalBytes); + + /** The file is fully downloaded, verified, and moved to its final path. */ + void onDone(String path); + + void onError(String message); +} diff --git a/App_Resources/Android/src/main/native/llama/CMakeLists.txt b/App_Resources/Android/src/main/native/llama/CMakeLists.txt new file mode 100644 index 00000000..7ab3677e --- /dev/null +++ b/App_Resources/Android/src/main/native/llama/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.22) +project(faceclaw_llama CXX C) + +# llama.cpp source tree location is injected by the gradle task (it downloads +# and extracts a pinned release tag before configuring this project). +if(NOT DEFINED LLAMA_CPP_SOURCE_DIR) + message(FATAL_ERROR "Pass -DLLAMA_CPP_SOURCE_DIR=") +endif() + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +# Static llama/ggml folded into our single JNI .so. CPU only: Vulkan on Mali +# and mixed Adreno drivers is not worth the instability for short assistant +# turns. The -march baseline lives in the gradle invocation (armv8.2-a with +# dotprod+fp16: the floor for Tensor G2's X1/A78/A55, so no i8mm). +set(BUILD_SHARED_LIBS OFF) +set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE) +set(LLAMA_CURL OFF CACHE BOOL "" FORCE) +set(GGML_OPENMP OFF CACHE BOOL "" FORCE) +set(GGML_NATIVE OFF CACHE BOOL "" FORCE) +set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE) + +add_subdirectory(${LLAMA_CPP_SOURCE_DIR} llama.cpp-build EXCLUDE_FROM_ALL) + +add_library(faceclaw_llama SHARED faceclaw_llama.cpp) +target_link_libraries(faceclaw_llama PRIVATE llama android log) diff --git a/App_Resources/Android/src/main/native/llama/faceclaw_llama.cpp b/App_Resources/Android/src/main/native/llama/faceclaw_llama.cpp new file mode 100644 index 00000000..ef639ed9 --- /dev/null +++ b/App_Resources/Android/src/main/native/llama/faceclaw_llama.cpp @@ -0,0 +1,391 @@ +// JNI bridge for the on-phone assistant model (llama.cpp, CPU inference). +// Java surface: FaceclawLlamaRunner's nativeLoadModel / nativeGenerate / +// nativeCancel / nativeFree. One model+context per handle; the Java side +// serializes all calls except nativeCancel onto a single executor thread, +// so only the cancel flag needs to be thread-safe. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "ggml-cpu.h" +#include "llama.h" + +#define TAG "FaceclawLlama" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) + +namespace { + +struct FcLlama { + llama_model * model = nullptr; + llama_context * ctx = nullptr; + ggml_threadpool * threadpool = nullptr; + const llama_vocab * vocab = nullptr; + int n_ctx = 0; + // Tokens currently materialized in the KV cache (prompt + generated), + // used to skip re-decoding the common prefix on the next call. Within a + // turn the next prompt extends the previous one, so this usually reduces + // prefill to just the new tool results. + std::vector cache_tokens; + std::atomic cancel{false}; +}; + +void android_llama_log(ggml_log_level level, const char * text, void * /*user*/) { + if (level < GGML_LOG_LEVEL_WARN) return; // llama.cpp is chatty at INFO during load + __android_log_write(level >= GGML_LOG_LEVEL_ERROR ? ANDROID_LOG_ERROR : ANDROID_LOG_WARN, TAG, text); +} + +void ensure_backend_init() { + static std::once_flag once; + std::call_once(once, [] { + llama_log_set(android_llama_log, nullptr); + llama_backend_init(); + }); +} + +int64_t now_ms() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +// Big-core detection for big.LITTLE phones. Without explicit affinity the +// scheduler is free to put our compute threads on the little cores (in-order +// A55s on Tensor G2), which slows prefill by 5-10x; pin to the cores whose +// max frequency is within 75% of the fastest core's (on a Pixel 7/7a this +// selects the 2x X1 + 2x A78 and excludes the 4x A55). +std::vector detect_fast_cpus() { + std::vector freqs; + for (int cpu = 0; cpu < GGML_MAX_N_THREADS; cpu++) { + char path[128]; + snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", cpu); + FILE * f = fopen(path, "r"); + if (f == nullptr) break; + long freq = 0; + if (fscanf(f, "%ld", &freq) != 1) freq = 0; + fclose(f); + freqs.push_back(freq); + } + std::vector fast; + long max_freq = 0; + for (long f : freqs) max_freq = std::max(max_freq, f); + if (max_freq <= 0) return fast; + for (size_t cpu = 0; cpu < freqs.size(); cpu++) { + if (freqs[cpu] * 4 >= max_freq * 3) fast.push_back((int) cpu); + } + // A uniform-frequency CPU has no little cores to avoid. + if (fast.size() == freqs.size()) fast.clear(); + return fast; +} + +std::string jstring_to_utf8(JNIEnv * env, jstring value) { + if (value == nullptr) return ""; + const char * chars = env->GetStringUTFChars(value, nullptr); + std::string out = chars ? chars : ""; + env->ReleaseStringUTFChars(value, chars); + return out; +} + +// Callback shims into the Java listener (an in-process wrapper object that +// posts to the JS thread; these calls happen on the executor thread). +struct Listener { + JNIEnv * env; + jobject obj; + jmethodID on_token; + jmethodID on_done; + jmethodID on_error; + + static Listener resolve(JNIEnv * env, jobject listener) { + jclass cls = env->GetObjectClass(listener); + return Listener{ + env, + listener, + env->GetMethodID(cls, "onToken", "(Ljava/lang/String;)V"), + env->GetMethodID(cls, "onDone", "(Ljava/lang/String;)V"), + env->GetMethodID(cls, "onError", "(Ljava/lang/String;)V"), + }; + } + + void token(const std::string & piece) const { call(on_token, piece); } + void done(const char * reason) const { call(on_done, reason); } + void error(const std::string & message) const { call(on_error, message); } + + private: + void call(jmethodID method, const std::string & text) const { + jstring jtext = env->NewStringUTF(text.c_str()); + env->CallVoidMethod(obj, method, jtext); + env->DeleteLocalRef(jtext); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } + } +}; + +// NewStringUTF aborts on invalid (modified-)UTF-8, and token pieces can end +// mid-codepoint, so hold back any incomplete trailing UTF-8 sequence. +size_t utf8_complete_prefix(const std::string & data) { + size_t len = data.size(); + size_t i = len; + // Find the start byte of the final codepoint (at most 3 bytes back). + while (i > 0 && len - i < 4 && (static_cast(data[i - 1]) & 0xC0) == 0x80) i--; + if (i == 0) return len; // malformed; let it through rather than stall + unsigned char start = static_cast(data[i - 1]); + size_t expected; + if (start < 0x80) expected = 1; + else if ((start & 0xE0) == 0xC0) expected = 2; + else if ((start & 0xF0) == 0xE0) expected = 3; + else if ((start & 0xF8) == 0xF0) expected = 4; + else return len; // malformed start byte; pass through + size_t have = len - (i - 1); + return have >= expected ? len : i - 1; +} + +std::vector tokenize(const llama_vocab * vocab, const std::string & text, bool add_special) { + int n = -llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), nullptr, 0, add_special, true); + if (n <= 0) return {}; + std::vector tokens(n); + int written = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), n, add_special, true); + if (written < 0) return {}; + tokens.resize(written); + return tokens; +} + +std::string token_piece(const llama_vocab * vocab, llama_token token) { + char buf[256]; + int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true); + if (n <= 0) return ""; + return std::string(buf, n); +} + +} // namespace + +extern "C" JNIEXPORT jlong JNICALL +Java_com_faceclaw_app_FaceclawLlamaRunner_nativeLoadModel( + JNIEnv * env, jclass, jstring jpath, jint n_ctx, jint n_threads) { + ensure_backend_init(); + const std::string path = jstring_to_utf8(env, jpath); + const int64_t t_start = now_ms(); + + // Cap threads to the big-core count and pin them there (see + // detect_fast_cpus): little cores would otherwise both drag the barrier + // and invite the scheduler to keep us off the fast cores entirely. + const std::vector fast_cpus = detect_fast_cpus(); + int threads = (int) n_threads; + if (!fast_cpus.empty()) { + threads = std::min(threads, (int) fast_cpus.size()); + } + + llama_model_params mparams = llama_model_default_params(); + mparams.n_gpu_layers = 0; + llama_model * model = llama_model_load_from_file(path.c_str(), mparams); + if (model == nullptr) { + LOGW("model load failed: %s", path.c_str()); + return 0; + } + const int64_t t_model = now_ms(); + + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = (uint32_t) n_ctx; + cparams.n_batch = 512; + cparams.n_threads = threads; + cparams.n_threads_batch = threads; + llama_context * ctx = llama_init_from_model(model, cparams); + if (ctx == nullptr) { + LOGW("context init failed"); + llama_model_free(model); + return 0; + } + + ggml_threadpool * threadpool = nullptr; + if (!fast_cpus.empty()) { + ggml_threadpool_params tpp = ggml_threadpool_params_default(threads); + for (size_t i = 0; i < fast_cpus.size() && fast_cpus[i] < GGML_MAX_N_THREADS; i++) { + tpp.cpumask[fast_cpus[i]] = true; + } + tpp.strict_cpu = true; + threadpool = ggml_threadpool_new(&tpp); + if (threadpool != nullptr) { + llama_attach_threadpool(ctx, threadpool, threadpool); + std::string cpus; + for (int cpu : fast_cpus) cpus += std::to_string(cpu) + " "; + LOGI("pinned %d threads to fast cpus: %s", threads, cpus.c_str()); + } else { + LOGW("threadpool creation failed; using default scheduling"); + } + } + + auto * handle = new FcLlama(); + handle->model = model; + handle->ctx = ctx; + handle->threadpool = threadpool; + handle->vocab = llama_model_get_vocab(model); + handle->n_ctx = (int) llama_n_ctx(ctx); + LOGI("model loaded: %s (n_ctx=%d, threads=%d, model_load_ms=%lld, ctx_init_ms=%lld)", + path.c_str(), handle->n_ctx, threads, + (long long) (t_model - t_start), (long long) (now_ms() - t_model)); + return reinterpret_cast(handle); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_faceclaw_app_FaceclawLlamaRunner_nativeFree(JNIEnv *, jclass, jlong jhandle) { + auto * handle = reinterpret_cast(jhandle); + if (handle == nullptr) return; + llama_free(handle->ctx); + llama_model_free(handle->model); + if (handle->threadpool != nullptr) { + ggml_threadpool_free(handle->threadpool); + } + delete handle; + LOGI("model freed"); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_faceclaw_app_FaceclawLlamaRunner_nativeCancel(JNIEnv *, jclass, jlong jhandle) { + auto * handle = reinterpret_cast(jhandle); + if (handle != nullptr) handle->cancel.store(true); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_faceclaw_app_FaceclawLlamaRunner_nativeGenerate( + JNIEnv * env, jclass, jlong jhandle, jstring jprompt, jstring jgrammar, + jint max_tokens, jfloat temperature, jfloat top_p, jint top_k, + jobject jlistener) { + auto * handle = reinterpret_cast(jhandle); + Listener listener = Listener::resolve(env, jlistener); + if (handle == nullptr) { + listener.error("Model is not loaded"); + return; + } + handle->cancel.store(false); + + const std::string prompt = jstring_to_utf8(env, jprompt); + const std::string grammar = jstring_to_utf8(env, jgrammar); + + std::vector tokens = tokenize(handle->vocab, prompt, true); + if (tokens.empty()) { + listener.error("Prompt tokenization failed"); + return; + } + if ((int) tokens.size() + 16 > handle->n_ctx) { + listener.error("Conversation too long for the on-phone model"); + return; + } + const int max_gen = std::min((int) max_tokens, handle->n_ctx - (int) tokens.size()); + + // Reuse the KV prefix shared with the previous call; drop the rest. Keep + // at least the final prompt token out of the reused prefix so this decode + // produces logits for it. + size_t common = 0; + while (common < tokens.size() && common < handle->cache_tokens.size() && + tokens[common] == handle->cache_tokens[common]) { + common++; + } + if (common == tokens.size()) common = tokens.size() - 1; + llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) common, -1); + handle->cache_tokens.assign(tokens.begin(), tokens.begin() + common); + + // Sampler chain per generation: Qwen3 instruct posture (temp 0.7, top-p + // 0.8, top-k 20) plus an optional lazy grammar that activates at the + // special token and forces a schema-valid call. + llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); + if (!grammar.empty()) { + llama_sampler * grammar_sampler = nullptr; + std::vector trigger = tokenize(handle->vocab, "", false); + if (trigger.size() == 1) { + grammar_sampler = llama_sampler_init_grammar_lazy_patterns( + handle->vocab, grammar.c_str(), "root", nullptr, 0, trigger.data(), 1); + } else { + static const char * patterns[] = {"([\\s\\S]*)"}; + grammar_sampler = llama_sampler_init_grammar_lazy_patterns( + handle->vocab, grammar.c_str(), "root", patterns, 1, nullptr, 0); + } + if (grammar_sampler != nullptr) { + llama_sampler_chain_add(chain, grammar_sampler); + } else { + LOGW("tool grammar failed to parse; generating unconstrained"); + } + } + llama_sampler_chain_add(chain, llama_sampler_init_top_k(top_k)); + llama_sampler_chain_add(chain, llama_sampler_init_top_p(top_p, 1)); + llama_sampler_chain_add(chain, llama_sampler_init_temp(temperature)); + llama_sampler_chain_add(chain, llama_sampler_init_dist(LLAMA_DEFAULT_SEED)); + + const auto fail = [&](const std::string & message) { + llama_sampler_free(chain); + // The KV state past the reused prefix is now unknown; drop it so the + // next call starts from a consistent cache. + llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) handle->cache_tokens.size(), -1); + listener.error(message); + }; + + // Prefill in n_batch chunks. + const int64_t t_prefill_start = now_ms(); + const int n_batch = (int) llama_n_batch(handle->ctx); + for (size_t i = common; i < tokens.size(); i += n_batch) { + if (handle->cancel.load()) { + llama_sampler_free(chain); + listener.done("cancelled"); + return; + } + const int chunk = (int) std::min((size_t) n_batch, tokens.size() - i); + if (llama_decode(handle->ctx, llama_batch_get_one(tokens.data() + i, chunk)) != 0) { + fail("Prompt evaluation failed"); + return; + } + handle->cache_tokens.insert(handle->cache_tokens.end(), tokens.begin() + i, tokens.begin() + i + chunk); + } + + // Decode loop: sample, stream, feed back. + const int64_t t_decode_start = now_ms(); + const size_t n_prefilled = tokens.size() - common; + int n_decoded = 0; + std::string pending_utf8; + const char * stop_reason = "length"; + for (int i = 0; i < max_gen; i++) { + if (handle->cancel.load()) { + stop_reason = "cancelled"; + break; + } + llama_token token = llama_sampler_sample(chain, handle->ctx, -1); + if (llama_vocab_is_eog(handle->vocab, token)) { + stop_reason = "stop"; + break; + } + pending_utf8 += token_piece(handle->vocab, token); + size_t emit = utf8_complete_prefix(pending_utf8); + if (emit > 0) { + listener.token(pending_utf8.substr(0, emit)); + pending_utf8.erase(0, emit); + } + if (llama_decode(handle->ctx, llama_batch_get_one(&token, 1)) != 0) { + fail("Token evaluation failed"); + return; + } + handle->cache_tokens.push_back(token); + n_decoded++; + } + + const int64_t t_end = now_ms(); + const int64_t prefill_ms = t_decode_start - t_prefill_start; + const int64_t decode_ms = t_end - t_decode_start; + LOGI("timing: prompt=%zu reused=%zu prefill=%zu tok in %lld ms (%.1f tok/s), decode=%d tok in %lld ms (%.1f tok/s)", + tokens.size(), common, n_prefilled, (long long) prefill_ms, + prefill_ms > 0 ? 1000.0 * (double) n_prefilled / (double) prefill_ms : 0.0, + n_decoded, (long long) decode_ms, + decode_ms > 0 ? 1000.0 * (double) n_decoded / (double) decode_ms : 0.0); + + if (!pending_utf8.empty()) { + size_t emit = utf8_complete_prefix(pending_utf8); + if (emit > 0) listener.token(pending_utf8.substr(0, emit)); + } + llama_sampler_free(chain); + listener.done(stop_reason); +} diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index dc15bd00..12e5cbbc 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -1,4 +1,5 @@ import { streamAnthropicMessage } from "../native/anthropic"; +import { streamLocalQwen } from "../native/llama"; import { streamOpenAiResponse } from "../native/openai"; import type { LlmContentBlock, @@ -20,6 +21,8 @@ import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types"; /** Safety caps so a tool-thrashing model can't loop or run forever. */ const MAX_ITERATIONS = 8; +/** The on-phone model is slower per step and unreliable on long tool chains. */ +const MAX_ITERATIONS_LOCAL = 4; const MAX_TURN_MS = 2 * 60 * 1000; export type DirectTurnOptions = { @@ -54,9 +57,10 @@ export class DirectAssistantBackend { options.callbacks.onError(message); }; + const maxIterations = options.provider === "local" ? MAX_ITERATIONS_LOCAL : MAX_ITERATIONS; const runIteration = (iteration: number): void => { if (cancelled) return; - if (iteration >= MAX_ITERATIONS) { + if (iteration >= maxIterations) { finishError("Assistant stopped: too many tool steps"); return; } @@ -115,9 +119,12 @@ export class DirectAssistantBackend { finishError(message); }, }; - streamHandle = options.provider === "openai" - ? streamOpenAiResponse(streamOptions) - : streamAnthropicMessage(streamOptions); + streamHandle = + options.provider === "openai" + ? streamOpenAiResponse(streamOptions) + : options.provider === "local" + ? streamLocalQwen(streamOptions) + : streamAnthropicMessage(streamOptions); }; runIteration(0); diff --git a/app/assistant/gbnf.ts b/app/assistant/gbnf.ts new file mode 100644 index 00000000..0010bb95 --- /dev/null +++ b/app/assistant/gbnf.ts @@ -0,0 +1,145 @@ +import type { LlmToolDefinition } from "./llm-protocol"; + +/** + * Generate a GBNF grammar (llama.cpp grammar dialect) that constrains a + * Qwen-style tool call to be well-formed: one block whose JSON + * names a real tool and whose arguments match that tool's schema. Used as a + * *lazy* grammar — sampling is unconstrained until the model emits + * , so plain text answers are unaffected, and once the block + * closes the grammar is complete (the model can only end the message, which + * serializes multi-call turns into one call per loop iteration — deliberate: + * small models are far more reliable one call at a time). + * + * Our tool schemas are flat objects of string/number/boolean/enum properties. + * Anything fancier degrades to a generic JSON value rule rather than failing. + */ + +type JsonSchema = { + type?: string; + enum?: unknown[]; + properties?: Record; + required?: string[]; + description?: string; +}; + +/** Longest run of optional properties we'll enumerate subsets for. */ +const MAX_SUBSET_PROPS = 5; + +export function buildToolCallGrammar(tools: LlmToolDefinition[]): string | null { + if (!tools.length) return null; + const used = new Set(); + const toolRules: string[] = []; + for (let i = 0; i < tools.length; i++) { + const args = argsRule(tools[i]!.input_schema as JsonSchema, used); + toolRules.push( + `tc${i} ::= "" nl "{" sp ${jsonKey("name")} sp ":" sp "\\"${gbnfEscape( + tools[i]!.name, + )}\\"" sp "," sp ${jsonKey("arguments")} sp ":" sp ${args} sp "}" nl ""`, + ); + } + used.add("sp"); + used.add("nl"); + + const lines = [`root ::= ${toolRules.map((_, i) => `tc${i}`).join(" | ")}`, ...toolRules]; + for (const [name, rule] of SHARED_RULES) { + if (used.has(name)) lines.push(rule); + } + return lines.join("\n"); +} + +/** Build the arguments-object expression for one tool schema. */ +function argsRule(schema: JsonSchema | undefined, used: Set): string { + const properties = schema?.properties ?? {}; + const names = Object.keys(properties); + if (!names.length) return `"{" sp "}"`; + const required = new Set(schema?.required ?? []); + const optionalCount = names.filter((n) => !required.has(n)).length; + if (names.length > MAX_SUBSET_PROPS && optionalCount > 0) { + used.add("jobj"); + return "jobj"; + } + + // Enumerate every subset of properties that includes all required ones, + // keeping declaration order. Small N keeps this tiny (<= 2^5 alternatives). + const members = names.map((name) => { + const value = valueExpr(properties[name]!, used); + return `${jsonKey(name)} sp ":" sp ${value}`; + }); + const alternatives: string[] = []; + for (let mask = 0; mask < 1 << names.length; mask++) { + const chosen: string[] = []; + let valid = true; + for (let j = 0; j < names.length; j++) { + const included = (mask & (1 << j)) !== 0; + if (included) chosen.push(members[j]!); + else if (required.has(names[j]!)) valid = false; + } + if (!valid) continue; + if (!chosen.length) { + alternatives.push(`"{" sp "}"`); + } else { + alternatives.push(`"{" sp ${chosen.join(` sp "," sp `)} sp "}"`); + } + } + return alternatives.length === 1 ? alternatives[0]! : `(${alternatives.join(" | ")})`; +} + +/** Grammar expression for one property value. */ +function valueExpr(schema: JsonSchema, used: Set): string { + if (Array.isArray(schema.enum) && schema.enum.length) { + if (schema.enum.every((v) => typeof v === "string")) { + return `(${schema.enum.map((v) => `"\\"${gbnfEscape(String(v))}\\""`).join(" | ")})`; + } + if (schema.enum.every((v) => typeof v === "number" || typeof v === "boolean")) { + return `(${schema.enum.map((v) => `"${String(v)}"`).join(" | ")})`; + } + } + switch (schema.type) { + case "string": + used.add("jstring"); + used.add("jchar"); + used.add("jhex"); + return "jstring"; + case "number": + used.add("jnumber"); + return "jnumber"; + case "integer": + used.add("jint"); + return "jint"; + case "boolean": + used.add("jbool"); + return "jbool"; + default: + for (const name of ["jvalue", "jstring", "jchar", "jhex", "jnumber", "jbool", "jarray", "jobj", "jmember"]) { + used.add(name); + } + return "jvalue"; + } +} + +/** A JSON object key as a GBNF literal: "name" -> "\"name\"". */ +function jsonKey(name: string): string { + return `"\\"${gbnfEscape(name)}\\""`; +} + +/** Escape for the inside of a GBNF double-quoted literal. */ +function gbnfEscape(text: string): string { + return text.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n"); +} + +// Terminal rules, emitted only when referenced. jchar mirrors llama.cpp's +// json.gbnf string character rule. +const SHARED_RULES: Array<[string, string]> = [ + ["sp", `sp ::= " "?`], + ["nl", `nl ::= "\\n"`], + ["jstring", `jstring ::= "\\"" jchar* "\\""`], + ["jchar", `jchar ::= [^"\\\\\\x7F\\x00-\\x1F] | "\\\\" (["\\\\bfnrt] | "u" jhex jhex jhex jhex)`], + ["jhex", `jhex ::= [0-9a-fA-F]`], + ["jnumber", `jnumber ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? (("e" | "E") ("-" | "+")? [0-9]+)?`], + ["jint", `jint ::= "-"? ("0" | [1-9] [0-9]*)`], + ["jbool", `jbool ::= "true" | "false"`], + ["jvalue", `jvalue ::= jstring | jnumber | jbool | "null" | jarray | jobj`], + ["jarray", `jarray ::= "[" sp (jvalue (sp "," sp jvalue)*)? sp "]"`], + ["jobj", `jobj ::= "{" sp (jmember (sp "," sp jmember)*)? sp "}"`], + ["jmember", `jmember ::= jstring sp ":" sp jvalue`], +]; diff --git a/app/assistant/qwen-prompt.ts b/app/assistant/qwen-prompt.ts new file mode 100644 index 00000000..b8da5265 --- /dev/null +++ b/app/assistant/qwen-prompt.ts @@ -0,0 +1,81 @@ +import type { LlmContentBlock, LlmMessage, LlmToolDefinition } from "./llm-protocol"; + +/** + * Render our provider-neutral message history into a Qwen3 (ChatML) prompt + * string, matching the model's chat template closely enough that its + * tool-calling priors apply: Hermes-style declarations in the system + * block, JSON blocks in assistant turns, and tool results wrapped + * in inside a user turn. The caller appends nothing — the + * returned string ends with the assistant generation header. + */ + +const IM_START = "<|im_start|>"; +const IM_END = "<|im_end|>"; + +export function buildQwenPrompt( + system: string | undefined, + messages: LlmMessage[], + tools: LlmToolDefinition[] | undefined, +): string { + const parts: string[] = []; + parts.push(`${IM_START}system\n${systemBlock(system, tools)}${IM_END}\n`); + for (const message of messages) { + if (message.role === "assistant") { + parts.push(`${IM_START}assistant\n${assistantBlock(message.content)}${IM_END}\n`); + } else { + parts.push(`${IM_START}user\n${userBlock(message.content)}${IM_END}\n`); + } + } + parts.push(`${IM_START}assistant\n`); + return parts.join(""); +} + +function systemBlock(system: string | undefined, tools: LlmToolDefinition[] | undefined): string { + const base = system?.trim() ?? "You are a helpful assistant."; + if (!tools?.length) return base; + const declarations = tools + .map((tool) => + JSON.stringify({ + type: "function", + function: { name: tool.name, description: tool.description, parameters: tool.input_schema }, + }), + ) + .join("\n"); + return ( + `${base}\n\n# Tools\n\n` + + `You may call one or more functions to assist with the user query.\n\n` + + `You are provided with function signatures within XML tags:\n` + + `\n${declarations}\n\n\n` + + `For each function call, return a json object with function name and arguments ` + + `within XML tags:\n` + + `\n{"name": , "arguments": }\n` + ); +} + +function assistantBlock(content: string | LlmContentBlock[]): string { + if (typeof content === "string") return content; + const parts: string[] = []; + for (const block of content) { + if (block.type === "text") { + if (block.text.trim()) parts.push(block.text.trim()); + } else if (block.type === "tool_use") { + parts.push(`\n${JSON.stringify({ name: block.name, arguments: block.input ?? {} })}\n`); + } + // provider_item blocks are OpenAI-specific continuation state; skip. + } + return parts.join("\n"); +} + +function userBlock(content: string | LlmContentBlock[]): string { + if (typeof content === "string") return content; + const parts: string[] = []; + for (const block of content) { + if (block.type === "text") { + if (block.text.trim()) parts.push(block.text.trim()); + } else if (block.type === "tool_result") { + const body = block.is_error ? `Error: ${block.content}` : block.content || "(no output)"; + parts.push(`\n${body}\n`); + } + } + return parts.join("\n"); +} diff --git a/app/assistant/session.ts b/app/assistant/session.ts index 103634a5..604095a9 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -1,4 +1,8 @@ -import { buildAssistantSystemPrompt } from "../prompts"; +import { + ASSISTANT_SYSTEM_PROMPT_BASE, + buildAssistantSystemPrompt, + describeAssistantContext, +} from "../prompts"; import { assistantBridge } from "./bridge-client"; import { DirectAssistantBackend } from "./direct-backend"; import type { LlmMessage, LlmToolDefinition } from "./llm-protocol"; @@ -108,15 +112,23 @@ export class AssistantSession { return; } - this.messages.push({ role: "user", content: text }); - this.trimHistory(); const llm = this.config.llm; + // Local provider: keep the system prompt (and thus the tool declarations + // that follow it in the rendered prompt) byte-stable so the on-phone + // model's KV prefix cache survives across turns; the volatile context + // (time, foreground app, battery) rides along with the utterance instead. + const isLocal = llm.provider === "local"; + this.messages.push({ + role: "user", + content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text, + }); + this.trimHistory(); this.turnHandle = this.directBackend.runTurn({ provider: llm.provider, apiKey: llm.apiKey, model: llm.model, effort: llm.effort, - system: buildAssistantSystemPrompt(ctx), + system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), messages: this.messages, buildTools: () => this.buildToolDefinitions(), registry: this.registry, diff --git a/app/native/llama.ts b/app/native/llama.ts new file mode 100644 index 00000000..c60ad4bb --- /dev/null +++ b/app/native/llama.ts @@ -0,0 +1,321 @@ +import { Utils } from "@nativescript/core"; +import { buildToolCallGrammar } from "../assistant/gbnf"; +import type { + LlmContentBlock, + LlmStreamHandle, + LlmStreamOptions, +} from "../assistant/llm-protocol"; +import { buildQwenPrompt } from "../assistant/qwen-prompt"; + +declare const com: any; +declare const java: any; + +/** + * On-phone LLM backend: model download management plus a streaming client + * with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so + * the direct assistant loop can treat "local" as just another provider. + * Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI). + */ + +export const LOCAL_MODEL = { + id: "qwen3-4b-instruct-2507", + label: "Qwen3 4B", + fileName: "Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + url: "https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/Qwen3-4B-Instruct-2507-Q4_K_M.gguf?download=true", + sha256: "3605803b982cb64aead44f6c1b2ae36e3acdb41d8e46c8a94c6533bc4c67e597", + sizeBytes: 2497281120, +}; + +// Bounded so KV cache stays ~0.5GB (Qwen3-4B KV is ~144KB/token at f16); +// the assistant trims history well before this in practice. +const N_CTX = 4096; +const N_THREADS = 4; +const DEFAULT_MAX_TOKENS = 512; +// Qwen3-4B-Instruct-2507 model-card sampling posture. +const TEMPERATURE = 0.7; +const TOP_P = 0.8; +const TOP_K = 20; +/** Free the ~3GB of model + KV this long after the last generation. */ +const IDLE_UNLOAD_MS = 2 * 60 * 1000; + +export type LocalModelState = { + status: "absent" | "downloading" | "ready"; + bytesDownloaded: number; + totalBytes: number; +}; + +let runner: any = null; +let downloader: any = null; +let downloadedBytes = 0; +let unloadTimer: ReturnType | null = null; +let toolUseCounter = 0; +const stateListeners = new Set<(state: LocalModelState) => void>(); + +function modelPath(): string { + const context = Utils.android.getApplicationContext(); + const dir = context.getExternalFilesDir("llm") ?? context.getFilesDir(); + return `${dir.getAbsolutePath()}/${LOCAL_MODEL.fileName}`; +} + +export function isLocalModelReady(): boolean { + if (!global.isAndroid) return false; + try { + const file = new java.io.File(modelPath()); + return file.exists() && file.length() > 0; + } catch { + return false; + } +} + +export function localModelState(): LocalModelState { + if (downloader) { + return { status: "downloading", bytesDownloaded: downloadedBytes, totalBytes: LOCAL_MODEL.sizeBytes }; + } + return { + status: isLocalModelReady() ? "ready" : "absent", + bytesDownloaded: 0, + totalBytes: LOCAL_MODEL.sizeBytes, + }; +} + +export function onLocalModelStateChanged(listener: (state: LocalModelState) => void): () => void { + stateListeners.add(listener); + return () => stateListeners.delete(listener); +} + +function notifyStateChanged(): void { + const state = localModelState(); + for (const listener of stateListeners) listener(state); +} + +export function startLocalModelDownload(): void { + if (!global.isAndroid || downloader || isLocalModelReady()) return; + const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({ + onProgress: (bytes: number, _total: number) => { + downloadedBytes = Number(bytes); + notifyStateChanged(); + }, + onDone: () => { + downloader = null; + notifyStateChanged(); + }, + onError: (message: string) => { + console.error(`Local model download failed: ${message}`); + downloader = null; + notifyStateChanged(); + }, + }); + downloader = new com.faceclaw.app.FaceclawModelDownloader( + LOCAL_MODEL.url, + modelPath(), + LOCAL_MODEL.sha256, + LOCAL_MODEL.sizeBytes, + listener, + ); + downloadedBytes = 0; + downloader.start(); + notifyStateChanged(); +} + +/** Stops the download; already-fetched bytes are kept and resumed next time. */ +export function cancelLocalModelDownload(): void { + if (!downloader) return; + downloader.cancel(); + downloader = null; + notifyStateChanged(); +} + +export function deleteLocalModel(): void { + if (!global.isAndroid) return; + cancelLocalModelDownload(); + unloadLocalModel(); + try { + new java.io.File(modelPath()).delete(); + new java.io.File(`${modelPath()}.part`).delete(); + } catch (error) { + console.error(`Local model delete failed: ${String(error)}`); + } + notifyStateChanged(); +} + +export function unloadLocalModel(): void { + if (unloadTimer) { + clearTimeout(unloadTimer); + unloadTimer = null; + } + runner?.unload(); +} + +function scheduleIdleUnload(): void { + if (unloadTimer) clearTimeout(unloadTimer); + unloadTimer = setTimeout(() => { + unloadTimer = null; + runner?.unload(); + }, IDLE_UNLOAD_MS); +} + +function ensureRunner(): any { + if (!runner) { + runner = new com.faceclaw.app.FaceclawLlamaRunner(); + } + return runner; +} + +const TOOL_CALL_OPEN = ""; +const TOOL_CALL_CLOSE = ""; + +/** + * Stream a completion from the on-phone model. Tool calls arrive inline as + * JSON blocks; they are hidden from the text stream and surfaced + * as tool_use content blocks in the final result, mirroring the cloud + * providers' contract (stopReason "tool_use" when the model called a tool). + */ +export function streamLocalQwen(options: LlmStreamOptions): LlmStreamHandle { + let settled = false; + let raw = ""; + let emitted = ""; + + const fail = (message: string) => { + if (settled) return; + settled = true; + scheduleIdleUnload(); + options.onError(message); + }; + + if (!global.isAndroid) { + setTimeout(() => fail("The on-phone model is only available on Android"), 0); + return { cancel: () => {} }; + } + if (!isLocalModelReady()) { + setTimeout(() => fail("The on-phone model isn't downloaded yet (see Settings)"), 0); + return { cancel: () => {} }; + } + if (unloadTimer) { + clearTimeout(unloadTimer); + unloadTimer = null; + } + + const prompt = buildQwenPrompt(options.system, options.messages, options.tools); + const grammar = options.tools?.length ? buildToolCallGrammar(options.tools) : null; + + const listener = new com.faceclaw.app.FaceclawLlamaListener({ + onToken: (piece: string) => { + if (settled) return; + raw += String(piece); + const visible = visibleText(raw); + if (visible.length > emitted.length) { + const delta = visible.slice(emitted.length); + emitted = visible; + options.onTextDelta?.(delta, emitted); + } + }, + onDone: (nativeStopReason: string) => { + if (settled) return; + settled = true; + scheduleIdleUnload(); + const { text, content, toolCount } = assembleContent(raw); + const stopReason = + toolCount > 0 ? "tool_use" : String(nativeStopReason) === "length" ? "max_tokens" : "end_turn"; + options.onDone({ text, content, stopReason }); + }, + onError: (message: string) => fail(String(message)), + }); + + try { + ensureRunner().generate( + modelPath(), + N_CTX, + N_THREADS, + prompt, + grammar ?? "", + options.maxTokens ?? DEFAULT_MAX_TOKENS, + TEMPERATURE, + TOP_P, + TOP_K, + listener, + ); + } catch (error) { + setTimeout(() => fail(`On-phone model failed: ${String((error as Error)?.message ?? error)}`), 0); + return { cancel: () => {} }; + } + + return { + cancel: () => { + settled = true; + scheduleIdleUnload(); + try { + runner?.cancel(); + } catch { + // ignore + } + }, + }; +} + +/** + * The streamable portion of the raw output: text outside completed + * blocks, holding back any trailing partial marker so a tool + * call never leaks into the visible stream. Monotonic under appends. + */ +function visibleText(raw: string): string { + let out = ""; + let i = 0; + for (;;) { + const start = raw.indexOf(TOOL_CALL_OPEN, i); + if (start === -1) { + const tail = raw.slice(i); + out += tail.slice(0, tail.length - partialSuffixLength(tail, TOOL_CALL_OPEN)); + return out; + } + out += raw.slice(i, start); + const end = raw.indexOf(TOOL_CALL_CLOSE, start); + if (end === -1) return out; // inside an unfinished tool call + i = end + TOOL_CALL_CLOSE.length; + } +} + +/** Length of the longest proper prefix of marker that text ends with. */ +function partialSuffixLength(text: string, marker: string): number { + const max = Math.min(text.length, marker.length - 1); + for (let k = max; k > 0; k--) { + if (text.endsWith(marker.slice(0, k))) return k; + } + return 0; +} + +/** Split the finished raw output into text + tool_use content blocks. */ +function assembleContent(raw: string): { text: string; content: LlmContentBlock[]; toolCount: number } { + const content: LlmContentBlock[] = []; + let text = ""; + let toolCount = 0; + let i = 0; + for (;;) { + const start = raw.indexOf(TOOL_CALL_OPEN, i); + if (start === -1) { + text += raw.slice(i); + break; + } + text += raw.slice(i, start); + const end = raw.indexOf(TOOL_CALL_CLOSE, start); + if (end === -1) break; // truncated call (length stop); drop it + const body = raw.slice(start + TOOL_CALL_OPEN.length, end).trim(); + try { + const parsed = JSON.parse(body); + if (parsed && typeof parsed.name === "string") { + toolCount++; + content.push({ + type: "tool_use", + id: `local_${Date.now()}_${++toolUseCounter}`, + name: parsed.name, + input: parsed.arguments ?? {}, + }); + } + } catch { + console.error(`Local model emitted unparseable tool call: ${body.slice(0, 200)}`); + } + i = end + TOOL_CALL_CLOSE.length; + } + text = text.trim(); + if (text) content.unshift({ type: "text", text }); + return { text, content, toolCount }; +} diff --git a/app/prompts.ts b/app/prompts.ts index 9d2ae343..d9a2b062 100644 --- a/app/prompts.ts +++ b/app/prompts.ts @@ -15,10 +15,17 @@ export const ASSISTANT_SYSTEM_PROMPT_BASE = [ /** Full assistant system prompt: the base plus the current device context. */ export function buildAssistantSystemPrompt(ctx: AssistantContext): string { - return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`; + return `${ASSISTANT_SYSTEM_PROMPT_BASE}\n\n${describeAssistantContext(ctx)}`; } -function describeContext(ctx: AssistantContext): string { +/** + * The volatile device context on its own. The local provider appends this to + * the user message instead of the system prompt: the system prompt + tool + * declarations are the bulk of the prompt, and keeping them byte-stable lets + * the on-phone model reuse its KV-cache prefix across turns instead of + * re-prefilling ~2-3k tokens because the clock changed. + */ +export function describeAssistantContext(ctx: AssistantContext): string { const parts = [`Current time: ${ctx.localTime}.`]; parts.push(`The glasses display is currently ${ctx.screenOn ? "on" : "off"}.`); if (ctx.foregroundApp) { diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 3cf738da..333c04d2 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -19,6 +19,7 @@ import { assistantModelProvider, type AssistantModel, } from "~/assistant/models"; +import { isLocalModelReady } from "../native/llama"; import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu"; import { DashboardInputEvent, Layer, type LayerContext } from "./layers"; import { GrayImage } from "~/graphics/image"; @@ -357,7 +358,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({ export type AssistantBackendKind = "direct" | "external"; const assistantBackendLabels: Record = { - direct: "On-phone (API key)", + direct: "On-phone", external: "My own agent (bridge)", }; @@ -369,7 +370,7 @@ export const assistantBackendSetting = new ConfigSettingEnum assistantBackendLabels[value] ?? value, description: - "Who answers assistant queries: an LLM called directly from the phone (needs an API key), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.", + "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.", }); export const assistantBridgeHostSetting = new ConfigSettingString({ @@ -468,9 +469,11 @@ export const assistantModelSetting = new ConfigSettingEnum({ const provider = assistantModelProvider(value); if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0; if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0; + if (provider === "local") return !isLocalModelReady(); return false; }, - description: "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set.", + description: + "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.", }); export const mapboxApiKeySetting = new ConfigSettingString({ diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 05d72e78..2f16fa93 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -3,8 +3,17 @@ import { getDefaultSmallFont } from "../../graphics/bdffont"; import type { GrayImage } from "../../graphics/image"; import { getDashboardLogo } from "../../graphics/logo"; import { wrapText } from "../../graphics/textwrap"; +import { + cancelLocalModelDownload, + deleteLocalModel, + LOCAL_MODEL, + localModelState, + onLocalModelStateChanged, + startLocalModelDownload, +} from "../../native/llama"; import { TextViewerLayer } from "../../apps/files/text-viewer"; -import type { MenuItem } from "../menu"; +import type { LayerContext } from "../layers"; +import { drawRightValueMenuItem, openModalMenu, type MenuItem } from "../menu"; import { shell } from "../shell/shell"; import { anthropicApiKeySetting, @@ -88,6 +97,7 @@ function settingsSections(): SettingsSection[] { // On-phone LLM loop vs the user's own agent via the bridge plugin. enumSettingMenuItem(assistantBackendSetting), enumSettingMenuItem(assistantModelSetting), + localModelMenuItem(), // When on, a wakeword utterance goes straight to the assistant with no // Send/Type menu step. toggleSettingMenuItem(assistantSkipConfirmationSetting), @@ -161,6 +171,77 @@ function settingsSections(): SettingsSection[] { ]; } +const LOCAL_MODEL_GB = `${(LOCAL_MODEL.sizeBytes / 1e9).toFixed(1)}GB`; + +// While a download is running, re-render on progress updates so the row's +// percentage stays live; the watch tears itself down when the download ends. +let localModelRenderUnsub: (() => void) | null = null; + +function watchLocalModelDownload(ctx: LayerContext): void { + localModelRenderUnsub?.(); + localModelRenderUnsub = onLocalModelStateChanged((state) => { + ctx.actions.requestRender(); + if (state.status !== "downloading") { + localModelRenderUnsub?.(); + localModelRenderUnsub = null; + } + }); +} + +function localModelStatusText(): string { + const state = localModelState(); + if (state.status === "ready") return "downloaded"; + if (state.status === "downloading") { + const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0; + return `${pct}% of ${LOCAL_MODEL_GB}`; + } + return "not downloaded"; +} + +/** Download/cancel/delete management for the on-phone assistant model. */ +function localModelMenuItem(): MenuItem { + return { + label: "On-phone model", + description: + `${LOCAL_MODEL.label} (${LOCAL_MODEL_GB} download over Wi-Fi recommended). ` + + "Answers assistant queries on the phone itself, with no API key or cloud service. " + + "Slower and simpler than the cloud models, but free and private. " + + "Used automatically when no API key is set. An interrupted download resumes where it left off.", + onSelect: (ctx) => { + const state = localModelState(); + const action: MenuItem = + state.status === "downloading" + ? { + label: "Cancel download", + onSelect: (innerCtx) => { + cancelLocalModelDownload(); + innerCtx.stack.pop(); + }, + } + : state.status === "ready" + ? { + label: "Delete model", + onSelect: (innerCtx) => { + deleteLocalModel(); + innerCtx.stack.pop(); + }, + } + : { + label: `Download (${LOCAL_MODEL_GB})`, + onSelect: (innerCtx) => { + startLocalModelDownload(); + watchLocalModelDownload(innerCtx); + innerCtx.stack.pop(); + }, + }; + openModalMenu(ctx, "On-phone model", [action], 0); + }, + render: ({ image, x, y, width }) => { + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", localModelStatusText()); + }, + }; +} + /** A row that opens one of the project docs (copied into the bundle under * about/ by webpack.config.js) in the paged text viewer. */ function bundledDocMenuItem(fileName: string, label: string): MenuItem { diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 90914d6c..87bcee21 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -801,7 +801,7 @@ class Shell { this.showAlert( assistantBackendSetting.get() === "external" ? "Configure the agent bridge host and token in Settings." - : "Set an API key for the selected assistant model in Settings.", + : "Set an API key or download the on-phone model in Settings.", ); return; }