Add support for a local model (Qwen3)

This commit is contained in:
James Babcock committed 2026-08-09 23:52:21 -07:00
1 parent 6103c00ca3
commit 6f45888627
16 files changed
+1532 -15

No files matched your search

@@ -0,0 +1,16 @@
package com.faceclaw.app;
/**
* Callbacks for one on-phone LLM generation. Like the other bridge listeners,
* implementations are typically created on the NativeScript JS thread and
* FaceclawLlamaRunner posts these callbacks back to that thread's Looper.
*/
public interface FaceclawLlamaListener {
/** A piece of generated text (may be a partial word). */
void onToken(String piece);
/** Generation finished: "stop" (EOG), "length", or "cancelled". */
void onDone(String stopReason);
void onError(String message);
}
@@ -0,0 +1,122 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* On-phone LLM inference for the voice assistant (llama.cpp over JNI).
*
* All model work (load, generate, free) runs on a single background executor
* thread so calls are naturally serialized; listener callbacks are posted to
* the Looper of the thread that constructed this object (the JS thread), the
* same contract as FaceclawSseRequest. cancel() may be called from any thread
* and interrupts the in-flight generation at the next token boundary.
*
* The model stays loaded between generations (turn iterations reuse the KV
* cache prefix); the TypeScript side calls unload() on an idle timer to give
* the ~3GB back.
*/
public class FaceclawLlamaRunner {
private static final String TAG = "FaceclawLlamaRunner";
static {
System.loadLibrary("faceclaw_llama");
}
private final Handler callbackHandler;
private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> {
Thread thread = new Thread(runnable, "FaceclawLlama");
thread.setPriority(Thread.NORM_PRIORITY);
return thread;
});
// Mutated only on the executor thread; volatile so cancel() can read it.
private volatile long handle;
private String loadedPath;
public FaceclawLlamaRunner() {
Looper looper = Looper.myLooper();
callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
}
public boolean isModelLoaded() {
return handle != 0;
}
/**
* Generate a completion, loading (or swapping) the model first if needed.
* grammar may be null/empty for unconstrained generation.
*/
public void generate(final String modelPath, final int nCtx, final int nThreads,
final String prompt, final String grammar, final int maxTokens,
final float temperature, final float topP, final int topK,
final FaceclawLlamaListener listener) {
executor.execute(() -> {
try {
if (handle != 0 && !modelPath.equals(loadedPath)) {
nativeFree(handle);
handle = 0;
loadedPath = null;
}
if (handle == 0) {
long loaded = nativeLoadModel(modelPath, nCtx, nThreads);
if (loaded == 0) {
post(() -> listener.onError("Could not load the on-phone model"));
return;
}
handle = loaded;
loadedPath = modelPath;
}
nativeGenerate(handle, prompt, grammar == null ? "" : grammar,
maxTokens, temperature, topP, topK, new FaceclawLlamaListener() {
@Override public void onToken(final String piece) {
post(() -> listener.onToken(piece));
}
@Override public void onDone(final String stopReason) {
post(() -> listener.onDone(stopReason));
}
@Override public void onError(final String message) {
post(() -> listener.onError(message));
}
});
} catch (Throwable t) {
Log.e(TAG, "generation failed", t);
final String message = String.valueOf(t);
post(() -> listener.onError("On-phone model failed: " + message));
}
});
}
/** Interrupt the current generation (its listener gets onDone("cancelled")). */
public void cancel() {
long current = handle;
if (current != 0) {
nativeCancel(current);
}
}
/** Free the model and its KV cache. Safe to call when nothing is loaded. */
public void unload() {
executor.execute(() -> {
if (handle != 0) {
nativeFree(handle);
handle = 0;
loadedPath = null;
}
});
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
private static native long nativeLoadModel(String path, int nCtx, int nThreads);
private static native void nativeGenerate(long handle, String prompt, String grammar,
int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener);
private static native void nativeCancel(long handle);
private static native void nativeFree(long handle);
}
@@ -0,0 +1,192 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.concurrent.TimeUnit;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.ResponseBody;
/**
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
* restarts, verifies a pinned sha256 (hashed incrementally, including the
* resumed prefix), then renames into place. Listener callbacks are posted to
* the constructing thread's Looper, like the other Faceclaw bridges.
*/
public class FaceclawModelDownloader {
private static final String TAG = "FaceclawModelDl";
private static final long PROGRESS_INTERVAL_MS = 500;
private static volatile OkHttpClient sharedClient;
private final Handler callbackHandler;
private final String url;
private final File destFile;
private final File partFile;
private final String expectedSha256;
private final long expectedTotalBytes;
private final FaceclawModelDownloaderListener listener;
private volatile boolean cancelled;
private volatile Call call;
private Thread worker;
public FaceclawModelDownloader(String url, String destPath, String expectedSha256,
long expectedTotalBytes, FaceclawModelDownloaderListener listener) {
if (listener == null) throw new IllegalArgumentException("listener is required");
Looper looper = Looper.myLooper();
this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
this.url = url;
this.destFile = new File(destPath);
this.partFile = new File(destPath + ".part");
this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase();
this.expectedTotalBytes = expectedTotalBytes;
this.listener = listener;
}
private static OkHttpClient getClient() {
if (sharedClient == null) {
synchronized (FaceclawModelDownloader.class) {
if (sharedClient == null) {
sharedClient = new OkHttpClient.Builder()
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.build();
}
}
}
return sharedClient;
}
public void start() {
worker = new Thread(this::run, "FaceclawModelDl");
worker.start();
}
/** Stops the download but keeps the .part file so a later start() resumes. */
public void cancel() {
cancelled = true;
Call current = call;
if (current != null) current.cancel();
}
private void run() {
try {
if (destFile.exists() && destFile.length() > 0) {
post(() -> listener.onDone(destFile.getAbsolutePath()));
return;
}
destFile.getParentFile().mkdirs();
MessageDigest digest = MessageDigest.getInstance("SHA-256");
long offset = 0;
if (partFile.exists()) {
offset = hashExistingPrefix(digest);
if (cancelled) return;
}
Request.Builder builder = new Request.Builder().url(url);
if (offset > 0) builder.header("Range", "bytes=" + offset + "-");
call = getClient().newCall(builder.build());
try (Response response = call.execute()) {
ResponseBody body = response.body();
if (!response.isSuccessful() || body == null) {
postError("Model download failed: HTTP " + response.code());
return;
}
boolean resumed = response.code() == 206;
if (offset > 0 && !resumed) {
// Server ignored the Range header; start over.
offset = 0;
digest = MessageDigest.getInstance("SHA-256");
}
long total = expectedTotalBytes > 0 ? expectedTotalBytes
: (body.contentLength() > 0 ? offset + body.contentLength() : -1);
try (FileOutputStream out = new FileOutputStream(partFile, resumed);
InputStream in = body.byteStream()) {
byte[] buffer = new byte[1 << 16];
long done = offset;
long lastProgressAt = 0;
int read;
while ((read = in.read(buffer)) != -1) {
if (cancelled) return;
out.write(buffer, 0, read);
digest.update(buffer, 0, read);
done += read;
long now = System.currentTimeMillis();
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
lastProgressAt = now;
final long doneFinal = done;
final long totalFinal = total;
post(() -> listener.onProgress(doneFinal, totalFinal));
}
}
}
}
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
postError("Model download ended early (" + partFile.length() + " of "
+ expectedTotalBytes + " bytes); try again to resume");
return;
}
String actualSha = toHex(digest.digest());
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
partFile.delete();
postError("Model download was corrupted (checksum mismatch); download it again");
return;
}
if (!partFile.renameTo(destFile)) {
postError("Could not move the downloaded model into place");
return;
}
post(() -> listener.onDone(destFile.getAbsolutePath()));
} catch (Exception e) {
if (cancelled) return;
Log.w(TAG, "download failed", e);
postError("Model download failed: " + e);
}
}
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
private long hashExistingPrefix(MessageDigest digest) throws IOException {
long hashed = 0;
try (FileInputStream in = new FileInputStream(partFile)) {
byte[] buffer = new byte[1 << 16];
int read;
while ((read = in.read(buffer)) != -1) {
if (cancelled) return hashed;
digest.update(buffer, 0, read);
hashed += read;
}
}
return hashed;
}
private static String toHex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) sb.append(String.format("%02x", b));
return sb.toString();
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
private void postError(final String message) {
post(() -> listener.onError(message));
}
}
@@ -0,0 +1,11 @@
package com.faceclaw.app;
/** Progress callbacks for FaceclawModelDownloader, posted to the creating thread's Looper. */
public interface FaceclawModelDownloaderListener {
void onProgress(long bytesDownloaded, long totalBytes);
/** The file is fully downloaded, verified, and moved to its final path. */
void onDone(String path);
void onError(String message);
}
@@ -0,0 +1,31 @@
cmake_minimum_required(VERSION 3.22)
project(faceclaw_llama CXX C)
# llama.cpp source tree location is injected by the gradle task (it downloads
# and extracts a pinned release tag before configuring this project).
if(NOT DEFINED LLAMA_CPP_SOURCE_DIR)
message(FATAL_ERROR "Pass -DLLAMA_CPP_SOURCE_DIR=<path to llama.cpp checkout>")
endif()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
# Static llama/ggml folded into our single JNI .so. CPU only: Vulkan on Mali
# and mixed Adreno drivers is not worth the instability for short assistant
# turns. The -march baseline lives in the gradle invocation (armv8.2-a with
# dotprod+fp16: the floor for Tensor G2's X1/A78/A55, so no i8mm).
set(BUILD_SHARED_LIBS OFF)
set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE)
set(LLAMA_CURL OFF CACHE BOOL "" FORCE)
set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
set(GGML_BACKEND_DL OFF CACHE BOOL "" FORCE)
add_subdirectory(${LLAMA_CPP_SOURCE_DIR} llama.cpp-build EXCLUDE_FROM_ALL)
add_library(faceclaw_llama SHARED faceclaw_llama.cpp)
target_link_libraries(faceclaw_llama PRIVATE llama android log)
@@ -0,0 +1,391 @@
// JNI bridge for the on-phone assistant model (llama.cpp, CPU inference).
// Java surface: FaceclawLlamaRunner's nativeLoadModel / nativeGenerate /
// nativeCancel / nativeFree. One model+context per handle; the Java side
// serializes all calls except nativeCancel onto a single executor thread,
// so only the cancel flag needs to be thread-safe.
#include <jni.h>
#include <android/log.h>
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstring>
#include <mutex>
#include <string>
#include <vector>
#include "ggml-cpu.h"
#include "llama.h"
#define TAG "FaceclawLlama"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
namespace {
struct FcLlama {
llama_model * model = nullptr;
llama_context * ctx = nullptr;
ggml_threadpool * threadpool = nullptr;
const llama_vocab * vocab = nullptr;
int n_ctx = 0;
// Tokens currently materialized in the KV cache (prompt + generated),
// used to skip re-decoding the common prefix on the next call. Within a
// turn the next prompt extends the previous one, so this usually reduces
// prefill to just the new tool results.
std::vector<llama_token> cache_tokens;
std::atomic<bool> cancel{false};
};
void android_llama_log(ggml_log_level level, const char * text, void * /*user*/) {
if (level < GGML_LOG_LEVEL_WARN) return; // llama.cpp is chatty at INFO during load
__android_log_write(level >= GGML_LOG_LEVEL_ERROR ? ANDROID_LOG_ERROR : ANDROID_LOG_WARN, TAG, text);
}
void ensure_backend_init() {
static std::once_flag once;
std::call_once(once, [] {
llama_log_set(android_llama_log, nullptr);
llama_backend_init();
});
}
int64_t now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
}
// Big-core detection for big.LITTLE phones. Without explicit affinity the
// scheduler is free to put our compute threads on the little cores (in-order
// A55s on Tensor G2), which slows prefill by 5-10x; pin to the cores whose
// max frequency is within 75% of the fastest core's (on a Pixel 7/7a this
// selects the 2x X1 + 2x A78 and excludes the 4x A55).
std::vector<int> detect_fast_cpus() {
std::vector<long> freqs;
for (int cpu = 0; cpu < GGML_MAX_N_THREADS; cpu++) {
char path[128];
snprintf(path, sizeof(path), "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", cpu);
FILE * f = fopen(path, "r");
if (f == nullptr) break;
long freq = 0;
if (fscanf(f, "%ld", &freq) != 1) freq = 0;
fclose(f);
freqs.push_back(freq);
}
std::vector<int> fast;
long max_freq = 0;
for (long f : freqs) max_freq = std::max(max_freq, f);
if (max_freq <= 0) return fast;
for (size_t cpu = 0; cpu < freqs.size(); cpu++) {
if (freqs[cpu] * 4 >= max_freq * 3) fast.push_back((int) cpu);
}
// A uniform-frequency CPU has no little cores to avoid.
if (fast.size() == freqs.size()) fast.clear();
return fast;
}
std::string jstring_to_utf8(JNIEnv * env, jstring value) {
if (value == nullptr) return "";
const char * chars = env->GetStringUTFChars(value, nullptr);
std::string out = chars ? chars : "";
env->ReleaseStringUTFChars(value, chars);
return out;
}
// Callback shims into the Java listener (an in-process wrapper object that
// posts to the JS thread; these calls happen on the executor thread).
struct Listener {
JNIEnv * env;
jobject obj;
jmethodID on_token;
jmethodID on_done;
jmethodID on_error;
static Listener resolve(JNIEnv * env, jobject listener) {
jclass cls = env->GetObjectClass(listener);
return Listener{
env,
listener,
env->GetMethodID(cls, "onToken", "(Ljava/lang/String;)V"),
env->GetMethodID(cls, "onDone", "(Ljava/lang/String;)V"),
env->GetMethodID(cls, "onError", "(Ljava/lang/String;)V"),
};
}
void token(const std::string & piece) const { call(on_token, piece); }
void done(const char * reason) const { call(on_done, reason); }
void error(const std::string & message) const { call(on_error, message); }
private:
void call(jmethodID method, const std::string & text) const {
jstring jtext = env->NewStringUTF(text.c_str());
env->CallVoidMethod(obj, method, jtext);
env->DeleteLocalRef(jtext);
if (env->ExceptionCheck()) {
env->ExceptionClear();
}
}
};
// NewStringUTF aborts on invalid (modified-)UTF-8, and token pieces can end
// mid-codepoint, so hold back any incomplete trailing UTF-8 sequence.
size_t utf8_complete_prefix(const std::string & data) {
size_t len = data.size();
size_t i = len;
// Find the start byte of the final codepoint (at most 3 bytes back).
while (i > 0 && len - i < 4 && (static_cast<unsigned char>(data[i - 1]) & 0xC0) == 0x80) i--;
if (i == 0) return len; // malformed; let it through rather than stall
unsigned char start = static_cast<unsigned char>(data[i - 1]);
size_t expected;
if (start < 0x80) expected = 1;
else if ((start & 0xE0) == 0xC0) expected = 2;
else if ((start & 0xF0) == 0xE0) expected = 3;
else if ((start & 0xF8) == 0xF0) expected = 4;
else return len; // malformed start byte; pass through
size_t have = len - (i - 1);
return have >= expected ? len : i - 1;
}
std::vector<llama_token> tokenize(const llama_vocab * vocab, const std::string & text, bool add_special) {
int n = -llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), nullptr, 0, add_special, true);
if (n <= 0) return {};
std::vector<llama_token> tokens(n);
int written = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), n, add_special, true);
if (written < 0) return {};
tokens.resize(written);
return tokens;
}
std::string token_piece(const llama_vocab * vocab, llama_token token) {
char buf[256];
int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true);
if (n <= 0) return "";
return std::string(buf, n);
}
} // namespace
extern "C" JNIEXPORT jlong JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeLoadModel(
JNIEnv * env, jclass, jstring jpath, jint n_ctx, jint n_threads) {
ensure_backend_init();
const std::string path = jstring_to_utf8(env, jpath);
const int64_t t_start = now_ms();
// Cap threads to the big-core count and pin them there (see
// detect_fast_cpus): little cores would otherwise both drag the barrier
// and invite the scheduler to keep us off the fast cores entirely.
const std::vector<int> fast_cpus = detect_fast_cpus();
int threads = (int) n_threads;
if (!fast_cpus.empty()) {
threads = std::min(threads, (int) fast_cpus.size());
}
llama_model_params mparams = llama_model_default_params();
mparams.n_gpu_layers = 0;
llama_model * model = llama_model_load_from_file(path.c_str(), mparams);
if (model == nullptr) {
LOGW("model load failed: %s", path.c_str());
return 0;
}
const int64_t t_model = now_ms();
llama_context_params cparams = llama_context_default_params();
cparams.n_ctx = (uint32_t) n_ctx;
cparams.n_batch = 512;
cparams.n_threads = threads;
cparams.n_threads_batch = threads;
llama_context * ctx = llama_init_from_model(model, cparams);
if (ctx == nullptr) {
LOGW("context init failed");
llama_model_free(model);
return 0;
}
ggml_threadpool * threadpool = nullptr;
if (!fast_cpus.empty()) {
ggml_threadpool_params tpp = ggml_threadpool_params_default(threads);
for (size_t i = 0; i < fast_cpus.size() && fast_cpus[i] < GGML_MAX_N_THREADS; i++) {
tpp.cpumask[fast_cpus[i]] = true;
}
tpp.strict_cpu = true;
threadpool = ggml_threadpool_new(&tpp);
if (threadpool != nullptr) {
llama_attach_threadpool(ctx, threadpool, threadpool);
std::string cpus;
for (int cpu : fast_cpus) cpus += std::to_string(cpu) + " ";
LOGI("pinned %d threads to fast cpus: %s", threads, cpus.c_str());
} else {
LOGW("threadpool creation failed; using default scheduling");
}
}
auto * handle = new FcLlama();
handle->model = model;
handle->ctx = ctx;
handle->threadpool = threadpool;
handle->vocab = llama_model_get_vocab(model);
handle->n_ctx = (int) llama_n_ctx(ctx);
LOGI("model loaded: %s (n_ctx=%d, threads=%d, model_load_ms=%lld, ctx_init_ms=%lld)",
path.c_str(), handle->n_ctx, threads,
(long long) (t_model - t_start), (long long) (now_ms() - t_model));
return reinterpret_cast<jlong>(handle);
}
extern "C" JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeFree(JNIEnv *, jclass, jlong jhandle) {
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
if (handle == nullptr) return;
llama_free(handle->ctx);
llama_model_free(handle->model);
if (handle->threadpool != nullptr) {
ggml_threadpool_free(handle->threadpool);
}
delete handle;
LOGI("model freed");
}
extern "C" JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeCancel(JNIEnv *, jclass, jlong jhandle) {
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
if (handle != nullptr) handle->cancel.store(true);
}
extern "C" JNIEXPORT void JNICALL
Java_com_faceclaw_app_FaceclawLlamaRunner_nativeGenerate(
JNIEnv * env, jclass, jlong jhandle, jstring jprompt, jstring jgrammar,
jint max_tokens, jfloat temperature, jfloat top_p, jint top_k,
jobject jlistener) {
auto * handle = reinterpret_cast<FcLlama *>(jhandle);
Listener listener = Listener::resolve(env, jlistener);
if (handle == nullptr) {
listener.error("Model is not loaded");
return;
}
handle->cancel.store(false);
const std::string prompt = jstring_to_utf8(env, jprompt);
const std::string grammar = jstring_to_utf8(env, jgrammar);
std::vector<llama_token> tokens = tokenize(handle->vocab, prompt, true);
if (tokens.empty()) {
listener.error("Prompt tokenization failed");
return;
}
if ((int) tokens.size() + 16 > handle->n_ctx) {
listener.error("Conversation too long for the on-phone model");
return;
}
const int max_gen = std::min((int) max_tokens, handle->n_ctx - (int) tokens.size());
// Reuse the KV prefix shared with the previous call; drop the rest. Keep
// at least the final prompt token out of the reused prefix so this decode
// produces logits for it.
size_t common = 0;
while (common < tokens.size() && common < handle->cache_tokens.size() &&
tokens[common] == handle->cache_tokens[common]) {
common++;
}
if (common == tokens.size()) common = tokens.size() - 1;
llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) common, -1);
handle->cache_tokens.assign(tokens.begin(), tokens.begin() + common);
// Sampler chain per generation: Qwen3 instruct posture (temp 0.7, top-p
// 0.8, top-k 20) plus an optional lazy grammar that activates at the
// <tool_call> special token and forces a schema-valid call.
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
if (!grammar.empty()) {
llama_sampler * grammar_sampler = nullptr;
std::vector<llama_token> trigger = tokenize(handle->vocab, "<tool_call>", false);
if (trigger.size() == 1) {
grammar_sampler = llama_sampler_init_grammar_lazy_patterns(
handle->vocab, grammar.c_str(), "root", nullptr, 0, trigger.data(), 1);
} else {
static const char * patterns[] = {"(<tool_call>[\\s\\S]*)"};
grammar_sampler = llama_sampler_init_grammar_lazy_patterns(
handle->vocab, grammar.c_str(), "root", patterns, 1, nullptr, 0);
}
if (grammar_sampler != nullptr) {
llama_sampler_chain_add(chain, grammar_sampler);
} else {
LOGW("tool grammar failed to parse; generating unconstrained");
}
}
llama_sampler_chain_add(chain, llama_sampler_init_top_k(top_k));
llama_sampler_chain_add(chain, llama_sampler_init_top_p(top_p, 1));
llama_sampler_chain_add(chain, llama_sampler_init_temp(temperature));
llama_sampler_chain_add(chain, llama_sampler_init_dist(LLAMA_DEFAULT_SEED));
const auto fail = [&](const std::string & message) {
llama_sampler_free(chain);
// The KV state past the reused prefix is now unknown; drop it so the
// next call starts from a consistent cache.
llama_memory_seq_rm(llama_get_memory(handle->ctx), 0, (llama_pos) handle->cache_tokens.size(), -1);
listener.error(message);
};
// Prefill in n_batch chunks.
const int64_t t_prefill_start = now_ms();
const int n_batch = (int) llama_n_batch(handle->ctx);
for (size_t i = common; i < tokens.size(); i += n_batch) {
if (handle->cancel.load()) {
llama_sampler_free(chain);
listener.done("cancelled");
return;
}
const int chunk = (int) std::min((size_t) n_batch, tokens.size() - i);
if (llama_decode(handle->ctx, llama_batch_get_one(tokens.data() + i, chunk)) != 0) {
fail("Prompt evaluation failed");
return;
}
handle->cache_tokens.insert(handle->cache_tokens.end(), tokens.begin() + i, tokens.begin() + i + chunk);
}
// Decode loop: sample, stream, feed back.
const int64_t t_decode_start = now_ms();
const size_t n_prefilled = tokens.size() - common;
int n_decoded = 0;
std::string pending_utf8;
const char * stop_reason = "length";
for (int i = 0; i < max_gen; i++) {
if (handle->cancel.load()) {
stop_reason = "cancelled";
break;
}
llama_token token = llama_sampler_sample(chain, handle->ctx, -1);
if (llama_vocab_is_eog(handle->vocab, token)) {
stop_reason = "stop";
break;
}
pending_utf8 += token_piece(handle->vocab, token);
size_t emit = utf8_complete_prefix(pending_utf8);
if (emit > 0) {
listener.token(pending_utf8.substr(0, emit));
pending_utf8.erase(0, emit);
}
if (llama_decode(handle->ctx, llama_batch_get_one(&token, 1)) != 0) {
fail("Token evaluation failed");
return;
}
handle->cache_tokens.push_back(token);
n_decoded++;
}
const int64_t t_end = now_ms();
const int64_t prefill_ms = t_decode_start - t_prefill_start;
const int64_t decode_ms = t_end - t_decode_start;
LOGI("timing: prompt=%zu reused=%zu prefill=%zu tok in %lld ms (%.1f tok/s), decode=%d tok in %lld ms (%.1f tok/s)",
tokens.size(), common, n_prefilled, (long long) prefill_ms,
prefill_ms > 0 ? 1000.0 * (double) n_prefilled / (double) prefill_ms : 0.0,
n_decoded, (long long) decode_ms,
decode_ms > 0 ? 1000.0 * (double) n_decoded / (double) decode_ms : 0.0);
if (!pending_utf8.empty()) {
size_t emit = utf8_complete_prefix(pending_utf8);
if (emit > 0) listener.token(pending_utf8.substr(0, emit));
}
llama_sampler_free(chain);
listener.done(stop_reason);
}