Add support for a local model (Qwen3)

This commit is contained in:
James Babcock committed 2026-08-09 23:52:21 -07:00
1 parent 6103c00ca3
commit 6f45888627
16 files changed
+1532 -15

No files matched your search

@@ -0,0 +1,16 @@
package com.faceclaw.app;
/**
* Callbacks for one on-phone LLM generation. Like the other bridge listeners,
* implementations are typically created on the NativeScript JS thread and
* FaceclawLlamaRunner posts these callbacks back to that thread's Looper.
*/
public interface FaceclawLlamaListener {
/** A piece of generated text (may be a partial word). */
void onToken(String piece);
/** Generation finished: "stop" (EOG), "length", or "cancelled". */
void onDone(String stopReason);
void onError(String message);
}
@@ -0,0 +1,122 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* On-phone LLM inference for the voice assistant (llama.cpp over JNI).
*
* All model work (load, generate, free) runs on a single background executor
* thread so calls are naturally serialized; listener callbacks are posted to
* the Looper of the thread that constructed this object (the JS thread), the
* same contract as FaceclawSseRequest. cancel() may be called from any thread
* and interrupts the in-flight generation at the next token boundary.
*
* The model stays loaded between generations (turn iterations reuse the KV
* cache prefix); the TypeScript side calls unload() on an idle timer to give
* the ~3GB back.
*/
public class FaceclawLlamaRunner {
private static final String TAG = "FaceclawLlamaRunner";
static {
System.loadLibrary("faceclaw_llama");
}
private final Handler callbackHandler;
private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> {
Thread thread = new Thread(runnable, "FaceclawLlama");
thread.setPriority(Thread.NORM_PRIORITY);
return thread;
});
// Mutated only on the executor thread; volatile so cancel() can read it.
private volatile long handle;
private String loadedPath;
public FaceclawLlamaRunner() {
Looper looper = Looper.myLooper();
callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
}
public boolean isModelLoaded() {
return handle != 0;
}
/**
* Generate a completion, loading (or swapping) the model first if needed.
* grammar may be null/empty for unconstrained generation.
*/
public void generate(final String modelPath, final int nCtx, final int nThreads,
final String prompt, final String grammar, final int maxTokens,
final float temperature, final float topP, final int topK,
final FaceclawLlamaListener listener) {
executor.execute(() -> {
try {
if (handle != 0 && !modelPath.equals(loadedPath)) {
nativeFree(handle);
handle = 0;
loadedPath = null;
}
if (handle == 0) {
long loaded = nativeLoadModel(modelPath, nCtx, nThreads);
if (loaded == 0) {
post(() -> listener.onError("Could not load the on-phone model"));
return;
}
handle = loaded;
loadedPath = modelPath;
}
nativeGenerate(handle, prompt, grammar == null ? "" : grammar,
maxTokens, temperature, topP, topK, new FaceclawLlamaListener() {
@Override public void onToken(final String piece) {
post(() -> listener.onToken(piece));
}
@Override public void onDone(final String stopReason) {
post(() -> listener.onDone(stopReason));
}
@Override public void onError(final String message) {
post(() -> listener.onError(message));
}
});
} catch (Throwable t) {
Log.e(TAG, "generation failed", t);
final String message = String.valueOf(t);
post(() -> listener.onError("On-phone model failed: " + message));
}
});
}
/** Interrupt the current generation (its listener gets onDone("cancelled")). */
public void cancel() {
long current = handle;
if (current != 0) {
nativeCancel(current);
}
}
/** Free the model and its KV cache. Safe to call when nothing is loaded. */
public void unload() {
executor.execute(() -> {
if (handle != 0) {
nativeFree(handle);
handle = 0;
loadedPath = null;
}
});
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
private static native long nativeLoadModel(String path, int nCtx, int nThreads);
private static native void nativeGenerate(long handle, String prompt, String grammar,
int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener);
private static native void nativeCancel(long handle);
private static native void nativeFree(long handle);
}
@@ -0,0 +1,192 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.concurrent.TimeUnit;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.ResponseBody;
/**
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
* restarts, verifies a pinned sha256 (hashed incrementally, including the
* resumed prefix), then renames into place. Listener callbacks are posted to
* the constructing thread's Looper, like the other Faceclaw bridges.
*/
public class FaceclawModelDownloader {
private static final String TAG = "FaceclawModelDl";
private static final long PROGRESS_INTERVAL_MS = 500;
private static volatile OkHttpClient sharedClient;
private final Handler callbackHandler;
private final String url;
private final File destFile;
private final File partFile;
private final String expectedSha256;
private final long expectedTotalBytes;
private final FaceclawModelDownloaderListener listener;
private volatile boolean cancelled;
private volatile Call call;
private Thread worker;
public FaceclawModelDownloader(String url, String destPath, String expectedSha256,
long expectedTotalBytes, FaceclawModelDownloaderListener listener) {
if (listener == null) throw new IllegalArgumentException("listener is required");
Looper looper = Looper.myLooper();
this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
this.url = url;
this.destFile = new File(destPath);
this.partFile = new File(destPath + ".part");
this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase();
this.expectedTotalBytes = expectedTotalBytes;
this.listener = listener;
}
private static OkHttpClient getClient() {
if (sharedClient == null) {
synchronized (FaceclawModelDownloader.class) {
if (sharedClient == null) {
sharedClient = new OkHttpClient.Builder()
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.build();
}
}
}
return sharedClient;
}
public void start() {
worker = new Thread(this::run, "FaceclawModelDl");
worker.start();
}
/** Stops the download but keeps the .part file so a later start() resumes. */
public void cancel() {
cancelled = true;
Call current = call;
if (current != null) current.cancel();
}
private void run() {
try {
if (destFile.exists() && destFile.length() > 0) {
post(() -> listener.onDone(destFile.getAbsolutePath()));
return;
}
destFile.getParentFile().mkdirs();
MessageDigest digest = MessageDigest.getInstance("SHA-256");
long offset = 0;
if (partFile.exists()) {
offset = hashExistingPrefix(digest);
if (cancelled) return;
}
Request.Builder builder = new Request.Builder().url(url);
if (offset > 0) builder.header("Range", "bytes=" + offset + "-");
call = getClient().newCall(builder.build());
try (Response response = call.execute()) {
ResponseBody body = response.body();
if (!response.isSuccessful() || body == null) {
postError("Model download failed: HTTP " + response.code());
return;
}
boolean resumed = response.code() == 206;
if (offset > 0 && !resumed) {
// Server ignored the Range header; start over.
offset = 0;
digest = MessageDigest.getInstance("SHA-256");
}
long total = expectedTotalBytes > 0 ? expectedTotalBytes
: (body.contentLength() > 0 ? offset + body.contentLength() : -1);
try (FileOutputStream out = new FileOutputStream(partFile, resumed);
InputStream in = body.byteStream()) {
byte[] buffer = new byte[1 << 16];
long done = offset;
long lastProgressAt = 0;
int read;
while ((read = in.read(buffer)) != -1) {
if (cancelled) return;
out.write(buffer, 0, read);
digest.update(buffer, 0, read);
done += read;
long now = System.currentTimeMillis();
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
lastProgressAt = now;
final long doneFinal = done;
final long totalFinal = total;
post(() -> listener.onProgress(doneFinal, totalFinal));
}
}
}
}
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
postError("Model download ended early (" + partFile.length() + " of "
+ expectedTotalBytes + " bytes); try again to resume");
return;
}
String actualSha = toHex(digest.digest());
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
partFile.delete();
postError("Model download was corrupted (checksum mismatch); download it again");
return;
}
if (!partFile.renameTo(destFile)) {
postError("Could not move the downloaded model into place");
return;
}
post(() -> listener.onDone(destFile.getAbsolutePath()));
} catch (Exception e) {
if (cancelled) return;
Log.w(TAG, "download failed", e);
postError("Model download failed: " + e);
}
}
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
private long hashExistingPrefix(MessageDigest digest) throws IOException {
long hashed = 0;
try (FileInputStream in = new FileInputStream(partFile)) {
byte[] buffer = new byte[1 << 16];
int read;
while ((read = in.read(buffer)) != -1) {
if (cancelled) return hashed;
digest.update(buffer, 0, read);
hashed += read;
}
}
return hashed;
}
private static String toHex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) sb.append(String.format("%02x", b));
return sb.toString();
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
private void postError(final String message) {
post(() -> listener.onError(message));
}
}
@@ -0,0 +1,11 @@
package com.faceclaw.app;
/** Progress callbacks for FaceclawModelDownloader, posted to the creating thread's Looper. */
public interface FaceclawModelDownloaderListener {
void onProgress(long bytesDownloaded, long totalBytes);
/** The file is fully downloaded, verified, and moved to its final path. */
void onDone(String path);
void onError(String message);
}