forked from Blendman974/faceclaw
Add support for a local model (Qwen3)
This commit is contained in:
1 parent
6103c00ca3
commit
6f45888627
16 files changed
+1532
-15
No files matched your search
@@ -0,0 +1,16 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/**
|
||||
* Callbacks for one on-phone LLM generation. Like the other bridge listeners,
|
||||
* implementations are typically created on the NativeScript JS thread and
|
||||
* FaceclawLlamaRunner posts these callbacks back to that thread's Looper.
|
||||
*/
|
||||
public interface FaceclawLlamaListener {
|
||||
/** A piece of generated text (may be a partial word). */
|
||||
void onToken(String piece);
|
||||
|
||||
/** Generation finished: "stop" (EOG), "length", or "cancelled". */
|
||||
void onDone(String stopReason);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
/**
|
||||
* On-phone LLM inference for the voice assistant (llama.cpp over JNI).
|
||||
*
|
||||
* All model work (load, generate, free) runs on a single background executor
|
||||
* thread so calls are naturally serialized; listener callbacks are posted to
|
||||
* the Looper of the thread that constructed this object (the JS thread), the
|
||||
* same contract as FaceclawSseRequest. cancel() may be called from any thread
|
||||
* and interrupts the in-flight generation at the next token boundary.
|
||||
*
|
||||
* The model stays loaded between generations (turn iterations reuse the KV
|
||||
* cache prefix); the TypeScript side calls unload() on an idle timer to give
|
||||
* the ~3GB back.
|
||||
*/
|
||||
public class FaceclawLlamaRunner {
|
||||
private static final String TAG = "FaceclawLlamaRunner";
|
||||
|
||||
static {
|
||||
System.loadLibrary("faceclaw_llama");
|
||||
}
|
||||
|
||||
private final Handler callbackHandler;
|
||||
private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "FaceclawLlama");
|
||||
thread.setPriority(Thread.NORM_PRIORITY);
|
||||
return thread;
|
||||
});
|
||||
|
||||
// Mutated only on the executor thread; volatile so cancel() can read it.
|
||||
private volatile long handle;
|
||||
private String loadedPath;
|
||||
|
||||
public FaceclawLlamaRunner() {
|
||||
Looper looper = Looper.myLooper();
|
||||
callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
|
||||
}
|
||||
|
||||
public boolean isModelLoaded() {
|
||||
return handle != 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a completion, loading (or swapping) the model first if needed.
|
||||
* grammar may be null/empty for unconstrained generation.
|
||||
*/
|
||||
public void generate(final String modelPath, final int nCtx, final int nThreads,
|
||||
final String prompt, final String grammar, final int maxTokens,
|
||||
final float temperature, final float topP, final int topK,
|
||||
final FaceclawLlamaListener listener) {
|
||||
executor.execute(() -> {
|
||||
try {
|
||||
if (handle != 0 && !modelPath.equals(loadedPath)) {
|
||||
nativeFree(handle);
|
||||
handle = 0;
|
||||
loadedPath = null;
|
||||
}
|
||||
if (handle == 0) {
|
||||
long loaded = nativeLoadModel(modelPath, nCtx, nThreads);
|
||||
if (loaded == 0) {
|
||||
post(() -> listener.onError("Could not load the on-phone model"));
|
||||
return;
|
||||
}
|
||||
handle = loaded;
|
||||
loadedPath = modelPath;
|
||||
}
|
||||
nativeGenerate(handle, prompt, grammar == null ? "" : grammar,
|
||||
maxTokens, temperature, topP, topK, new FaceclawLlamaListener() {
|
||||
@Override public void onToken(final String piece) {
|
||||
post(() -> listener.onToken(piece));
|
||||
}
|
||||
@Override public void onDone(final String stopReason) {
|
||||
post(() -> listener.onDone(stopReason));
|
||||
}
|
||||
@Override public void onError(final String message) {
|
||||
post(() -> listener.onError(message));
|
||||
}
|
||||
});
|
||||
} catch (Throwable t) {
|
||||
Log.e(TAG, "generation failed", t);
|
||||
final String message = String.valueOf(t);
|
||||
post(() -> listener.onError("On-phone model failed: " + message));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Interrupt the current generation (its listener gets onDone("cancelled")). */
|
||||
public void cancel() {
|
||||
long current = handle;
|
||||
if (current != 0) {
|
||||
nativeCancel(current);
|
||||
}
|
||||
}
|
||||
|
||||
/** Free the model and its KV cache. Safe to call when nothing is loaded. */
|
||||
public void unload() {
|
||||
executor.execute(() -> {
|
||||
if (handle != 0) {
|
||||
nativeFree(handle);
|
||||
handle = 0;
|
||||
loadedPath = null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void post(Runnable runnable) {
|
||||
callbackHandler.post(runnable);
|
||||
}
|
||||
|
||||
private static native long nativeLoadModel(String path, int nCtx, int nThreads);
|
||||
private static native void nativeGenerate(long handle, String prompt, String grammar,
|
||||
int maxTokens, float temperature, float topP, int topK, FaceclawLlamaListener listener);
|
||||
private static native void nativeCancel(long handle);
|
||||
private static native void nativeFree(long handle);
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import okhttp3.Call;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import okhttp3.ResponseBody;
|
||||
|
||||
/**
|
||||
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
|
||||
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
|
||||
* restarts, verifies a pinned sha256 (hashed incrementally, including the
|
||||
* resumed prefix), then renames into place. Listener callbacks are posted to
|
||||
* the constructing thread's Looper, like the other Faceclaw bridges.
|
||||
*/
|
||||
public class FaceclawModelDownloader {
|
||||
private static final String TAG = "FaceclawModelDl";
|
||||
private static final long PROGRESS_INTERVAL_MS = 500;
|
||||
|
||||
private static volatile OkHttpClient sharedClient;
|
||||
|
||||
private final Handler callbackHandler;
|
||||
private final String url;
|
||||
private final File destFile;
|
||||
private final File partFile;
|
||||
private final String expectedSha256;
|
||||
private final long expectedTotalBytes;
|
||||
private final FaceclawModelDownloaderListener listener;
|
||||
|
||||
private volatile boolean cancelled;
|
||||
private volatile Call call;
|
||||
private Thread worker;
|
||||
|
||||
public FaceclawModelDownloader(String url, String destPath, String expectedSha256,
|
||||
long expectedTotalBytes, FaceclawModelDownloaderListener listener) {
|
||||
if (listener == null) throw new IllegalArgumentException("listener is required");
|
||||
Looper looper = Looper.myLooper();
|
||||
this.callbackHandler = new Handler(looper != null ? looper : Looper.getMainLooper());
|
||||
this.url = url;
|
||||
this.destFile = new File(destPath);
|
||||
this.partFile = new File(destPath + ".part");
|
||||
this.expectedSha256 = expectedSha256 == null ? "" : expectedSha256.toLowerCase();
|
||||
this.expectedTotalBytes = expectedTotalBytes;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
private static OkHttpClient getClient() {
|
||||
if (sharedClient == null) {
|
||||
synchronized (FaceclawModelDownloader.class) {
|
||||
if (sharedClient == null) {
|
||||
sharedClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
return sharedClient;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
worker = new Thread(this::run, "FaceclawModelDl");
|
||||
worker.start();
|
||||
}
|
||||
|
||||
/** Stops the download but keeps the .part file so a later start() resumes. */
|
||||
public void cancel() {
|
||||
cancelled = true;
|
||||
Call current = call;
|
||||
if (current != null) current.cancel();
|
||||
}
|
||||
|
||||
private void run() {
|
||||
try {
|
||||
if (destFile.exists() && destFile.length() > 0) {
|
||||
post(() -> listener.onDone(destFile.getAbsolutePath()));
|
||||
return;
|
||||
}
|
||||
destFile.getParentFile().mkdirs();
|
||||
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
long offset = 0;
|
||||
if (partFile.exists()) {
|
||||
offset = hashExistingPrefix(digest);
|
||||
if (cancelled) return;
|
||||
}
|
||||
|
||||
Request.Builder builder = new Request.Builder().url(url);
|
||||
if (offset > 0) builder.header("Range", "bytes=" + offset + "-");
|
||||
call = getClient().newCall(builder.build());
|
||||
|
||||
try (Response response = call.execute()) {
|
||||
ResponseBody body = response.body();
|
||||
if (!response.isSuccessful() || body == null) {
|
||||
postError("Model download failed: HTTP " + response.code());
|
||||
return;
|
||||
}
|
||||
boolean resumed = response.code() == 206;
|
||||
if (offset > 0 && !resumed) {
|
||||
// Server ignored the Range header; start over.
|
||||
offset = 0;
|
||||
digest = MessageDigest.getInstance("SHA-256");
|
||||
}
|
||||
long total = expectedTotalBytes > 0 ? expectedTotalBytes
|
||||
: (body.contentLength() > 0 ? offset + body.contentLength() : -1);
|
||||
|
||||
try (FileOutputStream out = new FileOutputStream(partFile, resumed);
|
||||
InputStream in = body.byteStream()) {
|
||||
byte[] buffer = new byte[1 << 16];
|
||||
long done = offset;
|
||||
long lastProgressAt = 0;
|
||||
int read;
|
||||
while ((read = in.read(buffer)) != -1) {
|
||||
if (cancelled) return;
|
||||
out.write(buffer, 0, read);
|
||||
digest.update(buffer, 0, read);
|
||||
done += read;
|
||||
long now = System.currentTimeMillis();
|
||||
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
|
||||
lastProgressAt = now;
|
||||
final long doneFinal = done;
|
||||
final long totalFinal = total;
|
||||
post(() -> listener.onProgress(doneFinal, totalFinal));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
|
||||
postError("Model download ended early (" + partFile.length() + " of "
|
||||
+ expectedTotalBytes + " bytes); try again to resume");
|
||||
return;
|
||||
}
|
||||
String actualSha = toHex(digest.digest());
|
||||
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
|
||||
partFile.delete();
|
||||
postError("Model download was corrupted (checksum mismatch); download it again");
|
||||
return;
|
||||
}
|
||||
if (!partFile.renameTo(destFile)) {
|
||||
postError("Could not move the downloaded model into place");
|
||||
return;
|
||||
}
|
||||
post(() -> listener.onDone(destFile.getAbsolutePath()));
|
||||
} catch (Exception e) {
|
||||
if (cancelled) return;
|
||||
Log.w(TAG, "download failed", e);
|
||||
postError("Model download failed: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
|
||||
private long hashExistingPrefix(MessageDigest digest) throws IOException {
|
||||
long hashed = 0;
|
||||
try (FileInputStream in = new FileInputStream(partFile)) {
|
||||
byte[] buffer = new byte[1 << 16];
|
||||
int read;
|
||||
while ((read = in.read(buffer)) != -1) {
|
||||
if (cancelled) return hashed;
|
||||
digest.update(buffer, 0, read);
|
||||
hashed += read;
|
||||
}
|
||||
}
|
||||
return hashed;
|
||||
}
|
||||
|
||||
private static String toHex(byte[] bytes) {
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) sb.append(String.format("%02x", b));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void post(Runnable runnable) {
|
||||
callbackHandler.post(runnable);
|
||||
}
|
||||
|
||||
private void postError(final String message) {
|
||||
post(() -> listener.onError(message));
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/** Progress callbacks for FaceclawModelDownloader, posted to the creating thread's Looper. */
|
||||
public interface FaceclawModelDownloaderListener {
|
||||
void onProgress(long bytesDownloaded, long totalBytes);
|
||||
|
||||
/** The file is fully downloaded, verified, and moved to its final path. */
|
||||
void onDone(String path);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
Reference in new issue
Block a user