Merge main into ios-port and update the iOS CFW transport

This commit is contained in:
James Babcock committed 2026-09-17 23:33:51 -07:00
commit e8e144f3f2
156 files changed
+10528 -1679

No files matched your search

@@ -27,15 +27,14 @@ import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
@SuppressLint("MissingPermission")
public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final String TAG = "FaceclawComm";
// The EvenHub image container is a memory carrier only. Its 576x288 geometry
// gives the firmware separate 165888-byte display and reconstruction
// allocations: CFW reuses the former for its 640x480 packed-4bpp shadow and
// leaves the latter wholly available for compressed incoming messages.
// Local metadata for custom-command bookkeeping, not an EvenHub container.
// Submitted frames supply pixel geometry; the stock layout only captures input.
private static final BleProtocol.ImageTileOptions DASHBOARD_TILE =
new BleProtocol.ImageTileOptions("img00", 10, 0, 0, 576, 288);
@@ -77,7 +76,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private volatile String lastFirmwareCapabilities = "";
private volatile Thread workerThread;
private volatile boolean running;
private volatile boolean userDisconnectRequested;
@@ -130,7 +128,14 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private int faceclawFramebufferControlGeneration;
private int faceclawFramebufferControlSentCount;
private int faceclawWakePendingNonce = -1;
private boolean cfwCleanupSupported;
/**
* The last firmware-info read said the glasses run Faceclaw's custom
* firmware. Gates the private modes (cleanup, texture cache, ...) so stock
* or third-party firmware never sees them; the TS side checks the actual
* revision and disconnects on a mismatch, so no per-feature gating is
* needed here.
*/
private boolean customFirmwareDetected;
private boolean cfwCleanupDelivered;
private int lastCfwCleanupAckMagic;
@@ -162,6 +167,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private long lastShutdownExitAtMs = 0;
private int headsetBattery = -1;
private int headsetCharging = -1;
private int ringBattery = -1;
private int ringCharging = -1;
// Silent mode: 1 = on, 0 = off, -1 = not yet known. See updateSilentModeLocked.
private int silentMode = -1;
private int wearState = -1;
@@ -176,6 +183,38 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private volatile FaceclawAudioPacketListener audioPacketListener;
private PowerManager.WakeLock g2ScreenWakeLock;
// BLE bandwidth benchmark (Developer app). Streams no-op image payloads
// (CFW mode 7 with an unused sub-op: parsed, acked, and discarded — stock
// firmware likewise ignores unknown image modes) for a fixed duration with
// a selectable message size and pipeline window, then reports throughput.
// While active, desired-frame sends are held back and heartbeats are
// satisfied by the benchmark's own acks, so the stream is the only image
// traffic. All state below is guarded by `lock`; results are read with
// getBandwidthBenchmarkStatus() and survive until the next run starts.
private boolean benchmarkActive;
private boolean benchmarkAborted;
private final Random benchmarkRandom = new Random();
private int benchmarkMessageSize;
private int benchmarkWindowSize;
private int benchmarkDurationMs;
private int benchmarkLinkMode;
private boolean benchmarkLinkPending;
private long benchmarkReadyAtMs;
private long benchmarkStartAtMs; // first benchmark write; 0 until then
private long benchmarkDeadlineAtMs; // start + duration; MAX_VALUE until first write
private long benchmarkLastAckAtMs;
private long benchmarkEndAtMs; // 0 while running; set when the run drains
private int benchmarkMessagesSent;
private int benchmarkMessagesAcked;
private int benchmarkTimeouts;
private long benchmarkPayloadBytesAcked;
private long benchmarkWireBytesAcked;
// A timed-out benchmark message aborts the run, but its already-in-flight
// peers still time out one by one; keep the window comfortably below
// MAX_CONSECUTIVE_ACK_TIMEOUTS so a dead run can't escalate into a
// transport-failure reconnect all by itself.
private static final int BENCHMARK_MAX_WINDOW = 6;
private final BroadcastReceiver phoneLockReceiver = new BroadcastReceiver() {
@Override public void onReceive(Context context, Intent intent) {
emitPhoneLockStateIfChanged(true);
@@ -226,11 +265,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
// firmware frees the cache with the fb lease, and after any resync the
// cheap safe assumption is an empty cache (glyphs re-upload lazily).
private final TextureCacheState textureCache = new TextureCacheState();
private boolean textureCacheSupported;
private boolean textureImagesSupported;
private boolean fwTextSupported;
private final ArrayDeque<OutboundMessage> pendingMessages = new ArrayDeque<>();
private final CfwTransport[] cfwTransports = { new CfwTransport(), new CfwTransport() };
private final ArrayDeque<OutboundMessage> inFlightMessages = new ArrayDeque<>();
private OutboundMessage prewrittenMessage;
private List<byte[]> prewrittenFrames = Collections.emptyList();
@@ -580,6 +617,200 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
interruptibleSleep.interrupt();
}
/**
* Start the BLE bandwidth benchmark: stream messageSize-byte no-op image
* payloads for durationMs, keeping up to windowSize messages awaiting ack
* at once, then leave the results for getBandwidthBenchmarkStatus().
* Returns false when a run is already active or the image path is not
* ready. The duration clock starts at the first benchmark write, so
* traffic already queued ahead of the run doesn't count against it.
*/
public boolean startBandwidthBenchmark(int messageSize, int windowSize, int durationMs) {
return startBandwidthBenchmarkWithLinkMode(messageSize, windowSize, durationMs, 0);
}
// 0: current link; 1: re-request HIGH; 2: request 2M; 3: both.
public boolean startBandwidthBenchmarkWithLinkMode(int messageSize, int windowSize,
int durationMs, int linkMode) {
synchronized (lock) {
if (benchmarkActive || !running || !sessionReady || !fixedLayoutCreated
|| shutdownRequested || chargingMode) {
logLine("skip bandwidth benchmark; already running or image path not ready");
return false;
}
benchmarkMessageSize = Math.max(2, Math.min(messageSize, ConnectionOptions.IMAGE_FRAGMENT_SIZE));
benchmarkWindowSize = Math.max(1, Math.min(windowSize, BENCHMARK_MAX_WINDOW));
benchmarkDurationMs = Math.max(1_000, durationMs);
benchmarkLinkMode = linkMode & 3;
benchmarkLinkPending = true;
benchmarkReadyAtMs = Long.MAX_VALUE;
benchmarkStartAtMs = 0;
benchmarkDeadlineAtMs = Long.MAX_VALUE;
benchmarkLastAckAtMs = 0;
benchmarkEndAtMs = 0;
benchmarkMessagesSent = 0;
benchmarkMessagesAcked = 0;
benchmarkTimeouts = 0;
benchmarkPayloadBytesAcked = 0;
benchmarkWireBytesAcked = 0;
benchmarkAborted = false;
benchmarkActive = true;
logLine("bandwidth benchmark start: size=" + benchmarkMessageSize
+ "B window=" + benchmarkWindowSize + " duration=" + benchmarkDurationMs
+ "ms linkMode=" + benchmarkLinkMode);
}
interruptibleSleep.interrupt();
return true;
}
/**
* Cancel an in-progress benchmark (benchmark page closed). Queued no-op
* messages are dropped; in-flight ones drain through their normal acks.
*/
public void cancelBandwidthBenchmark() {
synchronized (lock) {
if (!benchmarkActive) {
return;
}
clearMessagesOfKindLocked("bandwidth");
finishBenchmarkLocked(true, "cancelled");
}
interruptibleSleep.interrupt();
}
/** Status/results of the current or most recent benchmark run, as JSON. */
public String getBandwidthBenchmarkStatus() {
synchronized (lock) {
String state = benchmarkActive
? (benchmarkStartAtMs == 0 ? "starting" : "running")
: (benchmarkEndAtMs != 0 ? "done" : "idle");
long end = benchmarkActive ? SystemClock.elapsedRealtime() : benchmarkEndAtMs;
long elapsed = benchmarkStartAtMs == 0 ? 0 : Math.max(0, end - benchmarkStartAtMs);
try {
org.json.JSONObject status = new org.json.JSONObject();
status.put("state", state);
status.put("messageSize", benchmarkMessageSize);
status.put("windowSize", benchmarkWindowSize);
status.put("linkMode", benchmarkLinkMode);
status.put("elapsedMs", elapsed);
status.put("messagesSent", benchmarkMessagesSent);
status.put("messagesAcked", benchmarkMessagesAcked);
status.put("timeouts", benchmarkTimeouts);
status.put("payloadBytesAcked", benchmarkPayloadBytesAcked);
status.put("wireBytesAcked", benchmarkWireBytesAcked);
status.put("aborted", benchmarkAborted);
return status.toString();
} catch (org.json.JSONException e) {
return "{\"state\":\"idle\"}";
}
}
}
/** Pending + in-flight benchmark no-op messages. */
private int benchmarkOutstandingLocked() {
int count = 0;
for (OutboundMessage message : pendingMessages) {
if ("bandwidth".equals(message.kind)) count++;
}
for (OutboundMessage message : inFlightMessages) {
if ("bandwidth".equals(message.kind)) count++;
}
return count;
}
/**
* Keep the benchmark stream fed: top the pending queue up so the send
* window never starves, stop enqueueing once the run expires (or a message
* times out), and finish the run when the last outstanding message drains.
*/
private void maintainBenchmarkLocked(long now) {
if (!sessionReady || !fixedLayoutCreated || shutdownRequested) {
finishBenchmarkLocked(true, "session no longer ready");
return;
}
if (benchmarkLinkPending || now < benchmarkReadyAtMs) {
return;
}
if (benchmarkAborted || now >= benchmarkDeadlineAtMs) {
// The run is over: drop queued-but-unsent no-ops (sending them
// would stretch the run past its deadline) and finish once the
// in-flight tail has acked or timed out.
Iterator<OutboundMessage> pendingIterator = pendingMessages.iterator();
while (pendingIterator.hasNext()) {
OutboundMessage queued = pendingIterator.next();
if ("bandwidth".equals(queued.kind)) {
pendingIterator.remove();
magicPool.release(queued.sid, queued.magic, queued.label, "benchmark over");
}
}
if (benchmarkOutstandingLocked() == 0) {
finishBenchmarkLocked(false, "complete");
}
return;
}
// One more than the window so a fresh message is always ready to write
// the moment an ack frees a slot.
for (int outstanding = benchmarkOutstandingLocked(); outstanding <= benchmarkWindowSize; outstanding++) {
enqueueBenchmarkMessageLocked();
}
}
private void finishBenchmarkLocked(boolean aborted, String reason) {
if (!benchmarkActive) {
return;
}
benchmarkActive = false;
benchmarkAborted |= aborted;
// Prefer the last ack as the end time so drain lag after the deadline
// doesn't dilute the throughput figure.
benchmarkEndAtMs = benchmarkLastAckAtMs != 0 ? benchmarkLastAckAtMs : SystemClock.elapsedRealtime();
logLine("bandwidth benchmark " + (benchmarkAborted ? "aborted" : "finished") + " (" + reason + "): "
+ benchmarkMessagesAcked + "/" + benchmarkMessagesSent + " acked, "
+ benchmarkPayloadBytesAcked + "B payload, " + benchmarkTimeouts + " timeouts");
}
private void enqueueBenchmarkMessageLocked() {
// Fresh bytes per message: the transport's persistent compression history
// would compress even a random payload if we reused it across the run.
byte[] payload = new byte[benchmarkMessageSize];
benchmarkRandom.nextBytes(payload);
payload[0] = 7; // CFW diagnostic-control mode...
payload[1] = (byte) 0x7f; // ...with an unused sub-op: acked, no effect
OutboundMessage message = messageBuilder.imagePayload(
"bandwidth",
DASHBOARD_TILE,
nextMapSessionId(),
payload,
"bandwidth no-op " + payload.length + "B",
connectionOptions.sendImagesToLeft);
final int payloadBytes = payload.length;
// Logical custom-message bytes; excludes length tag and packet framing.
final int wireBytes = message.message.length;
message.onSent = () -> {
benchmarkMessagesSent++;
if (benchmarkStartAtMs == 0) {
benchmarkStartAtMs = SystemClock.elapsedRealtime();
benchmarkDeadlineAtMs = benchmarkStartAtMs + benchmarkDurationMs;
}
};
message.onAck = () -> {
long ackedAtMs = SystemClock.elapsedRealtime();
benchmarkMessagesAcked++;
benchmarkPayloadBytesAcked += payloadBytes;
benchmarkWireBytesAcked += wireBytes;
benchmarkLastAckAtMs = ackedAtMs;
// No-op payloads ride the image path, so the firmware resets its
// heartbeat timer on them just like real image messages.
lastHeartbeatAckedAtMs = ackedAtMs;
};
message.onTimeout = () -> {
benchmarkTimeouts++;
benchmarkAborted = true;
logLine("bandwidth benchmark ack timeout");
};
pendingMessages.addLast(message);
}
public void addImuListener(FaceclawImuListener listener) {
if (listener != null) {
imuListeners.add(listener);
@@ -592,11 +823,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
public String getFirmwareCapabilities() {
return lastFirmwareCapabilities;
}
public void addAmbientLightListener(FaceclawAmbientLightListener listener) {
if (listener != null) {
ambientLightListeners.add(listener);
@@ -1100,7 +1326,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if (cfwCleanupDelivered) {
return true;
}
if (!cfwCleanupSupported || !running || !sessionReady
if (!customFirmwareDetected || !running || !sessionReady
|| shutdownRequested || !fixedLayoutCreated) {
logLine("skip CFW cleanup; mode 11 unavailable or image path not ready");
return false;
@@ -1432,6 +1658,33 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(uuid)) {
return;
}
if (data.length >= 7 && (data[6] & 255) == CfwTransport.SID) {
CfwTransport.Ack[] acks = CfwTransport.parseAcks(data);
if (acks == null) return;
synchronized (lock) {
lastIncomingAtMs = SystemClock.elapsedRealtime();
for (CfwTransport.Ack ack : acks) {
for (OutboundMessage message : inFlightMessages) {
String ingress = message.isLeftArmMessage ? leftAddress : rightAddress;
if (message.sid == CfwTransport.SID && address.equalsIgnoreCase(ingress) && message.magic == ack.streamId) {
message.acceptCfwAck(ack);
Log.i(TAG, "CFW " + (ack.nack ? "NACK" : "ACK") + " id=" + ack.streamId
+ " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2] & 255)
+ " redundant=" + (ack != acks[0])
+ " size=" + ack.size + " crc=" + ack.checksum
+ " expected=" + message.message.length + "/" + message.cfwChecksum
+ " ackedLenses=" + message.cfwAckLenses
+ " ageMs=" + (lastIncomingAtMs - message.sentAtMs));
message.ackPayload = Arrays.copyOf(data, data.length);
break;
}
}
}
drainCfwAcknowledgementsLocked();
}
interruptibleSleep.interrupt();
return;
}
Log.d(TAG, "onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.length);
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
int decodedWearState = BleProtocol.parseWearState(frame);
@@ -1461,17 +1714,44 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
);
faceclawWakeNotification = true;
lastConnectionOrInputAtMs = lastIncomingAtMs;
// The event type says which gesture woke the glasses: TS
// gives a head-up the Glanceboard and a double tap the
// regular UI. The claim handshake is the same for both.
boolean headUp = BleProtocol.parseFaceclawWakeEventCode(frame.pb)
== BleProtocol.FACECLAW_WAKE_EVENT_HEAD_UP;
event = new G2Event(
"display-wake",
"",
BleProtocol.EVENT_DOUBLE_CLICK,
headUp ? BleProtocol.EVENT_HEAD_UP : BleProtocol.EVENT_DOUBLE_CLICK,
0,
0
);
logLine("claimed deferred dashboard wake nonce=" + wakeNonce);
logLine("claimed deferred dashboard wake nonce=" + wakeNonce
+ (headUp ? " (head-up)" : ""));
}
}
if (!faceclawWakeNotification
&& shutdownRequested
&& frame.ok
&& frame.sid == BleProtocol.SID_UI_SETTING
&& address.equalsIgnoreCase(rightAddress)) {
// CFW idle-gesture forwarding (firmware revision 2+): with no
// EvenHub page the stock display thread drops taps, long
// presses and releases; the CFW reports them on the settings
// sid while our wake lease is held. Deliver them as the
// sys-events a live page would have produced, so TS treats a
// sleep-time tap or hold exactly like one during soft sleep
// (the Glanceboard).
BleProtocol.FaceclawGestureEvent gesture = BleProtocol.parseFaceclawGestureEvent(frame.pb);
if (gesture != null) {
lastConnectionOrInputAtMs = lastIncomingAtMs;
event = new G2Event("sys-event", "", gesture.eventType, gesture.eventSource, 0);
logLine("idle gesture forwarded by CFW: type=" + gesture.eventType
+ " source=" + gesture.eventSource);
}
}
if (!faceclawWakeNotification
&& event == null
&& shutdownRequested
&& address.equalsIgnoreCase(rightAddress)
&& BleProtocol.isDisplayWakeStateChange(frame)) {
@@ -1490,6 +1770,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if (!faceclawWakeNotification
&& frame.ok
&& frame.sid == BleProtocol.SID_UI_SETTING) {
// Mode-17 query notifications; settings READs are handled by
// createBatteryQueryMessageLocked so headset/ring update together.
if (address.equalsIgnoreCase(rightAddress)
&& (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) {
BleProtocol.RingBatterySnapshot ring = BleProtocol.parseRingBattery(frame.pb);
if (ring != null) {
ringBattery = ring.battery;
ringCharging = ring.charging;
emitBatteryState(headsetBattery, headsetCharging);
}
}
// CFW mic status (field 104) rides both standalone pushes and
// settings read acks, from each temple on its own link.
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
@@ -1534,6 +1825,15 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
&& address.equalsIgnoreCase(rightAddress)
&& (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) {
event = G2Event.decode(frame);
if (event != null
&& "sys-event".equals(event.kind)
&& event.eventType == BleProtocol.EVENT_HEAD_UP) {
// CFW forwards the IMU head-up while our page is on screen
// (soft sleep). Surface it as the same wake-only input the
// deferred head-up wake produces from a dark display.
event = new G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, event.eventSource, 0);
logLine("head-up forwarded by CFW while page on screen");
}
if (event != null) {
// Pure IMU samples arrive continuously; don't let them count
// as user input (which would starve battery polling).
@@ -1703,6 +2003,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
audioCaptureActive = false;
faceclawWakePendingNonce = -1;
cfwCleanupDelivered = false;
for (CfwTransport transport : cfwTransports) transport.reset();
lastCfwCleanupAckMagic = 0;
lastFaceclawWakeLeaseQueuedAtMs = 0;
lastFaceclawFramebufferLeaseQueuedAtMs = 0;
@@ -1722,7 +2023,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
logLine("session ready");
synchronized (lock) {
// Query settings promptly on the first session so firmware
// version/capabilities (and battery) arrive without waiting for
// version/extension (and battery) arrive without waiting for
// the input-quiet battery poll. The settings response doubles as
// the firmware-compatibility check surfaced during onboarding.
if (!firmwareInfoQueried) {
@@ -1954,6 +2255,33 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private long driveSession() {
//Log.d(TAG, "driveSession called (pendingMessages.size=" + pendingMessages.size() + " inFlightMessages.size=" + inFlightMessages.size() + ")");
while (true) {
// Run on the sender thread, outside `lock`: Bluetooth API calls can
// wait on the global GATT lock, and callbacks need the session lock.
// Keep negotiation outside the timed stream. A fixed settling period
// is only an experiment boundary; HCI must confirm actual parameters.
int linkMode = -1;
String[] benchmarkAddresses = null;
synchronized (lock) {
if (benchmarkActive && benchmarkLinkPending) {
benchmarkLinkPending = false;
linkMode = benchmarkLinkMode;
benchmarkAddresses = new String[] {leftAddress, rightAddress};
}
}
if (benchmarkAddresses != null) {
for (String address : benchmarkAddresses) {
try {
bleManager.prepareBenchmarkLink(address, linkMode);
} catch (RuntimeException error) {
logLine("benchmark link request failed: " + safeMessage(error));
}
}
synchronized (lock) {
// Cancellation/new-run races leave the new run pending; its
// own preparation will replace this deadline before sending.
benchmarkReadyAtMs = SystemClock.elapsedRealtime() + 1_000;
}
}
OutboundMessage messageToWrite = null;
OutboundMessage messageToPrewrite = null;
long now = SystemClock.elapsedRealtime();
@@ -1961,6 +2289,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
maybeFinishNoChangeDesiredFrame();
synchronized (lock) {
drainCfwAcknowledgementsLocked();
if (cfwCleanupDelivered) {
/* Successful mode 11 must be the last Faceclaw write. Drop
* anything a late external producer attempted to enqueue
@@ -1988,11 +2317,36 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
if (!inFlightMessages.isEmpty()) {
OutboundMessage oldest = inFlightMessages.peekFirst();
List<OutboundMessage> replay = CfwMessageWindow.replayWindow(inFlightMessages, now);
if (!replay.isEmpty()) {
logLine("CFW recovery: replay " + replay.size()
+ " unresolved message(s) from id=" + replay.get(0).magic);
for (OutboundMessage candidate : replay) {
if (candidate.cfwRetries >= CfwMessageWindow.MAX_RETRIES) {
handleTransportFailure("CFW recovery retry limit");
return 0;
}
logLine("CFW replay id=" + candidate.magic + " label=" + candidate.label
+ " cause=" + (candidate.cfwRetryPending ? "NACK"
: candidate.cfwAckLenses == CfwTransport.BOTH ? "ordered tail" : "missing ACK")
+ " ackedLenses=" + candidate.cfwAckLenses
+ " ageMs=" + (now - candidate.sentAtMs));
inFlightMessages.remove(candidate);
magicPool.release(candidate.sid, candidate.magic, candidate.label, "CFW replay");
candidate.prepareCfwReplay(magicPool.allocate());
}
for (int i = replay.size() - 1; i >= 0; --i) pendingMessages.addFirst(replay.get(i));
return 0;
}
if (oldest != null && oldest.ackDeadlineAtMs <= now) {
Log.i(TAG, "message timed out: " + oldest.label);
inFlightMessages.removeFirst();
logLine("message timed out: " + oldest.label);
logLine("message timed out: " + oldest.label + " sid=" + oldest.sid
+ " id=" + oldest.magic + " ackedLenses=" + oldest.cfwAckLenses
+ " ageMs=" + (now - oldest.sentAtMs));
magicPool.release(oldest.sid, oldest.magic, oldest.label, "timeout");
if (oldest.sid == CfwTransport.SID)
cfwTransports[oldest.isLeftArmMessage ? 0 : 1].reset();
handleAckTimeoutLocked(oldest);
return 0;
}
@@ -2050,9 +2404,15 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
enqueueCompassControlLocked(false, wanted);
}
if (benchmarkActive) {
maintainBenchmarkLocked(now);
}
// Up to WINDOW_SIZE messages may be in flight at once (full
// pipelining); a slot frees when an ack arrives.
boolean windowHasRoom = inFlightMessages.size() < Math.max(1, connectionOptions.WINDOW_SIZE);
// pipelining); a slot frees when an ack arrives. An active
// bandwidth benchmark measures its own selected window instead.
boolean windowHasRoom = inFlightMessages.size()
< Math.max(1, benchmarkActive ? benchmarkWindowSize : connectionOptions.WINDOW_SIZE);
// A frame ready to send right now: don't inject a fresh
// heartbeat in front of it (the image's own ack resets the
// firmware heartbeat timer, so the heartbeat is redundant).
@@ -2063,11 +2423,15 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
// window unprotected and a heartbeat that came due there
// cost the frame a full ack round trip (measured 124ms on
// frame#232 of the 2026-08-20 02:27 capture).
// A benchmark run counts as image traffic here for the same
// reason: its acks reset the firmware heartbeat timer, so a
// fresh heartbeat in front of it is redundant.
boolean imageWaiting = !shutdownRequested && fixedLayoutCreated
&& !hasPendingOrInflightKindLocked("heartbeat")
&& now >= imageRetryAfterMs
&& (hasPendingImageLocked()
|| !getDesiredFingerprint().equals(lastEnqueuedFingerprint));
&& (benchmarkActive
|| (now >= imageRetryAfterMs
&& (hasPendingImageLocked()
|| !getDesiredFingerprint().equals(lastEnqueuedFingerprint))));
noteImageStallLocked(now, windowHasRoom);
if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) {
return ConnectionOptions.IDLE_SLEEP_MS;
@@ -2076,7 +2440,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty()) {
messageToWrite = pendingMessages.removeFirst();
Log.i(TAG, "sending pending message: " + messageToWrite.label);
} else if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
} else if (messageToPrewrite == null && !shutdownRequested && !benchmarkActive
&& fixedLayoutCreated
&& windowHasRoom && !hasPendingImageLocked()
&& now >= imageRetryAfterMs
&& !getDesiredFingerprint().equals(lastEnqueuedFingerprint)) {
@@ -2145,7 +2510,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
private boolean canPrewriteCandidate(OutboundMessage message) {
if (message == null || !message.isLeftArmMessage) {
if (message == null || message.sid == CfwTransport.SID || !message.isLeftArmMessage) {
return false;
}
if (!"image".equals(message.kind)) {
@@ -2240,6 +2605,11 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
frames = Collections.singletonList(prewrittenFrames.get(prewrittenFrames.size() - 1));
prewrittenMessage = null;
prewrittenFrames = Collections.emptyList();
} else if (message.sid == CfwTransport.SID) {
if (message.cfwRetries > 0) cfwTransports[message.isLeftArmMessage ? 0 : 1].reset();
frames = cfwTransports[message.isLeftArmMessage ? 0 : 1].encode(
message.message, message.magic, CfwTransport.BOTH,
bleManager.getNegotiatedMtu(writeAddress));
} else {
frames = BleProtocol.framePb(
message.message,
@@ -2267,6 +2637,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
if (!result && message.sid == CfwTransport.SID)
cfwTransports[message.isLeftArmMessage ? 0 : 1].reset();
return result;
}
@@ -2354,6 +2726,18 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
/** Also drain on the worker: a clear/timeout may have removed a blocking
* head since the last notification. Fully ACKed controls must never reach
* the generic timeout path just because no further BLE reply arrived. */
private void drainCfwAcknowledgementsLocked() {
while (true) {
OutboundMessage first = CfwMessageWindow.acknowledgedHead(inFlightMessages);
if (first == null) return;
lastAckAtMs = SystemClock.elapsedRealtime();
resolveAckLocked(first, first.ackPayload);
}
}
private void resolveAckLocked(int sid, int magic, byte[] pb) {
Iterator<OutboundMessage> iterator = inFlightMessages.iterator();
while (iterator.hasNext()) {
@@ -2375,6 +2759,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
message.onAck.run();
}
consecutiveAckTimeouts = 0;
// onAck may have just satisfied a waiter blocked on lock.wait() (e.g.
// awaitEvenHubSessionReady polling fixedLayoutCreated/displayedFingerprint
// after a create-layout or image ack). Without this, that waiter only
// notices on its own up-to-100ms poll tick, adding avoidable latency to
// every EvenHub wake. Always called with lock held (see call site).
lock.notifyAll();
}
private void logImageUpdateSendLandmarkLocked(OutboundMessage message) {
@@ -2432,10 +2822,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
private void enqueueCreateLayoutLocked() {
// New session/container: re-assert the firmware-debug-flags overlay once
// New session: re-assert the firmware-debug-flags overlay once
// the layout is ready (the mode-7 send is gated on this having reset).
firmwareDebugFlagsLastSent = -1;
OutboundMessage message = messageBuilder.createLayout(DASHBOARD_TILE);
OutboundMessage message = messageBuilder.createLayout();
message.onAck = () -> {
startupProbePending = false;
clearMessagesOfKindLocked("startup-text-probe");
@@ -2480,10 +2870,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
/**
* Send the CFW mode-7 diagnostic-flag control op to the dashboard container:
* Send the CFW mode-7 diagnostic-flag control op through the private stream:
* [7][2] to show the on-glasses debug-flag overlay, [7][1] to hide it. Uses the
* arbitrary-payload image path (no bmp/dedup/frame-timing interaction) and does
* nothing on stock firmware (which ignores unknown image modes).
* arbitrary-payload custom path (no bmp/dedup/frame-timing interaction).
*/
private void enqueueFirmwareDebugFlagsLocked() {
boolean show = firmwareDebugFlagsEnabled;
@@ -2567,7 +2956,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
// ink out of the baked deltas. Uploads (mode 12) ride ahead of the
// image message on the ordered transport. Falls through to the plain
// paths whenever the planner has nothing to draw.
if (textureCacheSupported && connectionOptions.TEXTURE_CACHE_FRAMES
if (customFirmwareDetected && connectionOptions.TEXTURE_CACHE_FRAMES
&& draws != null && draws.length > 0 && packed.length > 0) {
byte[] deltaBase = (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedPacked.length > 0
&& lastEnqueuedWidth == width && lastEnqueuedHeight == height)
@@ -2576,8 +2965,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
TexturePlanner.Result tex = TexturePlanner.plan(
deltaBase, packed, width, height, draws, textureCache,
nextImageFrameId,
connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS,
textureImagesSupported, fwTextSupported);
connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS);
if (tex != null) {
nextImageFrameId = tex.nextFid;
for (byte[] upload : tex.uploads) {
@@ -2585,7 +2973,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
BleImageOptimizer.TileImagePlan plan = new BleImageOptimizer.TileImagePlan(
0, DASHBOARD_TILE, packed, width, height, nextMapSessionId(), tex.payload);
plan.fragments = BleImageOptimizer.planImageFragments(plan.payload, ConnectionOptions.IMAGE_FRAGMENT_SIZE);
FrameTimings.getInstance().spanEnd(frameId, "texture-plan");
String texLog = "texture update " + (tex.fullFrame ? "full" : ("rects=" + tex.rectCount))
+ " glyphs=" + tex.drawnGlyphs + " runs=" + tex.runCount
@@ -2648,7 +3035,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
BleImageOptimizer.TileImagePlan plan = incrementalPayload != null
? new BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packed, width, height, nextMapSessionId(), incrementalPayload)
: new BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packed, width, height, nextMapSessionId());
plan.fragments = BleImageOptimizer.planImageFragments(plan.payload, ConnectionOptions.IMAGE_FRAGMENT_SIZE);
FrameTimings.getInstance().spanEnd(frameId, "compress-and-plan");
if (incrementalLog != null) {
FrameTimings.getInstance().log(frameId, incrementalLog);
@@ -2656,15 +3042,26 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId);
}
/** Shared tail of enqueueDesiredImageLocked: enqueue fragments, advance the delta base, log. */
/** Queue complete private commands, advance the delta base, and retain frame/ACK bookkeeping. */
private void finishEnqueueDesiredImageLocked(
BleImageOptimizer.TileImagePlan plan, String fingerprint, int paintMs, int frameId) {
int updateId = nextImageUpdateId++;
int messageCount = plan.fragments.size();
List<byte[]> commands = new ArrayList<>();
if (plan.payload.length <= CfwTransport.MAX_MESSAGE) {
commands.add(plan.payload);
} else {
// A noisy full frame can exceed the stream record limit. Repaint it
// with independently decodable RLE bands below the decoded-message limit.
commands.addAll(BleImageOptimizer.encodeFullFrameBands(
plan.packed, plan.width, plan.height, nextImageFrameId));
for (int i = 0; i < commands.size(); i++) {
nextImageFrameId = nextImageFrameId >= 0xfffe ? 1 : nextImageFrameId + 1;
}
}
int messageCount = commands.size();
imageUpdateStats.put(updateId, new BleImageOptimizer.ImageUpdateStats(paintMs, 1, frameId));
for (int i = 0; i < plan.fragments.size(); i++) {
BleProtocol.ImageFragment fragment = plan.fragments.get(i);
enqueueImageFragmentLocked(plan, fragment, fingerprint, updateId, i + 1, messageCount, true);
for (int i = 0; i < commands.size(); i++) {
enqueueCustomImageLocked(plan, commands.get(i), fingerprint, updateId, i + 1, messageCount);
}
// This frame is now the base for the next delta (it will be the firmware
// shadow once applied), even though it hasn't been acked yet — that is what
@@ -2704,16 +3101,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
logLine("queue " + message.label);
}
private void enqueueImageFragmentLocked(
private void enqueueCustomImageLocked(
BleImageOptimizer.TileImagePlan plan,
BleProtocol.ImageFragment fragment,
byte[] payload,
String fingerprint,
int updateId,
int messageNumber,
int messageCount,
boolean requestAck
int messageCount
) {
OutboundMessage message = messageBuilder.imageFragment(fragment, plan, requestAck, connectionOptions.sendImagesToLeft);
OutboundMessage message = messageBuilder.customMessage("image", payload,
"image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft);
message.setImageUpdatePosition(updateId, messageNumber, messageCount);
message.onAck = () -> {
imageRetryAfterMs = 0;
@@ -2777,6 +3174,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private OutboundMessage createBatteryQueryMessageLocked() {
OutboundMessage message = messageBuilder.batteryQuery();
message.onAck = () -> {
BleProtocol.RingBatterySnapshot ring = BleProtocol.parseRingBattery(message.ackPayload);
// Old firmware and missing/malformed extensions must clear any prior reading.
ringBattery = ring == null ? -1 : ring.battery;
ringCharging = ring == null ? -1 : ring.charging;
BleProtocol.BatterySnapshot snapshot = BleProtocol.parseSettingsBattery(message.ackPayload);
if (snapshot != null) {
headsetBattery = snapshot.battery;
@@ -2792,12 +3193,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
if (firmwareInfo != null) {
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
&& hasCapability(firmwareInfo.capabilities, "texstr14");
textureImagesSupported = hasCapability(firmwareInfo.capabilities, "teximg13");
fwTextSupported = hasCapability(firmwareInfo.capabilities, "font15");
customFirmwareDetected = firmwareInfo.isFaceclawFirmware();
emitFirmwareInfo(firmwareInfo);
}
};
@@ -3339,6 +3735,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
Log.e(TAG, "Transport failure: "+reason);
synchronized (lock) {
maybeEmitEvenAppConflictLocked(reason);
finishBenchmarkLocked(true, "transport failure");
sessionReady = false;
fixedLayoutCreated = false;
startupProbePending = false;
@@ -3361,6 +3758,8 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
private void resetSessionStateLocked() {
ringBattery = -1;
ringCharging = -1;
sessionReady = false;
shutdownRequested = false;
fixedLayoutCreated = false;
@@ -3388,6 +3787,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
lastFaceclawWakeLeaseQueuedAtMs = 0;
faceclawWakeControlSentCount = 0;
cfwCleanupDelivered = false;
for (CfwTransport transport : cfwTransports) transport.reset();
lastCfwCleanupAckMagic = 0;
wearState = -1;
displayedFingerprint = "";
@@ -3396,14 +3796,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
// the very cause of the session teardown that got us here.
}
private static boolean hasCapability(String capabilities, String token) {
if (capabilities == null || token == null || token.isEmpty()) return false;
for (String capability : capabilities.trim().split("\\s+")) {
if (token.equals(capability)) return true;
}
return false;
}
private void emitRingEvent(String kind, String containerName, int eventType, int eventSource, int systemExitReasonCode, int frameId) {
final FaceclawBleCommunicatorListener current = listener;
if (current == null) {
@@ -3528,13 +3920,15 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
private void emitBatteryState(int headsetBattery, int headsetCharging) {
final int reportedRingBattery = ringBattery;
final int reportedRingCharging = ringCharging;
final FaceclawBleCommunicatorListener current = listener;
if (current == null) {
return;
}
mainHandler.post(() -> {
try {
current.onBatteryState(headsetBattery, headsetCharging);
current.onBatteryState(headsetBattery, headsetCharging, reportedRingBattery, reportedRingCharging);
} catch (Throwable t) {
Log.w(TAG, "listener onBatteryState failed", t);
}
@@ -3548,7 +3942,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
mainHandler.post(() -> {
try {
current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.capabilities);
current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.extension);
} catch (Throwable t) {
Log.w(TAG, "listener onFirmwareInfo failed", t);
}
@@ -4,12 +4,12 @@ public interface FaceclawBleCommunicatorListener {
void onLog(String line);
void onStateChange(String phase, String status);
void onRingEvent(String kind, String containerName, int eventType, int eventSource, int systemExitReasonCode, int frameId);
void onBatteryState(int headsetBattery, int headsetCharging);
void onBatteryState(int headsetBattery, int headsetCharging, int ringBattery, int ringCharging);
void onSilentMode(boolean silent);
void onWearState(boolean wearing);
void onPhoneLockState(boolean locked);
void onEvenAppConflict(String message);
void onFrameMetrics(int paintMs, int transmitMs, int tileCount);
void onFrameFinished(int frameId, String outcome);
void onFirmwareInfo(String leftVersion, String rightVersion, String capabilities);
void onFirmwareInfo(String leftVersion, String rightVersion, String extension);
}
@@ -42,6 +42,7 @@ public class FaceclawBleManager {
private final ConcurrentHashMap<String, Integer> servicesStatuses = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, CountDownLatch> mtuLatches = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Integer> negotiatedMtus = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Integer> mtuStatuses = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, CountDownLatch> descriptorLatches = new ConcurrentHashMap<>();
@@ -99,6 +100,29 @@ public class FaceclawBleManager {
}
}
/**
* Android's bond state for this address (BluetoothDevice.BOND_NONE /
* BOND_BONDING / BOND_BONDED), or -1 when it cannot be determined.
* BOND_BONDING covers the whole OS pairing flow, including a pairing
* dialog that is still waiting for the user.
*/
public int getBondState(String address) {
if (address == null || address.trim().isEmpty()) {
return -1;
}
try {
BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address);
return device == null ? -1 : device.getBondState();
} catch (Throwable t) {
return -1;
}
}
/** Whether a GATT client for this address is open, i.e. no disconnect callback has arrived for it. */
public boolean isConnected(String address) {
return address != null && gattClients.containsKey(address);
}
public boolean connect(String address, int timeoutMs) {
if (address == null || address.trim().isEmpty()) {
throw new IllegalArgumentException("address is required");
@@ -142,6 +166,7 @@ public class FaceclawBleManager {
connectLatches.remove(address);
connectResults.remove(address);
gattClients.remove(address, gatt);
negotiatedMtus.remove(address);
gatt.disconnect();
gatt.close();
return false;
@@ -160,6 +185,28 @@ public class FaceclawBleManager {
}
}
/** One-shot benchmark preferences; success is established by callbacks/HCI, not submission. */
public void prepareBenchmarkLink(String address, int mode) {
synchronized (gattLock(address)) {
BluetoothGatt gatt = requireGatt(address);
if ((mode & 1) != 0) {
boolean accepted = gatt.requestConnectionPriority(BluetoothGatt.CONNECTION_PRIORITY_HIGH);
Log.i(TAG, "benchmark HIGH address=" + address + " submitted=" + accepted);
}
if ((mode & 2) != 0) {
Log.i(TAG, "benchmark request 2M address=" + address);
gatt.setPreferredPhy(BluetoothDevice.PHY_LE_2M_MASK,
BluetoothDevice.PHY_LE_2M_MASK, BluetoothDevice.PHY_OPTION_NO_PREFERRED);
}
gatt.readPhy();
}
}
public int getNegotiatedMtu(String address) {
Integer mtu = negotiatedMtus.get(address);
return mtu == null ? 23 : mtu;
}
public boolean requestMtu(String address, int mtu, int timeoutMs) {
synchronized (gattLock(address)) {
CountDownLatch latch = new CountDownLatch(1);
@@ -350,6 +397,7 @@ public class FaceclawBleManager {
Object gattLock = gattLock(address);
synchronized (gattLock) {
BluetoothGatt gatt = gattClients.remove(address);
negotiatedMtus.remove(address);
if (gatt == null) {
return;
}
@@ -423,6 +471,7 @@ public class FaceclawBleManager {
Object gattLock = gattLock(address);
synchronized (gattLock) {
gattClients.remove(address, gatt);
negotiatedMtus.remove(address);
gatt.close();
}
dispatchConnectionState(address, false);
@@ -442,6 +491,7 @@ public class FaceclawBleManager {
@Override
public void onMtuChanged(BluetoothGatt gatt, int mtu, int status) {
String address = gatt.getDevice().getAddress();
if (status == BluetoothGatt.GATT_SUCCESS) negotiatedMtus.put(address, mtu);
mtuStatuses.put(address, status);
CountDownLatch latch = mtuLatches.remove(address);
if (latch != null) {
@@ -451,7 +501,14 @@ public class FaceclawBleManager {
@Override
public void onPhyRead(BluetoothGatt gatt, int txPhy, int rxPhy, int status) {
Log.i(TAG, "onPhyRead: txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status);
Log.i(TAG, "onPhyRead: address=" + gatt.getDevice().getAddress()
+ " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status);
}
@Override
public void onPhyUpdate(BluetoothGatt gatt, int txPhy, int rxPhy, int status) {
Log.i(TAG, "onPhyUpdate: address=" + gatt.getDevice().getAddress()
+ " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status);
}
@Override
@@ -1,25 +1,41 @@
package com.faceclaw.app;
import android.bluetooth.BluetoothDevice;
import android.bluetooth.BluetoothGatt;
import android.content.Context;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Log;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* Connects to the glasses (stock-firmware compatible), reads the device-info /
* settings response, and reports the firmware versions plus the CFW capability
* string (empty on stock firmware). Used by onboarding to decide whether to
* settings response, and reports the firmware versions plus the firmware-
* extension string (empty on stock firmware). Used by onboarding to decide whether to
* flash. Reuses FaceclawBleManager + BleProtocol; owns its own connection and
* runs on a single worker thread. Shows nothing on the lens.
*/
public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
private static final String TAG = "FaceclawDeviceInfo";
private static final int QUERY_TIMEOUT_MS = 4_000;
/** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */
private static final int ARM_ATTEMPTS = 3;
private static final int ARM_RETRY_DELAY_MS = 1_000;
/**
* Upper bound on waiting for the auth success while Android reports the arm
* as BOND_BONDING — i.e. an OS pairing dialog may be sitting there waiting
* for the user, which can take a lot longer than the normal 30 s window.
*/
private static final int PAIRING_WAIT_CAP_MS = 90_000;
/** After a fresh bond, how long the firmware gets to notify success on its own before the request is re-sent. */
private static final int POST_BOND_GRACE_MS = 2_000;
private static final int POLL_MS = 250;
private final Context context;
private final String rightAddress;
@@ -36,6 +52,17 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
private byte[] awaitPb = null;
private CountDownLatch awaitLatch = null;
/**
* Security-auth wait, kept separate from the generic ack wait: only the
* firmware's SUCCESS result (empty result message) may satisfy it. The
* firmware answers the request as soon as it arrives, but with a
* non-success result until the link is encrypted; accepting that first
* reply as "the ack" is what let the probe move on to the other arm while
* Android's pairing dialog for this one was still up.
*/
private final Set<Integer> authMagics = new HashSet<>();
private CountDownLatch authLatch = null;
/**
* A sid-0x09 frame that carried firmware versions without matching the
* awaited magic — some firmware pushes the settings snapshot on a magic it
@@ -92,16 +119,52 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
return;
}
byte[] ack = probeArm(rightAddress, "right");
boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress);
// Bring up and authenticate BOTH arms before asking for versions.
// Each arm is its own peripheral with its own bond, so on a fresh
// phone each raises its own pairing prompt; completing both here
// means the flash step (and the app proper) find both bonds in
// place. Strictly one arm at a time: Android pairs with one device
// at a time, and a second arm connected while the first was still
// pairing has been seen to drop within a second and never pair
// (2026-09-11 logcat).
boolean rightAuthenticated = bringUpArm(rightAddress, "right");
boolean leftAuthenticated = false;
if (haveLeft && !cancelled) {
leftAuthenticated = bringUpArm(leftAddress, "left");
}
if (cancelled) {
emitError("Cancelled.");
return;
}
if (!bleManager.isConnected(rightAddress)) {
// The right lens may have dropped while the left one paired;
// it is bonded by now, so this comes back quickly.
emitLog("right lens disconnected while the left lens was set up; reconnecting");
rightAuthenticated = bringUpArm(rightAddress, "right");
if (cancelled) {
emitError("Cancelled.");
return;
}
}
byte[] ack = null;
String rightFailure = null;
try {
ack = queryArm(rightAddress, "right", rightAuthenticated);
} catch (IllegalStateException e) {
rightFailure = e.getMessage() == null ? e.toString() : e.getMessage();
emitLog("right-lens query failed: " + rightFailure);
}
// The right lens is the documented control endpoint, but a silent
// right lens has been observed on stock 2.2.9 even after a
// successful security auth — run the same probe against the left
// lens rather than giving up, and log which lens answered.
if (ack == null && !cancelled && !leftAddress.trim().isEmpty()
&& !leftAddress.equalsIgnoreCase(rightAddress)) {
if (ack == null && !cancelled && haveLeft) {
emitLog("right lens did not answer the settings query; probing the left lens");
try {
ack = probeArm(leftAddress, "left");
ack = queryArm(leftAddress, "left", leftAuthenticated);
} catch (Exception e) {
emitLog("left-lens probe failed: " + (e.getMessage() == null ? e.toString() : e.getMessage()));
}
@@ -111,17 +174,18 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
return;
}
if (ack == null) {
throw new IllegalStateException(
"no response to the device-info query on either lens — check `adb logcat -s "
throw new IllegalStateException(rightFailure != null
? rightFailure
: "no response to the device-info query on either lens — check `adb logcat -s "
+ TAG + "` for the frame trace");
}
BleProtocol.FirmwareInfo info = BleProtocol.parseSettingsFirmwareInfo(ack);
String left = info == null ? "" : info.leftVersion;
String right = info == null ? "" : info.rightVersion;
String caps = info == null ? "" : info.capabilities;
emitLog("device-info: L=" + left + " R=" + right + " caps=[" + caps + "]");
emitResult(left, right, caps);
String extension = info == null ? "" : info.extension;
emitLog("device-info: L=" + left + " R=" + right + " ext=[" + extension + "]");
emitResult(left, right, extension);
} catch (Exception e) {
String message = cancelled ? "Cancelled." : (e.getMessage() == null ? e.toString() : e.getMessage());
emitError(message);
@@ -134,34 +198,223 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
}
/**
* Connect + authenticate + prelude + settings read on one lens. Returns the
* settings ack protobuf, an unsolicited settings push that carried firmware
* versions, or null when the lens never answered the read. Throws on
* connect or prelude failure; the caller wraps the fallback lens's attempt
* Connect to one arm and complete its security-auth exchange, retrying
* when the link drops mid-pairing. Returns whether the auth success was
* seen (false = connected but unconfirmed, tolerated because the custom
* firmware's response to the exchange is not yet hardware-verified);
* throws when the arm cannot be brought up at all.
*/
private boolean bringUpArm(String address, String label) throws InterruptedException {
String lastFailure = null;
for (int attempt = 1; attempt <= ARM_ATTEMPTS && !cancelled; attempt++) {
if (attempt > 1) {
// A link Android dropped while it was still pairing comes back
// once the bond settles; reconnecting sooner just fails again.
waitForBondToSettle(address, label);
if (cancelled) {
break;
}
emitLog("retrying the " + label + " lens (attempt " + attempt + "/" + ARM_ATTEMPTS + "): " + lastFailure);
Thread.sleep(ARM_RETRY_DELAY_MS);
}
emitState("connecting", label);
try {
connectArm(address);
} catch (IllegalStateException e) {
lastFailure = e.getMessage() == null ? e.toString() : e.getMessage();
emitLog(label + " lens: " + lastFailure);
continue;
}
if (cancelled) {
break;
}
// Firmware 2.2.9 answers no queries until the security-auth
// exchange completes over an encrypted link; on a phone with no
// existing bond this is also what triggers SMP pairing (and its OS
// prompt), so it must come before the prelude and query.
emitState("authenticating", label);
AuthResult result = authenticate(address, label);
if (result == AuthResult.SUCCESS) {
return true;
}
if (result == AuthResult.UNCONFIRMED) {
return false;
}
lastFailure = "link dropped during authentication";
}
if (cancelled) {
return false;
}
throw new IllegalStateException("could not connect to the " + label + " lens (" + address + "): "
+ lastFailure + " — if Android showed a Bluetooth pairing request, accept it and try again");
}
private enum AuthResult { SUCCESS, UNCONFIRMED, LINK_DROPPED }
/**
* Send the sid-0x80 authentication request and wait for the firmware's
* SUCCESS notification, which only arrives once the link is encrypted.
* On an unbonded phone that means waiting through Android's pairing flow
* (BOND_BONDING, possibly with a dialog up), so the wait is extended while
* the OS reports pairing in progress and the request is re-sent once
* after a fresh bond in case the pre-pairing write was dropped. Bond
* state is polled rather than awaited via broadcast; 250 ms granularity
* is plenty here.
*/
private AuthResult authenticate(String address, String label) throws InterruptedException {
long start = SystemClock.elapsedRealtime();
long deadline = start + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS;
long hardDeadline = start + PAIRING_WAIT_CAP_MS;
CountDownLatch latch = new CountDownLatch(1);
synchronized (lock) {
authMagics.clear();
authLatch = latch;
}
int initialBond = bleManager.getBondState(address);
emitLog("security auth: " + label + " lens, Android bond state " + bondStateName(initialBond));
try {
int sends = 0;
boolean lastSendFailed = false;
long nextSendAt = start;
long bondedAt = -1;
boolean resentAfterBond = false;
boolean loggedBonding = false;
while (!cancelled) {
long now = SystemClock.elapsedRealtime();
int bond = bleManager.getBondState(address);
if (bond == BluetoothDevice.BOND_BONDING) {
// OS pairing in progress — the firmware will answer once the
// link is encrypted. Don't write, don't give up, don't move
// on to the other arm.
if (!loggedBonding) {
emitLog("Android is pairing with the " + label + " lens; waiting for the user to accept");
loggedBonding = true;
}
if (now >= hardDeadline) {
break;
}
} else {
if (bond == BluetoothDevice.BOND_BONDED && initialBond != BluetoothDevice.BOND_BONDED
&& bondedAt < 0) {
bondedAt = now;
emitLog("Android bond established with the " + label + " lens; waiting for the auth success");
deadline = Math.max(deadline, now + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS);
}
boolean wantSend = sends == 0
|| (lastSendFailed && sends < 3 && now >= nextSendAt)
|| (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= POST_BOND_GRACE_MS);
if (wantSend) {
if (bondedAt >= 0) {
resentAfterBond = true;
if (sends > 0) {
emitLog("re-sending the auth request after pairing (" + label + " lens)");
}
}
lastSendFailed = !sendAuthRequest(address);
sends++;
nextSendAt = SystemClock.elapsedRealtime() + ARM_RETRY_DELAY_MS;
if (lastSendFailed && !bleManager.isConnected(address)) {
return AuthResult.LINK_DROPPED;
}
}
if (now >= deadline) {
break;
}
}
if (latch.await(POLL_MS, TimeUnit.MILLISECONDS)) {
emitLog("security auth complete: " + label + " lens (" + address + ")");
return AuthResult.SUCCESS;
}
if (!bleManager.isConnected(address)) {
emitLog("security auth: " + label + " lens disconnected while waiting"
+ (bond == BluetoothDevice.BOND_BONDING ? " (Android was still pairing)" : ""));
return AuthResult.LINK_DROPPED;
}
}
if (cancelled) {
return AuthResult.UNCONFIRMED;
}
emitLog("security auth unconfirmed: " + label + " lens (" + address + "), Android bond state "
+ bondStateName(bleManager.getBondState(address)));
return AuthResult.UNCONFIRMED;
} finally {
synchronized (lock) {
authLatch = null;
authMagics.clear();
}
}
}
private boolean sendAuthRequest(String address) {
int magic = allocMagic();
int seq;
synchronized (lock) {
authMagics.add(magic);
seq = nextSeq++ & 0xff;
}
byte[] payload = BleProtocol.buildAuthenticationRequest(magic);
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, magic, seq, payload.length));
List<byte[]> frames = BleProtocol.framePb(payload, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, seq);
boolean written;
try {
written = bleManager.writeFrames(
address, BleProtocol.WRITE_CHAR_UUID, frames, ConnectionOptions.WRITE_TYPE, ConnectionOptions.WRITE_TIMEOUT_MS);
} catch (IllegalStateException e) {
// "Not connected" — the link went away underneath us.
written = false;
}
if (!written) {
emitLog("tx write FAILED sid=0x80 (" + address + ")");
}
return written;
}
/** Block while Android reports this arm as BOND_BONDING (pairing dialog up / SMP in flight). */
private void waitForBondToSettle(String address, String label) throws InterruptedException {
long cap = SystemClock.elapsedRealtime() + PAIRING_WAIT_CAP_MS;
boolean logged = false;
while (!cancelled && bleManager.getBondState(address) == BluetoothDevice.BOND_BONDING
&& SystemClock.elapsedRealtime() < cap) {
if (!logged) {
emitLog("waiting for Android to finish pairing with the " + label + " lens before reconnecting");
logged = true;
}
Thread.sleep(POLL_MS);
}
}
private static String bondStateName(int state) {
switch (state) {
case BluetoothDevice.BOND_NONE:
return "none";
case BluetoothDevice.BOND_BONDING:
return "bonding";
case BluetoothDevice.BOND_BONDED:
return "bonded";
default:
return "unknown";
}
}
/**
* Prelude + settings read on an already connected/authenticated lens.
* Returns the settings ack protobuf, an unsolicited settings push that
* carried firmware versions, or null when the lens never answered the read.
* Throws on prelude failure; the caller wraps the fallback lens's attempt
* so its failure cannot mask the primary lens's outcome.
*/
private byte[] probeArm(String address, String label) throws InterruptedException {
emitState("connecting", label);
connectArm(address);
if (cancelled) {
return null;
}
// Firmware 2.2.9 answers no queries until the security-auth exchange
// completes over an encrypted link; on a phone with no existing bond
// this is also what triggers SMP pairing (and its OS prompt), so it
// must come before the prelude and query. Soft: this probe also runs
// against the custom firmware, whose response to the exchange is not
// yet hardware-verified, so an unconfirmed auth falls through.
emitState("authenticating", label);
boolean authenticated = authenticate(address);
private byte[] queryArm(String address, String label, boolean authenticated) throws InterruptedException {
if (cancelled) {
return null;
}
emitState("querying", label);
// Session prelude, then a settings/device-info read (both arms'
// versions and the CFW capability string ride back in one response).
// versions and the firmware-extension string ride back in one response).
if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
@@ -191,15 +444,6 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
return null;
}
private boolean authenticate(String address) throws InterruptedException {
int magic = allocMagic();
byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH,
magic, BleProtocol.buildAuthenticationRequest(magic), ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS);
boolean success = ack != null && BleProtocol.isAuthenticationSuccess(ack, magic);
emitLog(success ? "security auth complete: " + address : "security auth unconfirmed: " + address);
return success;
}
private void connectArm(String address) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw new IllegalStateException("connect failed");
@@ -284,7 +528,32 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
unsolicitedSettingsPb = frame.pb;
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
return; // async event, not an ack
// Async event, not an ack. This includes the lens's own periodic
// sid-0x80 notifies (device-chosen magic, non-empty result).
return;
}
if (frame.sid == BleProtocol.SID_SECURITY_AUTH) {
boolean success = false;
boolean ours = false;
synchronized (lock) {
if (authLatch != null) {
ours = authMagics.contains(frame.msgSeq);
for (int magic : authMagics) {
if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) {
success = true;
authLatch.countDown();
break;
}
}
}
}
if (ours && !success) {
// Expected before the link is encrypted: the firmware answers
// the request straight away with a non-success result and
// sends the real success once pairing/encryption completes.
emitLog("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting");
}
return;
}
synchronized (lock) {
if (awaitLatch != null && frame.sid == awaitSid && frame.msgSeq == awaitMagic) {
@@ -329,11 +598,11 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
});
}
private void emitResult(String left, String right, String caps) {
private void emitResult(String left, String right, String extension) {
mainHandler.post(() -> {
FaceclawDeviceInfoProbeListener current = listener;
if (current != null) {
current.onResult(left, right, caps);
current.onResult(left, right, extension);
}
});
}
@@ -4,11 +4,11 @@ package com.faceclaw.app;
public interface FaceclawDeviceInfoProbeListener {
void onLog(String line);
/** Lifecycle: connecting, querying. */
/** Lifecycle: connecting/authenticating for each arm, then querying. */
void onState(String state, String detail);
/** Terminal success: firmware versions and the CFW capability string (may be empty). */
void onResult(String leftVersion, String rightVersion, String capabilities);
/** Terminal success: firmware versions and the firmware-extension string (empty on stock). */
void onResult(String leftVersion, String rightVersion, String extension);
/** Terminal failure (couldn't connect or read the version). */
void onError(String message);
@@ -3,6 +3,7 @@ package com.faceclaw.app;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.net.Uri;
import android.provider.Settings;
import android.util.Log;
@@ -47,19 +48,26 @@ public final class FaceclawEvenAppDetector {
return hasNotfication;
}
public static void openEvenAppSettings(Context context) {
/** Returns "opened", "not-installed", or "failed" for the phone UI. */
public static String openEvenAppSettings(Context context) {
if (context == null) {
return;
return "failed";
}
Context appContext = context.getApplicationContext();
Intent intent = new Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS);
intent.setData(Uri.parse("package:" + EVEN_PACKAGE_NAME));
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
try {
// Settings may launch successfully and then immediately close for a
// missing package, so check before starting the activity.
appContext.getPackageManager().getApplicationInfo(EVEN_PACKAGE_NAME, 0);
Intent intent = new Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS);
intent.setData(Uri.parse("package:" + EVEN_PACKAGE_NAME));
intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
appContext.startActivity(intent);
} catch (Throwable t) {
return "opened";
} catch (PackageManager.NameNotFoundException e) {
return "not-installed";
} catch (Exception t) {
Log.w(TAG, "failed to open Even app settings", t);
openNotificationAccessSettings(appContext);
return "failed";
}
}
@@ -22,7 +22,12 @@ public class FaceclawForegroundService extends Service {
public static final String ACTION_STOP = "com.faceclaw.app.action.STOP";
public static final String EXTRA_TEXT = "text";
private static final String CHANNEL_ID = "faceclaw-dashboard";
// Channel id was bumped from "faceclaw-dashboard" when the badge setting
// changed: Android freezes a channel's showBadge flag at creation, so the
// old channel (which let Samsung's launcher count the pinned notification
// as a red "1" badge) is deleted on upgrade rather than reused.
private static final String LEGACY_CHANNEL_ID = "faceclaw-dashboard";
private static final String CHANNEL_ID = "faceclaw-connection";
private static final int NOTIFICATION_ID = 4201;
@Override
@@ -43,7 +48,7 @@ public class FaceclawForegroundService extends Service {
ensureNotificationChannel();
Notification notification = buildNotification(
text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected"
text != null && !text.trim().isEmpty() ? text : "Connected to glasses"
);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
@@ -69,14 +74,18 @@ public class FaceclawForegroundService extends Service {
NotificationChannel channel = new NotificationChannel(
CHANNEL_ID,
"Faceclaw dashboard",
"Glasses connection",
NotificationManager.IMPORTANCE_LOW
);
channel.setDescription("Keeps the Faceclaw dashboard connected to the glasses.");
channel.setDescription("Keeps Faceclaw connected to the glasses.");
// The pinned status notification must not count toward the launcher
// icon's notification badge.
channel.setShowBadge(false);
NotificationManager manager = getSystemService(NotificationManager.class);
if (manager != null) {
manager.createNotificationChannel(channel);
manager.deleteNotificationChannel(LEGACY_CHANNEL_ID);
}
}
@@ -96,7 +105,7 @@ public class FaceclawForegroundService extends Service {
: new Notification.Builder(this);
return builder
.setContentTitle("faceclaw dashboard")
.setContentTitle("Faceclaw")
.setContentText(text)
.setSmallIcon(getApplicationInfo().icon)
.setContentIntent(contentIntent)
@@ -5,6 +5,7 @@ import android.content.Context;
import android.content.Intent;
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.database.ContentObserver;
import android.graphics.Bitmap;
import android.media.AudioManager;
import android.media.MediaDescription;
@@ -27,6 +28,8 @@ import java.util.HashSet;
import java.util.Set;
public class FaceclawMediaController {
private static final String ENABLED_NOTIFICATION_LISTENERS = "enabled_notification_listeners";
private final Context appContext;
private final Object lock = new Object();
private final Handler mainHandler = new Handler(Looper.getMainLooper());
@@ -70,9 +73,27 @@ public class FaceclawMediaController {
}
};
/**
* Fires when the user grants or revokes notification-listener access in
* system settings. Session access depends on that grant, so the sessions
* listener is (re)registered and the state re-emitted whenever it changes;
* otherwise a grant made after start() would never be noticed.
*/
private final ContentObserver notificationAccessObserver = new ContentObserver(mainHandler) {
@Override
public void onChange(boolean selfChange) {
synchronized (lock) {
if (!started) return;
syncSessionsListenerLocked();
refreshActiveControllerLocked(null);
}
}
};
private volatile FaceclawMediaControllerListener listener;
private MediaController activeController;
private boolean started;
private boolean sessionsListenerRegistered;
private Set<String> ignoredPackages = new HashSet<>();
public FaceclawMediaController(Context context) {
@@ -96,26 +117,48 @@ public class FaceclawMediaController {
return;
}
started = true;
if (!isNotificationAccessEnabled()) {
emitStateLocked();
return;
}
if (sessionManager != null) {
try {
sessionManager.addOnActiveSessionsChangedListener(
sessionsChangedListener,
listenerComponent,
mainHandler
);
} catch (SecurityException ignored) {
emitStateLocked();
return;
}
try {
appContext.getContentResolver().registerContentObserver(
Settings.Secure.getUriFor(ENABLED_NOTIFICATION_LISTENERS),
false,
notificationAccessObserver
);
} catch (Exception e) {
Log.w("FaceclawMedia", "notification access observer registration failed", e);
}
syncSessionsListenerLocked();
refreshActiveControllerLocked(null);
}
}
/**
* Keep the active-sessions listener registered exactly while notification
* access is granted. Registration throws SecurityException without the
* grant, so this is retried from the settings observer rather than only
* attempted once at start().
*/
private void syncSessionsListenerLocked() {
if (sessionManager == null) return;
boolean accessEnabled = started && isNotificationAccessEnabled();
if (accessEnabled && !sessionsListenerRegistered) {
try {
sessionManager.addOnActiveSessionsChangedListener(
sessionsChangedListener,
listenerComponent,
mainHandler
);
sessionsListenerRegistered = true;
} catch (SecurityException ignored) {
}
} else if (!accessEnabled && sessionsListenerRegistered) {
try {
sessionManager.removeOnActiveSessionsChangedListener(sessionsChangedListener);
} catch (SecurityException ignored) {
}
sessionsListenerRegistered = false;
}
}
/** Apply the complete preference set and immediately choose an allowed session. */
public void setIgnoredPackagesJson(String packagesJson) {
Set<String> packages = new HashSet<>();
@@ -139,12 +182,11 @@ public class FaceclawMediaController {
return;
}
started = false;
if (sessionManager != null) {
try {
sessionManager.removeOnActiveSessionsChangedListener(sessionsChangedListener);
} catch (SecurityException ignored) {
}
try {
appContext.getContentResolver().unregisterContentObserver(notificationAccessObserver);
} catch (Exception ignored) {
}
syncSessionsListenerLocked();
setActiveControllerLocked(null);
emitStateLocked();
}
@@ -377,7 +419,7 @@ public class FaceclawMediaController {
private boolean isNotificationAccessEnabled() {
String enabledListeners = Settings.Secure.getString(
appContext.getContentResolver(),
"enabled_notification_listeners"
ENABLED_NOTIFICATION_LISTENERS
);
if (enabledListeners == null || enabledListeners.isEmpty()) {
return false;
@@ -13,6 +13,7 @@ import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig;
import java.io.File;
import java.nio.charset.StandardCharsets;
@@ -61,12 +62,43 @@ public class FaceclawVoiceController {
"decoder_model_merged.ort",
"tokens.txt"
};
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
//
// On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0)
// runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of
// zero tail padding, capped at 30s. Re-decoding on Moonshine's
// TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that
// encode, growing with the utterance, roughly 1.4x/second while the user is
// still speaking, so onboardModelKind gates that loop off for Whisper --
// see processRecognizer(). Whisper is also known to hallucinate text on
// near-silent input; recognizeTranscriptSegment() gates that too.
private static final String ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8";
private static final String[] ASR_WHISPER_MODEL_FILES = {
"base.en-encoder.int8.onnx",
"base.en-decoder.int8.onnx",
"base.en-tokens.txt"
};
// Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f
// buffer) a segment is treated as silence and never reaches the Whisper
// recognizer at all, rather than risking a hallucinated non-answer. Picked
// conservatively low (well under typical mic noise floor already seen in
// this pipeline's normalization target) -- UNTESTED on real hardware, tune
// against real glasses captures rather than trusting this number.
private static final float WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f;
private enum VoiceInputMode {
ONBOARD, // on-phone Moonshine transcription
ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind)
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
}
/** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */
private enum OnboardModelKind {
MOONSHINE,
WHISPER
}
private final Context appContext;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
@@ -86,6 +118,7 @@ public class FaceclawVoiceController {
private volatile boolean usePhoneMic;
private boolean activePhoneMic;
private VoiceInputMode mode = VoiceInputMode.CLOUD;
private volatile OnboardModelKind onboardModelKind = OnboardModelKind.MOONSHINE;
private OfflineRecognizer recognizer;
private FaceclawLc3Decoder lc3Decoder;
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
@@ -150,6 +183,17 @@ public class FaceclawVoiceController {
this.saveRecordings = saveRecordings;
}
/**
* Which on-device model {@link #start}("onboard") should load: "whisper"
* selects the second on-device model (sherpa-onnx offline Whisper); any
* other value (including null/absent) keeps the existing Moonshine model,
* so callers that never call this see unchanged behavior. Must be set
* before {@link #start}; has no effect in CLOUD mode.
*/
public void setOnboardModelKind(String kind) {
this.onboardModelKind = "whisper".equals(kind) ? OnboardModelKind.WHISPER : OnboardModelKind.MOONSHINE;
}
/**
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
@@ -222,20 +266,26 @@ public class FaceclawVoiceController {
}
public void start(String requestedMode) {
start(requestedMode, 0);
}
public void start(String requestedMode, int captureId) {
synchronized (lock) {
if (started) {
emitStatus("Voice control is already listening.");
emitStopped(captureId);
return;
}
if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) {
emitStatus("Voice control needs an active G2 connection.");
emitStopped(captureId);
return;
}
mode = parseMode(requestedMode);
activePhoneMic = usePhoneMic;
started = true;
audioStarted = false;
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController");
workerThread.start();
}
}
@@ -306,18 +356,19 @@ public class FaceclawVoiceController {
return VoiceInputMode.ONBOARD;
}
private void runLoop() {
private void runLoop(int captureId) {
try {
deleteLegacyKwsFiles();
VoiceInputMode currentMode = mode;
OnboardModelKind currentOnboardKind = onboardModelKind;
if (currentMode == VoiceInputMode.ONBOARD) {
File modelDir = findAsrModelDir();
File modelDir = findAsrModelDir(currentOnboardKind);
if (modelDir == null) {
emitStatus("Voice model not downloaded (see Settings > Voice).");
return;
}
emitStatus("Loading transcription model...");
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind));
resetTranscriptState();
lastTranscript = "";
}
@@ -387,6 +438,7 @@ public class FaceclawVoiceController {
audioStarted = false;
workerThread = null;
}
emitStopped(captureId);
}
}
@@ -453,32 +505,48 @@ public class FaceclawVoiceController {
return b.array();
}
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir, OnboardModelKind kind) {
OfflineModelConfig.Builder modelConfig = OfflineModelConfig.builder()
.setNumThreads(1);
if (kind == OnboardModelKind.WHISPER) {
modelConfig
.setWhisper(OfflineWhisperModelConfig.builder()
.setEncoder(new File(modelDir, "base.en-encoder.int8.onnx").getAbsolutePath())
.setDecoder(new File(modelDir, "base.en-decoder.int8.onnx").getAbsolutePath())
.setLanguage("en")
.setTask("transcribe")
.build())
.setTokens(new File(modelDir, "base.en-tokens.txt").getAbsolutePath());
} else {
modelConfig
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath());
}
return OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setNumThreads(1)
.build())
.setModelConfig(modelConfig.build())
.build();
}
/**
* The Moonshine model directory, populated by the download flow in
* asr-model.ts (releases before 0.5.0 copied the same files out of the
* APK, so upgraded installs are already complete). Null when any file is
* The on-device model directory for the given kind, populated by the
* download flow in asr-model.ts (releases before 0.5.0 copied the
* Moonshine files out of the APK directly, so upgraded installs are
* already complete for that model). Null when any expected file is
* missing, i.e. the model still needs to be downloaded.
*/
private File findAsrModelDir() {
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR);
for (String fileName : ASR_MODEL_FILES) {
private File findAsrModelDir(OnboardModelKind kind) {
boolean whisper = kind == OnboardModelKind.WHISPER;
String dirName = whisper ? ASR_WHISPER_MODEL_DIR : ASR_MODEL_DIR;
String[] fileNames = whisper ? ASR_WHISPER_MODEL_FILES : ASR_MODEL_FILES;
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + dirName);
for (String fileName : fileNames) {
File file = new File(modelDir, fileName);
if (!file.exists() || file.length() == 0) {
return null;
@@ -658,6 +726,16 @@ public class FaceclawVoiceController {
private void processRecognizer(float[] samples) {
appendTranscriptSamples(samples);
// Each Whisper call re-encodes the whole buffer plus ~10s of tail
// padding (see the ASR_WHISPER_* comment above),
// so it skips the live-partial redecode Moonshine does on this
// interval and only decodes when a segment commits (8s buffer fill)
// or the utterance ends (decodeTranscript(true) in runLoop()). This
// means no live preview text while speaking in Whisper mode -- status
// stays "Listening..." until release. This is a deliberate tradeoff.
if (onboardModelKind == OnboardModelKind.WHISPER) {
return;
}
long now = SystemClock.elapsedRealtime();
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES
&& now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) {
@@ -767,6 +845,14 @@ public class FaceclawVoiceController {
return "";
}
float[] segment = Arrays.copyOf(transcriptSamples, sampleCount);
// Whisper hallucinates text on near-silent input (a known quirk of the
// model, not this pipeline); gate it on the PRE-normalization peak, since
// normalizePeak() below would otherwise amplify true silence right up to
// the target level and hide the very thing being checked for. Moonshine
// does not share this failure mode in practice, so it is left unchanged.
if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) {
return "";
}
normalizePeak(segment);
OfflineStream offlineStream = currentRecognizer.createStream();
try {
@@ -780,7 +866,7 @@ public class FaceclawVoiceController {
}
}
private static void normalizePeak(float[] samples) {
private static float peakAmplitude(float[] samples) {
float peak = 0f;
for (float s : samples) {
float a = Math.abs(s);
@@ -788,6 +874,11 @@ public class FaceclawVoiceController {
peak = a;
}
}
return peak;
}
private static void normalizePeak(float[] samples) {
float peak = peakAmplitude(samples);
if (peak <= 0f) {
return;
}
@@ -805,7 +896,7 @@ public class FaceclawVoiceController {
(committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE;
String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS
? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "...";
Log.i(TAG, "Moonshine decode final=" + isFinal
Log.i(TAG, (onboardModelKind == OnboardModelKind.WHISPER ? "Whisper" : "Moonshine") + " decode final=" + isFinal
+ " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec)
+ " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE)
+ " textLen=" + text.length() + " text=\"" + preview + "\"");
@@ -930,6 +1021,13 @@ public class FaceclawVoiceController {
mainHandler.post(currentListener::onSpeechEnd);
}
private void emitStopped(int captureId) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener != null) {
mainHandler.post(() -> currentListener.onStopped(captureId));
}
}
/**
* Decides when a hands-free utterance is over, so "Hey Even" capture can
* stop without a button release.
@@ -10,6 +10,9 @@ public interface FaceclawVoiceControllerListener {
*/
void onTranscript(String text, boolean isFinal);
/** Capture has finished, after the final transcript or a startup/recognition error. */
void onStopped(int captureId);
/**
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
@@ -60,6 +60,10 @@ public final class FaceclawWearBridge {
public static final String PATH_PREFIX = "/faceclaw";
public static final String PATH_STATE = "/faceclaw/state";
public static final String PATH_STATE_REQUEST = "/faceclaw/state/request";
/** Watch -> phone: the watch's own battery; carries no seq and is never acked. */
public static final String PATH_WATCH_BATTERY = "/faceclaw/battery";
/** Phone -> watch: answered with PATH_WATCH_BATTERY even while the watch app is closed. */
public static final String PATH_WATCH_BATTERY_REQUEST = "/faceclaw/battery/request";
public static final String PATH_ACK = "/faceclaw/ack";
public static final String PATH_EVENT = "/faceclaw/event";
@@ -284,6 +288,10 @@ public final class FaceclawWearBridge {
boolean delivered = dispatchMessage(path, json, nodeId);
if (delivered) return;
// A battery report answers a poll the dashboard sent; with no
// dashboard to receive it, it is simply dropped (the dashboard polls
// again when it starts), and the watch does not expect an ack.
if (PATH_WATCH_BATTERY.equals(path)) return;
// No JS listener yet: tell the watch so it can explain why nothing is
// happening. Only state requests are kept for replay (see class doc);
@@ -1,11 +1,9 @@
package com.faceclaw.app;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.zip.Deflater;
import android.util.Log;
public final class BleImageOptimizer {
@@ -79,7 +77,7 @@ public final class BleImageOptimizer {
public final byte[] packed;
public final int width;
public final int height;
final byte[] payload; // bytes actually streamed: mode-6 zlib(rle(4bpp))
final byte[] payload; // bytes actually streamed: mode-6 rle(4bpp)
final int sessionId;
List<BleProtocol.ImageFragment> fragments = Collections.emptyList();
@@ -113,7 +111,7 @@ public final class BleImageOptimizer {
* Build a mode-3 bounding-box incremental payload, or null when a full
* update should be sent instead (frames not comparable, or the changed
* region spans the whole screen). Wire format:
* [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][zlib(rle(headerless 4bpp region))]
* [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][rle(headerless 4bpp region)]
* The region is top-down rows of the NEW frame, stride width/2 bytes. The
* box is aligned so left/width are multiples of 4 pixels and top/height
* multiples of 2 rows, letting each coordinate fit one byte and avoiding
@@ -260,7 +258,7 @@ public final class BleImageOptimizer {
/**
* Encode one CFW mode-3 rectangle delta:
* [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][zlib(rle(box pixels))]
* [3][left/4][top/2][width/4][height/2][fid_lo][fid_hi][rle(box pixels)]
* left/width must be multiples of 4 (=> left>>1, width>>1 are whole bytes), top/
* height multiples of 2. The box pixels are top-down rows of `next` (4bpp packed,
* width>>1 bytes/row), run-length encoded before deflate. Shared by the single-bbox
@@ -273,7 +271,7 @@ public final class BleImageOptimizer {
for (int y = 0; y < boxHeight; y++) {
System.arraycopy(next, (top + y) * stride + (left >> 1), region, y * regionStride, regionStride);
}
byte[] compressed = deflate(rleEncode(region));
byte[] compressed = rleEncode(region);
byte[] out = new byte[7 + compressed.length];
out[0] = 3;
out[1] = (byte) (left / 4);
@@ -286,6 +284,22 @@ public final class BleImageOptimizer {
return out;
}
/** A full repaint in independently decodable commands below the uint16 record limit. */
static List<byte[]> encodeFullFrameBands(byte[] packed, int width, int height, int firstFid) {
if (width <= 0 || width > 640 || (width & 3) != 0 || height <= 0 || height > 480
|| (height & 1) != 0 || packed.length != width * height / 2) {
throw new IllegalArgumentException("unsupported custom framebuffer dimensions");
}
List<byte[]> commands = new ArrayList<>();
int fid = firstFid;
for (int top = 0; top < height; top += 64) {
commands.add(encodeMode3Rect(packed, width / 2, 0, top, width,
Math.min(64, height - top), fid));
fid = fid >= 0xfffe ? 1 : fid + 1;
}
return commands;
}
/**
* A CFW mode-8 multi-segment payload (`[8][count]` then per-rect
* `[len16][mode-3 submsg]`) carrying several tight rectangle deltas instead of
@@ -310,8 +324,7 @@ public final class BleImageOptimizer {
}
// Break-even tuning for the rect splitter. Splitting adds ~15 fixed bytes per
// rect (seglen + mode-3 header + fid + zlib framing) and loses cross-rect
// dictionary sharing, so only split across gaps big enough to pay for that.
// rect (seglen + mode-3 header + fid), so only split across gaps big enough to pay for that.
// Gaps are in the changed-mask's native units: whole 4bpp bytes (2px) for
// columns, rows for the vertical bands. Kept above the 4px/2px box alignment so
// aligned rects never overlap.
@@ -461,8 +474,8 @@ public final class BleImageOptimizer {
}
/**
* RLE- then zlib-compress headerless 4bpp pixels for CFW load_image_z mode 6.
* Wire format: [6][zlib(rle(4bpp pixels))]. Always use mode 6: the logical
* Run-length encode headerless 4bpp pixels for CFW load_image_z mode 6.
* Wire format: [6][rle(4bpp pixels)]. Always use mode 6: the logical
* image can be larger than its EvenHub carrier, so a raw BMP fallback would
* carry dimensions the legacy container loader cannot accept.
*/
@@ -470,7 +483,7 @@ public final class BleImageOptimizer {
if (packed == null || packed.length == 0) {
return packed;
}
byte[] z = deflate(rleEncode(packed));
byte[] z = rleEncode(packed);
byte[] out = new byte[z.length + 1];
out[0] = 6;
System.arraycopy(z, 0, out, 1, z.length);
@@ -479,7 +492,7 @@ public final class BleImageOptimizer {
/**
* Run-length encode packed 4bpp pixels for CFW modes 3 and 6, whose payload is
* zlib(rle(pixels)) rather than zlib(pixels). Runs are over the pixel NIBBLES of
* RLE tokens; deflate belongs to the transport. Runs are over the pixel NIBBLES of
* {@code pix} in wire order (high nibble = left pixel), including the pad nibble
* that ends each row at odd widths — i.e. {@code pix} read as 2*length nibbles.
* One token is:
@@ -527,35 +540,6 @@ public final class BleImageOptimizer {
return (i & 1) != 0 ? (b & 0x0f) : (b >> 4);
}
// Level 6: BEST_COMPRESSION cost 18-109ms per frame on the BLE worker,
// but BEST_SPEED inflated typical payloads from ~2.7KB past the 3800-byte
// fragment boundary, adding a whole extra ack round trip (~350ms). The
// default level keeps payloads under one fragment at about half the CPU.
// One long-lived Deflater per thread: constructing one allocates and
// initializes a native zlib stream, which is measurable at per-frame rates.
private static final ThreadLocal<Deflater> DEFLATER =
new ThreadLocal<Deflater>() {
@Override protected Deflater initialValue() {
return new Deflater(Deflater.DEFAULT_COMPRESSION);
}
};
private static byte[] deflate(byte[] data) {
Deflater deflater = DEFLATER.get();
try {
deflater.setInput(data);
deflater.finish();
ByteArrayOutputStream out = new ByteArrayOutputStream(Math.max(64, data.length / 3));
byte[] buf = new byte[4096];
while (!deflater.finished()) {
out.write(buf, 0, deflater.deflate(buf));
}
return out.toByteArray();
} finally {
deflater.reset();
}
}
public static final class ImageUpdateStats {
final int paintMs;
final int tileCount;
@@ -50,16 +50,15 @@ public class BleProtocol {
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
/**
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
* CFW mic_control: host writes an ['M','C',ver,op,...] record as settings
* field 103; each temple
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
* to settings read responses and as standalone commandId=3 pushes.
*/
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
/**
* CFW als_sensor (EVENCFW/18, caps token als16): ambient-light reports on
* settings-channel field 105, 24-byte ['A','L',ver,reason,...] records from
* CFW als_sensor: ambient-light reports on settings-channel field 105, 24-byte ['A','L',ver,reason,...] records from
* the master temple (decoded in app/native/ambient-light.ts).
*/
public static final int FACECLAW_ALS_REPORT_FIELD = 105;
@@ -72,6 +71,19 @@ public class BleProtocol {
public static final int FACECLAW_WEAR_OP_QUERY = 7;
public static final int CFW_IMAGE_MODE_CLEANUP = 11;
private static final int FACECLAW_WAKE_EVENT = 1;
// Field-102 deferred wake from the IMU head-up (Faceclaw firmware revision
// 3+); same nonce handshake as the double-tap wake event 1.
public static final int FACECLAW_WAKE_EVENT_HEAD_UP = 5;
// Field-102 idle-gesture events (Faceclaw firmware revision 2+): gestures the
// stock display thread drops while no app is on screen, forwarded while the
// wake lease is held. Their last two bytes are the raw touch source and 0.
public static final int FACECLAW_GESTURE_EVENT_TAP = 2;
public static final int FACECLAW_GESTURE_EVENT_LONG_PRESS = 3;
public static final int FACECLAW_GESTURE_EVENT_LONG_PRESS_RELEASE = 4;
// The display thread's raw touch sources, as the firmware reports them.
private static final int FACECLAW_RAW_SOURCE_LEFT_TEMPLE = 0;
private static final int FACECLAW_RAW_SOURCE_RIGHT_TEMPLE = 1;
private static final int FACECLAW_RAW_SOURCE_RING = 4;
private static final int FACECLAW_WAKE_PROTOCOL_VERSION = 1;
// UI_FOREGROUND_EVEN_AI_ID: the stock "Even AI" assistant app. It is a
// FOREGROUND app, whereas EvenHub (and therefore faceclaw) is the
@@ -112,6 +124,9 @@ public class BleProtocol {
public static final int EVENT_RING_LONG_PRESS_RELEASE = 10;
// CFW extension: G2 2.2.9 tap-then-hold gesture forwarded to the phone.
public static final int EVENT_SHORT_THEN_LONG_PRESS = 11;
// CFW extension (firmware revision 3+): the IMU head-up, forwarded as a
// sys-event while an EvenHub page is on screen (soft sleep).
public static final int EVENT_HEAD_UP = 12;
public static final int EVENT_SOURCE_GLASSES_R = 1;
public static final int EVENT_SOURCE_RING = 2;
@@ -170,13 +185,11 @@ public class BleProtocol {
}
public static byte[] buildCreateMixedImagePage(int magic, ImageTileOptions[] tiles) {
/** Retain the text container for forwarded inputs, without an image container. */
public static byte[] buildCreateInputPage(int magic) {
List<byte[]> innerParts = new ArrayList<>();
innerParts.add(encodeVarintField(1, 1 + tiles.length));
innerParts.add(encodeVarintField(1, 1));
innerParts.add(encodeMessageField(3, encodeTextObject("dashboard", 1, 0, 0, 576, 288, " ", true)));
for (ImageTileOptions tile : tiles) {
innerParts.add(encodeMessageField(4, encodeImageObject(tile)));
}
innerParts.add(encodeVarintField(5, 10000));
byte[] inner = concat(innerParts);
return wrapEvenHub(0, magic, 3, inner);
@@ -475,24 +488,127 @@ public class BleProtocol {
return body;
}
/** Field 106: RB/version 1/flags/percentage, from the CFW ring-battery cache. */
public static RingBatterySnapshot parseRingBattery(byte[] pb) {
if (pb == null) return null;
byte[] body = readFieldBytes(stripTrailingCrc(pb), 106);
if (body == null || body.length != 5 || body[0] != 'R' || body[1] != 'B'
|| body[2] != 1) return null;
int flags = body[3] & 0xff;
int level = body[4] & 0xff;
if ((flags & ~7) != 0) return null;
if ((flags & 2) != 0) {
if ((flags & 1) == 0 || level > 100) return null;
return new RingBatterySnapshot(level, (flags & 4) != 0 ? 1 : 0);
}
if (level != 255 || (flags & 4) != 0) return null;
return new RingBatterySnapshot(-1, -1);
}
public static final class RingBatterySnapshot {
final int battery;
final int charging;
RingBatterySnapshot(int battery, int charging) {
this.battery = battery;
this.charging = charging;
}
}
/**
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
* when this is an ordinary settings frame.
* Return the uint16 wake nonce from a CFW field-102 deferred-wake
* notification (double tap or head-up), or -1 when this is an ordinary
* settings frame (idle-gesture events included).
*/
public static int parseFaceclawWakeEvent(byte[] pb) {
byte[] event = faceclawWakeEventBytes(pb);
return event == null ? -1 : (event[4] & 0xff) | ((event[5] & 0xff) << 8);
}
/**
* Which gesture a CFW deferred-wake notification reports: 1 (double tap)
* or FACECLAW_WAKE_EVENT_HEAD_UP; -1 when the frame is not a wake event.
*/
public static int parseFaceclawWakeEventCode(byte[] pb) {
byte[] event = faceclawWakeEventBytes(pb);
return event == null ? -1 : (event[3] & 0xff);
}
private static byte[] faceclawWakeEventBytes(byte[] pb) {
if (pb == null) {
return -1;
return null;
}
byte[] event = readFieldBytes(stripTrailingCrc(pb), FACECLAW_WAKE_EVENT_FIELD);
if (event == null
|| event.length != 6
|| event[0] != 'F'
|| event[1] != 'C'
|| (event[2] & 0xff) != FACECLAW_WAKE_PROTOCOL_VERSION
|| (event[3] & 0xff) != FACECLAW_WAKE_EVENT) {
return -1;
|| (event[2] & 0xff) != FACECLAW_WAKE_PROTOCOL_VERSION) {
return null;
}
return (event[4] & 0xff) | ((event[5] & 0xff) << 8);
int code = event[3] & 0xff;
return code == FACECLAW_WAKE_EVENT || code == FACECLAW_WAKE_EVENT_HEAD_UP ? event : null;
}
/** An idle gesture from a CFW field-102 notification, in EvenHub sys-event terms. */
public static final class FaceclawGestureEvent {
public final int eventType;
public final int eventSource;
FaceclawGestureEvent(int eventType, int eventSource) {
this.eventType = eventType;
this.eventSource = eventSource;
}
}
/**
* Decode a CFW field-102 idle-gesture event (tap, long press, release
* while no app is on screen) into the EvenHub sys-event type and source
* the same gesture would carry from a live page, or null when the frame
* is not one (ordinary settings frames and the double-tap wake event
* included).
*/
public static FaceclawGestureEvent parseFaceclawGestureEvent(byte[] pb) {
if (pb == null) {
return null;
}
byte[] event = readFieldBytes(stripTrailingCrc(pb), FACECLAW_WAKE_EVENT_FIELD);
if (event == null
|| event.length != 6
|| event[0] != 'F'
|| event[1] != 'C'
|| (event[2] & 0xff) != FACECLAW_WAKE_PROTOCOL_VERSION) {
return null;
}
int eventType;
switch (event[3] & 0xff) {
case FACECLAW_GESTURE_EVENT_TAP:
eventType = EVENT_CLICK;
break;
case FACECLAW_GESTURE_EVENT_LONG_PRESS:
eventType = EVENT_RING_LONG_PRESS;
break;
case FACECLAW_GESTURE_EVENT_LONG_PRESS_RELEASE:
eventType = EVENT_RING_LONG_PRESS_RELEASE;
break;
default:
return null;
}
int eventSource;
switch (event[4] & 0xff) {
case FACECLAW_RAW_SOURCE_LEFT_TEMPLE:
eventSource = EVENT_SOURCE_GLASSES_L;
break;
case FACECLAW_RAW_SOURCE_RIGHT_TEMPLE:
eventSource = EVENT_SOURCE_GLASSES_R;
break;
case FACECLAW_RAW_SOURCE_RING:
eventSource = EVENT_SOURCE_RING;
break;
default:
eventSource = 0;
break;
}
return new FaceclawGestureEvent(eventType, eventSource);
}
/**
@@ -559,11 +675,14 @@ public class BleProtocol {
}
/**
* Firmware versions and the CFW capability advertisement from a settings
* Firmware versions and the firmware-extension string from a settings
* READ ack. Versions are fields 5/6 of the deviceReceiveRequestFromApp
* submessage (field 4); the custom firmware additionally appends top-level
* field 100, a string like "EVENCFW/6 img576 img640 ... directfb fbguard", which
* stock firmware never sends. Returns null when the ack carries none of it.
* submessage (field 4); Faceclaw's custom firmware additionally appends
* top-level field 100 with its revision ("Faceclaw/<n>"; older builds sent
* "EVENCFW/<ver> <tokens>"), which stock firmware never sends. Returns null
* when the ack carries none of it. Compatibility is judged on the TS side
* (app/g2/firmware-compat.ts); Java only needs to know whether the firmware
* is ours at all.
*/
public static FirmwareInfo parseSettingsFirmwareInfo(byte[] pb) {
byte[] root = stripTrailingCrc(pb);
@@ -573,11 +692,11 @@ public class BleProtocol {
}
String leftVersion = readStringFieldValue(request, 5);
String rightVersion = readStringFieldValue(request, 6);
String capabilities = readStringFieldValue(root, 100);
if (leftVersion.isEmpty() && rightVersion.isEmpty() && capabilities.isEmpty()) {
String extension = readStringFieldValue(root, 100);
if (leftVersion.isEmpty() && rightVersion.isEmpty() && extension.isEmpty()) {
return null;
}
return new FirmwareInfo(leftVersion, rightVersion, capabilities);
return new FirmwareInfo(leftVersion, rightVersion, extension);
}
public static byte[] wrapEvenHub(int cmd, int magic, int innerFieldNumber, byte[] inner) {
@@ -1094,14 +1213,28 @@ public class BleProtocol {
}
public static final class FirmwareInfo {
/** Prefix of the firmware-extension string on Faceclaw's custom firmware. */
public static final String FACECLAW_EXTENSION_PREFIX = "Faceclaw/";
final String leftVersion;
final String rightVersion;
final String capabilities;
/** Raw field-100 string ("" on stock firmware). */
final String extension;
FirmwareInfo(String leftVersion, String rightVersion, String capabilities) {
FirmwareInfo(String leftVersion, String rightVersion, String extension) {
this.leftVersion = leftVersion;
this.rightVersion = rightVersion;
this.capabilities = capabilities;
this.extension = extension == null ? "" : extension;
}
/**
* True when the glasses run Faceclaw's custom firmware (any revision).
* Whether the revision is the one this app needs is decided on the TS
* side, which disconnects on a mismatch; this only guards the private
* modes against stock or third-party firmware in the meantime.
*/
public boolean isFaceclawFirmware() {
return extension.trim().startsWith(FACECLAW_EXTENSION_PREFIX);
}
}
}
@@ -0,0 +1,38 @@
package com.faceclaw.app;
import java.util.ArrayList;
import java.util.List;
/** Ordered completion and go-back-N recovery for the custom-message window. */
final class CfwMessageWindow {
// Flappy capture: p99 ACK 63ms, max 77ms. Leave room for full-frame
// processing while recovering a lost reply well before the stock timeout.
static final int ACK_TIMEOUT_MS = 500;
static final int MAX_RETRIES = 3;
private CfwMessageWindow() {}
static OutboundMessage acknowledgedHead(Iterable<OutboundMessage> messages) {
for (OutboundMessage message : messages) {
if (message.sid != CfwTransport.SID) continue;
return !message.cfwRetryPending && message.cfwAckLenses == CfwTransport.BOTH ? message : null;
}
return null;
}
static List<OutboundMessage> replayWindow(Iterable<OutboundMessage> messages, long nowMs) {
List<OutboundMessage> replay = new ArrayList<>();
boolean failed = false;
for (OutboundMessage message : messages) {
if (message.sid != CfwTransport.SID) continue;
failed |= message.cfwRetryPending || (message.cfwAckLenses != CfwTransport.BOTH
&& message.ackDeadlineAtMs > 0 && message.ackDeadlineAtMs <= nowMs);
replay.add(message);
}
// A lost ACK, first fragment, or NACK can leave an older attempt
// unresolved. Replaying only the named failure strands that older
// attempt at the head until its timeout. Rewind the unresolved window.
if (!failed) replay.clear();
return replay;
}
}
@@ -0,0 +1,141 @@
package com.faceclaw.app;
import java.io.ByteArrayOutputStream;
import java.util.zip.Deflater;
import java.util.ArrayList;
import java.util.List;
/** Faceclaw/10 records: clear flags, wire length and decoded CRC; persistent zlib body. */
public final class CfwTransport {
public static final int SID = 0xf0;
public static final int BOTH = 3;
public static final int MAX_MESSAGE = 65535;
public static final int COMPRESSED = 4, RESET_CONTEXT = 8;
private final Deflater deflater = new Deflater();
private boolean resetPending = true;
private int previousLenses;
public CfwTransport() {}
public synchronized void reset() {
deflater.reset();
resetPending = true;
previousLenses = 0;
}
/** Compress only when writing, in exactly the order seen by the receiver. */
public synchronized List<byte[]> encode(byte[] message, int streamId, int lenses, int mtu) {
validate(message, streamId, lenses, mtu);
if (previousLenses != lenses) reset();
boolean reset = resetPending;
deflater.setInput(message);
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] chunk = new byte[4096];
int count;
do {
count = deflater.deflate(chunk, 0, chunk.length, Deflater.SYNC_FLUSH);
out.write(chunk, 0, count);
} while (count == chunk.length);
byte[] body = out.toByteArray();
int flags = lenses | COMPRESSED | (reset ? RESET_CONTEXT : 0);
if (body.length == 0) flags &= ~COMPRESSED; // repeated empty sync flush
if (body.length > MAX_MESSAGE) {
// Incompressible maximum-size records still fit as raw records.
// The attempted deflate advanced history: reset both ends before reuse.
reset();
body = message;
flags = lenses | RESET_CONTEXT;
} else {
resetPending = false;
previousLenses = lenses;
}
return frameRecord(body, crc(message), streamId, flags, mtu);
}
private static void validate(byte[] message, int streamId, int lenses, int mtu) {
if (message == null || message.length > MAX_MESSAGE || streamId < 0 || streamId > 255
|| lenses < 1 || lenses > BOTH || mtu < 23 || mtu > 517)
throw new IllegalArgumentException("invalid CFW stream parameters");
}
/** One logical command per stream for now. Packets are bounded by actual ATT MTU. */
public static List<byte[]> frame(byte[] message, int streamId, int lenses, int mtu) {
validate(message, streamId, lenses, mtu);
return frameRecord(message, crc(message), streamId, lenses | RESET_CONTEXT, mtu);
}
private static List<byte[]> frameRecord(byte[] body, int checksum, int streamId, int flags, int mtu) {
int lenses = flags & BOTH;
int capacity = Math.min(252, mtu - 14);
byte[] stream = new byte[body.length + 5];
stream[0] = (byte) flags;
stream[1] = (byte) body.length;
stream[2] = (byte) (body.length >>> 8);
stream[3] = (byte) checksum;
stream[4] = (byte) (checksum >>> 8);
System.arraycopy(body, 0, stream, 5, body.length);
List<byte[]> packets = new ArrayList<>();
for (int offset = 0, index = 0; offset < stream.length; offset += capacity, index++) {
int count = Math.min(capacity, stream.length - offset);
byte[] packet = new byte[count + 11];
packet[0] = (byte) 0xaa;
packet[1] = 0x21;
packet[2] = (byte) (streamId + index);
packet[3] = (byte) (count + 3);
packet[4] = packet[5] = 1;
packet[6] = (byte) SID;
packet[8] = (byte) (lenses | (offset == 0 ? 0x80 : 0)
| (offset + count == stream.length ? 0x40 : 0));
System.arraycopy(stream, offset, packet, 9, count);
int crc = crc(packet, 8, count + 1);
packet[packet.length - 2] = (byte) crc;
packet[packet.length - 1] = (byte) (crc >>> 8);
packets.add(packet);
}
return packets;
}
public static int crc(byte[] data) { return crc(data, 0, data.length); }
private static int crc(byte[] data, int offset, int count) {
int crc = 0xffff;
for (int i = offset; i < offset + count; i++) {
crc ^= (data[i] & 255) << 8;
for (int bit = 0; bit < 8; bit++)
crc = ((crc << 1) ^ ((crc & 0x8000) != 0 ? 0x1021 : 0)) & 65535;
}
return crc;
}
private static int u16(byte[] data, int offset) {
return (data[offset] & 255) | ((data[offset + 1] & 255) << 8);
}
public static Ack parseAck(byte[] packet) {
Ack[] acks = parseAcks(packet);
return acks == null ? null : acks[0];
}
/** Primary ACK followed by up to three explicit preceding successes from
* the same processing lens. Validate the entire packet before applying any. */
public static Ack[] parseAcks(byte[] packet) {
if (packet == null || packet.length < 19 || packet.length > 40
|| (packet.length - 19) % 7 != 0 || (packet[0] & 255) != 0xaa
|| packet[1] != 0x12 || (packet[3] & 255) != packet.length - 8 || packet[4] != 1 || packet[5] != 1
|| (packet[6] & 255) != SID || packet[7] != 0 || (packet[8] != 1 && packet[8] != 3)
|| (packet[12] != 1 && packet[12] != 2)
|| (packet[8] == 3 && packet.length != 19)
|| crc(packet, 8, packet.length - 10) != u16(packet, packet.length - 2)) return null;
Ack[] acks = new Ack[1 + (packet.length - 19) / 7];
acks[0] = new Ack(packet[8] == 3, packet[9] & 255, u16(packet, 10), packet[12], u16(packet, 13), u16(packet, 15));
for (int i = 1, offset = 17; i < acks.length; i++, offset += 7)
acks[i] = new Ack(false, packet[offset] & 255, u16(packet, offset + 1),
packet[12], u16(packet, offset + 3), u16(packet, offset + 5));
return acks;
}
public static final class Ack {
public final boolean nack;
public final int streamId, messageId, lens, size, checksum;
Ack(boolean nack, int streamId, int messageId, int lens, int size, int checksum) {
this.nack = nack;
this.streamId = streamId; this.messageId = messageId; this.lens = lens;
this.size = size; this.checksum = checksum;
}
}
}
@@ -72,35 +72,18 @@ public class MessageBuilder {
String label,
boolean leftArm
) {
BleProtocol.ImageFragment fragment =
new BleProtocol.ImageFragment(0, payload, payload.length);
int magic = magicPool.allocate();
return new OutboundMessage(
kind,
label,
BleProtocol.SID_EVENHUB,
BleProtocol.FLAG_REQUEST,
magic,
BleProtocol.buildImageRawData(tile, sessionId, payload.length, fragment, magic),
ACK_TIMEOUT_MS,
-1,
leftArm
);
// Keep these arguments for source compatibility with existing callers;
// the private command no longer carries a tile, session, or image header.
return customMessage(kind, payload, label, -1, leftArm);
}
public OutboundMessage imageFragment(BleProtocol.ImageFragment fragment, BleImageOptimizer.TileImagePlan plan, boolean requestAck, boolean leftArm) {
int magic = requestAck ? magicPool.allocate() : 0;
return new OutboundMessage(
"image",
"image " + plan.tile.name + "#" + fragment.index,
BleProtocol.SID_EVENHUB,
BleProtocol.FLAG_REQUEST,
magic,
BleProtocol.buildImageRawData(plan.tile, plan.sessionId, plan.payload.length, fragment, magic),
ACK_TIMEOUT_MS,
plan.tileIndex,
leftArm
);
public OutboundMessage customMessage(String kind, byte[] payload, String label,
int tileIndex, boolean leftArm) {
if (payload == null || payload.length > CfwTransport.MAX_MESSAGE)
throw new IllegalArgumentException("CFW message exceeds uint16 length");
int streamId = magicPool.allocate();
return new OutboundMessage(kind, label, CfwTransport.SID, 0, streamId, payload,
CfwMessageWindow.ACK_TIMEOUT_MS, tileIndex, leftArm);
}
public OutboundMessage cfwCleanup(
@@ -179,7 +162,7 @@ public class MessageBuilder {
);
}
public OutboundMessage createLayout(BleProtocol.ImageTileOptions... tiles) {
public OutboundMessage createLayout() {
int magic = magicPool.allocate();
return new OutboundMessage(
"create-layout",
@@ -187,7 +170,7 @@ public class MessageBuilder {
BleProtocol.SID_EVENHUB,
BleProtocol.FLAG_REQUEST,
magic,
BleProtocol.buildCreateMixedImagePage(magic, tiles),
BleProtocol.buildCreateInputPage(magic),
ACK_TIMEOUT_MS,
-1,
false
@@ -6,12 +6,16 @@ public final class OutboundMessage {
final String label;
final int sid;
final int flag;
final int magic;
int magic;
final byte[] message;
final int ackTimeoutMs;
final int tileIndex;
final boolean isLeftArmMessage;
int cfwAckLenses, cfwRetries;
boolean cfwRetryPending;
final int cfwChecksum;
int imageUpdateId;
int imageMessageNumber;
int imageMessageCount;
@@ -30,11 +34,35 @@ public final class OutboundMessage {
this.flag = flag;
this.magic = magic;
this.message = message == null ? new byte[0] : Arrays.copyOf(message, message.length);
this.cfwChecksum = sid == CfwTransport.SID ? CfwTransport.crc(this.message) : 0;
this.ackTimeoutMs = ackTimeoutMs;
this.tileIndex = tileIndex;
this.isLeftArmMessage = isLeftArmMessage;
}
/** NACK names this attempt's stream/message sequence, even without a decoded size. */
boolean acceptCfwAck(CfwTransport.Ack ack) {
if (ack == null || sid != CfwTransport.SID || magic != ack.streamId
|| ack.messageId != 0 || (ack.lens != 1 && ack.lens != 2)) return false;
if (ack.nack) {
cfwRetryPending = true;
return false;
}
if (message.length != ack.size || cfwChecksum != ack.checksum) return false;
cfwAckLenses |= ack.lens;
return !cfwRetryPending && cfwAckLenses == CfwTransport.BOTH;
}
/** Replace an attempt atomically under the communicator lock. */
void prepareCfwReplay(int streamId) {
magic = streamId;
++cfwRetries;
cfwAckLenses = 0;
cfwRetryPending = false;
ackDeadlineAtMs = 0;
ackPayload = new byte[0];
}
void setImageUpdatePosition(int updateId, int messageNumber, int messageCount) {
this.imageUpdateId = updateId;
this.imageMessageNumber = messageNumber;
@@ -7,31 +7,31 @@ import java.util.List;
import java.util.Map;
/**
* Phone-side model of the CFW's flat 64 KiB texture cache (zlib_glue mode 12).
* Phone-side model of the CFW's flat 256 KiB texture cache (zlib_glue mode 18).
* The firmware holds only bytes; every structural decision — where font
* tables and glyph images live, what is resident, when to start over — is
* made here. Layout per font: a 96-entry uint16 offset table (chars 32..127)
* made here. Layout per font: a 96-entry uint32 offset table (chars 32..127)
* whose entries are written lazily as each glyph is uploaded; glyph images
* are the GlyphAtlas cached bytes ([w][h][4bpp RLE]).
*
* Allocation is a bump pointer with reset-on-full: the realistic working set
* (a few fonts' tables plus ~hundreds of glyphs at tens of bytes each) is far
* below 64 KiB, so eviction sophistication buys nothing. After a reset the
* below 256 KiB, so eviction sophistication buys nothing. After a reset the
* firmware cache still holds stale bytes; that is safe because a draw only
* ever references (font table entry, glyph image) pairs uploaded in the
* current generation — stale table entries are never named in a mode-14
* current generation — stale table entries are never named in a mode-20
* string.
*
* Upload entries accumulate as complete [offset u16][len u16][data] records
* and drain into one or more standalone mode-12 payloads, each independently
* Upload entries accumulate as complete [offset u32][len u16][data] records
* and drain into one or more standalone mode-18 payloads, each independently
* valid, enqueued ahead of the image message that references them (the
* transport is FIFO, so no ack round-trip is needed before use).
*
* Not thread-safe by itself: the communicator calls it under its own lock.
*/
public final class TextureCacheState {
public static final int CACHE_SIZE = 65536;
private static final int FONT_TABLE_BYTES = 96 * 2;
public static final int CACHE_SIZE = 262144;
private static final int FONT_TABLE_BYTES = 96 * 4;
private int allocEnd = 0;
private final Map<Integer, Integer> fontTableOffsets = new HashMap<>();
@@ -64,7 +64,7 @@ public final class TextureCacheState {
/**
* Bytes currently allocated out of CACHE_SIZE, for logging/tuning (a
* sudden drop in the frame log means a reset-on-full re-upload cycle —
* the signal that the 64 KiB budget is being outgrown).
* the signal that the 256 KiB budget is being outgrown).
*/
public int usedBytes() {
return allocEnd;
@@ -78,9 +78,9 @@ public final class TextureCacheState {
/**
* Ensure a glyph image (and its font table slot) is resident, queuing the
* mode-12 upload entries for anything newly placed. Returns the glyph's
* mode-18 upload entries for anything newly placed. Returns the glyph's
* image offset, or -1 when the cache is full (caller resets and retries)
* or the encoding is outside the mode-14 table (32..127).
* or the encoding is outside the mode-20 table (32..127).
*/
public int ensureGlyph(int fontId, int encoding, GlyphAtlas.Glyph glyph) {
if (glyph == null || encoding < 32 || encoding > 127) return -1;
@@ -104,13 +104,14 @@ public final class TextureCacheState {
glyphOffsets.put(key, offset);
addEntry(offset, glyph.cachedBytes);
addEntry(table + 2 * (encoding - 32),
new byte[]{(byte) (offset & 0xff), (byte) ((offset >> 8) & 0xff)});
addEntry(table + 4 * (encoding - 32),
new byte[]{(byte) offset, (byte) (offset >> 8),
(byte) (offset >> 16), (byte) (offset >> 24)});
return offset;
}
/**
* Ensure an ImageAtlas entry is resident (mode-13 draws reference it by
* Ensure an ImageAtlas entry is resident (mode-19 draws reference it by
* raw cache offset; no table involved). Returns the image offset, or -1
* when the cache is full (caller resets and retries).
*/
@@ -127,12 +128,14 @@ public final class TextureCacheState {
}
private void addEntry(int offset, byte[] data) {
byte[] entry = new byte[4 + data.length];
byte[] entry = new byte[6 + data.length];
entry[0] = (byte) (offset & 0xff);
entry[1] = (byte) ((offset >> 8) & 0xff);
entry[2] = (byte) (data.length & 0xff);
entry[3] = (byte) ((data.length >> 8) & 0xff);
System.arraycopy(data, 0, entry, 4, data.length);
entry[2] = (byte) ((offset >> 16) & 0xff);
entry[3] = (byte) ((offset >> 24) & 0xff);
entry[4] = (byte) (data.length & 0xff);
entry[5] = (byte) ((data.length >> 8) & 0xff);
System.arraycopy(data, 0, entry, 6, data.length);
pendingEntries.add(entry);
pendingBytes += entry.length;
}
@@ -147,7 +150,7 @@ public final class TextureCacheState {
}
/**
* Drain the queued entries into complete mode-12 payloads ([12][entries]),
* Drain the queued entries into complete mode-18 payloads ([18][entries]),
* each at most maxPayloadBytes so a payload fits one BLE image message
* without fragmentation. A single entry larger than the cap still gets its
* own payload (the transport can fragment it).
@@ -162,7 +165,7 @@ public final class TextureCacheState {
}
if (current == null) {
current = new ByteArrayOutputStream(Math.min(maxPayloadBytes, 4 + entry.length));
current.write(12);
current.write(18);
}
current.write(entry, 0, entry.length);
}
@@ -7,7 +7,7 @@ import java.util.List;
/**
* Plans a screen update that ships text and icons as on-glasses cached draws
* instead of pixels (CFW modes 12/13/14; see g2flash/patches/zlib_glue.c and
* instead of pixels (CFW modes 18/19/20; see g2flash/patches/zlib_glue.c and
* texture_cache.c).
*
* Approach: dirty rects come from comparing the fully composited old and new
@@ -19,8 +19,8 @@ import java.util.List;
* occlusion/clipping, and draw-on-draw overlap). A draw that passes and is
* (or can be made) resident in the texture cache is replayed on-glasses: its
* written pixels are punched to 0 in the delta rect content — which is what
* makes text and icons nearly free to compress — and re-emitted as a mode-14
* string or mode-13 image sub-message in the same atomic mode-8 batch, so
* makes text and icons nearly free to compress — and re-emitted as a mode-20
* string or mode-19 image sub-message in the same atomic mode-8 batch, so
* the shadow after apply equals the full composite exactly. Everything else
* stays baked.
*
@@ -30,7 +30,7 @@ import java.util.List;
public final class TexturePlanner {
private TexturePlanner() {}
/** Cap on mode-14 string sub-messages (mode-8 count is a u8). */
/** Cap on mode-20 string sub-messages (mode-8 count is a u8). */
private static final int MAX_GLYPH_RUNS = 180;
/**
* Soft cap on cached-glyph selection per update, before run grouping.
@@ -38,21 +38,21 @@ public final class TexturePlanner {
* screens draw everything; the run budget is the real bound.
*/
private static final int MAX_SELECTED_GLYPHS = 4600;
/** Cap on mode-13 image sub-messages per update. */
/** Cap on mode-19 image sub-messages per update. */
private static final int MAX_IMAGE_DRAWS = 60;
/** Cap on mode-15 builtin-font string sub-messages per update. */
private static final int MAX_FWTEXT_RUNS = 80;
/** Mode-12 payload size that fits one BLE image message comfortably. */
/** Mode-18 payload size that fits one BLE image message comfortably. */
private static final int UPLOAD_PAYLOAD_MAX = 3600;
/** Max x-adjust control bytes between two glyphs before starting a new run. */
private static final int MAX_ADJUST_BYTES = 4;
/** Options: identity LUT (top 15) + transparent, for mode-13 image draws. */
/** Options: identity LUT (top 15) + transparent, for mode-19 image draws. */
private static final int IMAGE_DRAW_OPTIONS = 0x1f;
public static final class Result {
/** Complete image payload: a mode-8 batch of rect deltas + cached draws. */
public final byte[] payload;
/** Mode-12 upload payloads to enqueue BEFORE the image message. */
/** Mode-18 upload payloads to enqueue BEFORE the image message. */
public final List<byte[]> uploads;
/** Next mode-3 frame id (deltas consumed some). */
public final int nextFid;
@@ -118,15 +118,14 @@ public final class TexturePlanner {
/**
* Plan a cached-draw update. previous is the delta base (the frame the
* shadow currently holds), or null/mismatched for a full-frame keyframe.
* allowImages requires the teximg13 capability; allowFwText requires
* font15. Returns null when the plain paths should run instead (no
* replayable draws, or identical frames).
* Returns null when the plain paths should run instead (no replayable
* draws, or identical frames).
*/
public static Result plan(
byte[] previous, byte[] next, int width, int height,
SurfaceCompositor.ScreenDraw[] draws,
TextureCacheState cache, int fidStart,
boolean allowMultiRect, int maxRects, boolean allowImages, boolean allowFwText) {
boolean allowMultiRect, int maxRects) {
if (next == null || draws == null || draws.length == 0 || width <= 0 || height <= 0) {
return null;
}
@@ -171,14 +170,12 @@ public final class TexturePlanner {
List<FwPunch> fwPunches = new ArrayList<>();
int fwGlyphCount = 0;
int fwBaked = 0;
if (allowFwText) {
for (SurfaceCompositor.ScreenDraw draw : draws) {
if (draw.kind != SurfaceCompositor.ScreenDraw.KIND_FWTEXT) continue;
fwBaked += planFwRun(draw, rects, next, stride, width, height, fwSubs, fwPunches);
}
fwGlyphCount = fwPunches.size();
bakedCandidates += fwBaked;
for (SurfaceCompositor.ScreenDraw draw : draws) {
if (draw.kind != SurfaceCompositor.ScreenDraw.KIND_FWTEXT) continue;
fwBaked += planFwRun(draw, rects, next, stride, width, height, fwSubs, fwPunches);
}
fwGlyphCount = fwPunches.size();
bakedCandidates += fwBaked;
for (SurfaceCompositor.ScreenDraw draw : draws) {
if (draw.kind == SurfaceCompositor.ScreenDraw.KIND_GLYPH) {
GlyphAtlas.Glyph atlas = GlyphAtlas.get(draw.fontId, draw.encoding);
@@ -204,7 +201,6 @@ public final class TexturePlanner {
}
selected.add(new Selected(draw, atlas, null, gx, top));
} else {
if (!allowImages) continue; // firmware lacks teximg13: stays baked
ImageAtlas.Entry atlas = ImageAtlas.get(draw.imageId);
if (atlas == null) continue;
if (!intersectsAny(rects, draw.x, draw.y, atlas.width, atlas.height)) {
@@ -239,7 +235,7 @@ public final class TexturePlanner {
: ensureResident(cache, selected);
bakedCandidates += selected.size() - drawable.size();
// Group glyphs into mode-14 runs; leftover-budget glyphs bake.
// Group glyphs into mode-20 runs; leftover-budget glyphs bake.
List<Selected> glyphDrawable = new ArrayList<>();
List<Selected> imageDrawable = new ArrayList<>();
for (Selected sel : drawable) {
@@ -509,7 +505,7 @@ public final class TexturePlanner {
/**
* Whether every in-panel nonzero pixel of the cached image equals the
* packed new frame — the mode-13 transparent draw writes exactly those.
* packed new frame — the mode-19 transparent draw writes exactly those.
*/
private static boolean imageMatchesComposite(byte[] packed, int stride, int width, int height,
ImageAtlas.Entry atlas, int left, int top) {
@@ -596,24 +592,26 @@ public final class TexturePlanner {
return Collections.emptyList(); // unreachable
}
/** Mode-13 cached-image draw: [13][offset u16][x u16][y u16][options u8]. */
/** Mode-19 cached-image draw: [19][offset u32][x u16][y u16][options u8]. */
private static byte[] encodeImageDraw(TextureCacheState cache, Selected sel) {
int offset = cache.ensureImage(sel.draw.imageId, sel.imageAtlas); // resident: returns the offset
byte[] sub = new byte[8];
sub[0] = 13;
byte[] sub = new byte[10];
sub[0] = 19;
sub[1] = (byte) (offset & 0xff);
sub[2] = (byte) ((offset >> 8) & 0xff);
sub[3] = (byte) (sel.draw.x & 0xff);
sub[4] = (byte) ((sel.draw.x >> 8) & 0xff);
sub[5] = (byte) (sel.draw.y & 0xff);
sub[6] = (byte) ((sel.draw.y >> 8) & 0xff);
sub[7] = (byte) IMAGE_DRAW_OPTIONS;
sub[3] = (byte) ((offset >> 16) & 0xff);
sub[4] = (byte) ((offset >> 24) & 0xff);
sub[5] = (byte) (sel.draw.x & 0xff);
sub[6] = (byte) ((sel.draw.x >> 8) & 0xff);
sub[7] = (byte) (sel.draw.y & 0xff);
sub[8] = (byte) ((sel.draw.y >> 8) & 0xff);
sub[9] = (byte) IMAGE_DRAW_OPTIONS;
return sub;
}
/**
* Group selected glyphs into mode-14 string sub-messages:
* [14][fontTable u16][x u16][y u16][options u8][strlen u8][string]
* Group selected glyphs into mode-20 string sub-messages:
* [20][fontTable u32][x u16][y u16][options u8][strlen u8][string]
* One run per (font, line y, top color) span; within a run, control bytes
* 1..31 adjust x by -10..+20 to hit each glyph's exact position, and each
* glyph advances x by its cached width. Order across runs is free: every
@@ -664,17 +662,19 @@ public final class TexturePlanner {
}
if (!runGlyphs.isEmpty()) {
byte[] stringBytes = string.toByteArray();
byte[] run = new byte[9 + stringBytes.length];
run[0] = 14;
byte[] run = new byte[11 + stringBytes.length];
run[0] = 20;
run[1] = (byte) (fontTable & 0xff);
run[2] = (byte) ((fontTable >> 8) & 0xff);
run[3] = (byte) (first.gx & 0xff);
run[4] = (byte) ((first.gx >> 8) & 0xff);
run[5] = (byte) (first.draw.y & 0xff);
run[6] = (byte) ((first.draw.y >> 8) & 0xff);
run[7] = (byte) (first.top | 0x10); // top color + transparent
run[8] = (byte) stringBytes.length;
System.arraycopy(stringBytes, 0, run, 9, stringBytes.length);
run[3] = (byte) ((fontTable >> 16) & 0xff);
run[4] = (byte) ((fontTable >> 24) & 0xff);
run[5] = (byte) (first.gx & 0xff);
run[6] = (byte) ((first.gx >> 8) & 0xff);
run[7] = (byte) (first.draw.y & 0xff);
run[8] = (byte) ((first.draw.y >> 8) & 0xff);
run[9] = (byte) (first.top | 0x10); // top color + transparent
run[10] = (byte) stringBytes.length;
System.arraycopy(stringBytes, 0, run, 11, stringBytes.length);
outRuns.add(run);
drawn.addAll(runGlyphs);
i = j;
@@ -162,7 +162,7 @@ public class OfflineModelConfig {
public static class Builder {
private OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig();
private OfflineParaformerModelConfig paraformer = new OfflineParaformerModelConfig();
private OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig();
private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build();
private OfflineFireRedAsrModelConfig fireRedAsr = new OfflineFireRedAsrModelConfig();
private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build();
private OfflineNemoEncDecCtcModelConfig nemo = new OfflineNemoEncDecCtcModelConfig();
@@ -191,6 +191,11 @@ public class OfflineModelConfig {
return this;
}
public Builder setWhisper(OfflineWhisperModelConfig whisper) {
this.whisper = whisper;
return this;
}
public Builder setTokens(String tokens) {
this.tokens = tokens;
return this;
@@ -228,16 +233,6 @@ class OfflineParaformerModelConfig {
private final QnnConfig qnnConfig = new QnnConfig();
}
class OfflineWhisperModelConfig {
private final String encoder = "";
private final String decoder = "";
private final String language = "en";
private final String task = "transcribe";
private final int tailPaddings = 1000;
private final boolean enableTokenTimestamps = false;
private final boolean enableSegmentTimestamps = false;
}
class OfflineFireRedAsrModelConfig {
private final String encoder = "";
private final String decoder = "";
@@ -0,0 +1,114 @@
package com.k2fsa.sherpa.onnx;
/**
* Config for sherpa-onnx's offline (non-streaming) Whisper backend.
*
* Field names and types here are load-bearing: the native JNI glue reads
* them by name via reflection (GetFieldID), not through the getters below,
* so they must match sherpa-onnx's own OfflineWhisperModelConfig exactly
* (verified against sherpa-onnx/jni/offline-recognizer.cc at the pinned
* release tag: it unconditionally reads encoder, decoder, language, task,
* tailPaddings, enableTokenTimestamps and enableSegmentTimestamps off the
* OfflineModelConfig's "whisper" field). This mirrors the
* OfflineMoonshineModelConfig.java pattern already used in this package.
*/
public class OfflineWhisperModelConfig {
private final String encoder;
private final String decoder;
private final String language;
private final String task;
private final int tailPaddings;
private final boolean enableTokenTimestamps;
private final boolean enableSegmentTimestamps;
private OfflineWhisperModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
this.language = builder.language;
this.task = builder.task;
this.tailPaddings = builder.tailPaddings;
this.enableTokenTimestamps = builder.enableTokenTimestamps;
this.enableSegmentTimestamps = builder.enableSegmentTimestamps;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public String getLanguage() {
return language;
}
public String getTask() {
return task;
}
public int getTailPaddings() {
return tailPaddings;
}
public boolean getEnableTokenTimestamps() {
return enableTokenTimestamps;
}
public boolean getEnableSegmentTimestamps() {
return enableSegmentTimestamps;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
private String language = "en";
private String task = "transcribe";
private int tailPaddings = 1000;
private boolean enableTokenTimestamps = false;
private boolean enableSegmentTimestamps = false;
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public Builder setLanguage(String language) {
this.language = language;
return this;
}
public Builder setTask(String task) {
this.task = task;
return this;
}
public Builder setTailPaddings(int tailPaddings) {
this.tailPaddings = tailPaddings;
return this;
}
public Builder setEnableTokenTimestamps(boolean enableTokenTimestamps) {
this.enableTokenTimestamps = enableTokenTimestamps;
return this;
}
public Builder setEnableSegmentTimestamps(boolean enableSegmentTimestamps) {
this.enableSegmentTimestamps = enableSegmentTimestamps;
return this;
}
public OfflineWhisperModelConfig build() {
return new OfflineWhisperModelConfig(this);
}
}
}
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<!-- Overrides the CLI-generated app_name (derived from package.json's
lowercase "faceclaw") so the launcher shows a capitalized label. -->
<string name="app_name">Faceclaw</string>
<string name="title_activity_kimera">Faceclaw</string>
</resources>