Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-10-06 13:32:49 -07:00
commit e12cfaaf38
309 files changed
+15906 -3352

No files matched your search

@@ -15,8 +15,18 @@ actual class Sha256Digest actual constructor() {
actual fun currentTimeMillis(): Long = System.currentTimeMillis()
actual fun formatLocalTime(epochMs: Long, pattern: String): String =
SimpleDateFormat(pattern, Locale.US).format(Date(epochMs))
// SimpleDateFormat is costly to build (pattern parse, Calendar, locale data) and not thread-safe,
// so keep one per thread per pattern; the send loop formats a timestamp for every image it logs.
private val localTimeFormats = object : ThreadLocal<HashMap<String, SimpleDateFormat>>() {
override fun initialValue() = HashMap<String, SimpleDateFormat>()
}
actual fun formatLocalTime(epochMs: Long, pattern: String): String {
val format = localTimeFormats.get()!!.getOrPut(pattern) { SimpleDateFormat(pattern, Locale.US) }
// Re-read the default zone each call, as a fresh instance would.
format.timeZone = java.util.TimeZone.getDefault()
return format.format(Date(epochMs))
}
actual fun localUtcOffsetMs(epochMs: Long): Int = java.util.TimeZone.getDefault().getOffset(epochMs)
@@ -11,5 +11,5 @@ private object DrawRedrawScheduler {
internal actual fun scheduleDrawRedraw(delayMs: Int, action: () -> Unit): () -> Unit {
val future = DrawRedrawScheduler.executor.schedule(action, delayMs.toLong(), TimeUnit.MILLISECONDS)
return { future.cancel(false); Unit }
return { future.cancel(false) }
}
@@ -0,0 +1,10 @@
package com.faceclaw.app
/** Callbacks for FaceclawFileDownload, delivered on the constructing thread's Looper. */
interface FaceclawFileDownloadListener {
/** The response finished; for a 2xx status the body is at the destination path, otherwise nothing is written. */
fun onDone(status: Int, contentLength: Long): Unit
/** Network-level failure (connect, TLS, timeout, mid-body drop); the destination is removed. */
fun onFailure(message: String?): Unit
}
@@ -895,8 +895,8 @@ class BleProtocol {
* are fields 5/6 of the deviceReceiveRequestFromApp submessage (field 4); Faceclaw's custom
* firmware additionally appends top-level field 100 with its revision ("Faceclaw/<n>";
* older builds sent "EVENCFW/<ver> <tokens>"), which stock firmware never sends. Returns
* null when the ack carries none of it. Compatibility is judged on the TS side
* (app/g2/firmware-compat.ts); Java only needs to know whether the firmware is ours at all.
* null when the ack carries none of it. The session halts when [FirmwareInfo.isCompatible]
* fails; the user-facing classification and messages live in app/g2/firmware-compat.ts.
*/
@JvmStatic
fun parseSettingsFirmwareInfo(pb: ByteArray): FirmwareInfo? {
@@ -963,7 +963,7 @@ class BleProtocol {
private fun concat(parts: MutableList<ByteArray>): ByteArray {
var out: ByteSink = ByteSink()
for (part in parts) {
if (((part != null) && (part.size > 0))) {
if (part.size > 0) {
out.write(part, 0, part.size)
}
}
@@ -1575,11 +1575,34 @@ class BleProtocol {
/**
* True when the glasses run Faceclaw's custom firmware (any revision). Whether the revision
* is the one this app needs is decided on the TS side, which disconnects on a mismatch;
* this only guards the private modes against stock or third-party firmware in the meantime.
* is one this app can run is [isCompatible]; this only guards the private modes against
* stock or third-party firmware.
*/
fun isFaceclawFirmware(): Boolean {
return extension.trim().startsWith(FACECLAW_EXTENSION_PREFIX)
}
/**
* The revision from a "Faceclaw/<n>" extension, or -1 for stock, pre-revision
* ("EVENCFW/...") and third-party firmware. Mirrors parseFirmwareExtension in
* app/g2/firmware-compat.ts.
*/
fun faceclawRevision(): Int {
val text = extension.trim()
if (!text.startsWith(FACECLAW_EXTENSION_PREFIX)) {
return -1
}
val digits = text.substring(FACECLAW_EXTENSION_PREFIX.length).trimStart().takeWhile { it in '0'..'9' }
return digits.toIntOrNull() ?: -1
}
/**
* True when this is Faceclaw's firmware at [requiredRevision] or newer (revisions only add
* to the contract). Mirrors hasCompatibleFirmware in app/g2/firmware-compat.ts.
*/
fun isCompatible(requiredRevision: Int): Boolean {
val revision = faceclawRevision()
return revision >= 0 && revision >= requiredRevision
}
}
}
@@ -35,5 +35,6 @@ const val CFW_MSG_SET_ROOT_DISPLAY_LIST = 27
const val CFW_MSG_PRESENT = 28
const val CFW_MSG_CREATE_SURFACE = 29
const val CFW_MSG_BRIGHTNESS = 30
const val CFW_MSG_ENTER_EVENHUB = 31
const val CFW_MSG_TYPE_MASK = 127
const val CFW_MSG_FLAG_LENSES_DIFFER = 128
@@ -5,11 +5,23 @@ import kotlin.jvm.JvmStatic
/** Ordered completion and go-back-N recovery for the custom-message window. */
class CfwMessageWindow private constructor() {
companion object {
// Flappy capture: p99 ACK 63ms, max 77ms. Allow full-frame processing
// while recovering a lost reply well before the stock timeout.
const val ACK_TIMEOUT_MS = 500
// An unresolved message is presumed lost only once both kinds of progress on
// its arm have stopped: no write completion for DATA_STALL_MS, and no CFW ack
// for ACK_STALL_MS since it was sent. Android withholds write completions
// while its outgoing queue is congested, so completions mean data is still
// draining; acks mean the firmware is still working through the window. A
// single 500 ms deadline from send replayed large uploads whose acks were
// just slow (an 11 KB upload acked by one lens at 536 ms), and under
// bandwidth pressure (LE Audio streaming) it tore the session down.
const val DATA_STALL_MS = 500
const val ACK_STALL_MS = 3_000
const val MAX_RETRIES = 3
/** When a message sent at [sentAtMs] counts as lost, given its arm's latest write completion and CFW ack. */
@JvmStatic
fun ackDeadline(sentAtMs: Long, lastArmWriteAtMs: Long, lastArmAckAtMs: Long): Long =
maxOf(lastArmWriteAtMs + DATA_STALL_MS, maxOf(sentAtMs, lastArmAckAtMs) + ACK_STALL_MS)
/** ID reuse must not invalidate an earlier command still eligible for replay. */
@JvmStatic
fun canSend(messages: Iterable<OutboundMessage>, candidate: OutboundMessage): Boolean =
@@ -8,6 +8,10 @@ class ConnectionOptions {
const val DESIRED_MTU = 512
const val CONNECT_TIMEOUT_MS = 5_000
// How long one session connect attempt keeps both arms' connections pending before
// giving up. Both are dialed at once, so a slow or missing arm gets the whole
// window (it stays under Android's own ~30 s direct-connect timeout).
const val ARM_CONNECT_WINDOW_MS = 20_000
const val SERVICES_TIMEOUT_MS = 5_000
const val DESCRIPTOR_TIMEOUT_MS = 5_000
const val RING_DESIRED_MTU = 247
@@ -43,7 +47,13 @@ class ConnectionOptions {
const val MAX_CONSECUTIVE_ACK_TIMEOUTS = 8
const val EVEN_APP_WRITE_FAILURE_WINDOW_MS = 15_000
const val IDLE_SLEEP_MS = 100
const val RECONNECT_DELAY_MS = 2_000
// Delay before each consecutive reconnect attempt (the last entry repeats). Android
// keeps an LE link up for a few seconds after its last GATT client closes (4 s on a
// Pixel), so the quick first retry reuses the old links and the glasses never see a
// disconnect; later retries wait long enough for the links to really drop.
@JvmField val RECONNECT_BACKOFF_MS = intArrayOf(2_000, 6_000, 10_000)
// A session that stayed up this long starts the backoff over when it fails.
const val STABLE_SESSION_MS = 30_000
// Cap on rects per batch. Each rect consumes a distinct CFW frame id, and the
// firmware's duplicate-fid ring holds 16, so keep several batches of history
// within it. Above this the split is abandoned for a single bounding box.
@@ -21,3 +21,5 @@ const val CFW_TEXTURE_OPT_INVERSE = 32
/** Rounded-rectangle border sentinel, not a color or flag bit. */
const val DRAW_ROUNDED_RECT_NO_BORDER = 16
/** Revision 36: the rounded rectangle's optional outside color is absent (bridge sentinel). */
const val DRAW_ROUNDED_RECT_NO_OUTSIDE = 16
@@ -13,7 +13,7 @@ class DrawClip(val x: Int, val y: Int, val width: Int, val height: Int) {
fun translated(dx: Int, dy: Int) = DrawClip(x + dx, y + dy, width, height)
}
/** Revision 35 wire grammar, shared by scene planning and the local renderer. */
/** Revision 36 wire grammar, shared by scene planning and the local renderer. */
object DrawProtocol {
/** Mirrors g2flash/patches/zlib_glue.c. */
const val DRAW = CFW_MSG_DRAW_CALLS
@@ -62,6 +62,27 @@ object DrawProtocol {
write: DrawWriter.() -> Unit): ByteArray =
call(op, encode(write), target, depth, clip)
/**
* [call] moved [extra] depth units further, as if played from a list called at that depth:
* adds to its depth field, or inserts one (after any target override) when it has none. The
* sum saturates at the field's range rather than failing the whole presentation.
*/
fun withAddedDepth(call: ByteArray, extra: Int): ByteArray {
if (extra == 0) return call
val flags = call[1].toInt() and 255
val at = if (flags and DRAW_FLAG_RESOURCE_TARGET != 0) 4 else 2
val result = if (flags and DRAW_FLAG_DEPTH != 0) {
call.copyOf().also { it[at] = addDepth(call[at].toInt(), extra).toByte() }
} else {
call.copyOfRange(0, at) + byteArrayOf(addDepth(0, extra).toByte()) + call.copyOfRange(at, call.size)
}
result[1] = (flags or DRAW_FLAG_DEPTH).toByte()
return result
}
/** Two depths combined, saturated to the s8 depth field. */
fun addDepth(depth: Int, extra: Int): Int = (depth + extra).coerceIn(-128, 127)
fun depthOffset(depth: Int, right: Boolean): Int =
if (right) -floorHalf(depth + 1) else floorHalf(depth)
@@ -72,12 +93,17 @@ object DrawProtocol {
border: Int = DRAW_ROUNDED_RECT_NO_BORDER, depth: Int? = null,
): ByteArray = roundedRect(DrawValue.Integer(x), DrawValue.Integer(y), width, height, radius, background, border, depth)
/**
* The background is max-blended, so 0 leaves the interior as it is. [outside] (revision 36),
* when given, is the color for the bounding box's pixels outside the rounded shape.
*/
fun roundedRect(
x: DrawValue, y: DrawValue, width: Int, height: Int, radius: Int, background: Int,
border: Int = DRAW_ROUNDED_RECT_NO_BORDER, depth: Int? = null, clip: DrawClip? = null,
outside: Int? = null,
): ByteArray {
require(width in 1..640 && height in 1..480 && radius in 0..65535)
require(background in 0..15 && border in 0..DRAW_ROUNDED_RECT_NO_BORDER)
require(background in 0..15 && border in 0..DRAW_ROUNDED_RECT_NO_BORDER && (outside == null || outside in 0..15))
return call(DRAW_OP_ROUNDED_RECT, depth = depth, clip = clip) {
writeExtended(x)
writeExtended(y)
@@ -86,6 +112,7 @@ object DrawProtocol {
writeU16(radius)
writeU8(background)
writeU8(border)
if (outside != null) writeU8(outside)
}
}
@@ -113,7 +113,7 @@ class MessageBuilder {
0,
streamId,
payload,
CfwMessageWindow.ACK_TIMEOUT_MS,
CfwMessageWindow.ACK_STALL_MS,
tileIndex,
leftArm,
)
@@ -203,6 +203,20 @@ class MessageBuilder {
)
}
/**
* CFW mode 31: the firmware queues its own copy of [createLayout]'s page, and ACKs whether
* or not that page already existed. Stock firmware never ACKs a repeated create.
*/
fun enterEvenHub(leftArm: Boolean): OutboundMessage {
return customMessage(
"enter-evenhub",
byteArrayOf(CFW_MSG_ENTER_EVENHUB.toByte()),
"enter EvenHub",
-1,
leftArm,
)
}
fun startupTextProbe(): OutboundMessage {
var magic: Int = magicPool.allocate()
return OutboundMessage(
@@ -2,9 +2,13 @@ package com.faceclaw.app
import kotlin.jvm.JvmOverloads
/** The delta base contains only app pixels. Shell resources are composed at presentation. */
/**
* The delta base contains only app pixels, except in frames that bake shell content (see [plan]).
* Shell resources are composed at presentation.
*/
class ScenePlanner(private val cache: ResourceCacheState) {
class Plan(val commands: List<ByteArray>, val nextFid: Int)
/** [bakedLayers] and [bakedSelections] say what a frame over the resource budget baked into its pixels. */
class Plan(val commands: List<ByteArray>, val nextFid: Int, val bakedLayers: Int = 0, val bakedSelections: Boolean = false)
private val owned = HashMap<String, CachedResource>()
private val pixels = HashMap<String, ByteArray>()
private var previous: ByteArray? = null
@@ -12,6 +16,14 @@ class ScenePlanner(private val cache: ResourceCacheState) {
private var screenHeight = 0
private var epoch = -1
private var root = -1
/**
* When the shell's surfaces and retained drawings don't all fit the resource budget, the frame
* bakes the bottom of the scene into its screen pixels until the rest fits: first the scene's
* own retained drawings, then layers from the bottom (the chrome) up, so the top overlay stays
* a live surface longest. A frame that can't be planned even fully flattened throws, leaving
* the cache and this planner as they were.
*/
@JvmOverloads
fun plan(screen: ByteArray, width: Int, height: Int, draws: Array<SurfaceCompositor.ScreenDraw>?, scene: ShellScene, firstId: Int, textures: Boolean = true, incremental: Boolean = true, multiRect: Boolean = true, maxRects: Int = 6): Plan {
require(width in 1..640 && height in 1..480)
@@ -19,42 +31,66 @@ class ScenePlanner(private val cache: ResourceCacheState) {
require(screen.size == ((width + 1) / 2) * height)
if (width != screenWidth || height != screenHeight) previous = null
screenWidth = width; screenHeight = height
val rollback = cache.checkpoint()
try {
planScene(screen, width, height, draws, scene, firstId, textures, incremental, multiRect, maxRects)?.let { return it }
for (layerCount in 0..scene.layers.size) {
if (layerCount == 0 && scene.selections.isEmpty()) continue
rollback()
val kept = ShellScene(scene.layers.drop(layerCount), screenDepth = scene.screenDepth)
planScene(screen, width, height, draws, kept, firstId, textures, incremental, multiRect, maxRects) {
scene.flatten(screen, width, height, layerCount)
}?.let { return Plan(it.commands, it.nextFid, layerCount, scene.selections.isNotEmpty()) }
}
error("Shell scene exceeds the resource budget even fully flattened")
} catch (t: Throwable) {
rollback()
throw t
}
}
/**
* One attempt at [plan], or null if [scene] doesn't fit the budget. [baked] supplies the
* screen pixels once the shell fits. Planner state changes only when the attempt succeeds.
*/
private fun planScene(screen: ByteArray, width: Int, height: Int, draws: Array<SurfaceCompositor.ScreenDraw>?, scene: ShellScene, firstId: Int, textures: Boolean, incremental: Boolean, multiRect: Boolean, maxRects: Int, baked: (() -> ByteArray)? = null): Plan? {
val commands = ArrayList<ByteArray>()
val keys = scene.layers.map { "shell:${it.key}:${it.width}x${it.height}" }
val surfaces = scene.layers.mapIndexed { i, layer -> owned.getOrPut(keys[i]) {
CachedResource(DrawProtocol.rawImage(layer.width, layer.height), keys[i])
} }
owned.keys.retainAll(keys.toSet()); pixels.keys.retainAll(keys.toSet())
val surfaces = scene.layers.mapIndexed { i, layer ->
owned[keys[i]] ?: CachedResource(DrawProtocol.rawImage(layer.width, layer.height), keys[i])
}
val selectedResources = scene.retainedResources
cache.pin(surfaces + selectedResources)
val absent = surfaces.map { cache.resourceId(it) < 0 }
val prepared = cache.prepare(surfaces + selectedResources)
if (prepared.any { it < 0 }) return null
val ids = prepared.copyOfRange(0, surfaces.size)
val selectedIds = prepared.copyOfRange(surfaces.size, prepared.size)
check(prepared.all { it >= 0 }) { "Shell surfaces exceed the resource budget" }
val frame = baked?.invoke() ?: screen
commands.addAll(cache.drainCommands(3600))
val rootCalls = scene.calls(width, height, ids, selectedIds)
for ((i, layer) in scene.layers.withIndex()) {
val old = if (absent[i]) null else pixels[keys[i]]
commands.addAll(DrawProtocol.messages(pixelCalls(old, layer.packed, layer.width, layer.height, ids[i])))
pixels[keys[i]] = layer.packed
}
val base = if (incremental) previous else null
val texture = if (textures) TexturePlanner.plan(base, screen, width, height, draws, cache, firstId, multiRect, maxRects) else null
val texture = if (textures) TexturePlanner.plan(base, frame, width, height, draws, cache, firstId, multiRect, maxRects) else null
if (texture != null) {
commands.addAll(texture.resourceCommands)
commands.addAll(DrawProtocol.messages(DrawProtocol.fromOptimized(texture.payload, width, height)))
} else commands.addAll(DrawProtocol.messages(pixelCalls(base, screen, width, height, null)))
} else commands.addAll(DrawProtocol.messages(pixelCalls(base, frame, width, height, null)))
val list = CachedResource(DrawProtocol.displayList(rootCalls))
val rootId = cache.prepare(listOf(list))[0]
check(rootId >= 0) { "No space for root display list" }
if (rootId < 0) return null
commands.addAll(cache.drainCommands(3600))
cache.pin(surfaces + selectedResources + list)
val evicted = commands.any { it[0].toInt() == ResourceCacheState.EVICT_MODE }
if (evicted) commands.add(0, DrawProtocol.root(DrawProtocol.SCREEN))
if (root != rootId || evicted) commands.add(DrawProtocol.root(rootId))
commands.add(byteArrayOf(DrawProtocol.PRESENT.toByte()))
root = rootId; previous = screen.copyOf()
owned.keys.retainAll(keys.toSet()); pixels.keys.retainAll(keys.toSet())
for ((i, key) in keys.withIndex()) { owned[key] = surfaces[i]; pixels[key] = scene.layers[i].packed }
root = rootId; previous = frame.copyOf()
return Plan(commands, texture?.nextFid ?: firstId)
}
private fun pixelCalls(old: ByteArray?, next: ByteArray, width: Int, height: Int, target: Int?): List<ByteArray> {
@@ -63,7 +99,7 @@ class ScenePlanner(private val cache: ResourceCacheState) {
val calls = ArrayList<ByteArray>()
for (top in 0 until height step 32) {
val rows = minOf(32, height - top)
if (old != null && (top * stride until (top + rows) * stride).all { old[it] == next[it] }) continue
if (old != null && sameBytes(old, next, top * stride, (top + rows) * stride)) continue
var left = width; var right = -1
for (y in top until top + rows) for (x in 0 until width) {
val p = y * stride + x / 2; val shift = if (x % 2 == 0) 4 else 0
@@ -75,4 +111,12 @@ class ScenePlanner(private val cache: ResourceCacheState) {
}
return calls
}
private fun sameBytes(a: ByteArray, b: ByteArray, from: Int, to: Int): Boolean {
var i = from
while (i < to) {
if (a[i] != b[i]) return false
i++
}
return true
}
}
@@ -308,7 +308,7 @@ class TexturePlanner {
i++
continue
}
var entry: FwGlyphAtlas.Entry? = FwGlyphAtlas.get(draw.fwCps!![i])
var entry: FwGlyphAtlas.Entry? = FwGlyphAtlas.get(draw.fwCps[i])
if ((entry == null)) {
baked++
i++
@@ -351,7 +351,7 @@ class TexturePlanner {
var bytesToLastInk: Int = 0
var bytesSoFar: Int = 0
while (((j < n) && ok[j])) {
var encodedLength: Int = utf8Length(draw.fwCps!![j])
var encodedLength: Int = utf8Length(draw.fwCps[j])
if (((bytesSoFar + encodedLength) > 255)) {
break
}
@@ -379,7 +379,7 @@ class TexturePlanner {
run {
var k: Int = i
while ((k <= lastInk)) {
var encoded: ByteArray = codePointUtf8(draw.fwCps!![k])
var encoded: ByteArray = codePointUtf8(draw.fwCps[k])
encoded.copyInto(sub, pos, 0, 0 + encoded.size)
pos += encoded.size
k++
@@ -393,7 +393,7 @@ class TexturePlanner {
outPunches.add(
FwPunch(
entries[k]!!,
((draw.x + draw.fwDx!![k]) + entries[k]!!.ofsX),
((draw.x + draw.fwDx[k]) + entries[k]!!.ofsX),
(draw.y + entries[k]!!.inkTop),
)
)
@@ -359,23 +359,7 @@ internal fun GlassesSessionCore.firstUnpairedArm(): String? {
*/
internal fun GlassesSessionCore.handleUnpairedFailure(address: String) {
logError("Connect failed and $address is not paired; suspending reconnect")
monitor.withLock {
reconnectHalted = true
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
shutdownRequested = false
chargingMode = false
imageRetryAfterMs = 0
displayedFingerprint = ""
faceclawWakePendingNonce = -1
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
clearAllMessagesLocked("arm not paired: $address")
reconnectAfterMs = Long.MAX_VALUE
link.disconnect(rightAddress)
link.disconnect(leftAddress)
}
monitor.withLock { haltReconnectLocked("arm not paired: $address") }
if (!userDisconnectRequested) {
setStateDisplay(
"unpaired",
@@ -386,11 +370,70 @@ internal fun GlassesSessionCore.handleUnpairedFailure(address: String) {
interruptibleSleep.interrupt()
}
internal fun GlassesSessionCore.handleTransportFailure(reason: String?) {
/**
* The settings ack reported firmware this app can't run: stock, another
* project's, or an older Faceclaw revision (see
* [BleProtocol.FirmwareInfo.isCompatible]). Everything else this session
* would send is a private-mode message the glasses may misread (stock leaves
* them unacked until the CFW retry limit drops the link), and a redial would
* only find the same firmware, so park the worker loop like
* [handleUnpairedFailure], without any cleanup messages. The firmware info
* was emitted just before this, so the TS side can explain the mismatch and
* offer the custom-firmware install.
*/
internal fun GlassesSessionCore.handleIncompatibleFirmware() {
val info = monitor.withLock {
val pending = incompatibleFirmware
incompatibleFirmware = null
pending
} ?: return
logError("Glasses report incompatible firmware L=" + info.leftVersion + " R=" + info.rightVersion
+ " ext=\"" + info.extension + "\" (need Faceclaw/" + requiredFirmwareRevision + "); suspending reconnect")
monitor.withLock { haltReconnectLocked("incompatible firmware") }
if (!userDisconnectRequested) {
setStateDisplay(
"incompatible-firmware",
"The glasses firmware is not compatible with this version of Faceclaw."
+ " Install the custom firmware, then connect."
)
}
interruptibleSleep.interrupt()
}
/**
* Park the worker loop for good: drop both arms, discard queued traffic and
* stop redialing. Only an explicit connect (start() on a fresh communicator)
* tries again.
*/
internal fun GlassesSessionCore.haltReconnectLocked(reason: String) {
reconnectHalted = true
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
shutdownRequested = false
chargingMode = false
imageRetryAfterMs = 0
displayedFingerprint = ""
faceclawWakePendingNonce = -1
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
clearAllMessagesLocked(reason)
reconnectAfterMs = Long.MAX_VALUE
link.disconnect(rightAddress)
link.disconnect(leftAddress)
}
/**
* Tear the session down and schedule a reconnect. [displayStatus], when given, replaces the
* generic "Reconnecting after <reason>" status line.
*/
internal fun GlassesSessionCore.handleTransportFailure(reason: String?, displayStatus: String? = null) {
logError("Transport failure: $reason")
val retryDelayMs: Long
monitor.withLock {
maybeEmitEvenAppConflictLocked(reason)
finishBenchmarkLocked(true, "transport failure")
val sessionWasReady = sessionReady
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
@@ -402,16 +445,38 @@ internal fun GlassesSessionCore.handleTransportFailure(reason: String?) {
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
clearAllMessagesLocked("transport failure: $reason")
reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS
retryDelayMs = scheduleReconnectLocked(sessionWasReady)
link.disconnect(rightAddress)
link.disconnect(leftAddress)
}
logLine("reconnect attempt in " + retryDelayMs + "ms")
if (!userDisconnectRequested) {
setStateDisplay("retrying", if (reason == null || reason.isEmpty()) "Reconnecting..." else "Reconnecting after $reason")
setStateDisplay("retrying", when {
displayStatus != null -> displayStatus + " Retrying in " + ((retryDelayMs + 999) / 1000) + "s."
reason == null || reason.isEmpty() -> "Reconnecting..."
else -> "Reconnecting after $reason"
})
}
interruptibleSleep.interrupt()
}
/**
* Set [reconnectAfterMs] for the next attempt after a failure and return the delay. Repeated
* failures back off along RECONNECT_BACKOFF_MS; a session that stayed up for
* STABLE_SESSION_MS before failing ([sessionWasReady]) starts the schedule over.
*/
internal fun GlassesSessionCore.scheduleReconnectLocked(sessionWasReady: Boolean): Long {
val now = now()
if (sessionWasReady && now - lastSessionReadyAtMs >= ConnectionOptions.STABLE_SESSION_MS) {
consecutiveReconnects = 0
}
val schedule = ConnectionOptions.RECONNECT_BACKOFF_MS
val delayMs = schedule[minOf(consecutiveReconnects, schedule.size - 1)].toLong()
consecutiveReconnects++
reconnectAfterMs = now + delayMs
return delayMs
}
internal fun GlassesSessionCore.resetSessionStateLocked() {
ringBattery = -1
ringCharging = -1
@@ -426,7 +491,9 @@ internal fun GlassesSessionCore.resetSessionStateLocked() {
ringConnected = false
ringNotificationsReady = false
reconnectAfterMs = 0
consecutiveReconnects = 0
reconnectHalted = false
incompatibleFirmware = null
ringReconnectAfterMs = 0
lastAckAtMs = 0
lastIncomingAtMs = 0
@@ -490,6 +557,22 @@ internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String)
}
}
internal fun GlassesSessionCore.emitAudioMonitorPacket(data: ByteArray, arm: String, arrivalMs: Long) {
if (audioMonitorListeners.isEmpty()) {
return
}
val copy = data.copyOf()
host.postToMain {
for (monitorListener in audioMonitorListeners) {
try {
monitorListener.onAudioPacket(copy, arm, arrivalMs)
} catch (t: Throwable) {
logWarn("audio monitor listener failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitRingEvent(event: G2Event, frameId: Int) {
val current = listener
if (current == null) {
@@ -124,12 +124,19 @@ class GlassesSessionCore(
internal val compassSubscriptions = CopyOnWriteList<CompassSubscription>(platform)
internal val ambientLightListeners = CopyOnWriteList<FaceclawAmbientLightListener>(platform)
internal val micStatusListeners = CopyOnWriteList<FaceclawMicStatusListener>(platform)
internal val audioMonitorListeners = CopyOnWriteList<FaceclawAudioPacketListener>(platform)
@Volatile internal var running = false
@Volatile internal var userDisconnectRequested = false
// Set when a connect attempt failed while an arm's OS bond is gone:
// retrying is pointless until the user re-pairs, so the worker loop parks
// Set when a connect attempt failed while an arm's OS bond is gone, or the
// glasses reported firmware this app can't run: retrying is pointless until
// the user re-pairs or installs the custom firmware, so the worker loop parks
// instead of redialing. Cleared by start() (a fresh explicit connect).
@Volatile internal var reconnectHalted = false
// The Faceclaw firmware revision this app needs; see setRequiredFirmwareRevision.
@Volatile internal var requiredFirmwareRevision = 0
// Firmware info from a settings ack that failed requiredFirmwareRevision. While
// set, driveSession sends nothing; the worker loop halts on it (handleIncompatibleFirmware).
internal var incompatibleFirmware: BleProtocol.FirmwareInfo? = null
internal var phase = "disconnected"
internal var status = "Disconnected."
@@ -275,9 +282,9 @@ class GlassesSessionCore(
/**
* The last firmware-info read said the glasses run Faceclaw's custom
* firmware. Gates the private modes (cleanup, resource cache, ...) so stock
* or third-party firmware never sees them; the TS side checks the actual
* revision and disconnects on a mismatch, so no per-feature gating is
* needed here.
* or third-party firmware never sees them; a revision mismatch halts the
* session (handleIncompatibleFirmware), so no per-feature gating is needed
* here.
*/
internal var customFirmwareDetected = false
internal val brightnessPolicy = BrightnessPolicy()
@@ -289,6 +296,8 @@ class GlassesSessionCore(
internal var lastCfwCleanupAckMagic = 0
internal var reconnectAfterMs = 0L
// Failed connects/sessions since the last stable session; indexes RECONNECT_BACKOFF_MS.
internal var consecutiveReconnects = 0
internal var ringReconnectAfterMs = 0L
internal var lastAckAtMs = 0L
internal var lastIncomingAtMs = 0L
@@ -350,7 +359,6 @@ class GlassesSessionCore(
internal var phoneLockState = -1
internal var lastPhoneLockCheckAtMs = 0L
internal var audioCaptureActive = false
internal var firmwareInfoQueried = false
// Glasses are in the charging case: nobody is wearing them, so display
// communication pauses and only battery polls flow (see driveSession).
internal var chargingMode = false
@@ -397,15 +405,17 @@ class GlassesSessionCore(
internal val desiredTilesLock: ProtocolLock = platform.createLock()
internal var desiredFingerprint = ""
// Headerless packed 4bpp frame (see BmpUtil.pack4bppFromGray8) plus its
// pixel dimensions.
internal var desiredPacked: ByteArray? = ByteArray(0)
// Composited 8bpp frame plus its pixel dimensions. The send loop packs it to
// the headerless 4bpp wire format (BmpUtil.pack4bppFromGray8) when it picks
// the frame up, so frames superseded before sending are never packed and
// the packing stays off the submitting (main) thread.
internal var desiredGray: ByteArray? = ByteArray(0)
internal var desiredWidth = 0
internal var desiredHeight = 0
internal var desiredPaintMs = 0
internal var desiredFrameId = 0
// Screen-space deferred draws (glyphs + images) whose pixels are baked
// into desiredPacked; the resource-cache planner may replay them as
// into desiredGray; the resource-cache planner may replay them as
// on-glasses cached draws.
internal var desiredDraws: Array<SurfaceCompositor.ScreenDraw>? = arrayOf()
// (frame, reason) of the last "waiting to send" line, so a frame that
@@ -432,6 +442,11 @@ class GlassesSessionCore(
internal val pendingMessages = ArrayDeque<OutboundMessage>()
internal val cfwTransports = arrayOf(CfwTransport(platform), CfwTransport(platform))
// Per arm (0 = left, 1 = right, like cfwTransports): when a write to it last
// completed and when a CFW ack last arrived from it. The CFW window's liveness
// signals (see CfwMessageWindow.ackDeadline).
internal val lastArmWriteAtMs = LongArray(2)
internal val lastArmCfwAckAtMs = LongArray(2)
internal val inFlightMessages = ArrayDeque<OutboundMessage>()
internal var prewrittenMessage: OutboundMessage? = null
internal var prewrittenFrames: List<ByteArray> = emptyList()
@@ -468,8 +483,10 @@ class GlassesSessionCore(
val active = monitor.withLock { running }
if (!active) return false
return try {
link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE,
val result = link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE,
ConnectionOptions.WRITE_TIMEOUT_MS)
if (result) monitor.withLock { noteWriteCompletedLocked(address, now()) }
result
} catch (t: Throwable) {
logLine("raw packet write failed: " + safeMessage(t))
false
@@ -482,6 +499,15 @@ class GlassesSessionCore(
emitPhoneLockStateIfChanged(true)
}
/**
* The Faceclaw firmware revision this app needs (REQUIRED_FACECLAW_FIRMWARE_VERSION in
* app/g2/firmware-compat.ts; the TS bridges pass it in before start()). A settings ack
* reporting anything else halts the session. 0 accepts any Faceclaw revision.
*/
fun setRequiredFirmwareRevision(revision: Int) {
requiredFirmwareRevision = revision
}
/** Start the worker (through the host); false when it was already running. */
fun start(): Boolean {
monitor.withLock {
@@ -491,6 +517,7 @@ class GlassesSessionCore(
running = true
userDisconnectRequested = false
reconnectHalted = false
incompatibleFirmware = null
shutdownRequested = false
host.startWorker { run() }
if (hasRingAddress()) {
@@ -1108,6 +1135,24 @@ class GlassesSessionCore(
}
}
/**
* Observe every packet on the audio (render) characteristic, on the main
* thread, whether or not any capture is active. Purely passive: sends
* nothing to the glasses and does not affect the capture listener, so it
* shows what another app's mic activation actually produces.
*/
fun addAudioMonitorListener(listener: FaceclawAudioPacketListener?) {
if (listener != null) {
audioMonitorListeners.add(listener)
}
}
fun removeAudioMonitorListener(listener: FaceclawAudioPacketListener?) {
if (listener != null) {
audioMonitorListeners.remove(listener)
}
}
/**
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
@@ -1200,6 +1245,9 @@ class GlassesSessionCore(
/** The current composite for previews/screenshots, or null before any surface exists. */
fun previewComposite(): SurfaceCompositor.Composite? = compositor.previewComposite()
/** See SurfaceCompositor.setPreviewAnimationListener; runs on the redraw scheduler's thread. */
fun setPreviewAnimationListener(listener: (() -> Unit)?) = compositor.setPreviewAnimationListener(listener)
/**
* Show or hide a compositor surface, immediately submitting the resulting
* frame. Recompositing here (rather than waiting for the next surface
@@ -1217,9 +1265,7 @@ class GlassesSessionCore(
val frameId = frameTimings.startFrame(
"compositor:visible $id=$visible")
compositor.setSurfaceVisible(id, visible)
val composite = compositor.composite()
val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height)
storeDesiredComposite(composite, packed, 0, frameId)
storeDesiredComposite(compositor.composite(), 0, frameId)
}
/**
@@ -1231,9 +1277,7 @@ class GlassesSessionCore(
val frameId = frameTimings.startFrame(
"compositor:" + (if (blanked) "blank" else "unblank"))
compositor.setBlanked(blanked)
val composite = compositor.composite()
val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height)
storeDesiredComposite(composite, packed, 0, frameId)
storeDesiredComposite(compositor.composite(), 0, frameId)
}
/**
@@ -1253,7 +1297,7 @@ class GlassesSessionCore(
fun submitShellScene(bytes: ByteReader, paintMs: Int, frameId: Int) {
compositor.setShellScene(bytes)
val composite = compositor.composite()
storeDesiredComposite(composite, BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height), paintMs, frameId)
storeDesiredComposite(composite, paintMs, frameId)
}
/** Legacy compositor dimming for callers without a shell scene. */
@@ -1290,17 +1334,15 @@ class GlassesSessionCore(
val composite = compositor.applyAndComposite(
surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs)
frameTimings.spanEnd(frameId, "composite")
// Pack the composited 8bpp buffer down to the headerless 4bpp frame
// format the wire planners consume; BMP framing is added later only for
// the uncompressed fallback.
frameTimings.spanStart(frameId, "pack-4bpp")
val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height)
frameTimings.spanEnd(frameId, "pack-4bpp")
storeDesiredComposite(composite, packed, paintMs, frameId)
storeDesiredComposite(composite, paintMs, frameId)
}
/** See [SurfaceCompositor.isSurfaceCurrent]. */
fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean =
compositor.isSurfaceCurrent(surfaceId, fingerprint)
/** Store a composite as the desired frame unless a newer one won the race. */
internal fun storeDesiredComposite(composite: SurfaceCompositor.Composite, packed: ByteArray, paintMs: Int, frameId: Int) {
internal fun storeDesiredComposite(composite: SurfaceCompositor.Composite, paintMs: Int, frameId: Int) {
var supersededFrameId = 0
var stale = false
desiredTilesLock.locked {
@@ -1311,7 +1353,7 @@ class GlassesSessionCore(
} else {
lastStoredCompositeSeq = composite.seq
supersededFrameId = desiredFrameId
desiredPacked = packed
desiredGray = composite.screenGray
desiredWidth = composite.width
desiredHeight = composite.height
desiredFingerprint = composite.fingerprint
@@ -1547,6 +1589,10 @@ class GlassesSessionCore(
break
}
emitPhoneLockStateIfChanged(false)
if (monitor.withLock { incompatibleFirmware != null }) {
handleIncompatibleFirmware()
continue
}
if (!sessionReady) {
if (reconnectHalted) {
interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS.toLong())
@@ -1600,6 +1646,8 @@ class GlassesSessionCore(
val acks = CfwTransport.parseAcks(data) ?: return
monitor.withLock {
lastIncomingAtMs = now()
val arm = armIndex(address)
if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs
for (ack in acks) {
for (message in inFlightMessages) {
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
@@ -1637,10 +1685,15 @@ class GlassesSessionCore(
else
null
var emitWearState = false
var putOnWhileCharging = false
var event: G2Event? = null
monitor.withLock {
lastIncomingAtMs = now()
if (decodedWearState >= 0 && decodedWearState != wearState) {
// An observed OFF_HEAD -> ON_HEAD means charging is over, up
// to CHARGING_BATTERY_POLL_MS before a battery poll would say
// so. Like TS's put-on wake, a first snapshot doesn't count.
putOnWhileCharging = chargingMode && wearState == 0 && decodedWearState == 1
wearState = decodedWearState
emitWearState = true
}
@@ -1805,13 +1858,26 @@ class GlassesSessionCore(
updateSilentModeLocked(false)
}
if ("sys-event" == decoded.kind) {
if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT || decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) {
if (decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) {
if (shutdownRequested) {
lastShutdownExitAtMs = now()
}
fixedLayoutCreated = false
displayedFingerprint = ""
clearAllMessagesLocked("firmware exit event")
} else if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_ENTER) {
logLine("firmware overlay opened over the EvenHub page")
} else if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT) {
// Despite its name this does not end the page: stock
// evenhub_page_event_handler sends it when a firmware
// overlay announced by FOREGROUND_ENTER closes, and
// restores the page and its keepalive. Teardown is
// always SYSTEM_EXIT/ABNORMAL_EXIT. Re-creating the
// layout here sent a create the firmware ignores while
// the page exists, so the session timed out (#42).
// Re-present over whatever the overlay left instead.
logLine("firmware overlay closed; re-presenting")
lastEnqueuedFingerprint = ""
}
}
}
@@ -1822,6 +1888,11 @@ class GlassesSessionCore(
logLine(if (decodedWearState > 0) "wear state ON_HEAD" else "wear state OFF_HEAD")
emitWearState(decodedWearState > 0)
}
if (putOnWhileCharging) {
// After the emit, so TS sees the put-on while still in its
// charging phase and wakes once the rebuilt session is ready.
monitor.withLock { endChargingModeLocked("glasses put on while charging") }
}
if (compassEvent != null) {
emitCompassEvent(compassEvent)
}
@@ -1877,10 +1948,11 @@ class GlassesSessionCore(
lastIncomingAtMs = arrivalMs
listenerToCall = if (audioCaptureActive) audioPacketListener else null
}
val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?"
emitAudioMonitorPacket(data, arm, arrivalMs)
if (listenerToCall == null) {
return
}
val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?"
try {
listenerToCall.onAudioPacket(data.copyOf(), arm, arrivalMs)
} catch (t: Throwable) {
@@ -1889,16 +1961,19 @@ class GlassesSessionCore(
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
monitor.withLock {
if (address == null) {
return
}
if (isConfiguredRingAddress(address)) {
if (address == null) {
return
}
if (isConfiguredRingAddress(address)) {
monitor.withLock {
if (connected != ringConnected) ringConnectedChangedAtMs = now()
if (connected && !ringConnected) ringConnectCount++
ringConnected = connected
ringNotificationsReady = false
if (!connected) {
if (connected) {
// A background connect landed: finish setting it up right away.
ringReconnectAfterMs = 0
} else {
ringNotificationsReady = false
ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS
// Half-received fragments and unsent ACKs do not survive the
// link; the sequence counter restarts on the next connect.
@@ -1908,10 +1983,14 @@ class GlassesSessionCore(
directRingCharging = -1
emitBatteryState(headsetBattery, headsetCharging)
}
logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected")
// Wakes the ring worker, which owns ring connect/setup.
monitor.signalAll()
return
}
logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected")
return
}
var lostSession = false
monitor.withLock {
if (address.equals(rightAddress, ignoreCase = true)) {
rightConnected = connected
} else if (address.equals(leftAddress, ignoreCase = true)) {
@@ -1920,6 +1999,10 @@ class GlassesSessionCore(
return
}
if (!connected) {
// Before the session is ready, this is an attempt failing (or an arm
// dropping mid-setup): the connect sequence notices on its own and
// schedules the retry, so leave the backoff alone.
lostSession = sessionReady
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
@@ -1928,17 +2011,14 @@ class GlassesSessionCore(
audioPacketListener = null
clearAllMessagesLocked("connection lost")
displayedFingerprint = ""
if (!reconnectHalted) {
reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS
if (lostSession && !reconnectHalted) {
scheduleReconnectLocked(true)
}
}
}
interruptibleSleep.interrupt()
if (connected) {
setStateDisplay("connected", "Connected.")
} else if (!reconnectHalted) {
// While parked on a missing bond, keep the "unpaired" display: this
// callback is just the teardown of the arm that did connect.
if (lostSession && !reconnectHalted) {
// While parked on a missing bond, keep the "unpaired" display.
setStateDisplay("connecting", "Connecting to the glasses...")
}
}
@@ -1947,10 +2027,21 @@ class GlassesSessionCore(
// Connect sequence (worker thread)
internal fun connectLoopOnce() {
monitor.withLock {
// TS treats a wear snapshot as per-transport and forgets it on a
// reconnect; forget ours too, or the new session's first report
// (the CFW query reply) is deduped away and TS never relearns it.
wearState = -1
}
setStateDisplay("connecting", "Connecting to the glasses...")
try {
connectArm(rightAddress, true)
connectArm(leftAddress, true)
// Dial both arms at once, so the left arm's attempt is already pending (and
// can land) while the right arm is being set up; they share one window.
val deadlineMs = now() + ConnectionOptions.ARM_CONNECT_WINDOW_MS
link.beginConnect(rightAddress, false)
link.beginConnect(leftAddress, false)
connectArm(rightAddress, true, deadlineMs)
connectArm(leftAddress, true, deadlineMs)
if (!sleepDuringConnectSettling(800)) {
return
}
@@ -1998,16 +2089,16 @@ class GlassesSessionCore(
setStateDisplay("connected", "Connected.")
logLine("session ready")
monitor.withLock {
// Query settings promptly on the first session so firmware
// Query settings at the start of every session so firmware
// version/extension (and battery) arrive without waiting for
// the input-quiet battery poll. The settings response doubles as
// the firmware-compatibility check surfaced during onboarding.
if (!firmwareInfoQueried) {
firmwareInfoQueried = true
lastBatteryRefreshAtMs = now()
pendingMessages.addLast(createBatteryQueryMessageLocked())
logLine("queue settings query for firmware info")
}
// the input-quiet battery poll. The reply is the firmware-
// compatibility check, and the drive loop only creates the
// layout once the queue is empty, so it lands before any
// custom-firmware traffic -- including after a reconnect to
// glasses that were reflashed in the meantime.
lastBatteryRefreshAtMs = now()
pendingMessages.addLast(createBatteryQueryMessageLocked())
logLine("queue settings query for firmware info")
}
// Wake the ring worker; it owns the (blocking) ring connect.
monitor.signalAll()
@@ -2016,6 +2107,17 @@ class GlassesSessionCore(
val unpairedArm = firstUnpairedArm()
if (unpairedArm != null) {
handleUnpairedFailure(unpairedArm)
} else if (t is ArmUnreachableException) {
val otherArm = if (t.side == "left") rightAddress else leftAddress
if (link.isConnected(otherArm)) {
handleTransportFailure(
t.side + " arm not found",
"Can't reach the " + t.side + " arm of the glasses. It may be connected to"
+ " another device, or be off or out of range."
)
} else {
handleTransportFailure("glasses not found", "Can't find the glasses. They may be off or out of range.")
}
} else {
handleTransportFailure("connect failed")
}
@@ -2036,9 +2138,10 @@ class GlassesSessionCore(
}
}
internal fun connectArm(address: String, enableRenderNotify: Boolean) {
if (!link.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw IllegalStateException("connect failed: $address")
internal fun connectArm(address: String, enableRenderNotify: Boolean, deadlineMs: Long) {
val timeoutMs = maxOf(deadlineMs - now(), ConnectionOptions.CONNECT_TIMEOUT_MS.toLong())
if (!link.connect(address, timeoutMs.toInt())) {
throw ArmUnreachableException(if (address.equals(leftAddress, ignoreCase = true)) "left" else "right", address)
}
// requestConnectionPriority has no callback in this Android compile target, so there is
// no reliable completion point to keep it in the global GATT operation pipeline. But it's
@@ -2083,6 +2186,17 @@ class GlassesSessionCore(
return
}
try {
// The ring is often out of range or bound to the glasses. Rather than block the
// ring worker (and the health ACKs behind it) on a timed connect, keep a
// low-duty background connection pending; its callback wakes the worker to
// finish setup. The periodic recheck re-arms it if the attempt died without a
// callback.
if (!link.isConnected(ringAddress) && link.beginConnect(ringAddress, true)) {
monitor.withLock {
ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS
}
return
}
connectRing()
} catch (t: Throwable) {
// Back off exponentially: each attempt's setup steps hold the
@@ -2100,6 +2214,9 @@ class GlassesSessionCore(
}
logLine("direct ring connect failed (" + reason + "): " + safeMessage(t)
+ "; retry in " + (retryDelayMs / 1000) + "s")
// Start the next attempt from a fresh connection rather than retrying setup on
// one that just failed it.
link.disconnect(ringAddress)
}
}
@@ -2286,6 +2403,13 @@ class GlassesSessionCore(
return id
}
/** 0 for the left arm, 1 for the right (the cfwTransports order), -1 for anything else. */
internal fun armIndex(address: String): Int = when {
address.equals(leftAddress, ignoreCase = true) -> 0
address.equals(rightAddress, ignoreCase = true) -> 1
else -> -1
}
internal fun hasRingAddress(): Boolean {
return ringAddress.trim().isNotEmpty()
}
@@ -2308,3 +2432,7 @@ class GlassesSessionCore(
}
}
}
/** An arm's connection never came up within the connect window ([side] is "left" or "right"). */
internal class ArmUnreachableException(val side: String, address: String) :
IllegalStateException("connect failed: $side arm $address")
@@ -46,11 +46,16 @@ internal fun GlassesSessionCore.driveSession(): Long {
}
var messageToWrite: OutboundMessage? = null
var messageToPrewrite: OutboundMessage? = null
var heartbeatToWrite: OutboundMessage? = null
val now = now()
maybeFinishNoChangeDesiredFrame()
monitor.withLock {
if (incompatibleFirmware != null) {
// Incompatible firmware: hand back to the worker loop to halt.
return 0
}
drainCfwAcknowledgementsLocked()
if (cfwCleanupDelivered) {
/* Successful mode 11 must be the last Faceclaw write. Drop
@@ -78,6 +83,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
)
}
if (!inFlightMessages.isEmpty()) {
refreshCfwAckDeadlinesLocked()
val oldest = inFlightMessages.firstOrNull()
val replay = CfwMessageWindow.replayWindow(inFlightMessages, now)
if (!replay.isEmpty()) {
@@ -130,7 +136,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
finishDesiredFrameLocked("discarded: glasses charging")
if (sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) {
messageToWrite = pendingMessages.removeFirst()
logLine("sending pending message (charging): " + messageToWrite!!.label)
logLine("sending pending message (charging): " + messageToWrite.label)
} else if (sessionReady && pendingMessages.isEmpty() && inFlightMessages.isEmpty()
&& now - lastBatteryRefreshAtMs >= ConnectionOptions.CHARGING_BATTERY_POLL_MS) {
logLine("Writing charging-mode battery poll")
@@ -196,14 +202,21 @@ internal fun GlassesSessionCore.driveSession(): Long {
&& (hasPendingImageLocked()
|| desiredFingerprintSnapshot() != lastEnqueuedFingerprint)))
noteImageStallLocked(now, windowHasRoom)
if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) {
val heartbeatStep = if (messageToPrewrite == null) heartbeatStepLocked(imageWaiting) else HeartbeatStep.NONE
if (heartbeatStep == HeartbeatStep.HOLD) {
return ConnectionOptions.IDLE_SLEEP_MS.toLong()
}
if (heartbeatStep == HeartbeatStep.SEND) {
logLine("Writing heartbeat")
heartbeatToWrite = createHeartbeatMessage()
lastHeartbeatSentAtMs = now()
return@withLock
}
if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty()
&& CfwMessageWindow.canSend(inFlightMessages, pendingMessages.firstOrNull()!!)) {
messageToWrite = pendingMessages.removeFirst()
logLine("sending pending message: " + messageToWrite!!.label)
logLine("sending pending message: " + messageToWrite.label)
} else if (messageToPrewrite == null && !shutdownRequested && !benchmarkActive
&& fixedLayoutCreated
&& windowHasRoom && !hasPendingImageLocked()
@@ -227,6 +240,15 @@ internal fun GlassesSessionCore.driveSession(): Long {
}
}
val heartbeat = heartbeatToWrite
if (heartbeat != null) {
// Like every write, outside `monitor`: waiting for the write to complete while
// holding it would stall notification handling, which needs it. A failed write
// just leaves the heartbeat to its ack timeout (see createHeartbeatMessage).
writeMessage(heartbeat)
return ConnectionOptions.IDLE_SLEEP_MS.toLong()
}
val prewrite = messageToPrewrite
if (prewrite != null) {
if (prewriteMessage(prewrite)) {
@@ -285,7 +307,17 @@ internal fun GlassesSessionCore.canPrewriteCandidate(message: OutboundMessage?):
return message.message.size + 2 > 232
}
internal fun GlassesSessionCore.handleHeartbeat(imageWaiting: Boolean): Boolean {
/** What the heartbeat schedule wants from one driveSession pass. */
internal enum class HeartbeatStep {
/** Nothing due: carry on with other traffic. */
NONE,
/** Send nothing else this pass (a heartbeat is pending, or overdue but blocked). */
HOLD,
/** Write a heartbeat now, then hold. */
SEND,
}
internal fun GlassesSessionCore.heartbeatStepLocked(imageWaiting: Boolean): HeartbeatStep {
val now = now()
val heartbeatEligible = !shutdownRequested && fixedLayoutCreated
val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat")
@@ -300,22 +332,18 @@ internal fun GlassesSessionCore.handleHeartbeat(imageWaiting: Boolean): Boolean
// the heartbeat still fires once we reach the URGENT threshold
// if rendering goes quiet again. Preserves the pending-heartbeat
// inter-lens-sync invariant below (that path is untouched).
return false
return HeartbeatStep.NONE
}
logLine("Writing heartbeat")
val heartbeatMessage = createHeartbeatMessage()
lastHeartbeatSentAtMs = now
writeMessage(heartbeatMessage)
return true
return HeartbeatStep.SEND
} else if (heartbeatUrgent) {
return true
return HeartbeatStep.HOLD
} else if (heartbeatPending) {
// Don't send other message types while a heartbeat is pending because that
// can lead to inter-lens sync issues
return true
return HeartbeatStep.HOLD
}
return false
return HeartbeatStep.NONE
}
internal fun GlassesSessionCore.createHeartbeatMessage(): OutboundMessage {
@@ -392,6 +420,7 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean
monitor.withLock {
val sentAtMs = now()
if (result) noteWriteCompletedLocked(writeAddress, sentAtMs)
message.sentAtMs = sentAtMs
message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs
logImageUpdateSendLandmarkLocked(message)
@@ -440,6 +469,7 @@ internal fun GlassesSessionCore.prewriteMessage(message: OutboundMessage): Boole
if (!result) {
return false
}
monitor.withLock { noteWriteCompletedLocked(writeAddress, now()) }
prewrittenMessage = message
prewrittenFrames = frames.toList()
@@ -463,13 +493,39 @@ internal fun GlassesSessionCore.spoilPrewrittenMessage(reason: String): Boolean
val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress
logLine("spoiling prewritten " + message.label + ": " + reason)
return link.writeFrames(
val result = link.writeFrames(
writeAddress,
BleProtocol.WRITE_CHAR_UUID,
listOf(finalFrame),
ConnectionOptions.WRITE_MODE,
ConnectionOptions.WRITE_TIMEOUT_MS
)
if (result) monitor.withLock { noteWriteCompletedLocked(writeAddress, now()) }
return result
}
/**
* Record that writes to [address] just completed: every frame's onCharacteristicWrite
* fired. The drive loop only checks ack deadlines between writes, so noting it when
* writeFrames returns is as good as noting each callback.
*/
internal fun GlassesSessionCore.noteWriteCompletedLocked(address: String, atMs: Long) {
val arm = armIndex(address)
if (arm >= 0) lastArmWriteAtMs[arm] = atMs
}
/**
* Re-derive each written, unresolved CFW message's ack deadline from its arm's latest
* write completion and CFW ack (CfwMessageWindow.ackDeadline), so either kind of
* progress keeps the window waiting. Deadlines only move later.
*/
internal fun GlassesSessionCore.refreshCfwAckDeadlinesLocked() {
for (message in inFlightMessages) {
if (message.sid != CfwTransport.SID || message.ackDeadlineAtMs <= 0) continue
val arm = if (message.isLeftArmMessage) 0 else 1
message.ackDeadlineAtMs = CfwMessageWindow.ackDeadline(
message.sentAtMs, lastArmWriteAtMs[arm], lastArmCfwAckAtMs[arm])
}
}
internal fun GlassesSessionCore.removePreparedMessageLocked(message: OutboundMessage?): Boolean {
@@ -590,6 +646,10 @@ internal fun GlassesSessionCore.enqueueCreateLayoutLocked() {
// New session: re-assert the firmware-debug-flags overlay once
// the layout is ready (the mode-7 send is gated on this having reset).
firmwareDebugFlagsLastSent = -1
if (customFirmwareDetected) {
enqueueEnterEvenHubLocked()
return
}
val message = messageBuilder.createLayout()
message.onAck = MessageCallback {
startupProbePending = false
@@ -611,6 +671,25 @@ internal fun GlassesSessionCore.enqueueCreateLayoutLocked() {
logLine("queue create layout")
}
/**
* The custom-firmware replacement for the stock create layout (CFW mode 31).
* A reconnect can find the page from the previous session still up, and stock
* firmware never ACKs a repeated create; the CFW ACKs whatever state the
* glasses were in. A lost message goes through the CFW replay window like any
* other private message, which is safe because this one is idempotent. The
* settings query that sets customFirmwareDetected is ahead of this in the
* queue on every session.
*/
private fun GlassesSessionCore.enqueueEnterEvenHubLocked() {
val message = messageBuilder.enterEvenHub(connectionOptions.sendImagesToLeft)
message.onAck = MessageCallback {
fixedLayoutCreated = true
displayedFingerprint = ""
}
pendingMessages.addLast(message)
logLine("queue enter EvenHub")
}
internal fun GlassesSessionCore.enqueueStartupProbeLocked() {
enqueueCreateLayoutLocked()
@@ -711,7 +790,7 @@ internal fun GlassesSessionCore.enqueueAmbientLightControlLocked(payload: ByteAr
internal fun GlassesSessionCore.enqueueDesiredImageLocked() {
val fingerprint: String
val packedSnapshot: ByteArray?
val graySnapshot: ByteArray?
val width: Int
val height: Int
val paintMs: Int
@@ -720,7 +799,7 @@ internal fun GlassesSessionCore.enqueueDesiredImageLocked() {
val scene: ShellScene
desiredTilesLock.locked {
fingerprint = desiredFingerprint
packedSnapshot = desiredPacked
graySnapshot = desiredGray
width = desiredWidth
height = desiredHeight
paintMs = desiredPaintMs
@@ -729,13 +808,35 @@ internal fun GlassesSessionCore.enqueueDesiredImageLocked() {
scene = desiredShellScene
desiredFrameId = 0
}
val packedFrame: ByteArray = packedSnapshot ?: ByteArray(0)
// Pack here rather than at submit: a frame superseded before the send loop
// gets to it is never packed, and the main thread doesn't pay for it.
frameTimings.spanStart(frameId, "pack-4bpp")
val packedFrame: ByteArray =
if (graySnapshot == null || graySnapshot.isEmpty()) ByteArray(0)
else BmpUtil.pack4bppFromGray8(graySnapshot, width, height)
frameTimings.spanEnd(frameId, "pack-4bpp")
// Visibility belongs to this immutable composite, not the latest UI request.
enqueueBrightnessLocked(!fingerprint.startsWith("blanked:"))
if (customFirmwareDetected && packedFrame.size > 0) {
val rendered = scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId,
connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES,
connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS)
val rendered = try {
scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId,
connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES,
connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS)
} catch (e: Exception) {
// Planning depends only on this frame, so reconnecting would fail
// the same way (#42). Drop the frame (the planner and resource
// cache are as they were) and wait for the scene to change.
lastEnqueuedFingerprint = fingerprint
logLine("frame plan failed; frame dropped: " + safeMessage(e))
finishFrame(frameId, "discarded: frame plan failed")
return
}
if (rendered.bakedLayers > 0 || rendered.bakedSelections) {
val baked = "shell over resource budget: baked " + rendered.bakedLayers + "/" + scene.layers.size +
" layers" + (if (rendered.bakedSelections) " and retained drawings" else "") + " into the frame"
frameTimings.log(frameId, baked)
logLine("$baked fingerprint=$fingerprint")
}
nextImageFrameId = rendered.nextFid
val commands = rendered.commands
for (i in 0 until commands.size - 1) enqueueResourceCommandLocked(commands[i])
@@ -949,6 +1050,13 @@ internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessa
if (firmwareInfo != null) {
customFirmwareDetected = firmwareInfo.isFaceclawFirmware()
emitFirmwareInfo(firmwareInfo)
if (sessionReady && !firmwareInfo.isCompatible(requiredFirmwareRevision)) {
// Halting tears down the link, which belongs on the worker thread
// (this runs on the notification path); until it does, driveSession
// sends nothing more.
incompatibleFirmware = firmwareInfo
interruptibleSleep.interrupt()
}
}
}
message.onTimeout = MessageCallback {
@@ -997,12 +1105,24 @@ internal fun GlassesSessionCore.updateChargingModeLocked(charging: Boolean, batt
logLine("glasses are charging; pausing display communication")
setStateDisplay("charging", GlassesSessionCore.chargingStatusText(battery))
} else {
chargingMode = false
logLine("glasses removed from charger; reconnecting")
handleTransportFailure("charging ended")
endChargingModeLocked("glasses removed from charger")
}
}
/**
* Leave charging mode, whether a battery poll or putting the glasses on
* (they can't be worn in the case) said charging is over. The rebuild goes
* through the normal reconnect loop.
*/
internal fun GlassesSessionCore.endChargingModeLocked(why: String) {
if (!chargingMode) {
return
}
chargingMode = false
logLine("$why; reconnecting")
handleTransportFailure("charging ended")
}
/**
* Record, into the frame that is waiting, why it did not go out on this
* pass of the send loop. Without this the export shows a bare multi-second
@@ -1081,7 +1201,7 @@ internal fun GlassesSessionCore.describeWriteBlockerLocked(now: Long, windowHasR
if (hasPendingOrInflightKindLocked("heartbeat")) {
return "heartbeat in flight"
}
// handleHeartbeat is a barrier: while one is due it holds back every
// heartbeatStepLocked is a barrier: while one is due it holds back every
// other write, so a frame queued at the wrong moment waits a heartbeat
// round trip. Reported explicitly because it is otherwise invisible --
// heartbeats belong to no frame.
@@ -13,6 +13,7 @@ import kotlin.contracts.contract
* must call from whatever thread the platform stack delivers on.
*/
interface SessionLink {
/** Connect, waiting up to [timeoutMs]; waits on an attempt already started by [beginConnect]. */
fun connect(address: String, timeoutMs: Int): Boolean
/**
@@ -21,6 +22,17 @@ interface SessionLink {
*/
fun connect(address: String, timeoutMs: Int, autoConnect: Boolean): Boolean = connect(address, timeoutMs)
/**
* Start connecting without waiting, so the attempt can land while the caller does other
* work. [background] asks for a low-duty-cycle attempt that stays pending until the
* device shows up. Returns true while a connection exists or is pending, false when the
* link can't do this (callers then fall back to a blocking [connect]).
*/
fun beginConnect(address: String, background: Boolean): Boolean = false
/** Whether a connection to [address] is up (not merely pending). */
fun isConnected(address: String): Boolean = false
/** Best effort; no completion point (see connectArm). */
fun requestHighPriority(address: String)
@@ -1,10 +1,14 @@
package com.faceclaw.app
/** Reference interpreter. It uses the firmware's packed rows, clipping, LUT and call grammar. */
/**
* Reference interpreter. It uses the firmware's packed rows, clipping, LUT and call grammar.
* [flat] ignores stereo depth, for pixels both lenses share (see ShellScene.flatten).
*/
class DisplayListRenderer(
private val resources: Map<Int, ByteArray>,
private val builtin: ((Int, Int) -> BuiltinGlyph)? = null,
private val rightLens: Boolean = false,
private val flat: Boolean = false,
) {
class BuiltinGlyph(val image: ByteArray, val advance: Int, val x: Int = 0, val y: Int = 0)
@@ -239,7 +243,8 @@ class DisplayListRenderer(
target = target(id).shifted(inherited.shiftX)
}
if (flags and DRAW_FLAG_DEPTH != 0) {
target = target.shifted(DrawProtocol.depthOffset(reader.readS8(), rightLens))
val depth = reader.readS8()
if (!flat) target = target.shifted(DrawProtocol.depthOffset(depth, rightLens))
}
if (flags and DRAW_FLAG_CLIP != 0) {
target = target.clipped(reader.readS16(), reader.readS16(), reader.readU16(), reader.readU16())
@@ -298,11 +303,78 @@ class DisplayListRenderer(
}
require(width > 0 && height > 0 && x + width <= source.width && y + height <= source.height)
// Mirrors the firmware's overflow guard before the depth shift.
if (walk.apply && dx in -65536..65536) {
// Snapshot the rectangle so overlapping copies have memmove semantics.
val copy = IntArray(width * height) { source.get(x + it % width, y + it / width) }
for (i in copy.indices) target.put(dx + i % width, dy + i / width, copy[i])
if (!walk.apply || dx !in -65536..65536) return
// Clip the destination to the writable area, in shifted target pixels.
// The source is read unshifted, like Target.get.
val originX = dx + target.shiftX
var left = maxOf(originX, 0)
var right = minOf(originX + width, target.width)
var top = maxOf(dy, 0)
var bottom = minOf(dy.toLong() + height, target.height.toLong()).toInt()
target.clip?.let { clip ->
left = maxOf(left, clip[0]); top = maxOf(top, clip[1])
right = minOf(right, clip[2]); bottom = minOf(bottom, clip[3])
}
if (left >= right || top >= bottom) return
val count = right - left
val rows = bottom - top
var sourceX = x + (left - originX)
var sourceBytes = source.bytes
var sourceStride = source.stride
var sourceRow = source.offset + (y + (top - dy)) * sourceStride
if (sourceBytes === target.bytes) {
// Snapshot the packed source span so overlapping copies have memmove semantics.
val first = sourceX / 2
val span = (sourceX + count - 1) / 2 - first + 1
val snapshot = ByteArray(span * rows)
for (row in 0 until rows) {
val start = sourceRow + row * sourceStride + first
sourceBytes.copyInto(snapshot, row * span, start, start + span)
}
sourceBytes = snapshot
sourceStride = span
sourceRow = 0
sourceX = sourceX and 1
}
for (row in 0 until rows) {
copyPixels(sourceBytes, sourceRow + row * sourceStride, sourceX,
target.bytes, target.offset + (top + row) * target.stride, left, count)
}
}
/** Copy [count] packed 4bpp pixels between rows that must not overlap. */
private fun copyPixels(source: ByteArray, sourceRow: Int, sourceX: Int,
destination: ByteArray, destinationRow: Int, destinationX: Int, count: Int) {
var sx = sourceX
var dx = destinationX
var remaining = count
if (dx and 1 != 0) {
putPixel(destination, destinationRow, dx++, getPixel(source, sourceRow, sx++))
remaining--
}
val pairs = remaining / 2
var si = sourceRow + sx / 2
var di = destinationRow + dx / 2
if (sx and 1 == 0) {
source.copyInto(destination, di, si, si + pairs)
} else {
// Misaligned: each destination byte straddles two source bytes.
repeat(pairs) {
destination[di++] = ((source[si].toInt() shl 4) or ((source[++si].toInt() and 255) ushr 4)).toByte()
}
}
if (remaining and 1 != 0) {
putPixel(destination, destinationRow, dx + pairs * 2, getPixel(source, sourceRow, sx + pairs * 2))
}
}
private fun getPixel(bytes: ByteArray, row: Int, x: Int): Int =
(bytes[row + x / 2].toInt() ushr (if (x and 1 == 0) 4 else 0)) and 15
private fun putPixel(bytes: ByteArray, row: Int, x: Int, value: Int) {
val index = row + x / 2
val old = bytes[index].toInt()
bytes[index] = if (x and 1 == 0) ((old and 15) or (value shl 4)).toByte() else ((old and 240) or value).toByte()
}
private fun drawStockText(reader: DrawReader, target: Target, walk: Walk) {
@@ -420,16 +492,21 @@ class DisplayListRenderer(
val radius = reader.readU16()
val fill = reader.readU8()
val border = reader.readU8()
// Revision 36: an optional trailing color for the corners outside the shape.
val outside = if (reader.remaining == 1) reader.readU8().also { require(it <= 15) } else DRAW_ROUNDED_RECT_NO_OUTSIDE
reader.requireDone()
require(width in 1..640 && height in 1..480 && fill <= 15 && border <= DRAW_ROUNDED_RECT_NO_BORDER)
if (!walk.apply) return
if (x < -width - target.shiftX || x >= target.width - target.shiftX ||
y < -height || y >= target.height) return
for (yy in 0 until height) for (xx in 0 until width) {
if (!roundedContains(xx, yy, width, height, radius)) continue
val tx = x + xx + target.shiftX
val ty = y + yy
if (tx !in 0 until target.width || ty !in 0 until target.height) continue
if (!roundedContains(xx, yy, width, height, radius)) {
if (outside != DRAW_ROUNDED_RECT_NO_OUTSIDE) target.put(x + xx, ty, outside)
continue
}
val edge = !roundedContains(xx - 1, yy - 1, width - 2, height - 2, maxOf(0, radius - 1))
val value = if (edge && border < DRAW_ROUNDED_RECT_NO_BORDER) border else maxOf(fill, target.get(tx, ty))
target.put(x + xx, ty, value)
@@ -45,7 +45,7 @@ class FrameDisplayList private constructor(
val clip = c.clip?.translated(x, y)
when (c.op) {
DRAW_OP_ROUNDED_RECT -> DrawProtocol.roundedRect(bind(c.x, elapsed, x), bind(c.y, elapsed, y),
v[0], v[1], v[2], v[3], v[4], depth + c.depth, clip)
v[0], v[1], v[2], v[3], v[4], depth + c.depth, clip, v[5].takeIf { it != DRAW_ROUNDED_RECT_NO_OUTSIDE })
DRAW_OP_IMAGE -> DrawProtocol.image(resource(v[0]), bind(c.x, elapsed, x + v[1]), bind(c.y, elapsed, y + v[2]),
v[3], depth = depth + c.depth, clip = clip)
DRAW_OP_TEXT -> DrawProtocol.text(resource(v[0]), bind(c.x, elapsed, x + v[1]), bind(c.y, elapsed, y + v[2]),
@@ -185,9 +185,9 @@ class FrameDisplayList private constructor(
DRAW_OP_ROUNDED_RECT -> {
val xx = value(); val yy = value()
val w = r.readU16(); val h = r.readU16(); val radius = r.readU16()
val background = r.readU8(); val border = r.readU8()
require(w in 1..640 && h in 1..480 && background in 0..15 && border in 0..16)
commands.add(Command(op, d, intArrayOf(w, h, radius, background, border), xx, yy, clip = clip))
val background = r.readU8(); val border = r.readU8(); val outside = r.readU8()
require(w in 1..640 && h in 1..480 && background in 0..15 && border in 0..16 && outside in 0..16)
commands.add(Command(op, d, intArrayOf(w, h, radius, background, border, outside), xx, yy, clip = clip))
}
DRAW_OP_IMAGE -> {
val id = resource(); require(id != DrawProtocol.SCREEN)
@@ -2,11 +2,14 @@ package com.faceclaw.app
/** A retained selection or image, replayed at stereo depth over the screen and shell surfaces. */
class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val radius: Int,
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List<IntArray> = emptyList()) : RetainedDrawing {
val resource = CachedResource(DrawProtocol.rawImage(width, height, packed))
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List<IntArray> = emptyList(),
/** [packed] as a raw-image resource; translated copies share it (and its hash) instead of rebuilding it. */
val resource: CachedResource = CachedResource(DrawProtocol.rawImage(width, height, packed))) : RetainedDrawing {
override val resources = listOf(resource)
override val fingerprint = "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
override fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask)
override val fingerprint: String by lazy {
"$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
}
override fun translated(dx: Int, dy: Int) = if (dx == 0 && dy == 0) this else MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask, resource)
override fun calls(ids: IntArray, nowMs: Long) = calls(ids.single())
fun calls(id: Int): List<ByteArray> {
val image = when (kind) {
@@ -36,7 +39,7 @@ class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val
val radius = reader.getShort().toInt() and 65535
val background = BmpUtil.nibbleForGray(reader.get() and 255)
val border = BmpUtil.nibbleForGray(reader.get() and 255)
val depth = reader.get().toByte().toInt()
val depth = reader.get().toInt()
val count = reader.getShort().toInt() and 65535
require(count <= 2048)
require(w in 1..640 && h in 1..480 && 5 + (w + 1) / 2 * h <= 65536 && reader.remaining() >= w * h)
@@ -1,39 +1,78 @@
package com.faceclaw.app
/** Immutable shell snapshot. Keys identify writable surfaces across repaints, not their pixels. */
/**
* Immutable shell snapshot. Keys identify writable surfaces across repaints, not their pixels.
* [screenDepth] is the stereo depth of the whole presentation: the screen copy and everything
* drawn over it shift together per lens (see [calls]).
*/
class ShellScene(val layers: List<Layer>, val selections: List<RetainedDrawing> = emptyList(), val screenDepth: Int = 0) {
class Layer(val key: Int, val x: Int, val y: Int, val width: Int, val height: Int, val dim: Int, val packed: ByteArray, val selections: List<RetainedDrawing> = emptyList(), val depth: Int = 0)
class Layer(val key: Int, val x: Int, val y: Int, val width: Int, val height: Int, val dim: Int, val packed: ByteArray, val selections: List<RetainedDrawing> = emptyList(), val depth: Int = 0) {
// Layers are reused by every composite until the shell repaints, and hashing packed
// byte-by-byte on each one was a measurable share of the compositor's time.
internal val fingerprint: String by lazy {
"$key,$x,$y,$width,$height,$dim,$depth,${CachedResource(packed).hash},${selections.joinToString { row -> row.fingerprint }}"
}
}
val allSelections = selections + layers.flatMap { it.selections }
val retainedResources = allSelections.flatMap { it.resources }
init { require(retainedResources.size + layers.size < 511 && screenDepth in -128..127) }
val fingerprint: String = layers.joinToString(";") { "${it.key},${it.x},${it.y},${it.width},${it.height},${it.dim},${it.depth},${CachedResource(it.packed).hash},${it.selections.joinToString { row -> row.fingerprint }}" } + selections.joinToString { it.fingerprint } + "|depth:$screenDepth"
val fingerprint: String = layers.joinToString(";") { it.fingerprint } + selections.joinToString { it.fingerprint } + "|depth:$screenDepth"
/**
* The root list. A nonzero [screenDepth] adds to every positional call's own depth, so the
* scene moves as one; depths are even (see uiDepthSetting) so the per-lens halves add
* exactly. Dim LUTs stay put: they cover the whole target, and a shifted one would leave an
* undimmed strip at one edge.
*/
fun calls(width: Int, height: Int, surfaces: IntArray, selected: IntArray): List<ByteArray> {
val calls = ArrayList<ByteArray>(); var rowId = 0
val now = drawAnimationTimeMs()
fun shifted(call: ByteArray) = DrawProtocol.withAddedDepth(call, screenDepth)
calls.addAll(DrawProtocol.screenCopy(width, height, screenDepth))
fun add(row: RetainedDrawing) {
calls.addAll(row.calls(selected.copyOfRange(rowId, rowId + row.resources.size), now))
for (call in row.calls(selected.copyOfRange(rowId, rowId + row.resources.size), now)) calls.add(shifted(call))
rowId += row.resources.size
}
for (row in selections) add(row)
for ((index, layer) in layers.withIndex()) {
if (layer.dim < 256) calls.add(DrawProtocol.lut(width, height, layer.dim))
calls.add(DrawProtocol.image(surfaces[index], layer.x, layer.y, depth = layer.depth))
calls.add(DrawProtocol.image(surfaces[index], layer.x, layer.y, depth = DrawProtocol.addDepth(layer.depth, screenDepth)))
for (row in layer.selections) add(row)
}
return calls
}
/** This scene without its whole-presentation shift: the phone mirror's view (see SurfaceCompositor.previewScene). */
fun unshifted(): ShellScene = if (screenDepth == 0) this else ShellScene(layers, selections, 0)
fun preview(screenGray: ByteArray, width: Int, height: Int, rightLens: Boolean = false): ByteArray {
val screen = BmpUtil.pack4bppFromGray8(screenGray, width, height)
val output = ByteArray(screen.size); val resources = HashMap<Int, ByteArray>()
val output = ByteArray(screen.size)
DisplayListRenderer(rendererResources(width, height), rightLens = rightLens).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height))
val target = DisplayListRenderer.Target(output, width, height)
return ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() }
}
/**
* [screen] (packed 4bpp) with this scene's own retained drawings and its bottom [layerCount]
* layers drawn in, for a frame whose shell exceeds the resource budget (see ScenePlanner.plan).
* Both lenses share the result, so stereo depth is dropped, and animations are drawn settled.
*/
fun flatten(screen: ByteArray, width: Int, height: Int, layerCount: Int): ByteArray {
val baked = ShellScene(layers.take(layerCount), selections)
val output = ByteArray(screen.size)
DisplayListRenderer(baked.rendererResources(width, height), flat = true).render(511,
DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height), SETTLED_MS)
return output
}
/** Layer surfaces, then retained resources, by index, and the root list at 511. */
private fun rendererResources(width: Int, height: Int): HashMap<Int, ByteArray> {
val resources = HashMap<Int, ByteArray>()
val surfaces = IntArray(layers.size) { id ->
val layer = layers[id]; resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed); id
}
val rows = IntArray(retainedResources.size) { id -> resources[id + layers.size] = retainedResources[id].bytes; id + layers.size }
resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows))
DisplayListRenderer(resources, rightLens = rightLens).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height))
val target = DisplayListRenderer.Target(output, width, height)
return ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() }
return resources
}
internal class AnimatedPreview(val player: DisplayListPlayer, var pixels: ByteArray)
@@ -43,20 +82,9 @@ class ShellScene(val layers: List<Layer>, val selections: List<RetainedDrawing>
): AnimatedPreview {
val screen = BmpUtil.pack4bppFromGray8(screenGray, width, height)
val output = ByteArray(screen.size)
val resources = HashMap<Int, ByteArray>()
val surfaces = IntArray(layers.size) { id ->
val layer = layers[id]
resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed)
id
}
val rows = IntArray(retainedResources.size) { id ->
resources[id + layers.size] = retainedResources[id].bytes
id + layers.size
}
resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows))
val target = DisplayListRenderer.Target(output, width, height)
lateinit var preview: AnimatedPreview
val player = DisplayListPlayer(DisplayListRenderer(resources),
val player = DisplayListPlayer(DisplayListRenderer(rendererResources(width, height)),
DisplayListRenderer.Target(screen, width, height), target,
clock = { drawAnimationTimeMs() }, schedule = schedule,
displayed = { preview.pixels = ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() } })
@@ -67,6 +95,8 @@ class ShellScene(val layers: List<Layer>, val selections: List<RetainedDrawing>
companion object {
val EMPTY = ShellScene(emptyList())
/** Past every animation's end: TIME counts from PRESENT and clamps to the animation's length. */
private const val SETTLED_MS = 3_600_000L
fun decode(reader: ByteReader): ShellScene {
val count = reader.getShort().toInt() and 65535; require(count <= 64)
val layers = ArrayList<Layer>()
@@ -83,10 +113,11 @@ class ShellScene(val layers: List<Layer>, val selections: List<RetainedDrawing>
val selections = List(selectionCount) { readRetainedDrawing(reader, reader.get().toInt()) }
layers.add(Layer(key, x, y, w, h, dim, BmpUtil.pack4bppFromGray8(gray, w, h), selections, depth))
}
require(reader.remaining() == 0)
require(reader.remaining() == 2)
val screenDepth = reader.getShort().toInt()
require(layers.sumOf { layer -> layer.selections.sumOf { it.resources.size } } + layers.size < 511)
require(layers.map { it.key }.toSet().size == layers.size)
return ShellScene(layers)
return ShellScene(layers, screenDepth = screenDepth)
}
}
}
@@ -1,5 +1,6 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
import kotlin.jvm.JvmOverloads
@@ -43,11 +44,19 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
return maxOf(1, (((value * dim) + 128) shr 8))
}
/**
* The rest of [reader] as a heap-array reader. Parsing reads field by field, and through a
* platform buffer (Android's direct ByteBuffer) each byte was its own native peek; one bulk
* copy first is far cheaper.
*/
private fun heapReader(reader: ByteReader): ByteReader =
if (reader is ArrayByteReader) reader else ArrayByteReader(ByteArray(reader.remaining()).also { reader.get(it) })
private fun parseDraws(draws: ByteReader?): Array<ScreenDraw> {
if (((draws == null) || (draws.remaining() < 1))) {
return NO_DRAWS
}
val cursor: ByteReader = draws
val cursor: ByteReader = heapReader(draws)
var out: MutableList<ScreenDraw> = ArrayList()
while ((cursor.remaining() >= 1)) {
var kind: Int = (cursor.get() and 0xff)
@@ -253,8 +262,11 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private var shellScene: ShellScene? = null
fun setShellScene(reader: ByteReader) {
val scene = ShellScene.decode(reader)
lock.withLock { shellScene = scene }
val scene = ShellScene.decode(heapReader(reader))
lock.withLock {
shellScene = scene
changedSinceComposite = true
}
}
private class Surface {
@@ -284,6 +296,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
/** Surface-local deferred draws of the retained content (already baked into pixels). */
@JvmField var draws: Array<ScreenDraw> = NO_DRAWS
/**
* [draws]' retained selections in screen coordinates, as built for [placedFor] at
* ([placedX], [placedY]). Reused by every composite until either changes, so the shell
* scene built from them (and its fingerprint) can be reused too.
*/
@JvmField var placedSelections: List<RetainedDrawing> = emptyList()
@JvmField var placedFor: Array<ScreenDraw>? = null
@JvmField var placedX: Int = 0
@JvmField var placedY: Int = 0
constructor(id: String) {
this.id = id
}
@@ -291,13 +313,35 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private val lock = protocolPlatform().createLock()
private var animatedPreview: ShellScene.AnimatedPreview? = null
/** The last composite's scene; reused while its inputs are unchanged (see compositeLocked). */
private var lastScene: ShellScene? = null
private var previewKey: String? = null
@Volatile private var previewAnimationListener: (() -> Unit)? = null
/**
* Called after each timer redraw of an animated preview (including the one where it settles),
* outside the lock and on the redraw scheduler's thread. The mirror pulls previews on new
* frames only, and the animation advances between them, so without this the mirror keeps
* whichever mid-motion frame it last pulled.
*/
fun setPreviewAnimationListener(listener: (() -> Unit)?) {
previewAnimationListener = listener
}
/**
* The phone mirror's view of [scene]: without the whole-screen stereo shift, which would move
* it off centre by half the depth and put mirror touches that far from what they land on.
*/
private fun previewScene(scene: ShellScene, gray: ByteArray, key: String): ByteArray {
if (previewKey != key) {
animatedPreview?.player?.stop()
animatedPreview = scene.animatedPreview(gray, screenWidth, screenHeight,
schedule = { delay, action -> scheduleDrawRedraw(delay) { lock.withLock(action) } })
animatedPreview = scene.unshifted().animatedPreview(gray, screenWidth, screenHeight,
schedule = { delay, action ->
scheduleDrawRedraw(delay) {
lock.withLock(action)
previewAnimationListener?.invoke()
}
})
previewKey = key
}
return animatedPreview!!.pixels
@@ -318,6 +362,13 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private var nextCompositeSeq: Long = 1
/**
* Set by changes that take effect at the next composite (geometry, dim, depth, blanking,
* visibility, a new shell scene, content retained without compositing) and cleared by a
* composite that is kept (not a preview); see [isSurfaceCurrent].
*/
private var changedSinceComposite: Boolean = false
/** Set the output frame size. Must be called before any surface work. */
fun configureScreen(width: Int, height: Int): Unit {
if (((width <= 0) || (height <= 0))) {
@@ -362,10 +413,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
surface = Surface(id)
surfaces.put(id, surface)
}
if (
(((surface.pixels == null) || (surface.width != width)) ||
(surface.height != height))
) {
if ((surface.width != width) || (surface.height != height)) {
surface.pixels = ByteArray((width * height))
surface.fingerprint = ""
surface.draws = NO_DRAWS
@@ -376,12 +424,27 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
surface.height = height
surface.zOrder = zOrder
surface.transparency = transparency
changedSinceComposite = true
}
}
fun removeSurface(id: String): Unit {
lock.withLock {
surfaces.remove(id)
changedSinceComposite = true
}
}
/**
* Whether a full-surface update carrying [fingerprint] would change nothing: the surface
* already retains that content and every other change has reached a kept composite. A
* submitter can then drop an identical repaint without flattening or sending it.
*/
fun isSurfaceCurrent(id: String, fingerprint: String): Boolean {
lock.withLock {
if (changedSinceComposite) return false
val surface = surfaces.get(id) ?: return false
return surface.fingerprint.isNotEmpty() && surface.fingerprint == fingerprint
}
}
@@ -398,6 +461,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
lock.withLock {
underlayDimBelowZOrder = belowZOrder
underlayDim = maxOf(0, minOf(256, factor256))
changedSinceComposite = true
}
}
@@ -419,6 +483,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
throw IllegalArgumentException(("unknown surface " + id))
}
surface.visible = visible
changedSinceComposite = true
}
}
@@ -432,15 +497,20 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
lock.withLock {
val surface = surfaces.get(id) ?: throw IllegalArgumentException(("unknown surface " + id))
surface.depth = depth
changedSinceComposite = true
}
}
/** The screen's stereo depth: see setSurfaceDepth. */
/**
* The screen's stereo depth: a covering surface's own (see setSurfaceDepth), else the shell
* scene's depth for the whole display (the regular UI's Depth setting).
*/
private fun screenDepthLocked(ordered: List<Surface>): Int {
val top = ordered.lastOrNull { it.visible } ?: return 0
val uiDepth = shellScene?.screenDepth ?: 0
val top = ordered.lastOrNull { it.visible } ?: return uiDepth
val covers = top.transparency == TRANSPARENCY_OPAQUE && top.x <= 0 && top.y <= 0 &&
top.x + top.width >= screenWidth && top.y + top.height >= screenHeight
return if (covers) top.depth else 0
return if (covers) top.depth else uiDepth
}
/**
@@ -450,6 +520,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
fun setBlanked(blanked: Boolean): Unit {
lock.withLock {
this.blanked = blanked
changedSinceComposite = true
}
}
@@ -582,7 +653,11 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
expectedBytes)
)
}
run {
if (rectX == 0 && rectWidth == surface.width) {
// Full-width rect (every current submitter): one bulk read. Per-row reads from a
// direct ByteBuffer each cross into native code, which showed up in profiles.
pixels.get(surface.pixels, rectY * surface.width, expectedBytes)
} else run {
var row: Int = 0
while ((row < rectHeight)) {
var dstOffset: Int = (((rectY + row) * surface.width) + rectX)
@@ -592,6 +667,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
surface.fingerprint = (if ((contentFingerprint == null)) "" else contentFingerprint)
surface.draws = parsed
if (!composeAfter) changedSinceComposite = true
return if (composeAfter) compositeLocked() else null
}
}
@@ -614,8 +690,10 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
return null
}
val seq = nextCompositeSeq
val changed = changedSinceComposite
val result = compositeLocked(includePreview = true)
nextCompositeSeq = seq
changedSinceComposite = changed
return result
}
}
@@ -636,6 +714,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
private fun compositeLocked(includePreview: Boolean = includePreviewInFrames): Composite {
changedSinceComposite = false
var gray: ByteArray = ByteArray((screenWidth * screenHeight))
if (blanked) {
animatedPreview?.player?.stop()
@@ -662,9 +741,9 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
blendLocked(gray, surface)
var dim: Int = dimForLocked(surface)
selections.addAll(placedSelectionsLocked(surface))
for (draw in surface.draws) {
val selected = draw.selection
if (selected != null) { selections.add(selected.translated(surface.x, surface.y)); continue }
if (draw.selection != null) continue
if (((dim < 256) && (draw.kind == ScreenDraw.KIND_IMAGE))) {
continue
}
@@ -706,8 +785,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
}
val screenDepth = screenDepthLocked(ordered)
val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene(emptyList(), screenDepth = screenDepth)
else ShellScene(shellScene?.layers ?: emptyList(), selections, screenDepth)
val overlaid = surfaces.values.any { it.visible && it.zOrder > 1 }
val sceneLayers = if (overlaid) emptyList() else shellScene?.layers ?: emptyList()
val sceneSelections: List<RetainedDrawing> = if (overlaid) emptyList() else selections
// Built per composite, a scene rebuilt its fingerprint and resource lists every time even
// though its inputs (the decoded shell layers, cached placed selections) rarely change.
val previous = lastScene
val scene = if (previous != null && previous.layers === sceneLayers && previous.screenDepth == screenDepth &&
sameElements(previous.selections, sceneSelections)) previous
else ShellScene(sceneLayers, sceneSelections, screenDepth)
lastScene = scene
val sceneKey = fingerprint.append("|shell:").append(scene.fingerprint).toString()
val preview = if (includePreview && (shellScene != null || selections.isNotEmpty())) {
previewScene(scene, gray, sceneKey)
@@ -723,6 +810,22 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
nextCompositeSeq++, draws.toTypedArray()).also { it.screenGray = gray; it.shellScene = scene }
}
private fun placedSelectionsLocked(surface: Surface): List<RetainedDrawing> {
if (surface.placedFor !== surface.draws || surface.placedX != surface.x || surface.placedY != surface.y) {
surface.placedSelections = surface.draws.mapNotNull { it.selection?.translated(surface.x, surface.y) }
surface.placedFor = surface.draws
surface.placedX = surface.x
surface.placedY = surface.y
}
return surface.placedSelections
}
private fun sameElements(a: List<RetainedDrawing>, b: List<RetainedDrawing>): Boolean {
if (a.size != b.size) return false
for (i in a.indices) if (a[i] !== b[i]) return false
return true
}
private fun blendLocked(gray: ByteArray, surface: Surface): Unit {
var srcX: Int = maxOf(0, -surface.x)
var srcY: Int = maxOf(0, -surface.y)
@@ -744,7 +847,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
var col: Int = 0
while ((col < copyWidth)) {
var value: Int = (surface.pixels[(srcOffset + col)] and 0xff)
if ((value.toInt() != 0)) {
if ((value != 0)) {
gray[(dstOffset + col)] = (dimValue(value, dim)).toByte()
} else {
if ((surface.transparency == TRANSPARENCY_OPAQUE)) {
@@ -0,0 +1,133 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/**
* Scales one decoded video frame's luma plane into an 8bpp gray rect for a compositor surface:
* a box filter (averaging up to [MAX_TAPS] x [MAX_TAPS] evenly spaced samples per output pixel,
* nearest-neighbor when upscaling), then the container's display rotation, then a tone lookup.
*
* Only luma is read: the glasses show gray, and Y is exactly the gray a decoder hands back
* without any color conversion. Platform code owns decoding and copies the plane into a
* ByteArray; this class owns the pixel math. One instance serves one (source size, rotation,
* output size) combination and reuses its scratch buffers across frames; not thread-safe.
*/
class VideoFrameScaler(
/** Visible luma size in coded (unrotated) orientation. */
val sourceWidth: Int,
val sourceHeight: Int,
/** Clockwise display rotation in degrees: 0, 90, 180 or 270. */
val rotation: Int,
/** Output size in display orientation. */
val outputWidth: Int,
val outputHeight: Int,
) {
companion object {
/** Most samples averaged along each axis per output pixel; bounds work for big downscales. */
const val MAX_TAPS = 4
/** Snap any angle to the nearest of 0/90/180/270, the only rotations containers use. */
@JvmStatic
fun normalizeRotation(degrees: Int): Int = (((((degrees % 360) + 360) % 360 + 45) / 90) % 4) * 90
/**
* Luma to display gray. Limited ("TV") range video codes black at 16 and white at 235, so
* it is stretched to 0..255 first. Monochrome thresholds at mid-gray to 0 or 255 (a
* 1-bit image compresses and delta-encodes far better than 16 levels); otherwise the
* gamma curve applies, as for photos (see GrayPacket.fromArgb).
*/
@JvmStatic
fun toneLut(limitedRange: Boolean, gamma: Float, monochrome: Boolean): ByteArray {
val curve = GrayPacket.toneCurve(gamma)
val lut = ByteArray(256)
for (y in 0 until 256) {
val full = if (limitedRange) ((y - 16) * 255 + 109) / 219 else y
val v = full.coerceIn(0, 255)
lut[y] = (if (monochrome) (if (v >= 128) 255 else 0) else curve[v]).toByte()
}
return lut
}
/**
* Sample positions for each output index along one axis: [count] entries per output,
* evenly spaced inside that output's box of source pixels.
*/
private fun taps(source: Int, output: Int, count: Int): IntArray {
val positions = IntArray(output * count)
for (i in 0 until output) {
for (j in 0 until count) {
// Center of sub-box j of output i, in source pixels: (i + (j + 0.5)/count) * source / output.
val numerator = (i.toLong() * count + j) * 2 + 1
val position = (numerator * source / (2L * count * output)).toInt()
positions[i * count + j] = position.coerceIn(0, source - 1)
}
}
return positions
}
}
private val swapsAxes = rotation == 90 || rotation == 270
/** Scaled size before rotation (coded orientation). */
private val scaledWidth = if (swapsAxes) outputHeight else outputWidth
private val scaledHeight = if (swapsAxes) outputWidth else outputHeight
private val tapsX = ((sourceWidth + scaledWidth - 1) / scaledWidth).coerceIn(1, MAX_TAPS)
private val tapsY = ((sourceHeight + scaledHeight - 1) / scaledHeight).coerceIn(1, MAX_TAPS)
private val xs = taps(sourceWidth, scaledWidth, tapsX)
private val ys = taps(sourceHeight, scaledHeight, tapsY)
private val sums = IntArray(scaledWidth)
/** Unrotated scaled frame, needed only when a rotation follows. */
private val scaled: ByteArray? = if (rotation == 0) null else ByteArray(scaledWidth * scaledHeight)
init {
require(sourceWidth > 0 && sourceHeight > 0) { "bad source size ${sourceWidth}x$sourceHeight" }
require(outputWidth > 0 && outputHeight > 0) { "bad output size ${outputWidth}x$outputHeight" }
require(rotation == 0 || rotation == 90 || rotation == 180 || rotation == 270) { "bad rotation $rotation" }
}
/**
* Scale the luma plane into [out] (outputWidth * outputHeight bytes, row-major). The visible
* pixel (x, y) is at luma[offset + y * rowStride + x * pixelStride]; [lut] is a 256-entry
* table from [toneLut].
*/
fun scale(luma: ByteArray, offset: Int, rowStride: Int, pixelStride: Int, lut: ByteArray, out: ByteArray) {
require(out.size >= outputWidth * outputHeight) { "output buffer too small" }
val target = scaled ?: out
val count = tapsX * tapsY
val half = count / 2
for (oy in 0 until scaledHeight) {
sums.fill(0)
for (ty in 0 until tapsY) {
val row = offset + ys[oy * tapsY + ty] * rowStride
var tap = 0
for (ox in 0 until scaledWidth) {
var sum = 0
for (tx in 0 until tapsX) {
sum += luma[row + xs[tap++] * pixelStride].toInt() and 0xff
}
sums[ox] += sum
}
}
val base = oy * scaledWidth
for (ox in 0 until scaledWidth) {
target[base + ox] = lut[(sums[ox] + half) / count]
}
}
if (scaled != null) rotate(scaled, out)
}
/** Rotate the coded-orientation frame clockwise by [rotation] into display orientation. */
private fun rotate(source: ByteArray, out: ByteArray) {
val w = scaledWidth
val h = scaledHeight
for (y in 0 until outputHeight) {
val base = y * outputWidth
for (x in 0 until outputWidth) {
out[base + x] = when (rotation) {
90 -> source[(h - 1 - x) * w + y]
180 -> source[(h - 1 - y) * w + (w - 1 - x)]
else -> source[x * w + (w - 1 - y)]
}
}
}
}
}
@@ -92,7 +92,7 @@ class FrameTimingsCore(
fun root(): Frame {
var frame: Frame = this
while (frame.parent != null) {
frame = frame.parent!!
frame = frame.parent
}
return frame
}
@@ -268,13 +268,16 @@ class FrameTimingsCore(
get() = lock.withLock { exportDirty }
/** The full export text (does not clear the dirty flag). */
fun buildExport(): String = lock.withLock { buildExportLocked() }
fun buildExport(): String = formatExport(lock.withLock { exportSnapshotLocked() })
/** The export text if anything changed since the last take, else null; clears the dirty flag. */
fun takeExport(): String? = lock.withLock {
if (!exportDirty) return null
exportDirty = false
buildExportLocked()
fun takeExport(): String? {
val snapshot = lock.withLock {
if (!exportDirty) return null
exportDirty = false
exportSnapshotLocked()
}
return formatExport(snapshot)
}
// ---------------------------------------------------------------------
@@ -360,30 +363,72 @@ class FrameTimingsCore(
)
}
private fun buildExportLocked(): String {
/**
* Frame state as of one moment, formatted into the export text outside the lock. Formatting
* the full export takes tens of milliseconds, and holding the lock for it stalled every
* thread recording a span meanwhile, the main thread included.
*/
private class FrameSnapshot(
val id: Int,
val reason: String,
val startedAtMs: Long,
val startedWallClockMs: Long,
val durationMs: Long,
val open: Boolean,
val outcome: String?,
val subtreeEndAtMs: Long,
val subtreeHasOpenFrame: Boolean,
val lines: List<Line>,
val children: List<FrameSnapshot>,
)
private class ExportSnapshot(
val wallClockMs: Long,
val statsSummary: String,
val slowest: List<FrameSnapshot>,
val recent: List<FrameSnapshot>,
)
private fun snapshotLocked(frame: Frame, now: Long): FrameSnapshot = FrameSnapshot(
frame.id, frame.reason, frame.startedAtMs, frame.startedWallClockMs, frame.durationMs(), frame.isOpen(),
frame.outcome, frame.subtreeEndAtMs(now), frame.subtreeHasOpenFrame(), frame.lines.toList(),
frame.children.map { snapshotLocked(it, now) },
)
private fun exportSnapshotLocked(): ExportSnapshot {
val now = platform.elapsedRealtimeMs()
// Stable sort, descending by whole-tree duration (matches Collections.sort).
val treeDurations = HashMap<Frame, Long>()
for (frame in slowestRoots) treeDurations[frame] = frame.subtreeEndAtMs(now) - frame.startedAtMs
slowestRoots.sortWith(compareByDescending { treeDurations.getValue(it) })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
return ExportSnapshot(
wallClockMs(),
statsSummaryLocked(),
slowestRoots.map { snapshotLocked(it, now) },
recentRoots.map { snapshotLocked(it, now) },
)
}
private fun formatExport(snapshot: ExportSnapshot): String {
val out = StringBuilder(64 * 1024)
out.append("FrameTimings export at ").append(formatWallClock(wallClockMs())).append('\n')
out.append(statsSummaryLocked()).append('\n')
out.append("FrameTimings export at ").append(formatWallClock(snapshot.wallClockMs)).append('\n')
out.append(snapshot.statsSummary).append('\n')
out.append('\n')
out.append("Frames are trees: an input event's own frame is the root, and the renders it\n")
out.append("caused are indented under it with offsets measured from the root's start.\n")
out.append('\n')
// Stable sort, descending by whole-tree duration (matches Collections.sort).
slowestRoots.sortWith(compareByDescending { it.subtreeEndAtMs(now) - it.startedAtMs })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
out.append("=== slowest frames that reached the glasses ===\n")
for (frame in slowestRoots) {
appendTreeLocked(out, frame, frame, now)
for (frame in snapshot.slowest) {
appendTree(out, frame, frame, 0)
}
out.append("=== recent frames (oldest first) ===\n")
for (frame in recentRoots) {
appendTreeLocked(out, frame, frame, now)
for (frame in snapshot.recent) {
appendTree(out, frame, frame, 0)
}
return out.toString()
}
@@ -416,28 +461,22 @@ class FrameTimingsCore(
}
/** Print a frame and its descendants, all timed against root's start. */
private fun appendTreeLocked(out: StringBuilder, frame: Frame, root: Frame, now: Long) {
var depth = 0
var walk: Frame? = frame
while (walk !== root) {
depth++
walk = walk!!.parent
}
private fun appendTree(out: StringBuilder, frame: FrameSnapshot, root: FrameSnapshot, depth: Int) {
val indent = " ".repeat(depth)
val startOffsetMs = frame.startedAtMs - root.startedAtMs
out.append(indent).append("frame#").append(frame.id)
.append(" [").append(frame.reason).append(']')
if (frame === root) {
out.append(" started ").append(formatWallClock(frame.startedWallClockMs))
val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs
out.append(" duration ").append(frame.durationMs()).append("ms")
val totalMs = frame.subtreeEndAtMs - frame.startedAtMs
out.append(" duration ").append(frame.durationMs).append("ms")
if (frame.children.isNotEmpty()) {
out.append(" (tree ").append(totalMs).append("ms")
.append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')')
.append(if (frame.subtreeHasOpenFrame) ", still open" else "").append(')')
}
} else {
out.append(" started +").append(startOffsetMs).append("ms")
.append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms"))
.append(" duration ").append(if (frame.open) "open" else (frame.durationMs.toString() + "ms"))
}
out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n')
for (line in frame.lines) {
@@ -445,7 +484,7 @@ class FrameTimingsCore(
.append("ms [").append(line.thread).append("] ").append(line.message).append('\n')
}
for (child in frame.children) {
appendTreeLocked(out, child, root, now)
appendTree(out, child, root, depth + 1)
}
}
}
@@ -1,35 +0,0 @@
package com.faceclaw.shared
/** NativeScript implements this Kotlin-owned contract in TypeScript. */
interface KotlinBridgeListener {
fun transform(message: String): String
}
/**
* Portable Kotlin at the same commonMain source path used by ios-port. Calls and callbacks are
* synchronous on the caller's thread. The listener is used only during roundTrip and is never
* retained.
*/
class KotlinBridge {
fun platform(): String = PlatformInfo.name
fun threadName(): String = PlatformInfo.threadName()
fun isMainThread(): Boolean = PlatformInfo.isMainThread()
fun greet(name: String): String = "Hello, $name, from Kotlin"
fun roundTrip(message: String, listener: KotlinBridgeListener): String {
val reply = listener.transform("Kotlin received: $message")
// Consume the TypeScript return value, then return it across the bridge.
return "Kotlin received callback: $reply"
}
}
internal expect object PlatformInfo {
val name: String
fun threadName(): String
fun isMainThread(): Boolean
}
@@ -0,0 +1,9 @@
package com.faceclaw.shared
internal expect object PlatformInfo {
val name: String
fun threadName(): String
fun isMainThread(): Boolean
}
@@ -1,4 +1,4 @@
@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class)
@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class, kotlinx.cinterop.BetaInteropApi::class)
package com.faceclaw.app
@@ -277,4 +277,7 @@ class IosSurfaceCompositor(width: Int, height: Int) {
fun composite(): NSData = compositor.composite().gray.data()
fun compositeFrame(): IosTextureFrame = IosTextureFrame(compositor.composite())
/** Called on the main queue after each step of an animation the preview is replaying (see SurfaceCompositor). */
fun setPreviewAnimationListener(listener: (() -> Unit)?) = compositor.setPreviewAnimationListener(listener)
}
@@ -52,6 +52,9 @@ fun interface IosAncsAuthorizationListener {
fun onAncsAuthorization(identifier: String, authorized: Boolean)
}
/** The canonical 8-4-4-4-12 hex form -[NSUUID initWithUUIDString:] accepts. */
private val UUID_STRING = Regex("^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$")
/**
* CoreBluetooth central used by the shared session and stock-firmware flows on iOS. It is
* the iOS counterpart of Android's FaceclawBleManager: every [SessionLink]/[StockLink] call
@@ -268,7 +271,10 @@ class IosBleCentral(private val platform: ProtocolPlatform = IosProtocolPlatform
var retrieved: CBPeripheral? = null
dispatch_sync(queue) {
val manager = central ?: return@dispatch_sync
val uuid = NSUUID(uUIDString = identifier) ?: return@dispatch_sync
// NSUUID's initializer can return nil, but Kotlin imports it as a
// non-null constructor: screen out malformed identifiers first.
if (!UUID_STRING.matches(identifier)) return@dispatch_sync
val uuid = NSUUID(uUIDString = identifier)
retrieved = manager.retrievePeripheralsWithIdentifiers(listOf(uuid)).firstOrNull() as? CBPeripheral
}
retrieved?.let { return remember(it) }
@@ -70,6 +70,8 @@ class IosGlassesSession internal constructor(
fun setListener(listener: FaceclawBleCommunicatorListener?) = core.setListener(listener)
fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision)
fun start(): Boolean = core.start()
fun disconnect() = core.disconnect()
@@ -244,6 +246,9 @@ class IosGlassesSession internal constructor(
/** The current composited screen (gray, 8bpp) or null before any surface was configured. */
fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) }
/** Called on the main queue after each step of an animation the preview is replaying (see SurfaceCompositor). */
fun setPreviewAnimationListener(listener: (() -> Unit)?) = core.setPreviewAnimationListener(listener)
fun compositeWidth(): Int = core.previewComposite()?.width ?: 0
fun compositeHeight(): Int = core.previewComposite()?.height ?: 0
@@ -8,7 +8,6 @@ import platform.Foundation.NSData
import platform.Foundation.NSError
import platform.Foundation.NSHTTPURLResponse
import platform.Foundation.NSMutableURLRequest
import platform.Foundation.NSString
import platform.Foundation.NSURL
import platform.Foundation.NSURLRequest
import platform.Foundation.NSURLResponse
@@ -19,9 +18,6 @@ import platform.Foundation.NSURLSessionDataTask
import platform.Foundation.NSURLSessionResponseAllow
import platform.Foundation.NSURLSessionResponseDisposition
import platform.Foundation.NSURLSessionTask
import platform.Foundation.NSUTF8StringEncoding
import platform.Foundation.create
import platform.Foundation.dataUsingEncoding
import platform.Foundation.setHTTPBody
import platform.Foundation.setHTTPMethod
import platform.Foundation.setValue
@@ -31,7 +27,7 @@ import platform.darwin.NSObject
internal fun buildRequest(url: NSURL, method: String, body: String?, headers: List<Pair<String, String>>): NSMutableURLRequest {
val request = NSMutableURLRequest.requestWithURL(url)
request.setHTTPMethod(method)
if (body != null) request.setHTTPBody((body as NSString).dataUsingEncoding(NSUTF8StringEncoding))
if (body != null) request.setHTTPBody(body.encodeToByteArray().data())
var hasUserAgent = false
for ((name, value) in headers) {
if (name.equals("User-Agent", ignoreCase = true)) hasUserAgent = true