forked from Blendman974/faceclaw
Less-sensitive disconnect detection to avoid reconnect loops under BLE contention
This commit is contained in:
1 parent
7d8f03ffe5
commit
7684403491
6 files changed
+128
-6
No files matched your search
+15
-3
@@ -5,11 +5,23 @@ import kotlin.jvm.JvmStatic
|
|||||||
/** Ordered completion and go-back-N recovery for the custom-message window. */
|
/** Ordered completion and go-back-N recovery for the custom-message window. */
|
||||||
class CfwMessageWindow private constructor() {
|
class CfwMessageWindow private constructor() {
|
||||||
companion object {
|
companion object {
|
||||||
// Flappy capture: p99 ACK 63ms, max 77ms. Allow full-frame processing
|
// An unresolved message is presumed lost only once both kinds of progress on
|
||||||
// while recovering a lost reply well before the stock timeout.
|
// its arm have stopped: no write completion for DATA_STALL_MS, and no CFW ack
|
||||||
const val ACK_TIMEOUT_MS = 500
|
// for ACK_STALL_MS since it was sent. Android withholds write completions
|
||||||
|
// while its outgoing queue is congested, so completions mean data is still
|
||||||
|
// draining; acks mean the firmware is still working through the window. A
|
||||||
|
// single 500 ms deadline from send replayed large uploads whose acks were
|
||||||
|
// just slow (an 11 KB upload acked by one lens at 536 ms), and under
|
||||||
|
// bandwidth pressure (LE Audio streaming) it tore the session down.
|
||||||
|
const val DATA_STALL_MS = 500
|
||||||
|
const val ACK_STALL_MS = 3_000
|
||||||
const val MAX_RETRIES = 3
|
const val MAX_RETRIES = 3
|
||||||
|
|
||||||
|
/** When a message sent at [sentAtMs] counts as lost, given its arm's latest write completion and CFW ack. */
|
||||||
|
@JvmStatic
|
||||||
|
fun ackDeadline(sentAtMs: Long, lastArmWriteAtMs: Long, lastArmAckAtMs: Long): Long =
|
||||||
|
maxOf(lastArmWriteAtMs + DATA_STALL_MS, maxOf(sentAtMs, lastArmAckAtMs) + ACK_STALL_MS)
|
||||||
|
|
||||||
/** ID reuse must not invalidate an earlier command still eligible for replay. */
|
/** ID reuse must not invalidate an earlier command still eligible for replay. */
|
||||||
@JvmStatic
|
@JvmStatic
|
||||||
fun canSend(messages: Iterable<OutboundMessage>, candidate: OutboundMessage): Boolean =
|
fun canSend(messages: Iterable<OutboundMessage>, candidate: OutboundMessage): Boolean =
|
||||||
|
|||||||
+1
-1
@@ -97,7 +97,7 @@ class MessageBuilder {
|
|||||||
0,
|
0,
|
||||||
streamId,
|
streamId,
|
||||||
payload,
|
payload,
|
||||||
CfwMessageWindow.ACK_TIMEOUT_MS,
|
CfwMessageWindow.ACK_STALL_MS,
|
||||||
tileIndex,
|
tileIndex,
|
||||||
leftArm,
|
leftArm,
|
||||||
)
|
)
|
||||||
|
|||||||
+17
-1
@@ -314,6 +314,11 @@ class GlassesSessionCore(
|
|||||||
|
|
||||||
internal val pendingMessages = ArrayDeque<OutboundMessage>()
|
internal val pendingMessages = ArrayDeque<OutboundMessage>()
|
||||||
internal val cfwTransports = arrayOf(CfwTransport(platform), CfwTransport(platform))
|
internal val cfwTransports = arrayOf(CfwTransport(platform), CfwTransport(platform))
|
||||||
|
// Per arm (0 = left, 1 = right, like cfwTransports): when a write to it last
|
||||||
|
// completed and when a CFW ack last arrived from it. The CFW window's liveness
|
||||||
|
// signals (see CfwMessageWindow.ackDeadline).
|
||||||
|
internal val lastArmWriteAtMs = LongArray(2)
|
||||||
|
internal val lastArmCfwAckAtMs = LongArray(2)
|
||||||
internal val inFlightMessages = ArrayDeque<OutboundMessage>()
|
internal val inFlightMessages = ArrayDeque<OutboundMessage>()
|
||||||
internal var prewrittenMessage: OutboundMessage? = null
|
internal var prewrittenMessage: OutboundMessage? = null
|
||||||
internal var prewrittenFrames: List<ByteArray> = emptyList()
|
internal var prewrittenFrames: List<ByteArray> = emptyList()
|
||||||
@@ -350,8 +355,10 @@ class GlassesSessionCore(
|
|||||||
val active = monitor.withLock { running }
|
val active = monitor.withLock { running }
|
||||||
if (!active) return false
|
if (!active) return false
|
||||||
return try {
|
return try {
|
||||||
link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE,
|
val result = link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE,
|
||||||
ConnectionOptions.WRITE_TIMEOUT_MS)
|
ConnectionOptions.WRITE_TIMEOUT_MS)
|
||||||
|
if (result) monitor.withLock { noteWriteCompletedLocked(address, now()) }
|
||||||
|
result
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
logLine("raw packet write failed: " + safeMessage(t))
|
logLine("raw packet write failed: " + safeMessage(t))
|
||||||
false
|
false
|
||||||
@@ -1348,6 +1355,8 @@ class GlassesSessionCore(
|
|||||||
val acks = CfwTransport.parseAcks(data) ?: return
|
val acks = CfwTransport.parseAcks(data) ?: return
|
||||||
monitor.withLock {
|
monitor.withLock {
|
||||||
lastIncomingAtMs = now()
|
lastIncomingAtMs = now()
|
||||||
|
val arm = armIndex(address)
|
||||||
|
if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs
|
||||||
for (ack in acks) {
|
for (ack in acks) {
|
||||||
for (message in inFlightMessages) {
|
for (message in inFlightMessages) {
|
||||||
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
|
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
|
||||||
@@ -2018,6 +2027,13 @@ class GlassesSessionCore(
|
|||||||
return id
|
return id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 0 for the left arm, 1 for the right (the cfwTransports order), -1 for anything else. */
|
||||||
|
internal fun armIndex(address: String): Int = when {
|
||||||
|
address.equals(leftAddress, ignoreCase = true) -> 0
|
||||||
|
address.equals(rightAddress, ignoreCase = true) -> 1
|
||||||
|
else -> -1
|
||||||
|
}
|
||||||
|
|
||||||
internal fun hasRingAddress(): Boolean {
|
internal fun hasRingAddress(): Boolean {
|
||||||
return ringAddress.trim().isNotEmpty()
|
return ringAddress.trim().isNotEmpty()
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-1
@@ -79,6 +79,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
if (!inFlightMessages.isEmpty()) {
|
if (!inFlightMessages.isEmpty()) {
|
||||||
|
refreshCfwAckDeadlinesLocked()
|
||||||
val oldest = inFlightMessages.firstOrNull()
|
val oldest = inFlightMessages.firstOrNull()
|
||||||
val replay = CfwMessageWindow.replayWindow(inFlightMessages, now)
|
val replay = CfwMessageWindow.replayWindow(inFlightMessages, now)
|
||||||
if (!replay.isEmpty()) {
|
if (!replay.isEmpty()) {
|
||||||
@@ -415,6 +416,7 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean
|
|||||||
|
|
||||||
monitor.withLock {
|
monitor.withLock {
|
||||||
val sentAtMs = now()
|
val sentAtMs = now()
|
||||||
|
if (result) noteWriteCompletedLocked(writeAddress, sentAtMs)
|
||||||
message.sentAtMs = sentAtMs
|
message.sentAtMs = sentAtMs
|
||||||
message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs
|
message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs
|
||||||
logImageUpdateSendLandmarkLocked(message)
|
logImageUpdateSendLandmarkLocked(message)
|
||||||
@@ -463,6 +465,7 @@ internal fun GlassesSessionCore.prewriteMessage(message: OutboundMessage): Boole
|
|||||||
if (!result) {
|
if (!result) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
monitor.withLock { noteWriteCompletedLocked(writeAddress, now()) }
|
||||||
|
|
||||||
prewrittenMessage = message
|
prewrittenMessage = message
|
||||||
prewrittenFrames = frames.toList()
|
prewrittenFrames = frames.toList()
|
||||||
@@ -486,13 +489,39 @@ internal fun GlassesSessionCore.spoilPrewrittenMessage(reason: String): Boolean
|
|||||||
|
|
||||||
val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress
|
val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress
|
||||||
logLine("spoiling prewritten " + message.label + ": " + reason)
|
logLine("spoiling prewritten " + message.label + ": " + reason)
|
||||||
return link.writeFrames(
|
val result = link.writeFrames(
|
||||||
writeAddress,
|
writeAddress,
|
||||||
BleProtocol.WRITE_CHAR_UUID,
|
BleProtocol.WRITE_CHAR_UUID,
|
||||||
listOf(finalFrame),
|
listOf(finalFrame),
|
||||||
ConnectionOptions.WRITE_MODE,
|
ConnectionOptions.WRITE_MODE,
|
||||||
ConnectionOptions.WRITE_TIMEOUT_MS
|
ConnectionOptions.WRITE_TIMEOUT_MS
|
||||||
)
|
)
|
||||||
|
if (result) monitor.withLock { noteWriteCompletedLocked(writeAddress, now()) }
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record that writes to [address] just completed: every frame's onCharacteristicWrite
|
||||||
|
* fired. The drive loop only checks ack deadlines between writes, so noting it when
|
||||||
|
* writeFrames returns is as good as noting each callback.
|
||||||
|
*/
|
||||||
|
internal fun GlassesSessionCore.noteWriteCompletedLocked(address: String, atMs: Long) {
|
||||||
|
val arm = armIndex(address)
|
||||||
|
if (arm >= 0) lastArmWriteAtMs[arm] = atMs
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Re-derive each written, unresolved CFW message's ack deadline from its arm's latest
|
||||||
|
* write completion and CFW ack (CfwMessageWindow.ackDeadline), so either kind of
|
||||||
|
* progress keeps the window waiting. Deadlines only move later.
|
||||||
|
*/
|
||||||
|
internal fun GlassesSessionCore.refreshCfwAckDeadlinesLocked() {
|
||||||
|
for (message in inFlightMessages) {
|
||||||
|
if (message.sid != CfwTransport.SID || message.ackDeadlineAtMs <= 0) continue
|
||||||
|
val arm = if (message.isLeftArmMessage) 0 else 1
|
||||||
|
message.ackDeadlineAtMs = CfwMessageWindow.ackDeadline(
|
||||||
|
message.sentAtMs, lastArmWriteAtMs[arm], lastArmCfwAckAtMs[arm])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal fun GlassesSessionCore.removePreparedMessageLocked(message: OutboundMessage?): Boolean {
|
internal fun GlassesSessionCore.removePreparedMessageLocked(message: OutboundMessage?): Boolean {
|
||||||
|
|||||||
@@ -464,6 +464,59 @@ class GlassesSessionCoreTest {
|
|||||||
s.core.disconnect()
|
s.core.disconnect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun aCfwAckSlowerThanHalfASecondIsNotReplayed() {
|
||||||
|
val s = Session()
|
||||||
|
s.link.muted.add("image")
|
||||||
|
s.startAndAwaitLayout()
|
||||||
|
s.core.configureCompositorScreen(64, 32)
|
||||||
|
s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE)
|
||||||
|
try {
|
||||||
|
s.submitFrame(3, 0x20, 1)
|
||||||
|
assertTrue(waitUntil(5_000) { s.link.cfwMessages().size == 1 })
|
||||||
|
// Well past the old 500 ms deadline, but inside the 3 s ack window.
|
||||||
|
s.platform.offsetMs += 1_500
|
||||||
|
sleepMs(400)
|
||||||
|
assertFalse(s.host.hasLog("CFW recovery"), s.host.logs().takeLast(10).toString())
|
||||||
|
assertEquals(1, s.link.cfwMessages().size)
|
||||||
|
// The late ack still completes the frame.
|
||||||
|
val first = s.link.writes.first { it.sid == CfwTransport.SID && it.kind == "image" }
|
||||||
|
val message = assertNotNull(s.core.monitor.withLock { s.core.inFlightMessages.firstOrNull { it.magic == first.magic } })
|
||||||
|
for (lens in 1..2) {
|
||||||
|
s.core.onNotification(LEFT, BleProtocol.NOTIFY_CHAR_UUID, cfwAck(first.magic, lens, message.message.size, message.cfwChecksum))
|
||||||
|
}
|
||||||
|
assertTrue(waitUntil(3_000) { s.displayedMatchesDesired() })
|
||||||
|
} finally {
|
||||||
|
s.core.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun cfwAckDeadlinesFollowTheirOwnArmsWritesAndAcks() {
|
||||||
|
val core = Session().core
|
||||||
|
fun cfw(magic: Int, leftArm: Boolean) = OutboundMessage(
|
||||||
|
"image", "m$magic", CfwTransport.SID, 0, magic, byteArrayOf(1), CfwMessageWindow.ACK_STALL_MS, 0, leftArm)
|
||||||
|
val left = cfw(1, true)
|
||||||
|
val right = cfw(2, false)
|
||||||
|
core.monitor.withLock {
|
||||||
|
for (message in listOf(left, right)) {
|
||||||
|
message.sentAtMs = 1_000
|
||||||
|
message.ackDeadlineAtMs = 4_000
|
||||||
|
core.inFlightMessages.addLast(message)
|
||||||
|
}
|
||||||
|
core.lastArmCfwAckAtMs[0] = 2_000
|
||||||
|
core.lastArmWriteAtMs[1] = 5_000
|
||||||
|
core.refreshCfwAckDeadlinesLocked()
|
||||||
|
}
|
||||||
|
assertEquals(5_000L, left.ackDeadlineAtMs)
|
||||||
|
assertEquals(5_500L, right.ackDeadlineAtMs)
|
||||||
|
// Any CFW ack from an arm counts as that arm's progress, matched or not.
|
||||||
|
val before = core.now()
|
||||||
|
core.onNotification(LEFT, BleProtocol.NOTIFY_CHAR_UUID, cfwAck(77, 1, 1, 0))
|
||||||
|
assertTrue(core.monitor.withLock { core.lastArmCfwAckAtMs[0] } >= before)
|
||||||
|
assertEquals(0L, core.monitor.withLock { core.lastArmCfwAckAtMs[1] })
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun reconnectBackoffGrowsAndRestartsAfterAStableSession() {
|
fun reconnectBackoffGrowsAndRestartsAfterAStableSession() {
|
||||||
val core = Session().core
|
val core = Session().core
|
||||||
|
|||||||
@@ -212,6 +212,18 @@ class ProtocolTest {
|
|||||||
assertTrue(CfwMessageWindow.canSend(listOf(eviction), upload))
|
assertTrue(CfwMessageWindow.canSend(listOf(eviction), upload))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun cfwAckDeadlineWaitsWhileEitherWritesOrAcksProgress() {
|
||||||
|
// Sent at 1000 into an idle arm: 3 s from the send.
|
||||||
|
assertEquals(4_000L, CfwMessageWindow.ackDeadline(1_000, 1_000, 0))
|
||||||
|
// An ack from the arm restarts the 3 s ack window.
|
||||||
|
assertEquals(5_500L, CfwMessageWindow.ackDeadline(1_000, 1_000, 2_500))
|
||||||
|
// Writes still completing keep it waiting past the ack window, 500 ms at a time.
|
||||||
|
assertEquals(6_700L, CfwMessageWindow.ackDeadline(1_000, 6_200, 2_500))
|
||||||
|
// An ack from before the send doesn't shorten anything.
|
||||||
|
assertEquals(4_000L, CfwMessageWindow.ackDeadline(1_000, 1_000, 900))
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun orderedCompletionAndWholeWindowRecovery() {
|
fun orderedCompletionAndWholeWindowRecovery() {
|
||||||
val head = message()
|
val head = message()
|
||||||
|
|||||||
Reference in new issue
Block a user