Improve connection stability, I hope

This commit is contained in:
James Babcock committed 2026-09-29 14:33:23 -07:00
1 parent 48f7f3d01f
commit 6267afaff7
9 files changed
+497 -243

No files matched your search

@@ -6,6 +6,10 @@ import android.bluetooth.BluetoothGatt
class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLink {
override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs)
override fun beginConnect(address: String, background: Boolean): Boolean = bleManager.beginConnect(address, background)
override fun isConnected(address: String): Boolean = bleManager.isConnected(address)
override fun requestHighPriority(address: String) {
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
}
@@ -11,14 +11,13 @@ import android.bluetooth.BluetoothGattService
import android.bluetooth.BluetoothManager
import android.bluetooth.BluetoothProfile
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import android.util.Log
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
@@ -28,6 +27,26 @@ class FaceclawBleManager(context: Context) {
private const val TAG = "FaceclawBle"
private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200)
// Handed to an operation waiter whose link dropped before its completion
// callback arrived; never a real GATT status.
private const val STATUS_LINK_LOST = -1
// Android's Bluetooth stack has process-wide command-pipeline constraints on some
// devices, so every GATT operation in the process is serialized on this one lock,
// across manager instances too (the session and each stock-firmware flow own one).
// An operation holds it until its completion callback; waiting for a connection
// to come up does not.
private val bluetoothApiLock = Any()
// Listener callbacks run here, in arrival order, instead of on the binder thread
// that delivered them. BluetoothGatt's callback interface is oneway, so a listener
// that blocks (the session core waits for its monitor) would otherwise hold back
// every later callback for that device, including the write completion that the
// monitor's holder may itself be waiting for.
private val callbackExecutor: ExecutorService = Executors.newSingleThreadExecutor { runnable ->
Thread(runnable, "FaceclawBleCallbacks").apply { isDaemon = true }
}
// Process-wide outbound-traffic totals, sampled by the phone UI's BLE
// bandwidth indicator. Static so counts from every manager instance and
// isolate land in the same totals.
@@ -48,28 +67,34 @@ class FaceclawBleManager(context: Context) {
}
}
/**
* One BluetoothGatt client for an address: a pending connection attempt until
* [connected]. [gatt] is set once connectGatt returns; a callback can arrive before
* that, while it is still null.
*/
private class GattLink {
@Volatile var gatt: BluetoothGatt? = null
@Volatile var connected = false
val connectLatch = CountDownLatch(1)
}
private enum class OpKind { MTU, SERVICES, DESCRIPTOR, WRITE }
/** The GATT operation awaiting its completion callback (the global lock allows one at a time). */
private class PendingOp(val address: String, val kind: OpKind) {
val latch = CountDownLatch(1)
@Volatile var status: Int? = null
}
private val context: Context
private val bluetoothManager: BluetoothManager
private val bluetoothAdapter: BluetoothAdapter
private val mainHandler = Handler(Looper.getMainLooper())
private val gattClients = ConcurrentHashMap<String, BluetoothGatt>()
private val bluetoothApiLock = Any()
private val connectLatches = ConcurrentHashMap<String, CountDownLatch>()
private val connectResults = ConcurrentHashMap<String, Boolean>()
private val servicesLatches = ConcurrentHashMap<String, CountDownLatch>()
private val servicesStatuses = ConcurrentHashMap<String, Int>()
private val mtuLatches = ConcurrentHashMap<String, CountDownLatch>()
private val links = ConcurrentHashMap<String, GattLink>()
private val negotiatedMtus = ConcurrentHashMap<String, Int>()
private val mtuStatuses = ConcurrentHashMap<String, Int>()
private val descriptorLatches = ConcurrentHashMap<String, CountDownLatch>()
private val descriptorStatuses = ConcurrentHashMap<String, Int>()
private val writeLatches = ConcurrentHashMap<String, CountDownLatch>()
private val writeStatuses = ConcurrentHashMap<String, Int>()
@Volatile
private var pendingOp: PendingOp? = null
@Volatile
private var listener: FaceclawBleListener? = null
@@ -80,6 +105,7 @@ class FaceclawBleManager(context: Context) {
if (bluetoothManager == null || bluetoothManager.adapter == null) {
throw IllegalStateException("Bluetooth adapter unavailable")
}
this.bluetoothManager = bluetoothManager
this.bluetoothAdapter = bluetoothManager.adapter
}
@@ -122,67 +148,81 @@ class FaceclawBleManager(context: Context) {
}
}
/** Whether a GATT client for this address is open, i.e. no disconnect callback has arrived for it. */
/** Whether a connection to this address is up (not merely pending). */
fun isConnected(address: String?): Boolean {
return address != null && gattClients.containsKey(address)
return address != null && links[address]?.connected == true
}
/**
* Start connecting without waiting; a later [connect] waits on the same attempt.
* [background] uses Android's autoConnect: a low-duty-cycle attempt with no timeout
* that lands whenever the device next advertises (a direct attempt scans hard but
* gives up after about 30 s). Returns true while a connection exists or is pending.
*/
fun beginConnect(address: String?, background: Boolean): Boolean {
return startConnect(requireAddress(address), background) != null
}
fun connect(address: String?, timeoutMs: Int): Boolean {
if (address == null || address.trim().isEmpty()) {
throw IllegalArgumentException("address is required")
val key = requireAddress(address)
val link = startConnect(key, false) ?: return false
if (link.connected) {
return true
}
val latch = CountDownLatch(1)
connectLatches[address] = latch
connectResults.remove(address)
if (!awaitLatch(link.connectLatch, timeoutMs)) {
Log.w(TAG, "connect timed out after ${timeoutMs}ms: address=$key"
+ " (no connection event; the device may not be advertising, or may be connected to something else)")
dropLink(key, link)
return false
}
return link.connected
}
val gattLock = gattLock(address)
synchronized(gattLock) {
val existing = gattClients[address]
/** The current client for [address], starting a connection attempt when there is none. */
private fun startConnect(address: String, background: Boolean): GattLink? {
synchronized(bluetoothApiLock) {
val existing = links[address]
if (existing != null) {
connectLatches.remove(address)
return true
if (!existing.connected || isOsConnected(existing)) {
return existing
}
// A client can go stale without a disconnect callback (the Bluetooth
// service drops its clients when the adapter turns off).
Log.w(TAG, "dropping stale client: address=$address (Android reports no GATT connection)")
dropLink(address, existing)
}
val device = bluetoothAdapter.getRemoteDevice(address)
if (device == null) {
connectLatches.remove(address)
throw IllegalArgumentException("remote device not found: $address")
}
val link = GattLink()
links[address] = link
val gatt: BluetoothGatt? = device.connectGatt(
context,
false,
background,
gattCallback,
BluetoothDevice.TRANSPORT_LE,
BluetoothDevice.PHY_LE_2M or BluetoothDevice.PHY_LE_1M
)
if (gatt != null) {
gattClients[address] = gatt
}
if (gatt == null) {
connectLatches.remove(address)
return false
links.remove(address, link)
Log.w(TAG, "connectGatt returned null: address=$address")
return null
}
link.gatt = gatt
Log.i(TAG, "connect started: address=$address background=$background")
return link
}
}
if (!awaitLatch(latch, timeoutMs)) {
connectLatches.remove(address)
connectResults.remove(address)
gattClients.remove(address, gatt)
negotiatedMtus.remove(address)
gatt.disconnect()
gatt.close()
return false
}
val connected = connectResults.remove(address)
connectLatches.remove(address)
return connected == true
private fun isOsConnected(link: GattLink): Boolean {
val gatt = link.gatt ?: return false
return try {
bluetoothManager.getConnectionState(gatt.device, BluetoothProfile.GATT) == BluetoothProfile.STATE_CONNECTED
} catch (t: Throwable) {
true
}
}
fun requestConnectionPriority(address: String, priority: Int): Boolean {
synchronized(gattLock(address)) {
synchronized(bluetoothApiLock) {
val gatt = requireGatt(address)
return gatt.requestConnectionPriority(priority)
}
@@ -190,7 +230,7 @@ class FaceclawBleManager(context: Context) {
/** One-shot benchmark preferences; success is established by callbacks/HCI, not submission. */
fun prepareBenchmarkLink(address: String, mode: Int) {
synchronized(gattLock(address)) {
synchronized(bluetoothApiLock) {
val gatt = requireGatt(address)
if ((mode and 1) != 0) {
val accepted = gatt.requestConnectionPriority(BluetoothGatt.CONNECTION_PRIORITY_HIGH)
@@ -211,69 +251,27 @@ class FaceclawBleManager(context: Context) {
}
fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean {
synchronized(gattLock(address)) {
val latch = CountDownLatch(1)
mtuLatches[address] = latch
mtuStatuses.remove(address)
val gatt = requireGatt(address)
val started = gatt.requestMtu(mtu)
if (!started) {
mtuLatches.remove(address)
return false
}
if (!awaitLatch(latch, timeoutMs)) {
mtuLatches.remove(address)
mtuStatuses.remove(address)
return false
}
val status = mtuStatuses.remove(address)
mtuLatches.remove(address)
return status != null && status == BluetoothGatt.GATT_SUCCESS
}
val status = runOp(address, OpKind.MTU, timeoutMs) { gatt -> gatt.requestMtu(mtu) }
return status == BluetoothGatt.GATT_SUCCESS
}
fun discoverServices(address: String, timeoutMs: Int): Boolean {
synchronized(gattLock(address)) {
val latch = CountDownLatch(1)
servicesLatches[address] = latch
servicesStatuses.remove(address)
val gatt = requireGatt(address)
val started = gatt.discoverServices()
if (!started) {
servicesLatches.remove(address)
return false
}
if (!awaitLatch(latch, timeoutMs)) {
servicesLatches.remove(address)
servicesStatuses.remove(address)
return false
}
val status = servicesStatuses.remove(address)
servicesLatches.remove(address)
return status != null && status == BluetoothGatt.GATT_SUCCESS
}
val status = runOp(address, OpKind.SERVICES, timeoutMs) { gatt -> gatt.discoverServices() }
return status == BluetoothGatt.GATT_SUCCESS
}
fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean {
synchronized(gattLock(address)) {
val latch = CountDownLatch(1)
descriptorLatches[address] = latch
descriptorStatuses.remove(address)
synchronized(bluetoothApiLock) {
val gatt = requireGatt(address)
val characteristic = requireCharacteristic(gatt, characteristicUuid)
val notificationSet = gatt.setCharacteristicNotification(characteristic, enable)
if (!notificationSet) {
descriptorLatches.remove(address)
return false
}
val descriptor = characteristic.getDescriptor(java.util.UUID.fromString(BleProtocol.CCCD_UUID))
if (descriptor == null) {
descriptorLatches.remove(address)
return true
}
@@ -281,20 +279,39 @@ class FaceclawBleManager(context: Context) {
BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE
else
BluetoothGattDescriptor.DISABLE_NOTIFICATION_VALUE
val started = gatt.writeDescriptor(descriptor)
if (!started) {
descriptorLatches.remove(address)
descriptorStatuses.remove(address)
return false
val status = runOp(address, OpKind.DESCRIPTOR, timeoutMs) { client -> client.writeDescriptor(descriptor) }
return status == BluetoothGatt.GATT_SUCCESS
}
}
/**
* Start one GATT operation and wait for its completion callback, holding the global
* lock throughout. Returns the callback's status ([STATUS_LINK_LOST] when the link
* dropped first), or null when Android refused the request or it timed out.
*/
private fun runOp(address: String, kind: OpKind, timeoutMs: Int, start: (BluetoothGatt) -> Boolean): Int? {
synchronized(bluetoothApiLock) {
val op = PendingOp(address, kind)
// Published before the connected check in requireGatt, so a disconnect racing
// with this start either fails that check or finds the op to release.
pendingOp = op
try {
val gatt = requireGatt(address)
if (!start(gatt)) {
Log.w(TAG, "$kind not started: address=$address")
return null
}
if (!awaitLatch(op.latch, timeoutMs)) {
Log.w(TAG, "$kind timed out after ${timeoutMs}ms: address=$address")
return null
}
if (op.status == STATUS_LINK_LOST) {
Log.w(TAG, "$kind abandoned: link lost: address=$address")
}
return op.status
} finally {
pendingOp = null
}
if (!awaitLatch(latch, timeoutMs)) {
descriptorLatches.remove(address)
descriptorStatuses.remove(address)
return false
}
val status = descriptorStatuses.remove(address)
descriptorLatches.remove(address)
return status != null && status == BluetoothGatt.GATT_SUCCESS
}
}
@@ -310,7 +327,7 @@ class FaceclawBleManager(context: Context) {
}
val startMs = System.currentTimeMillis()
synchronized(gattLock(address)) {
synchronized(bluetoothApiLock) {
val gatt = requireGatt(address)
val characteristic = requireCharacteristic(gatt, characteristicUuid)
@@ -338,45 +355,45 @@ class FaceclawBleManager(context: Context) {
): Boolean {
var retryCount = 0
while (true) {
val latch = CountDownLatch(1)
writeLatches[address] = latch
writeStatuses.remove(address)
val currentTime = System.currentTimeMillis()
val result = gatt.writeCharacteristic(characteristic, data!!, writeType)
val timeAsleep = System.currentTimeMillis() - currentTime
//Log.i(TAG, "writeCharacteristic: spent " + timeAsleep + "ms");
if (result == android.bluetooth.BluetoothStatusCodes.ERROR_GATT_WRITE_REQUEST_BUSY) {
writeLatches.remove(address, latch)
writeStatuses.remove(address)
if (!sleepBeforeWriteRetry(address, "busy", retryCount++)) {
val op = PendingOp(address, OpKind.WRITE)
pendingOp = op
try {
if (!isConnected(address)) {
Log.w(TAG, "writeCharacteristic abandoned: link lost: address=$address")
return false
}
continue
} else {
//Log.i(TAG, "writeCharacteristic with writeType=" + writeType + " result=" + result + " retryCount=" + retryCount);
val result = gatt.writeCharacteristic(characteristic, data!!, writeType)
if (result != android.bluetooth.BluetoothStatusCodes.SUCCESS) {
writeLatches.remove(address, latch)
writeStatuses.remove(address)
if (!sleepBeforeWriteRetry(address, "start result=$result", retryCount++)) {
if (!isConnected(address)) {
Log.w(TAG, "writeCharacteristic abandoned: link lost: address=$address")
return false
}
val reason = if (result == android.bluetooth.BluetoothStatusCodes.ERROR_GATT_WRITE_REQUEST_BUSY)
"busy"
else
"start result=$result"
if (!sleepBeforeWriteRetry(address, reason, retryCount++)) {
return false
}
continue
}
if (!awaitLatch(latch, timeoutMs)) {
writeLatches.remove(address, latch)
writeStatuses.remove(address)
if (!awaitLatch(op.latch, timeoutMs)) {
Log.w(TAG, "writeCharacteristic timed out after ${timeoutMs}ms waiting for completion: address=$address")
return false
}
val status = writeStatuses.remove(address)
writeLatches.remove(address)
if (status != null && status == BluetoothGatt.GATT_SUCCESS) {
val status = op.status
if (status == BluetoothGatt.GATT_SUCCESS) {
return true
}
if (status == STATUS_LINK_LOST) {
Log.w(TAG, "writeCharacteristic abandoned: link lost: address=$address")
return false
}
if (!sleepBeforeWriteRetry(address, "callback status=$status", retryCount++)) {
return false
}
} finally {
pendingOp = null
}
}
}
@@ -397,28 +414,70 @@ class FaceclawBleManager(context: Context) {
}
}
/** Close this address's client, cancelling a pending connection attempt. */
fun disconnect(address: String?) {
val gattLock = gattLock(address)
synchronized(gattLock) {
val gatt = if (address != null) gattClients.remove(address) else null
if (address != null) negotiatedMtus.remove(address)
if (gatt == null) {
return
}
if (address == null) {
return
}
val link = links[address] ?: return
dropLink(address, link)
}
fun close() {
for (address in links.keys) {
disconnect(address)
}
}
/**
* Forget [link] if it is still current for [address], release anything waiting on it,
* then disconnect and close its client. Closing suppresses the client's remaining
* callbacks, so listeners hear nothing about a link dropped here.
*/
private fun dropLink(address: String, link: GattLink) {
if (!links.remove(address, link)) {
return
}
link.connected = false
negotiatedMtus.remove(address)
failPendingOp(address)
link.connectLatch.countDown()
// startConnect holds the lock until link.gatt is set, so it is set by the time
// this block runs (or connectGatt failed and there is nothing to close).
synchronized(bluetoothApiLock) {
val gatt = link.gatt ?: return
gatt.disconnect()
gatt.close()
}
}
fun close() {
for (address in gattClients.keys) {
disconnect(address)
private fun failPendingOp(address: String) {
val op = pendingOp
if (op != null && op.address == address && op.latch.count > 0) {
op.status = STATUS_LINK_LOST
op.latch.countDown()
}
}
private fun completeOp(address: String, kind: OpKind, status: Int) {
val op = pendingOp
if (op != null && op.address == address && op.kind == kind) {
op.status = status
op.latch.countDown()
}
}
private fun requireAddress(address: String?): String {
if (address == null || address.trim().isEmpty()) {
throw IllegalArgumentException("address is required")
}
return address
}
private fun requireGatt(address: String): BluetoothGatt {
val gatt = gattClients[address]
if (gatt == null) {
val link = links[address]
val gatt = link?.gatt
if (link == null || gatt == null || !link.connected) {
throw IllegalStateException("Not connected: $address")
}
return gatt
@@ -444,52 +503,54 @@ class FaceclawBleManager(context: Context) {
}
}
private fun gattLock(address: String?): Any {
// Android's Bluetooth stack has process-wide command-pipeline constraints on some devices.
// Keep every BluetoothGatt API call serialized globally, even for different MAC addresses.
return bluetoothApiLock
}
private val gattCallback: BluetoothGattCallback = object : BluetoothGattCallback() {
override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) {
Log.i(TAG, "onConnectionStateChange: status=$status newState=$newState")
val address = gatt.device.address
if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) {
connectResults[address] = true
val latch = connectLatches.remove(address)
latch?.countDown()
dispatchConnectionState(address, true)
Log.i(TAG, "onConnectionStateChange: address=$address status=$status newState=$newState")
val link = links[address]
val current = link?.gatt
if (link == null || (current != null && current !== gatt)) {
// A client this manager no longer tracks: make sure it is released.
synchronized(bluetoothApiLock) {
if (newState != BluetoothProfile.STATE_DISCONNECTED) gatt.disconnect()
gatt.close()
}
return
}
if (newState == BluetoothProfile.STATE_DISCONNECTED) {
connectResults[address] = false
val latch = connectLatches.remove(address)
latch?.countDown()
val gattLock = gattLock(address)
synchronized(gattLock) {
gattClients.remove(address, gatt)
negotiatedMtus.remove(address)
gatt.close()
}
dispatchConnectionState(address, false)
if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) {
link.connected = true
link.connectLatch.countDown()
dispatchConnectionState(address, link, true)
return
}
if (newState != BluetoothProfile.STATE_DISCONNECTED && status == BluetoothGatt.GATT_SUCCESS) {
return
}
// Disconnected, or a failed attempt (including a connection reported with an
// error status). Wake the waiters first: the lock below may be held by one of
// them, waiting for a callback that will now never come.
links.remove(address, link)
link.connected = false
negotiatedMtus.remove(address)
failPendingOp(address)
link.connectLatch.countDown()
dispatchConnectionState(address, link, false)
synchronized(bluetoothApiLock) {
if (newState != BluetoothProfile.STATE_DISCONNECTED) gatt.disconnect()
gatt.close()
}
}
override fun onServicesDiscovered(gatt: BluetoothGatt, status: Int) {
val address = gatt.device.address
servicesStatuses[address] = status
val latch = servicesLatches.remove(address)
latch?.countDown()
completeOp(gatt.device.address, OpKind.SERVICES, status)
}
override fun onMtuChanged(gatt: BluetoothGatt, mtu: Int, status: Int) {
val address = gatt.device.address
if (status == BluetoothGatt.GATT_SUCCESS) negotiatedMtus[address] = mtu
mtuStatuses[address] = status
val latch = mtuLatches.remove(address)
latch?.countDown()
completeOp(address, OpKind.MTU, status)
}
override fun onPhyRead(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) {
@@ -503,17 +564,11 @@ class FaceclawBleManager(context: Context) {
}
override fun onDescriptorWrite(gatt: BluetoothGatt, descriptor: BluetoothGattDescriptor, status: Int) {
val address = gatt.device.address
descriptorStatuses[address] = status
val latch = descriptorLatches.remove(address)
latch?.countDown()
completeOp(gatt.device.address, OpKind.DESCRIPTOR, status)
}
override fun onCharacteristicWrite(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, status: Int) {
val address = gatt.device.address
writeStatuses[address] = status
val latch = writeLatches.remove(address)
latch?.countDown()
completeOp(gatt.device.address, OpKind.WRITE, status)
}
override fun onCharacteristicChanged(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, value: ByteArray) {
@@ -526,14 +581,31 @@ class FaceclawBleManager(context: Context) {
}
}
private fun dispatchConnectionState(address: String, connected: Boolean) {
val current = listener ?: return
current.onConnectionStateChange(address, connected)
private fun dispatchConnectionState(address: String, link: GattLink, connected: Boolean) {
callbackExecutor.execute {
// Drop news about a client that a newer attempt has already replaced.
val current = links[address]
if (if (connected) current !== link else current != null && current !== link) {
return@execute
}
val target = listener ?: return@execute
try {
target.onConnectionStateChange(address, connected)
} catch (t: Throwable) {
Log.w(TAG, "connection-state listener failed: address=$address", t)
}
}
}
private fun dispatchNotification(address: String, characteristicUuid: String, data: ByteArray?) {
val current = listener ?: return
val copy = data?.clone() ?: ByteArray(0)
current.onNotification(address, characteristicUuid, copy)
callbackExecutor.execute {
val target = listener ?: return@execute
try {
target.onNotification(address, characteristicUuid, copy)
} catch (t: Throwable) {
Log.w(TAG, "notification listener failed: address=$address", t)
}
}
}
}
@@ -8,6 +8,10 @@ class ConnectionOptions {
const val DESIRED_MTU = 512
const val CONNECT_TIMEOUT_MS = 5_000
// How long one session connect attempt keeps both arms' connections pending before
// giving up. Both are dialed at once, so a slow or missing arm gets the whole
// window (it stays under Android's own ~30 s direct-connect timeout).
const val ARM_CONNECT_WINDOW_MS = 20_000
const val SERVICES_TIMEOUT_MS = 5_000
const val DESCRIPTOR_TIMEOUT_MS = 5_000
const val RING_DESIRED_MTU = 247
@@ -36,7 +40,13 @@ class ConnectionOptions {
const val MAX_CONSECUTIVE_ACK_TIMEOUTS = 8
const val EVEN_APP_WRITE_FAILURE_WINDOW_MS = 15_000
const val IDLE_SLEEP_MS = 100
const val RECONNECT_DELAY_MS = 2_000
// Delay before each consecutive reconnect attempt (the last entry repeats). Android
// keeps an LE link up for a few seconds after its last GATT client closes (4 s on a
// Pixel), so the quick first retry reuses the old links and the glasses never see a
// disconnect; later retries wait long enough for the links to really drop.
@JvmField val RECONNECT_BACKOFF_MS = intArrayOf(2_000, 6_000, 10_000)
// A session that stayed up this long starts the backoff over when it fails.
const val STABLE_SESSION_MS = 30_000
// Cap on rects per batch. Each rect consumes a distinct CFW frame id, and the
// firmware's duplicate-fid ring holds 16, so keep several batches of history
// within it. Above this the split is abandoned for a single bounding box.
@@ -386,11 +386,17 @@ internal fun GlassesSessionCore.handleUnpairedFailure(address: String) {
interruptibleSleep.interrupt()
}
internal fun GlassesSessionCore.handleTransportFailure(reason: String?) {
/**
* Tear the session down and schedule a reconnect. [displayStatus], when given, replaces the
* generic "Reconnecting after <reason>" status line.
*/
internal fun GlassesSessionCore.handleTransportFailure(reason: String?, displayStatus: String? = null) {
logError("Transport failure: $reason")
val retryDelayMs: Long
monitor.withLock {
maybeEmitEvenAppConflictLocked(reason)
finishBenchmarkLocked(true, "transport failure")
val sessionWasReady = sessionReady
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
@@ -402,16 +408,38 @@ internal fun GlassesSessionCore.handleTransportFailure(reason: String?) {
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
clearAllMessagesLocked("transport failure: $reason")
reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS
retryDelayMs = scheduleReconnectLocked(sessionWasReady)
link.disconnect(rightAddress)
link.disconnect(leftAddress)
}
logLine("reconnect attempt in " + retryDelayMs + "ms")
if (!userDisconnectRequested) {
setStateDisplay("retrying", if (reason == null || reason.isEmpty()) "Reconnecting..." else "Reconnecting after $reason")
setStateDisplay("retrying", when {
displayStatus != null -> displayStatus + " Retrying in " + ((retryDelayMs + 999) / 1000) + "s."
reason == null || reason.isEmpty() -> "Reconnecting..."
else -> "Reconnecting after $reason"
})
}
interruptibleSleep.interrupt()
}
/**
* Set [reconnectAfterMs] for the next attempt after a failure and return the delay. Repeated
* failures back off along RECONNECT_BACKOFF_MS; a session that stayed up for
* STABLE_SESSION_MS before failing ([sessionWasReady]) starts the schedule over.
*/
internal fun GlassesSessionCore.scheduleReconnectLocked(sessionWasReady: Boolean): Long {
val now = now()
if (sessionWasReady && now - lastSessionReadyAtMs >= ConnectionOptions.STABLE_SESSION_MS) {
consecutiveReconnects = 0
}
val schedule = ConnectionOptions.RECONNECT_BACKOFF_MS
val delayMs = schedule[minOf(consecutiveReconnects, schedule.size - 1)].toLong()
consecutiveReconnects++
reconnectAfterMs = now + delayMs
return delayMs
}
internal fun GlassesSessionCore.resetSessionStateLocked() {
ringBattery = -1
ringCharging = -1
@@ -424,6 +452,7 @@ internal fun GlassesSessionCore.resetSessionStateLocked() {
ringConnected = false
ringNotificationsReady = false
reconnectAfterMs = 0
consecutiveReconnects = 0
reconnectHalted = false
ringReconnectAfterMs = 0
lastAckAtMs = 0
@@ -192,6 +192,8 @@ class GlassesSessionCore(
internal var lastCfwCleanupAckMagic = 0
internal var reconnectAfterMs = 0L
// Failed connects/sessions since the last stable session; indexes RECONNECT_BACKOFF_MS.
internal var consecutiveReconnects = 0
internal var ringReconnectAfterMs = 0L
internal var lastAckAtMs = 0L
internal var lastIncomingAtMs = 0L
@@ -1611,19 +1613,26 @@ class GlassesSessionCore(
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
monitor.withLock {
if (address == null) {
return
}
if (isConfiguredRingAddress(address)) {
if (address == null) {
return
}
if (isConfiguredRingAddress(address)) {
monitor.withLock {
ringConnected = connected
ringNotificationsReady = false
if (!connected) {
if (connected) {
// A background connect landed: finish setting it up right away.
ringReconnectAfterMs = 0
} else {
ringNotificationsReady = false
ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS
}
logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected")
return
}
logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected")
interruptibleSleep.interrupt()
return
}
var lostSession = false
monitor.withLock {
if (address.equals(rightAddress, ignoreCase = true)) {
rightConnected = connected
} else if (address.equals(leftAddress, ignoreCase = true)) {
@@ -1632,6 +1641,10 @@ class GlassesSessionCore(
return
}
if (!connected) {
// Before the session is ready, this is an attempt failing (or an arm
// dropping mid-setup): the connect sequence notices on its own and
// schedules the retry, so leave the backoff alone.
lostSession = sessionReady
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
@@ -1640,17 +1653,14 @@ class GlassesSessionCore(
audioPacketListener = null
clearAllMessagesLocked("connection lost")
displayedFingerprint = ""
if (!reconnectHalted) {
reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS
if (lostSession && !reconnectHalted) {
scheduleReconnectLocked(true)
}
}
}
interruptibleSleep.interrupt()
if (connected) {
setStateDisplay("connected", "Connected.")
} else if (!reconnectHalted) {
// While parked on a missing bond, keep the "unpaired" display: this
// callback is just the teardown of the arm that did connect.
if (lostSession && !reconnectHalted) {
// While parked on a missing bond, keep the "unpaired" display.
setStateDisplay("connecting", "Connecting to the glasses...")
}
}
@@ -1667,8 +1677,13 @@ class GlassesSessionCore(
}
setStateDisplay("connecting", "Connecting to the glasses...")
try {
connectArm(rightAddress, true)
connectArm(leftAddress, true)
// Dial both arms at once, so the left arm's attempt is already pending (and
// can land) while the right arm is being set up; they share one window.
val deadlineMs = now() + ConnectionOptions.ARM_CONNECT_WINDOW_MS
link.beginConnect(rightAddress, false)
link.beginConnect(leftAddress, false)
connectArm(rightAddress, true, deadlineMs)
connectArm(leftAddress, true, deadlineMs)
if (!sleepDuringConnectSettling(800)) {
return
}
@@ -1733,6 +1748,17 @@ class GlassesSessionCore(
val unpairedArm = firstUnpairedArm()
if (unpairedArm != null) {
handleUnpairedFailure(unpairedArm)
} else if (t is ArmUnreachableException) {
val otherArm = if (t.side == "left") rightAddress else leftAddress
if (link.isConnected(otherArm)) {
handleTransportFailure(
t.side + " arm not found",
"Can't reach the " + t.side + " arm of the glasses. It may be connected to"
+ " another device, or be off or out of range."
)
} else {
handleTransportFailure("glasses not found", "Can't find the glasses. They may be off or out of range.")
}
} else {
handleTransportFailure("connect failed")
}
@@ -1753,9 +1779,10 @@ class GlassesSessionCore(
}
}
internal fun connectArm(address: String, enableRenderNotify: Boolean) {
if (!link.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw IllegalStateException("connect failed: $address")
internal fun connectArm(address: String, enableRenderNotify: Boolean, deadlineMs: Long) {
val timeoutMs = maxOf(deadlineMs - now(), ConnectionOptions.CONNECT_TIMEOUT_MS.toLong())
if (!link.connect(address, timeoutMs.toInt())) {
throw ArmUnreachableException(if (address.equals(leftAddress, ignoreCase = true)) "left" else "right", address)
}
// requestConnectionPriority has no callback in this Android compile target, so there is
// no reliable completion point to keep it in the global GATT operation pipeline. But it's
@@ -1802,6 +1829,16 @@ class GlassesSessionCore(
return
}
try {
// The ring is often out of range or bound to the glasses. Rather than block this
// worker (and every frame and heartbeat behind it) on a timed connect, keep a
// low-duty background connection pending; its callback wakes us to finish setup.
// The periodic recheck re-arms it if the attempt died without a callback.
if (!link.isConnected(ringAddress) && link.beginConnect(ringAddress, true)) {
monitor.withLock {
ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS
}
return
}
connectRing()
} catch (t: Throwable) {
monitor.withLock {
@@ -1810,6 +1847,9 @@ class GlassesSessionCore(
ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS
}
logLine("direct ring connect failed (" + reason + "): " + safeMessage(t))
// Start the next attempt from a fresh connection rather than retrying setup on
// one that just failed it.
link.disconnect(ringAddress)
}
}
@@ -1980,3 +2020,7 @@ class GlassesSessionCore(
}
}
}
/** An arm's connection never came up within the connect window ([side] is "left" or "right"). */
internal class ArmUnreachableException(val side: String, address: String) :
IllegalStateException("connect failed: $side arm $address")
@@ -46,6 +46,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
}
var messageToWrite: OutboundMessage? = null
var messageToPrewrite: OutboundMessage? = null
var heartbeatToWrite: OutboundMessage? = null
val now = now()
maybeFinishNoChangeDesiredFrame()
@@ -196,9 +197,16 @@ internal fun GlassesSessionCore.driveSession(): Long {
&& (hasPendingImageLocked()
|| desiredFingerprintSnapshot() != lastEnqueuedFingerprint)))
noteImageStallLocked(now, windowHasRoom)
if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) {
val heartbeatStep = if (messageToPrewrite == null) heartbeatStepLocked(imageWaiting) else HeartbeatStep.NONE
if (heartbeatStep == HeartbeatStep.HOLD) {
return ConnectionOptions.IDLE_SLEEP_MS.toLong()
}
if (heartbeatStep == HeartbeatStep.SEND) {
logLine("Writing heartbeat")
heartbeatToWrite = createHeartbeatMessage()
lastHeartbeatSentAtMs = now()
return@withLock
}
if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty()
&& CfwMessageWindow.canSend(inFlightMessages, pendingMessages.firstOrNull()!!)) {
@@ -227,6 +235,15 @@ internal fun GlassesSessionCore.driveSession(): Long {
}
}
val heartbeat = heartbeatToWrite
if (heartbeat != null) {
// Like every write, outside `monitor`: waiting for the write to complete while
// holding it would stall notification handling, which needs it. A failed write
// just leaves the heartbeat to its ack timeout (see createHeartbeatMessage).
writeMessage(heartbeat)
return ConnectionOptions.IDLE_SLEEP_MS.toLong()
}
val prewrite = messageToPrewrite
if (prewrite != null) {
if (prewriteMessage(prewrite)) {
@@ -285,7 +302,17 @@ internal fun GlassesSessionCore.canPrewriteCandidate(message: OutboundMessage?):
return message.message.size + 2 > 232
}
internal fun GlassesSessionCore.handleHeartbeat(imageWaiting: Boolean): Boolean {
/** What the heartbeat schedule wants from one driveSession pass. */
internal enum class HeartbeatStep {
/** Nothing due: carry on with other traffic. */
NONE,
/** Send nothing else this pass (a heartbeat is pending, or overdue but blocked). */
HOLD,
/** Write a heartbeat now, then hold. */
SEND,
}
internal fun GlassesSessionCore.heartbeatStepLocked(imageWaiting: Boolean): HeartbeatStep {
val now = now()
val heartbeatEligible = !shutdownRequested && fixedLayoutCreated
val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat")
@@ -300,22 +327,18 @@ internal fun GlassesSessionCore.handleHeartbeat(imageWaiting: Boolean): Boolean
// the heartbeat still fires once we reach the URGENT threshold
// if rendering goes quiet again. Preserves the pending-heartbeat
// inter-lens-sync invariant below (that path is untouched).
return false
return HeartbeatStep.NONE
}
logLine("Writing heartbeat")
val heartbeatMessage = createHeartbeatMessage()
lastHeartbeatSentAtMs = now
writeMessage(heartbeatMessage)
return true
return HeartbeatStep.SEND
} else if (heartbeatUrgent) {
return true
return HeartbeatStep.HOLD
} else if (heartbeatPending) {
// Don't send other message types while a heartbeat is pending because that
// can lead to inter-lens sync issues
return true
return HeartbeatStep.HOLD
}
return false
return HeartbeatStep.NONE
}
internal fun GlassesSessionCore.createHeartbeatMessage(): OutboundMessage {
@@ -1099,7 +1122,7 @@ internal fun GlassesSessionCore.describeWriteBlockerLocked(now: Long, windowHasR
if (hasPendingOrInflightKindLocked("heartbeat")) {
return "heartbeat in flight"
}
// handleHeartbeat is a barrier: while one is due it holds back every
// heartbeatStepLocked is a barrier: while one is due it holds back every
// other write, so a frame queued at the wrong moment waits a heartbeat
// round trip. Reported explicitly because it is otherwise invisible --
// heartbeats belong to no frame.
@@ -13,8 +13,20 @@ import kotlin.contracts.contract
* must call from whatever thread the platform stack delivers on.
*/
interface SessionLink {
/** Connect, waiting up to [timeoutMs]; waits on an attempt already started by [beginConnect]. */
fun connect(address: String, timeoutMs: Int): Boolean
/**
* Start connecting without waiting, so the attempt can land while the caller does other
* work. [background] asks for a low-duty-cycle attempt that stays pending until the
* device shows up. Returns true while a connection exists or is pending, false when the
* link can't do this (callers then fall back to a blocking [connect]).
*/
fun beginConnect(address: String, background: Boolean): Boolean = false
/** Whether a connection to [address] is up (not merely pending). */
fun isConnected(address: String): Boolean = false
/** Best effort; no completion point (see connectArm). */
fun requestHighPriority(address: String)
+3 -1
View File
@@ -18,7 +18,9 @@ kotlin {
minSdk = 24
withJava()
compilerOptions { jvmTarget.set(JvmTarget.JVM_17) }
withHostTestBuilder {}.configure {}
// Production code logs through android.util.Log, which the host-test android.jar
// stubs to throw; a throw on the session worker turns into a spurious reconnect.
withHostTestBuilder {}.configure { isReturnDefaultValues = true }
}
iosArm64()
iosSimulatorArm64()
@@ -62,6 +62,8 @@ private class FakeLink(private val platform: ProtocolPlatform) : SessionLink {
val connected = HashSet<String>()
val disconnected = ArrayList<String>()
@Volatile var closed = false
/** An address whose connection attempts fail, like an arm that isn't advertising. */
@Volatile var unreachable: String? = null
var seq = 0
val writes: List<Write>
@@ -71,10 +73,13 @@ private class FakeLink(private val platform: ProtocolPlatform) : SessionLink {
fun cfwMessages(): List<ByteArray> = writes.filter { it.sid == CfwTransport.SID && it.kind == "image" }.mapNotNull { it.message }
override fun connect(address: String, timeoutMs: Int): Boolean {
connected.add(address)
if (address == unreachable) return false
lock.withLock { connected.add(address) }
return true
}
override fun isConnected(address: String): Boolean = lock.withLock { address in connected }
override fun requestHighPriority(address: String) {}
override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean = true
@@ -115,8 +120,10 @@ private class FakeLink(private val platform: ProtocolPlatform) : SessionLink {
}
override fun disconnect(address: String) {
disconnected.add(address)
connected.remove(address)
lock.withLock {
disconnected.add(address)
connected.remove(address)
}
}
override fun close() {
@@ -196,12 +203,14 @@ private class FakeHost(private val platform: ProtocolPlatform) : SessionHost {
private class FakeListener : FaceclawBleCommunicatorListener {
val phases = ArrayList<String>()
@Volatile var lastStatus = ""
val events = ArrayList<String>()
val finished = ArrayList<String>()
val batteries = ArrayList<String>()
override fun onStateChange(phase: String?, status: String?) {
phases.add(phase ?: "")
lastStatus = status ?: ""
}
override fun onRingEvent(kind: String?, containerName: String?, eventType: Int, eventSource: Int, systemExitReasonCode: Int, frameId: Int, ringTick: Long, ringType: Int, ringAux: Int, ringSpeed: Int) {
@@ -454,4 +463,53 @@ class GlassesSessionCoreTest {
assertContentEquals(first.message, second.message)
s.core.disconnect()
}
@Test
fun reconnectBackoffGrowsAndRestartsAfterAStableSession() {
val core = Session().core
core.monitor.withLock {
assertEquals(2_000L, core.scheduleReconnectLocked(false))
assertEquals(6_000L, core.scheduleReconnectLocked(false))
assertEquals(10_000L, core.scheduleReconnectLocked(false))
assertEquals(10_000L, core.scheduleReconnectLocked(false))
// A session that dies soon after coming up keeps backing off...
core.lastSessionReadyAtMs = core.now() - 1_000
assertEquals(10_000L, core.scheduleReconnectLocked(true))
// ...one that stayed up starts the schedule over.
core.lastSessionReadyAtMs = core.now() - ConnectionOptions.STABLE_SESSION_MS
assertEquals(2_000L, core.scheduleReconnectLocked(true))
assertEquals(6_000L, core.scheduleReconnectLocked(false))
}
}
@Test
fun unreachableArmIsNamedAndRetriedWithBackoff() {
val s = Session()
s.link.unreachable = LEFT
fun retryDelayMs() = s.core.monitor.withLock { s.core.reconnectAfterMs } - s.platform.elapsedRealtimeMs()
fun attempts() = s.core.monitor.withLock { s.core.consecutiveReconnects }
try {
assertTrue(s.core.start())
assertTrue(waitUntil(5_000) { attempts() == 1 }, s.host.logs().takeLast(10).toString())
assertTrue(s.host.hasLog("Transport failure: left arm not found"))
assertTrue(s.listener.lastStatus.startsWith("Can't reach the left arm"), s.listener.lastStatus)
assertTrue(retryDelayMs() in 1_000L..2_000L)
// Both arms are torn down, so the right arm doesn't sit connected on its own.
assertTrue(s.link.disconnected.containsAll(listOf(RIGHT, LEFT)))
s.platform.offsetMs += 2_000
assertTrue(waitUntil(5_000) { attempts() == 2 })
assertTrue(retryDelayMs() in 5_000L..6_000L)
s.platform.offsetMs += 6_000
assertTrue(waitUntil(5_000) { attempts() == 3 })
assertTrue(retryDelayMs() in 9_000L..10_000L)
// Once the arm answers again, the session comes up.
s.link.unreachable = null
s.platform.offsetMs += 10_000
assertTrue(waitUntil(5_000) { s.layoutCreated() }, s.host.logs().takeLast(10).toString())
} finally {
s.core.close()
}
}
}