diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt new file mode 100644 index 00000000..8f7cb4cb --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt @@ -0,0 +1,41 @@ +package com.faceclaw.app + +import android.bluetooth.BluetoothGatt + +/** [SessionLink] over the Android GATT boundary ([FaceclawBleManager]). */ +class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLink { + override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs) + + override fun requestHighPriority(address: String) { + bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + } + + override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean = + bleManager.requestMtu(address, mtu, timeoutMs) + + override fun negotiatedMtu(address: String): Int = bleManager.getNegotiatedMtu(address) + + override fun discoverServices(address: String, timeoutMs: Int): Boolean = + bleManager.discoverServices(address, timeoutMs) + + override fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean = + bleManager.enableNotifications(address, characteristicUuid, enable, timeoutMs) + + override fun writeFrames( + address: String, + characteristicUuid: String, + frames: List, + mode: GattWriteMode, + timeoutMs: Int, + ): Boolean = bleManager.writeFrames(address, characteristicUuid, frames, AndroidProtocolPlatform.writeType(mode), timeoutMs) + + override fun disconnect(address: String) = bleManager.disconnect(address) + + override fun close() = bleManager.close() + + override fun isBonded(address: String): Boolean = bleManager.isBonded(address) + + override fun prepareBenchmarkLink(address: String, mode: Int) = bleManager.prepareBenchmarkLink(address, mode) + + override fun recordDisplayFrameSent() = FaceclawBleManager.recordDisplayFrameSent() +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt new file mode 100644 index 00000000..2b149f09 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSpeechEngines.kt @@ -0,0 +1,147 @@ +package com.faceclaw.app + +import android.util.Log + +import com.k2fsa.sherpa.onnx.FeatureConfig +import com.k2fsa.sherpa.onnx.OfflineModelConfig +import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizer +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizerResult +import com.k2fsa.sherpa.onnx.OfflineStream +import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig + +import java.io.File + +/** + * Android implementations of the shared voice ports: the sherpa-onnx offline recognizer as an + * [OfflineTranscriber], the phone microphone as a [PcmAudioSource], and the JNI LC3 decoder as + * an [AudioPacketDecoder]. The pipelines themselves live in the shared VoiceCaptureSession and + * CaptionSession. + */ +internal object AndroidSpeechEngines { + private const val TAG = "FaceclawVoice" + private const val SAMPLE_RATE = 16000 + private const val FEATURE_DIM = 80 + + /** Sherpa-onnx configuration for the on-device model files in [modelDir]. */ + fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig { + val modelConfig = OfflineModelConfig.builder() + .setNumThreads(1) + if (kind == VoiceModelKind.WHISPER) { + modelConfig + .setWhisper(OfflineWhisperModelConfig.builder() + .setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath) + .setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath) + .setLanguage("en") + .setTask("transcribe") + .build()) + .setTokens(File(modelDir, "base.en-tokens.txt").absolutePath) + } else { + modelConfig + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(File(modelDir, "encoder_model.ort").absolutePath) + .setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath) + .build()) + .setTokens(File(modelDir, "tokens.txt").absolutePath) + } + return OfflineRecognizerConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setModelConfig(modelConfig.build()) + .build() + } + + class SherpaTranscriber(config: OfflineRecognizerConfig) : OfflineTranscriber { + private var recognizer: OfflineRecognizer? = OfflineRecognizer(config) + + override fun recognize(samples: FloatArray, count: Int): String { + val currentRecognizer = recognizer ?: return "" + val segment = if (count == samples.size) samples else samples.copyOf(count) + val offlineStream: OfflineStream = currentRecognizer.createStream() + try { + offlineStream.acceptWaveform(segment, SAMPLE_RATE) + currentRecognizer.decode(offlineStream) + val result: OfflineRecognizerResult? = currentRecognizer.getResult(offlineStream) + val raw: String? = if (result == null) "" else result.text + return raw?.trim() ?: "" + } finally { + offlineStream.release() + } + } + + override fun release() { + recognizer?.release() + recognizer = null + } + } + + /** The JNI LC3 decoder behind the shared packet-decoder port. */ + class Lc3DecoderAdapter : AudioPacketDecoder { + private val decoder = FaceclawLc3Decoder() + + override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int = decoder.decodePacket(packet, pcmOut) + + override val lastAngleDegrees: Int get() = decoder.getLastAngleDegrees() + override val lastSsr: Int get() = decoder.getLastSsr() + override val realPackets: Long get() = decoder.getRealPackets() + override val duplicatePackets: Long get() = decoder.getDuplicatePackets() + override val missingPackets: Long get() = decoder.getMissingPackets() + override val decodeErrors: Long get() = decoder.getDecodeErrors() + + override fun close() = decoder.close() + } + + /** + * Open the phone's own microphone at the pipeline's native format (16 kHz + * mono PCM16), or null when it cannot start — the permission is missing + * (SecurityException) or the device refuses the configuration. + */ + fun openPhoneMic(chunkSamples: Int): PcmAudioSource? { + var record: android.media.AudioRecord? = null + try { + val minBytes = android.media.AudioRecord.getMinBufferSize( + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT) + val bufferBytes = Math.max(minBytes, chunkSamples * 2 * 4) + record = android.media.AudioRecord( + android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION, + SAMPLE_RATE, + android.media.AudioFormat.CHANNEL_IN_MONO, + android.media.AudioFormat.ENCODING_PCM_16BIT, + bufferBytes) + if (record.state != android.media.AudioRecord.STATE_INITIALIZED) { + record.release() + return null + } + record.startRecording() + if (record.recordingState != android.media.AudioRecord.RECORDSTATE_RECORDING) { + record.release() + return null + } + return AudioRecordSource(record) + } catch (t: Throwable) { + Log.w(TAG, "phone mic open failed", t) + if (record != null) { + record.release() + } + return null + } + } + + private class AudioRecordSource(private val record: android.media.AudioRecord) : PcmAudioSource { + override fun read(pcm: ShortArray): Int = record.read(pcm, 0, pcm.size) + + override fun close() { + try { + record.stop() + } catch (ignored: Throwable) { + // Already stopped or never recording; release below either way. + } + record.release() + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidStockLink.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidStockLink.kt new file mode 100644 index 00000000..3e7d264e --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidStockLink.kt @@ -0,0 +1,64 @@ +package com.faceclaw.app + +import android.bluetooth.BluetoothDevice +import android.bluetooth.BluetoothGatt +import android.content.Context + +/** + * Android [StockLink]: a 1:1 wrapper over FaceclawBleManager for the stock-firmware flows + * (device-info probe, flash prompt, OTA flasher). Owns its own GATT manager, exactly as each + * flow did before the shared cores existed. + */ +class AndroidStockLink(context: Context) : StockLink, FaceclawBleListener { + private val bleManager = FaceclawBleManager(context.applicationContext) + + @Volatile + private var listener: StockLinkListener? = null + + init { + bleManager.setListener(this) + } + + override fun setListener(listener: StockLinkListener?) { + this.listener = listener + } + + override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs) + + override fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int) { + bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) + bleManager.requestMtu(address, desiredMtu, timeoutMs) + } + + override fun discoverServices(address: String, timeoutMs: Int): Boolean = bleManager.discoverServices(address, timeoutMs) + + override fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean = + bleManager.enableNotifications(address, characteristicUuid, true, timeoutMs) + + override fun writeFrames(address: String, characteristicUuid: String, frames: List, mode: GattWriteMode, timeoutMs: Int): Boolean = + bleManager.writeFrames(address, characteristicUuid, frames, AndroidProtocolPlatform.writeType(mode), timeoutMs) + + override fun isConnected(address: String): Boolean = bleManager.isConnected(address) + + override fun bondState(address: String): BondState = + when (bleManager.getBondState(address)) { + BluetoothDevice.BOND_NONE -> BondState.NONE + BluetoothDevice.BOND_BONDING -> BondState.BONDING + BluetoothDevice.BOND_BONDED -> BondState.BONDED + else -> BondState.UNKNOWN + } + + override fun disconnect(address: String) = bleManager.disconnect(address) + + override fun close() = bleManager.close() + + override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { + if (address == null || characteristicUuid == null || data == null) return + listener?.onNotification(address, characteristicUuid, data) + } + + override fun onConnectionStateChange(address: String?, connected: Boolean) { + if (address == null) return + listener?.onConnectionStateChange(address, connected) + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt index 65ea4dd2..3bddef2b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarmService.kt @@ -22,9 +22,6 @@ import android.os.PowerManager import android.os.VibrationEffect import android.os.Vibrator -import java.util.ArrayList -import java.util.Collections -import java.util.LinkedHashMap /** * The ringing foreground service. One instance carries every item currently @@ -37,22 +34,14 @@ import java.util.LinkedHashMap * to total silence) Do Not Disturb do not silence it. */ class FaceclawAlarmService : Service() { + /** The shared item model; kept as a nested subclass so the lock-screen activity's references hold. */ internal class Ringing( - @JvmField val id: Long, + id: Long, title: String?, text: String?, kind: String?, snoozeMinutes: Int - ) { - @JvmField val title: String = if (title == null || title.trim().isEmpty()) "Alarm" else title - @JvmField val text: String = text ?: "" - @JvmField val kind: String = kind ?: FaceclawAlarms.KIND_ALARM - @JvmField val snoozeMinutes: Int = Math.max(1, snoozeMinutes) - @JvmField val ringAtMs: Long = System.currentTimeMillis() - @JvmField var deliveredAtMs: Long = 0 - @JvmField var escalated: Boolean = false - @JvmField var silenced: Boolean = false - } + ) : RingingItem(id, title, text, kind, snoozeMinutes, System.currentTimeMillis()) companion object { const val CHANNEL_ID = "faceclaw-alarms" @@ -72,17 +61,11 @@ class FaceclawAlarmService : Service() { /** Alarm-stream level (fraction of max) used when the wearer has it muted. */ internal const val MUTED_VOLUME_FRACTION = 0.6f - /** Ringing items, oldest first; static so the activity can read them. */ - private val ringing: MutableMap = Collections.synchronizedMap(LinkedHashMap()) /** - * Signals that arrived before the ring intent was processed (the JS - * engine fires, launches its window and reports delivery on the same - * main thread the service's onStartCommand queues behind), keyed by id - * with their arrival time. Consumed by startRinging. + * The escalation policy is process-wide (the activity reads it, the static entry points + * consult it before the service exists); the live service attaches its timers/effects. */ - private val deliveredEarly: MutableMap = Collections.synchronizedMap(LinkedHashMap()) - private val stoppedEarly: MutableMap = Collections.synchronizedMap(LinkedHashMap()) - private const val EARLY_SIGNAL_MAX_AGE_MS = 15_000L + private val policy = AlarmRingingPolicy(FaceclawAlarms.glassesGate) // ------------------------------------------------------------------ // Static entry points @@ -102,8 +85,7 @@ class FaceclawAlarmService : Service() { @JvmStatic fun delivered(context: Context, id: Long) { - if (!ringing.containsKey(id)) { - deliveredEarly[id] = System.currentTimeMillis() + if (!policy.deliveredOrDefer(id)) { return } val appContext = context.applicationContext @@ -114,8 +96,7 @@ class FaceclawAlarmService : Service() { @JvmStatic fun stopItem(context: Context, id: Long) { - if (!ringing.containsKey(id)) { - stoppedEarly[id] = System.currentTimeMillis() + if (!policy.stopOrDefer(id)) { return } val appContext = context.applicationContext @@ -125,21 +106,13 @@ class FaceclawAlarmService : Service() { } @JvmStatic - fun isRinging(id: Long): Boolean { - return ringing.containsKey(id) - } + fun isRinging(id: Long): Boolean = policy.isRinging(id) @JvmStatic - fun isAnythingRinging(): Boolean { - return !ringing.isEmpty() - } + fun isAnythingRinging(): Boolean = policy.isAnythingRinging() @JvmStatic - internal fun snapshot(): List { - synchronized(ringing) { - return ArrayList(ringing.values) - } - } + internal fun snapshot(): List = policy.snapshot().map { it as Ringing } @JvmStatic internal fun actionIntent(appContext: Context, action: String, id: Long): PendingIntent { @@ -157,16 +130,6 @@ class FaceclawAlarmService : Service() { return PendingIntent.getService(appContext, code, intent, flags) } - /** Consume an early signal for the id; stale entries are dropped. */ - private fun takeEarlySignal(signals: MutableMap, id: Long): Boolean { - val at = signals.remove(id) - val now = System.currentTimeMillis() - synchronized(signals) { - signals.values.removeIf { stamp -> now - stamp > EARLY_SIGNAL_MAX_AGE_MS } - } - return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS - } - private fun notificationId(id: Long): Int { return 0x41000000 or (FaceclawAlarms.requestCode(id) and 0x00ffffff) } @@ -188,6 +151,11 @@ class FaceclawAlarmService : Service() { return null } + override fun onCreate() { + super.onCreate() + policy.attach(timers, effects) + } + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { val action = intent?.action val id = if (intent == null) 0L else intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0) @@ -206,13 +174,13 @@ class FaceclawAlarmService : Service() { intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10) )) ACTION_DELIVERED -> - markDelivered(id) + policy.markDelivered(id) ACTION_DISMISS -> - dismiss(id) + policy.dismiss(id) ACTION_SNOOZE -> - snooze(id) + policy.snooze(id) ACTION_STOP_ITEM -> - stopItemQuietly(id, "acknowledged") + policy.stopQuietly(id, "acknowledged") else -> {} } finishIfIdle() @@ -220,6 +188,7 @@ class FaceclawAlarmService : Service() { } override fun onDestroy() { + policy.detach() handler.removeCallbacksAndMessages(null) stopSound() releaseWakeLock() @@ -234,120 +203,69 @@ class FaceclawAlarmService : Service() { return } ensureChannel() - val existing = ringing[item.id] - if (existing != null) { - // The engine's JS timeout and the AlarmManager alarm both fired: one ring. - showNotification(existing) - return - } - if (takeEarlySignal(stoppedEarly, item.id)) { - // Acknowledged or cancelled before this intent was processed. - FaceclawAlarms.log(this, "ring " + item.id + " already acknowledged") - return - } - ringing[item.id] = item - FaceclawAlarms.log(this, "ring " + item.kind + " " + item.id + " (" + FaceclawAlarms.glassesStatusDescription() + ")") - acquireWakeLock() - showNotification(item) - if (!FaceclawAlarms.glassesCanCarryAlarm()) { - escalate(item, FaceclawAlarms.glassesStatusDescription()) - return - } - if (takeEarlySignal(deliveredEarly, item.id)) { - markDelivered(item.id) - return - } - // The glasses could carry it: give the JS side a moment to confirm - // they are actually showing it, then hold for the acknowledgement. - handler.postDelayed({ - val current = ringing[item.id] - if (current != null && !current.escalated && current.deliveredAtMs == 0L) { - escalate(current, "not delivered to the glasses") + policy.ring(item) + } + + /** Escalation timers on the service's main-thread Handler. */ + private val timers = object : AlarmScheduler { + private val pending = HashMap() + + override fun postDelayed(token: Any, delayMs: Long, action: () -> Unit) { + pending.remove(token)?.let { handler.removeCallbacks(it) } + val runnable = Runnable { + pending.remove(token) + action() } - }, DELIVERY_WAIT_MS) + pending[token] = runnable + handler.postDelayed(runnable, delayMs) + } + + override fun cancel(token: Any) { + pending.remove(token)?.let { handler.removeCallbacks(it) } + } + + override fun cancelAll() { + pending.clear() + handler.removeCallbacksAndMessages(null) + } } - private fun markDelivered(id: Long) { - val item = ringing[id] - if (item == null || item.deliveredAtMs != 0L) { - return - } - item.deliveredAtMs = System.currentTimeMillis() - FaceclawAlarms.log(this, "delivered $id to glasses") - handler.postDelayed({ - val current = ringing[id] - if (current != null && !current.escalated) { - escalate(current, "not acknowledged on the glasses") + /** The phone-side effects of the shared policy: notifications, sound, schedule, journal. */ + private val effects = object : RingingSink { + override fun showNotification(item: RingingItem) = this@FaceclawAlarmService.showNotification(item as Ringing) + + override fun itemRemoved(item: RingingItem) { + val manager = getSystemService(NotificationManager::class.java) + manager?.cancel(notificationId(item.id)) + if (item.id == foregroundId) { + promoteAnotherToForeground() } - }, ACK_WAIT_MS) - } - - private fun escalate(item: Ringing, reason: String) { - if (item.escalated) { - return } - item.escalated = true - FaceclawAlarms.log(this, "phone sound for " + item.id + ": " + reason) - showNotification(item) - startSound() - handler.postDelayed({ - val current = ringing[item.id] - if (current != null && current.escalated && !current.silenced) { - current.silenced = true - FaceclawAlarms.log(this, "auto-silenced " + current.id) - showNotification(current) - syncSound() - } - }, AUTO_SILENCE_MS) - } - private fun dismiss(id: Long) { - val item = ringing[id] ?: return - FaceclawAlarms.recordPhoneAction(this, id, "dismiss", 0) - removeItem(item) - // Also drop the schedule entry (a one-off is over; the engine re-arms repeats). - FaceclawAlarms.cancel(this, id) - } + override fun ringingStarted() = acquireWakeLock() - private fun snooze(id: Long) { - val item = ringing[id] ?: return - val minutes = item.snoozeMinutes - FaceclawAlarms.recordPhoneAction(this, id, "snooze", minutes) - removeItem(item) - // Re-arm on the phone directly so the snooze holds even with the JS - // side gone; the engine replays the journal and lands on the same id. - FaceclawAlarms.schedule( - this, - id, - System.currentTimeMillis() + minutes * 60_000L, - item.title, - item.text, - item.kind, - minutes - ) - } + override fun startSound() = this@FaceclawAlarmService.startSound() - private fun stopItemQuietly(id: Long, reason: String) { - val item = ringing[id] ?: return - FaceclawAlarms.log(this, "$reason $id") - removeItem(item) - } + override fun stopSound() = this@FaceclawAlarmService.stopSound() - private fun removeItem(item: Ringing) { - ringing.remove(item.id) - val manager = getSystemService(NotificationManager::class.java) - manager?.cancel(notificationId(item.id)) - syncSound() - if (item.id == foregroundId) { - promoteAnotherToForeground() + override fun reschedule(item: RingingItem, atMs: Long, minutes: Int) { + // The engine replays the journal and lands on the same id. + FaceclawAlarms.schedule(this@FaceclawAlarmService, item.id, atMs, item.title, item.text, item.kind, minutes) } + + override fun cancelSchedule(id: Long) = FaceclawAlarms.cancel(this@FaceclawAlarmService, id) + + override fun recordPhoneAction(id: Long, action: String, minutes: Int) = + FaceclawAlarms.recordPhoneAction(this@FaceclawAlarmService, id, action, minutes) + + override fun log(line: String) = FaceclawAlarms.log(this@FaceclawAlarmService, line) } private fun finishIfIdle() { - if (!ringing.isEmpty()) { + if (policy.isAnythingRinging()) { return } - handler.removeCallbacksAndMessages(null) + policy.cancelTimers() stopSound() releaseWakeLock() stopForeground(STOP_FOREGROUND_REMOVE) @@ -359,7 +277,7 @@ class FaceclawAlarmService : Service() { private fun showNotification(item: Ringing) { val notification = buildNotification(item) - if (foregroundId == 0L || foregroundId == item.id || !ringing.containsKey(foregroundId)) { + if (foregroundId == 0L || foregroundId == item.id || !policy.isRinging(foregroundId)) { foregroundId = item.id startForegroundCompat(notificationId(item.id), notification) } else { @@ -456,21 +374,6 @@ class FaceclawAlarmService : Service() { // ------------------------------------------------------------------ // Sound and vibration (only while some item is escalated and not yet silenced) - private fun syncSound() { - var wanted = false - for (item in snapshot()) { - if (item.escalated && !item.silenced) { - wanted = true - break - } - } - if (wanted) { - startSound() - } else { - stopSound() - } - } - private fun startSound() { if (player != null) { return diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt index 43ffcfc1..fa6ad2ed 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAlarms.kt @@ -47,32 +47,19 @@ class FaceclawAlarms private constructor() { internal const val EXTRA_KIND = "kind" internal const val EXTRA_SNOOZE_MINUTES = "snoozeMinutes" - const val KIND_TIMER = "timer" - const val KIND_ALARM = "alarm" + const val KIND_TIMER = AlarmSchedule.KIND_TIMER + const val KIND_ALARM = AlarmSchedule.KIND_ALARM private const val PREFS_NAME = "faceclaw-alarms" - private const val KEY_SCHEDULED = "scheduled" - private const val KEY_JOURNAL = "journal" - private const val KEY_LOG = "log" - private const val LOG_LINES = 60 - /** - * A schedule entry found already past due (after a reboot, or when the - * process was asleep) rings if it is at most this late; older ones are - * dropped as missed. Matches the JS engine's grace window. - */ - internal const val LATE_GRACE_MS = 5 * 60_000L - /** A glasses status report older than this is not trusted (the JS side is probably gone). */ - internal const val GLASSES_STATUS_MAX_AGE_MS = 3 * 60_000L + /** Grace window for past-due entries found on replay (shared policy). */ + internal const val LATE_GRACE_MS = AlarmSchedule.LATE_GRACE_MS + /** A glasses status report older than this is not trusted (shared policy). */ + internal const val GLASSES_STATUS_MAX_AGE_MS = GlassesStatusGate.MAX_AGE_MS - @Volatile - private var glassesConnected = false - @Volatile - private var glassesWorn = false - @Volatile - private var glassesCharging = false - @Volatile - private var glassesStatusAtMs = 0L + /** The last glasses status pushed by the JS side; read by the ringing service. */ + @JvmField + internal val glassesGate = GlassesStatusGate() @Volatile private var listener: FaceclawAlarmListener? = null @@ -95,66 +82,30 @@ class FaceclawAlarms private constructor() { return base.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) } - // The Java statics were `synchronized`, i.e. locked on the class object; - // the explicit blocks below keep that single monitor. - private fun readArray(context: Context, key: String): JSONArray { - synchronized(FaceclawAlarms::class.java) { - val raw = prefs(context).getString(key, "[]") - return try { - JSONArray(raw) - } catch (error: JSONException) { - JSONArray() - } - } - } + @Volatile + private var scheduleInstance: AlarmSchedule? = null - private fun writeArray(context: Context, key: String, array: JSONArray) { + /** The shared schedule/journal/log core over the device-protected prefs (one per process). */ + @JvmStatic + internal fun scheduleCore(context: Context): AlarmSchedule { + scheduleInstance?.let { return it } synchronized(FaceclawAlarms::class.java) { - prefs(context).edit().putString(key, array.toString()).apply() + scheduleInstance?.let { return it } + val prefs = prefs(context) + val created = AlarmSchedule(object : AlarmStore { + override fun read(key: String): String? = prefs.getString(key, null) + + override fun write(key: String, value: String?) { + prefs.edit().putString(key, value).apply() + } + }) + scheduleInstance = created + return created } } @JvmStatic - internal fun findScheduled(context: Context, id: Long): JSONObject? { - val scheduled = readArray(context, KEY_SCHEDULED) - for (index in 0 until scheduled.length()) { - val entry = scheduled.optJSONObject(index) - if (entry != null && entry.optLong("id") == id) { - return entry - } - } - return null - } - - private fun putScheduled(context: Context, entry: JSONObject) { - synchronized(FaceclawAlarms::class.java) { - val scheduled = readArray(context, KEY_SCHEDULED) - val next = JSONArray() - val id = entry.optLong("id") - for (index in 0 until scheduled.length()) { - val existing = scheduled.optJSONObject(index) - if (existing != null && existing.optLong("id") != id) { - next.put(existing) - } - } - next.put(entry) - writeArray(context, KEY_SCHEDULED, next) - } - } - - private fun removeScheduled(context: Context, id: Long) { - synchronized(FaceclawAlarms::class.java) { - val scheduled = readArray(context, KEY_SCHEDULED) - val next = JSONArray() - for (index in 0 until scheduled.length()) { - val existing = scheduled.optJSONObject(index) - if (existing != null && existing.optLong("id") != id) { - next.put(existing) - } - } - writeArray(context, KEY_SCHEDULED, next) - } - } + internal fun findScheduled(context: Context, id: Long): AlarmEntry? = scheduleCore(context).find(id) // ------------------------------------------------------------------ // Scheduling @@ -177,26 +128,16 @@ class FaceclawAlarms private constructor() { snoozeMinutes: Int ) { val appContext = context.applicationContext - val entry = JSONObject() - try { - entry.put("id", id) - entry.put("at", triggerAtMs) - entry.put("title", title ?: "") - entry.put("text", text ?: "") - entry.put("kind", kind ?: KIND_ALARM) - entry.put("snoozeMinutes", snoozeMinutes) - } catch (ignored: JSONException) { - return - } - putScheduled(appContext, entry) + val entry = AlarmEntry(id, triggerAtMs, title, text, kind ?: KIND_ALARM, snoozeMinutes) + scheduleCore(appContext).put(entry) armAlarm(appContext, entry) } - private fun armAlarm(appContext: Context, entry: JSONObject) { + private fun armAlarm(appContext: Context, entry: AlarmEntry) { val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? ?: return - val id = entry.optLong("id") - val triggerAt = Math.max(System.currentTimeMillis(), entry.optLong("at")) + val id = entry.id + val triggerAt = Math.max(System.currentTimeMillis(), entry.at) val fire = expiryPendingIntent(appContext, entry) try { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { @@ -219,15 +160,9 @@ class FaceclawAlarms private constructor() { val appContext = context.applicationContext val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? if (manager != null) { - val entry = JSONObject() - try { - entry.put("id", id) - } catch (ignored: JSONException) { - // unreachable for a numeric put - } - manager.cancel(expiryPendingIntent(appContext, entry)) + manager.cancel(expiryPendingIntent(appContext, AlarmEntry(id, 0L, null, null, null, 10))) } - removeScheduled(appContext, id) + scheduleCore(appContext).remove(id) FaceclawAlarmService.stopItem(appContext, id) } @@ -239,25 +174,7 @@ class FaceclawAlarms private constructor() { @JvmStatic fun rescheduleAll(context: Context, reason: String?) { val appContext = context.applicationContext - val scheduled = readArray(appContext, KEY_SCHEDULED) - val now = System.currentTimeMillis() - var armed = 0 - for (index in 0 until scheduled.length()) { - val entry = scheduled.optJSONObject(index) ?: continue - val at = entry.optLong("at") - if (at <= now) { - if (now - at <= LATE_GRACE_MS) { - ringEntry(appContext, entry) - } else { - log(appContext, "missed " + entry.optString("kind") + " " + entry.optLong("id") + " (" + reason + ")") - removeScheduled(appContext, entry.optLong("id")) - } - continue - } - armAlarm(appContext, entry) - armed++ - } - log(appContext, "rescheduled $armed ($reason)") + scheduleCore(appContext).replay(reason, { ringEntry(appContext, it) }, { armAlarm(appContext, it) }) } /** @@ -271,24 +188,13 @@ class FaceclawAlarms private constructor() { return false } val appContext = context.applicationContext - val scheduled = readArray(appContext, KEY_SCHEDULED) - val now = System.currentTimeMillis() - var soonest = Long.MAX_VALUE - for (index in 0 until scheduled.length()) { - val entry = scheduled.optJSONObject(index) - if (entry != null && entry.optLong("at") > now) { - soonest = Math.min(soonest, entry.optLong("at")) - } - } - if (soonest == Long.MAX_VALUE) { + val core = scheduleCore(appContext) + if (core.soonestPendingAt() == null) { return false } val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? ?: return false - val next = manager.nextAlarmClock - // Another app's alarm may legitimately be sooner; only a later (or - // absent) system alarm proves ours is missing. - return next == null || next.triggerTime > soonest + 1000 + return core.looksStale(manager.nextAlarmClock?.triggerTime) } // ------------------------------------------------------------------ @@ -296,15 +202,8 @@ class FaceclawAlarms private constructor() { /** An item came due: hand it to the ringing service (idempotent per id). */ @JvmStatic - internal fun ringEntry(appContext: Context, entry: JSONObject) { - FaceclawAlarmService.ring( - appContext, - entry.optLong("id"), - entry.optString("title"), - entry.optString("text"), - entry.optString("kind", KIND_ALARM), - entry.optInt("snoozeMinutes", 10) - ) + internal fun ringEntry(appContext: Context, entry: AlarmEntry) { + FaceclawAlarmService.ring(appContext, entry.id, entry.title, entry.text, entry.kind, entry.snoozeMinutes) } /** The JS engine's own expiry path; the service deduplicates against the alarm's. */ @@ -313,13 +212,7 @@ class FaceclawAlarms private constructor() { val appContext = context.applicationContext val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager? if (manager != null) { - val probe = JSONObject() - try { - probe.put("id", id) - } catch (ignored: JSONException) { - // unreachable for a numeric put - } - manager.cancel(expiryPendingIntent(appContext, probe)) + manager.cancel(expiryPendingIntent(appContext, AlarmEntry(id, 0L, null, null, null, 10))) } FaceclawAlarmService.ring(appContext, id, title, text, kind, snoozeMinutes) } @@ -334,7 +227,7 @@ class FaceclawAlarms private constructor() { @JvmStatic fun acknowledge(context: Context, id: Long) { val appContext = context.applicationContext - removeScheduled(appContext, id) + scheduleCore(appContext).remove(id) FaceclawAlarmService.stopItem(appContext, id) } @@ -343,42 +236,15 @@ class FaceclawAlarms private constructor() { @JvmStatic fun setGlassesStatus(connected: Boolean, worn: Boolean, charging: Boolean) { - glassesConnected = connected - glassesWorn = worn - glassesCharging = charging - glassesStatusAtMs = System.currentTimeMillis() + glassesGate.set(connected, worn, charging) } /** True only when a fresh report says the glasses are connected, on a head, and not charging. */ @JvmStatic - internal fun glassesCanCarryAlarm(): Boolean { - val age = System.currentTimeMillis() - glassesStatusAtMs - if (glassesStatusAtMs == 0L || age > GLASSES_STATUS_MAX_AGE_MS) { - return false - } - return glassesConnected && glassesWorn && !glassesCharging - } + internal fun glassesCanCarryAlarm(): Boolean = glassesGate.canCarryAlarm() @JvmStatic - internal fun glassesStatusDescription(): String { - if (glassesStatusAtMs == 0L) { - return "no glasses status" - } - val age = System.currentTimeMillis() - glassesStatusAtMs - if (age > GLASSES_STATUS_MAX_AGE_MS) { - return "glasses status stale" - } - if (!glassesConnected) { - return "glasses not connected" - } - if (glassesCharging) { - return "glasses charging" - } - if (!glassesWorn) { - return "glasses not worn" - } - return "glasses worn" - } + internal fun glassesStatusDescription(): String = glassesGate.description() // ------------------------------------------------------------------ // Phone actions -> JS @@ -396,21 +262,7 @@ class FaceclawAlarms private constructor() { @JvmStatic internal fun recordPhoneAction(context: Context, id: Long, action: String, minutes: Int) { val appContext = context.applicationContext - val event = JSONObject() - try { - event.put("id", id) - event.put("action", action) - event.put("minutes", minutes) - event.put("at", System.currentTimeMillis()) - } catch (ignored: JSONException) { - return - } - synchronized(FaceclawAlarms::class.java) { - val journal = readArray(appContext, KEY_JOURNAL) - journal.put(event) - writeArray(appContext, KEY_JOURNAL, journal) - } - log(appContext, "phone " + action + " " + id + (if (minutes > 0) " ($minutes min)" else "")) + scheduleCore(appContext).appendJournal(id, action, minutes) val current = listener if (current != null) { mainHandler.post { @@ -425,14 +277,7 @@ class FaceclawAlarms private constructor() { /** Hand the journal to the JS side (as a JSON array) and clear it. */ @JvmStatic - fun drainJournal(context: Context): String { - synchronized(FaceclawAlarms::class.java) { - val appContext = context.applicationContext - val journal = readArray(appContext, KEY_JOURNAL) - writeArray(appContext, KEY_JOURNAL, JSONArray()) - return journal.toString() - } - } + fun drainJournal(context: Context): String = scheduleCore(context.applicationContext).drainJournal() // ------------------------------------------------------------------ // Reliability self-check @@ -577,45 +422,25 @@ class FaceclawAlarms private constructor() { @JvmStatic internal fun log(context: Context, line: String) { - synchronized(FaceclawAlarms::class.java) { - val appContext = context.applicationContext - val log = readArray(appContext, KEY_LOG) - val next = JSONArray() - val start = Math.max(0, log.length() - (LOG_LINES - 1)) - for (index in start until log.length()) { - next.put(log.optString(index)) - } - next.put(System.currentTimeMillis().toString() + " " + line) - writeArray(appContext, KEY_LOG, next) - } + scheduleCore(context.applicationContext).log(line) } /** The ring / miss / action log, newest last, one line per entry. */ @JvmStatic - fun readLog(context: Context): String { - val log = readArray(context.applicationContext, KEY_LOG) - val out = StringBuilder() - for (index in 0 until log.length()) { - if (index > 0) { - out.append('\n') - } - out.append(log.optString(index)) - } - return out.toString() - } + fun readLog(context: Context): String = scheduleCore(context.applicationContext).readLog() // ------------------------------------------------------------------ // Intents - private fun expiryPendingIntent(appContext: Context, entry: JSONObject): PendingIntent { + private fun expiryPendingIntent(appContext: Context, entry: AlarmEntry): PendingIntent { val intent = Intent(appContext, FaceclawAlarmReceiver::class.java) intent.action = ACTION_EXPIRE - val id = entry.optLong("id") + val id = entry.id intent.putExtra(EXTRA_ID, id) - intent.putExtra(EXTRA_TITLE, entry.optString("title")) - intent.putExtra(EXTRA_TEXT, entry.optString("text")) - intent.putExtra(EXTRA_KIND, entry.optString("kind", KIND_ALARM)) - intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.optInt("snoozeMinutes", 10)) + intent.putExtra(EXTRA_TITLE, entry.title) + intent.putExtra(EXTRA_TEXT, entry.text) + intent.putExtra(EXTRA_KIND, entry.kind) + intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.snoozeMinutes) var flags = PendingIntent.FLAG_UPDATE_CURRENT if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { flags = flags or PendingIntent.FLAG_IMMUTABLE @@ -635,9 +460,7 @@ class FaceclawAlarms private constructor() { } @JvmStatic - internal fun requestCode(id: Long): Int { - return (id xor (id ushr 32)).toInt() and 0x7fffffff - } + internal fun requestCode(id: Long): Int = AlarmSchedule.requestCode(id) @JvmStatic internal fun startServiceCompat(appContext: Context, intent: Intent) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt index 1ffe9d16..58e016cd 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt @@ -9,8 +9,6 @@ import android.util.Log import java.io.File import java.io.FileInputStream import java.io.IOException -import java.nio.ByteBuffer -import java.nio.ByteOrder /** * Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A @@ -28,69 +26,25 @@ class FaceclawAudioTranscoder( private const val TAG = "FaceclawTranscode" private const val CODEC_TIMEOUT_US: Long = 10_000 - /** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */ + /** Shared RIFF walk over the file's first bytes: locates fmt and data chunks, 16-bit PCM only. */ @JvmStatic @Throws(IOException::class) - private fun readWavHeader(path: String?): WavInfo { - FileInputStream(path).use { `in` -> - val head = ByteArray(12) - if (`in`.read(head) != 12 || head[0] != 'R'.code.toByte() || head[1] != 'I'.code.toByte() || head[8] != 'W'.code.toByte()) { - throw IOException("not a WAV file") + private fun readWavHeader(path: String?): WavPcmReader.WavInfo { + val file = File(path) + val fileLength = file.length() + FileInputStream(file).use { `in` -> + val prefix = ByteArray(Math.min(fileLength, WavPcmReader.HEADER_PREFIX_BYTES.toLong()).toInt()) + var filled = 0 + while (filled < prefix.size) { + val read = `in`.read(prefix, filled, prefix.size - filled) + if (read <= 0) break + filled += read } - val info = WavInfo() - var offset: Long = 12 - val chunkHead = ByteArray(8) - while (`in`.read(chunkHead) == 8) { - val b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN) - val id = b.getInt() - val size = b.getInt().toLong() and 0xffffffffL - offset += 8 - if (id == 0x20746d66) { // "fmt " - val fmt = ByteArray(Math.min(size, 16L).toInt()) - if (`in`.read(fmt) != fmt.size) { - throw IOException("truncated fmt chunk") - } - val f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN) - val audioFormat = f.getShort().toInt() - info.channels = f.getShort().toInt() - info.sampleRate = f.getInt() - f.getInt() - f.getShort() - val bits = f.getShort().toInt() - if (audioFormat != 1 || bits != 16) { - throw IOException("only 16-bit PCM WAV is supported") - } - var skip = size - fmt.size - while (skip > 0) { - val step = `in`.skip(skip) - if (step <= 0) { - break - } - skip -= step - } - offset += size - } else if (id == 0x61746164) { // "data" - info.dataOffset = offset - // A zero size means the header was never patched (crash - // mid-recording); recover the length from the file size. - info.dataBytes = if (size > 0) size else File(path).length() - offset - break - } else { - var skip = size - while (skip > 0) { - val step = `in`.skip(skip) - if (step <= 0) { - break - } - skip -= step - } - offset += size - } + try { + return WavPcmReader.parseHeader(if (filled == prefix.size) prefix else prefix.copyOf(filled), fileLength) + } catch (e: IllegalArgumentException) { + throw IOException(e.message ?: "not a WAV file") } - if (info.sampleRate == 0 || info.dataOffset == 0L) { - throw IOException("missing fmt or data chunk") - } - return info } } } @@ -226,11 +180,4 @@ class FaceclawAudioTranscoder( } } } - - private class WavInfo { - var sampleRate = 0 - var channels = 0 - var dataOffset: Long = 0 - var dataBytes: Long = 0 - } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index 97e8d343..d07ffde4 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -1,7 +1,6 @@ package com.faceclaw.app import android.annotation.SuppressLint -import android.bluetooth.BluetoothGatt import android.app.KeyguardManager import android.content.BroadcastReceiver import android.content.Context @@ -10,41 +9,21 @@ import android.content.IntentFilter import android.os.Handler import android.os.Looper import android.os.PowerManager -import android.os.SystemClock import android.util.Log -import java.io.PrintWriter -import java.io.StringWriter -import java.util.ArrayDeque -import java.util.ArrayList -import java.util.Arrays -import java.util.Collections -import java.util.HashMap -import java.util.Locale -import java.util.Random - +/** + * Android facade over the shared [GlassesSessionCore]: the JVM API TypeScript + * and the other Android classes call by name, plus the Android-only pieces: + * the GATT link ([AndroidSessionLink] over [FaceclawBleManager]), main-thread + * marshalling, the worker thread, the screen wake lock, keyguard/screen + * broadcast state, the Even-app conflict check, and the Bitmap/ByteBuffer/ + * Context entry points for previews, screenshots and recordings. + */ @SuppressLint("MissingPermission") -class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddress: String?, ringAddress: String?) : FaceclawBleListener, Runnable { +class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddress: String?, ringAddress: String?) { companion object { - private const val TAG = "FaceclawComm" - - // Local metadata for custom-command bookkeeping, not an EvenHub container. - // Submitted frames supply pixel geometry; the stock layout only captures input. - private val DASHBOARD_TILE: BleProtocol.ImageTileOptions = - BleProtocol.ImageTileOptions("img00", 10, 0, 0, 576, 288) - + private const val TAG = GlassesSessionCore.TAG private const val G2_SCREEN_WAKE_LOCK_TAG = "Faceclaw:G2Screen" - private const val FACECLAW_WAKE_LEASE_RENEW_MS = 45_000L - private const val FACECLAW_WAKE_CONTROL_WAIT_MS = 1_500L - private const val CFW_CLEANUP_WAIT_MS = 4_000L - private const val COMPASS_REPORT_INTERVAL_MS = 100 - private const val COMPASS_MIN_CHANGE_DEGREES = 0 - - // A timed-out benchmark message aborts the run, but its already-in-flight - // peers still time out one by one; keep the window comfortably below - // MAX_CONSECUTIVE_ACK_TIMEOUTS so a dead run can't escalate into a - // transport-failure reconnect all by itself. - private const val BENCHMARK_MAX_WINDOW = 6 // The most recently started communicator; lets app worker threads submit // surface frames without holding a cross-isolate reference to the bridge @@ -55,53 +34,6 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre fun getActive(): FaceclawBleCommunicator? { return activeInstance } - - private fun chargingStatusText(battery: Int): String { - return if (battery >= 0) - "Glasses charging. Battery $battery%." - else - "Glasses charging." - } - - private fun timestamp(elapsedMs: Long): String { - val wallMs = System.currentTimeMillis() - return String.format(Locale.US, "%tF %tT.%tL elapsed=%dms", wallMs, wallMs, wallMs, elapsedMs) - } - - private fun requireAddress(name: String, address: String?): String { - if (address == null || address.trim().isEmpty()) { - throw IllegalArgumentException("$name is required") - } - return address.trim() - } - - private fun hex(data: ByteArray?): String { - if (data == null || data.isEmpty()) { - return "" - } - val out = CharArray(data.size * 2) - val digits = "0123456789abcdef".toCharArray() - for (i in data.indices) { - val value = data[i].toInt() and 0xff - out[i * 2] = digits[value ushr 4] - out[i * 2 + 1] = digits[value and 0x0f] - } - return String(out) - } - - private fun safeMessage(t: Throwable?): String { - if (t == null) { - return "unknown" - } - val writer = StringWriter() - t.printStackTrace(PrintWriter(writer)) - val trace = writer.toString() - if (trace.trim().isNotEmpty()) { - return trace - } - val message = t.message - return if (message == null || message.trim().isEmpty()) t.toString() else message - } } private val appContext: Context = context.applicationContext @@ -111,222 +43,71 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre private val powerManager: PowerManager = appContext.getSystemService(Context.POWER_SERVICE) as PowerManager private val keyguardManager: KeyguardManager? = appContext.getSystemService(Context.KEYGUARD_SERVICE) as KeyguardManager? private val bleManager: FaceclawBleManager = FaceclawBleManager(appContext) - private val interruptibleSleep = InterruptibleSleep() - private val lock = java.lang.Object() private val mainHandler = Handler(Looper.getMainLooper()) - private val rightAddress: String = requireAddress("rightAddress", rightAddress) - private val leftAddress: String = requireAddress("leftAddress", leftAddress) - private val ringAddress: String = ringAddress?.trim() ?: "" - - @Volatile private var listener: FaceclawBleCommunicatorListener? = null - private val imuListeners: MutableList = - java.util.concurrent.CopyOnWriteArrayList() - /** A compass subscriber plus the Looper it registered from (see addCompassListener). */ - private class CompassSubscription( - @JvmField val listener: FaceclawCompassListener, - @JvmField val handler: Handler, - ) - - private val compassSubscriptions: MutableList = - java.util.concurrent.CopyOnWriteArrayList() - private val ambientLightListeners: MutableList = - java.util.concurrent.CopyOnWriteArrayList() - private val micStatusListeners: MutableList = - java.util.concurrent.CopyOnWriteArrayList() + private val mainDispatcher = SessionDispatcher { action -> mainHandler.post { action() } } @Volatile private var workerThread: Thread? = null - @Volatile private var running = false - @Volatile private var userDisconnectRequested = false - // Set when a connect attempt failed while an arm's Android bond is gone: - // retrying is pointless until the user re-pairs, so the worker loop parks - // instead of redialing. Cleared by start() (a fresh explicit connect). - @Volatile private var reconnectHalted = false - - private var phase = "disconnected" - private var status = "Disconnected." - - private var rightConnected = false - private var leftConnected = false - private var ringConnected = false - private var ringNotificationsReady = false - private var sessionReady = false - private var fixedLayoutCreated = false - private var shutdownRequested = false - // CFW firmware-debug-flags overlay (mode 7). Desired value pushed from TS; the - // sub-op last sent this session (-1 = not yet), reset on (re)connect so the - // overlay state is re-asserted on every reconnect and whenever the value changes. - @Volatile private var firmwareDebugFlagsEnabled = false - private var firmwareDebugFlagsLastSent = -1 - // Desired CFW mode-10 compass state. It survives reconnects; lastSent is - // reset with each session so an open Compass window is re-asserted. - /** - * Who currently wants the stock compass running (the Compass window, the - * Navigate worker, ...). The magnetometer is one shared resource, so it - * stays on while any owner holds it and is released when the last lets go; - * this keeps one app's release from silently switching off another's feed. - */ - private val compassOwners: MutableSet = HashSet() - private var compassControlLastSent = -1 - // Whether the glasses-side compass may still be running: set when an enable - // is enqueued, cleared only when a disable is acked. Drives the forced - // disable sent ahead of an EvenHub shutdown/suspend, since a pending - // disable can be wiped by the shutdown's queue flush and the retry loop - // does not run while shutdownRequested (magnetometer left on = battery drain). - private var compassMaybeOn = false - private var startupProbePending = false - // Desired ownership of CFW's fail-open stock-wake lease (dashboard launch - // and Even AI foreground takeover). This survives a transport reconnect; - // the lease itself is volatile firmware state and is re-acquired once both - // arms are ready. - private var faceclawWakeLeaseEnabled = false - private var lastFaceclawWakeLeaseQueuedAtMs = 0L - private var faceclawWakeControlGeneration = 0 - private var faceclawWakeControlSentCount = 0 - private var lastFaceclawFramebufferLeaseQueuedAtMs = 0L - private var faceclawFramebufferControlGeneration = 0 - private var faceclawFramebufferControlSentCount = 0 - private var faceclawWakePendingNonce = -1 - /** - * The last firmware-info read said the glasses run Faceclaw's custom - * firmware. Gates the private modes (cleanup, resource cache, ...) so stock - * or third-party firmware never sees them; the TS side checks the actual - * revision and disconnects on a mismatch, so no per-feature gating is - * needed here. - */ - private var customFirmwareDetected = false - private var cfwCleanupDelivered = false - private var lastCfwCleanupAckMagic = 0 - - private var reconnectAfterMs = 0L - private var ringReconnectAfterMs = 0L - private var lastAckAtMs = 0L - private var lastIncomingAtMs = 0L - private var lastHeartbeatSentAtMs = 0L - private var lastHeartbeatAckedAtMs = 0L - private var lastConnectionOrInputAtMs = 0L - private var lastBatteryRefreshAtMs = 0L - private var imageRetryAfterMs = 0L - private var lastSessionReadyAtMs = 0L - private var lastEvenAppConflictAtMs = 0L - private var consecutiveAckTimeouts = 0 - private var lastAudioControlAckMagic = 0 - - private val connectionOptions = ConnectionOptions() - private val magicPool = BleMagicPool(AndroidProtocolPlatform) - private val messageBuilder = MessageBuilder(magicPool) - private var nextTransportSeq = 0x40 - private var nextMapSessionId = 0 - private var nextImageUpdateId = 1 - // Wire frame id for mode-3 deltas (CFW reorder/skip/dup diagnostic). uint16, - // advanced by 1 per emitted delta; kept in [1, 0xfffe] to avoid the CFW's - // 0xffff "empty" sentinel. - private var nextImageFrameId = 1 - private var lastShutdownAckMagic = 0 - private var lastShutdownExitAtMs = 0L - private var headsetBattery = -1 - private var headsetCharging = -1 - private var ringBattery = -1 - private var ringCharging = -1 - // Silent mode: 1 = on, 0 = off, -1 = not yet known. See updateSilentModeLocked. - private var silentMode = -1 - private var wearState = -1 - private var phoneLockState = -1 - private var lastPhoneLockCheckAtMs = 0L - private var phoneLockReceiverRegistered = false - private var audioCaptureActive = false - private var firmwareInfoQueried = false - // Glasses are in the charging case: nobody is wearing them, so display - // communication pauses and only battery polls flow (see driveSession). - private var chargingMode = false - @Volatile private var audioPacketListener: FaceclawAudioPacketListener? = null private var g2ScreenWakeLock: PowerManager.WakeLock? = null + private var phoneLockReceiverRegistered = false - // BLE bandwidth benchmark (Developer app). Streams no-op image payloads - // (CFW mode 7 with an unused sub-op: parsed, acked, and discarded — stock - // firmware likewise ignores unknown image modes) for a fixed duration with - // a selectable message size and pipeline window, then reports throughput. - // While active, desired-frame sends are held back and heartbeats are - // satisfied by the benchmark's own acks, so the stream is the only image - // traffic. All state below is guarded by `lock`; results are read with - // getBandwidthBenchmarkStatus() and survive until the next run starts. - private var benchmarkActive = false - private var benchmarkAborted = false - private val benchmarkRandom = Random() - private var benchmarkMessageSize = 0 - private var benchmarkWindowSize = 0 - private var benchmarkDurationMs = 0 - private var benchmarkLinkMode = 0 - private var benchmarkLinkPending = false - private var benchmarkReadyAtMs = 0L - private var benchmarkStartAtMs = 0L // first benchmark write; 0 until then - private var benchmarkDeadlineAtMs = 0L // start + duration; MAX_VALUE until first write - private var benchmarkLastAckAtMs = 0L - private var benchmarkEndAtMs = 0L // 0 while running; set when the run drains - private var benchmarkMessagesSent = 0 - private var benchmarkMessagesAcked = 0 - private var benchmarkTimeouts = 0 - private var benchmarkPayloadBytesAcked = 0L - private var benchmarkWireBytesAcked = 0L + private val host = object : SessionHost { + override fun postToMain(action: () -> Unit) { + mainHandler.post { action() } + } - private val phoneLockReceiver: BroadcastReceiver = object : BroadcastReceiver() { - override fun onReceive(context: Context?, intent: Intent?) { - emitPhoneLockStateIfChanged(true) - interruptibleSleep.interrupt() + override fun currentThreadDispatcher(): SessionDispatcher { + val looper = Looper.myLooper() ?: return mainDispatcher + val handler = Handler(looper) + return SessionDispatcher { action -> handler.post { action() } } + } + + override fun isPhoneLocked(): Boolean = keyguardManager != null && keyguardManager.isDeviceLocked + + override fun setScreenWakeLock(on: Boolean) = updateG2ScreenWakeLock(on) + + override fun isEvenAppActive(): Boolean = FaceclawEvenAppDetector.isEvenNotificationActive(appContext) + + override fun startWorker(body: () -> Unit) { + val thread = Thread(body, "FaceclawBleCommunicator") + workerThread = thread + thread.start() + } + + override fun joinWorker(timeoutMs: Long) { + val threadToJoin = workerThread ?: return + threadToJoin.interrupt() + try { + threadToJoin.join(timeoutMs) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + } + workerThread = null + } + + override fun log(level: SessionLogLevel, tag: String, message: String, error: Throwable?) { + when (level) { + SessionLogLevel.DEBUG -> Log.d(tag, message) + SessionLogLevel.INFO -> Log.i(tag, message) + SessionLogLevel.WARN -> if (error != null) Log.w(tag, message, error) else Log.w(tag, message) + SessionLogLevel.ERROR -> if (error != null) Log.e(tag, message, error) else Log.e(tag, message) + } } } - private var displayedFingerprint = "" - // The frame the firmware shadow will hold once the current image pipeline - // drains: the most recently ENQUEUED image (headerless packed 4bpp, see - // BmpUtil.pack4bppFromGray8), which is the correct base for the next delta - // when frames are pipelined. Set at enqueue; cleared whenever the image - // pipeline is cleared (clearAllMessagesLocked / clearMessagesOfKindLocked - // "image"), so it is only ever read while it holds a valid current-session base. - private var lastEnqueuedPacked: ByteArray = ByteArray(0) - private var lastEnqueuedWidth = 0 - private var lastEnqueuedHeight = 0 - private var lastEnqueuedFingerprint = "" - private val imageUpdateStats: MutableMap = HashMap() + private val core = GlassesSessionCore( + AndroidSessionLink(bleManager), + host, + FrameTimings.getInstance().core, + AndroidProtocolPlatform, + rightAddress, + leftAddress, + ringAddress, + ) - private val desiredTilesLock = Any() - private var desiredFingerprint = "" - // Headerless packed 4bpp frame (see BmpUtil.pack4bppFromGray8) plus its - // pixel dimensions. - private var desiredPacked: ByteArray? = ByteArray(0) - private var desiredWidth = 0 - private var desiredHeight = 0 - private var desiredPaintMs = 0 - private var desiredFrameId = 0 - // Screen-space deferred draws (glyphs + images) whose pixels are baked - // into desiredPacked; the resource-cache planner may replay them as - // on-glasses cached draws. - private var desiredDraws: Array? = arrayOf() - // (frame, reason) of the last "waiting to send" line, so a frame that - // stalls for seconds records one line per state change (see - // noteImageStallLocked). Send-loop thread only. - private var stallFrameId = 0 - private var stallReason = "" - // Highest compositor sequence stored as the desired frame; composites that - // lost a store race to a newer one are discarded (their content is already - // included in the newer composite). - private var lastStoredCompositeSeq = 0L - - // Wire submissions need screenGray + the shell scene, never preview pixels. - // Preview/screenshot/recording callers render those explicitly on demand. - private val compositor = SurfaceCompositor(false) - - // Phone-side model of the CFW's 192 KiB resource cache. - // Reset whenever the image pipeline / EvenHub session is torn down: the - // firmware frees the cache with the fb lease, and after any resync the - // cheap safe assumption is an empty cache (glyphs re-upload lazily). - private val resourceCache = ResourceCacheState() - private val scenePlanner = ScenePlanner(resourceCache) - private var desiredShellScene: ShellScene = ShellScene.EMPTY - - private val pendingMessages = ArrayDeque() - private val cfwTransports = arrayOf(CfwTransport(AndroidProtocolPlatform), CfwTransport(AndroidProtocolPlatform)) - private val inFlightMessages = ArrayDeque() - private var prewrittenMessage: OutboundMessage? = null - private var prewrittenFrames: List = Collections.emptyList() + private val phoneLockReceiver: BroadcastReceiver = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + core.onPhoneLockSignal() + } + } // Active animated-GIF screen recording, or null when idle. Frames are // pushed by recordScreenFrame(), which the TS side calls at each @@ -334,7 +115,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre @Volatile private var screenRecorder: GifScreenRecorder? = null init { - bleManager.setListener(this) + bleManager.setListener(core) val phoneLockFilter = IntentFilter() phoneLockFilter.addAction(Intent.ACTION_SCREEN_ON) phoneLockFilter.addAction(Intent.ACTION_SCREEN_OFF) @@ -343,746 +124,98 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre phoneLockReceiverRegistered = true } - - fun setListener(listener: FaceclawBleCommunicatorListener?) { - this.listener = listener - emitState() - emitPhoneLockStateIfChanged(true) - } + fun setListener(listener: FaceclawBleCommunicatorListener?) = core.setListener(listener) fun start() { - synchronized(lock) { - if (running) { - return - } - running = true - userDisconnectRequested = false - reconnectHalted = false - shutdownRequested = false + if (core.start()) { activeInstance = this - val thread = Thread(this, "FaceclawBleCommunicator") - workerThread = thread - thread.start() } } - fun disconnect() { - /* On the normal path DashboardController already sent mode 11 after - * quiescing its producers. Also cover direct/early close callers here; - * a successful cleanup must remain the final BLE message. Older CFWs - * fall back to the standalone framebuffer-lease release. */ - if (!cfwCleanupDelivered && !sendCfwCleanup()) { - releaseFaceclawFramebufferLease() - } - val threadToJoin: Thread? - synchronized(lock) { - userDisconnectRequested = true - running = false - audioCaptureActive = false - audioPacketListener = null - threadToJoin = workerThread - } - setStateDisplay("disconnecting", "Disconnecting...") - interruptibleSleep.interrupt() - if (threadToJoin != null) { - threadToJoin.interrupt() - try { - threadToJoin.join(5_000) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - } - } - synchronized(lock) { - workerThread = null - resetSessionStateLocked() - clearAllMessagesLocked("disconnect") - // Unknown until the next connection's first push or settings poll. - silentMode = -1 - } - bleManager.disconnect(rightAddress) - bleManager.disconnect(leftAddress) - if (hasRingAddress()) { - bleManager.disconnect(ringAddress) - } - bleManager.close() - releaseG2ScreenWakeLock() - setStateDisplay("disconnected", "Disconnected.") - } + fun disconnect() = core.disconnect() fun close() { if (activeInstance === this) { activeInstance = null } - disconnect() - for (transport in cfwTransports) transport.close() + core.close() if (phoneLockReceiverRegistered) { phoneLockReceiverRegistered = false appContext.unregisterReceiver(phoneLockReceiver) } } - fun setG2ScreenOn(screenOn: Boolean) { - mainHandler.post { updateG2ScreenWakeLock(screenOn) } - } + fun setG2ScreenOn(screenOn: Boolean) = core.setG2ScreenOn(screenOn) - fun setFirmwareDebugFlags(enabled: Boolean) { - // Just record it; the drive loop emits the mode-7 control message when the - // display path is ready and idle, and re-emits when this value changes. - firmwareDebugFlagsEnabled = enabled - } + fun setFirmwareDebugFlags(enabled: Boolean) = core.setFirmwareDebugFlags(enabled) - fun startG2AudioCapture(listener: FaceclawAudioPacketListener?): Boolean { - if (listener == null) { - throw IllegalArgumentException("listener is required") - } - var magic = 0 - synchronized(lock) { - if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { - logLine("skip G2 mic enable; EvenHub display path not ready") - return false - } - audioPacketListener = listener - val message = createAudioControlMessageLocked(true) - magic = message.magic - pendingMessages.addFirst(message) - logLine("queue G2 mic enable") - } - interruptibleSleep.interrupt() - return waitForAudioControlAck(magic, "enable") - } + fun startG2AudioCapture(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioCapture(listener) - fun stopG2AudioCapture() { - var magic = 0 - synchronized(lock) { - audioPacketListener = null - audioCaptureActive = false - clearMessagesOfKindLocked("audio-control") - if (running && sessionReady) { - val message = createAudioControlMessageLocked(false) - magic = message.magic - pendingMessages.addFirst(message) - logLine("queue G2 mic disable") - } - } - interruptibleSleep.interrupt() - if (magic != 0) { - waitForAudioControlAck(magic, "disable") - } - } + fun stopG2AudioCapture() = core.stopG2AudioCapture() - fun isSessionReady(): Boolean { - synchronized(lock) { - return running && sessionReady - } - } + fun isSessionReady(): Boolean = core.isSessionReady() - /** - * Whether the glasses mic is enabled right now. The enable lives in the - * current EvenHub session, so it dies with a transport drop, the charging - * case, or a suspend — silently, from the phone's point of view. Callers - * that track a capture across those events must check this rather than - * assume their earlier enable still holds. - */ - fun isAudioCaptureActive(): Boolean { - synchronized(lock) { - return running && sessionReady && !shutdownRequested && audioCaptureActive - } - } + /** See [GlassesSessionCore.isAudioCaptureActive]. */ + fun isAudioCaptureActive(): Boolean = core.isAudioCaptureActive() - /** - * Acquire/renew or release CFW's volatile wake-takeover lease on both - * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure - * the fail-open firmware policy is installed before relying on wakeword - * interception or suspending EvenHub. - */ - fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean { - val generation: Int - synchronized(lock) { - faceclawWakeLeaseEnabled = enabled - if (!running || !sessionReady) { - return !enabled - } - generation = enqueueFaceclawWakeControlLocked( - if (enabled) BleProtocol.FACECLAW_WAKE_OP_ACQUIRE else BleProtocol.FACECLAW_WAKE_OP_RELEASE, - 0, - true - ) - if (!enabled) { - faceclawWakePendingNonce = -1 - } - } - interruptibleSleep.interrupt() - return waitForFaceclawWakeControlDelivery(generation, FACECLAW_WAKE_CONTROL_WAIT_MS) - } + fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled) - /** - * Wait until the recreated layout and retained compositor frame have both - * landed. If this wake came from CFW's deferred double tap, READY - * is then sent to both arms to cancel their stock-dashboard fallback. - */ - fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean { - val deadline = SystemClock.elapsedRealtime() + Math.max(0, timeoutMs) - var readyGeneration = 0 - synchronized(lock) { - while (running && sessionReady) { - var frameReady = false - synchronized(desiredTilesLock) { - frameReady = desiredFingerprint.isNotEmpty() - && desiredFingerprint == displayedFingerprint - } - if (!shutdownRequested && fixedLayoutCreated && frameReady) { - if (faceclawWakePendingNonce >= 0) { - readyGeneration = enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_READY, - faceclawWakePendingNonce, - true - ) - faceclawWakePendingNonce = -1 - } - break - } - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - return false - } - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - return false - } - } - if (!running || !sessionReady) { - return false - } - } - if (readyGeneration != 0) { - interruptibleSleep.interrupt() - if (!waitForFaceclawWakeControlDelivery(readyGeneration, FACECLAW_WAKE_CONTROL_WAIT_MS)) { - logLine("wake READY delivery not confirmed before fallback deadline") - } - } - return true - } + fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs) - /** - * Enable or disable the IMU (accelerometer) report stream. Fire-and-forget: - * the control message is queued ahead of other traffic; readings arrive via - * registered FaceclawImuListeners. reportFrq is the requested sample rate - * (ignored on disable). - */ - fun setImuReportEnabled(enable: Boolean, reportFrq: Int) { - synchronized(lock) { - if (!running || !sessionReady) { - logLine("skip IMU " + (if (enable) "enable" else "disable") + "; session not ready") - return - } - clearMessagesOfKindLocked("imu-control") - val message = messageBuilder.enableOrDisableImu(enable, reportFrq) - message.onTimeout = MessageCallback { logLine("IMU control ack timeout") } - pendingMessages.addFirst(message) - logLine("queue IMU " + (if (enable) "enable freq=$reportFrq" else "disable")) - } - interruptibleSleep.interrupt() - } + fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq) - /** - * Enable/disable the stock compass through CFW image-handler mode 10. The - * desired state is retained across reconnects; headings arrive through - * stock sid-0x08 navigation notifications and FaceclawCompassListeners. - */ - fun setCompassEnabled(enable: Boolean) { - setCompassEnabled("compass", enable) - } + fun setCompassEnabled(enable: Boolean) = core.setCompassEnabled(enable) - /** - * As above, on behalf of a named owner. The compass runs while at least - * one owner has enabled it; an owner disabling it only takes effect once - * no other owner still wants it. - */ - fun setCompassEnabled(owner: String?, enable: Boolean) { - synchronized(lock) { - val before = compassOwners.isNotEmpty() - if (enable) { - compassOwners.add(owner) - } else { - compassOwners.remove(owner) - } - val wanted = compassOwners.isNotEmpty() - if (wanted == before && compassControlLastSent == (if (wanted) 1 else 0)) { - logLine("compass " + (if (enable) "enable" else "disable") + " by " + owner - + "; state unchanged (owners=" + compassOwners + ")") - return - } - compassControlLastSent = -1 - clearMessagesOfKindLocked("compass-control") - if (running && sessionReady && !shutdownRequested && fixedLayoutCreated) { - enqueueCompassControlLocked(true, wanted) - } else { - logLine("defer compass " + (if (enable) "enable" else "disable") + "; display path not ready") - } - } - interruptibleSleep.interrupt() - } + fun setCompassEnabled(owner: String?, enable: Boolean) = core.setCompassEnabled(owner, enable) - /** - * Set the lens brightness. Fire-and-forget, like the IMU control: the - * message is queued ahead of other traffic and any not-yet-sent brightness - * message is superseded. When autoAdjust is true the ambient-light sensor - * drives brightness and brightnessLevel is ignored; otherwise - * brightnessLevel (0-100) is applied directly. - */ - fun setBrightness(autoAdjust: Boolean, brightnessLevel: Int) { - synchronized(lock) { - if (!running || !sessionReady) { - logLine("skip brightness set; session not ready") - return - } - clearMessagesOfKindLocked("brightness-control") - val message = messageBuilder.setBrightness(autoAdjust, brightnessLevel) - message.onTimeout = MessageCallback { logLine("brightness control ack timeout") } - pendingMessages.addFirst(message) - logLine("queue brightness " + (if (autoAdjust) "auto" else "level=$brightnessLevel")) - } - interruptibleSleep.interrupt() - } + fun setBrightness(autoAdjust: Boolean, brightnessLevel: Int) = core.setBrightness(autoAdjust, brightnessLevel) - /** - * Enable the stock wear detector, then ask CFW to emit its current cached - * state. The queue order matters: the query must run after the setting is - * applied, including on a fresh install where wear detection was disabled. - */ - fun enableWearDetectionAndRequestState() { - synchronized(lock) { - if (!running || !sessionReady) { - logLine("skip wear detector setup; session not ready") - return - } - clearMessagesOfKindLocked("wear-detection-control") - clearMessagesOfKindLocked("wear-query-control") - val queryLeft = messageBuilder.faceclawWearQuery(true) - val queryRight = messageBuilder.faceclawWearQuery(false) - val enable = messageBuilder.setWearDetection(true) - enable.onTimeout = MessageCallback { logLine("wear detection enable ack timeout") } - pendingMessages.addFirst(queryLeft) - pendingMessages.addFirst(queryRight) - pendingMessages.addFirst(enable) - logLine("queue wear detection enable + current-state query") - } - interruptibleSleep.interrupt() - } + fun enableWearDetectionAndRequestState() = core.enableWearDetectionAndRequestState() - /** - * Start the BLE bandwidth benchmark: stream messageSize-byte no-op image - * payloads for durationMs, keeping up to windowSize messages awaiting ack - * at once, then leave the results for getBandwidthBenchmarkStatus(). - * Returns false when a run is already active or the image path is not - * ready. The duration clock starts at the first benchmark write, so - * traffic already queued ahead of the run doesn't count against it. - */ - fun startBandwidthBenchmark(messageSize: Int, windowSize: Int, durationMs: Int): Boolean { - return startBandwidthBenchmarkWithLinkMode(messageSize, windowSize, durationMs, 0) - } + fun startBandwidthBenchmark(messageSize: Int, windowSize: Int, durationMs: Int): Boolean = + core.startBandwidthBenchmark(messageSize, windowSize, durationMs) // 0: current link; 1: re-request HIGH; 2: request 2M; 3: both. fun startBandwidthBenchmarkWithLinkMode(messageSize: Int, windowSize: Int, - durationMs: Int, linkMode: Int): Boolean { - synchronized(lock) { - if (benchmarkActive || !running || !sessionReady || !fixedLayoutCreated - || shutdownRequested || chargingMode) { - logLine("skip bandwidth benchmark; already running or image path not ready") - return false - } - benchmarkMessageSize = Math.max(2, Math.min(messageSize, ConnectionOptions.IMAGE_FRAGMENT_SIZE)) - benchmarkWindowSize = Math.max(1, Math.min(windowSize, BENCHMARK_MAX_WINDOW)) - benchmarkDurationMs = Math.max(1_000, durationMs) - benchmarkLinkMode = linkMode and 3 - benchmarkLinkPending = true - benchmarkReadyAtMs = Long.MAX_VALUE - benchmarkStartAtMs = 0 - benchmarkDeadlineAtMs = Long.MAX_VALUE - benchmarkLastAckAtMs = 0 - benchmarkEndAtMs = 0 - benchmarkMessagesSent = 0 - benchmarkMessagesAcked = 0 - benchmarkTimeouts = 0 - benchmarkPayloadBytesAcked = 0 - benchmarkWireBytesAcked = 0 - benchmarkAborted = false - benchmarkActive = true - logLine("bandwidth benchmark start: size=" + benchmarkMessageSize - + "B window=" + benchmarkWindowSize + " duration=" + benchmarkDurationMs - + "ms linkMode=" + benchmarkLinkMode) - } - interruptibleSleep.interrupt() - return true - } + durationMs: Int, linkMode: Int): Boolean = + core.startBandwidthBenchmarkWithLinkMode(messageSize, windowSize, durationMs, linkMode) - /** - * Cancel an in-progress benchmark (benchmark page closed). Queued no-op - * messages are dropped; in-flight ones drain through their normal acks. - */ - fun cancelBandwidthBenchmark() { - synchronized(lock) { - if (!benchmarkActive) { - return - } - clearMessagesOfKindLocked("bandwidth") - finishBenchmarkLocked(true, "cancelled") - } - interruptibleSleep.interrupt() - } + fun cancelBandwidthBenchmark() = core.cancelBandwidthBenchmark() /** Status/results of the current or most recent benchmark run, as JSON. */ - fun getBandwidthBenchmarkStatus(): String { - synchronized(lock) { - val state = if (benchmarkActive) - (if (benchmarkStartAtMs == 0L) "starting" else "running") - else - (if (benchmarkEndAtMs != 0L) "done" else "idle") - val end = if (benchmarkActive) SystemClock.elapsedRealtime() else benchmarkEndAtMs - val elapsed = if (benchmarkStartAtMs == 0L) 0L else Math.max(0L, end - benchmarkStartAtMs) - try { - val status = org.json.JSONObject() - status.put("state", state) - status.put("messageSize", benchmarkMessageSize) - status.put("windowSize", benchmarkWindowSize) - status.put("linkMode", benchmarkLinkMode) - status.put("elapsedMs", elapsed) - status.put("messagesSent", benchmarkMessagesSent) - status.put("messagesAcked", benchmarkMessagesAcked) - status.put("timeouts", benchmarkTimeouts) - status.put("payloadBytesAcked", benchmarkPayloadBytesAcked) - status.put("wireBytesAcked", benchmarkWireBytesAcked) - status.put("aborted", benchmarkAborted) - return status.toString() - } catch (e: org.json.JSONException) { - return "{\"state\":\"idle\"}" - } - } - } + fun getBandwidthBenchmarkStatus(): String = core.getBandwidthBenchmarkStatus() - /** Pending + in-flight benchmark no-op messages. */ - private fun benchmarkOutstandingLocked(): Int { - var count = 0 - for (message in pendingMessages) { - if ("bandwidth" == message.kind) count++ - } - for (message in inFlightMessages) { - if ("bandwidth" == message.kind) count++ - } - return count - } + fun addImuListener(listener: FaceclawImuListener?) = core.addImuListener(listener) - /** - * Keep the benchmark stream fed: top the pending queue up so the send - * window never starves, stop enqueueing once the run expires (or a message - * times out), and finish the run when the last outstanding message drains. - */ - private fun maintainBenchmarkLocked(now: Long) { - if (!sessionReady || !fixedLayoutCreated || shutdownRequested) { - finishBenchmarkLocked(true, "session no longer ready") - return - } - if (benchmarkLinkPending || now < benchmarkReadyAtMs) { - return - } - if (benchmarkAborted || now >= benchmarkDeadlineAtMs) { - // The run is over: drop queued-but-unsent no-ops (sending them - // would stretch the run past its deadline) and finish once the - // in-flight tail has acked or timed out. - val pendingIterator = pendingMessages.iterator() - while (pendingIterator.hasNext()) { - val queued = pendingIterator.next() - if ("bandwidth" == queued.kind) { - pendingIterator.remove() - magicPool.release(queued.sid, queued.magic, queued.label, "benchmark over") - } - } - if (benchmarkOutstandingLocked() == 0) { - finishBenchmarkLocked(false, "complete") - } - return - } - // One more than the window so a fresh message is always ready to write - // the moment an ack frees a slot. - var outstanding = benchmarkOutstandingLocked() - while (outstanding <= benchmarkWindowSize) { - enqueueBenchmarkMessageLocked() - outstanding++ - } - } + fun removeImuListener(listener: FaceclawImuListener?) = core.removeImuListener(listener) - private fun finishBenchmarkLocked(aborted: Boolean, reason: String) { - if (!benchmarkActive) { - return - } - benchmarkActive = false - benchmarkAborted = benchmarkAborted or aborted - // Prefer the last ack as the end time so drain lag after the deadline - // doesn't dilute the throughput figure. - benchmarkEndAtMs = if (benchmarkLastAckAtMs != 0L) benchmarkLastAckAtMs else SystemClock.elapsedRealtime() - logLine("bandwidth benchmark " + (if (benchmarkAborted) "aborted" else "finished") + " (" + reason + "): " - + benchmarkMessagesAcked + "/" + benchmarkMessagesSent + " acked, " - + benchmarkPayloadBytesAcked + "B payload, " + benchmarkTimeouts + " timeouts") - } + fun addAmbientLightListener(listener: FaceclawAmbientLightListener?) = core.addAmbientLightListener(listener) - private fun enqueueBenchmarkMessageLocked() { - // Fresh bytes per message: the transport's persistent compression history - // would compress even a random payload if we reused it across the run. - val payload = ByteArray(benchmarkMessageSize) - benchmarkRandom.nextBytes(payload) - payload[0] = CFW_MSG_DIAGNOSTICS.toByte() // CFW diagnostic-control mode... - payload[1] = 0x7f.toByte() // ...with an unused sub-op: acked, no effect - val message = messageBuilder.imagePayload( - "bandwidth", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - "bandwidth no-op " + payload.size + "B", - connectionOptions.sendImagesToLeft) - val payloadBytes = payload.size - // Logical custom-message bytes; excludes length tag and packet framing. - val wireBytes = message.message.size - message.onSent = MessageCallback { - benchmarkMessagesSent++ - if (benchmarkStartAtMs == 0L) { - benchmarkStartAtMs = SystemClock.elapsedRealtime() - benchmarkDeadlineAtMs = benchmarkStartAtMs + benchmarkDurationMs - } - } - message.onAck = MessageCallback { - val ackedAtMs = SystemClock.elapsedRealtime() - benchmarkMessagesAcked++ - benchmarkPayloadBytesAcked += payloadBytes - benchmarkWireBytesAcked += wireBytes - benchmarkLastAckAtMs = ackedAtMs - // No-op payloads ride the image path, so the firmware resets its - // heartbeat timer on them just like real image messages. - lastHeartbeatAckedAtMs = ackedAtMs - } - message.onTimeout = MessageCallback { - benchmarkTimeouts++ - benchmarkAborted = true - logLine("bandwidth benchmark ack timeout") - } - pendingMessages.addLast(message) - } + fun removeAmbientLightListener(listener: FaceclawAmbientLightListener?) = core.removeAmbientLightListener(listener) - fun addImuListener(listener: FaceclawImuListener?) { - if (listener != null) { - imuListeners.add(listener) - } - } + fun queryAmbientLight() = core.queryAmbientLight() - fun removeImuListener(listener: FaceclawImuListener?) { - if (listener != null) { - imuListeners.remove(listener) - } - } - - fun addAmbientLightListener(listener: FaceclawAmbientLightListener?) { - if (listener != null) { - ambientLightListeners.add(listener) - } - } - - fun removeAmbientLightListener(listener: FaceclawAmbientLightListener?) { - if (listener != null) { - ambientLightListeners.remove(listener) - } - } - - private fun emitAmbientLight(body: ByteArray) { - if (ambientLightListeners.isEmpty()) { - return - } - val copy = Arrays.copyOf(body, body.size) - mainHandler.post { - for (alsListener in ambientLightListeners) { - try { - alsListener.onAmbientLight(copy) - } catch (t: Throwable) { - Log.w(TAG, "ambient light listener failed", t) - } - } - } - } - - /** Request one CFW ambient-light report (image-handler mode 16 op 0). */ - fun queryAmbientLight() { - synchronized(lock) { - enqueueAmbientLightControlLocked(byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 0.toByte()), "als query", false) - } - interruptibleSleep.interrupt() - } - - /** - * Start or stop CFW passive light-sensor polling (image-handler mode 16 - * ops 1/2). While polling, the firmware reads its OPT3001 every intervalMs - * (clamped 100..5000 by the firmware), never steps the panel brightness - * itself, and pushes a field-105 report when the reading moved by at least - * minDelta or heartbeatMs elapsed. bindToLease stops polling automatically - * when the Faceclaw framebuffer lease is released or lapses. Starting is - * idempotent and re-opens the sensor if the stock firmware closed it. - */ fun setAmbientLightPolling(enable: Boolean, intervalMs: Int, minDelta: Int, - heartbeatMs: Int, bindToLease: Boolean) { - val payload = if (enable) - byteArrayOf( - CFW_MSG_AMBIENT_LIGHT.toByte(), - 1.toByte(), - (if (bindToLease) 1 else 0).toByte(), - (intervalMs and 0xff).toByte(), - ((intervalMs shr 8) and 0xff).toByte(), - (minDelta and 0xff).toByte(), - ((minDelta shr 8) and 0xff).toByte(), - (heartbeatMs and 0xff).toByte(), - ((heartbeatMs shr 8) and 0xff).toByte(), - ) - else - byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 2.toByte()) - synchronized(lock) { - clearMessagesOfKindLocked("als-control") - enqueueAmbientLightControlLocked(payload, "als polling " + (if (enable) "start" else "stop"), true) - } - interruptibleSleep.interrupt() - } + heartbeatMs: Int, bindToLease: Boolean) = + core.setAmbientLightPolling(enable, intervalMs, minDelta, heartbeatMs, bindToLease) - private fun enqueueAmbientLightControlLocked(payload: ByteArray, label: String, priority: Boolean) { - if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { - logLine("skip $label; display path not ready") - return - } - val message = messageBuilder.imagePayload( - "als-control", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - label, - connectionOptions.sendImagesToLeft) - message.onTimeout = MessageCallback { logLine("$label ack timeout") } - if (priority) pendingMessages.addFirst(message) - else pendingMessages.addLast(message) - logLine("queue $label") - } + fun addMicStatusListener(listener: FaceclawMicStatusListener?) = core.addMicStatusListener(listener) - fun addMicStatusListener(listener: FaceclawMicStatusListener?) { - if (listener != null) { - micStatusListeners.add(listener) - } - } + fun removeMicStatusListener(listener: FaceclawMicStatusListener?) = core.removeMicStatusListener(listener) - fun removeMicStatusListener(listener: FaceclawMicStatusListener?) { - if (listener != null) { - micStatusListeners.remove(listener) - } - } + fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) = + core.sendFaceclawMicControl(record, label, rightTemple, leftTemple) - private fun emitMicStatus(body: ByteArray, address: String) { - if (micStatusListeners.isEmpty()) { - return - } - val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" - else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?" - val copy = Arrays.copyOf(body, body.size) - mainHandler.post { - for (micListener in micStatusListeners) { - try { - micListener.onMicStatus(copy, arm) - } catch (t: Throwable) { - Log.w(TAG, "mic status listener failed", t) - } - } - } - } + fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener) - /** - * Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings - * field-103 write to both temples, or to a single one. Fire-and-forget: - * the firmware answers with a field-104 status notify per temple, which - * arrives through addMicStatusListener. - */ - fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) { - if (record == null || record.size < 4) { - return - } - synchronized(lock) { - if (!running || !sessionReady) { - logLine("skip mic control ($label); session not ready") - return - } - if (rightTemple) { - pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false)) - } - if (leftTemple) { - pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true)) - } - logLine("queue mic control $label") - } - interruptibleSleep.interrupt() - } + fun stopG2AudioForwarding() = core.stopG2AudioForwarding() - /** - * Forward render-characteristic audio packets to the listener WITHOUT - * sending the stock EvenHub audio-control enable. Used for the CFW - * mic_control streaming path, where capture is armed through settings - * field 103 and the temples emit 'SM' frames on the same characteristic - * that stock mono LC3 uses. Returns false when no session is up. - */ - fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean { - if (listener == null) { - throw IllegalArgumentException("listener is required") - } - synchronized(lock) { - if (!running || !sessionReady || shutdownRequested) { - logLine("skip G2 audio forwarding; session not ready") - return false - } - audioPacketListener = listener - audioCaptureActive = true - logLine("G2 audio forwarding enabled") - } - return true - } + fun addCompassListener(listener: FaceclawCompassListener?) = core.addCompassListener(listener) - fun stopG2AudioForwarding() { - synchronized(lock) { - audioPacketListener = null - audioCaptureActive = false - logLine("G2 audio forwarding disabled") - } - } - - /** - * Subscribe to compass events. Callbacks are delivered on the Looper of - * the thread that registered (falling back to the main thread), so app - * worker isolates can listen without a cross-thread hop into their JS. - */ - fun addCompassListener(listener: FaceclawCompassListener?) { - if (listener == null) { - return - } - val looper = Looper.myLooper() - val handler = if (looper != null) Handler(looper) else mainHandler - compassSubscriptions.add(CompassSubscription(listener, handler)) - } - - fun removeCompassListener(listener: FaceclawCompassListener?) { - if (listener == null) { - return - } - for (subscription in compassSubscriptions) { - if (subscription.listener === listener) { - compassSubscriptions.remove(subscription) - } - } - } + fun removeCompassListener(listener: FaceclawCompassListener?) = core.removeCompassListener(listener) /** Set the compositor's output frame size. Call before configuring surfaces. */ - fun configureCompositorScreen(width: Int, height: Int) { - compositor.configureScreen(width, height) - } + fun configureCompositorScreen(width: Int, height: Int) = core.configureCompositorScreen(width, height) /** * The current composited screen as a phone-UI preview bitmap, or null @@ -1096,7 +229,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre /** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */ fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean): android.graphics.Bitmap? { - val composite = compositor.previewComposite() ?: return null + val composite = core.previewComposite() ?: return null return PreviewBitmapUtil.fromGray( java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } @@ -1104,7 +237,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @Throws(java.io.IOException::class) fun saveCompositePngScreenshot(): String { - val composite = compositor.previewComposite() ?: return "" + val composite = core.previewComposite() ?: return "" return ScreenshotUtil.savePngScreenshot(appContext, composite.gray, composite.width, composite.height) } @@ -1115,7 +248,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre */ @Throws(java.io.IOException::class) fun saveCompositePngScreenshot(cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): String { - val composite = compositor.previewComposite() ?: return "" + val composite = core.previewComposite() ?: return "" val x = Math.max(0, cropX) val y = Math.max(0, cropY) val width = Math.min(composite.width - x, cropWidth - (x - cropX)) @@ -1138,7 +271,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre /** Capture the current composite into the active recording; no-op when idle. */ fun recordScreenFrame() { val recorder = screenRecorder ?: return - val composite = compositor.previewComposite() ?: return + val composite = core.previewComposite() ?: return recorder.addFrame(composite.gray, composite.width, composite.height, System.currentTimeMillis()) } @@ -1151,74 +284,30 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre return "" } if (recorder.isOverflowed()) { - logLine("screen recording hit its frame cap; the tail was dropped") + Log.i(TAG, "screen recording hit its frame cap; the tail was dropped") } return recorder.save(appContext) } - /** - * Show or hide a compositor surface, immediately submitting the resulting - * frame. Recompositing here (rather than waiting for the next surface - * update) is what makes a just-foregrounded window's retained frame - * actually appear — otherwise a static window (e.g. the terminal hub) whose - * frame landed while briefly hidden would stay blank until its next repaint. - */ - fun setSurfaceVisible(id: String, visible: Boolean) { - // Its own frame: this recomposite is a real screen update with real - // latency, and without one it would show up in other frames' logs only - // as an anonymous "superseded by frame#0". - val frameId = FrameTimings.getInstance().startFrame( - "compositor:visible $id=$visible") - compositor.setSurfaceVisible(id, visible) - val composite = compositor.composite() - val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) - storeDesiredComposite(composite, packed, 0, frameId) - } + fun setSurfaceVisible(id: String, visible: Boolean) = core.setSurfaceVisible(id, visible) - /** - * Blank (screen off) or unblank the composited output, immediately - * submitting the resulting frame. Retained surface state is untouched, so - * unblanking restores the previous screen content without repaints. - */ - fun setScreenBlanked(blanked: Boolean) { - val frameId = FrameTimings.getInstance().startFrame( - "compositor:" + (if (blanked) "blank" else "unblank")) - compositor.setBlanked(blanked) - val composite = compositor.composite() - val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) - storeDesiredComposite(composite, packed, 0, frameId) - } + fun setScreenBlanked(blanked: Boolean) = core.setScreenBlanked(blanked) - /** - * Create or reconfigure a compositor surface. transparency is one of the - * SurfaceCompositor.TRANSPARENCY_* constants. Geometry changes take effect - * when the next frame is submitted. - */ - fun configureSurface(id: String?, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) { - compositor.configureSurface(id, x, y, width, height, zOrder, transparency) - } + fun configureSurface(id: String?, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) = + core.configureSurface(id, x, y, width, height, zOrder, transparency) - fun removeSurface(id: String) { - compositor.removeSurface(id) - } + fun removeSurface(id: String) = core.removeSurface(id) /** Stage an immutable shell snapshot alongside the retained app screen. */ - fun submitShellScene(bytes: java.nio.ByteBuffer, paintMs: Int, frameId: Int) { - compositor.setShellScene(AndroidByteReader(bytes)) - val composite = compositor.composite() - storeDesiredComposite(composite, BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height), paintMs, frameId) - } + fun submitShellScene(bytes: java.nio.ByteBuffer, paintMs: Int, frameId: Int) = + core.submitShellScene(AndroidByteReader(bytes), paintMs, frameId) /** Legacy compositor dimming for callers without a shell scene. */ - fun setUnderlayDim(belowZOrder: Int, factor256: Int) { - compositor.setUnderlayDim(belowZOrder, factor256) - } + fun setUnderlayDim(belowZOrder: Int, factor256: Int) = core.setUnderlayDim(belowZOrder, factor256) /** * Apply an update to one compositor surface and submit the recomposited - * screen as the desired frame. The update covers the rect (rectX, rectY, - * rectWidth, rectHeight) in surface-local coordinates; contentFingerprint - * identifies the surface's full content after the update. + * screen as the desired frame (see [GlassesSessionCore.submitSurfaceFrame]). * * pixels8bpp arrives as a ByteBuffer because NativeScript marshals a JS * ArrayBuffer to one without the per-element bridge copy that a byte[] @@ -1239,13 +328,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre contentFingerprint, paintMs, frameId, null) } - /** - * As above, with the frame's glyph draws (see SurfaceCompositor's glyph - * overload for the buffer format): the surface's full text content as - * structured draws, letting the resource-cache planner ship glyphs as - * on-glasses cached draws instead of pixels. Null when the submitter has - * no glyph metadata; the pixels alone remain fully correct. - */ + /** As above, with the frame's glyph draws (null when the submitter has no glyph metadata). */ fun submitSurfaceFrame( pixels8bpp: java.nio.ByteBuffer, surfaceId: String, @@ -1258,2759 +341,28 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre frameId: Int, glyphs: java.nio.ByteBuffer? ) { - Log.i(TAG, "Received an updated frame for surface $surfaceId") - FrameTimings.getInstance().log(frameId, "surface " + surfaceId + " updated rect=" - + rectWidth + "x" + rectHeight + "+" + rectX + "+" + rectY - + (if (glyphs == null) " (no glyph draws)" else "")) - FrameTimings.getInstance().spanStart(frameId, "composite") - val composite = compositor.applyAndComposite( - surfaceId, AndroidByteReader(pixels8bpp), rectX, rectY, rectWidth, rectHeight, contentFingerprint, - if (glyphs == null) null else AndroidByteReader(glyphs)) - FrameTimings.getInstance().spanEnd(frameId, "composite") - // Pack the composited 8bpp buffer down to the headerless 4bpp frame - // format the wire planners consume; BMP framing is added later only for - // the uncompressed fallback. - FrameTimings.getInstance().spanStart(frameId, "pack-4bpp") - val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) - FrameTimings.getInstance().spanEnd(frameId, "pack-4bpp") - storeDesiredComposite(composite, packed, paintMs, frameId) + core.submitSurfaceFrame(AndroidByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, + contentFingerprint, paintMs, frameId, if (glyphs == null) null else AndroidByteReader(glyphs)) } - /** Store a composite as the desired frame unless a newer one won the race. */ - private fun storeDesiredComposite(composite: SurfaceCompositor.Composite, packed: ByteArray, paintMs: Int, frameId: Int) { - var supersededFrameId = 0 - var stale = false - synchronized(desiredTilesLock) { - if (composite.seq <= lastStoredCompositeSeq) { - // A concurrent submission composited after us and stored first; - // its composite already includes this surface update. - stale = true - } else { - lastStoredCompositeSeq = composite.seq - supersededFrameId = desiredFrameId - desiredPacked = packed - desiredWidth = composite.width - desiredHeight = composite.height - desiredFingerprint = composite.fingerprint - desiredPaintMs = paintMs - desiredFrameId = frameId - desiredDraws = composite.draws - desiredShellScene = composite.shellScene - } - } - if (stale) { - finishFrame(frameId, "discarded: composite superseded before store") - return - } - if (supersededFrameId != 0 && supersededFrameId != frameId) { - finishFrame(supersededFrameId, "discarded: superseded by frame#$frameId before send") - } - FrameTimings.getInstance().log(frameId, "image submitted as desired frame") - interruptibleSleep.interrupt() - } - - /** - * Play a tone sequence via CFW load_image_z mode 5 kind 4. The payload is - * the complete wire buffer ([5][4][nSteps][freqLo,freqHi,duty,msLo,msHi]*n, - * up to 48 steps), built on the TS side; it rides the arbitrary-payload - * image path like the other mode-5 controls. - */ + /** Play a tone sequence via CFW load_image_z mode 5 kind 4 (payload built on the TS side). */ fun playBuzzerSequence(payload: java.nio.ByteBuffer?) { - synchronized(lock) { - if (!running || !sessionReady || !fixedLayoutCreated) { - logLine("skip buzzer sequence; session not ready") - return - } - val bytes = ByteArray(if (payload == null) 0 else payload.remaining()) - if (payload != null) { - payload.get(bytes) - } - if (bytes.size < 3) { - logLine("skip buzzer sequence; empty payload") - return - } - val message = messageBuilder.imagePayload( - DASHBOARD_TILE, - nextMapSessionId(), - bytes, - "buzzer sequence " + bytes.size + "B", - connectionOptions.sendImagesToLeft - ) - message.onTimeout = MessageCallback { - handleTransportFailure("buzzer sequence ack timeout") - } - pendingMessages.addLast(message) - logLine("queue " + message.label) + val bytes = ByteArray(if (payload == null) 0 else payload.remaining()) + if (payload != null) { + payload.get(bytes) } - interruptibleSleep.interrupt() + core.playBuzzerSequence(bytes) } - fun sendShutdown(exitMode: Int): Boolean { - return sendShutdownInternal(exitMode, true) - } - - /** - * Send CFW image-handler mode 11 after quiescing normal traffic. A successful - * return means the cleanup was ACKed and no later Faceclaw message should be - * emitted before closing BLE. Unsupported/older CFWs return false so callers - * can use the legacy shutdown-and-lease-release path. - */ - fun sendCfwCleanup(): Boolean { - val magic: Int - synchronized(lock) { - if (cfwCleanupDelivered) { - return true - } - if (!customFirmwareDetected || !running || !sessionReady - || shutdownRequested || !fixedLayoutCreated) { - logLine("skip CFW cleanup; mode 11 unavailable or image path not ready") - return false - } - - /* Stop auto-renewals, heartbeats, image generation, and control - * retries, then discard everything that has not reached BLE yet. */ - shutdownRequested = true - lastCfwCleanupAckMagic = 0 - clearPendingMessagesLocked("CFW cleanup requested") - logLine("quiescing transport for CFW cleanup") - } - interruptibleSleep.interrupt() - - /* WINDOW_SIZE can exceed one, so merely appending cleanup would allow it - * to overlap previously-written image fragments. Wait until all of those - * ACK or time out; shutdownRequested prevents the drive loop from adding - * any fresh automatic traffic meanwhile. */ - val drainDeadline = SystemClock.elapsedRealtime() + CFW_CLEANUP_WAIT_MS - synchronized(lock) { - while (running && sessionReady && !inFlightMessages.isEmpty()) { - val remaining = drainDeadline - SystemClock.elapsedRealtime() - if (remaining <= 0) break - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - if (!running || !sessionReady || !inFlightMessages.isEmpty()) { - shutdownRequested = false - logLine("CFW cleanup could not drain prior traffic") - return false - } - - /* External producers are expected to be stopped by the caller, but - * clear once more at the barrier so cleanup is definitely last. */ - clearPendingMessagesLocked("CFW cleanup barrier") - val message = messageBuilder.cfwCleanup( - DASHBOARD_TILE, - nextMapSessionId(), - connectionOptions.sendImagesToLeft - ) - magic = message.magic - message.onAck = MessageCallback { - lastCfwCleanupAckMagic = message.magic - cfwCleanupDelivered = true - compassMaybeOn = false - faceclawWakeLeaseEnabled = false - logLine("CFW cleanup completed") - } - message.onTimeout = MessageCallback { logLine("CFW cleanup ack timeout") } - pendingMessages.addLast(message) - logLine("queue CFW cleanup") - } - interruptibleSleep.interrupt() - - val deadline = SystemClock.elapsedRealtime() + CFW_CLEANUP_WAIT_MS - synchronized(lock) { - while (running - && sessionReady - && lastCfwCleanupAckMagic != magic - && hasPendingOrInflightMagicLocked(magic)) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) break - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - val acked = lastCfwCleanupAckMagic == magic - if (!acked) shutdownRequested = false - return acked - } - } - - /** - * End the EvenHub page while retaining both arm GATT connections and all - * notification subscriptions. A missed ACK is intentionally non-fatal: - * reconnecting Bluetooth here would defeat the power-saving mode. - */ - fun suspendEvenHubSession(): Boolean { - // Ending the plugin task releases the fb lease, which frees the - // on-glasses resource cache; forget it phone-side either way (a lost - // ack may still have taken effect). - synchronized(lock) { - resourceCache.reset() - } - if (sendShutdownInternal(0, false)) { - return true - } - synchronized(lock) { - // A shutdown ACK can be lost even though the command took effect. - // If the transport remains intentionally quiesced, callers still - // need to remember to run the resume path on the next wake. - return running && sessionReady && shutdownRequested - } - } - - /** - * Start a fresh EvenHub plugin task on the existing BLE transport, then let - * the session driver create the layout, warm up the image path, and send - * the desired frame. - */ - fun resumeEvenHubSession(): Boolean { - var claimGeneration = 0 - synchronized(lock) { - if (!running || !sessionReady || chargingMode) { - logLine("skip EvenHub resume; transport not ready") - return false - } - if (!shutdownRequested) { - return true - } - if (faceclawWakePendingNonce >= 0 - && hasPendingOrInflightKindLocked("wake-lease-control")) { - claimGeneration = faceclawWakeControlGeneration - } - logLine("replaying session prelude for EvenHub resume") - } - - // A custom double-tap wake has only a short unclaimed fail-open - // deadline. Let the worker put CLAIM on both arms before the direct - // prelude write begins. - if (claimGeneration != 0) { - waitForFaceclawWakeControlDelivery(claimGeneration, 500) - } - - try { - // Empty-name Cmd=9 tears down the whole plugin task, not just its - // image container. Re-run the launch prelude before Cmd=0 CREATE. - sendPrelude(true) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - handleTransportFailure("EvenHub resume prelude interrupted") - return false - } catch (t: Throwable) { - logLine("EvenHub resume prelude failed: " + safeMessage(t)) - handleTransportFailure("EvenHub resume prelude failed") - return false - } - - synchronized(lock) { - if (!running || !sessionReady || chargingMode) { - return false - } - shutdownRequested = false - fixedLayoutCreated = false - startupProbePending = false - audioCaptureActive = false - clearAllMessagesPreservingWakeLeaseLocked("EvenHub resume") - displayedFingerprint = "" - imageRetryAfterMs = 0 - lastHeartbeatSentAtMs = 0 - lastHeartbeatAckedAtMs = 0 - logLine("EvenHub session resume requested") - } - interruptibleSleep.interrupt() - return true - } - - private fun sendShutdownInternal(exitMode: Int, reconnectOnTimeout: Boolean): Boolean { - val magic: Int - val startedAtMs = SystemClock.elapsedRealtime() - synchronized(lock) { - if (!running || !sessionReady) { - logLine("skip shutdown; session not ready") - return false - } - if (shutdownRequested) { - return true - } - shutdownRequested = true - // Magic values wrap, so an ACK from a much older suspend must not - // satisfy this request after enough sleep/wake cycles. - lastShutdownAckMagic = 0 - clearPendingMessagesLocked("shutdown requested") - val message = messageBuilder.shutdown(exitMode) - magic = message.magic - message.onAck = MessageCallback { - lastShutdownAckMagic = message.magic - fixedLayoutCreated = false - displayedFingerprint = "" - } - message.onTimeout = MessageCallback { - if (reconnectOnTimeout) { - handleTransportFailure("shutdown ack timeout") - } else { - logLine("EvenHub shutdown ack timeout; keeping BLE connected") - } - } - pendingMessages.addFirst(message) - logLine("queue shutdown") - // The stock compass keeps the magnetometer sampling independently of - // the plugin task, so ending the page does not stop it. Force a - // disable ahead of the shutdown command whenever it may be running: - // this also covers a disable that was wiped by the queue flush above - // or whose ack was lost, and the charging-mode/exit paths where the - // Compass window never got a chance to release it. - if (compassMaybeOn && fixedLayoutCreated) { - enqueueCompassControlLocked(true, false) - } - } - interruptibleSleep.interrupt() + fun sendShutdown(exitMode: Int): Boolean = core.sendShutdown(exitMode) - val ackDeadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + 500 - synchronized(lock) { - while (running - && sessionReady - && lastShutdownAckMagic != magic - && lastShutdownExitAtMs < startedAtMs - && hasPendingOrInflightMagicLocked(magic)) { - val remaining = ackDeadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - val acked = lastShutdownAckMagic == magic - if (acked) { - val exitDeadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + 500 - while (running && sessionReady && lastShutdownExitAtMs < startedAtMs) { - val remaining = exitDeadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - } - return acked || lastShutdownExitAtMs >= startedAtMs - } - } - - private fun waitForAudioControlAck(magic: Int, operation: String): Boolean { - val deadline = SystemClock.elapsedRealtime() + ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 500 - synchronized(lock) { - while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - val acked = lastAudioControlAckMagic == magic - if (!acked) { - if ("enable" == operation) { - audioPacketListener = null - audioCaptureActive = false - } - logLine("G2 mic $operation ack timeout") - } - return acked - } - } - - - override fun run() { - logLine(String.format(Locale.US, "communicator start R=%s L=%s ring=%s", rightAddress, leftAddress, ringAddress)) - while (true) { - try { - if (!running) { - Log.w(TAG, "Exiting event looop") - break - } - emitPhoneLockStateIfChanged(false) - if (!sessionReady) { - if (reconnectHalted) { - interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS.toLong()) - continue - } - val now = SystemClock.elapsedRealtime() - if (now < reconnectAfterMs) { - interruptibleSleep.sleep(Math.min(ConnectionOptions.IDLE_SLEEP_MS.toLong(), reconnectAfterMs - now)) - continue - } - Log.w(TAG, "Attempting to connect") - connectLoopOnce() - continue - } - - if (shouldAttemptRingConnect()) { - tryConnectRing("retry") - continue - } - - val sleepMs = driveSession() - if (sleepMs > 0) { - interruptibleSleep.sleep(sleepMs) - } - } catch (t: Throwable) { - logLine("communicator loop error: " + safeMessage(t)) - handleTransportFailure("loop error") - } - } - logLine("communicator stop") - } - - override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { - if (address == null || characteristicUuid == null || data == null) { - return - } - val uuid = characteristicUuid.lowercase(Locale.US) - if (isDirectRingNotification(address, uuid)) { - handleDirectRingNotification(uuid, data) - return - } - if (BleProtocol.RENDER_NOTIFY_UUID == uuid) { - handleRenderNotification(address, data) - return - } - if (BleProtocol.NOTIFY_CHAR_UUID != uuid) { - return - } - if (data.size >= 7 && (data[6].toInt() and 255) == CfwTransport.SID) { - val acks = CfwTransport.parseAcks(data) ?: return - synchronized(lock) { - lastIncomingAtMs = SystemClock.elapsedRealtime() - for (ack in acks) { - for (message in inFlightMessages) { - val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress - if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) { - message.acceptCfwAck(ack) - Log.i(TAG, "CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId - + " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2].toInt() and 255) - + " redundant=" + (ack !== acks[0]) - + " size=" + ack.size + " crc=" + ack.checksum - + " expected=" + message.message.size + "/" + message.cfwChecksum - + " ackedLenses=" + message.cfwAckLenses - + " ageMs=" + (lastIncomingAtMs - message.sentAtMs)) - message.ackPayload = Arrays.copyOf(data, data.size) - break - } - } - } - drainCfwAcknowledgementsLocked() - } - interruptibleSleep.interrupt() - return - } - Log.d(TAG, "onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size) - val frame = BleProtocol.parseFrame(data) - val decodedWearState = BleProtocol.parseWearState(frame) - val compassEvent = if (address.equals(rightAddress, ignoreCase = true)) - BleProtocol.parseCompassEvent(frame) - else - null - var emitWearState = false - var event: G2Event? = null - synchronized(lock) { - lastIncomingAtMs = SystemClock.elapsedRealtime() - if (decodedWearState >= 0 && decodedWearState != wearState) { - wearState = decodedWearState - emitWearState = true - } - var faceclawWakeNotification = false - if (shutdownRequested - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING - && address.equals(rightAddress, ignoreCase = true)) { - val wakeNonce = BleProtocol.parseFaceclawWakeEvent(frame.pb) - if (wakeNonce >= 0) { - faceclawWakePendingNonce = wakeNonce - enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_CLAIM, - wakeNonce, - true - ) - faceclawWakeNotification = true - lastConnectionOrInputAtMs = lastIncomingAtMs - // The event type says which gesture woke the glasses: TS - // gives a head-up the Glanceboard and a double tap the - // regular UI. The claim handshake is the same for both. - val headUp = BleProtocol.parseFaceclawWakeEventCode(frame.pb) == - BleProtocol.FACECLAW_WAKE_EVENT_HEAD_UP - event = G2Event( - "display-wake", - "", - if (headUp) BleProtocol.EVENT_HEAD_UP else BleProtocol.EVENT_DOUBLE_CLICK, - 0, - 0 - ) - logLine("claimed deferred dashboard wake nonce=" + wakeNonce - + (if (headUp) " (head-up)" else "")) - } - } - if (!faceclawWakeNotification - && shutdownRequested - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING - && address.equals(rightAddress, ignoreCase = true)) { - // CFW idle-gesture forwarding (firmware revision 2+): with no - // EvenHub page the stock display thread drops taps, long - // presses and releases; the CFW reports them on the settings - // sid while our wake lease is held. Deliver them as the - // sys-events a live page would have produced, so TS treats a - // sleep-time tap or hold exactly like one during soft sleep - // (the Glanceboard). - val gesture = BleProtocol.parseFaceclawGestureEvent(frame.pb) - if (gesture != null) { - lastConnectionOrInputAtMs = lastIncomingAtMs - event = G2Event("sys-event", "", gesture.eventType, gesture.eventSource, 0) - logLine("idle gesture forwarded by CFW: type=" + gesture.eventType - + " source=" + gesture.eventSource) - } - } - if (!faceclawWakeNotification - && event == null - && shutdownRequested - && address.equals(rightAddress, ignoreCase = true) - && BleProtocol.isDisplayWakeStateChange(frame)) { - // With no EvenHub page, ring/arm double-taps are handled by the - // stock display lifecycle and surface only as this state ping. - // Translate it back into an input event so TS can wake the shell - // and recreate the page. - event = G2Event( - "display-wake", - "", - BleProtocol.EVENT_DOUBLE_CLICK, - 0, - 0 - ) - } - if (!faceclawWakeNotification - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING) { - // Mode-17 query notifications; settings READs are handled by - // createBatteryQueryMessageLocked so headset/ring update together. - if (address.equals(rightAddress, ignoreCase = true) - && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { - val ring = BleProtocol.parseRingBattery(frame.pb) - if (ring != null) { - ringBattery = ring.battery - ringCharging = ring.charging - emitBatteryState(headsetBattery, headsetCharging) - } - } - // CFW mic status (field 104) rides both standalone pushes and - // settings read acks, from each temple on its own link. - val micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb) - if (micStatus != null) { - emitMicStatus(micStatus, address) - } - } - if (!faceclawWakeNotification - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING) { - // CFW ambient-light report (field 105) from the master temple. - val alsReport = BleProtocol.parseFaceclawAlsReport(frame.pb) - if (alsReport != null) { - emitAmbientLight(alsReport) - } - } - if (!faceclawWakeNotification - && decodedWearState < 0 - && frame.ok - && frame.sid == BleProtocol.SID_UI_SETTING) { - // Device-initiated settings push. It carries a magic the glasses - // chose, so it would otherwise fall through to resolveAckLocked, - // match nothing, and be logged as an unexpected ack. - val pushedSilentMode = BleProtocol.parseSilentModePush(frame.pb) - if (pushedSilentMode >= 0) { - updateSilentModeLocked(pushedSilentMode > 0) - return - } - } - if (!faceclawWakeNotification - && decodedWearState < 0 - && frame.ok - && frame.msgSeq >= 0 - && frame.flag != BleProtocol.FLAG_NOTIFY - && frame.flag != BleProtocol.FLAG_NOTIFY_ALT) { - lastAckAtMs = lastIncomingAtMs - resolveAckLocked(frame.sid, frame.msgSeq, frame.pb) - } - if (!faceclawWakeNotification - && event == null - && frame.ok - && address.equals(rightAddress, ignoreCase = true) - && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { - var decoded = G2Event.decode(frame) - if (decoded != null - && "sys-event" == decoded.kind - && decoded.eventType == BleProtocol.EVENT_HEAD_UP) { - // CFW forwards the IMU head-up while our page is on screen - // (soft sleep). Surface it as the same wake-only input the - // deferred head-up wake produces from a dark display. - decoded = G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, decoded.eventSource, 0) - logLine("head-up forwarded by CFW while page on screen") - } - event = decoded - if (decoded != null) { - // Pure IMU samples arrive continuously; don't let them count - // as user input (which would starve battery polling). - val pureImuSample = "sys-event" == decoded.kind - && decoded.eventType == BleProtocol.EVENT_IMU_DATA_REPORT - if (!pureImuSample) { - lastConnectionOrInputAtMs = lastIncomingAtMs - } - if ("list-click" == decoded.kind || "text-click" == decoded.kind) { - // Container-routed touchpad input reached us, so the - // firmware is dispatching input: silent mode is off, - // whether or not its end-of-silent push arrived. - updateSilentModeLocked(false) - } - if ("sys-event" == decoded.kind) { - if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT || decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) { - if (shutdownRequested) { - lastShutdownExitAtMs = SystemClock.elapsedRealtime() - } - fixedLayoutCreated = false - displayedFingerprint = "" - clearAllMessagesLocked("firmware exit event") - } - } - } - } - } - interruptibleSleep.interrupt() - if (emitWearState) { - logLine(if (decodedWearState > 0) "wear state ON_HEAD" else "wear state OFF_HEAD") - emitWearState(decodedWearState > 0) - } - if (compassEvent != null) { - emitCompassEvent(compassEvent) - } - val finalEvent = event - if (finalEvent != null) { - if (finalEvent.hasImu) { - emitImuData(finalEvent.imuX, finalEvent.imuY, finalEvent.imuZ, finalEvent.eventSource) - } - // A standalone IMU_DATA_REPORT is a sensor sample, not a gesture: - // deliver it only to IMU listeners, skipping the input pipeline (and - // its per-frame latency bookkeeping) to avoid flooding it. - val pureImuSample = "sys-event" == finalEvent.kind - && finalEvent.eventType == BleProtocol.EVENT_IMU_DATA_REPORT - if (!pureImuSample) { - val frameId = FrameTimings.getInstance().startFrame( - "input:" + finalEvent.kind + " type=" + finalEvent.eventType + " src=" + finalEvent.eventSource) - FrameTimings.getInstance().log(frameId, "input event decoded from BLE notification") - emitRingEvent(finalEvent, frameId) - } - } - } - - private fun handleDirectRingNotification(characteristicUuid: String, data: ByteArray) { - val decoded = FaceclawRingEventDecoder.decode(data) - if (decoded == null) { - Log.d(TAG, "direct ring notify ignored: characteristicUuid=" + characteristicUuid + " raw=" + hex(data)) - return - } + fun sendCfwCleanup(): Boolean = core.sendCfwCleanup() - val event = decoded.event - val arrivalMs = SystemClock.elapsedRealtime() - synchronized(lock) { - lastIncomingAtMs = arrivalMs - lastConnectionOrInputAtMs = arrivalMs - } - logLine("direct ring " + decoded.label + " " + decoded.detail + " raw=" + hex(data)) - val frameId = FrameTimings.getInstance().startFrame("input:ring:" + decoded.label) - FrameTimings.getInstance().log(frameId, "input event decoded from direct ring notification") - emitRingEvent(event, frameId) - interruptibleSleep.interrupt() - } - - private fun handleRenderNotification(address: String, data: ByteArray) { - val listenerToCall: FaceclawAudioPacketListener? - val arrivalMs = SystemClock.elapsedRealtime() - synchronized(lock) { - lastIncomingAtMs = arrivalMs - listenerToCall = if (audioCaptureActive) audioPacketListener else null - } - if (listenerToCall == null) { - return - } - val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?" - try { - listenerToCall.onAudioPacket(Arrays.copyOf(data, data.size), arm, arrivalMs) - } catch (t: Throwable) { - logLine("G2 mic packet listener failed: " + safeMessage(t)) - } - } - - override fun onConnectionStateChange(address: String?, connected: Boolean) { - synchronized(lock) { - if (address == null) { - return - } - if (isConfiguredRingAddress(address)) { - ringConnected = connected - ringNotificationsReady = false - if (!connected) { - ringReconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RING_RECONNECT_DELAY_MS - } - logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected") - return - } - if (address.equals(rightAddress, ignoreCase = true)) { - rightConnected = connected - } else if (address.equals(leftAddress, ignoreCase = true)) { - leftConnected = connected - } else { - return - } - if (!connected) { - sessionReady = false - fixedLayoutCreated = false - startupProbePending = false - chargingMode = false - audioCaptureActive = false - audioPacketListener = null - clearAllMessagesLocked("connection lost") - displayedFingerprint = "" - if (!reconnectHalted) { - reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS - } - } - } - interruptibleSleep.interrupt() - if (connected) { - setStateDisplay("connected", "Connected.") - } else if (!reconnectHalted) { - // While parked on a missing bond, keep the "unpaired" display: this - // callback is just the teardown of the arm that did connect. - setStateDisplay("connecting", "Connecting to the glasses...") - } - } - - @Throws(InterruptedException::class) - private fun connectLoopOnce() { - setStateDisplay("connecting", "Connecting to the glasses...") - try { - connectArm(rightAddress, true) - connectArm(leftAddress, true) - if (!sleepDuringConnectSettling(800)) { - return - } - authenticateArms() - sendPrelude() - - synchronized(lock) { - sessionReady = true - // A fresh transport prelude always starts an active EvenHub - // lifecycle, even if the previous connection dropped while - // its page was intentionally suspended. - shutdownRequested = false - fixedLayoutCreated = false - clearAllMessagesLocked("session ready") - displayedFingerprint = "" - lastAckAtMs = SystemClock.elapsedRealtime() - lastIncomingAtMs = lastAckAtMs - lastConnectionOrInputAtMs = lastAckAtMs - lastSessionReadyAtMs = lastAckAtMs - lastBatteryRefreshAtMs = 0 - imageRetryAfterMs = 0 - lastHeartbeatSentAtMs = 0 - lastHeartbeatAckedAtMs = 0 - consecutiveAckTimeouts = 0 - lastAudioControlAckMagic = 0 - audioCaptureActive = false - faceclawWakePendingNonce = -1 - cfwCleanupDelivered = false - for (transport in cfwTransports) transport.reset() - lastCfwCleanupAckMagic = 0 - lastFaceclawWakeLeaseQueuedAtMs = 0 - lastFaceclawFramebufferLeaseQueuedAtMs = 0 - enqueueFaceclawFramebufferControlLocked( - BleProtocol.FACECLAW_FB_OP_ACQUIRE, - true - ) - if (faceclawWakeLeaseEnabled) { - enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, - 0, - true - ) - } - } - setStateDisplay("connected", "Connected.") - logLine("session ready") - synchronized(lock) { - // Query settings promptly on the first session so firmware - // version/extension (and battery) arrive without waiting for - // the input-quiet battery poll. The settings response doubles as - // the firmware-compatibility check surfaced during onboarding. - if (!firmwareInfoQueried) { - firmwareInfoQueried = true - lastBatteryRefreshAtMs = SystemClock.elapsedRealtime() - pendingMessages.addLast(createBatteryQueryMessageLocked()) - logLine("queue settings query for firmware info") - } - } - tryConnectRing("initial") - } catch (t: Throwable) { - logLine("connect failed: " + safeMessage(t)) - val unpairedArm = firstUnpairedArm() - if (unpairedArm != null) { - handleUnpairedFailure(unpairedArm) - } else { - handleTransportFailure("connect failed") - } - } - } - - @Throws(InterruptedException::class) - private fun sleepDuringConnectSettling(delayMs: Long): Boolean { - val deadline = SystemClock.elapsedRealtime() + delayMs - synchronized(lock) { - while (running && !userDisconnectRequested) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - return true - } - lock.wait(Math.min(remaining, 100L)) - } - return false - } - } - - private fun connectArm(address: String, enableRenderNotify: Boolean) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw IllegalStateException("connect failed: $address") - } - // requestConnectionPriority has no callback in this Android compile target, so there is - // no reliable completion point to keep it in the global GATT operation pipeline. But it's - // important enough for performance that we call it anyways. - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) - - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) - - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw IllegalStateException("discoverServices failed: $address") - } - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - throw IllegalStateException("enableNotifications failed: " + address + " " + BleProtocol.NOTIFY_CHAR_UUID) - } - if (enableRenderNotify) { - bleManager.enableNotifications(address, BleProtocol.RENDER_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS) - } - synchronized(lock) { - if (address.equals(rightAddress, ignoreCase = true)) { - rightConnected = true - } else if (address.equals(leftAddress, ignoreCase = true)) { - leftConnected = true - } - } - } - - private fun shouldAttemptRingConnect(): Boolean { - if (!hasRingAddress()) { - return false - } - val now = SystemClock.elapsedRealtime() - synchronized(lock) { - return running - && sessionReady - && !ringNotificationsReady - && now >= ringReconnectAfterMs - && pendingMessages.isEmpty() - && inFlightMessages.isEmpty() - } - } - - private fun tryConnectRing(reason: String) { - if (!hasRingAddress()) { - return - } - try { - connectRing() - } catch (t: Throwable) { - synchronized(lock) { - ringConnected = false - ringNotificationsReady = false - ringReconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RING_RECONNECT_DELAY_MS - } - logLine("direct ring connect failed (" + reason + "): " + safeMessage(t)) - } - } - - private fun connectRing() { - logLine("connecting direct ring $ringAddress") - if (!bleManager.connect(ringAddress, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw IllegalStateException("connect failed: $ringAddress") - } - - bleManager.requestConnectionPriority(ringAddress, BluetoothGatt.CONNECTION_PRIORITY_HIGH) - bleManager.requestMtu(ringAddress, ConnectionOptions.RING_DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) - - if (!bleManager.discoverServices(ringAddress, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw IllegalStateException("discoverServices failed: $ringAddress") - } - - val phoneNotify = enableRingNotification(BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID) - val dataNotify = enableRingNotification(BleProtocol.R1_NOTIFY_CHAR_UUID) - if (!phoneNotify && !dataNotify) { - throw IllegalStateException("no R1 notify characteristic subscribed") - } - - synchronized(lock) { - ringConnected = true - ringNotificationsReady = true - ringReconnectAfterMs = 0 - } - logLine("direct ring ready phoneNotify=$phoneNotify dataNotify=$dataNotify") - } - - private fun enableRingNotification(characteristicUuid: String): Boolean { - try { - return bleManager.enableNotifications( - ringAddress, - characteristicUuid, - true, - ConnectionOptions.DESCRIPTOR_TIMEOUT_MS - ) - } catch (t: Throwable) { - Log.d(TAG, "direct ring notify subscribe skipped: " + characteristicUuid + " " + safeMessage(t)) - return false - } - } - - /** - * Complete the sid-0x80 security-auth exchange on both freshly opened arm - * connections. Firmware 2.2.9 answers no queries until it completes over an - * encrypted link and closes unauthenticated links after ~30 s (see - * ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is - * also what triggers SMP pairing. Deliberately soft: on timeout we log and - * continue rather than fail the connect — the custom firmware's response - * behavior is not yet hardware-verified, and on stock firmware an - * unanswered auth just means the prelude fails exactly as it did before. - * A pairing prompt accepted after our window still bonds at the OS level, - * so the next reconnect attempt authenticates promptly. - */ - @Throws(InterruptedException::class) - private fun authenticateArms() { - val right = messageBuilder.securityAuth(false) - val left = messageBuilder.securityAuth(true) - val now = SystemClock.elapsedRealtime() - for (message in arrayOf(right, left)) { - message.onAck = MessageCallback { - } - message.onTimeout = MessageCallback { - } - message.sentAtMs = now - writeMessage(message) - } - val deadline = SystemClock.elapsedRealtime() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS - while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { - synchronized(lock) { - if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { - break - } - } - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - interruptibleSleep.sleep(Math.min(remaining, 100L)) - } - synchronized(lock) { - if (!inFlightMessages.isEmpty()) { - clearInFlightMessagesLocked("security auth timeout") - logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)") - return - } - } - val rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic) - val leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic) - logLine("security auth R=" + (if (rightOk) "ok" else "unconfirmed") + " L=" + (if (leftOk) "ok" else "unconfirmed")) - } - - @Throws(InterruptedException::class) - private fun sendPrelude() { - sendPrelude(false) - } - - @Throws(InterruptedException::class) - private fun sendPrelude(preserveWakeLeaseControls: Boolean) { - synchronized(lock) { - if (preserveWakeLeaseControls) { - clearAllMessagesPreservingWakeLeaseLocked("prelude") - } else { - clearAllMessagesLocked("prelude") - } - } - val now = SystemClock.elapsedRealtime() - val prelude = messageBuilder.prelude() - prelude.onAck = MessageCallback { - } - prelude.onTimeout = MessageCallback { - handleTransportFailure("ack timeout") - } - prelude.sentAtMs = now - writeMessage(prelude) - - val deadline = SystemClock.elapsedRealtime() + ConnectionOptions.PRELUDE_TIMEOUT_MS - while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { - synchronized(lock) { - if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { - break - } - } - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - interruptibleSleep.sleep(Math.min(remaining, 100L)) - } - synchronized(lock) { - if (!inFlightMessages.isEmpty()) { - clearInFlightMessagesLocked("prelude timeout") - throw IllegalStateException("prelude ack timeout") - } - } - } - - private fun driveSession(): Long { - //Log.d(TAG, "driveSession called (pendingMessages.size=" + pendingMessages.size() + " inFlightMessages.size=" + inFlightMessages.size() + ")"); - while (true) { - // Run on the sender thread, outside `lock`: Bluetooth API calls can - // wait on the global GATT lock, and callbacks need the session lock. - // Keep negotiation outside the timed stream. A fixed settling period - // is only an experiment boundary; HCI must confirm actual parameters. - var linkMode = -1 - var benchmarkAddresses: Array? = null - synchronized(lock) { - if (benchmarkActive && benchmarkLinkPending) { - benchmarkLinkPending = false - linkMode = benchmarkLinkMode - benchmarkAddresses = arrayOf(leftAddress, rightAddress) - } - } - val addressesToPrepare = benchmarkAddresses - if (addressesToPrepare != null) { - for (address in addressesToPrepare) { - try { - bleManager.prepareBenchmarkLink(address, linkMode) - } catch (error: RuntimeException) { - logLine("benchmark link request failed: " + safeMessage(error)) - } - } - synchronized(lock) { - // Cancellation/new-run races leave the new run pending; its - // own preparation will replace this deadline before sending. - benchmarkReadyAtMs = SystemClock.elapsedRealtime() + 1_000 - } - } - var messageToWrite: OutboundMessage? = null - var messageToPrewrite: OutboundMessage? = null - val now = SystemClock.elapsedRealtime() - - maybeFinishNoChangeDesiredFrame() - - synchronized(lock) { - drainCfwAcknowledgementsLocked() - if (cfwCleanupDelivered) { - /* Successful mode 11 must be the last Faceclaw write. Drop - * anything a late external producer attempted to enqueue - * while DashboardController was closing the transport. */ - clearPendingMessagesLocked("after CFW cleanup") - return 250 - } - if (sessionReady - && now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS - && !hasPendingOrInflightKindLocked("framebuffer-lease-control")) { - enqueueFaceclawFramebufferControlLocked( - BleProtocol.FACECLAW_FB_OP_ACQUIRE, - false - ) - } - if (faceclawWakeLeaseEnabled - && sessionReady - && now - lastFaceclawWakeLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS - && !hasPendingOrInflightKindLocked("wake-lease-control")) { - enqueueFaceclawWakeControlLocked( - BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, - 0, - false - ) - } - if (!inFlightMessages.isEmpty()) { - val oldest = inFlightMessages.peekFirst() - val replay = CfwMessageWindow.replayWindow(inFlightMessages, now) - if (!replay.isEmpty()) { - logLine("CFW recovery: replay " + replay.size - + " unresolved message(s) from id=" + replay[0].magic) - for (candidate in replay) { - if (candidate.cfwRetries >= CfwMessageWindow.MAX_RETRIES) { - handleTransportFailure("CFW recovery retry limit") - return 0 - } - logLine("CFW replay id=" + candidate.magic + " label=" + candidate.label - + " cause=" + (if (candidate.cfwRetryPending) "NACK" - else if (candidate.cfwAckLenses == CfwTransport.BOTH) "ordered tail" else "missing ACK") - + " ackedLenses=" + candidate.cfwAckLenses - + " ageMs=" + (now - candidate.sentAtMs)) - inFlightMessages.remove(candidate) - magicPool.release(candidate.sid, candidate.magic, candidate.label, "CFW replay") - candidate.prepareCfwReplay(magicPool.allocate()) - } - for (i in replay.indices.reversed()) pendingMessages.addFirst(replay[i]) - return 0 - } - if (oldest != null && oldest.ackDeadlineAtMs <= now) { - Log.i(TAG, "message timed out: " + oldest.label) - inFlightMessages.removeFirst() - logLine("message timed out: " + oldest.label + " sid=" + oldest.sid - + " id=" + oldest.magic + " ackedLenses=" + oldest.cfwAckLenses - + " ageMs=" + (now - oldest.sentAtMs)) - magicPool.release(oldest.sid, oldest.magic, oldest.label, "timeout") - if (oldest.sid == CfwTransport.SID) - cfwTransports[if (oldest.isLeftArmMessage) 0 else 1].reset() - handleAckTimeoutLocked(oldest) - return 0 - } - if (connectionOptions.WINDOW_SIZE <= 1 - && sessionReady - && prewrittenMessage == null - && !pendingMessages.isEmpty() - && canPrewriteCandidate(pendingMessages.peekFirst()) - && !shouldBlockPrewriteForHeartbeatLocked(now)) { - // Only the serial (window==1) path pre-sends the all-but-last - // packet; with a real window we just send the next message fully. - messageToPrewrite = pendingMessages.peekFirst() - } - } - - if (chargingMode) { - // Glasses are in the case: no display traffic, only battery - // polls (which also detect the end of charging). - finishDesiredFrameLocked("discarded: glasses charging") - if (sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) { - messageToWrite = pendingMessages.removeFirst() - Log.i(TAG, "sending pending message (charging): " + messageToWrite!!.label) - } else if (sessionReady && pendingMessages.isEmpty() && inFlightMessages.isEmpty() - && now - lastBatteryRefreshAtMs >= ConnectionOptions.CHARGING_BATTERY_POLL_MS) { - Log.i(TAG, "Writing charging-mode battery poll") - messageToWrite = createBatteryQueryMessageLocked() - lastBatteryRefreshAtMs = now - } else { - return 1_000 - } - } else { - if (messageToPrewrite == null - && !shutdownRequested - && !fixedLayoutCreated - && pendingMessages.isEmpty() - && inFlightMessages.isEmpty()) { - Log.i(TAG, "enqueueing create layout") - enqueueCreateLayoutLocked() - } else if (messageToPrewrite != null) { - // Prewrite outside the lock; the logical message remains pending until - // its final BLE frame is sent after the current protocol ACK. - } - if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated - && (if (firmwareDebugFlagsEnabled) 2 else 1) != firmwareDebugFlagsLastSent - && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { - Log.i(TAG, "enqueueing firmware debug flags " + (if (firmwareDebugFlagsEnabled) "show" else "hide")) - enqueueFirmwareDebugFlagsLocked() - } - - if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated - && (if (compassOwners.isEmpty()) 0 else 1) != compassControlLastSent - && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { - val wanted = compassOwners.isNotEmpty() - Log.i(TAG, "enqueueing compass " + (if (wanted) "enable" else "disable")) - enqueueCompassControlLocked(false, wanted) - } - - if (benchmarkActive) { - maintainBenchmarkLocked(now) - } - - // Up to WINDOW_SIZE messages may be in flight at once (full - // pipelining); a slot frees when an ack arrives. An active - // bandwidth benchmark measures its own selected window instead. - val windowHasRoom = inFlightMessages.size < - Math.max(1, if (benchmarkActive) benchmarkWindowSize else connectionOptions.WINDOW_SIZE) - // A frame ready to send right now: don't inject a fresh - // heartbeat in front of it (the image's own ack resets the - // firmware heartbeat timer, so the heartbeat is redundant). - // "Ready" covers both a fresh composite still to be planned - // and a planned image already queued but not yet written -- - // enqueueing sets lastEnqueuedFingerprint, so testing only - // the desired fingerprint left the queued-but-unwritten - // window unprotected and a heartbeat that came due there - // cost the frame a full ack round trip (measured 124ms on - // frame#232 of the 2026-08-20 02:27 capture). - // A benchmark run counts as image traffic here for the same - // reason: its acks reset the firmware heartbeat timer, so a - // fresh heartbeat in front of it is redundant. - val imageWaiting = !shutdownRequested && fixedLayoutCreated - && !hasPendingOrInflightKindLocked("heartbeat") - && (benchmarkActive - || (now >= imageRetryAfterMs - && (hasPendingImageLocked() - || getDesiredFingerprint() != lastEnqueuedFingerprint))) - noteImageStallLocked(now, windowHasRoom) - if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) { - return ConnectionOptions.IDLE_SLEEP_MS.toLong() - } - - if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty() - && CfwMessageWindow.canSend(inFlightMessages, pendingMessages.peekFirst()!!)) { - messageToWrite = pendingMessages.removeFirst() - Log.i(TAG, "sending pending message: " + messageToWrite!!.label) - } else if (messageToPrewrite == null && !shutdownRequested && !benchmarkActive - && fixedLayoutCreated - && windowHasRoom && !hasPendingImageLocked() - && now >= imageRetryAfterMs - && getDesiredFingerprint() != lastEnqueuedFingerprint) { - // Enqueue the next frame's delta against lastEnqueuedPacked - // (what the shadow will be), so it can pipeline behind an - // image still awaiting its ack. - Log.i(TAG, "Enqueued image update") - enqueueDesiredImageLocked() - return 0 - } else if (messageToPrewrite == null && shouldPollBatteryLocked(now)) { - Log.i(TAG, "Writing battery query") - messageToWrite = createBatteryQueryMessageLocked() - lastBatteryRefreshAtMs = now - } else if (messageToPrewrite == null && (!pendingMessages.isEmpty() || !inFlightMessages.isEmpty())) { - return ConnectionOptions.IDLE_SLEEP_MS.toLong() - } else if (messageToPrewrite == null) { - return 250 - } - } - } - - val prewrite = messageToPrewrite - if (prewrite != null) { - if (prewriteMessage(prewrite)) { - return 0 - } - return ConnectionOptions.IDLE_SLEEP_MS.toLong() - } - - val write = messageToWrite!! - if (!writeMessage(write)) { - synchronized(lock) { - if (removePreparedMessageLocked(write) || write.magic == 0) { - handleTransportFailure("write failed") - } - } - return 0 - } - } - } - - /** - * If the desired image already matches what the glasses display, nothing will - * ever be enqueued for it, so finish its frame now (otherwise the TS side - * would block on it until its backpressure timeout). - */ - private fun maybeFinishNoChangeDesiredFrame() { - var frameIdToFinish = 0 - synchronized(lock) { - synchronized(desiredTilesLock) { - if (desiredFrameId != 0 && lastEnqueuedFingerprint.isNotEmpty() && desiredFingerprint == lastEnqueuedFingerprint) { - frameIdToFinish = desiredFrameId - desiredFrameId = 0 - } - } - } - if (frameIdToFinish != 0) { - finishFrame(frameIdToFinish, "discarded: no change from displayed image") - } - } - - private fun shouldBlockPrewriteForHeartbeatLocked(now: Long): Boolean { - if (shutdownRequested || !fixedLayoutCreated) { - return false - } - return hasPendingOrInflightKindLocked("heartbeat") - || now - lastHeartbeatAckedAtMs >= ConnectionOptions.HEARTBEAT_READY_MS - } - - private fun canPrewriteCandidate(message: OutboundMessage?): Boolean { - if (message == null || message.sid == CfwTransport.SID || !message.isLeftArmMessage) { - return false - } - if ("image" != message.kind) { - return false - } - return message.message.size + 2 > 232 - } - - private fun handleHeartbeat(imageWaiting: Boolean): Boolean { - val now = SystemClock.elapsedRealtime() - val heartbeatEligible = !shutdownRequested && fixedLayoutCreated - val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat") - val heartbeatElapsedMs = now - lastHeartbeatAckedAtMs - val heartbeatReady = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS - val heartbeatUrgent = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_URGENT_MS - val heartbeatBlocksLeftWrites = heartbeatReady || heartbeatPending - - if (heartbeatReady && !heartbeatPending && inFlightMessages.isEmpty()) { - if (imageWaiting && !heartbeatUrgent) { - // Defer to the waiting frame: sending it now satisfies the - // firmware heartbeat deadline (its ack resets the timer), and - // the heartbeat still fires once we reach the URGENT threshold - // if rendering goes quiet again. Preserves the pending-heartbeat - // inter-lens-sync invariant below (that path is untouched). - return false - } - Log.i(TAG, "Writing heartbeat") - val heartbeatMessage = createHeartbeatMessage() - lastHeartbeatSentAtMs = now - writeMessage(heartbeatMessage) - return true - } else if (heartbeatUrgent) { - return true - } else if (heartbeatPending) { - // Don't send other message types while a heartbeat is pending because that - // can lead to inter-lens sync issues - return true - } - - return false - } - - private fun createHeartbeatMessage(): OutboundMessage { - val message = messageBuilder.heartbeat() - message.onAck = MessageCallback { - synchronized(lock) { - lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime() - } - } - message.onTimeout = MessageCallback { - // If a heartbeat fails to ack and we're over the heartbeat deadline, assume the connection is failed and reconnect. - // Otherwise ignore it, which will cause a retransmission attempt. - val isPastDeadline: Boolean - synchronized(lock) { - isPastDeadline = SystemClock.elapsedRealtime() - lastHeartbeatSentAtMs >= ConnectionOptions.HEARTBEAT_FAILURE_DEADLINE_MS - } - if (isPastDeadline) { - handleTransportFailure("heartbeat ack timeout") - } - } - return message - } - - - private fun writeMessage(message: OutboundMessage): Boolean { - val now = SystemClock.elapsedRealtime() - message.writeStartedAtMs = now - message.sentAtMs = now - message.ackDeadlineAtMs = now + message.ackTimeoutMs + ConnectionOptions.WRITE_TIMEOUT_MS - if (message.magic != 0) { - synchronized(lock) { - inFlightMessages.addLast(message) - } - } - if (message.imageUpdateId > 0 && message.imageMessageNumber == 1) { - synchronized(lock) { - val stats = imageUpdateStats[message.imageUpdateId] - if (stats != null && stats.firstWriteStartedAtMs <= 0) { - stats.firstWriteStartedAtMs = now - } - } - } - - val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress - val frames: List - if (prewrittenMessage != null && prewrittenMessage !== message) { - if (!spoilPrewrittenMessage("before " + message.label)) { - return false - } - } - if (prewrittenMessage === message) { - frames = Collections.singletonList(prewrittenFrames[prewrittenFrames.size - 1]) - prewrittenMessage = null - prewrittenFrames = Collections.emptyList() - } else if (message.sid == CfwTransport.SID) { - if (message.cfwRetries > 0) cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset() - frames = cfwTransports[if (message.isLeftArmMessage) 0 else 1].encode( - message.message, message.magic, CfwTransport.BOTH, - bleManager.getNegotiatedMtu(writeAddress)) - } else { - frames = BleProtocol.framePb( - message.message, - message.sid, - message.flag, - nextTransportSeq++ - ) - } - val result = bleManager.writeFrames( - writeAddress, - BleProtocol.WRITE_CHAR_UUID, - frames, - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS - ) - - synchronized(lock) { - val sentAtMs = SystemClock.elapsedRealtime() - message.sentAtMs = sentAtMs - message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs - logImageUpdateSendLandmarkLocked(message) - val onSent = message.onSent - if (result && onSent != null) { - onSent.run() - lock.notifyAll() - } - } - - if (!result && message.sid == CfwTransport.SID) - cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset() - return result - } - - private fun prewriteMessage(message: OutboundMessage): Boolean { - if (prewrittenMessage === message) { - return true - } - if (prewrittenMessage != null && !spoilPrewrittenMessage("before prewrite " + message.label)) { - return false - } - if (!canPrewriteCandidate(message)) { - return false - } - - val frames: List = BleProtocol.framePb( - message.message, - message.sid, - message.flag, - nextTransportSeq++ - ) - if (frames.size <= 1) { - return false - } - - val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress - val prefixFrames = frames.subList(0, frames.size - 1) - val result = bleManager.writeFrames( - writeAddress, - BleProtocol.WRITE_CHAR_UUID, - prefixFrames, - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS - ) - if (!result) { - return false - } - - prewrittenMessage = message - prewrittenFrames = ArrayList(frames) - logLine("prewrote " + message.label + " frames=" + prefixFrames.size + "/" + frames.size) - return true - } - - private fun spoilPrewrittenMessage(reason: String): Boolean { - val message = prewrittenMessage - if (message == null || prewrittenFrames.isEmpty()) { - prewrittenMessage = null - prewrittenFrames = Collections.emptyList() - return true - } - val finalFrame = Arrays.copyOf( - prewrittenFrames[prewrittenFrames.size - 1], - prewrittenFrames[prewrittenFrames.size - 1].size - ) - if (finalFrame.size > 8) { - finalFrame[finalFrame.size - 1] = (finalFrame[finalFrame.size - 1].toInt() xor 0xff).toByte() - } - prewrittenMessage = null - prewrittenFrames = Collections.emptyList() - - val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress - logLine("spoiling prewritten " + message.label + ": " + reason) - return bleManager.writeFrames( - writeAddress, - BleProtocol.WRITE_CHAR_UUID, - Collections.singletonList(finalFrame), - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS - ) - } - - private fun removePreparedMessageLocked(message: OutboundMessage?): Boolean { - if (message == null || message.magic == 0) { - return false - } - val iterator = inFlightMessages.iterator() - while (iterator.hasNext()) { - if (iterator.next() === message) { - iterator.remove() - magicPool.release(message.sid, message.magic, message.label, "write failed") - return true - } - } - return false - } - - - /** Also drain on the worker: a clear/timeout may have removed a blocking - * head since the last notification. Fully ACKed controls must never reach - * the generic timeout path just because no further BLE reply arrived. */ - private fun drainCfwAcknowledgementsLocked() { - while (true) { - val first = CfwMessageWindow.acknowledgedHead(inFlightMessages) ?: return - lastAckAtMs = SystemClock.elapsedRealtime() - resolveAckLocked(first, first.ackPayload) - } - } - - private fun resolveAckLocked(sid: Int, magic: Int, pb: ByteArray?) { - val iterator = inFlightMessages.iterator() - while (iterator.hasNext()) { - val message = iterator.next() - if (message.sid == sid && message.magic == magic) { - resolveAckLocked(message, pb) - return - } - } - recordUnexpectedAckLocked(sid, magic) - } - - private fun resolveAckLocked(message: OutboundMessage, pb: ByteArray?) { - Log.i(TAG, "Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")") - inFlightMessages.remove(message) - message.ackPayload = if (pb == null) ByteArray(0) else Arrays.copyOf(pb, pb.size) - magicPool.release(message.sid, message.magic, message.label, "ack") - val onAck = message.onAck - if (onAck != null) { - onAck.run() - } - consecutiveAckTimeouts = 0 - // onAck may have just satisfied a waiter blocked on lock.wait() (e.g. - // awaitEvenHubSessionReady polling fixedLayoutCreated/displayedFingerprint - // after a create-layout or image ack). Without this, that waiter only - // notices on its own up-to-100ms poll tick, adding avoidable latency to - // every EvenHub wake. Always called with lock held (see call site). - lock.notifyAll() - } - - private fun logImageUpdateSendLandmarkLocked(message: OutboundMessage) { - if (message.imageUpdateId <= 0) { - return - } - val stats = imageUpdateStats[message.imageUpdateId] - val frameId = if (stats == null) 0 else stats.frameId - if (message.imageMessageNumber == 1) { - if (stats != null && stats.firstWriteStartedAtMs <= 0) { - stats.firstWriteStartedAtMs = if (message.writeStartedAtMs > 0) message.writeStartedAtMs else message.sentAtMs - } - FrameTimings.getInstance().log(frameId, "first bluetooth packet sent") - logImageUpdateLandmarkLocked("first bluetooth message sent", message, message.sentAtMs) - } - if (message.imageMessageNumber == message.imageMessageCount) { - FrameTimings.getInstance().log(frameId, - "last bluetooth packet sent (message " + message.imageMessageNumber + "/" + message.imageMessageCount + ")") - logImageUpdateLandmarkLocked("last bluetooth message sent", message, message.sentAtMs) - } - } - - private fun logImageUpdateAckLandmarkLocked(message: OutboundMessage) { - if (message.imageUpdateId <= 0 || message.imageMessageNumber != message.imageMessageCount) { - return - } - val ackedAtMs = SystemClock.elapsedRealtime() - FaceclawBleManager.recordDisplayFrameSent() - val stats = imageUpdateStats.remove(message.imageUpdateId) - if (stats != null && stats.firstWriteStartedAtMs > 0) { - emitFrameMetrics(stats.paintMs, Math.max(0L, ackedAtMs - stats.firstWriteStartedAtMs).toInt(), stats.tileCount) - } - if (stats != null) { - finishFrame(stats.frameId, "sent") - } - logImageUpdateLandmarkLocked("last bluetooth message acked", message, ackedAtMs) - } - - /** Remove the stats entry for an image update that will not complete, finishing its frame. */ - private fun discardImageUpdateStatsLocked(imageUpdateId: Int, reason: String) { - if (imageUpdateId <= 0) { - return - } - val stats = imageUpdateStats.remove(imageUpdateId) - if (stats != null) { - finishFrame(stats.frameId, "discarded: $reason") - } - } - - private fun logImageUpdateLandmarkLocked(event: String, message: OutboundMessage, elapsedMs: Long) { - logLine("image update#" + message.imageUpdateId + " " + event - + " at " + timestamp(elapsedMs) - + " message=" + message.imageMessageNumber + "/" + message.imageMessageCount - + " label=" + message.label) - } - - private fun enqueueCreateLayoutLocked() { - // New session: re-assert the firmware-debug-flags overlay once - // the layout is ready (the mode-7 send is gated on this having reset). - firmwareDebugFlagsLastSent = -1 - val message = messageBuilder.createLayout() - message.onAck = MessageCallback { - startupProbePending = false - clearMessagesOfKindLocked("startup-text-probe") - fixedLayoutCreated = true - displayedFingerprint = "" - } - message.onTimeout = MessageCallback { - if (startupProbePending) { - logLine("create layout timed out while startup text probe is pending") - if (hasPendingOrInflightKindLocked("startup-text-probe")) { - return@MessageCallback - } - startupProbePending = false - } - handleTransportFailure("ack timeout") - } - pendingMessages.addLast(message) - logLine("queue create layout") - } + fun suspendEvenHubSession(): Boolean = core.suspendEvenHubSession() - private fun enqueueStartupProbeLocked() { - enqueueCreateLayoutLocked() + fun resumeEvenHubSession(): Boolean = core.resumeEvenHubSession() - val message = messageBuilder.startupTextProbe() - message.onAck = MessageCallback { - startupProbePending = false - clearMessagesOfKindLocked("create-layout") - fixedLayoutCreated = true - displayedFingerprint = "" - logLine("existing dashboard layout accepted text probe") - } - message.onTimeout = MessageCallback { - startupProbePending = false - if (hasPendingOrInflightKindLocked("create-layout")) { - return@MessageCallback - } - handleTransportFailure("ack timeout") - } - pendingMessages.addLast(message) - startupProbePending = true - logLine("queue startup text probe") - } - - /** - * Send the CFW mode-7 diagnostic-flag control op through the private stream: - * [7][2] to show the on-glasses debug-flag overlay, [7][1] to hide it. Uses the - * arbitrary-payload custom path (no bmp/dedup/frame-timing interaction). - */ - private fun enqueueFirmwareDebugFlagsLocked() { - val show = firmwareDebugFlagsEnabled - val sub = if (show) 2 else 1 - val payload = byteArrayOf(CFW_MSG_DIAGNOSTICS.toByte(), sub.toByte()) - val message = messageBuilder.imagePayload( - DASHBOARD_TILE, nextMapSessionId(), payload, - "fw-debug-flags " + (if (show) "show" else "hide"), - connectionOptions.sendImagesToLeft) - pendingMessages.addLast(message) - firmwareDebugFlagsLastSent = sub - logLine("queue firmware debug flags " + (if (show) "show" else "hide")) - } - - /** - * Send CFW image-handler mode 10. Enable uses the configurable form - * [10][2][interval-ms LE16][minimum-change-degrees LE16]; disable remains [10][0]. - */ - private fun enqueueCompassControlLocked(priority: Boolean, enable: Boolean) { - val sentState = if (enable) 1 else 0 - val payload = if (enable) - byteArrayOf( - CFW_MSG_COMPASS.toByte(), - 2.toByte(), - (COMPASS_REPORT_INTERVAL_MS and 0xff).toByte(), - ((COMPASS_REPORT_INTERVAL_MS shr 8) and 0xff).toByte(), - (COMPASS_MIN_CHANGE_DEGREES and 0xff).toByte(), - ((COMPASS_MIN_CHANGE_DEGREES shr 8) and 0xff).toByte(), - ) - else - byteArrayOf(CFW_MSG_COMPASS.toByte(), 0.toByte()) - val message = messageBuilder.imagePayload( - "compass-control", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - "compass " + (if (enable) "enable" else "disable"), - connectionOptions.sendImagesToLeft) - message.onTimeout = MessageCallback { - compassControlLastSent = -1 - logLine("compass control ack timeout") - } - if (enable) { - compassMaybeOn = true - } else { - message.onAck = MessageCallback { compassMaybeOn = false } - } - if (priority) pendingMessages.addFirst(message) - else pendingMessages.addLast(message) - compassControlLastSent = sentState - logLine("queue " + message.label) - } - - private fun enqueueDesiredImageLocked() { - val fingerprint = getDesiredFingerprint() - val packedSnapshot: ByteArray? - val width: Int - val height: Int - val paintMs: Int - val frameId: Int - val draws: Array? - val scene: ShellScene - synchronized(desiredTilesLock) { - packedSnapshot = desiredPacked - width = desiredWidth - height = desiredHeight - paintMs = desiredPaintMs - frameId = desiredFrameId - draws = desiredDraws - scene = desiredShellScene - desiredFrameId = 0 - } - val packedFrame: ByteArray = packedSnapshot ?: ByteArray(0) - if (customFirmwareDetected && packedFrame.size > 0) { - val rendered = scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId, - connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES, - connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS) - nextImageFrameId = rendered.nextFid - val commands = rendered.commands - for (i in 0 until commands.size - 1) enqueueResourceCommandLocked(commands[i]) - val plan = BleImageOptimizer.TileImagePlan( - 0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId(), commands[commands.size - 1]) - finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId) - return - } - - FrameTimings.getInstance().spanStart(frameId, "compress-and-plan") - // Incremental (mode 3 bounding box) update against the last ENQUEUED frame - // (the base the firmware shadow will hold when this update is applied). - // lastEnqueuedPacked is cleared whenever the image pipeline is cleared, so - // a non-empty value means the display base is trusted. - var incrementalPayload: ByteArray? = null - var incrementalLog: String? = null - if (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedPacked.size > 0 - && lastEnqueuedWidth == width && lastEnqueuedHeight == height) { - val baseFid = nextImageFrameId - val single = - BleImageOptimizer.buildIncrementalImagePayload(lastEnqueuedPacked, packedFrame, width, height, baseFid) - if (single != null) { - incrementalPayload = single.payload - // advance only when a delta is actually emitted, so consecutive - // deltas carry consecutive ids (CFW skip/reorder detection) - nextImageFrameId = if (nextImageFrameId >= 0xfffe) 1 else nextImageFrameId + 1 - incrementalLog = "incremental update bbox=" + - ((single.payload[3].toInt() and 0xff) * 4) + "x" + ((single.payload[4].toInt() and 0xff) * 2) + - "+" + ((single.payload[1].toInt() and 0xff) * 4) + "+" + ((single.payload[2].toInt() and 0xff) * 2) + - " changed=" + single.changedBytes + "/" + single.boxBytes + "B" + - " clusters=" + single.clusterCount - - // When the bounding box spans multiple clusters or is sizeable, try - // splitting into tight rects (CFW mode-8). Only replace the single - // box if the multi-rect message is actually smaller on the wire. - if (connectionOptions.MULTI_RECT_FRAMES - && (single.clusterCount > 1 || single.payload.size > ConnectionOptions.MULTI_RECT_MIN_PAYLOAD)) { - val multi = BleImageOptimizer.buildMultiRectImagePayload( - lastEnqueuedPacked, packedFrame, width, height, baseFid, ConnectionOptions.MULTI_RECT_MAX_RECTS) - if (multi != null && multi.payload.size < single.payload.size) { - incrementalPayload = multi.payload - nextImageFrameId = multi.nextFid // rectCount fids consumed - incrementalLog = "multi-rect update n=" + multi.rectCount + - " covered=" + multi.coveredBytes + "B" + - " payload=" + multi.payload.size + "B (vs bbox " + single.payload.size + "B)" - } - } - } - } - val plan = if (incrementalPayload != null) - BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId(), incrementalPayload) - else - BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId()) - FrameTimings.getInstance().spanEnd(frameId, "compress-and-plan") - if (incrementalLog != null) { - FrameTimings.getInstance().log(frameId, incrementalLog) - } - finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId) - } - - /** Queue complete private commands, advance the delta base, and retain frame/ACK bookkeeping. */ - private fun finishEnqueueDesiredImageLocked( - plan: BleImageOptimizer.TileImagePlan, fingerprint: String, paintMs: Int, frameId: Int) { - val updateId = nextImageUpdateId++ - val commands: MutableList = ArrayList() - if (plan.payload.size <= CfwTransport.MAX_MESSAGE) { - commands.add(plan.payload) - } else { - // A noisy full frame can exceed the stream record limit. Repaint it - // with independently decodable RLE bands below the decoded-message limit. - commands.addAll(BleImageOptimizer.encodeFullFrameBands( - plan.packed!!, plan.width, plan.height, nextImageFrameId)) - for (i in 0 until commands.size) { - nextImageFrameId = if (nextImageFrameId >= 0xfffe) 1 else nextImageFrameId + 1 - } - } - val messageCount = commands.size - imageUpdateStats[updateId] = BleImageOptimizer.ImageUpdateStats(paintMs, 1, frameId) - for (i in 0 until commands.size) { - enqueueCustomImageLocked(plan, commands[i], fingerprint, updateId, i + 1, messageCount) - } - // This frame is now the base for the next delta (it will be the firmware - // shadow once applied), even though it hasn't been acked yet — that is what - // lets the next frame pipeline behind it. plan.packed is the full frame; - // frames are immutable by convention, so referencing it is safe. - lastEnqueuedPacked = plan.packed!! - lastEnqueuedWidth = plan.width - lastEnqueuedHeight = plan.height - lastEnqueuedFingerprint = fingerprint - - FrameTimings.getInstance().log(frameId, "queued image update#" + updateId - + " messages=" + messageCount + " payload=" + plan.payload.size + "B") - logLine("queue image update#" + updateId + " fingerprint=" + fingerprint - + " messages=" + messageCount) - } - - /** - * Enqueue one resource eviction/upload command ahead of the image message that - * references its glyphs (the transport is FIFO, so no ack round trip is - * needed before use). A timeout means the on-glasses cache state is - * unknown; forget everything phone-side (glyphs re-upload lazily) and let - * the accompanying image update's own timeout drive the frame resync. - */ - private fun enqueueResourceCommandLocked(payload: ByteArray) { - val message = messageBuilder.imagePayload( - "resources", - DASHBOARD_TILE, - nextMapSessionId(), - payload, - "resource command " + payload.size + "B", - connectionOptions.sendImagesToLeft) - message.onTimeout = MessageCallback { - resourceCache.reset() - logLine("resource command ack timeout; resource cache state reset") - } - pendingMessages.addLast(message) - logLine("queue " + message.label) - } - - private fun enqueueCustomImageLocked( - plan: BleImageOptimizer.TileImagePlan, - payload: ByteArray, - fingerprint: String, - updateId: Int, - messageNumber: Int, - messageCount: Int - ) { - val message = messageBuilder.customMessage("image", payload, - "image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft) - message.setImageUpdatePosition(updateId, messageNumber, messageCount) - message.onAck = MessageCallback { - imageRetryAfterMs = 0 - // Firmware >= 2.2.4.34 resets its heartbeat timer when it receives - // image messages (not just heartbeats), so an acked image fragment - // satisfies the heartbeat deadline and heartbeats stop contending - // with active rendering. - lastHeartbeatAckedAtMs = SystemClock.elapsedRealtime() - logImageUpdateAckLandmarkLocked(message) - var imageStillInFlight = false - for (inFlight in inFlightMessages) { - if ("image" == inFlight.kind) { - imageStillInFlight = true - break - } - } - if (!imageStillInFlight) { - var imageStillQueued = false - for (queued in pendingMessages) { - if ("image" == queued.kind) { - imageStillQueued = true - break - } - } - if (!imageStillQueued) { - displayedFingerprint = fingerprint - } - } - } - message.onTimeout = MessageCallback { - discardImageUpdateStatsLocked(message.imageUpdateId, "image ack timeout (will retry)") - clearMessagesOfKindLocked("image") - displayedFingerprint = "" - imageRetryAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.IMAGE_RETRY_DELAY_MS - } - pendingMessages.addLast(message) - } - - private fun createAudioControlMessageLocked(enable: Boolean): OutboundMessage { - val message = messageBuilder.enableOrDisableMic(enable) - message.onAck = MessageCallback { - lastAudioControlAckMagic = message.magic - audioCaptureActive = message.label?.contains("enable") == true - logLine(if (audioCaptureActive) "G2 mic enabled" else "G2 mic disabled") - } - message.onTimeout = MessageCallback { - handleTransportFailure("audio control ack timeout") - } - return message - } - - private fun shouldPollBatteryLocked(now: Long): Boolean { - return !shutdownRequested - && sessionReady - && pendingMessages.isEmpty() - && inFlightMessages.isEmpty() - && now - lastConnectionOrInputAtMs >= ConnectionOptions.BATTERY_INPUT_QUIET_MS - && (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS) - } - - private fun createBatteryQueryMessageLocked(): OutboundMessage { - val message = messageBuilder.batteryQuery() - message.onAck = MessageCallback { - val ring = BleProtocol.parseRingBattery(message.ackPayload) - // Old firmware and missing/malformed extensions must clear any prior reading. - ringBattery = if (ring == null) -1 else ring.battery - ringCharging = if (ring == null) -1 else ring.charging - val snapshot = BleProtocol.parseSettingsBattery(message.ackPayload) - if (snapshot != null) { - headsetBattery = snapshot.battery - headsetCharging = snapshot.charging - emitBatteryState(headsetBattery, headsetCharging) - if (snapshot.silentMode >= 0) { - // Backstop for the push in onNotification: the firmware is - // confirmed to push silent-mode-on, but the off transition is - // not, so re-read the authoritative value on every poll. - updateSilentModeLocked(snapshot.silentMode > 0) - } - updateChargingModeLocked(snapshot.charging > 0, snapshot.battery) - } - val firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload) - if (firmwareInfo != null) { - customFirmwareDetected = firmwareInfo.isFaceclawFirmware() - emitFirmwareInfo(firmwareInfo) - } - } - message.onTimeout = MessageCallback { - logLine("Battery query timed out") - } - return message - } - - /** - * Track silent mode, which the wearer toggles by long-pressing both - * touchpads at once. While it is on the firmware refuses input events and - * app launches and powers the display down, so the glasses look dead even - * though the BLE session is healthy; the phone UI says so explicitly. - */ - private fun updateSilentModeLocked(silent: Boolean) { - val next = if (silent) 1 else 0 - if (silentMode == next) { - return - } - silentMode = next - logLine(if (silent) "glasses entered silent mode" else "glasses left silent mode") - emitSilentMode(silent) - } - - /** - * Track whether the glasses are in the charging case. Charging means nobody - * is wearing them: display communication pauses (no heartbeats, so the - * firmware tears down its EvenHub context on its own) and only battery polls - * continue. When charging stops, tear the transport down and let the normal - * reconnect loop rebuild the session, layout, and first frame. - */ - private fun updateChargingModeLocked(charging: Boolean, battery: Int) { - if (charging == chargingMode) { - if (chargingMode) { - setStateDisplay("charging", chargingStatusText(battery)) - } - return - } - if (charging) { - chargingMode = true - clearAllMessagesLocked("glasses charging") - fixedLayoutCreated = false - startupProbePending = false - displayedFingerprint = "" - finishDesiredFrameLocked("discarded: glasses charging") - logLine("glasses are charging; pausing display communication") - setStateDisplay("charging", chargingStatusText(battery)) - } else { - chargingMode = false - logLine("glasses removed from charger; reconnecting") - handleTransportFailure("charging ended") - } - } - - /** - * Record, into the frame that is waiting, why it did not go out on this - * pass of the send loop. Without this the export shows a bare multi-second - * jump between "image submitted as desired frame" and the first BLE - * packet, with no hint whether we were blocked on a heartbeat, the - * BLE window, or another message queued ahead. Deduped on (frame, reason), - * so a frame stalled for seconds gets one line per state change rather - * than one per loop pass. - */ - private fun noteImageStallLocked(now: Long, windowHasRoom: Boolean) { - var frameId: Int - synchronized(desiredTilesLock) { - frameId = desiredFrameId - } - val reason: String? - if (frameId != 0 && getDesiredFingerprint() != lastEnqueuedFingerprint) { - reason = describeEnqueueBlockerLocked(now, windowHasRoom) - } else { - // Nothing waiting to be planned; an already-planned image may still - // be queued behind other traffic. - val queuedImage = firstPendingImageLocked() - frameId = if (queuedImage == null) 0 else imageUpdateFrameIdLocked(queuedImage.imageUpdateId) - reason = if (queuedImage == null) null else describeWriteBlockerLocked(now, windowHasRoom, queuedImage) - } - if (frameId == 0 || reason == null) { - stallFrameId = 0 - stallReason = "" - return - } - if (frameId == stallFrameId && reason == stallReason) { - return - } - stallFrameId = frameId - stallReason = reason - FrameTimings.getInstance().log(frameId, "waiting to send: $reason") - } - - /** Why the desired composite has not been turned into wire messages yet, or null. */ - private fun describeEnqueueBlockerLocked(now: Long, windowHasRoom: Boolean): String? { - if (shutdownRequested) { - return "shutdown requested" - } - if (!sessionReady) { - return "BLE session not ready" - } - if (!fixedLayoutCreated) { - return "display layout not created yet" - } - if (now < imageRetryAfterMs) { - return "image retry backoff (" + (imageRetryAfterMs - now) + "ms left)" - } - if (hasPendingImageLocked()) { - return "an earlier image is still queued" - } - if (!windowHasRoom) { - return "BLE window full (" + inFlightMessages.size + " message(s) in flight)" - } - if (hasPendingOrInflightKindLocked("heartbeat")) { - return "heartbeat in flight" - } - if (!pendingMessages.isEmpty()) { - return pendingMessages.size.toString() + " message(s) queued ahead, next " + - pendingMessages.peekFirst().label - } - return null - } - - /** Why a planned image message has not been written to BLE yet, or null. */ - private fun describeWriteBlockerLocked(now: Long, windowHasRoom: Boolean, queuedImage: OutboundMessage): String? { - if (!sessionReady) { - return "BLE session not ready" - } - if (!windowHasRoom) { - return "BLE window full (" + inFlightMessages.size + " message(s) in flight)" - } - if (hasPendingOrInflightKindLocked("heartbeat")) { - return "heartbeat in flight" - } - // handleHeartbeat is a barrier: while one is due it holds back every - // other write, so a frame queued at the wrong moment waits a heartbeat - // round trip. Reported explicitly because it is otherwise invisible -- - // heartbeats belong to no frame. - val heartbeatElapsedMs = now - lastHeartbeatAckedAtMs - if (fixedLayoutCreated && !shutdownRequested - && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS) { - return "heartbeat due (" + heartbeatElapsedMs + "ms since the last one acked)" - } - val head = pendingMessages.peekFirst() - if (head != null && head !== queuedImage) { - return "queued behind " + head.label - } - return null - } - - private fun firstPendingImageLocked(): OutboundMessage? { - for (message in pendingMessages) { - if (message.imageUpdateId > 0) { - return message - } - } - return null - } - - private fun imageUpdateFrameIdLocked(imageUpdateId: Int): Int { - val stats = imageUpdateStats[imageUpdateId] - return if (stats == null) 0 else stats.frameId - } - - private fun finishDesiredFrameLocked(outcome: String) { - val frameIdToFinish: Int - synchronized(desiredTilesLock) { - frameIdToFinish = desiredFrameId - desiredFrameId = 0 - } - finishFrame(frameIdToFinish, outcome) - } - - private fun hasPendingOrInflightKindLocked(kind: String): Boolean { - for (queued in pendingMessages) { - if (kind == queued.kind) { - return true - } - } - for (inFlight in inFlightMessages) { - if (kind == inFlight.kind) { - return true - } - } - return false - } - - private fun hasPendingOrInflightMagicLocked(magic: Int): Boolean { - for (queued in pendingMessages) { - if (queued.magic == magic) { - return true - } - } - for (inFlight in inFlightMessages) { - if (inFlight.magic == magic) { - return true - } - } - return false - } - - private fun hasPendingMagicLocked(sid: Int, magic: Int): Boolean { - for (queued in pendingMessages) { - if (queued.sid == sid && queued.magic == magic) { - return true - } - } - return false - } - - private fun handleAckTimeoutLocked(message: OutboundMessage) { - consecutiveAckTimeouts += 1 - - val onTimeout = message.onTimeout - if (onTimeout != null) { - onTimeout.run() - } - - if (consecutiveAckTimeouts > ConnectionOptions.MAX_CONSECUTIVE_ACK_TIMEOUTS) { - handleTransportFailure("too many ack timeouts") - } - } - - /** - * Replace any stale lease control with one right-arm and one left-arm - * fire-and-forget write. With priority=true the right arm is sent first so - * CLAIM reaches the lens that originated the deferred wake immediately. - */ - private fun enqueueFaceclawWakeControlLocked(operation: Int, nonce: Int, priority: Boolean): Int { - clearMessagesOfKindLocked("wake-lease-control") - val generation = ++faceclawWakeControlGeneration - faceclawWakeControlSentCount = 0 - if (operation == BleProtocol.FACECLAW_WAKE_OP_ACQUIRE) { - lastFaceclawWakeLeaseQueuedAtMs = SystemClock.elapsedRealtime() - } - val onSent = MessageCallback { - if (faceclawWakeControlGeneration == generation) { - faceclawWakeControlSentCount += 1 - } - } - val right = messageBuilder.faceclawWakeControl(operation, nonce, false) - val left = messageBuilder.faceclawWakeControl(operation, nonce, true) - right.onSent = onSent - left.onSent = onSent - if (priority) { - pendingMessages.addFirst(left) - pendingMessages.addFirst(right) - } else { - pendingMessages.addLast(right) - pendingMessages.addLast(left) - } - logLine("queue " + right.label + " + L") - return generation - } - - private fun waitForFaceclawWakeControlDelivery(generation: Int, timeoutMs: Long): Boolean { - val deadline = SystemClock.elapsedRealtime() + Math.max(0L, timeoutMs) - synchronized(lock) { - while (running - && sessionReady - && faceclawWakeControlGeneration == generation - && faceclawWakeControlSentCount < 2) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - return faceclawWakeControlGeneration == generation - && faceclawWakeControlSentCount >= 2 - } - } - - /** - * Acquire/renew or release CFW's independent direct-framebuffer repaint - * guard on both arms. It is separate from the optional idle-wake lease: - * every Faceclaw display session needs this guard while its EvenHub layout - * contains swipe-capturing stock widgets. - */ - private fun enqueueFaceclawFramebufferControlLocked(operation: Int, priority: Boolean): Int { - clearMessagesOfKindLocked("framebuffer-lease-control") - val generation = ++faceclawFramebufferControlGeneration - faceclawFramebufferControlSentCount = 0 - if (operation == BleProtocol.FACECLAW_FB_OP_ACQUIRE) { - lastFaceclawFramebufferLeaseQueuedAtMs = SystemClock.elapsedRealtime() - } - val onSent = MessageCallback { - if (faceclawFramebufferControlGeneration == generation) { - faceclawFramebufferControlSentCount += 1 - } - } - val right = messageBuilder.faceclawFramebufferControl(operation, false) - val left = messageBuilder.faceclawFramebufferControl(operation, true) - right.onSent = onSent - left.onSent = onSent - if (priority) { - pendingMessages.addFirst(left) - pendingMessages.addFirst(right) - } else { - pendingMessages.addLast(right) - pendingMessages.addLast(left) - } - logLine("queue " + right.label + " + L") - return generation - } - - private fun waitForFaceclawFramebufferControlDelivery(generation: Int, timeoutMs: Long): Boolean { - val deadline = SystemClock.elapsedRealtime() + Math.max(0L, timeoutMs) - synchronized(lock) { - while (running - && sessionReady - && faceclawFramebufferControlGeneration == generation - && faceclawFramebufferControlSentCount < 2) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - try { - lock.wait(Math.min(remaining, 100L)) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - break - } - } - return faceclawFramebufferControlGeneration == generation - && faceclawFramebufferControlSentCount >= 2 - } - } - - private fun releaseFaceclawFramebufferLease(): Boolean { - val generation: Int - synchronized(lock) { - if (!running || !sessionReady) { - return true - } - generation = enqueueFaceclawFramebufferControlLocked( - BleProtocol.FACECLAW_FB_OP_RELEASE, - true - ) - } - interruptibleSleep.interrupt() - return waitForFaceclawFramebufferControlDelivery( - generation, - FACECLAW_WAKE_CONTROL_WAIT_MS - ) - } - - private fun clearMessagesOfKindLocked(kind: String) { - val pendingIterator = pendingMessages.iterator() - while (pendingIterator.hasNext()) { - val message = pendingIterator.next() - if (kind == message.kind) { - pendingIterator.remove() - discardPrewriteIfMatchesLocked(message) - if ("image" == kind) { - discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared ($kind)") - } - magicPool.release(message.sid, message.magic, message.label, "cleared pending $kind") - } - } - val inFlightIterator = inFlightMessages.iterator() - while (inFlightIterator.hasNext()) { - val message = inFlightIterator.next() - if (kind == message.kind) { - inFlightIterator.remove() - if ("image" == kind) { - discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared ($kind)") - } - magicPool.release(message.sid, message.magic, message.label, "cleared inflight $kind") - } - } - if ("image" == kind) { - // The image pipeline was flushed (e.g. ack timeout -> keyframe resync): - // drop the pipelined delta base so the next image is a full keyframe. - lastEnqueuedPacked = ByteArray(0) - lastEnqueuedFingerprint = "" - } - } - - private fun clearAllMessagesLocked(reason: String) { - clearPendingMessagesLocked(reason) - clearInFlightMessagesLocked(reason) - // The image pipeline is gone: drop the pipelined delta base so the next - // image is a full keyframe rather than a delta onto a stale base. - lastEnqueuedPacked = ByteArray(0) - lastEnqueuedFingerprint = "" - // Queued resource commands (if any) were dropped with the rest, and the - // session churn behind a full clear may have freed the on-glasses - // cache; forget it phone-side so glyphs re-upload lazily. - resourceCache.reset() - } - - /** - * A custom wake queues CLAIM before NativeScript asks for a resume. Keep - * that private control while flushing stale EvenHub traffic around the - * direct prelude write. - */ - private fun clearAllMessagesPreservingWakeLeaseLocked(reason: String) { - val pendingIterator = pendingMessages.iterator() - while (pendingIterator.hasNext()) { - val message = pendingIterator.next() - if ("wake-lease-control" == message.kind) { - continue - } - pendingIterator.remove() - discardPrewriteIfMatchesLocked(message) - discardImageUpdateStatsLocked( - message.imageUpdateId, - "pending messages cleared: $reason" - ) - magicPool.release( - message.sid, - message.magic, - message.label, - "cleared pending: $reason" - ) - } - clearInFlightMessagesLocked(reason) - lastEnqueuedPacked = ByteArray(0) - lastEnqueuedFingerprint = "" - resourceCache.reset() - } - - /** Any image update whose fragments are still queued (not yet sent). */ - private fun hasPendingImageLocked(): Boolean { - for (message in pendingMessages) { - if ("image" == message.kind) { - return true - } - } - return false - } - - private fun clearPendingMessagesLocked(reason: String) { - while (!pendingMessages.isEmpty()) { - val message = pendingMessages.removeFirst() - discardPrewriteIfMatchesLocked(message) - discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared: $reason") - magicPool.release(message.sid, message.magic, message.label, "cleared pending: $reason") - } - } - - private fun clearInFlightMessagesLocked(reason: String) { - while (!inFlightMessages.isEmpty()) { - val message = inFlightMessages.removeFirst() - discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared: $reason") - magicPool.release(message.sid, message.magic, message.label, "cleared inflight: $reason") - } - } - - private fun discardPrewriteIfMatchesLocked(message: OutboundMessage?) { - if (message != null && message === prewrittenMessage) { - prewrittenMessage = null - prewrittenFrames = Collections.emptyList() - } - } - - private fun recordUnexpectedAckLocked(sid: Int, magic: Int) { - if (magic < BleMagicPool.MIN_MAGIC || magic > BleMagicPool.MAX_MAGIC) { - return - } - val previous = magicPool.getReleaseRecord(sid, magic) - if (previous == null) { - val pendingNote = if (hasPendingMagicLocked(sid, magic)) " while that magic is only pending locally" else "" - logLine("unexpected ACK sid=" + sid + " magic=" + magic + pendingNote - + "; possible Even app BLE contention") - return - } - if ("timeout" == previous.reason) { - logLine("late ACK after timeout sid=" + sid + " magic=" + magic - + " label=" + previous.label - + "; ACK timeout may be too short") - return - } - if ("ack" == previous.reason) { - logLine("duplicate ACK for already-acked message sid=" + sid + " magic=" + magic - + " label=" + previous.label - + "; possible Even app BLE contention") - return - } - logLine("late ACK for released message sid=" + sid + " magic=" + magic - + " label=" + previous.label - + " release=" + previous.reason) - } - - /** The address of a configured arm Android no longer holds a bond for, or null. */ - private fun firstUnpairedArm(): String? { - if (!bleManager.isBonded(rightAddress)) return rightAddress - if (!bleManager.isBonded(leftAddress)) return leftAddress - return null - } - - /** - * A connect failure while an arm's bond is missing (the pairing was - * forgotten in Android settings, or the address was typed in by hand and - * never paired) will repeat forever, so instead of scheduling a retry, - * park the worker loop and tell the user to re-pair. Only an explicit - * connect (which builds a fresh communicator) starts a new attempt. - */ - private fun handleUnpairedFailure(address: String) { - Log.e(TAG, "Connect failed and $address is not paired; suspending reconnect") - synchronized(lock) { - reconnectHalted = true - sessionReady = false - fixedLayoutCreated = false - startupProbePending = false - shutdownRequested = false - chargingMode = false - imageRetryAfterMs = 0 - displayedFingerprint = "" - faceclawWakePendingNonce = -1 - lastFaceclawWakeLeaseQueuedAtMs = 0 - faceclawWakeControlSentCount = 0 - clearAllMessagesLocked("arm not paired: $address") - reconnectAfterMs = Long.MAX_VALUE - bleManager.disconnect(rightAddress) - bleManager.disconnect(leftAddress) - } - if (!userDisconnectRequested) { - setStateDisplay( - "unpaired", - "The glasses (" + address + ") are not paired with this phone." - + " Use \"Pair glasses\" to pair them again, then connect." - ) - } - interruptibleSleep.interrupt() - } - - private fun handleTransportFailure(reason: String?) { - Log.e(TAG, "Transport failure: $reason") - synchronized(lock) { - maybeEmitEvenAppConflictLocked(reason) - finishBenchmarkLocked(true, "transport failure") - sessionReady = false - fixedLayoutCreated = false - startupProbePending = false - shutdownRequested = false - chargingMode = false - imageRetryAfterMs = 0 - displayedFingerprint = "" - faceclawWakePendingNonce = -1 - lastFaceclawWakeLeaseQueuedAtMs = 0 - faceclawWakeControlSentCount = 0 - clearAllMessagesLocked("transport failure: $reason") - reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS - bleManager.disconnect(rightAddress) - bleManager.disconnect(leftAddress) - } - if (!userDisconnectRequested) { - setStateDisplay("retrying", if (reason == null || reason.isEmpty()) "Reconnecting..." else "Reconnecting after $reason") - } - interruptibleSleep.interrupt() - } - - private fun resetSessionStateLocked() { - ringBattery = -1 - ringCharging = -1 - sessionReady = false - shutdownRequested = false - fixedLayoutCreated = false - chargingMode = false - rightConnected = false - leftConnected = false - ringConnected = false - ringNotificationsReady = false - reconnectAfterMs = 0 - reconnectHalted = false - ringReconnectAfterMs = 0 - lastAckAtMs = 0 - lastIncomingAtMs = 0 - lastHeartbeatSentAtMs = 0 - lastSessionReadyAtMs = 0 - consecutiveAckTimeouts = 0 - lastAudioControlAckMagic = 0 - audioCaptureActive = false - audioPacketListener = null - compassControlLastSent = -1 - // A dead transport orphans any glasses-side compass state; the fresh - // session re-asserts the desired state once its layout is ready. - compassMaybeOn = false - faceclawWakePendingNonce = -1 - lastFaceclawWakeLeaseQueuedAtMs = 0 - faceclawWakeControlSentCount = 0 - cfwCleanupDelivered = false - for (transport in cfwTransports) transport.reset() - lastCfwCleanupAckMagic = 0 - wearState = -1 - displayedFingerprint = "" - // Deliberately not clearing silentMode: it is a property of the glasses, - // not of our session, and silent mode blocks app launches, so it can be - // the very cause of the session teardown that got us here. - } - - private fun emitRingEvent(event: G2Event, frameId: Int) { - val current = listener - if (current == null) { - FrameTimings.getInstance().finishFrame(frameId, "discarded: no listener attached") - return - } - val containerNameSnapshot = event.containerName - mainHandler.post { - FrameTimings.getInstance().log(frameId, "dispatching input event on main thread") - try { - current.onRingEvent(event.kind, containerNameSnapshot, event.eventType, event.eventSource, event.systemExitReasonCode, frameId, - event.ringTick, event.ringType, event.ringAux, event.ringSpeed) - } catch (t: Throwable) { - Log.w(TAG, "listener onRingEvent failed", t) - FrameTimings.getInstance().finishFrame(frameId, "discarded: listener onRingEvent failed") - } - } - } - - /** Finish a frame owned by the communicator and tell the TS side, which may be awaiting it. */ - private fun finishFrame(frameId: Int, outcome: String) { - if (frameId <= 0) { - return - } - FrameTimings.getInstance().finishFrame(frameId, outcome) - val current = listener ?: return - mainHandler.post { - try { - current.onFrameFinished(frameId, outcome) - } catch (t: Throwable) { - Log.w(TAG, "listener onFrameFinished failed", t) - } - } - } - - private fun emitImuData(x: Double, y: Double, z: Double, eventSource: Int) { - if (imuListeners.isEmpty()) { - return - } - mainHandler.post { - for (imuListener in imuListeners) { - try { - imuListener.onImuData(x, y, z, eventSource) - } catch (t: Throwable) { - Log.w(TAG, "listener onImuData failed", t) - } - } - } - } - - private fun emitCompassEvent(event: BleProtocol.CompassEvent) { - if (event.diagnosticFlags >= 0) { - val sources = arrayOf("unknown", "GRV", "GMRV", "RV") - Log.i("FaceclawCompass", "heading=" + event.headingDegrees - + " magneticAccuracy=" + event.magneticAccuracy - + " magneticAnomalies=" + event.magneticAnomalies - + " orientationSource=" + sources[event.orientationSource] - + " flags=0x" + Integer.toHexString(event.diagnosticFlags) - + " sampleTimeMs=" + event.sampleTimeMs) - } - for (subscription in compassSubscriptions) { - subscription.handler.post { - try { - subscription.listener.onCompassEvent(event.command, event.headingDegrees, - event.magneticAccuracy, event.magneticAnomalies, event.orientationSource, - event.diagnosticFlags, event.sampleTimeMs) - } catch (t: Throwable) { - Log.w(TAG, "listener onCompassEvent failed", t) - } - } - } - } - - private fun emitSilentMode(silent: Boolean) { - val current = listener ?: return - mainHandler.post { - try { - current.onSilentMode(silent) - } catch (t: Throwable) { - Log.w(TAG, "listener onSilentMode failed", t) - } - } - } - - private fun emitWearState(wearing: Boolean) { - val current = listener ?: return - mainHandler.post { - try { - current.onWearState(wearing) - } catch (t: Throwable) { - Log.w(TAG, "listener onWearState failed", t) - } - } - } - - private fun emitPhoneLockStateIfChanged(force: Boolean) { - val locked: Boolean - synchronized(lock) { - val now = SystemClock.elapsedRealtime() - if (!force && now - lastPhoneLockCheckAtMs < 1_000) return - lastPhoneLockCheckAtMs = now - locked = keyguardManager != null && keyguardManager.isDeviceLocked - val value = if (locked) 1 else 0 - if (!force && value == phoneLockState) return - phoneLockState = value - } - val current = listener ?: return - mainHandler.post { - try { - current.onPhoneLockState(locked) - } catch (t: Throwable) { - Log.w(TAG, "listener onPhoneLockState failed", t) - } - } - } - - private fun emitBatteryState(headsetBattery: Int, headsetCharging: Int) { - val reportedRingBattery = ringBattery - val reportedRingCharging = ringCharging - val current = listener ?: return - mainHandler.post { - try { - current.onBatteryState(headsetBattery, headsetCharging, reportedRingBattery, reportedRingCharging) - } catch (t: Throwable) { - Log.w(TAG, "listener onBatteryState failed", t) - } - } - } - - private fun emitFirmwareInfo(info: BleProtocol.FirmwareInfo) { - val current = listener ?: return - mainHandler.post { - try { - current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.extension) - } catch (t: Throwable) { - Log.w(TAG, "listener onFirmwareInfo failed", t) - } - } - } - - private fun emitFrameMetrics(paintMs: Int, transmitMs: Int, tileCount: Int) { - val current = listener ?: return - mainHandler.post { - try { - current.onFrameMetrics(paintMs, transmitMs, tileCount) - } catch (t: Throwable) { - Log.w(TAG, "listener onFrameMetrics failed", t) - } - } - } - - private fun maybeEmitEvenAppConflictLocked(reason: String?) { - if ("write failed" != reason) { - return - } - val now = SystemClock.elapsedRealtime() - if (lastSessionReadyAtMs <= 0 || now - lastSessionReadyAtMs > ConnectionOptions.EVEN_APP_WRITE_FAILURE_WINDOW_MS) { - return - } - if (lastEvenAppConflictAtMs > 0 && now - lastEvenAppConflictAtMs < 60_000) { - return - } - if (!FaceclawEvenAppDetector.isEvenNotificationActive(appContext)) { - return - } - lastEvenAppConflictAtMs = now - emitEvenAppConflict("The Even Realities app still appears to be running. It can hold the glasses BLE link and cause Faceclaw write failures. Open its app settings and force stop it, then reconnect Faceclaw.") - } - - private fun emitEvenAppConflict(message: String?) { - val current = listener ?: return - val messageSnapshot = message ?: "" - mainHandler.post { - try { - current.onEvenAppConflict(messageSnapshot) - } catch (t: Throwable) { - Log.w(TAG, "listener onEvenAppConflict failed", t) - } - } - } - - private fun setStateDisplay(nextPhase: String, nextStatus: String) { - synchronized(lock) { - phase = nextPhase - status = nextStatus - } - emitState() - } - - private fun emitState() { - val current = listener ?: return - val phaseSnapshot: String - val statusSnapshot: String - synchronized(lock) { - phaseSnapshot = phase - statusSnapshot = status - } - mainHandler.post { - try { - current.onStateChange(phaseSnapshot, statusSnapshot) - } catch (t: Throwable) { - Log.w(TAG, "listener onStateChange failed", t) - } - } - } + // --------------------------------------------------------------------- private fun updateG2ScreenWakeLock(screenOn: Boolean) { if (screenOn) { @@ -4022,51 +374,14 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre } if (!wakeLock.isHeld) { wakeLock.acquire() - logLine("G2 screen wake lock acquired") + Log.i(TAG, "G2 screen wake lock acquired") } return } - releaseG2ScreenWakeLock() - } - - private fun releaseG2ScreenWakeLock() { val wakeLock = g2ScreenWakeLock if (wakeLock != null && wakeLock.isHeld) { wakeLock.release() - logLine("G2 screen wake lock released") - } - } - - private fun logLine(line: String) { - Log.i(TAG, line) - } - - private fun nextMapSessionId(): Int { - val id = nextMapSessionId - val increment = if (connectionOptions.skipSessionIds) 2 else 1 - nextMapSessionId = (nextMapSessionId + increment) and 0xff - return id - } - - private fun hasRingAddress(): Boolean { - return ringAddress.trim().isNotEmpty() - } - - private fun isConfiguredRingAddress(address: String?): Boolean { - return hasRingAddress() && address != null && address.equals(ringAddress, ignoreCase = true) - } - - private fun isDirectRingNotification(address: String?, characteristicUuid: String?): Boolean { - if (!isConfiguredRingAddress(address) || characteristicUuid == null) { - return false - } - return BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID == characteristicUuid - || BleProtocol.R1_NOTIFY_CHAR_UUID == characteristicUuid - } - - private fun getDesiredFingerprint(): String { - synchronized(desiredTilesLock) { - return desiredFingerprint + Log.i(TAG, "G2 screen wake lock released") } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt index c0cc2e04..d985766e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt @@ -2,455 +2,74 @@ package com.faceclaw.app import android.os.Handler import android.os.Looper -import android.util.Log - -import com.k2fsa.sherpa.onnx.FeatureConfig -import com.k2fsa.sherpa.onnx.OfflineModelConfig -import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig -import com.k2fsa.sherpa.onnx.OfflineRecognizer -import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig -import com.k2fsa.sherpa.onnx.OfflineRecognizerResult -import com.k2fsa.sherpa.onnx.OfflineStream import java.io.File -import java.util.ArrayDeque -import java.util.Arrays /** - * Continuous captioning over the decoded mic PCM: segments speech into - * utterances with an adaptive energy gate, transcribes each utterance with - * the on-device Moonshine model, and attaches a speaker voice-print per - * utterance. PCM is pushed in from the TS side (which owns mic arbitration - * and any beam-direction gating), so this engine has no BLE dependencies. + * Continuous captioning over the decoded mic PCM: the shared [CaptionSession] segments speech + * into utterances with an adaptive energy gate, transcribes each with the on-device Moonshine + * model and attaches a speaker voice-print. This Android class only supplies the sherpa-onnx + * recognizer and speaker model and delivers callbacks on the main thread. PCM is pushed in + * from the TS side (which owns mic arbitration and any beam-direction gating), so the engine + * has no BLE dependencies. * - * Utterance boundaries are measured on the sample clock, not the wall clock: - * BLE delivers mic packets in bursts, so elapsed real time overestimates the - * audio heard. The TS side converts startMs/endMs to wall-clock times using - * the engine start timestamp. + * Utterance boundaries are measured on the sample clock, not the wall clock; the TS side + * converts startMs/endMs to wall-clock times using the engine start timestamp. */ class FaceclawCaptionEngine { - companion object { - private const val TAG = "FaceclawCaptions" - private const val SAMPLE_RATE = 16000 - private const val FEATURE_DIM = 80 - private const val MAX_QUEUE_PACKETS = 200 + private val mainHandler = Handler(Looper.getMainLooper()) - // Segmentation: RMS thresholds relative to a rolling noise floor, in the - // same spirit as FaceclawVoiceController.EndpointDetector but recurring. - private const val ONSET_FACTOR = 3.0 - private const val RELEASE_FACTOR = 1.8 - private const val MIN_RMS = 220.0 - private const val NOISE_EMA_ALPHA = 0.05 - private const val PRE_ROLL_MS = 400 - private const val DEFAULT_SILENCE_MS = 800 - private const val MIN_UTTERANCE_MS = 350 - private const val MAX_UTTERANCE_MS = 15000 - // Moonshine v2 fails past ~9.1 s of input; decode long utterances in - // segments cut at the quietest window (mirrors FaceclawVoiceController). - private const val DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8 - private const val CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2 - private const val CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000 - private const val NORMALIZE_TARGET_PEAK = 0.9f - private const val NORMALIZE_MAX_GAIN = 30f - // Voice-prints degrade on very long inputs; embed at most the first 10 s. - private const val EMBED_MAX_SAMPLES = SAMPLE_RATE * 10 + private val host = object : CaptionHost { + override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } } + + override fun loadTranscriber(modelDir: String): OfflineTranscriber? { + if (!File(modelDir, "tokens.txt").exists()) { + return null + } + return AndroidSpeechEngines.SherpaTranscriber( + AndroidSpeechEngines.recognizerConfig(File(modelDir), VoiceModelKind.MOONSHINE)) + } + + override fun loadEmbedder(modelPath: String): SpeakerEmbedder? { + if (!File(modelPath).exists()) { + return null + } + val speakerId = FaceclawSpeakerId(modelPath) + speakerId.ensureLoaded() + return speakerId + } } - private val mainHandler = Handler(Looper.getMainLooper()) - private val lock = Any() - private val queueLock = Any() - private val queue = ArrayDeque() - - @Volatile - private var listener: FaceclawCaptionEngineListener? = null - @Volatile - private var asrModelDir: String? = null - @Volatile - private var speakerModelPath: String? = null - @Volatile - private var silenceMs = DEFAULT_SILENCE_MS - private var workerThread: Thread? = null - @Volatile - private var started = false - - private var recognizer: OfflineRecognizer? = null - private var speakerId: FaceclawSpeakerId? = null - - // Segmentation state (worker thread only). - private var totalSamples: Long = 0 - private var noiseFloor = 0.0 - private var inUtterance = false - private var utteranceStartSample: Long = 0 - private var silenceRunSamples: Long = 0 - private var utterancePeakRms = 0.0 - private var utterance = ShortArray(SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)) - private var utteranceLength = 0 - private val preRoll = ShortArray(SAMPLE_RATE * PRE_ROLL_MS / 1000) - private var preRollLength = 0 + private val session = CaptionSession(host) fun setListener(listener: FaceclawCaptionEngineListener?) { - this.listener = listener + session.setListener(listener) } /** Directory holding the Moonshine model files, or null to disable ASR. */ fun setAsrModelDir(dir: String?) { - this.asrModelDir = dir + session.setAsrModelDir(dir) } /** Speaker-embedding ONNX model path, or null to disable voice-prints. */ fun setSpeakerModelPath(path: String?) { - this.speakerModelPath = path + session.setSpeakerModelPath(path) } fun setSilenceMs(ms: Int) { - this.silenceMs = Math.max(200, Math.min(3000, ms)) + session.setSilenceMs(ms) } fun start() { - synchronized(lock) { - if (started) { - return - } - started = true - workerThread = Thread(Runnable { runLoop() }, "FaceclawCaptionEngine") - workerThread!!.start() - } + session.start() } fun stop() { - val threadToJoin: Thread? - synchronized(lock) { - if (!started) { - return - } - started = false - threadToJoin = workerThread - workerThread = null - } - synchronized(queueLock) { - (queueLock as java.lang.Object).notifyAll() - } - if (threadToJoin != null) { - threadToJoin.interrupt() - if (Thread.currentThread() !== threadToJoin) { - try { - threadToJoin.join(2000) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - } - } - } + session.stop() } /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ fun acceptPcm(pcm16le: ByteArray?) { - if (!started || pcm16le == null || pcm16le.size < 2) { - return - } - synchronized(queueLock) { - if (queue.size >= MAX_QUEUE_PACKETS) { - queue.removeFirst() - } - queue.addLast(pcm16le) - (queueLock as java.lang.Object).notifyAll() - } - } - - private fun runLoop() { - try { - loadModels() - resetSegmentation() - while (started && !Thread.currentThread().isInterrupted) { - val chunk = takeChunk() - if (chunk == null) { - continue - } - processChunk(chunk) - } - // Flush a trailing utterance so its text isn't lost on stop. - if (inUtterance && utteranceLength > 0) { - finalizeUtterance() - } - } catch (t: Throwable) { - Log.e(TAG, "caption engine failed", t) - emitStatus("Captions failed: " + t.message) - } finally { - releaseModels() - } - } - - private fun loadModels() { - val modelDir = asrModelDir - if (modelDir != null && File(modelDir, "tokens.txt").exists()) { - emitStatus("Loading caption model...") - recognizer = OfflineRecognizer(OfflineRecognizerConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setModelConfig(OfflineModelConfig.builder() - .setMoonshine(OfflineMoonshineModelConfig.builder() - .setEncoder(File(modelDir, "encoder_model.ort").absolutePath) - .setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath) - .build()) - .setTokens(File(modelDir, "tokens.txt").absolutePath) - .setNumThreads(1) - .build()) - .build()) - } else { - recognizer = null - } - val embedModel = speakerModelPath - if (embedModel != null && File(embedModel).exists()) { - speakerId = FaceclawSpeakerId(embedModel) - speakerId!!.ensureLoaded() - } else { - speakerId = null - } - emitStatus(if (recognizer != null) "Captions listening..." else "Captions listening (no ASR model)...") - } - - private fun releaseModels() { - if (recognizer != null) { - recognizer!!.release() - recognizer = null - } - if (speakerId != null) { - speakerId!!.close() - speakerId = null - } - } - - private fun takeChunk(): ByteArray? { - synchronized(queueLock) { - while (started && queue.isEmpty()) { - try { - (queueLock as java.lang.Object).wait(250) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - return null - } - } - return queue.pollFirst() - } - } - - private fun resetSegmentation() { - totalSamples = 0 - noiseFloor = 0.0 - inUtterance = false - utteranceLength = 0 - preRollLength = 0 - silenceRunSamples = 0 - utterancePeakRms = 0.0 - } - - private fun processChunk(chunk: ByteArray) { - val count = chunk.size / 2 - val pcm = ShortArray(count) - var sumSquares = 0.0 - for (i in 0 until count) { - val s = ((chunk[i * 2].toInt() and 0xff) or (chunk[i * 2 + 1].toInt() shl 8)).toShort() - pcm[i] = s - sumSquares += s.toDouble() * s - } - val rms = Math.sqrt(sumSquares / Math.max(1, count)) - totalSamples += count.toLong() - - if (!inUtterance) { - // Track the noise floor only while idle so speech doesn't raise it. - noiseFloor = if (noiseFloor == 0.0) rms else noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA - val threshold = Math.max(noiseFloor, MIN_RMS) - if (rms >= threshold * ONSET_FACTOR) { - inUtterance = true - utteranceLength = 0 - silenceRunSamples = 0 - utterancePeakRms = rms - appendUtterance(preRoll, preRollLength) - utteranceStartSample = Math.max(0L, totalSamples - count - preRollLength) - appendUtterance(pcm, count) - emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE) - } else { - appendPreRoll(pcm, count) - } - return - } - - appendUtterance(pcm, count) - utterancePeakRms = Math.max(utterancePeakRms, rms) - val threshold = Math.max(noiseFloor, MIN_RMS) - if (rms < threshold * RELEASE_FACTOR) { - silenceRunSamples += count.toLong() - } else { - silenceRunSamples = 0 - } - val utteranceMs = utteranceLength.toLong() * 1000 / SAMPLE_RATE - val silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs - if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) { - finalizeUtterance() - inUtterance = false - preRollLength = 0 - } - } - - private fun appendPreRoll(pcm: ShortArray, count: Int) { - // Keep the last PRE_ROLL_MS of idle audio so onset consonants survive. - if (count >= preRoll.size) { - System.arraycopy(pcm, count - preRoll.size, preRoll, 0, preRoll.size) - preRollLength = preRoll.size - return - } - val keep = Math.min(preRollLength, preRoll.size - count) - System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep) - System.arraycopy(pcm, 0, preRoll, keep, count) - preRollLength = keep + count - } - - private fun appendUtterance(pcm: ShortArray, count: Int) { - val room = utterance.size - utteranceLength - val copied = Math.min(room, count) - if (copied > 0) { - System.arraycopy(pcm, 0, utterance, utteranceLength, copied) - utteranceLength += copied - } - } - - private fun finalizeUtterance() { - val length = utteranceLength - val startMs = utteranceStartSample * 1000 / SAMPLE_RATE - val endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE - if (endMs - startMs < MIN_UTTERANCE_MS) { - return - } - val text = recognizeUtterance(length) - val embedding = embedUtterance(length) - emitUtterance(text, embedding, startMs, endMs, utterancePeakRms) - } - - private fun recognizeUtterance(length: Int): String { - if (recognizer == null || length <= 0) { - return "" - } - val joined = StringBuilder() - var offset = 0 - while (offset < length) { - val remaining = length - offset - var segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES) - if (remaining > DECODE_SEGMENT_MAX_SAMPLES) { - segment = findQuietCut(offset, segment) - } - val part = recognizeRange(offset, segment) - if (part.length > 0) { - if (joined.length > 0 && ".,!?;:%)]}".indexOf(part[0]) < 0) { - joined.append(' ') - } - joined.append(part) - } - offset += segment - } - return joined.toString().trim() - } - - /** - * End a decode segment at the center of the quietest window near its end - * so the cut lands between words (same approach as the PTT controller). - */ - private fun findQuietCut(offset: Int, segment: Int): Int { - val searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES) - val win = CUT_WINDOW_SAMPLES - if (segment - searchStart <= win) { - return segment - } - var sum = 0.0 - for (i in searchStart until searchStart + win) { - val s = utterance[offset + i].toDouble() - sum += s * s - } - var best = sum - var bestStart = searchStart - var start = searchStart + 1 - while (start + win <= segment) { - val dropped = utterance[offset + start - 1].toDouble() - val added = utterance[offset + start + win - 1].toDouble() - sum += added * added - dropped * dropped - if (sum < best) { - best = sum - bestStart = start - } - start++ - } - return bestStart + win / 2 - } - - private fun recognizeRange(offset: Int, count: Int): String { - val samples = FloatArray(count) - var peak = 0f - for (i in 0 until count) { - val v = utterance[offset + i] / 32768.0f - samples[i] = v - val a = Math.abs(v) - if (a > peak) { - peak = a - } - } - if (peak > 0f) { - val gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN) - if (gain > 1f) { - for (i in 0 until count) { - samples[i] *= gain - } - } - } - val recognizer = this.recognizer!! - val stream: OfflineStream = recognizer.createStream() - try { - stream.acceptWaveform(samples, SAMPLE_RATE) - recognizer.decode(stream) - val result: OfflineRecognizerResult? = recognizer.getResult(stream) - val raw: String? = if (result == null) "" else result.text - return raw?.trim() ?: "" - } finally { - stream.release() - } - } - - private fun embedUtterance(length: Int): FloatArray? { - val currentSpeakerId = speakerId - if (currentSpeakerId == null || length <= 0) { - return null - } - val count = Math.min(length, EMBED_MAX_SAMPLES) - val le = ByteArray(count * 2) - for (i in 0 until count) { - val s = utterance[i].toInt() - le[i * 2] = (s and 0xff).toByte() - le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() - } - return currentSpeakerId.embed(le, SAMPLE_RATE) - } - - private fun emitUtterance(text: String?, embedding: FloatArray?, startMs: Long, endMs: Long, peakRms: Double) { - val currentListener = listener - if (currentListener == null) { - return - } - val embeddingCopy = if (embedding == null) null else Arrays.copyOf(embedding, embedding.size) - mainHandler.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) } - } - - private fun emitSpeechStart(startMs: Long) { - val currentListener = listener - if (currentListener == null) { - return - } - mainHandler.post { currentListener.onSpeechStart(startMs) } - } - - private fun emitStatus(status: String?) { - val currentListener = listener - if (currentListener == null) { - return - } - mainHandler.post { currentListener.onStatus(status) } + session.acceptPcm(pcm16le) } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt index d9002ede..2d9f316f 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt @@ -13,8 +13,6 @@ import org.json.JSONException import org.json.JSONObject import java.io.File -import java.nio.ByteBuffer -import java.nio.ByteOrder import java.util.ArrayList /** @@ -47,41 +45,6 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp } } - // ---- blob helpers ---- - - private fun blobToFloats(blob: ByteArray?): FloatArray? { - if (blob == null || blob.size < 4 || blob.size % 4 != 0) { - return null - } - val buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN) - val out = FloatArray(blob.size / 4) - for (i in out.indices) { - out[i] = buffer.getFloat() - } - return out - } - - private fun floatsToBlob(values: FloatArray): ByteArray { - val buffer = ByteBuffer.allocate(values.size * 4).order(ByteOrder.LITTLE_ENDIAN) - for (v in values) { - buffer.putFloat(v) - } - return buffer.array() - } - - private fun l2NormalizeInPlace(v: FloatArray) { - var sum = 0.0 - for (x in v) { - sum += x.toDouble() * x - } - val norm = Math.sqrt(sum) - if (norm <= 0) { - return - } - for (i in v.indices) { - v[i] = (v[i] / norm).toFloat() - } - } } override fun onCreate(db: SQLiteDatabase) { @@ -567,7 +530,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp fun updateSpeakerEmbedding(speakerId: Long, embeddingBase64: String?, maxCount: Int) { try { val incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP) - val update = blobToFloats(incoming) + val update = SpeakerEmbeddings.blobToFloats(incoming) if (update == null) { return } @@ -578,28 +541,16 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp arrayOf(speakerId.toString())) try { if (c.moveToFirst()) { - current = if (c.isNull(0)) null else blobToFloats(c.getBlob(0)) + current = if (c.isNull(0)) null else SpeakerEmbeddings.blobToFloats(c.getBlob(0)) count = c.getInt(1) } } finally { c.close() } - val merged: FloatArray - if (current == null || current.size != update.size) { - merged = update - count = 1 - } else { - val effective = Math.min(count, Math.max(1, maxCount)) - merged = FloatArray(current.size) - for (i in merged.indices) { - merged[i] = (current[i] * effective + update[i]) / (effective + 1) - } - l2NormalizeInPlace(merged) - count = count + 1 - } + val centroid = SpeakerEmbeddings.runningMean(current, count, update, maxCount) val values = ContentValues() - values.put("embedding", floatsToBlob(merged)) - values.put("embedding_count", count) + values.put("embedding", SpeakerEmbeddings.floatsToBlob(centroid.embedding)) + values.put("embedding_count", centroid.count) values.put("last_heard_at", System.currentTimeMillis()) writableDatabase.update("speakers", values, "id=?", arrayOf(speakerId.toString())) @@ -630,7 +581,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp try { while (c.moveToNext()) { val id = c.getLong(0) - val embedding = if (c.isNull(1)) null else blobToFloats(c.getBlob(1)) + val embedding = if (c.isNull(1)) null else SpeakerEmbeddings.blobToFloats(c.getBlob(1)) val count = c.getInt(2) if (id == fromId) { from = embedding @@ -644,21 +595,10 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp } finally { c.close() } - var merged: FloatArray? = into - if (from != null && into != null && from.size == into.size) { - val blended = FloatArray(into.size) - val total = Math.max(1, fromCount + intoCount) - for (i in blended.indices) { - blended[i] = (into[i] * intoCount + from[i] * fromCount) / total - } - l2NormalizeInPlace(blended) - merged = blended - } else if (into == null) { - merged = from - } + val merged = SpeakerEmbeddings.blend(from, fromCount, into, intoCount) val values = ContentValues() if (merged != null) { - values.put("embedding", floatsToBlob(merged)) + values.put("embedding", SpeakerEmbeddings.floatsToBlob(merged)) values.put("embedding_count", fromCount + intoCount) } if (fromWearer) { @@ -677,7 +617,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp val hasInsights = !c2.isNull(0) && !c2.getString(0).isEmpty() if (c2.position == 0) { targetHasInsights = hasInsights - } else if (!targetHasInsights && hasInsights) { + } else if (SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights, hasInsights)) { values.put("last_recap", c2.getString(0)) values.put("action_items", if (c2.isNull(1)) "" else c2.getString(1)) values.put("facts", if (c2.isNull(2)) "" else c2.getString(2)) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt index 9d734da2..5568a6ce 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.kt @@ -1,99 +1,44 @@ package com.faceclaw.app -import android.bluetooth.BluetoothDevice -import android.bluetooth.BluetoothGatt import android.content.Context import android.os.Handler import android.os.Looper -import android.os.SystemClock -import android.util.Log - -import java.util.HashSet -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit /** - * Connects to the glasses (stock-firmware compatible), reads the device-info / - * settings response, and reports the firmware versions plus the firmware- - * extension string (empty on stock firmware). Used by onboarding to decide whether to - * flash. Reuses FaceclawBleManager + BleProtocol; owns its own connection and - * runs on a single worker thread. Shows nothing on the lens. + * Android host for the shared [DeviceInfoProbeFlow]: owns the GATT link (AndroidStockLink), + * the worker thread the flow runs on, and delivers listener callbacks on the main thread. + * Public API unchanged for the TypeScript wrapper (app/native/device-info-probe.ts). */ -class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) : FaceclawBleListener { - companion object { - private const val TAG = "FaceclawDeviceInfo" - private const val QUERY_TIMEOUT_MS = 4_000 - /** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */ - private const val ARM_ATTEMPTS = 3 - private const val ARM_RETRY_DELAY_MS = 1_000 - /** - * Upper bound on waiting for the auth success while Android reports the arm - * as BOND_BONDING — i.e. an OS pairing dialog may be sitting there waiting - * for the user, which can take a lot longer than the normal 30 s window. - */ - private const val PAIRING_WAIT_CAP_MS = 90_000 - /** After a fresh bond, how long the firmware gets to notify success on its own before the request is re-sent. */ - private const val POST_BOND_GRACE_MS = 2_000 - private const val POLL_MS = 250 - - private fun bondStateName(state: Int): String { - return when (state) { - BluetoothDevice.BOND_NONE -> "none" - BluetoothDevice.BOND_BONDING -> "bonding" - BluetoothDevice.BOND_BONDED -> "bonded" - else -> "unknown" - } - } - } - - private val context: Context - private val rightAddress: String - private val leftAddress: String - private val bleManager: FaceclawBleManager +class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) { + private val link = AndroidStockLink(context.applicationContext) private val mainHandler = Handler(Looper.getMainLooper()) - private val lock = Any() - private var nextSeq = 0x40 - private var nextMagic = 100 - - private var awaitSid = -1 - private var awaitMagic = -1 - private var awaitPb: ByteArray? = null - private var awaitLatch: CountDownLatch? = null - - /** - * Security-auth wait, kept separate from the generic ack wait: only the - * firmware's SUCCESS result (empty result message) may satisfy it. The - * firmware answers the request as soon as it arrives, but with a - * non-success result until the link is encrypted; accepting that first - * reply as "the ack" is what let the probe move on to the other arm while - * Android's pairing dialog for this one was still up. - */ - private val authMagics: MutableSet = HashSet() - private var authLatch: CountDownLatch? = null - - /** - * A sid-0x09 frame that carried firmware versions without matching the - * awaited magic — some firmware pushes the settings snapshot on a magic it - * picked itself instead of (or in addition to) acking the read. - */ - @Volatile - private var unsolicitedSettingsPb: ByteArray? = null @Volatile private var listener: FaceclawDeviceInfoProbeListener? = null + @Volatile private var worker: Thread? = null - @Volatile - private var cancelled = false - init { - this.context = context.applicationContext - this.rightAddress = rightAddress ?: "" - this.leftAddress = leftAddress ?: "" - this.bleManager = FaceclawBleManager(this.context) - this.bleManager.setListener(this) - } + private val flow = + DeviceInfoProbeFlow( + link, + rightAddress ?: "", + leftAddress ?: "", + object : FaceclawDeviceInfoProbeListener { + override fun onLog(line: String?) { + android.util.Log.i(DeviceInfoProbeFlow.TAG, line ?: "") + post { it.onLog(line) } + } + + override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") } + + override fun onResult(leftVersion: String?, rightVersion: String?, extension: String?) = + post { it.onResult(leftVersion, rightVersion, extension) } + + override fun onError(message: String?) = post { it.onError(message ?: "") } + }, + ) fun setListener(listener: FaceclawDeviceInfoProbeListener?) { this.listener = listener @@ -104,525 +49,29 @@ class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddre if (worker != null) { return } - val w = Thread({ run() }, "faceclaw-device-info") + val w = Thread({ flow.run() }, "faceclaw-device-info") worker = w w.start() } } fun cancel() { - cancelled = true - val w = worker - if (w != null) { - w.interrupt() - } + flow.cancel() } fun close() { cancel() try { - bleManager.close() + link.close() } catch (ignored: Exception) { } } - private fun run() { - try { - if (rightAddress.trim().isEmpty()) { - emitError("No right-arm address configured.") - return - } - - val haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equals(rightAddress, ignoreCase = true) - - // Bring up and authenticate BOTH arms before asking for versions. - // Each arm is its own peripheral with its own bond, so on a fresh - // phone each raises its own pairing prompt; completing both here - // means the flash step (and the app proper) find both bonds in - // place. Strictly one arm at a time: Android pairs with one device - // at a time, and a second arm connected while the first was still - // pairing has been seen to drop within a second and never pair - // (2026-09-11 logcat). - var rightAuthenticated = bringUpArm(rightAddress, "right") - var leftAuthenticated = false - if (haveLeft && !cancelled) { - leftAuthenticated = bringUpArm(leftAddress, "left") - } - if (cancelled) { - emitError("Cancelled.") - return - } - if (!bleManager.isConnected(rightAddress)) { - // The right lens may have dropped while the left one paired; - // it is bonded by now, so this comes back quickly. - emitLog("right lens disconnected while the left lens was set up; reconnecting") - rightAuthenticated = bringUpArm(rightAddress, "right") - if (cancelled) { - emitError("Cancelled.") - return - } - } - - var ack: ByteArray? = null - var rightFailure: String? = null - try { - ack = queryArm(rightAddress, "right", rightAuthenticated) - } catch (e: IllegalStateException) { - rightFailure = if (e.message == null) e.toString() else e.message - emitLog("right-lens query failed: $rightFailure") - } - // The right lens is the documented control endpoint, but a silent - // right lens has been observed on stock 2.2.9 even after a - // successful security auth — run the same probe against the left - // lens rather than giving up, and log which lens answered. - if (ack == null && !cancelled && haveLeft) { - emitLog("right lens did not answer the settings query; probing the left lens") - try { - ack = queryArm(leftAddress, "left", leftAuthenticated) - } catch (e: Exception) { - emitLog("left-lens probe failed: " + (if (e.message == null) e.toString() else e.message)) - } - } - if (cancelled) { - emitError("Cancelled.") - return - } - if (ack == null) { - throw IllegalStateException(if (rightFailure != null) - rightFailure - else "no response to the device-info query on either lens — check `adb logcat -s " - + TAG + "` for the frame trace") - } - - val info: BleProtocol.FirmwareInfo? = BleProtocol.parseSettingsFirmwareInfo(ack) - val left = if (info == null) "" else info.leftVersion - val right = if (info == null) "" else info.rightVersion - val extension = if (info == null) "" else info.extension - emitLog("device-info: L=$left R=$right ext=[$extension]") - emitResult(left, right, extension) - } catch (e: Exception) { - val message = if (cancelled) "Cancelled." else (if (e.message == null) e.toString() else e.message) - emitError(message) - } finally { - try { - bleManager.close() - } catch (ignored: Exception) { - } - } - } - - /** - * Connect to one arm and complete its security-auth exchange, retrying - * when the link drops mid-pairing. Returns whether the auth success was - * seen (false = connected but unconfirmed, tolerated because the custom - * firmware's response to the exchange is not yet hardware-verified); - * throws when the arm cannot be brought up at all. - */ - @Throws(InterruptedException::class) - private fun bringUpArm(address: String, label: String): Boolean { - var lastFailure: String? = null - var attempt = 1 - while (attempt <= ARM_ATTEMPTS && !cancelled) { - if (attempt > 1) { - // A link Android dropped while it was still pairing comes back - // once the bond settles; reconnecting sooner just fails again. - waitForBondToSettle(address, label) - if (cancelled) { - break - } - emitLog("retrying the $label lens (attempt $attempt/$ARM_ATTEMPTS): $lastFailure") - Thread.sleep(ARM_RETRY_DELAY_MS.toLong()) - } - emitState("connecting", label) - try { - connectArm(address) - } catch (e: IllegalStateException) { - lastFailure = if (e.message == null) e.toString() else e.message - emitLog("$label lens: $lastFailure") - attempt++ - continue - } - if (cancelled) { - break - } - - // Firmware 2.2.9 answers no queries until the security-auth - // exchange completes over an encrypted link; on a phone with no - // existing bond this is also what triggers SMP pairing (and its OS - // prompt), so it must come before the prelude and query. - emitState("authenticating", label) - val result = authenticate(address, label) - if (result == AuthResult.SUCCESS) { - return true - } - if (result == AuthResult.UNCONFIRMED) { - return false - } - lastFailure = "link dropped during authentication" - attempt++ - } - if (cancelled) { - return false - } - throw IllegalStateException("could not connect to the $label lens ($address): " - + lastFailure + " — if Android showed a Bluetooth pairing request, accept it and try again") - } - - private enum class AuthResult { SUCCESS, UNCONFIRMED, LINK_DROPPED } - - /** - * Send the sid-0x80 authentication request and wait for the firmware's - * SUCCESS notification, which only arrives once the link is encrypted. - * On an unbonded phone that means waiting through Android's pairing flow - * (BOND_BONDING, possibly with a dialog up), so the wait is extended while - * the OS reports pairing in progress and the request is re-sent once - * after a fresh bond in case the pre-pairing write was dropped. Bond - * state is polled rather than awaited via broadcast; 250 ms granularity - * is plenty here. - */ - @Throws(InterruptedException::class) - private fun authenticate(address: String, label: String): AuthResult { - val start = SystemClock.elapsedRealtime() - var deadline = start + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS - val hardDeadline = start + PAIRING_WAIT_CAP_MS - val latch = CountDownLatch(1) - synchronized(lock) { - authMagics.clear() - authLatch = latch - } - val initialBond = bleManager.getBondState(address) - emitLog("security auth: $label lens, Android bond state " + bondStateName(initialBond)) - try { - var sends = 0 - var lastSendFailed = false - var nextSendAt = start - var bondedAt = -1L - var resentAfterBond = false - var loggedBonding = false - while (!cancelled) { - val now = SystemClock.elapsedRealtime() - val bond = bleManager.getBondState(address) - - if (bond == BluetoothDevice.BOND_BONDING) { - // OS pairing in progress — the firmware will answer once the - // link is encrypted. Don't write, don't give up, don't move - // on to the other arm. - if (!loggedBonding) { - emitLog("Android is pairing with the $label lens; waiting for the user to accept") - loggedBonding = true - } - if (now >= hardDeadline) { - break - } - } else { - if (bond == BluetoothDevice.BOND_BONDED && initialBond != BluetoothDevice.BOND_BONDED - && bondedAt < 0) { - bondedAt = now - emitLog("Android bond established with the $label lens; waiting for the auth success") - deadline = Math.max(deadline, now + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS) - } - val wantSend = sends == 0 - || (lastSendFailed && sends < 3 && now >= nextSendAt) - || (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= POST_BOND_GRACE_MS) - if (wantSend) { - if (bondedAt >= 0) { - resentAfterBond = true - if (sends > 0) { - emitLog("re-sending the auth request after pairing ($label lens)") - } - } - lastSendFailed = !sendAuthRequest(address) - sends++ - nextSendAt = SystemClock.elapsedRealtime() + ARM_RETRY_DELAY_MS - if (lastSendFailed && !bleManager.isConnected(address)) { - return AuthResult.LINK_DROPPED - } - } - if (now >= deadline) { - break - } - } - - if (latch.await(POLL_MS.toLong(), TimeUnit.MILLISECONDS)) { - emitLog("security auth complete: $label lens ($address)") - return AuthResult.SUCCESS - } - if (!bleManager.isConnected(address)) { - emitLog("security auth: $label lens disconnected while waiting" - + (if (bond == BluetoothDevice.BOND_BONDING) " (Android was still pairing)" else "")) - return AuthResult.LINK_DROPPED - } - } - if (cancelled) { - return AuthResult.UNCONFIRMED - } - emitLog("security auth unconfirmed: $label lens ($address), Android bond state " - + bondStateName(bleManager.getBondState(address))) - return AuthResult.UNCONFIRMED - } finally { - synchronized(lock) { - authLatch = null - authMagics.clear() - } - } - } - - private fun sendAuthRequest(address: String): Boolean { - val magic = allocMagic() - val seq: Int - synchronized(lock) { - authMagics.add(magic) - seq = nextSeq++ and 0xff - } - val payload = BleProtocol.buildAuthenticationRequest(magic) - emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", - address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, magic, seq, payload.size)) - val frames = BleProtocol.framePb(payload, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, seq) - var written: Boolean - try { - written = bleManager.writeFrames( - address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS) - } catch (e: IllegalStateException) { - // "Not connected" — the link went away underneath us. - written = false - } - if (!written) { - emitLog("tx write FAILED sid=0x80 ($address)") - } - return written - } - - /** Block while Android reports this arm as BOND_BONDING (pairing dialog up / SMP in flight). */ - @Throws(InterruptedException::class) - private fun waitForBondToSettle(address: String, label: String) { - val cap = SystemClock.elapsedRealtime() + PAIRING_WAIT_CAP_MS - var logged = false - while (!cancelled && bleManager.getBondState(address) == BluetoothDevice.BOND_BONDING - && SystemClock.elapsedRealtime() < cap) { - if (!logged) { - emitLog("waiting for Android to finish pairing with the $label lens before reconnecting") - logged = true - } - Thread.sleep(POLL_MS.toLong()) - } - } - - /** - * Prelude + settings read on an already connected/authenticated lens. - * Returns the settings ack protobuf, an unsolicited settings push that - * carried firmware versions, or null when the lens never answered the read. - * Throws on prelude failure; the caller wraps the fallback lens's attempt - * so its failure cannot mask the primary lens's outcome. - */ - @Throws(InterruptedException::class) - private fun queryArm(address: String, label: String, authenticated: Boolean): ByteArray? { - if (cancelled) { - return null - } - - emitState("querying", label) - // Session prelude, then a settings/device-info read (both arms' - // versions and the firmware-extension string ride back in one response). - if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST, - BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD, - ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { - throw IllegalStateException("session prelude not acked ($label lens)" - + (if (authenticated) "" else "; authentication did not complete —" - + " if Android shows a Bluetooth pairing request, accept it and try again")) - } - - // Two attempts: the first read straight after a fresh pairing has been - // seen to go unanswered while a later one succeeds. - var attempt = 0 - while (attempt < 2 && !cancelled) { - val magic = allocMagic() - val ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, - magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS) - if (ack != null) { - return ack - } - // A push with the firmware versions on the device's own magic is - // as good as the ack we asked for. - val pushed = unsolicitedSettingsPb - if (pushed != null) { - emitLog("using unsolicited settings push instead of the read ack ($label lens)") - return pushed - } - emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)") - attempt++ - } - return null - } - - private fun connectArm(address: String) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw IllegalStateException("connect failed") - } - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw IllegalStateException("service discovery failed") - } - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - throw IllegalStateException("could not subscribe to notifications") - } - } - - @Throws(InterruptedException::class) - private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? { - val latch = CountDownLatch(1) - synchronized(lock) { - awaitSid = sid - awaitMagic = magic - awaitPb = null - awaitLatch = latch - } - val seq: Int - synchronized(lock) { - seq = nextSeq++ and 0xff - } - emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", - address, sid, flag, magic, seq, payload.size)) - val frames = BleProtocol.framePb(payload, sid, flag, seq) - val written = bleManager.writeFrames( - address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS) - if (!written) { - emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid)) - synchronized(lock) { - awaitLatch = null - } - return null - } - val acked = latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS) - synchronized(lock) { - awaitLatch = null - return if (acked) awaitPb else null - } - } - - override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { - if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { - return - } - // One notification value can carry several envelope frames back to - // back; reading only the first would silently drop the rest. - val frames = BleProtocol.splitFrames(data) - if (frames.size > 1) { - emitLog("rx " + address + " value carries " + frames.size + " frames (raw " + data!!.size + " bytes)") - } - for (buf in frames) { - handleFrame(address, buf, data!!.size) - } - } - - private fun handleFrame(address: String?, buf: ByteArray, rawValueLength: Int) { - val frame = BleProtocol.parseFrame(buf) - if (!frame.ok) { - emitLog("rx " + address + " unparseable frame len=" + buf.size + " (raw value " + rawValueLength + ")" - + " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.size)))) - return - } - // Log every control frame while diagnosing 2.2.9: sid/flag/type/magic - // plus a payload prefix is enough to reconstruct what the lens said. - val declared = if (buf.size > 3) buf[3].toInt() and 0xff else 0 - val truncated = if (buf.size < 8 + declared) " TRUNCATED(declared=$declared)" else "" - emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s", - address, frame.sid, frame.flag, frame.msgType, frame.msgSeq, - if (buf.size > 5) buf[5].toInt() and 0xff else 0, if (buf.size > 4) buf[4].toInt() and 0xff else 0, frame.pb.size, truncated, - FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.size))))) - if (frame.sid == BleProtocol.SID_UI_SETTING - && BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) { - // Any settings frame carrying firmware versions answers the probe's - // question, whether or not it matches the magic we asked with. - unsolicitedSettingsPb = frame.pb - } - if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { - // Async event, not an ack. This includes the lens's own periodic - // sid-0x80 notifies (device-chosen magic, non-empty result). - return - } - if (frame.sid == BleProtocol.SID_SECURITY_AUTH) { - var success = false - var ours = false - synchronized(lock) { - val currentLatch = authLatch - if (currentLatch != null) { - ours = authMagics.contains(frame.msgSeq) - for (magic in authMagics) { - if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) { - success = true - currentLatch.countDown() - break - } - } - } - } - if (ours && !success) { - // Expected before the link is encrypted: the firmware answers - // the request straight away with a non-success result and - // sends the real success once pairing/encryption completes. - emitLog("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting") - } - return - } - synchronized(lock) { - val currentLatch = awaitLatch - if (currentLatch != null && frame.sid == awaitSid && frame.msgSeq == awaitMagic) { - awaitPb = frame.pb - currentLatch.countDown() - } - } - } - - override fun onConnectionStateChange(address: String?, connected: Boolean) { - if (!connected) { - Log.i(TAG, "disconnected: $address") - } - } - - private fun allocMagic(): Int { - synchronized(lock) { - val magic = nextMagic - nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1 - return magic - } - } - - private fun emitLog(line: String) { - Log.i(TAG, line) + private fun post(action: (FaceclawDeviceInfoProbeListener) -> Unit) { mainHandler.post { val current = listener if (current != null) { - current.onLog(line) - } - } - } - - private fun emitState(state: String, detail: String?) { - val safeDetail = detail ?: "" - mainHandler.post { - val current = listener - if (current != null) { - current.onState(state, safeDetail) - } - } - } - - private fun emitResult(left: String, right: String, extension: String) { - mainHandler.post { - val current = listener - if (current != null) { - current.onResult(left, right, extension) - } - } - } - - private fun emitError(message: String?) { - val safeMessage = message ?: "" - mainHandler.post { - val current = listener - if (current != null) { - current.onError(safeMessage) + action(current) } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt index 95cd7e98..86c911d6 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.kt @@ -129,17 +129,19 @@ class FaceclawDiarizer( val file = File(path) val fileLength = file.length() DataInputStream(FileInputStream(file)).use { `in` -> - val header = ByteArray(44) - `in`.readFully(header) - if (header[0] != 'R'.code.toByte() || header[1] != 'I'.code.toByte() || header[8] != 'W'.code.toByte()) { - throw IOException("not a WAV file") + // Shared RIFF walk (handles extra chunks and unpatched data sizes); the + // stream is then positioned at the first PCM sample. + val prefix = ByteArray(Math.min(fileLength, WavPcmReader.HEADER_PREFIX_BYTES.toLong()).toInt()) + `in`.readFully(prefix) + val info = try { + WavPcmReader.parseHeader(prefix, fileLength) + } catch (e: IllegalArgumentException) { + throw IOException(e.message ?: "not a WAV file") } - val channels = (header[22].toInt() and 0xff) or (header[23].toInt() shl 8) - val bits = (header[34].toInt() and 0xff) or (header[35].toInt() shl 8) - if (channels != 1 || bits != 16) { + if (info.channels != 1) { throw IOException("only 16-bit mono WAV is supported") } - val dataBytes = fileLength - 44 + val dataBytes = info.dataBytes val count = Math.min(dataBytes / 2, Int.MAX_VALUE.toLong()).toInt() val samples = FloatArray(count) val chunk = ByteArray(65536) @@ -147,6 +149,21 @@ class FaceclawDiarizer( var carried = 0 var carriedByte: Byte = 0 var read = 0 + // The prefix already holds the start of the PCM data; consume it first. + val prefixData = prefix.size - info.dataOffset.toInt() + if (prefixData > 0) { + val pairs = prefixData / 2 + var i = 0 + while (i < pairs && sampleIndex < count) { + val base = info.dataOffset.toInt() + i * 2 + samples[sampleIndex++] = AudioSegmentation.pcm16le(prefix[base], prefix[base + 1]) / 32768.0f + i++ + } + if ((prefixData and 1) == 1) { + carried = 1 + carriedByte = prefix[prefix.size - 1] + } + } while (sampleIndex < count && `in`.read(chunk).also { read = it } > 0) { var offset = 0 if (carried == 1) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt index bec2b8a3..af821b0b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt @@ -15,7 +15,6 @@ import java.io.FileInputStream import java.io.IOException import java.io.InputStream import java.nio.charset.StandardCharsets -import java.util.Locale /** * Serves an unpacked EvenHub app's files to its WebView from a fake per-app @@ -52,25 +51,25 @@ class FaceclawEvenHubWebViewClient( if (url == null || host != url.host) { return null // External hosts: normal network handling. } - var path = url.path - if (path == null || path.isEmpty() || path == "/") { - path = "/index.html" - } + val path = url.path try { + // Path rules, MIME table and shim injection are shared (EvenHubAssetServer); + // the canonical-file check below additionally defeats symlinks. + val asset = EvenHubAssetServer.resolve(path) val root = File(rootDir).canonicalFile - val target = File(root, path.substring(1)).canonicalFile - if (!target.path.startsWith(root.path) || !target.isFile) { + val target = asset?.let { File(root, it.relativePath).canonicalFile } + if (asset == null || target == null || !EvenHubAssetServer.isInsideRoot(root.path, target.path) || !target.isFile) { Log.w(TAG, "404 $path") return WebResourceResponse( "text/plain", "utf-8", ByteArrayInputStream("not found".toByteArray(StandardCharsets.UTF_8))) } - val mime = mimeTypeFor(target.name) - if (mime == "text/html") { - return WebResourceResponse(mime, "utf-8", ByteArrayInputStream(injectIntoHtml(target))) + if (asset.isHtml) { + val html = EvenHubAssetServer.injectIntoHtml(String(readAll(target), StandardCharsets.UTF_8), injectScript) + return WebResourceResponse(asset.mime, "utf-8", ByteArrayInputStream(html.toByteArray(StandardCharsets.UTF_8))) } val stream: InputStream = FileInputStream(target) - return WebResourceResponse(mime, null, stream) + return WebResourceResponse(asset.mime, null, stream) } catch (e: IOException) { Log.e(TAG, "serve failed for $path", e) return WebResourceResponse( @@ -94,30 +93,9 @@ class FaceclawEvenHubWebViewClient( } } - @Throws(IOException::class) - private fun injectIntoHtml(file: File): ByteArray { - val raw = readAll(file) - val html = String(raw, StandardCharsets.UTF_8) - val tag = "" - // After if present, else before everything. - val headIndex = indexOfIgnoreCase(html, "= 0) { - val close = html.indexOf('>', headIndex) - if (close >= 0) { - return (html.substring(0, close + 1) + tag + html.substring(close + 1)) - .toByteArray(StandardCharsets.UTF_8) - } - } - return (tag + html).toByteArray(StandardCharsets.UTF_8) - } - companion object { private const val TAG = "FaceclawEvenHub" - private fun indexOfIgnoreCase(haystack: String, needle: String): Int { - return haystack.lowercase(Locale.ROOT).indexOf(needle) - } - @Throws(IOException::class) private fun readAll(file: File): ByteArray { val input = FileInputStream(file) @@ -134,25 +112,5 @@ class FaceclawEvenHubWebViewClient( input.close() } } - - private fun mimeTypeFor(name: String): String { - val lower = name.lowercase(Locale.ROOT) - if (lower.endsWith(".html") || lower.endsWith(".htm")) return "text/html" - if (lower.endsWith(".js") || lower.endsWith(".mjs")) return "application/javascript" - if (lower.endsWith(".css")) return "text/css" - if (lower.endsWith(".json")) return "application/json" - if (lower.endsWith(".png")) return "image/png" - if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg" - if (lower.endsWith(".gif")) return "image/gif" - if (lower.endsWith(".svg")) return "image/svg+xml" - if (lower.endsWith(".webp")) return "image/webp" - if (lower.endsWith(".ico")) return "image/x-icon" - if (lower.endsWith(".wasm")) return "application/wasm" - if (lower.endsWith(".woff")) return "font/woff" - if (lower.endsWith(".woff2")) return "font/woff2" - if (lower.endsWith(".ttf")) return "font/ttf" - if (lower.endsWith(".txt") || lower.endsWith(".map")) return "text/plain" - return "application/octet-stream" - } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt index 714b3377..6169c965 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.kt @@ -1,185 +1,54 @@ package com.faceclaw.app -import android.bluetooth.BluetoothGatt import android.content.Context import android.os.Handler import android.os.Looper -import android.os.SystemClock -import android.util.Log - -import java.io.ByteArrayOutputStream -import java.io.FileInputStream -import java.io.IOException -import java.nio.charset.StandardCharsets -import java.util.ArrayList -import java.util.Arrays -import java.util.concurrent.CountDownLatch -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit -import java.util.concurrent.TimeoutException /** - * Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA - * firmware container to each lens over the firmware-data service using the - * aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol. - * - * Each lens connection completes the sid-0x80 security-auth exchange before - * OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing - * the transfer mid-component. No control-channel traffic is injected during - * the transfer itself: official OTA captures show none between BEGIN and the - * final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a - * concurrent control writer would interleave with the marker/data - * transactions. (See ../notes/ble-connections-2.2.9.md.) - * - * Lenses are flashed one at a time (left, then right). Finishing either lens - * reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect - * for it uses a generous window rather than failing fast. - * - * Separate from FaceclawBleCommunicator on purpose: this speaks the stock OTA - * protocol and must run before the custom firmware exists. It reuses the GATT - * wrapper (FaceclawBleManager) and framing (BleProtocol). + * Android host for the shared [OtaFlashFlow] (g2flash.py's OTA procedure over the stock + * firmware-data service): owns the GATT link (AndroidStockLink) and the worker thread, and + * delivers listener callbacks on the main thread. Separate from FaceclawBleCommunicator on + * purpose: this speaks the stock OTA protocol and must run before the custom firmware exists. + * Public API unchanged for app/native/firmware-flasher.ts. */ -class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) : FaceclawBleListener { - companion object { - private const val TAG = "FaceclawFlasher" - - // OTA message types (envelope sid byte) and control opcodes. - private const val SID_CTRL = 0xc0 - private const val SID_DATA = 0xc1 - private const val FLAG_OTA = 0x00 - private const val OP_BEGIN = 0x00 - private const val OP_FILE_CHECK = 0x01 - private const val OP_BLOCK = 0x02 - private const val OP_END = 0x03 - - private const val BLOCK_SIZE = 4096 - private const val BLOCK_ACK_TIMEOUT_MS = 4_000 - private const val CTRL_ACK_TIMEOUT_MS = 8_000 - private const val BLOCK_NAK_RETRIES = 3 - private const val COMPONENT_RETRIES = 3 - - // Reconnect windows. The first lens connects from an idle device; the second - // must ride out the post-first-lens reboot of both lenses. - private const val FIRST_LENS_CONNECT_WINDOW_MS = 30_000 - private const val SECOND_LENS_CONNECT_WINDOW_MS = 120_000 - private const val REBOOT_SETTLE_MS = 5_000 - private const val NOTIFY_SETTLE_MS = 2_500 - private const val RETRY_DELAY_MS = 2_500 - - // END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART. - private val END_OK = intArrayOf(0, 8, 9) - - // Firmware layout expectations (mirrors g2flash.py). Firmware 2.2.4 has 5 - // components; 2.2.6 has 6. - private const val MIN_SEGMENTS = 5 - private const val MAX_SEGMENTS = 6 - private const val REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin" - private const val APP_LOAD_ADDR = 0x00438000L - private const val APP_MAX_END = 0x007F0000L - private const val APP_PREAMBLE = 0x20 - - // ---- firmware container parsing / validation ----------------------------- - - private val EMPTY = ByteArray(0) - - // ---- helpers ------------------------------------------------------------- - - private fun isEndOk(status: Int): Boolean { - for (ok in END_OK) { - if (ok == status) { - return true - } - } - return false - } - - private fun readFile(path: String): ByteArray { - try { - FileInputStream(path).use { fis -> - val out = ByteArrayOutputStream() - val buf = ByteArray(65536) - var read: Int - while (fis.read(buf).also { read = it } >= 0) { - out.write(buf, 0, read) - } - return out.toByteArray() - } - } catch (e: IOException) { - throw IllegalStateException("could not read firmware: " + e.message, e) - } - } - - private fun readU32(buf: ByteArray, offset: Int): Long { - return (buf[offset].toLong() and 0xffL) or - ((buf[offset + 1].toLong() and 0xffL) shl 8) or - ((buf[offset + 2].toLong() and 0xffL) shl 16) or - ((buf[offset + 3].toLong() and 0xffL) shl 24) - } - - private fun readCString(buf: ByteArray, offset: Int, maxLen: Int): String { - var end = offset - val limit = Math.min(buf.size, offset + maxLen) - while (end < limit && buf[end].toInt() != 0) { - end++ - } - return String(buf, offset, end - offset, StandardCharsets.ISO_8859_1) - } - - private val CRC32C_TABLE: IntArray = buildCrc32cTable() - - private fun buildCrc32cTable(): IntArray { - val table = IntArray(256) - for (b in 0 until 256) { - var c = b shl 24 - for (i in 0 until 8) { - c = if ((c and 0x80000000.toInt()) != 0) (c shl 1) xor 0x1edc6f41 else c shl 1 - } - table[b] = c - } - return table - } - - /** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */ - private fun crc32cMsb(data: ByteArray): Int { - var crc = 0 - for (value in data) { - crc = (crc shl 8) xor CRC32C_TABLE[((crc ushr 24) xor (value.toInt() and 0xff)) and 0xff] - } - return crc - } - } - - private val context: Context - private val leftAddress: String - private val rightAddress: String - private val firmwarePath: String - private val bleManager: FaceclawBleManager +class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) { + private val link = AndroidStockLink(context.applicationContext) private val mainHandler = Handler(Looper.getMainLooper()) - private val lock = Any() - private val dataAcks = LinkedBlockingQueue() @Volatile private var listener: FaceclawFirmwareFlasherListener? = null + @Volatile private var worker: Thread? = null - @Volatile - private var cancelled = false - private var nextSeq = 0 - private var nextAuthMagic = 0x60 - private var pendingAuthMagic = -1 - private var authLatch: CountDownLatch? = null - private var authAckPb: ByteArray? = null + private val flow = + OtaFlashFlow( + link, + rightAddress ?: "", + leftAddress ?: "", + firmwarePath ?: "", + object : FaceclawFirmwareFlasherListener { + override fun onLog(line: String?) { + android.util.Log.i(OtaFlashFlow.TAG, line ?: "") + post { it.onLog(line) } + } - init { - this.context = context.applicationContext - this.rightAddress = rightAddress ?: "" - this.leftAddress = leftAddress ?: "" - this.firmwarePath = firmwarePath ?: "" - this.bleManager = FaceclawBleManager(this.context) - this.bleManager.setListener(this) - } + override fun onProgress( + lens: String?, + componentIndex: Int, + componentCount: Int, + blockIndex: Int, + blockCount: Int, + bytesSent: Long, + bytesTotal: Long, + ) = post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) } + + override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") } + + override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") } + }, + ) fun setListener(listener: FaceclawFirmwareFlasherListener?) { this.listener = listener @@ -190,517 +59,29 @@ class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddre if (worker != null) { return } - val w = Thread({ run() }, "faceclaw-flasher") + val w = Thread({ flow.run() }, "faceclaw-flasher") worker = w w.start() } } fun cancel() { - cancelled = true - val w = worker - if (w != null) { - w.interrupt() - } + flow.cancel() } fun close() { cancel() try { - bleManager.close() + link.close() } catch (ignored: Exception) { } } - private fun run() { - try { - if (leftAddress.trim().isEmpty() || rightAddress.trim().isEmpty()) { - throw IllegalStateException("Both lens addresses are required to flash.") - } - - emitState("validating", "") - val img = readFile(firmwarePath) - val segs = validate(img) - emitLog("firmware validated: " + segs.size + " components, " + img.size + " bytes") - - flashLens("left", leftAddress, img, segs, FIRST_LENS_CONNECT_WINDOW_MS) - if (cancelled) { - emitState("error", "Cancelled.") - emitComplete(false, "Cancelled after the left lens.") - return - } - - emitState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens.") - sleepInterruptibly(REBOOT_SETTLE_MS) - flashLens("right", rightAddress, img, segs, SECOND_LENS_CONNECT_WINDOW_MS) - - emitState("done", "") - emitComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware.") - } catch (e: Exception) { - var message = if (e.message == null) e.toString() else e.message - if (cancelled) { - message = "Cancelled." - } - emitState("error", message) - emitComplete(false, message) - } finally { - teardown() - } - } - - @Throws(TimeoutException::class) - private fun flashLens(lens: String, address: String, img: ByteArray, segs: List, connectWindowMs: Int) { - emitState("connecting", lens) - connectLensResilient(lens, address, connectWindowMs) - sleepInterruptibly(NOTIFY_SETTLE_MS) - drainAcks() - // Fresh envelope counter for the OTA stream (the auth exchange during - // bring-up used its own); mirrors the stock sequence. - resetSeq() - - emitState("flashing", lens) - val beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS) - if (!isEndOk(beginStatus)) { - emitLog("warning: unexpected begin status $beginStatus; continuing") - } - // Progress is reported in bytes across the whole image: the segments - // are one large firmware blob plus several small ones, so a bar - // stepped per segment would sit still through the big one and then - // sprint through the rest. - var totalBytes = 0L - for (seg in segs) { - totalBytes += seg.ps - } - var bytesBefore = 0L - for (i in segs.indices) { - if (cancelled) { - throw IllegalStateException("Cancelled.") - } - flashComponentWithRetry(lens, address, i, segs.size, segs[i], img, bytesBefore, totalBytes) - bytesBefore += segs[i].ps - } - emitLog("$lens lens: all components verified") - try { - bleManager.disconnect(address) - } catch (ignored: Exception) { - } - } - - private fun flashComponentWithRetry(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray, - bytesBefore: Long, totalBytes: Long) { - for (attempt in 0 until COMPONENT_RETRIES) { - if (attempt > 0) { - emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES) - } - var endStatus: Int - try { - endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes) - } catch (e: TimeoutException) { - emitLog(seg.name + ": block phase failed: " + e.message) - endStatus = -1 - } catch (e: RuntimeException) { - emitLog(seg.name + ": block phase failed: " + e.message) - endStatus = -1 - } - if (isEndOk(endStatus)) { - emitLog(seg.name + ": END verify OK (status " + endStatus + ")") - return - } - if (endStatus >= 0) { - emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")") - } - drainAcks() - sleepInterruptibly(1_500) - } - throw IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts") - } - - @Throws(TimeoutException::class) - private fun flashComponent(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray, - bytesBefore: Long, totalBytes: Long): Int { - val sub = Arrays.copyOfRange(img, seg.off, seg.off + 128) - val ps = seg.ps - val payloadStart = seg.off + 128 - - val checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, CTRL_ACK_TIMEOUT_MS) - if (checkStatus != 0) { - throw IllegalStateException("FILE_CHECK rejected status=$checkStatus") - } - - val blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE - for (b in 0 until blockCount) { - if (cancelled) { - throw IllegalStateException("Cancelled.") - } - val start = payloadStart + b * BLOCK_SIZE - val end = Math.min(start + BLOCK_SIZE, payloadStart + ps) - val block = Arrays.copyOfRange(img, start, end) - - var accepted = false - for (tries in 0 until BLOCK_NAK_RETRIES) { - val status = sendBlock(address, block) // throws TimeoutException -> component re-flash - if (status == 0) { - accepted = true - break - } - emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status - + " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES) - } - if (!accepted) { - throw IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times") - } - if (b % 20 == 0 || b == blockCount - 1) { - val bytesSent = bytesBefore + Math.min((b + 1).toLong() * BLOCK_SIZE, ps.toLong()) - emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes) - } - } - emitLog(seg.name + ": data phase done; sending END") - return sendCtrlAndWait(address, OP_END, EMPTY, CTRL_ACK_TIMEOUT_MS) - } - - /** Send one 4 KB block as a marker + data pair sharing one envelope seq. */ - @Throws(TimeoutException::class) - private fun sendBlock(address: String, block: ByteArray): Int { - val seq = nextSeq() - drainAcks() - writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, byteArrayOf(OP_BLOCK.toByte()), seq) - writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_DATA, block, seq) - return waitAck(OP_BLOCK, BLOCK_ACK_TIMEOUT_MS) - } - - @Throws(TimeoutException::class) - private fun sendCtrlAndWait(address: String, op: Int, data: ByteArray, timeoutMs: Int): Int { - val seq = nextSeq() - drainAcks() - val payload = ByteArray(1 + data.size) - payload[0] = op.toByte() - System.arraycopy(data, 0, payload, 1, data.size) - writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, payload, seq) - return waitAck(op, timeoutMs) - } - - private fun writeOta(address: String, writeChar: String, sid: Int, payload: ByteArray, seq: Int): Boolean { - val frames = BleProtocol.framePb(payload, sid, FLAG_OTA, seq) - return bleManager.writeFrames( - address, writeChar, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS) - } - - @Throws(TimeoutException::class) - private fun waitAck(wantOp: Int, timeoutMs: Int): Int { - val deadline = SystemClock.elapsedRealtime() + timeoutMs - while (true) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - break - } - val pb: ByteArray? - try { - pb = dataAcks.poll(remaining, TimeUnit.MILLISECONDS) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - throw TimeoutException("interrupted waiting for ack op=$wantOp") - } - if (pb == null) { - break - } - if (pb.size >= 2 && (pb[0].toInt() and 0xff) == wantOp) { - return pb[1].toInt() and 0xff - } - // Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting. - } - throw TimeoutException("no ack op=0x" + Integer.toHexString(wantOp) + " within " + timeoutMs + "ms") - } - - // ---- connection ---------------------------------------------------------- - - private fun connectLensResilient(lens: String, address: String, windowMs: Int) { - val deadline = SystemClock.elapsedRealtime() + windowMs - var lastError: String? = "unknown" - var attempt = 0 - while (SystemClock.elapsedRealtime() < deadline && !cancelled) { - attempt++ - try { - if (bringUpLens(address)) { - emitLog("connected $lens lens (attempt $attempt)") - return - } - lastError = "connect/discover incomplete" - } catch (e: Exception) { - lastError = if (e.message == null) e.toString() else e.message - } - emitLog("$lens connect attempt $attempt failed ($lastError); retrying...") - try { - bleManager.disconnect(address) - } catch (ignored: Exception) { - } - sleepInterruptibly(RETRY_DELAY_MS) - } - if (cancelled) { - throw IllegalStateException("Cancelled.") - } - throw IllegalStateException("could not reach $lens lens: $lastError") - } - - private fun bringUpLens(address: String): Boolean { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - return false - } - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - return false - } - // Control-channel notify carries the auth response; mandatory. - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - return false - } - if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - return false - } - if (!authenticate(address)) { - throw IllegalStateException( - "security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it") - } - return true - } - - /** - * Complete the sid-0x80 security-auth exchange on this connection. On an - * unbonded phone this triggers SMP pairing (possibly with an OS prompt), - * so the wait is generous. Must succeed before any OTA traffic. - */ - private fun authenticate(address: String): Boolean { - val magic: Int - val latch = CountDownLatch(1) - synchronized(lock) { - magic = nextAuthMagic - nextAuthMagic = if (nextAuthMagic >= 0x7f) 0x60 else nextAuthMagic + 1 - pendingAuthMagic = magic - authAckPb = null - authLatch = latch - } - try { - if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH, - BleProtocol.buildAuthenticationRequest(magic), nextSeq())) { - emitLog("auth write failed: $address") - return false - } - val signalled: Boolean - try { - signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - return false - } - val pb: ByteArray? - synchronized(lock) { - pb = authAckPb - } - if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) { - emitLog("security auth not acknowledged (" + address - + ") — if Android shows a Bluetooth pairing request, accept it") - return false - } - emitLog("security auth complete: $address") - return true - } finally { - synchronized(lock) { - pendingAuthMagic = -1 - authLatch = null - } - } - } - - // ---- firmware container parsing / validation ----------------------------- - - private class Segment( - @JvmField val off: Int, - @JvmField val ps: Int, - @JvmField val crc: Long, - @JvmField val name: String - ) - - private fun parseSegments(img: ByteArray): List { - if (img.size < 0x40) { - throw IllegalStateException("file is too small to be a firmware image") - } - val n = readU32(img, 8) - if (n <= 0 || n > 64) { - throw IllegalStateException("implausible component count $n (corrupt image?)") - } - val segs = ArrayList() - var i = 0 - while (i < n) { - val base = 0x40 + i * 16 - val crc = readU32(img, base + 12) - val off = readU32(img, base + 4).toInt() - if (off + 128 > img.size) { - throw IllegalStateException("segment $i subheader runs past end of file") - } - val ps = readU32(img, off + 8).toInt() - val name = readCString(img, off + 48, 80) - segs.add(Segment(off, ps, crc, name)) - i++ - } - return segs - } - - private fun validate(img: ByteArray): List { - val segs = parseSegments(img) - if (segs.size < MIN_SEGMENTS || segs.size > MAX_SEGMENTS) { - throw IllegalStateException( - "expected " + MIN_SEGMENTS + "-" + MAX_SEGMENTS + " components, found " + segs.size) - } - var main: Segment? = null - for (s in segs) { - val payload = Arrays.copyOfRange(img, s.off + 128, s.off + 128 + s.ps) - val calc = crc32cMsb(payload).toLong() and 0xffffffffL - val subCrc = readU32(img, s.off + 12) - if (calc != s.crc || calc != subCrc) { - throw IllegalStateException("component " + s.name + " CRC32C is stale (image not checksum-fixed)") - } - if (REQUIRED_SEGMENT == s.name) { - main = s - } - } - if (main == null) { - throw IllegalStateException("required component $REQUIRED_SEGMENT not found") - } - checkMainAppFitsMram(img, main) - return segs - } - - private fun checkMainAppFitsMram(img: ByteArray, main: Segment) { - if (main.ps < APP_PREAMBLE) { - throw IllegalStateException("main-app payload is smaller than its preamble") - } - val loadAddr = readU32(img, main.off + 128 + 0x14) - val preLen = readU32(img, main.off + 128) and 0xFFFFFFL - if (loadAddr != APP_LOAD_ADDR) { - throw IllegalStateException( - "main-app preamble load address is 0x" + java.lang.Long.toHexString(loadAddr) - + ", expected 0x" + java.lang.Long.toHexString(APP_LOAD_ADDR)) - } - if (preLen != main.ps.toLong()) { - throw IllegalStateException( - "main-app preamble length (" + preLen + ") != staged payload size (" + main.ps + ")") - } - val progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE - if (progEnd > APP_MAX_END) { - val over = progEnd - APP_MAX_END - throw IllegalStateException( - "main-app is too large: programmed region ends at 0x" + java.lang.Long.toHexString(progEnd) - + ", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)") - } - } - - // ---- listener callbacks -------------------------------------------------- - - override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { - if (BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { - // Control channel: only the security-auth response is awaited here. - // A value can carry several envelope frames back to back (e.g. the - // response packed with one of the lens's own sid-0x80 notifies). - for (buf in BleProtocol.splitFrames(data)) { - val frame = BleProtocol.parseFrame(buf) - if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) { - synchronized(lock) { - val currentLatch = authLatch - if (currentLatch != null && frame.msgSeq == pendingAuthMagic) { - authAckPb = frame.pb - currentLatch.countDown() - } - } - } - } - return - } - if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equals(characteristicUuid, ignoreCase = true)) { - return // OTA acks arrive on the data-notify char - } - val frame = BleProtocol.parseFrame(data) - if (!frame.ok || frame.pb.size < 2) { - return - } - dataAcks.add(Arrays.copyOf(frame.pb, Math.min(frame.pb.size, 2))) - } - - override fun onConnectionStateChange(address: String?, connected: Boolean) { - if (!connected) { - Log.i(TAG, "disconnected: $address") - } - } - - // ---- helpers ------------------------------------------------------------- - - private fun teardown() { - try { - bleManager.close() - } catch (ignored: Exception) { - } - } - - private fun resetSeq() { - synchronized(lock) { - nextSeq = 0 - } - } - - private fun nextSeq(): Int { - synchronized(lock) { - nextSeq = (nextSeq + 1) and 0xff - return nextSeq - } - } - - private fun drainAcks() { - dataAcks.clear() - } - - private fun sleepInterruptibly(ms: Int) { - try { - Thread.sleep(ms.toLong()) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - } - } - - private fun emitLog(line: String) { - Log.i(TAG, line) + private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) { mainHandler.post { val current = listener if (current != null) { - current.onLog(line) - } - } - } - - private fun emitProgress(lens: String, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int, - bytesSent: Long, bytesTotal: Long) { - mainHandler.post { - val current = listener - if (current != null) { - current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) - } - } - } - - private fun emitState(state: String, detail: String?) { - val safeDetail = detail ?: "" - mainHandler.post { - val current = listener - if (current != null) { - current.onState(state, safeDetail) - } - } - } - - private fun emitComplete(success: Boolean, detail: String?) { - val safeDetail = detail ?: "" - mainHandler.post { - val current = listener - if (current != null) { - current.onComplete(success, safeDetail) + action(current) } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt index b3cf1d83..37ad4593 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.kt @@ -1,110 +1,53 @@ package com.faceclaw.app -import android.bluetooth.BluetoothGatt import android.content.Context import android.os.Handler import android.os.Looper -import android.util.Log - -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.CountDownLatch -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.TimeUnit /** - * A minimal, stock-firmware-compatible BLE communicator used only to show the - * pre-flash confirmation prompt on the glasses and read back the user's Yes/No - * choice. It is deliberately separate from FaceclawBleCommunicator: the main - * app assumes the custom firmware (image containers, compositor, etc.), whereas - * this must work on unmodified glasses, so it speaks only the stock subset — - * session prelude, a text container, a list container, and heartbeats. No - * images. - * - * It reuses the low-level GATT wrapper (FaceclawBleManager) and the framing / - * protobuf helpers (BleProtocol), but owns its own GATT connection and runs its - * whole lifecycle on a single worker thread. It expects the main app to be - * disconnected while it runs (onboarding, before flashing). - * - * Both arms are connected and security-authenticated up front, even though the - * prompt itself only needs the right arm (the lenses relay messages to each - * other, and acks/events only ever come from the right arm): on an unbonded - * phone the auth exchange is what raises the OS pairing prompt, and doing both - * here means both prompts appear at the start rather than one popping up half - * way through flashing the second lens. While connected, each arm's battery - * level is read (they are independent batteries) so the caller can refuse to - * flash on a low charge. - * - * With `skipPrompt` the on-glasses confirmation is not shown: the flow is just - * connect + auth + battery read + result(approved). Used to re-check the - * battery after the user has already confirmed once. + * Android host for the shared [FlashPromptFlow] (the stock-firmware pre-flash Yes/No prompt + * plus battery read): owns the GATT link (AndroidStockLink) and the worker thread, and + * delivers listener callbacks on the main thread. It expects the main app to be disconnected + * while it runs (onboarding, before flashing). Public API unchanged for + * app/native/flash-prompt-communicator.ts. */ class FaceclawFlashPromptCommunicator( - context: Context, rightAddress: String?, leftAddress: String?, warningText: String?, skipPrompt: Boolean) : FaceclawBleListener { - companion object { - private const val TAG = "FaceclawFlashPrompt" - - private const val TEXT_NAME = "flashwarn" - private const val LIST_NAME = "flashmenu" - private const val TEXT_CONTAINER_ID = 1 - private const val LIST_CONTAINER_ID = 2 - // Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid. - private val ITEMS = arrayOf("No, cancel", "Yes, flash") - - private const val HEARTBEAT_INTERVAL_MS = 4_000 - private const val SELECTION_TIMEOUT_MS = 120_000 - private const val CREATE_ACK_TIMEOUT_MS = 3_000 - private const val BATTERY_ACK_TIMEOUT_MS = 3_000 - - private fun ackKey(sid: Int, magic: Int): String { - return "$sid:$magic" - } - } - - private val context: Context - private val rightAddress: String - private val leftAddress: String - private val warningText: String - private val skipPrompt: Boolean - private val bleManager: FaceclawBleManager + context: Context, + rightAddress: String?, + leftAddress: String?, + warningText: String?, + skipPrompt: Boolean, +) { + private val link = AndroidStockLink(context.applicationContext) private val mainHandler = Handler(Looper.getMainLooper()) - private val lock = Any() - private val pendingAcks = ConcurrentHashMap() - private val ackPayloads = ConcurrentHashMap() - private val selectionLatch = CountDownLatch(1) @Volatile private var listener: FaceclawFlashPromptListener? = null + @Volatile private var worker: Thread? = null - @Volatile - private var cancelled = false - @Volatile - private var finished = false - @Volatile - private var rightConnected = false - @Volatile - private var leftConnected = false - @Volatile - private var rightLost = false - @Volatile - private var approved: Boolean? = null - @Volatile - private var heartbeatExecutor: ScheduledExecutorService? = null - private var nextMagic = 100 - private var nextSeq = 0x40 + private val flow = + FlashPromptFlow( + link, + rightAddress ?: "", + leftAddress ?: "", + warningText ?: "", + skipPrompt, + object : FaceclawFlashPromptListener { + override fun onLog(line: String?) { + android.util.Log.i(FlashPromptFlow.TAG, line ?: "") + post { it.onLog(line) } + } - init { - this.context = context.applicationContext - this.rightAddress = rightAddress ?: "" - this.leftAddress = leftAddress ?: "" - this.warningText = warningText ?: "" - this.skipPrompt = skipPrompt - this.bleManager = FaceclawBleManager(this.context) - this.bleManager.setListener(this) - } + override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") } + + override fun onBattery(rightPercent: Int, leftPercent: Int) = post { it.onBattery(rightPercent, leftPercent) } + + override fun onResult(approved: Boolean) = post { it.onResult(approved) } + }, + ) fun setListener(listener: FaceclawFlashPromptListener?) { this.listener = listener @@ -115,7 +58,7 @@ class FaceclawFlashPromptCommunicator( if (worker != null) { return } - val w = Thread({ run() }, "faceclaw-flash-prompt") + val w = Thread({ flow.run() }, "faceclaw-flash-prompt") worker = w w.start() } @@ -123,443 +66,22 @@ class FaceclawFlashPromptCommunicator( /** Abort the prompt (e.g. the user backed out on the phone). */ fun cancel() { - cancelled = true - selectionLatch.countDown() - val w = worker - if (w != null) { - w.interrupt() - } + flow.cancel() } fun close() { cancel() - stopHeartbeat() try { - bleManager.close() + link.close() } catch (ignored: Exception) { } } - private fun run() { - try { - if (rightAddress.trim().isEmpty()) { - emitState("error", "No right-arm address configured.") - return - } - - emitState("connecting", "") - connectArm(rightAddress, "right") - rightConnected = true - if (cancelled) { - teardown() - return - } - if (!leftAddress.trim().isEmpty()) { - connectArm(leftAddress, "left") - leftConnected = true - if (cancelled) { - teardown() - return - } - } - - emitState("connected", "") - // Auth both arms now (pairing prompts, if any, happen here) before - // anything else, so both bonds exist by the time flashing starts. - authenticateArm(rightAddress, "right") - if (leftConnected) { - authenticateArm(leftAddress, "left") - } - if (cancelled) { - teardown() - return - } - sendPrelude(rightAddress) - - if (!skipPrompt) { - showPrompt(rightAddress) - startHeartbeat() - emitState("prompting", "") - - selectionLatch.await(SELECTION_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS) - stopHeartbeat() - - if (cancelled) { - emitState("cancelled", "") - teardown() - return - } - if (approved == null) { - if (rightLost) { - emitState("disconnected", "Lost connection to the glasses.") - } else { - emitState("timeout", "No response from the glasses.") - } - teardown() - return - } - try { - sendShutdown() - } catch (ignored: Exception) { - } - if (approved != true) { - emitResult(false) - emitState("result", "declined") - teardown() - return - } - } - - // Approved (or prompt skipped): read each arm's battery while the - // links are still up, then report the result. - emitState("battery", "") - readBatteries() - if (cancelled) { - emitState("cancelled", "") - teardown() - return - } - if (rightLost) { - emitState("disconnected", "Lost connection to the glasses.") - teardown() - return - } - finished = true - emitResult(true) - emitState("result", "approved") - teardown() - } catch (e: Exception) { - stopHeartbeat() - if (!cancelled) { - val message = if (e.message == null) e.toString() else e.message - emitState("error", message) - } - teardown() - } - } - - private fun connectArm(address: String, arm: String) { - if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { - throw IllegalStateException("could not connect to the $arm arm ($address)") - } - bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) - bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) - if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { - throw IllegalStateException("discoverServices failed: $arm arm ($address)") - } - if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { - throw IllegalStateException("enableNotifications failed: $arm arm ($address)") - } - emitLog("connected $arm arm") - } - - /** - * Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a - * session (and closes the link after ~30 s) without it, and on an - * unbonded phone this is what triggers SMP pairing, so it may sit - * waiting on an OS pairing prompt. - */ - @Throws(InterruptedException::class) - private fun authenticateArm(address: String, arm: String) { - val authMagic = allocMagic() - val authAck = writeAndAwaitAck( - address, - BleProtocol.SID_SECURITY_AUTH, - BleProtocol.FLAG_SECURITY_AUTH, - authMagic, - BleProtocol.buildAuthenticationRequest(authMagic), - ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS) - if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) { - throw IllegalStateException( - "could not authenticate with the $arm arm ($address" - + ") — if Android shows a Bluetooth pairing request, accept it and try again") - } - emitLog("security auth complete: $arm arm") - } - - /** Mandatory session prelude on sid=0x01 (app-launch). Right arm only; it relays. */ - @Throws(InterruptedException::class) - private fun sendPrelude(address: String) { - if (writeAndAwaitAck( - address, - BleProtocol.PRELUDE_ACK_SID, - BleProtocol.FLAG_REQUEST, - BleProtocol.PRELUDE_ACK_MAGIC, - BleProtocol.PRELUDE_F5872_PAYLOAD, - ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { - throw IllegalStateException("session prelude not acked: $address") - } - } - - @Throws(InterruptedException::class) - private fun showPrompt(address: String) { - // Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. - // Two attempts: on 2.2.9 the first request sent right after the - // prelude can be dropped while the lens emits its own sid-0x80 - // notifications (observed with the device-info settings read). - var pageAck: ByteArray? = null - var attempt = 0 - while (attempt < 2 && pageAck == null && !cancelled) { - val magic = allocMagic() - val page = BleProtocol.buildCreatePromptPage( - magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS) - pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS) - if (pageAck == null) { - emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address) - } - attempt++ - } - if (pageAck == null) { - throw IllegalStateException("prompt page not acked: $address") - } - emitLog("prompt page shown on $address") - } - - /** - * Read each arm's battery via a sid-0x09 settings read on that arm's own - * link (the arms have independent batteries and each answers with its - * own). An arm that does not answer reports -1; the caller decides what - * to do with a partial reading. - */ - @Throws(InterruptedException::class) - private fun readBatteries() { - val right = if (rightConnected) readBattery(rightAddress, "right") else -1 - val left = if (leftConnected) readBattery(leftAddress, "left") else -1 - emitLog("battery R=" + (if (right < 0) "?" else "$right%") + " L=" + (if (left < 0) "?" else "$left%")) - emitBattery(right, left) - } - - @Throws(InterruptedException::class) - private fun readBattery(address: String, arm: String): Int { - var attempt = 0 - while (attempt < 2 && !cancelled) { - val magic = allocMagic() - val ack = writeAndAwaitAck( - address, - BleProtocol.SID_UI_SETTING, - BleProtocol.FLAG_REQUEST, - magic, - BleProtocol.buildSettingsQuery(magic), - BATTERY_ACK_TIMEOUT_MS) - if (ack == null) { - emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm") - attempt++ - continue - } - val snapshot: BleProtocol.BatterySnapshot? = BleProtocol.parseSettingsBattery(ack) - if (snapshot != null) { - return snapshot.battery - } - emitLog("battery read ack had no battery field: $arm arm") - attempt++ - } - return -1 - } - - /** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */ - @Throws(InterruptedException::class) - private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? { - val latch = CountDownLatch(1) - val key = ackKey(sid, magic) - pendingAcks[key] = latch - try { - if (!writeFrame(address, sid, flag, payload)) { - return null - } - if (!latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS)) { - return null - } - val pb = ackPayloads[key] - return pb ?: ByteArray(0) - } finally { - pendingAcks.remove(key, latch) - ackPayloads.remove(key) - } - } - - private fun writeFrame(address: String, sid: Int, flag: Int, payload: ByteArray): Boolean { - val seq: Int - synchronized(lock) { - seq = nextSeq++ and 0xff - } - val frames = BleProtocol.framePb(payload, sid, flag, seq) - return bleManager.writeFrames( - address, - BleProtocol.WRITE_CHAR_UUID, - frames, - AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), - ConnectionOptions.WRITE_TIMEOUT_MS) - } - - private fun startHeartbeat() { - stopHeartbeat() - val executor = Executors.newSingleThreadScheduledExecutor { runnable -> - val t = Thread(runnable, "faceclaw-flash-hb") - t.isDaemon = true - t - } - heartbeatExecutor = executor - executor.scheduleWithFixedDelay( - { sendHeartbeat() }, HEARTBEAT_INTERVAL_MS.toLong(), HEARTBEAT_INTERVAL_MS.toLong(), TimeUnit.MILLISECONDS) - } - - private fun sendHeartbeat() { - if (finished || cancelled) { - return - } - try { - val heartbeat = BleProtocol.buildHeartbeat(allocMagic()) - if (rightConnected) { - writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat) - } - } catch (e: Exception) { - Log.w(TAG, "heartbeat failed: " + e.message) - } - } - - private fun stopHeartbeat() { - val executor = heartbeatExecutor - heartbeatExecutor = null - if (executor != null) { - executor.shutdownNow() - } - } - - private fun sendShutdown() { - val shutdown = BleProtocol.buildShutdown(allocMagic(), 0) - if (rightConnected) { - writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown) - } - } - - private fun teardown() { - finished = true - stopHeartbeat() - try { - bleManager.close() - } catch (ignored: Exception) { - } - rightConnected = false - leftConnected = false - } - - override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { - if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { - return - } - // A value can carry several envelope frames back to back. - for (buf in BleProtocol.splitFrames(data)) { - handleFrame(address, buf) - } - } - - private fun handleFrame(address: String?, buf: ByteArray) { - val frame = BleProtocol.parseFrame(buf) - if (!frame.ok) { - return - } - if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { - handleEvent(frame) - return - } - if (frame.msgSeq >= 0) { - val key = ackKey(frame.sid, frame.msgSeq) - val latch = pendingAcks[key] - if (latch != null) { - ackPayloads[key] = frame.pb - latch.countDown() - } - } - } - - private fun handleEvent(frame: BleProtocol.ParsedFrame) { - val selection: BleProtocol.ListSelection? = BleProtocol.parseListSelection(frame) - if (selection == null || LIST_NAME != selection.containerName) { - return - } - // Only a confirmed click is a decision; scroll/highlight changes are ignored. - if (selection.eventType != BleProtocol.EVENT_CLICK) { - return - } - val itemName = selection.itemName - val yes = selection.itemIndex == 1 - || (itemName != null && itemName.lowercase().startsWith("yes")) - synchronized(lock) { - if (finished) { - return - } - finished = true - approved = yes - } - emitLog("selection: " + (if (yes) "flash" else "cancel") + " (index " + selection.itemIndex + ")") - selectionLatch.countDown() - } - - override fun onConnectionStateChange(address: String?, connected: Boolean) { - if (connected) { - return - } - if (address!!.equals(leftAddress, ignoreCase = true)) { - // The prompt itself only needs the right arm; a dropped left link - // just means no left battery reading. - leftConnected = false - emitLog("left arm disconnected") - return - } - if (address.equals(rightAddress, ignoreCase = true)) { - rightConnected = false - synchronized(lock) { - if (!finished && !cancelled) { - rightLost = true - finished = true - selectionLatch.countDown() - } - } - } - } - - private fun allocMagic(): Int { - synchronized(lock) { - val magic = nextMagic - nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1 - return magic - } - } - - private fun emitLog(line: String) { - Log.i(TAG, line) + private fun post(action: (FaceclawFlashPromptListener) -> Unit) { mainHandler.post { val current = listener if (current != null) { - current.onLog(line) - } - } - } - - private fun emitState(state: String, detail: String?) { - val safeDetail = detail ?: "" - mainHandler.post { - val current = listener - if (current != null) { - current.onState(state, safeDetail) - } - } - } - - private fun emitBattery(rightPercent: Int, leftPercent: Int) { - mainHandler.post { - val current = listener - if (current != null) { - current.onBattery(rightPercent, leftPercent) - } - } - } - - private fun emitResult(approvedResult: Boolean) { - mainHandler.post { - val current = listener - if (current != null) { - current.onResult(approvedResult) + action(current) } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt index 6e5a2855..9f26efab 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawHttp.kt @@ -7,27 +7,18 @@ import okhttp3.Request import okhttp3.Response /** - * Shared HTTP identity for the Java networking helpers. The version lives in + * Android okhttp face of the shared HTTP identity. The version lives in * app/version.ts, so the TypeScript side pushes the real string in at startup - * (see installNativeUserAgent in app/util/http.ts); the default below is only - * a fallback for requests that somehow beat that call. + * (see installNativeUserAgent in app/util/http.ts); the value itself is held + * by the shared HttpIdentity so every platform's networking reads one string. */ class FaceclawHttp private constructor() { companion object { - @Volatile - private var userAgent = "Faceclaw" + @JvmStatic + fun setUserAgent(value: String?) = HttpIdentity.setUserAgent(value) @JvmStatic - fun setUserAgent(value: String?) { - if (value != null && !value.trim().isEmpty()) { - userAgent = value.trim() - } - } - - @JvmStatic - fun getUserAgent(): String { - return userAgent - } + fun getUserAgent(): String = HttpIdentity.getUserAgent() /** Stamps our User-Agent on any request that doesn't already carry one. */ @JvmStatic @@ -39,7 +30,7 @@ class FaceclawHttp private constructor() { if (request.header("User-Agent") != null) { return chain.proceed(request) } - return chain.proceed(request.newBuilder().header("User-Agent", userAgent).build()) + return chain.proceed(request.newBuilder().header("User-Agent", HttpIdentity.getUserAgent()).build()) } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt index ce53c010..1028f3bf 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.kt @@ -2,23 +2,23 @@ package com.faceclaw.app import android.util.Log +/** + * Android LC3 decoder for G2 microphone packets: the liblc3 JNI codec behind the shared + * [Lc3PacketFramer], which owns the counter/duplicate/trailer logic and statistics. + */ class FaceclawLc3Decoder { companion object { private const val TAG = "FaceclawLc3" - const val SAMPLE_RATE = 16000 - const val FRAME_US = 10000 - const val FRAME_BYTES = 40 - const val FRAMES_PER_PACKET = 5 - const val PACKET_BYTES = 205 - const val COUNTER_OFFSET = 204 - const val SAMPLES_PER_FRAME = 160 - const val SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME - // Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the - // glasses DSP appends two signed 16-bit LE values computed on the raw - // stereo capture before the mono downmix — a signal-strength ratio and the - // asin-derived direction-of-arrival in degrees — then the packet counter. - const val SSR_OFFSET = 200 - const val ANGLE_OFFSET = 202 + const val SAMPLE_RATE = Lc3PacketFramer.SAMPLE_RATE + const val FRAME_US = Lc3PacketFramer.FRAME_US + const val FRAME_BYTES = Lc3PacketFramer.FRAME_BYTES + const val FRAMES_PER_PACKET = Lc3PacketFramer.FRAMES_PER_PACKET + const val PACKET_BYTES = Lc3PacketFramer.PACKET_BYTES + const val COUNTER_OFFSET = Lc3PacketFramer.COUNTER_OFFSET + const val SAMPLES_PER_FRAME = Lc3PacketFramer.SAMPLES_PER_FRAME + const val SAMPLES_PER_PACKET = Lc3PacketFramer.SAMPLES_PER_PACKET + const val SSR_OFFSET = Lc3PacketFramer.SSR_OFFSET + const val ANGLE_OFFSET = Lc3PacketFramer.ANGLE_OFFSET init { System.loadLibrary("faceclaw_lc3") @@ -32,17 +32,15 @@ class FaceclawLc3Decoder { private external fun nativeDestroy(handle: Long) } - private var nativeHandle: Long - private var lastCounter = -1 - private var realPackets: Long = 0 - private var duplicatePackets: Long = 0 - private var missingPackets: Long = 0 - private var decodeErrors: Long = 0 - private var lastSsr = 0 - private var lastAngleDegrees = 0 + private var nativeHandle: Long = nativeCreate(FRAME_US, SAMPLE_RATE) + private val framer = Lc3PacketFramer(object : Lc3Codec { + override fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int { + // The JNI entry decodes the five 40-byte frames at the start of the packet. + return nativeDecodePacket(nativeHandle, packet, pcmOut) + } + }) init { - nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE) if (nativeHandle == 0L) { throw IllegalStateException("Could not create LC3 decoder") } @@ -54,70 +52,30 @@ class FaceclawLc3Decoder { throw IllegalStateException("LC3 decoder is closed") } if (packet == null || packet.size != PACKET_BYTES) { - decodeErrors++ Log.w(TAG, "unexpected G2 audio packet length=" + (packet?.size ?: -1)) - return 0 } - if (pcmOut == null || pcmOut.size < SAMPLES_PER_PACKET) { - throw IllegalArgumentException("pcmOut must hold $SAMPLES_PER_PACKET samples") - } - - val counter = packet[COUNTER_OFFSET].toInt() and 0xff - if (lastCounter >= 0) { - val gap = (counter - lastCounter) and 0xff - if (gap == 0) { - duplicatePackets++ - return 0 - } - val missing = gap - 1 - if (missing > 0) { - missingPackets += missing.toLong() - } - } - - val decoded = nativeDecodePacket(nativeHandle, packet, pcmOut) - if (decoded <= 0) { - decodeErrors++ - return 0 - } - realPackets++ - lastCounter = counter - lastSsr = ((packet[SSR_OFFSET].toInt() and 0xff) or (packet[SSR_OFFSET + 1].toInt() shl 8)).toShort().toInt() - lastAngleDegrees = ((packet[ANGLE_OFFSET].toInt() and 0xff) or (packet[ANGLE_OFFSET + 1].toInt() shl 8)).toShort().toInt() - return decoded + return framer.decodePacket(packet, pcmOut) } /** Firmware signal-strength ratio from the most recent decoded packet. */ @Synchronized - fun getLastSsr(): Int { - return lastSsr - } + fun getLastSsr(): Int = framer.lastSsr /** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */ @Synchronized - fun getLastAngleDegrees(): Int { - return lastAngleDegrees - } + fun getLastAngleDegrees(): Int = framer.lastAngleDegrees @Synchronized - fun getRealPackets(): Long { - return realPackets - } + fun getRealPackets(): Long = framer.realPackets @Synchronized - fun getDuplicatePackets(): Long { - return duplicatePackets - } + fun getDuplicatePackets(): Long = framer.duplicatePackets @Synchronized - fun getMissingPackets(): Long { - return missingPackets - } + fun getMissingPackets(): Long = framer.missingPackets @Synchronized - fun getDecodeErrors(): Long { - return decodeErrors - } + fun getDecodeErrors(): Long = framer.decodeErrors @Synchronized fun close() { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt index 376175d8..96cb52ff 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.kt @@ -2,13 +2,11 @@ package com.faceclaw.app import android.os.Handler import android.os.Looper -import android.util.Log import java.io.File import java.io.FileInputStream import java.io.FileOutputStream -import java.io.IOException -import java.security.MessageDigest +import java.io.InputStream import java.util.concurrent.TimeUnit import okhttp3.Call @@ -17,10 +15,10 @@ import okhttp3.Request /** * Large-file downloader for on-phone model weights (multi-GB, from Hugging - * Face). Downloads to ".part" with HTTP Range resume across app - * restarts, verifies a pinned sha256 (hashed incrementally, including the - * resumed prefix), then renames into place. Listener callbacks are posted to - * the constructing thread's Looper, like the other Faceclaw bridges. + * Face). The resume/hash/verify/rename policy is the shared ResumableDownload; + * this class supplies okhttp, java.io storage, the worker thread, and posts + * listener callbacks to the constructing thread's Looper, like the other + * Faceclaw bridges. */ class FaceclawModelDownloader( url: String, @@ -30,35 +28,31 @@ class FaceclawModelDownloader( listener: FaceclawModelDownloaderListener? ) { private val callbackHandler: Handler - private val url: String - private val destFile: File - private val partFile: File - private val expectedSha256: String - private val expectedTotalBytes: Long - private val listener: FaceclawModelDownloaderListener - - @Volatile - private var cancelled = false - @Volatile - private var call: Call? = null + private val transport = OkHttpTransport() + private val download: ResumableDownload private var worker: Thread? = null init { if (listener == null) throw IllegalArgumentException("listener is required") val looper = Looper.myLooper() this.callbackHandler = Handler(looper ?: Looper.getMainLooper()) - this.url = url - this.destFile = File(destPath) - this.partFile = File("$destPath.part") - this.expectedSha256 = expectedSha256?.lowercase() ?: "" - this.expectedTotalBytes = expectedTotalBytes - this.listener = listener + val posted = object : FaceclawModelDownloaderListener { + override fun onProgress(bytesDownloaded: Long, totalBytes: Long) { + callbackHandler.post { listener.onProgress(bytesDownloaded, totalBytes) } + } + + override fun onDone(path: String?) { + callbackHandler.post { listener.onDone(path) } + } + + override fun onError(message: String?) { + callbackHandler.post { listener.onError(message) } + } + } + download = ResumableDownload(url, File(destPath).absolutePath, expectedSha256, expectedTotalBytes, transport, FileStorage, posted) } companion object { - private const val TAG = "FaceclawModelDl" - private const val PROGRESS_INTERVAL_MS = 500L - @Volatile private var sharedClient: OkHttpClient? = null @@ -79,120 +73,76 @@ class FaceclawModelDownloader( } fun start() { - val thread = Thread({ run() }, "FaceclawModelDl") + val thread = Thread({ download.run() }, "FaceclawModelDl") worker = thread thread.start() } /** Stops the download but keeps the .part file so a later start() resumes. */ fun cancel() { - cancelled = true - val current = call - current?.cancel() + download.cancel() + transport.call?.cancel() } - private fun run() { - try { - if (destFile.exists() && destFile.length() > 0) { - post { listener.onDone(destFile.absolutePath) } - return - } - destFile.parentFile.mkdirs() - - var digest = MessageDigest.getInstance("SHA-256") - var offset = 0L - if (partFile.exists()) { - offset = hashExistingPrefix(digest) - if (cancelled) return - } + /** One call at a time per downloader, so cancel() can abort a connect in progress. */ + private class OkHttpTransport : DownloadTransport { + @Volatile + var call: Call? = null + override fun open(url: String, rangeFrom: Long): DownloadResponse { val builder = Request.Builder().url(url) - if (offset > 0) builder.header("Range", "bytes=$offset-") - val newCall = getClient().newCall(builder.build()) - call = newCall + if (rangeFrom > 0) builder.header("Range", "bytes=$rangeFrom-") + val call: Call = getClient().newCall(builder.build()) + this.call = call + val response = call.execute() + val body = response.body + val input: InputStream? = body?.byteStream() + return object : DownloadResponse { + override val code: Int = response.code + override val contentLength: Long = body?.contentLength() ?: -1L - newCall.execute().use { response -> - val body = response.body - if (!response.isSuccessful || body == null) { - postError("Model download failed: HTTP " + response.code) - return - } - val resumed = response.code == 206 - if (offset > 0 && !resumed) { - // Server ignored the Range header; start over. - offset = 0L - digest = MessageDigest.getInstance("SHA-256") - } - val total = if (expectedTotalBytes > 0) expectedTotalBytes - else (if (body.contentLength() > 0) offset + body.contentLength() else -1L) + override fun read(buffer: ByteArray): Int = input?.read(buffer) ?: -1 - FileOutputStream(partFile, resumed).use { out -> - body.byteStream().use { input -> - val buffer = ByteArray(1 shl 16) - var done = offset - var lastProgressAt = 0L - var read: Int - while (input.read(buffer).also { read = it } != -1) { - if (cancelled) return - out.write(buffer, 0, read) - digest.update(buffer, 0, read) - done += read - val now = System.currentTimeMillis() - if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) { - lastProgressAt = now - val doneFinal = done - val totalFinal = total - post { listener.onProgress(doneFinal, totalFinal) } - } - } - } + override fun close() { + call.cancel() + response.close() } } - - if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) { - postError("Model download ended early (" + partFile.length() + " of " - + expectedTotalBytes + " bytes); try again to resume") - return - } - val actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest()) - if (!expectedSha256.isEmpty() && actualSha != expectedSha256) { - partFile.delete() - postError("Model download was corrupted (checksum mismatch); download it again") - return - } - if (!partFile.renameTo(destFile)) { - postError("Could not move the downloaded model into place") - return - } - post { listener.onDone(destFile.absolutePath) } - } catch (e: Exception) { - if (cancelled) return - Log.w(TAG, "download failed", e) - postError("Model download failed: $e") } } - /** Hash the already-downloaded prefix so the final digest covers the whole file. */ - @Throws(IOException::class) - private fun hashExistingPrefix(digest: MessageDigest): Long { - var hashed = 0L - FileInputStream(partFile).use { input -> - val buffer = ByteArray(1 shl 16) - var read: Int - while (input.read(buffer).also { read = it } != -1) { - if (cancelled) return hashed - digest.update(buffer, 0, read) - hashed += read + private object FileStorage : DownloadStorage { + override fun length(path: String): Long = File(path).let { if (it.exists()) it.length() else -1L } + + override fun prepareParent(path: String) { + File(path).parentFile?.mkdirs() + } + + override fun openWrite(path: String, append: Boolean): FileSink { + val out = FileOutputStream(path, append) + return object : FileSink { + override fun write(bytes: ByteArray, offset: Int, length: Int) = out.write(bytes, offset, length) + + override fun close() = out.close() } } - return hashed - } - private fun post(runnable: Runnable) { - callbackHandler.post(runnable) - } + override fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long { + var consumed = 0L + FileInputStream(path).use { input -> + val buffer = ByteArray(1 shl 16) + while (true) { + val read = input.read(buffer) + if (read < 0) break + if (!consumer(buffer, read)) break + consumed += read + } + } + return consumed + } - private fun postError(message: String) { - post { listener.onError(message) } + override fun rename(from: String, to: String): Boolean = File(from).renameTo(File(to)) + + override fun delete(path: String): Boolean = File(path).delete() } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt index 358098ab..10203b7c 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.kt @@ -2,20 +2,19 @@ package com.faceclaw.app import android.os.Handler import android.os.Looper -import java.io.ByteArrayOutputStream +import java.io.IOException import java.net.InetAddress import java.net.NetworkInterface -import java.util.Collections -import org.json.JSONArray import java.net.ServerSocket import java.net.Socket -import java.nio.charset.StandardCharsets -import java.nio.ByteBuffer -import java.security.MessageDigest import java.security.SecureRandom -import org.json.JSONObject +import java.util.Collections -/** Explicit-address, bounded input port. Complete requests notify JS on the main thread. */ +/** + * Explicit-address, bounded input port. The request state machine lives in the shared + * [RemoteInputSession]; this adapter supplies java.net sockets, SecureRandom and main-thread + * delivery, and keeps the JVM API TypeScript calls (app/native/remote-input.ts). + */ class FaceclawRemoteInput { companion object { private val INSTANCE = FaceclawRemoteInput() @@ -23,130 +22,91 @@ class FaceclawRemoteInput { @JvmStatic fun interfaces(): String { - val result = JSONArray() + val all = ArrayList() try { for (device in Collections.list(NetworkInterface.getNetworkInterfaces())) { - if (!device.isUp) continue for (address in Collections.list(device.inetAddresses)) { - if (address.isLoopbackAddress || address.isLinkLocalAddress || address.isAnyLocalAddress) continue - result.put(JSONObject().put("name", device.name).put("address", address.hostAddress) - .put("pointToPoint", device.isPointToPoint)) + all.add(InterfaceAddress( + interfaceName = device.name, + address = address.hostAddress ?: continue, + up = device.isUp, + pointToPoint = device.isPointToPoint, + loopback = address.isLoopbackAddress, + linkLocal = address.isLinkLocalAddress, + anyLocal = address.isAnyLocalAddress, + )) } } } catch (ignored: Exception) {} - return result.toString() - } - - private fun hex(bytes: ByteArray): String { - val result = StringBuilder(bytes.size * 2) - for (b in bytes) result.append(String.format(java.util.Locale.ROOT, "%02x", b.toInt() and 255)) - return result.toString() + return RemoteInputSession.interfacesJson(all) } } - private var server: ServerSocket? = null - private var client: Socket? = null - private var pending: String? = null - private var reply: String? = null - private var nextId = 0L - private var requestId = 0L - private var deadline = 0L + private val mainHandler = Handler(Looper.getMainLooper()) - private var requestListener: Runnable? = null + private val session = RemoteInputSession( + port = AndroidLocalServerPort, + random = object : RandomSource { + private val secureRandom = SecureRandom() + override fun nextBytes(count: Int): ByteArray = ByteArray(count).also { secureRandom.nextBytes(it) } + }, + dispatch = { action -> mainHandler.post(action) }, + platform = AndroidProtocolPlatform, + ) - @Synchronized fun setRequestListener(listener: Runnable?) { requestListener = listener } + fun setRequestListener(listener: Runnable?) { session.setRequestListener(listener?.let { { it.run() } }) } - private fun notifyRequestReady(id: Long) { - // Called with this monitor held; posting avoids entering JS on the socket thread. - mainHandler.post { - val callback: Runnable? - synchronized(this) { - if (requestId != id || pending == null || System.currentTimeMillis() >= deadline) return@post - callback = requestListener - } - callback?.run() - } - } + fun start(port: Int): String { return startAddress(port, "127.0.0.1") } + fun startAddress(port: Int, address: String): String = session.start(port, address) + fun stop() = session.stop() + fun nextRequest(): String? = session.nextRequest() + fun complete(id: Long, value: String?) = session.complete(id, value) + fun randomSecret(): String = session.randomSecret() + fun hash(value: String): String = RemoteInputSession.tokenHash(value) +} - @Synchronized fun start(port: Int): String { return startAddress(port, "127.0.0.1") } - @Synchronized fun startAddress(port: Int, address: String): String { - if (server != null) return "" - try { - // Numeric local addresses only; never resolve hostnames or bind a wildcard. - if (!address.matches(Regex("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?"))) return "Invalid local address." +private object AndroidLocalServerPort : LocalServerPort { + override fun isLocalUnicastAddress(address: String): Boolean { + return try { + // Numeric literals only reach here, so getByName never resolves a hostname. val local = InetAddress.getByName(address) - if (local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null) - return "Address is not assigned to a local interface." - val listener = ServerSocket(port, 8, local) - server = listener - val thread = Thread({ serve(listener) }, "FaceclawInput") - thread.isDaemon = true - thread.start() - return "" - } catch (e: Exception) { return "Could not open local input port $port." } - } - @Synchronized fun stop() { - try { server?.close() } catch (ignored: Exception) {} - try { client?.close() } catch (ignored: Exception) {} - server = null; client = null; pending = null; requestId = 0; reply = null - (this as java.lang.Object).notifyAll() - } - @Synchronized fun nextRequest(): String? { - if (System.currentTimeMillis() >= deadline) { pending = null; return null } - val result = pending; pending = null; return result - } - @Synchronized fun complete(id: Long, value: String?) { - if (id == requestId && System.currentTimeMillis() < deadline) { reply = value; (this as java.lang.Object).notifyAll() } - } - private fun serve(listener: ServerSocket) { - while (!listener.isClosed) { - try { - listener.accept().use { socket -> - synchronized(this) { if (server !== listener) return; client = socket } - socket.soTimeout = 5000 - socket.tcpNoDelay = true - val readDeadline = System.currentTimeMillis() + 5000 - val body = ByteArrayOutputStream() - var b = -1 - while (true) { - val remaining = readDeadline - System.currentTimeMillis() - if (remaining <= 0) throw Exception("Read timeout") - socket.soTimeout = remaining.toInt() - b = socket.getInputStream().read() - if (b == -1 || b == '\n'.code) break - if (body.size() >= 65536) throw Exception("Invalid frame") - body.write(b) - } - if (b != '\n'.code) return@use - val text = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(body.toByteArray())).toString() - val response: String - synchronized(this) { - if (server !== listener) return - requestId = ++nextId - deadline = System.currentTimeMillis() + 5000 - reply = null - pending = JSONObject().put("id", requestId).put("expiresAt", deadline) - .put("body", text).toString() - notifyRequestReady(requestId) - while (reply == null && server === listener && System.currentTimeMillis() < deadline) { - (this as java.lang.Object).wait(Math.max(1L, deadline - System.currentTimeMillis())) - } - response = reply ?: "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}" - if (server === listener) { pending = null; reply = null; requestId = 0 } - } - socket.getOutputStream().write((response + "\n").toByteArray(StandardCharsets.UTF_8)) - } - } catch (ignored: Exception) { - // Never log request bodies: they contain credentials and possibly private text. - } finally { - synchronized(this) { if (server === listener) client = null } - } + !(local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null) + } catch (e: Exception) { + false } } - fun randomSecret(): String { - val bytes = ByteArray(32); SecureRandom().nextBytes(bytes); return hex(bytes) - } - fun hash(value: String): String { - try { return hex(MessageDigest.getInstance("SHA-256").digest(value.toByteArray(StandardCharsets.UTF_8))) } - catch (e: Exception) { throw IllegalStateException("SHA-256 unavailable", e) } + + override fun bind(address: String, port: Int): RemoteInputListener { + val server = ServerSocket(port, 8, InetAddress.getByName(address)) + return object : RemoteInputListener { + override fun accept(): RemoteInputConnection? { + val socket = try { server.accept() } catch (e: IOException) { return null } + return AndroidConnection(socket) + } + override val isClosed: Boolean get() = server.isClosed + override fun close() = server.close() + } } } + +private class AndroidConnection(private val socket: Socket) : RemoteInputConnection { + init { + socket.soTimeout = 5000 + socket.tcpNoDelay = true + } + + override fun readByte(timeoutMs: Long): Int { + return try { + socket.soTimeout = timeoutMs.coerceIn(1L, Int.MAX_VALUE.toLong()).toInt() + socket.getInputStream().read() + } catch (e: IOException) { + -1 + } + } + + override fun write(bytes: ByteArray) { + socket.getOutputStream().write(bytes) + socket.getOutputStream().flush() + } + + override fun close() = socket.close() +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt index abd6d7c1..6a6b1450 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSettings.kt @@ -4,9 +4,6 @@ import android.content.Context import android.content.SharedPreferences import android.os.Handler import android.os.Looper -import android.util.Log - -import java.util.concurrent.CopyOnWriteArrayList /** * App settings store shared by every JS isolate (main thread and app @@ -22,16 +19,26 @@ import java.util.concurrent.CopyOnWriteArrayList * NativeScript worker threads run a message loop, so both the main thread * and workers have a Looper; a listener registered from a Looper-less thread * is accepted but never notified (it can still read fresh values on demand). + * The reads/writes and the fan-out live in the shared SettingsChangeHub. */ class FaceclawSettings private constructor(context: Context) { private val prefs: SharedPreferences = context.applicationContext .getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) - private val listeners = CopyOnWriteArrayList() + private val hub = SettingsChangeHub(object : SettingsStorage { + override fun getString(key: String?, defaultValue: String?): String? = prefs.getString(key, defaultValue) - private class ListenerEntry(val listener: FaceclawSettingsListener, val handler: Handler?) + override fun putString(key: String?, value: String?) { + prefs.edit().putString(key, value).apply() + } + + override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = prefs.getBoolean(key, defaultValue) + + override fun putBoolean(key: String?, value: Boolean) { + prefs.edit().putBoolean(key, value).apply() + } + }) companion object { - private const val TAG = "FaceclawSettings" private const val PREFS_NAME = "faceclaw_settings" @Volatile private var instance: FaceclawSettings? = null @@ -58,23 +65,13 @@ class FaceclawSettings private constructor(context: Context) { } } - fun getString(key: String?, defaultValue: String?): String? { - return prefs.getString(key, defaultValue) - } + fun getString(key: String?, defaultValue: String?): String? = hub.getString(key, defaultValue) - fun setString(key: String?, value: String?) { - prefs.edit().putString(key, value).apply() - notifyChanged(key) - } + fun setString(key: String?, value: String?) = hub.setString(key, value) - fun getBoolean(key: String?, defaultValue: Boolean): Boolean { - return prefs.getBoolean(key, defaultValue) - } + fun getBoolean(key: String?, defaultValue: Boolean): Boolean = hub.getBoolean(key, defaultValue) - fun setBoolean(key: String?, value: Boolean) { - prefs.edit().putBoolean(key, value).apply() - notifyChanged(key) - } + fun setBoolean(key: String?, value: Boolean) = hub.setBoolean(key, value) /** * Register a change listener. Must be called from the thread whose @@ -83,30 +80,9 @@ class FaceclawSettings private constructor(context: Context) { */ fun registerListener(listener: FaceclawSettingsListener) { val looper = Looper.myLooper() - if (looper == null) { - Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified") - } - listeners.add(ListenerEntry(listener, if (looper != null) Handler(looper) else null)) + val handler = if (looper != null) Handler(looper) else null + hub.register(listener, if (handler != null) SettingsDispatcher { action -> handler.post(action) } else null) } - fun unregisterListener(listener: FaceclawSettingsListener?) { - for (entry in listeners) { - if (entry.listener === listener) { - listeners.remove(entry) - } - } - } - - private fun notifyChanged(key: String?) { - for (entry in listeners) { - val handler = entry.handler ?: continue - handler.post { - try { - entry.listener.onSettingChanged(key) - } catch (e: Exception) { - Log.w(TAG, "settings listener failed for key $key", e) - } - } - } - } + fun unregisterListener(listener: FaceclawSettingsListener?) = hub.unregister(listener) } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt index 5cbf7bd2..87dd6c22 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.kt @@ -11,28 +11,9 @@ import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig * utterance; matching/enrollment happens on the TS side (cosine similarity * against the stored speaker profiles). */ -class FaceclawSpeakerId(private val modelPath: String?) { +class FaceclawSpeakerId(private val modelPath: String?) : SpeakerEmbedder { companion object { private const val TAG = "FaceclawSpeakerId" - - @JvmStatic - private fun l2Normalize(v: FloatArray?): FloatArray? { - if (v == null) { - return null - } - var sum = 0.0 - for (x in v) { - sum += x.toDouble() * x - } - val norm = Math.sqrt(sum) - if (norm <= 0) { - return v - } - for (i in v.indices) { - v[i] = (v[i] / norm).toFloat() - } - return v - } } private var extractor: SpeakerEmbeddingExtractor? = null @@ -71,20 +52,18 @@ class FaceclawSpeakerId(private val modelPath: String?) { * is too short to embed (under ~0.5 s). */ @Synchronized - fun embed(pcm16le: ByteArray?, sampleRate: Int): FloatArray? { - if (pcm16le == null || pcm16le.size < sampleRate) { + override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedNullable(pcm16le, sampleRate) + + @Synchronized + fun embedNullable(pcm16le: ByteArray?, sampleRate: Int): FloatArray? { + if (pcm16le == null || !VoicePrint.isLongEnoughToEmbed(pcm16le.size, sampleRate)) { return null } if (!ensureLoaded()) { return null } val extractor = this.extractor!! - val count = pcm16le.size / 2 - val samples = FloatArray(count) - for (i in 0 until count) { - val s = ((pcm16le[i * 2].toInt() and 0xff) or (pcm16le[i * 2 + 1].toInt() shl 8)).toShort() - samples[i] = s / 32768.0f - } + val samples = AudioSegmentation.pcm16leToFloat(pcm16le) var stream: OnlineStream? = null try { stream = extractor.createStream() @@ -94,7 +73,7 @@ class FaceclawSpeakerId(private val modelPath: String?) { return null } val embedding = extractor.compute(stream) - return l2Normalize(embedding) + return VoicePrint.l2Normalize(embedding) } catch (t: Throwable) { Log.w(TAG, "embedding failed", t) return null @@ -105,6 +84,8 @@ class FaceclawSpeakerId(private val modelPath: String?) { } } + override fun release() = close() + @Synchronized fun close() { val extractor = this.extractor diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt index cf7a9514..245ebc95 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSseRequest.kt @@ -4,7 +4,6 @@ import android.os.Handler import android.os.Looper import android.util.Log -import java.io.BufferedReader import java.io.IOException import java.util.concurrent.TimeUnit @@ -23,6 +22,10 @@ import okhttp3.Response * is delivered line by line as it arrives, so the JS side can parse SSE * events incrementally. Like FaceclawWebSocket, listener callbacks are posted * to the Looper of the thread that constructed this object. + * + * The line splitting, error-body and overflow policy live in the shared + * SseStream; this class only supplies okhttp as the transport and the Handler + * delivery. */ class FaceclawSseRequest /** @@ -32,7 +35,8 @@ class FaceclawSseRequest */ constructor(url: String?, jsonBody: String?, headers: Array?, listener: FaceclawSseListener?) { private val callbackHandler: Handler - private val call: Call + private val stream = SseStream() + private val transfer: Cancellable @Volatile private var cancelled = false @@ -45,63 +49,49 @@ constructor(url: String?, jsonBody: String?, headers: Array?, listener: } val looper = Looper.myLooper() callbackHandler = Handler(looper ?: Looper.getMainLooper()) - val builder = Request.Builder() - .url(url.trim()) - .post((jsonBody ?: "").toRequestBody(JSON)) - if (headers != null) { - var i = 0 - while (i + 1 < headers.size) { - val name = headers[i] - val value = headers[i + 1] - if (name != null && !name.isEmpty() && value != null) { - builder.addHeader(name, value) - } - i += 2 - } - } - call = getClient().newCall(builder.build()) - call.enqueue(object : Callback { - override fun onFailure(call: Call, e: IOException) { - postFailure(listener, e.toString()) + val sink = object : ChunkSink { + override fun onStatus(code: Int) { + stream.onStatus(code) } - override fun onResponse(call: Call, response: Response) { - try { - response.body.use { body -> - if (!response.isSuccessful) { - val code = response.code - var errorBody = "" - try { - errorBody = body?.string() ?: "" - } catch (e: IOException) { - Log.w(TAG, "error body read failed", e) - } - val finalBody = errorBody - post { listener.onHttpError(code, finalBody) } - return - } - if (body == null) { - post { listener.onComplete() } - return - } - val reader = BufferedReader(body.charStream()) - var line: String? - while (reader.readLine().also { line = it } != null) { - val finalLine = line - post { listener.onLine(finalLine) } - } - post { listener.onComplete() } - } - } catch (e: IOException) { - postFailure(listener, e.toString()) - } + override fun onChunk(bytes: ByteArray, offset: Int, length: Int) { + stream.onChunk(bytes, offset, length) + drain(listener) } - }) + + override fun onComplete() { + stream.onComplete() + drain(listener) + } + + override fun onFailure(message: String) { + stream.onFailure(message) + drain(listener) + } + } + transfer = OkHttpStreaming.post(url.trim(), jsonBody ?: "", SseStream.unpackHeaders(headers), sink) } fun cancel() { cancelled = true - call.cancel() + stream.cancel() + transfer.cancel() + } + + /** Hands every queued event to the listener on the constructing thread, in order. */ + private fun drain(listener: FaceclawSseListener) { + val events = stream.takeEvents() + if (events.isEmpty()) return + post { + for (event in events) { + if (cancelled) return@post + try { + SseStream.deliver(event, listener) + } catch (t: Throwable) { + Log.w(TAG, "listener callback failed", t) + } + } + } } private fun post(runnable: Runnable) { @@ -112,19 +102,47 @@ constructor(url: String?, jsonBody: String?, headers: Array?, listener: if (cancelled) { return@post } - try { - runnable.run() - } catch (t: Throwable) { - Log.w(TAG, "listener callback failed", t) - } + runnable.run() } } - private fun postFailure(listener: FaceclawSseListener, message: String) { - if (cancelled) { - return + /** okhttp transport: enqueues the call and feeds the body to the sink in chunks. */ + private object OkHttpStreaming : StreamingHttp { + override fun post(url: String, body: String, headers: List>, sink: ChunkSink): Cancellable { + val builder = Request.Builder() + .url(url) + .post(body.toRequestBody(JSON)) + for ((name, value) in headers) builder.addHeader(name, value) + val call = getClient().newCall(builder.build()) + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + sink.onFailure(e.toString()) + } + + override fun onResponse(call: Call, response: Response) { + try { + response.use { + sink.onStatus(response.code) + val input = response.body?.byteStream() + if (input != null) { + val buffer = ByteArray(8192) + while (true) { + val read = input.read(buffer) + if (read < 0) break + if (read > 0) sink.onChunk(buffer, 0, read) + } + } + sink.onComplete() + } + } catch (e: IOException) { + sink.onFailure(e.toString()) + } + } + }) + return object : Cancellable { + override fun cancel() = call.cancel() + } } - post { listener.onFailure(message) } } companion object { @@ -141,9 +159,14 @@ constructor(url: String?, jsonBody: String?, headers: Array?, listener: // Streaming responses can pause between events (e.g. while // the model thinks), so the read timeout is generous. The // Anthropic API sends periodic ping events well within it. + // Redirects are not followed: provider credentials in the + // headers must never be forwarded to another endpoint (a + // 3xx surfaces to the listener as an HTTP error instead). sharedClient = OkHttpClient.Builder() .connectTimeout(15, TimeUnit.SECONDS) .readTimeout(180, TimeUnit.SECONDS) + .followRedirects(false) + .followSslRedirects(false) .addInterceptor(FaceclawHttp.userAgentInterceptor()) .build() } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt index 7c8be12c..b5108f48 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt @@ -14,10 +14,10 @@ import android.util.Log import java.io.ByteArrayOutputStream import java.lang.reflect.Method +/** Reads Wi-Fi/cell/hotspot state; the icon artwork is the shared StatusIconArt. */ class FaceclawSystemStatusIconProvider private constructor() { companion object { private const val TAG = "FaceclawStatusIcons" - private const val ICON_SIZE = 24 @JvmStatic fun getSystemStatusIconGrays(context: Context?, iconSize: Int): ByteArray { @@ -30,16 +30,16 @@ class FaceclawSystemStatusIconProvider private constructor() { val wifiLevel = getWifiLevel(appContext) if (wifiLevel >= 0) { - append(out, scaleIcon(drawWifiIcon(wifiLevel), size)) + append(out, StatusIconArt.scale(StatusIconArt.wifi(wifiLevel), size)) } val cellLevel = getCellLevel(appContext) if (cellLevel >= 0) { - append(out, scaleIcon(drawCellIcon(cellLevel), size)) + append(out, StatusIconArt.scale(StatusIconArt.cell(cellLevel), size)) } if (isHotspotEnabled(appContext)) { - append(out, scaleIcon(drawHotspotIcon(), size)) + append(out, StatusIconArt.scale(StatusIconArt.hotspot(), size)) } return out.toByteArray() @@ -106,86 +106,6 @@ class FaceclawSystemStatusIconProvider private constructor() { return rssi > -127 && rssi < 0 } - private fun drawWifiIcon(level: Int): ByteArray { - val icon = ByteArray(ICON_SIZE * ICON_SIZE) - val cx = 12 - val cy = 18 - fillRect(icon, cx - 1, cy - 1, 3, 3, 230) - val radii = intArrayOf(5, 9, 13, 17) - for (index in radii.indices) { - if (level <= index) { - continue - } - drawWifiArc(icon, cx, cy, radii[index], 210) - } - return icon - } - - private fun drawWifiArc(icon: ByteArray, cx: Int, cy: Int, radius: Int, value: Int) { - val inner = (radius - 1) * (radius - 1) - val outer = (radius + 1) * (radius + 1) - for (y in 0 until ICON_SIZE) { - for (x in 0 until ICON_SIZE) { - val dx = x - cx - val dy = y - cy - val dist = dx * dx + dy * dy - if (dy <= 0 && dist >= inner && dist <= outer && Math.abs(dx) <= radius) { - setPixel(icon, x, y, value) - } - } - } - } - - private fun drawCellIcon(level: Int): ByteArray { - val icon = ByteArray(ICON_SIZE * ICON_SIZE) - for (bar in 0 until 4) { - val x = 5 + bar * 4 - val height = 5 + bar * 4 - val value = if (level > bar) 220 else 55 - fillRect(icon, x, 20 - height, 3, height, value) - } - return icon - } - - private fun drawHotspotIcon(): ByteArray { - val icon = ByteArray(ICON_SIZE * ICON_SIZE) - fillRect(icon, 11, 17, 3, 3, 230) - drawWifiArc(icon, 12, 19, 7, 210) - drawWifiArc(icon, 12, 19, 12, 210) - fillRect(icon, 6, 5, 12, 2, 170) - return icon - } - - private fun scaleIcon(source: ByteArray, size: Int): ByteArray { - if (size == ICON_SIZE) { - return source - } - val scaled = ByteArray(size * size) - for (y in 0 until size) { - val sy = Math.min(ICON_SIZE - 1, (y * ICON_SIZE) / size) - for (x in 0 until size) { - val sx = Math.min(ICON_SIZE - 1, (x * ICON_SIZE) / size) - scaled[y * size + x] = source[sy * ICON_SIZE + sx] - } - } - return scaled - } - - private fun fillRect(icon: ByteArray, x: Int, y: Int, width: Int, height: Int, value: Int) { - for (row in y until y + height) { - for (col in x until x + width) { - setPixel(icon, col, row, value) - } - } - } - - private fun setPixel(icon: ByteArray, x: Int, y: Int, value: Int) { - if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) { - return - } - icon[y * ICON_SIZE + x] = Math.max(0, Math.min(255, value)).toByte() - } - private fun append(out: ByteArrayOutputStream, bytes: ByteArray) { out.write(bytes, 0, bytes.size) } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index 0e59e435..ad0531bb 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -3,52 +3,21 @@ package com.faceclaw.app import android.content.Context import android.os.Handler import android.os.Looper -import android.os.SystemClock import android.util.Log -import com.k2fsa.sherpa.onnx.FeatureConfig -import com.k2fsa.sherpa.onnx.OfflineModelConfig -import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig -import com.k2fsa.sherpa.onnx.OfflineRecognizer -import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig -import com.k2fsa.sherpa.onnx.OfflineRecognizerResult -import com.k2fsa.sherpa.onnx.OfflineStream -import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig - import java.io.File -import java.nio.charset.StandardCharsets -import java.util.ArrayDeque -import java.util.Arrays +/** + * Android host for the shared [VoiceCaptureSession]: supplies the glasses packet source (the + * BLE communicator), the phone microphone, the sherpa-onnx recognizer and speaker model, model + * directory lookup, recording storage, and main-thread callback delivery. The capture pipeline + * itself (packet queue, beam gate, endpointing, segmented transcript, verification) is shared + * with iOS. + */ class FaceclawVoiceController(context: Context) { companion object { private const val TAG = "FaceclawVoice" private const val SAMPLE_RATE = 16000 - private const val FEATURE_DIM = 80 - private const val MAX_AUDIO_QUEUE_PACKETS = 80 - private const val EXPECTED_PACKET_INTERVAL_MS = 50 - private const val LATE_PACKET_INTERVAL_MS = 90 - private const val STATS_INTERVAL_MS = 5_000 - // Push-to-talk utterance boundaries come from the button. We re-decode the - // current audio segment in full for each live partial and emit the complete - // utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder - // used here fails once a single input grows past roughly 9.1 seconds, so - // longer utterances are committed in model-safe segments. - private const val TRANSCRIPT_DECODE_INTERVAL_MS = 700 - private const val TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3 - private const val TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8 - // When a segment fills, cut at the quietest window within the last - // TRANSCRIPT_CUT_SEARCH_SAMPLES rather than mid-word at the 8s mark; the - // audio after the cut carries over into the next segment. - private const val TRANSCRIPT_CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2 - private const val TRANSCRIPT_CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000 - // Glasses-mic PCM peaks around 0.1 full scale, and at that level the - // quantized Moonshine model often returns empty or garbled text. Boost - // each decode window toward this peak, with a gain cap so near-silent - // buffers aren't amplified into pure noise. - private const val TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f - private const val TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f - private const val TRANSCRIPT_LOG_PREVIEW_CHARS = 80 // Model directory shared with the TS-side download flow (asr-model.ts), // which fetches the Moonshine files here on demand; they are no longer // bundled in the APK. @@ -66,62 +35,18 @@ class FaceclawVoiceController(context: Context) { // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, // int8-quantized -- see the model-choice note in asr-model.ts). Directory // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. - // - // On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0) - // runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of - // zero tail padding, capped at 30s. Re-decoding on Moonshine's - // TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that - // encode, growing with the utterance, roughly 1.4x/second while the user is - // still speaking, so onboardModelKind gates that loop off for Whisper -- - // see processRecognizer(). Whisper is also known to hallucinate text on - // near-silent input; recognizeTranscriptSegment() gates that too. + // Whisper re-encodes the whole buffer on every call, so the shared session + // skips live partials for it; see VoiceCaptureSession.processRecognizer. private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8" private val ASR_WHISPER_MODEL_FILES = arrayOf( "base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt" ) - // Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f - // buffer) a segment is treated as silence and never reaches the Whisper - // recognizer at all, rather than risking a hallucinated non-answer. Picked - // conservatively low (well under typical mic noise floor already seen in - // this pipeline's normalization target) -- UNTESTED on real hardware, tune - // against real glasses captures rather than trusting this number. - private const val WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f - - // Speaker verification against the enrolled wearer voice-print ("my voice - // only" command gating). Configured before start(); the utterance PCM is - // buffered (capped) and verified once at session end. - private const val VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10 - private const val VERIFY_MIN_SAMPLES = SAMPLE_RATE - // 50 ms chunks match the G2 packet cadence the rest of the pipeline // (endpointing, transcript pacing) is tuned for. private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20 - @JvmStatic - private fun buildWavHeader(pcmBytes: Int, sampleRate: Int, channels: Int, bitsPerSample: Int): ByteArray { - val byteRate = sampleRate * channels * bitsPerSample / 8 - val blockAlign = channels * bitsPerSample / 8 - val dataSize = pcmBytes - val riffSize = 36 + dataSize - val b = java.nio.ByteBuffer.allocate(44).order(java.nio.ByteOrder.LITTLE_ENDIAN) - b.put("RIFF".toByteArray(StandardCharsets.US_ASCII)) - b.putInt(riffSize) - b.put("WAVE".toByteArray(StandardCharsets.US_ASCII)) - b.put("fmt ".toByteArray(StandardCharsets.US_ASCII)) - b.putInt(16) // PCM fmt chunk size - b.putShort(1.toShort()) // PCM - b.putShort(channels.toShort()) - b.putInt(sampleRate) - b.putInt(byteRate) - b.putShort(blockAlign.toShort()) - b.putShort(bitsPerSample.toShort()) - b.put("data".toByteArray(StandardCharsets.US_ASCII)) - b.putInt(dataSize) - return b.array() - } - @JvmStatic private fun deleteRecursively(file: File) { if (!file.exists()) { @@ -136,46 +61,6 @@ class FaceclawVoiceController(context: Context) { file.delete() } - @JvmStatic - private fun peakAmplitude(samples: FloatArray): Float { - var peak = 0f - for (s in samples) { - val a = Math.abs(s) - if (a > peak) { - peak = a - } - } - return peak - } - - @JvmStatic - private fun normalizePeak(samples: FloatArray) { - val peak = peakAmplitude(samples) - if (peak <= 0f) { - return - } - val gain = Math.min(TRANSCRIPT_NORMALIZE_TARGET_PEAK / peak, TRANSCRIPT_NORMALIZE_MAX_GAIN) - if (gain <= 1f) { - return - } - for (i in samples.indices) { - samples[i] *= gain - } - } - - @JvmStatic - private fun joinTranscript(prefix: String?, suffix: String?): String { - if (prefix == null || prefix.length == 0) { - return suffix ?: "" - } - if (suffix == null || suffix.length == 0) { - return prefix - } - val first = suffix[0] - val attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0 - return prefix + (if (attachesToPrevious) "" else " ") + suffix - } - // The 28 MB embedding model takes seconds to load; keep one instance // across capture sessions so verification adds only the embed time. private var sharedSpeakerId: FaceclawSpeakerId? = null @@ -195,93 +80,68 @@ class FaceclawVoiceController(context: Context) { } } - private enum class VoiceInputMode { - ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind) - CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side - } - - /** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */ - private enum class OnboardModelKind { - MOONSHINE, - WHISPER - } - private val appContext: Context = context.applicationContext private val mainHandler = Handler(Looper.getMainLooper()) - private val lock = Any() - private val audioQueueLock = Any() - private val audioQueue = ArrayDeque() - @Volatile - private var listener: FaceclawVoiceControllerListener? = null @Volatile private var communicator: FaceclawBleCommunicator? = null - private var workerThread: Thread? = null - @Volatile - private var started = false - // Set once the worker has the glasses mic enabled for this session. - // Read and written under `lock`, so it flips with `started` atomically. - private var audioStarted = false - // Capture from the phone's own microphone instead of the G2 over BLE - // (preview-only mode, where no glasses are connected). Latched into - // activePhoneMic at start() (under `lock`) so a mid-session setter call - // can't switch pipelines underneath the worker. - @Volatile - private var usePhoneMic = false - private var activePhoneMic = false - private var mode = VoiceInputMode.CLOUD - @Volatile - private var onboardModelKind = OnboardModelKind.MOONSHINE - private var recognizer: OfflineRecognizer? = null - private var lc3Decoder: FaceclawLc3Decoder? = null - private val transcriptSamples = FloatArray(TRANSCRIPT_SEGMENT_MAX_SAMPLES) - private var transcriptSampleCount = 0 - private var committedTranscriptSampleCount: Long = 0 - private var committedTranscript = "" - private var currentSegmentTranscript = "" - private var lastTranscriptDecodeAtMs: Long = 0 - private var lastTranscript = "" - @Volatile - private var saveRecordings = false - @Volatile - private var endpointing = false - private val endpointDetector = VoiceEndpointDetector() - private var recordingPcm: java.io.ByteArrayOutputStream? = null - // Speaker verification against the enrolled wearer voice-print ("my voice - // only" command gating). Configured before start(); the utterance PCM is - // buffered (capped) and verified once at session end. - @Volatile - private var verifySpeakerModelPath: String? = null - @Volatile - private var verifyWearerEmbedding: FloatArray? = null - @Volatile - private var verifyThreshold = 0.8f - private var verifyBuffer: ShortArray? = null - private var verifyCount = 0 - // Global mic processing (Microphones app config): spectral noise - // suppression and firmware-DoA beam gating, applied to every capture - // session that opts in (assistant push-to-talk, Transcribe, hands-free). - // The raw tap opts out — raw means raw, and the Microphones session does - // its own beam-compensated processing on that path. - @Volatile - private var suppressionEnabled = false - @Volatile - private var beamFilterEnabled = false - @Volatile - private var beamCenterDeg = 0 - @Volatile - private var beamHalfWidthDeg = 180 - private var suppressor: FaceclawNoiseSuppressor? = null - private var queuedPackets: Long = 0 - private var queueDroppedPackets: Long = 0 - private var decodedSamples: Long = 0 - private var latePackets: Long = 0 - private var wrongArmPackets: Long = 0 - private var lastPacketArrivalMs: Long = 0 - private var maxInterPacketMs: Long = 0 - private var lastStatsAtMs: Long = 0 + + private val host = object : VoiceCaptureHost { + override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } } + + override fun onSessionStarting() { + deleteLegacyKwsFiles() + } + + override fun isG2SessionReady(): Boolean { + val currentCommunicator = communicator + return currentCommunicator != null && currentCommunicator.isSessionReady() + } + + override fun isG2AudioCaptureActive(): Boolean { + val currentCommunicator = communicator + return currentCommunicator != null && currentCommunicator.isAudioCaptureActive() + } + + override fun g2AudioSource(): PacketAudioSource? { + val currentCommunicator = communicator ?: return null + return object : PacketAudioSource { + override fun start(listener: FaceclawAudioPacketListener): Boolean = + currentCommunicator.startG2AudioCapture(listener) + + override fun stop() = currentCommunicator.stopG2AudioCapture() + + override fun isActive(): Boolean = currentCommunicator.isAudioCaptureActive() + } + } + + override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter() + + override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES) + + override fun hasTranscriberModel(kind: VoiceModelKind): Boolean = findAsrModelDir(kind) != null + + override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber { + val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing") + return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind)) + } + + override fun speakerEmbedder(modelPath: String): SpeakerEmbedder { + val cached = cachedSpeakerId(modelPath) + // The cached model outlives the session; never release it here. + return object : SpeakerEmbedder { + override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = cached.embed(pcm16le, sampleRate) + + override fun release() {} + } + } + + override fun recordingSink(): RecordingSink = RecordingSink { pcmBytes, sampleRate -> saveRecording(pcmBytes, sampleRate) } + } + + private val session = VoiceCaptureSession(host) fun setListener(listener: FaceclawVoiceControllerListener?) { - this.listener = listener + session.setListener(listener) } fun setCommunicator(communicator: FaceclawBleCommunicator?) { @@ -290,23 +150,22 @@ class FaceclawVoiceController(context: Context) { /** Source the next capture from the phone microphone (no glasses paired). */ fun setUsePhoneMic(usePhoneMic: Boolean) { - this.usePhoneMic = usePhoneMic + session.setUsePhoneMic(usePhoneMic) } /** When true, the decoded mic PCM for each session is saved as a WAV. */ fun setSaveRecordings(saveRecordings: Boolean) { - this.saveRecordings = saveRecordings + session.setSaveRecordings(saveRecordings) } /** * Which on-device model {@link #start}("onboard") should load: "whisper" * selects the second on-device model (sherpa-onnx offline Whisper); any - * other value (including null/absent) keeps the existing Moonshine model, - * so callers that never call this see unchanged behavior. Must be set - * before {@link #start}; has no effect in CLOUD mode. + * other value (including null/absent) keeps the existing Moonshine model. + * Must be set before {@link #start}; has no effect in CLOUD mode. */ fun setOnboardModelKind(kind: String?) { - this.onboardModelKind = if ("whisper" == kind) OnboardModelKind.WHISPER else OnboardModelKind.MOONSHINE + session.setOnboardModelKind(kind) } /** @@ -315,7 +174,7 @@ class FaceclawVoiceController(context: Context) { * button release to end the utterance. Must be set before {@link #start}. */ fun setEndpointing(endpointing: Boolean) { - this.endpointing = endpointing + session.setEndpointing(endpointing) } /** @@ -325,19 +184,16 @@ class FaceclawVoiceController(context: Context) { * to disable. */ fun setSpeakerVerification(speakerModelPath: String?, wearerEmbedding: FloatArray?, threshold: Float) { - this.verifySpeakerModelPath = speakerModelPath - this.verifyWearerEmbedding = wearerEmbedding - this.verifyThreshold = if (threshold > 0) threshold else 0.8f + session.setSpeakerVerification(speakerModelPath, wearerEmbedding, threshold) } fun clearSpeakerVerification() { - this.verifySpeakerModelPath = null - this.verifyWearerEmbedding = null + session.clearSpeakerVerification() } /** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */ fun setNoiseSuppression(enabled: Boolean) { - this.suppressionEnabled = enabled + session.setNoiseSuppression(enabled) } /** @@ -347,62 +203,15 @@ class FaceclawVoiceController(context: Context) { * consumer sees them. Safe to update mid-run. */ fun setBeamFilter(enabled: Boolean, centerDeg: Int, halfWidthDeg: Int) { - this.beamFilterEnabled = enabled - this.beamCenterDeg = centerDeg - this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg)) - } - - private fun withinBeam(angleDegrees: Int): Boolean { - var delta = angleDegrees - beamCenterDeg - while (delta > 180) delta -= 360 - while (delta < -180) delta += 360 - return Math.abs(delta) <= beamHalfWidthDeg - } - - private fun applySuppression(pcm: ShortArray, count: Int) { - try { - if (suppressor == null) { - suppressor = FaceclawNoiseSuppressor(SAMPLE_RATE) - } - val le = ByteArray(count * 2) - for (i in 0 until count) { - le[i * 2] = (pcm[i].toInt() and 0xff).toByte() - le[i * 2 + 1] = ((pcm[i].toInt() shr 8) and 0xff).toByte() - } - val cleaned = suppressor!!.process(le)!! - val cleanedCount = Math.min(count, cleaned.size / 2) - for (i in 0 until cleanedCount) { - pcm[i] = ((cleaned[i * 2].toInt() and 0xff) or (cleaned[i * 2 + 1].toInt() shl 8)).toShort() - } - } catch (t: Throwable) { - Log.w(TAG, "noise suppression failed; passing audio through", t) - suppressionEnabled = false - } + session.setBeamFilter(enabled, centerDeg, halfWidthDeg) } fun start(requestedMode: String?) { - start(requestedMode, 0) + session.start(requestedMode, 0) } fun start(requestedMode: String?, captureId: Int) { - synchronized(lock) { - if (started) { - emitStatus("Voice control is already listening.") - emitStopped(captureId) - return - } - if (!usePhoneMic && (communicator == null || !communicator!!.isSessionReady())) { - emitStatus("Voice control needs an active G2 connection.") - emitStopped(captureId) - return - } - mode = parseMode(requestedMode) - activePhoneMic = usePhoneMic - started = true - audioStarted = false - workerThread = Thread(Runnable { runLoop(captureId) }, "FaceclawVoiceController") - workerThread!!.start() - } + session.start(requestedMode, captureId) } /** @@ -412,223 +221,39 @@ class FaceclawVoiceController(context: Context) { * worker that will never see another packet. Anything deciding whether to * (re)start capture must ask this rather than assume its own bookkeeping. */ - fun isCapturing(): Boolean { - val audioUp: Boolean - val phoneMic: Boolean - synchronized(lock) { - if (!started) { - return false - } - audioUp = audioStarted - phoneMic = activePhoneMic - } - if (!audioUp) { - // The worker is still bringing the mic up; report it as running so - // a concurrent request shares it instead of restarting it. - return true - } - if (phoneMic) { - // AudioRecord has no session to lose the enable to; it runs until stop(). - return true - } - val currentCommunicator = communicator - return currentCommunicator != null && currentCommunicator.isAudioCaptureActive() - } + fun isCapturing(): Boolean = session.isCapturing() fun stop() { - val threadToJoin: Thread? - synchronized(lock) { - if (!started) { - return - } - started = false - threadToJoin = workerThread - } - stopG2Audio() - synchronized(audioQueueLock) { - (audioQueueLock as java.lang.Object).notifyAll() - } - if (threadToJoin != null) { - threadToJoin.interrupt() - if (Thread.currentThread() !== threadToJoin) { - try { - threadToJoin.join(1500) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - } - } - } + session.stop() } fun close() { stop() } - private fun parseMode(requestedMode: String?): VoiceInputMode { - if ("cloud" == requestedMode) { - return VoiceInputMode.CLOUD - } - return VoiceInputMode.ONBOARD - } - - private fun runLoop(captureId: Int) { - try { - deleteLegacyKwsFiles() - val currentMode = mode - val currentOnboardKind = onboardModelKind - if (currentMode == VoiceInputMode.ONBOARD) { - val modelDir = findAsrModelDir(currentOnboardKind) - if (modelDir == null) { - emitStatus("Voice model not downloaded (see Settings > Voice).") - return - } - emitStatus("Loading transcription model...") - recognizer = OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind)) - resetTranscriptState() - lastTranscript = "" - } - endpointDetector.reset() - if (suppressor != null) { - suppressor!!.reset() - } - recordingPcm = if (saveRecordings) java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) else null - val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null - verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null - verifyCount = 0 - if (activePhoneMic) { - val record = openPhoneMic() - if (record == null) { - emitStatus("Could not start the phone microphone.") - return - } - synchronized(lock) { - audioStarted = true - } - emitStatus(if (currentMode == VoiceInputMode.CLOUD) - "Listening (cloud)..." - else - "Listening...") - try { - processPhoneAudio(record) - } finally { - try { - record.stop() - } catch (ignored: Throwable) { - // Already stopped or never recording; release below either way. - } - record.release() - } - } else { - lc3Decoder = FaceclawLc3Decoder() - if (!startG2Audio()) { - emitStatus("Could not start G2 microphone input.") - return - } - synchronized(lock) { - audioStarted = true - } - emitStatus(if (currentMode == VoiceInputMode.CLOUD) - "Listening (cloud)..." - else - "Listening...") - processG2Audio() - } - // Verification result must precede the final transcript so the - // TS bridge can suppress a non-wearer command before it is acted - // on (the callbacks are posted in order to the main handler). - runSpeakerVerification() - // Button released / stop requested: emit one final full-utterance - // transcript so the UI can freeze it. - if (currentMode == VoiceInputMode.ONBOARD) { - decodeTranscript(true) - } - } catch (error: Throwable) { - Log.e(TAG, "Voice control failed", error) - emitStatus("Voice control failed: " + error.message) - } finally { - stopG2Audio() - writeRecordingIfAny() - releaseSherpa() - releaseLc3() - synchronized(lock) { - started = false - audioStarted = false - workerThread = null - } - emitStopped(captureId) - } - } - - private fun appendRecording(pcm: ShortArray, count: Int) { - val out = recordingPcm - if (out == null) { - return - } - for (i in 0 until count) { - val s = pcm[i].toInt() - out.write(s and 0xff) - out.write((s shr 8) and 0xff) - } - } - /** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */ - private fun writeRecordingIfAny() { - val out = recordingPcm - recordingPcm = null - if (out == null || out.size() == 0) { - return - } + private fun saveRecording(pcmBytes: ByteArray, sampleRate: Int) { try { - val pcmBytes = out.toByteArray() - val dir = java.io.File(appContext.getExternalFilesDir(null), "voice-recordings") + val dir = File(appContext.getExternalFilesDir(null), "voice-recordings") if (!dir.exists() && !dir.mkdirs()) { Log.w(TAG, "could not create voice-recordings dir") return } val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US) .format(java.util.Date()) - val file = java.io.File(dir, "voice-$stamp.wav") + val file = File(dir, "voice-$stamp.wav") java.io.FileOutputStream(file).use { fos -> - fos.write(buildWavHeader(pcmBytes.size, SAMPLE_RATE, 1, 16)) + fos.write(BinaryEncoding.wavHeader(pcmBytes.size, sampleRate, 1)) fos.write(pcmBytes) } Log.i(TAG, "saved voice recording " + file.absolutePath + " samples=" + (pcmBytes.size / 2) - + " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / SAMPLE_RATE)) + + " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / sampleRate)) } catch (t: Throwable) { Log.w(TAG, "failed to save voice recording", t) } } - private fun buildRecognizerConfig(modelDir: File, kind: OnboardModelKind): OfflineRecognizerConfig { - val modelConfig = OfflineModelConfig.builder() - .setNumThreads(1) - if (kind == OnboardModelKind.WHISPER) { - modelConfig - .setWhisper(OfflineWhisperModelConfig.builder() - .setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath) - .setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath) - .setLanguage("en") - .setTask("transcribe") - .build()) - .setTokens(File(modelDir, "base.en-tokens.txt").absolutePath) - } else { - modelConfig - .setMoonshine(OfflineMoonshineModelConfig.builder() - .setEncoder(File(modelDir, "encoder_model.ort").absolutePath) - .setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath) - .build()) - .setTokens(File(modelDir, "tokens.txt").absolutePath) - } - return OfflineRecognizerConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setModelConfig(modelConfig.build()) - .build() - } - /** * The on-device model directory for the given kind, populated by the * download flow in asr-model.ts (releases before 0.5.0 copied the @@ -636,8 +261,8 @@ class FaceclawVoiceController(context: Context) { * already complete for that model). Null when any expected file is * missing, i.e. the model still needs to be downloaded. */ - private fun findAsrModelDir(kind: OnboardModelKind): File? { - val whisper = kind == OnboardModelKind.WHISPER + private fun findAsrModelDir(kind: VoiceModelKind): File? { + val whisper = kind == VoiceModelKind.WHISPER val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName) @@ -657,530 +282,4 @@ class FaceclawVoiceController(context: Context) { Log.w(TAG, "failed to delete legacy wake-word files", t) } } - - private fun startG2Audio(): Boolean { - val currentCommunicator = communicator - if (currentCommunicator == null) { - return false - } - resetAudioStats() - synchronized(audioQueueLock) { - audioQueue.clear() - } - return currentCommunicator.startG2AudioCapture(object : FaceclawAudioPacketListener { - override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { - queueAudioPacket(data, arm, arrivalMs) - } - }) - } - - private fun processG2Audio() { - val pcm = ShortArray(FaceclawLc3Decoder.SAMPLES_PER_PACKET) - while (started && !Thread.currentThread().isInterrupted) { - val currentDecoder = lc3Decoder - if (currentDecoder == null) { - return - } - - val packet = takeAudioPacket() - if (packet == null) { - continue - } - - val count = currentDecoder.decodePacket(packet.data, pcm) - if (count <= 0) { - maybeEmitAudioStats(false) - continue - } - decodedSamples += count.toLong() - // Global mic processing from the Microphones app config: the - // beam filter drops packets whose firmware direction-of-arrival - // falls outside the listening wedge (isolating the aimed talker - // for every consumer, recognition included), and the spectral - // noise suppressor cleans what remains before it reaches the - // recognizer, cloud PCM, endpointing, or speaker verification. - val angleDegrees = currentDecoder.getLastAngleDegrees() - val ssr = currentDecoder.getLastSsr() - if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { - emitFrameMeta(angleDegrees, ssr) - maybeEmitAudioStats(false) - continue - } - processPcmChunk(pcm, count, angleDegrees, ssr, true) - maybeEmitAudioStats(false) - } - } - - /** - * Per-chunk processing shared by the G2 and phone-mic paths, downstream of - * decode and the beam filter. hasFrameMeta is false for the phone mic, - * which has no firmware DSP metadata to report. - */ - private fun processPcmChunk(pcm: ShortArray, count: Int, angleDegrees: Int, ssr: Int, hasFrameMeta: Boolean) { - if (suppressionEnabled) { - applySuppression(pcm, count) - } - if (recordingPcm != null) { - appendRecording(pcm, count) - } - val verifyBuffer = this.verifyBuffer - if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { - val copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount) - System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied) - verifyCount += copied - } - if (endpointing && endpointDetector.accept(pcm, count)) { - emitSpeechEnd() - } - // PCM and frame metadata flow in every mode so levels, recording, - // and the Microphones radar keep working alongside onboard ASR. - emitPcm(pcm, count) - if (hasFrameMeta) { - emitFrameMeta(angleDegrees, ssr) - } - if (mode != VoiceInputMode.CLOUD) { - val samples = FloatArray(count) - for (i in 0 until count) { - samples[i] = pcm[i] / 32768.0f - } - processRecognizer(samples) - } - } - - /** - * Open the phone's own microphone at the pipeline's native format - * (16 kHz mono PCM16), or null when it cannot start — the permission is - * missing (SecurityException) or the device refuses the configuration. - */ - private fun openPhoneMic(): android.media.AudioRecord? { - var record: android.media.AudioRecord? = null - try { - val minBytes = android.media.AudioRecord.getMinBufferSize( - SAMPLE_RATE, - android.media.AudioFormat.CHANNEL_IN_MONO, - android.media.AudioFormat.ENCODING_PCM_16BIT) - val bufferBytes = Math.max(minBytes, PHONE_MIC_CHUNK_SAMPLES * 2 * 4) - record = android.media.AudioRecord( - android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION, - SAMPLE_RATE, - android.media.AudioFormat.CHANNEL_IN_MONO, - android.media.AudioFormat.ENCODING_PCM_16BIT, - bufferBytes) - if (record.state != android.media.AudioRecord.STATE_INITIALIZED) { - record.release() - return null - } - record.startRecording() - if (record.recordingState != android.media.AudioRecord.RECORDSTATE_RECORDING) { - record.release() - return null - } - return record - } catch (t: Throwable) { - Log.w(TAG, "phone mic open failed", t) - if (record != null) { - record.release() - } - return null - } - } - - /** - * Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame - * metadata — AudioRecord already delivers the pipeline's PCM format. The - * blocking read returns every chunk (50 ms), which bounds how long a - * stop() waits for the loop to notice `started` dropped. - */ - private fun processPhoneAudio(record: android.media.AudioRecord) { - val pcm = ShortArray(PHONE_MIC_CHUNK_SAMPLES) - while (started && !Thread.currentThread().isInterrupted) { - val read = record.read(pcm, 0, pcm.size) - if (read < 0) { - Log.w(TAG, "phone mic read failed: $read") - return - } - if (read == 0) { - continue - } - decodedSamples += read.toLong() - processPcmChunk(pcm, read, 0, 0, false) - } - } - - private fun processRecognizer(samples: FloatArray) { - appendTranscriptSamples(samples) - // Each Whisper call re-encodes the whole buffer plus ~10s of tail - // padding (see the ASR_WHISPER_* comment above), - // so it skips the live-partial redecode Moonshine does on this - // interval and only decodes when a segment commits (8s buffer fill) - // or the utterance ends (decodeTranscript(true) in runLoop()). This - // means no live preview text while speaking in Whisper mode -- status - // stays "Listening..." until release. This is a deliberate tradeoff. - if (onboardModelKind == OnboardModelKind.WHISPER) { - return - } - val now = SystemClock.elapsedRealtime() - if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES - && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { - decodeTranscript(false) - lastTranscriptDecodeAtMs = now - } - } - - private fun appendTranscriptSamples(samples: FloatArray) { - var sourceOffset = 0 - while (sourceOffset < samples.size) { - val available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount - val count = Math.min(available, samples.size - sourceOffset) - System.arraycopy(samples, sourceOffset, transcriptSamples, transcriptSampleCount, count) - transcriptSampleCount += count - sourceOffset += count - - if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) { - commitTranscriptSegment() - } - } - } - - /** - * Decode the current model-safe segment and emit the best transcript of the - * complete utterance (REPLACE semantics — the caller displays it as-is). - */ - private fun decodeTranscript(isFinal: Boolean) { - if (recognizer == null || transcriptSampleCount <= 0) { - if (isFinal) { - emitTranscript(lastTranscript, true) - } - return - } - val segmentSampleCount = transcriptSampleCount - var segmentText = recognizeTranscriptSegment(segmentSampleCount) - if (segmentText.length > 0) { - currentSegmentTranscript = segmentText - } else { - segmentText = currentSegmentTranscript - } - val text = joinTranscript(committedTranscript, segmentText) - lastTranscript = text - logTranscriptDecode(isFinal, segmentSampleCount, text) - emitTranscript(text, isFinal) - } - - /** - * Finalize a full segment before accepting more audio. This keeps every - * Moonshine invocation below its failing sequence length while retaining - * all earlier text in the replace-semantics preview. - */ - private fun commitTranscriptSegment() { - val cut = findSegmentCutPoint() - var segmentText = recognizeTranscriptSegment(cut) - if (segmentText.length == 0) { - // Fallback text came from partial decodes of the full buffer, so it - // may include words from the carried-over tail; rare now that decode - // windows are peak-normalized. - segmentText = currentSegmentTranscript - } - committedTranscript = joinTranscript(committedTranscript, segmentText) - currentSegmentTranscript = "" - committedTranscriptSampleCount += cut.toLong() - val tail = transcriptSampleCount - cut - System.arraycopy(transcriptSamples, cut, transcriptSamples, 0, tail) - transcriptSampleCount = tail - lastTranscript = committedTranscript - lastTranscriptDecodeAtMs = SystemClock.elapsedRealtime() - logTranscriptDecode(false, cut, committedTranscript) - emitTranscript(committedTranscript, false) - } - - /** - * Pick where to end the committed segment: the center of the quietest - * window within the search region at the end of the buffer, so the cut - * lands between words instead of splitting one. - */ - private fun findSegmentCutPoint(): Int { - val count = transcriptSampleCount - val searchStart = Math.max(0, count - TRANSCRIPT_CUT_SEARCH_SAMPLES) - val win = TRANSCRIPT_CUT_WINDOW_SAMPLES - if (count - searchStart <= win) { - return count - } - var sum = 0.0 - for (i in searchStart until searchStart + win) { - sum += transcriptSamples[i].toDouble() * transcriptSamples[i] - } - var best = sum - var bestStart = searchStart - var start = searchStart + 1 - while (start + win <= count) { - val dropped = transcriptSamples[start - 1] - val added = transcriptSamples[start + win - 1] - sum += added.toDouble() * added - dropped.toDouble() * dropped - if (sum < best) { - best = sum - bestStart = start - } - start++ - } - return bestStart + win / 2 - } - - private fun recognizeTranscriptSegment(sampleCount: Int): String { - val currentRecognizer = recognizer - if (currentRecognizer == null || sampleCount <= 0) { - return "" - } - val segment = Arrays.copyOf(transcriptSamples, sampleCount) - // Whisper hallucinates text on near-silent input (a known quirk of the - // model, not this pipeline); gate it on the PRE-normalization peak, since - // normalizePeak() below would otherwise amplify true silence right up to - // the target level and hide the very thing being checked for. Moonshine - // does not share this failure mode in practice, so it is left unchanged. - if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) { - return "" - } - normalizePeak(segment) - val offlineStream: OfflineStream = currentRecognizer.createStream() - try { - offlineStream.acceptWaveform(segment, SAMPLE_RATE) - currentRecognizer.decode(offlineStream) - val result: OfflineRecognizerResult? = currentRecognizer.getResult(offlineStream) - val raw: String? = if (result == null) "" else result.text - return raw?.trim() ?: "" - } finally { - offlineStream.release() - } - } - - private fun logTranscriptDecode(isFinal: Boolean, segmentSampleCount: Int, text: String) { - val totalAudioSec = - (committedTranscriptSampleCount + transcriptSampleCount) / SAMPLE_RATE.toDouble() - val preview = if (text.length <= TRANSCRIPT_LOG_PREVIEW_CHARS) - text else text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..." - Log.i(TAG, (if (onboardModelKind == OnboardModelKind.WHISPER) "Whisper" else "Moonshine") + " decode final=" + isFinal - + " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec) - + " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / SAMPLE_RATE.toDouble()) - + " textLen=" + text.length + " text=\"" + preview + "\"") - } - - private fun resetTranscriptState() { - transcriptSampleCount = 0 - committedTranscriptSampleCount = 0 - committedTranscript = "" - currentSegmentTranscript = "" - lastTranscriptDecodeAtMs = 0 - } - - private fun emitPcm(pcm: ShortArray, count: Int) { - val currentListener = listener - if (currentListener == null || count <= 0) { - return - } - val le = ByteArray(count * 2) - for (i in 0 until count) { - val s = pcm[i].toInt() - le[i * 2] = (s and 0xff).toByte() - le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() - } - mainHandler.post { currentListener.onPcm(le) } - } - - /** - * Embed the session's buffered utterance and compare it to the enrolled - * wearer voice-print. Fails open: a session too short to verify, or a - * model that will not load, counts as the wearer rather than silencing - * every command. - */ - private fun runSpeakerVerification() { - val buffer = verifyBuffer - val wearer = verifyWearerEmbedding - val modelPath = verifySpeakerModelPath - verifyBuffer = null - if (buffer == null || wearer == null || modelPath == null) { - return - } - if (verifyCount < VERIFY_MIN_SAMPLES) { - emitSpeakerVerified(true, 0f) - return - } - val speakerId = cachedSpeakerId(modelPath) - try { - val le = ByteArray(verifyCount * 2) - for (i in 0 until verifyCount) { - val s = buffer[i].toInt() - le[i * 2] = (s and 0xff).toByte() - le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() - } - val embedding = speakerId.embed(le, SAMPLE_RATE) - if (embedding == null || embedding.size != wearer.size) { - emitSpeakerVerified(true, 0f) - return - } - var dot = 0.0 - for (i in embedding.indices) { - dot += embedding[i].toDouble() * wearer[i] - } - val isWearer = dot >= verifyThreshold - Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot) - + " threshold=" + verifyThreshold + " isWearer=" + isWearer) - emitSpeakerVerified(isWearer, dot.toFloat()) - } catch (t: Throwable) { - Log.w(TAG, "speaker verification failed", t) - emitSpeakerVerified(true, 0f) - } - } - - private fun emitSpeakerVerified(isWearer: Boolean, similarity: Float) { - val currentListener = listener - if (currentListener == null) { - return - } - mainHandler.post { currentListener.onSpeakerVerified(isWearer, similarity) } - } - - private fun emitFrameMeta(angleDegrees: Int, ssr: Int) { - val currentListener = listener - if (currentListener == null) { - return - } - mainHandler.post { currentListener.onFrameMeta(angleDegrees, ssr) } - } - - private fun emitSpeechEnd() { - val currentListener = listener - if (currentListener == null) { - return - } - mainHandler.post { currentListener.onSpeechEnd() } - } - - private fun emitStopped(captureId: Int) { - val currentListener = listener - if (currentListener != null) { - mainHandler.post { currentListener.onStopped(captureId) } - } - } - - private fun stopG2Audio() { - val currentCommunicator = communicator - if (currentCommunicator != null) { - currentCommunicator.stopG2AudioCapture() - } - maybeEmitAudioStats(true) - } - - private fun releaseSherpa() { - if (recognizer != null) { - recognizer!!.release() - recognizer = null - } - } - - private fun releaseLc3() { - if (lc3Decoder != null) { - lc3Decoder!!.close() - lc3Decoder = null - } - } - - private fun queueAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { - if (!started || data == null) { - return - } - if ("L" != arm) { - wrongArmPackets++ - } - synchronized(audioQueueLock) { - if (audioQueue.size >= MAX_AUDIO_QUEUE_PACKETS) { - audioQueue.removeFirst() - queueDroppedPackets++ - } - audioQueue.addLast(AudioPacket(data, arm, arrivalMs)) - queuedPackets++ - if (lastPacketArrivalMs > 0) { - val delta = arrivalMs - lastPacketArrivalMs - if (delta > maxInterPacketMs) { - maxInterPacketMs = delta - } - if (delta > LATE_PACKET_INTERVAL_MS) { - latePackets++ - } - } - lastPacketArrivalMs = arrivalMs - (audioQueueLock as java.lang.Object).notifyAll() - } - } - - private fun takeAudioPacket(): AudioPacket? { - synchronized(audioQueueLock) { - while (started && audioQueue.isEmpty()) { - try { - (audioQueueLock as java.lang.Object).wait(250) - maybeEmitAudioStats(false) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - return null - } - } - return audioQueue.pollFirst() - } - } - - private fun resetAudioStats() { - queuedPackets = 0 - queueDroppedPackets = 0 - decodedSamples = 0 - latePackets = 0 - wrongArmPackets = 0 - lastPacketArrivalMs = 0 - maxInterPacketMs = 0 - lastStatsAtMs = SystemClock.elapsedRealtime() - } - - private fun maybeEmitAudioStats(force: Boolean) { - val now = SystemClock.elapsedRealtime() - if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) { - return - } - lastStatsAtMs = now - val currentDecoder = lc3Decoder - val real = if (currentDecoder == null) 0L else currentDecoder.getRealPackets() - val duplicate = if (currentDecoder == null) 0L else currentDecoder.getDuplicatePackets() - val missing = if (currentDecoder == null) 0L else currentDecoder.getMissingPackets() - val decodeErrors = if (currentDecoder == null) 0L else currentDecoder.getDecodeErrors() - val status = "G2 mic packets=" + queuedPackets + - " decoded=" + real + - " missing=" + missing + - " duplicate=" + duplicate + - " late=" + latePackets + - " maxGapMs=" + maxInterPacketMs + - "\n" + - " queueDrop=" + queueDroppedPackets + - " decodeErrors=" + decodeErrors + - " wrongArm=" + wrongArmPackets + - " audioSec=" + String.format(java.util.Locale.US, "%.1f", decodedSamples / SAMPLE_RATE.toDouble()) - // Audio-pipeline stats are diagnostic; keep them in logcat only, out of - // the on-glasses voice UI. - Log.i(TAG, status.replace('\n', ' ') + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS) - } - - private fun emitStatus(status: String?) { - val currentListener = listener - if (currentListener == null) { - return - } - mainHandler.post { currentListener.onStatus(status) } - } - - private fun emitTranscript(text: String?, isFinal: Boolean) { - val currentListener = listener - if (currentListener == null) { - return - } - Log.i(TAG, "Emit transcript final=" + isFinal + " textLen=" + (text?.trim()?.length ?: 0)) - mainHandler.post { currentListener.onTranscript(text, isFinal) } - } - - private class AudioPacket(val data: ByteArray?, arm: String?, val arrivalMs: Long) { - val arm: String = arm ?: "?" - } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt index e6b717ec..fa33d1c3 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FontFileRenderer.kt @@ -11,7 +11,6 @@ import android.util.Log import java.io.File import java.io.RandomAccessFile import java.nio.ByteBuffer -import java.nio.ByteOrder import java.util.LinkedHashMap /** @@ -20,7 +19,9 @@ import java.util.LinkedHashMap * minikin/HarfBuzz. Output uses the same grayscale packet format as * ImageFileLoader ([widthLo, widthHi, heightLo, heightHi, pixels...], one * byte per pixel, row-major), which the TS side turns into a GrayImage and - * the compositor later quantizes to the 4bpp the firmware wants. + * the compositor later quantizes to the 4bpp the firmware wants. The packet + * layouts, gamma mapping and ink cropping are shared with iOS in GlyphPacket; + * only rasterization lives here. * * The gamma parameter maps antialiased coverage to output shade: * out = 255 * (coverage/255)^gamma. The G2 display response looks roughly @@ -88,27 +89,13 @@ class FontFileRenderer private constructor() { // keep at least the advance width so spacing stays truthful. var left = pad var right = pad + advance - val painted = paintedColumnRange(pixels, width, height) + val painted = GlyphPacket.paintedColumnRange(pixels, width, height) if (painted != null) { left = Math.min(left, painted[0]) right = Math.max(right, painted[1] + 1) } val outWidth = Math.max(1, right - left) - - val lut = gammaLut(gamma) - val out = ByteArray(4 + outWidth * height) - out[0] = (outWidth and 0xff).toByte() - out[1] = ((outWidth shr 8) and 0xff).toByte() - out[2] = (height and 0xff).toByte() - out[3] = ((height shr 8) and 0xff).toByte() - for (y in 0 until height) { - val srcRow = y * width - val dstRow = 4 + y * outWidth - for (x in 0 until outWidth) { - out[dstRow + x] = lut[pixels[srcRow + left + x].toInt() and 0xff] - } - } - return out + return GlyphPacket.packImageRegion(pixels, width, left, 0, outWidth, height, gamma) } catch (e: Exception) { Log.w(TAG, "text render failed: $path", e) return ByteArray(0) @@ -159,16 +146,7 @@ class FontFileRenderer private constructor() { val pixels = alpha8Pixels(bitmap, maxWidth, height) bitmap.recycle() - val lut = gammaLut(gamma) - val out = ByteArray(4 + pixels.size) - out[0] = (maxWidth and 0xff).toByte() - out[1] = ((maxWidth shr 8) and 0xff).toByte() - out[2] = (height and 0xff).toByte() - out[3] = ((height shr 8) and 0xff).toByte() - for (i in pixels.indices) { - out[4 + i] = lut[pixels[i].toInt() and 0xff] - } - return out + return GlyphPacket.packImage(pixels, maxWidth, height, gamma) } catch (e: Exception) { Log.w(TAG, "wrapped render failed: $path", e) return ByteArray(0) @@ -206,7 +184,7 @@ class FontFileRenderer private constructor() { val bounds = android.graphics.Rect() paint.getTextBounds(text, 0, text.length, bounds) if (bounds.isEmpty) { - return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null) + return GlyphPacket.emptyGlyphCell(advanceFixed) } // getTextBounds can be off by a hair on AA edges; render with // padding, then trim to the actually painted box. @@ -224,40 +202,11 @@ class FontFileRenderer private constructor() { val pixels = alpha8Pixels(bitmap, bw, bh) bitmap.recycle() - var x0 = bw - var x1 = -1 - var y0 = bh - var y1 = -1 - for (y in 0 until bh) { - val row = y * bw - for (x in 0 until bw) { - if (pixels[row + x].toInt() != 0) { - if (x < x0) x0 = x - if (x > x1) x1 = x - if (y < y0) y0 = y - if (y > y1) y1 = y - } - } - } - if (x1 < x0) { - return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null) - } - val width = x1 - x0 + 1 - val height = y1 - y0 + 1 + // Trim to the tight ink box; bearing/inkTop become pen- and + // line-relative inside the shared packer. val fm = paint.fontMetricsInt - val bearingX = x0 - penX - val inkTop = (-fm.ascent) + (y0 - baselineY) - - val lut = gammaLut(gamma) - val coverage = ByteArray(width * height) - for (y in 0 until height) { - val src = (y0 + y) * bw + x0 - val dst = y * width - for (x in 0 until width) { - coverage[dst + x] = lut[pixels[src + x].toInt() and 0xff] - } - } - return glyphCellPacket(advanceFixed, bearingX, inkTop, width, height, coverage) + return GlyphPacket.packGlyphCell( + advanceFixed, pixels, bw, bh, penX, baselineY, -fm.ascent, gamma) } catch (e: Exception) { Log.w(TAG, "glyph render failed: " + path + " U+" + Integer.toHexString(codePoint), e) return ByteArray(0) @@ -294,26 +243,6 @@ class FontFileRenderer private constructor() { return paint } - @JvmStatic - private fun glyphCellPacket( - advanceFixed: Int, bearingX: Int, inkTop: Int, width: Int, height: Int, coverage: ByteArray?): ByteArray { - val out = ByteArray(10 + (coverage?.size ?: 0)) - out[0] = (advanceFixed and 0xff).toByte() - out[1] = ((advanceFixed shr 8) and 0xff).toByte() - out[2] = (bearingX and 0xff).toByte() - out[3] = ((bearingX shr 8) and 0xff).toByte() - out[4] = (inkTop and 0xff).toByte() - out[5] = ((inkTop shr 8) and 0xff).toByte() - out[6] = (width and 0xff).toByte() - out[7] = ((width shr 8) and 0xff).toByte() - out[8] = (height and 0xff).toByte() - out[9] = ((height shr 8) and 0xff).toByte() - if (coverage != null) { - System.arraycopy(coverage, 0, out, 10, coverage.size) - } - return out - } - /** * Line metrics for a font at a pixel size, as "ascent descent lineGap" * (integers, pixels). Empty string if the font cannot be loaded. @@ -361,8 +290,21 @@ class FontFileRenderer private constructor() { */ @JvmStatic fun getFontName(path: String?): String { + if (path == null) { + return "" + } try { - return parseNameTable(path) + RandomAccessFile(path, "r").use { raf -> + return OpenTypeNames.parse(object : RandomAccessBytes { + override val length: Long + get() = raf.length() + + override fun read(offset: Long, into: ByteArray, count: Int) { + raf.seek(offset) + raf.readFully(into, 0, count) + } + }) + } } catch (e: Exception) { Log.w(TAG, "name table parse failed: $path", e) return "" @@ -431,144 +373,5 @@ class FontFileRenderer private constructor() { } return pixels } - - /** [first, last] painted (nonzero) column indexes, or null if blank. */ - @JvmStatic - private fun paintedColumnRange(pixels: ByteArray, width: Int, height: Int): IntArray? { - var first = width - var last = -1 - for (y in 0 until height) { - val row = y * width - var x = 0 - while (x < first) { - if (pixels[row + x].toInt() != 0) { - first = x - break - } - x++ - } - x = width - 1 - while (x > last) { - if (pixels[row + x].toInt() != 0) { - last = x - break - } - x-- - } - } - return if (last >= first) intArrayOf(first, last) else null - } - - @JvmStatic - private fun gammaLut(gammaIn: Double): ByteArray { - val lut = ByteArray(256) - var gamma = gammaIn - if (gamma <= 0) { - gamma = 1.0 - } - for (i in 0 until 256) { - lut[i] = Math.max(0, Math.min(255, - Math.round(255.0 * Math.pow(i / 255.0, gamma)).toInt())).toByte() - } - return lut - } - - // --- Minimal 'name' table reader (TTF/OTF/TTC) for preview titles --- - - @JvmStatic - @Throws(Exception::class) - private fun parseNameTable(path: String?): String { - RandomAccessFile(path, "r").use { raf -> - val fileSize = raf.length() - if (fileSize < 12) { - return "" - } - var offset = 0L - val tag = readU32(raf, 0) - if (tag == 0x74746366) { // 'ttcf': use the first face - if (readU32(raf, 8) < 1) { - return "" - } - offset = readU32(raf, 12).toLong() and 0xffffffffL - } - val numTables = readU16(raf, offset + 4) - var nameOffset = -1L - for (i in 0 until numTables) { - val rec = offset + 12 + i * 16L - if (rec + 16 > fileSize) { - return "" - } - if (readU32(raf, rec) == 0x6e616d65) { // 'name' - nameOffset = readU32(raf, rec + 8).toLong() and 0xffffffffL - break - } - } - if (nameOffset < 0 || nameOffset + 6 > fileSize) { - return "" - } - val count = readU16(raf, nameOffset + 2) - val stringStorage = nameOffset + readU16(raf, nameOffset + 4) - var family: String? = null - var style: String? = null - var preferredFamily: String? = null - var preferredStyle: String? = null - for (i in 0 until count) { - val rec = nameOffset + 6 + i * 12L - if (rec + 12 > fileSize) { - break - } - val platform = readU16(raf, rec) - val nameId = readU16(raf, rec + 6) - if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) { - continue - } - val length = readU16(raf, rec + 8) - val strOffset = stringStorage + readU16(raf, rec + 10) - if (strOffset + length > fileSize || length <= 0 || length > 512) { - continue - } - val data = ByteArray(length) - raf.seek(strOffset) - raf.readFully(data) - // Platform 0 (Unicode) and 3 (Windows) store UTF-16BE; - // platform 1 (Mac) is close enough to Latin-1 for names. - var value = if (platform == 1) - String(data, java.nio.charset.StandardCharsets.ISO_8859_1) - else String(data, java.nio.charset.StandardCharsets.UTF_16BE) - value = value.trim() - if (value.isEmpty()) { - continue - } - if (nameId == 1 && family == null) family = value - if (nameId == 2 && style == null) style = value - if (nameId == 16 && preferredFamily == null) preferredFamily = value - if (nameId == 17 && preferredStyle == null) preferredStyle = value - } - val outFamily = preferredFamily ?: family - val outStyle = preferredStyle ?: style - if (outFamily == null) { - return "" - } - return outFamily + "\n" + (outStyle ?: "") - } - } - - @JvmStatic - @Throws(Exception::class) - private fun readU16(raf: RandomAccessFile, offset: Long): Int { - raf.seek(offset) - val b = ByteArray(2) - raf.readFully(b) - return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getShort().toInt() and 0xffff - } - - @JvmStatic - @Throws(Exception::class) - private fun readU32(raf: RandomAccessFile, offset: Long): Int { - raf.seek(offset) - val b = ByteArray(4) - raf.readFully(b) - return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getInt() - } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt index fdf2ce4d..f59a9d56 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FrameTimings.kt @@ -9,33 +9,13 @@ import java.io.FileOutputStream import java.io.OutputStreamWriter import java.io.Writer import java.nio.charset.StandardCharsets -import java.text.SimpleDateFormat -import java.util.ArrayDeque -import java.util.ArrayList -import java.util.Collections -import java.util.Date -import java.util.LinkedHashMap -import java.util.Locale /** - * Per-frame timing instrumentation, shared between the Java BLE layer and the - * Typescript UI layer. A "frame" starts when we receive an input event (or a - * timer fires and we decide to redraw), and finishes when the resulting screen - * update has been fully transmitted to the glasses, is discarded unsent, or - * times out because nobody reported finishing it. - * - * Frames form a tree. One input event can fan out into several renders (the - * focused app's window plus the shell chrome, say), and each of those is its - * own frame linked back to the input frame that caused it via - * [startFrame]. The export nests descendants under their - * root and renders every line against the root's clock, so a single block - * shows the whole input-to-pixels story. Latency percentiles are measured over - * roots: root start to the first descendant that actually reached the glasses. - * - * All public methods are thread-safe and cheap enough to call from the BLE - * worker thread, the Android main thread, the JS thread, and app worker - * threads. Frame IDs are positive ints; 0 means "no frame" and is silently - * ignored everywhere, so callers do not need to null-check. + * Android facade over the shared [FrameTimingsCore] (per-frame latency + * instrumentation; see that class for the frame-tree model). This class keeps + * the process-wide singleton the BLE layer and TypeScript call by name, and owns + * the Android-only parts: the export directory derived from the Context and the + * daemon thread that sweeps timed-out frames and writes the export file. * * Statistics and full log lines for recent and slowest frames are periodically * exported to getExternalFilesDir()/frame-timings.txt, retrievable via @@ -43,131 +23,19 @@ import java.util.Locale */ class FrameTimings private constructor() { companion object { - private const val TAG = "FrameTimings" + private const val TAG = FrameTimingsCore.TAG private val INSTANCE = FrameTimings() - - /** Frames still open after this long are finished as "timeout". */ - private const val FRAME_TIMEOUT_MS = 30_000L - private const val EXPORT_INTERVAL_MS = 15_000L - private const val SWEEP_INTERVAL_MS = 5_000L - private const val RECENT_ROOTS_KEPT = 40 - private const val SLOWEST_ROOTS_KEPT = 20 - private const val SENT_DURATIONS_WINDOW = 512 - private const val MAX_LINES_PER_FRAME = 200 - /** - * Frames retained for ID lookup. Well above the number any export can - * reference, so a late log/span/finish call always finds its frame; frames - * older than this are unreachable and get collected. - */ - private const val FRAMES_RETAINED = 1024 private const val EXPORT_FILE_NAME = "frame-timings.txt" @JvmStatic fun getInstance(): FrameTimings { return INSTANCE } - - @JvmStatic - private fun percentileOfSorted(sorted: List, percentile: Int): Long { - val index = Math.ceil(percentile / 100.0 * sorted.size).toInt() - 1 - return sorted[Math.max(0, Math.min(sorted.size - 1, index))] - } - - @JvmStatic - private fun repeat(unit: String, times: Int): String { - val out = StringBuilder(unit.length * times) - for (i in 0 until times) { - out.append(unit) - } - return out.toString() - } - } - - private class Line( - val atMs: Long, // SystemClock.elapsedRealtime() - val thread: String, - val message: String? - ) - - private class Frame( - val id: Int, - var reason: String, - val startedAtMs: Long, // SystemClock.elapsedRealtime() - val startedWallClockMs: Long, - val parent: Frame? - ) { - val children: MutableList = ArrayList(0) - val lines: MutableList = ArrayList() - val openSpans: MutableMap = LinkedHashMap() - var finishedAtMs: Long = 0 - var outcome: String? = null // null while open; "sent", "discarded: ...", "timeout: ..." - /** Set on the root once some frame in its subtree reached the glasses. */ - var subtreeSentRecorded = false - - fun root(): Frame { - var frame: Frame = this - while (frame.parent != null) { - frame = frame.parent!! - } - return frame - } - - fun isOpen(): Boolean { - return outcome == null - } - - fun durationMs(): Long { - return finishedAtMs - startedAtMs - } - - fun wasSent(): Boolean { - val outcome = this.outcome - return outcome != null && outcome.startsWith("sent") - } - - /** Last moment anything in this subtree happened, for whole-tree duration. */ - fun subtreeEndAtMs(nowMs: Long): Long { - var end = if (isOpen()) nowMs else finishedAtMs - for (child in children) { - end = Math.max(end, child.subtreeEndAtMs(nowMs)) - } - return end - } - - fun subtreeHasOpenFrame(): Boolean { - if (isOpen()) { - return true - } - for (child in children) { - if (child.subtreeHasOpenFrame()) { - return true - } - } - return false - } } + /** The shared core; the glasses session core records frames into it directly. */ + val core = FrameTimingsCore(AndroidProtocolPlatform) private val lock = Any() - /** - * Every frame we still retain, keyed by ID: open frames plus enough - * history that late calls and the export can still resolve one. - */ - private val framesById: LinkedHashMap = - object : LinkedHashMap(64, 0.75f, false) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry): Boolean { - return size > FRAMES_RETAINED && !eldest.value.isOpen() - } - } - private val recentRoots = ArrayDeque() - private val slowestRoots: MutableList = ArrayList() - private val inputLatencies = ArrayDeque() - private val renderLatencies = ArrayDeque() - private var nextFrameId = 1 - private var framesStarted: Long = 0 - private var framesSent: Long = 0 - private var framesDiscarded: Long = 0 - private var framesTimedOut: Long = 0 - private var exportDirty = false private var exportDir: File? = null private var exportThread: Thread? = null @@ -189,245 +57,47 @@ class FrameTimings private constructor() { } /** Begin a root frame; reason is a short label like "input:sys-event type=0". */ - fun startFrame(reason: String?): Int { - return startFrame(reason, 0) - } + fun startFrame(reason: String?): Int = core.startFrame(reason) - /** - * Begin a frame caused by an existing one (parentFrameId; 0 for a root). - * The child is nested under its root in the export and its latency counts - * against the root, so an input event that fans out into an app render and - * a shell-chrome render reads as one timeline instead of three. - */ - fun startFrame(reason: String?, parentFrameId: Int): Int { - val now = SystemClock.elapsedRealtime() - val frame: Frame - synchronized(lock) { - val parent = framesById[parentFrameId] - frame = Frame(nextFrameId++, reason ?: "", now, - System.currentTimeMillis(), parent) - framesById[frame.id] = frame - framesStarted++ - if (parent != null) { - parent.children.add(frame) - addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")") - } - } - return frame.id - } + /** Begin a frame caused by an existing one (parentFrameId; 0 for a root). */ + fun startFrame(reason: String?, parentFrameId: Int): Int = core.startFrame(reason, parentFrameId) - /** - * Append to a frame's label, for facts only known after it started (which - * app is in the foreground, which window a render is for). Shows up in the - * frame's header line, so the export is scannable without reading bodies. - */ - fun annotate(frameId: Int, text: String?) { - if (text == null || text.isEmpty()) { - return - } - synchronized(lock) { - val frame = framesById[frameId] - if (frame == null) { - return - } - frame.reason = if (frame.reason.isEmpty()) text else frame.reason + " " + text - } - } + /** Append to a frame's label, for facts only known after it started. */ + fun annotate(frameId: Int, text: String?) = core.annotate(frameId, text) - fun log(frameId: Int, message: String?) { - val now = SystemClock.elapsedRealtime() - synchronized(lock) { - val frame = openFrameLocked(frameId) - if (frame == null) { - return - } - addLineLocked(frame, now, message) - } - } + fun log(frameId: Int, message: String?) = core.log(frameId, message) - fun spanStart(frameId: Int, name: String) { - val now = SystemClock.elapsedRealtime() - synchronized(lock) { - val frame = openFrameLocked(frameId) - if (frame == null) { - return - } - frame.openSpans[name] = now - addLineLocked(frame, now, "span $name start") - } - } + fun spanStart(frameId: Int, name: String) = core.spanStart(frameId, name) - fun spanEnd(frameId: Int, name: String) { - val now = SystemClock.elapsedRealtime() - synchronized(lock) { - val frame = openFrameLocked(frameId) - if (frame == null) { - return - } - val startedAt = frame.openSpans.remove(name) - if (startedAt == null) { - addLineLocked(frame, now, "span $name end (start not recorded)") - } else { - addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)") - } - } - } + fun spanEnd(frameId: Int, name: String) = core.spanEnd(frameId, name) /** * Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything - * starting with "discarded", or "timeout". First finish wins; later calls for - * the same frame are ignored, so racing completion paths are safe. + * starting with "discarded", or "timeout". First finish wins. */ - fun finishFrame(frameId: Int, outcome: String?) { - val now = SystemClock.elapsedRealtime() - val finished: Frame - synchronized(lock) { - val frame = openFrameLocked(frameId) - if (frame == null) { - return - } - finishFrameLocked(frame, now, outcome ?: "discarded: no outcome given") - finished = frame - } - Log.i(TAG, "frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome - + " in " + finished.durationMs() + "ms") - } + fun finishFrame(frameId: Int, outcome: String?) = core.finishFrame(frameId, outcome) /** One-line stats summary, e.g. for showing in the phone UI. */ - fun statsSummary(): String { - synchronized(lock) { - val input = percentilesLocked(inputLatencies) - return "frames started=" + framesStarted + " sent=" + framesSent + - " discarded=" + framesDiscarded + " timeout=" + framesTimedOut + - (if (input == null) - "" - else " | input-to-display p50=" + input[0] + "ms p90=" + input[1] + - "ms p99=" + input[2] + "ms max=" + input[3] + "ms") - } - } + fun statsSummary(): String = core.statsSummary() // --------------------------------------------------------------------- - /** The frame with this ID if it exists and is still open, else null. */ - private fun openFrameLocked(frameId: Int): Frame? { - if (frameId <= 0) { - return null - } - val frame = framesById[frameId] - return if (frame != null && frame.isOpen()) frame else null - } - - private fun addLineLocked(frame: Frame, now: Long, message: String?) { - if (frame.lines.size >= MAX_LINES_PER_FRAME) { - if (frame.lines.size == MAX_LINES_PER_FRAME) { - frame.lines.add(Line(now, Thread.currentThread().name, "... line cap reached")) - } - return - } - frame.lines.add(Line(now, Thread.currentThread().name, message)) - } - - private fun finishFrameLocked(frame: Frame, now: Long, outcome: String) { - frame.finishedAtMs = now - frame.outcome = outcome - for (open in frame.openSpans.entries) { - frame.lines.add(Line(now, Thread.currentThread().name, - "span " + open.key + " never ended (started at +" + - (open.value - frame.startedAtMs) + "ms)")) - } - frame.openSpans.clear() - addLineLocked(frame, now, "finished: $outcome") - - val root = frame.root() - if (frame.wasSent()) { - framesSent++ - // The user-visible latency is input (or timer) to the first pixels - // that actually reached the glasses, wherever in the tree that was. - if (!root.subtreeSentRecorded) { - root.subtreeSentRecorded = true - recordLatencyLocked(root, now - root.startedAtMs) - insertSlowestLocked(root) - } - } else if (outcome.startsWith("timeout")) { - framesTimedOut++ - } else { - framesDiscarded++ - } - - if (root === frame) { - recentRoots.addLast(frame) - while (recentRoots.size > RECENT_ROOTS_KEPT) { - recentRoots.removeFirst() - } - } - exportDirty = true - } - - private fun recordLatencyLocked(root: Frame, latencyMs: Long) { - val bucket = if (root.reason.startsWith("input:")) inputLatencies else renderLatencies - bucket.addLast(latencyMs) - while (bucket.size > SENT_DURATIONS_WINDOW) { - bucket.removeFirst() - } - } - - private fun insertSlowestLocked(root: Frame) { - if (!slowestRoots.contains(root)) { - slowestRoots.add(root) - } - // Sorted and trimmed at export time, when every subtree duration is final. - } - - /** {p50, p90, p99, max} over a rolling latency window, or null if empty. */ - private fun percentilesLocked(window: ArrayDeque): LongArray? { - if (window.isEmpty()) { - return null - } - val sorted: MutableList = ArrayList(window) - Collections.sort(sorted) - return longArrayOf( - percentileOfSorted(sorted, 50), - percentileOfSorted(sorted, 90), - percentileOfSorted(sorted, 99), - sorted[sorted.size - 1], - ) - } - - private fun sweepTimedOutFrames() { - val now = SystemClock.elapsedRealtime() - val timedOut: MutableList = ArrayList() - synchronized(lock) { - for (frame in ArrayList(framesById.values)) { - if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) { - finishFrameLocked(frame, now, "timeout: never reported finished") - timedOut.add(frame) - } - } - } - for (frame in timedOut) { - Log.w(TAG, "frame#" + frame.id + " [" + frame.reason + "] timed out after " + - frame.durationMs() + "ms without being finished") - } - } - private fun exportLoop() { var lastExportAtMs = 0L while (true) { try { - Thread.sleep(SWEEP_INTERVAL_MS) + Thread.sleep(FrameTimingsCore.SWEEP_INTERVAL_MS) } catch (e: InterruptedException) { return } try { - sweepTimedOutFrames() + core.sweepTimedOut() val now = SystemClock.elapsedRealtime() - val shouldExport: Boolean - synchronized(lock) { - shouldExport = exportDirty && exportDir != null && now - lastExportAtMs >= EXPORT_INTERVAL_MS - } - if (shouldExport) { + val dir: File? + synchronized(lock) { dir = exportDir } + if (dir != null && core.isExportDirty && now - lastExportAtMs >= FrameTimingsCore.EXPORT_INTERVAL_MS) { lastExportAtMs = now - exportToFile() + exportToFile(dir) } } catch (t: Throwable) { Log.w(TAG, "export loop error", t) @@ -435,14 +105,8 @@ class FrameTimings private constructor() { } } - private fun exportToFile() { - val content: String - val dir: File - synchronized(lock) { - dir = exportDir ?: return - content = buildExportLocked() - exportDirty = false - } + private fun exportToFile(dir: File) { + val content = core.takeExport() ?: return val target = File(dir, EXPORT_FILE_NAME) val temp = File(dir, "$EXPORT_FILE_NAME.tmp") try { @@ -456,99 +120,4 @@ class FrameTimings private constructor() { Log.w(TAG, "failed to export frame timings", t) } } - - private fun buildExportLocked(): String { - val now = SystemClock.elapsedRealtime() - val wallClockFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) - val out = StringBuilder(64 * 1024) - out.append("FrameTimings export at ").append(wallClockFormat.format(Date())).append('\n') - out.append(statsSummaryLocked()).append('\n') - out.append('\n') - out.append("Frames are trees: an input event's own frame is the root, and the renders it\n") - out.append("caused are indented under it with offsets measured from the root's start.\n") - out.append('\n') - - Collections.sort(slowestRoots, Comparator { a, b -> - java.lang.Long.compare(b.subtreeEndAtMs(now) - b.startedAtMs, a.subtreeEndAtMs(now) - a.startedAtMs) }) - while (slowestRoots.size > SLOWEST_ROOTS_KEPT) { - slowestRoots.removeAt(slowestRoots.size - 1) - } - - out.append("=== slowest frames that reached the glasses ===\n") - for (frame in slowestRoots) { - appendTreeLocked(out, frame, frame, wallClockFormat, now) - } - - out.append("=== recent frames (oldest first) ===\n") - for (frame in recentRoots) { - appendTreeLocked(out, frame, frame, wallClockFormat, now) - } - return out.toString() - } - - private fun statsSummaryLocked(): String { - val input = percentilesLocked(inputLatencies) - val render = percentilesLocked(renderLatencies) - val out = StringBuilder() - out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent) - .append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut) - .append(" open=").append(openFrameCountLocked()) - appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size, input) - appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size, render) - return out.toString() - } - - private fun appendPercentilesLocked(out: StringBuilder, label: String, count: Int, percentiles: LongArray?) { - if (percentiles == null) { - return - } - out.append('\n').append(label).append(" (last ").append(count).append("): p50=") - .append(percentiles[0]).append("ms p90=").append(percentiles[1]) - .append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms") - } - - private fun openFrameCountLocked(): Int { - var open = 0 - for (frame in framesById.values) { - if (frame.isOpen()) { - open++ - } - } - return open - } - - /** Print a frame and its descendants, all timed against root's start. */ - private fun appendTreeLocked( - out: StringBuilder, frame: Frame, root: Frame, wallClockFormat: SimpleDateFormat, now: Long) { - var depth = 0 - var walk: Frame? = frame - while (walk !== root) { - depth++ - walk = walk!!.parent - } - val indent = repeat(" ", depth) - val startOffsetMs = frame.startedAtMs - root.startedAtMs - out.append(indent).append("frame#").append(frame.id) - .append(" [").append(frame.reason).append(']') - if (frame === root) { - out.append(" started ").append(wallClockFormat.format(Date(frame.startedWallClockMs))) - val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs - out.append(" duration ").append(frame.durationMs()).append("ms") - if (!frame.children.isEmpty()) { - out.append(" (tree ").append(totalMs).append("ms") - .append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')') - } - } else { - out.append(" started +").append(startOffsetMs).append("ms") - .append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms")) - } - out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n') - for (line in frame.lines) { - out.append(indent).append(String.format(Locale.US, " %+7dms [%s] %s", - line.atMs - root.startedAtMs, line.thread, line.message)).append('\n') - } - for (child in frame.children) { - appendTreeLocked(out, child, root, wallClockFormat, now) - } - } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt index da181f0d..efaf4ac4 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/IconRenderer.kt @@ -8,9 +8,6 @@ import android.util.Log import androidx.core.graphics.PathParser -import java.util.regex.Matcher -import java.util.regex.Pattern - /** * Renders a small subset of SVG (the shapes used by Lucide / simple Noun * Project icons: path, circle, ellipse, rect, line, polyline, polygon) to a @@ -18,13 +15,14 @@ import java.util.regex.Pattern * (Lucide style: fill="none", 2px stroke, round caps/joins); the returned * bytes are one coverage value per pixel (0=transparent .. 255=opaque white), * row-major, size*size. Called once per icon and cached on the TS side. + * + * Parsing, placement math and the coverage quantization are shared with iOS + * (SvgIconSource); this class only maps shapes onto android.graphics.Path and + * rasterizes them. */ class IconRenderer private constructor() { companion object { private const val TAG = "IconRenderer" - private val TAG_RE: Pattern = Pattern.compile("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>") - private val VIEWBOX_RE: Pattern = Pattern.compile("viewBox\\s*=\\s*\"([^\"]*)\"") - private val SEPARATOR_RE: Pattern = Pattern.compile("[\\s,]+") @JvmStatic fun renderSvgGray(svg: String?, size: Int, strokeWidth: Float): ByteArray { @@ -32,27 +30,10 @@ class IconRenderer private constructor() { return ByteArray(0) } try { - var minX = 0f - var minY = 0f - var viewW = 24f - var viewH = 24f - val vb = VIEWBOX_RE.matcher(svg) - if (vb.find()) { - val parts = SEPARATOR_RE.split(vb.group(1)!!.trim()) - if (parts.size == 4) { - minX = java.lang.Float.parseFloat(parts[0]) - minY = java.lang.Float.parseFloat(parts[1]) - viewW = java.lang.Float.parseFloat(parts[2]) - viewH = java.lang.Float.parseFloat(parts[3]) - } - } - // Lucide-style icons are stroked outlines (fill="none"); everything - // else (Noun Project glyphs, brand logos) is a filled shape. - val stroked = svg.contains("fill=\"none\"") + val icon = SvgIconSource.parse(svg) ?: return ByteArray(0) val path = Path() - val m = TAG_RE.matcher(svg) - while (m.find()) { - appendElement(path, m.group(1)!!, m.group(2)!!) + for (shape in icon.shapes) { + appendShape(path, shape) } if (path.isEmpty) { return ByteArray(0) @@ -60,14 +41,14 @@ class IconRenderer private constructor() { val bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888) val canvas = Canvas(bitmap) - val scale = Math.min(size / viewW, size / viewH) // Center the (square) viewBox in the bitmap, then map its origin. - canvas.translate((size - viewW * scale) / 2f, (size - viewH * scale) / 2f) - canvas.scale(scale, scale) - canvas.translate(-minX, -minY) + val placement = icon.placement(size) + canvas.translate(placement[0], placement[1]) + canvas.scale(placement[2], placement[2]) + canvas.translate(-icon.minX, -icon.minY) val paint = Paint(Paint.ANTI_ALIAS_FLAG) paint.color = 0xffffffff.toInt() - if (stroked) { + if (icon.stroked) { paint.style = Paint.Style.STROKE paint.strokeWidth = strokeWidth paint.strokeCap = Paint.Cap.ROUND @@ -79,117 +60,47 @@ class IconRenderer private constructor() { val pixels = IntArray(size * size) bitmap.getPixels(pixels, 0, size, 0, 0, size, size) - val gray = ByteArray(size * size) - for (i in pixels.indices) { - gray[i] = quantizeCoverage((pixels[i] ushr 24) and 0xff).toByte() // alpha = coverage - } - return gray + return SvgIconSource.quantizeArgb(pixels) // alpha = coverage } catch (e: Exception) { Log.w(TAG, "icon render failed", e) return ByteArray(0) } } - /** - * Snap antialiased coverage toward hard edges so the 4bpp frames sent - * over BLE deflate better: mostly-transparent and mostly-opaque pixels - * become exactly 0/255 (long runs, and 0 stays color-key transparent for - * bitBlt), and only genuinely half-covered edge pixels keep one mid gray. - * The packed frame then uses a 3-symbol alphabet instead of all 16 - * levels, at icon sizes this is visually near-identical. - */ @JvmStatic - private fun quantizeCoverage(alpha: Int): Int { - if (alpha < 64) return 0 - if (alpha >= 192) return 255 - return 128 - } - - @JvmStatic - private fun appendElement(path: Path, tag: String, attrs: String) { - when (tag) { - "path" -> { - val d = attr(attrs, "d") - if (d != null) { - val sub = PathParser.createPathFromPathData(d) - if (sub != null) { - path.addPath(sub) - } + private fun appendShape(path: Path, shape: SvgShape) { + when (shape) { + is SvgShape.PathData -> { + val sub = PathParser.createPathFromPathData(shape.d) + if (sub != null) { + path.addPath(sub) } } - "circle" -> { - val cx = num(attrs, "cx", 0f) - val cy = num(attrs, "cy", 0f) - val r = num(attrs, "r", 0f) - if (r > 0) path.addCircle(cx, cy, r, Path.Direction.CW) - } - "ellipse" -> { - val cx = num(attrs, "cx", 0f) - val cy = num(attrs, "cy", 0f) - val rx = num(attrs, "rx", 0f) - val ry = num(attrs, "ry", 0f) - if (rx > 0 && ry > 0) path.addOval(cx - rx, cy - ry, cx + rx, cy + ry, Path.Direction.CW) - } - "rect" -> { - val x = num(attrs, "x", 0f) - val y = num(attrs, "y", 0f) - val w = num(attrs, "width", 0f) - val h = num(attrs, "height", 0f) - var rx = num(attrs, "rx", 0f) - var ry = num(attrs, "ry", if (rx > 0) rx else 0f) - if (rx <= 0) rx = ry - if (ry <= 0) ry = rx - if (w > 0 && h > 0) { - if (rx > 0 || ry > 0) { - path.addRoundRect(x, y, x + w, y + h, rx, ry, Path.Direction.CW) - } else { - path.addRect(x, y, x + w, y + h, Path.Direction.CW) - } + is SvgShape.Circle -> path.addCircle(shape.cx, shape.cy, shape.r, Path.Direction.CW) + is SvgShape.Ellipse -> path.addOval( + shape.cx - shape.rx, shape.cy - shape.ry, shape.cx + shape.rx, shape.cy + shape.ry, Path.Direction.CW) + is SvgShape.Rect -> { + if (shape.rx > 0 || shape.ry > 0) { + path.addRoundRect(shape.x, shape.y, shape.x + shape.width, shape.y + shape.height, + shape.rx, shape.ry, Path.Direction.CW) + } else { + path.addRect(shape.x, shape.y, shape.x + shape.width, shape.y + shape.height, Path.Direction.CW) } } - "line" -> { - path.moveTo(num(attrs, "x1", 0f), num(attrs, "y1", 0f)) - path.lineTo(num(attrs, "x2", 0f), num(attrs, "y2", 0f)) + is SvgShape.Line -> { + path.moveTo(shape.x1, shape.y1) + path.lineTo(shape.x2, shape.y2) } - "polyline", "polygon" -> { - val points = attr(attrs, "points") - if (points != null) { - val nums = SEPARATOR_RE.split(points.trim()) - var started = false - var i = 0 - while (i + 1 < nums.size) { - val px = java.lang.Float.parseFloat(nums[i]) - val py = java.lang.Float.parseFloat(nums[i + 1]) - if (!started) { - path.moveTo(px, py) - started = true - } else { - path.lineTo(px, py) - } - i += 2 - } - if (tag == "polygon") path.close() + is SvgShape.Polyline -> { + val p = shape.points + path.moveTo(p[0], p[1]) + var i = 2 + while (i + 1 < p.size) { + path.lineTo(p[i], p[i + 1]) + i += 2 } + if (shape.closed) path.close() } - else -> { - } - } - } - - @JvmStatic - private fun attr(attrs: String, name: String): String? { - val m: Matcher = Pattern.compile("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").matcher(attrs) - return if (m.find()) m.group(1) else null - } - - @JvmStatic - private fun num(attrs: String, name: String, fallback: Float): Float { - val value = attr(attrs, name) - if (value == null) return fallback - try { - return java.lang.Float.parseFloat(value.trim()) - } catch (e: NumberFormatException) { - return fallback } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt index d1b6c300..b73baee1 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt @@ -10,7 +10,8 @@ import android.util.Log * pixels...] with one byte per pixel, row-major, or an empty array on * failure. FaceclawMediaController's album art shares bitmapToGrayPacket; * the TS side (app/native/image-files.ts) owns the shared photo-tone preset - * passed as (gamma, dither). + * passed as (gamma, dither). The pixel math (tone curve, alpha darkening, + * dither) lives in the shared GrayPacket so iOS produces identical bytes. */ class ImageFileLoader private constructor() { companion object { @@ -151,24 +152,7 @@ class ImageFileLoader private constructor() { } val pixels = IntArray(width * height) scaled.getPixels(pixels, 0, width, 0, 0, width, height) - val out = ByteArray(4 + width * height) - out[0] = (width and 0xff).toByte() - out[1] = ((width shr 8) and 0xff).toByte() - out[2] = (height and 0xff).toByte() - out[3] = ((height shr 8) and 0xff).toByte() - val tone = toneCurve(gamma) - for (i in pixels.indices) { - val p = pixels[i] - val a = (p ushr 24) and 0xff - val r = (p shr 16) and 0xff - val g = (p shr 8) and 0xff - val b = p and 0xff - out[4 + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte() - } - if (dither) { - ditherToDisplayLevels(out, 4, width, height) - } - return out + return GrayPacket.fromArgb(pixels, width, height, gamma, dither) } catch (e: Exception) { Log.w(TAG, "bitmap conversion failed", e) return ByteArray(0) @@ -177,70 +161,5 @@ class ImageFileLoader private constructor() { return ByteArray(0) } } - - /** 256-entry map of source gray to displayed gray for out = in^gamma. */ - @JvmStatic - private fun toneCurve(gamma: Float): IntArray { - val curve = IntArray(256) - for (v in 0 until 256) { - curve[v] = if (gamma == 1f) - v - else Math.round(255f * Math.pow((v / 255f).toDouble(), gamma.toDouble()).toFloat()) - } - return curve - } - - /** - * Floyd-Steinberg error diffusion onto the display's 16 gray levels, in - * place over an 8bpp plane. Serpentine scanning keeps the diffusion from - * building up a directional texture across wide flat areas. - * - * Level n is written back as n * 16 rather than the n * 17 it stands for - * so that BmpUtil's (v + 8) >> 4 reproduces n exactly; level 0 is written - * as 1, not 0, because 0 is the shell's color-key for transparent. - */ - @JvmStatic - private fun ditherToDisplayLevels(plane: ByteArray, offset: Int, width: Int, height: Int) { - var curr = FloatArray(width) - var next = FloatArray(width) - for (x in 0 until width) { - curr[x] = (plane[offset + x].toInt() and 0xff).toFloat() - } - for (y in 0 until height) { - val rowStart = offset + y * width - val nextStart = rowStart + width - val hasNext = y + 1 < height - for (x in 0 until width) { - next[x] = if (hasNext) (plane[nextStart + x].toInt() and 0xff).toFloat() else 0f - } - val leftToRight = (y and 1) == 0 - val start = if (leftToRight) 0 else width - 1 - val step = if (leftToRight) 1 else -1 - for (i in 0 until width) { - val x = start + i * step - val wanted = curr[x] - val level = Math.round(Math.min(255f, Math.max(0f, wanted)) / 17f) - val error = wanted - level * 17f - plane[rowStart + x] = (if (level == 0) 1 else level * 16).toByte() - val ahead = x + step - if (ahead >= 0 && ahead < width) { - curr[ahead] += error * (7f / 16f) - } - if (hasNext) { - if (ahead >= 0 && ahead < width) { - next[ahead] += error * (1f / 16f) - } - next[x] += error * (5f / 16f) - val behind = x - step - if (behind >= 0 && behind < width) { - next[behind] += error * (3f / 16f) - } - } - } - val swap = curr - curr = next - next = swap - } - } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt index 96f98bda..54f31c42 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt @@ -9,34 +9,11 @@ import java.nio.ByteBuffer * grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS * ArrayBuffer to one without copying element-by-element); doing the * gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the - * JS/Java bridge, where it used to cost ~150ms per preview. + * JS/Java bridge, where it used to cost ~150ms per preview. The palette + * itself is the shared PreviewPalette. */ class PreviewBitmapUtil private constructor() { companion object { - private var cachedLut: IntArray? = null - private var cachedGammaBits: Long = 0 - private var cachedGreen: Boolean = false - - /** Reuse the current preview palette; rebuild only when gamma or color changes. */ - @JvmStatic - @Synchronized - private fun lookupTable(brightenGamma: Double, green: Boolean): IntArray { - val gammaBits = java.lang.Double.doubleToLongBits(brightenGamma) - val cached = cachedLut - if (cached != null && cachedGammaBits == gammaBits && cachedGreen == green) { - return cached - } - val lut = IntArray(256) - for (g in 0 until 256) { - val v = Math.max(0L, Math.min(255L, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))).toInt() - lut[g] = if (green) (0xff000000.toInt() or (v shl 8)) else (0xff000000.toInt() or (v shl 16) or (v shl 8) or v) - } - cachedGammaBits = gammaBits - cachedGreen = green - cachedLut = lut - return lut - } - @JvmStatic fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double): Bitmap { return fromGray(gray, width, height, brightenGamma, false) @@ -48,13 +25,9 @@ class PreviewBitmapUtil private constructor() { if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { throw IllegalArgumentException("invalid gray preview buffer") } - val lut = lookupTable(brightenGamma, green) val bytes = ByteArray(width * height) gray.get(bytes) - val colors = IntArray(width * height) - for (i in colors.indices) { - colors[i] = lut[bytes[i].toInt() and 0xff] - } + val colors = PreviewPalette.expand(bytes, 0, width * height, brightenGamma, green) return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt deleted file mode 100644 index 60153482..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt +++ /dev/null @@ -1,48 +0,0 @@ -package com.faceclaw.app - -import android.os.SystemClock - -class InterruptibleSleep { - private val lock = java.lang.Object() - private var interrupted = false - - fun sleep(ms: Long): Boolean { - val delayMs = Math.max(1L, ms) - val deadline = SystemClock.elapsedRealtime() + delayMs - synchronized(lock) { - // Consume (do not clear-and-ignore) an interrupt delivered before this - // sleep began. interrupt() runs on other threads with no lock held by - // the sleeper between driveSession() returning and sleep() starting, so - // clearing the flag at entry lost that wake and forced the worker to - // sleep the full interval — up to 250ms per idle frame. Honoring it - // costs at most one extra (cheap) driveSession pass. - if (interrupted) { - interrupted = false - return false - } - while (true) { - val remaining = deadline - SystemClock.elapsedRealtime() - if (remaining <= 0) { - return true - } - try { - lock.wait(remaining) - } catch (ignored: InterruptedException) { - interrupted = false - return false - } - if (interrupted) { - interrupted = false - return false - } - } - } - } - - fun interrupt() { - synchronized(lock) { - interrupted = true - lock.notifyAll() - } - } -} diff --git a/native/kotlin/README.md b/native/kotlin/README.md index e2939a80..40190770 100644 --- a/native/kotlin/README.md +++ b/native/kotlin/README.md @@ -8,7 +8,21 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore - `commonMain/kotlin`: wire protocol, fragmentation/reassembly, event decoding, image/RLE/texture planning, retained surface composition, glyph/image atlases, LVGL font parsing, spectral noise suppression, PNG/GIF/WAV encodings and - callback interfaces. + callback interfaces. Since 2026-09-22 also the platform-neutral cores behind + the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency + primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`, + `audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader), + `graphics/` (GrayPacket, PreviewPalette, StatusIconArt, GlyphPacket, + OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream, + ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`, + `alarms/` (AlarmSchedule, AlarmRingingPolicy), `settings/SettingsChangeHub`, + the stock-firmware flows over a `StockLink` GATT port (`g2protocol/StockLinkSession`, + `DeviceInfoProbeFlow`, `FlashPromptFlow`, `OtaFlashFlow`), the full glasses session + (`g2protocol/session/GlassesSessionCore` over `SessionLink`/`SessionHost`) and the + voice/caption pipelines (`audio/VoiceCaptureSession`, `CaptionEngineCore`). + Cores take port interfaces (GATT link, storage, transport, dispatcher) and call + listeners synchronously; the Android adapters in `App_Resources` own threads, + Handler marshalling and OS APIs. See `notes/stage2-shared-kotlin-plan.md`. - `androidMain/kotlin`: monotonic clock, reentrant locks, JVM zlib and file I/O, zero-copy `ByteBuffer` readers, firmware hashing/file writes and WAV file storage. - `iosMain/kotlin`: monotonic clock, recursive locks, native zlib and POSIX file @@ -22,7 +36,9 @@ bulk buffer crossing without putting Java APIs in common code. iOS uses ## Platform boundaries -`ProtocolPlatform` supplies clocks, locks and persistent deflaters. `expect` / +`ProtocolPlatform` supplies clocks, locks, persistent deflaters and condition +variables (`ProtocolCondition`, NOT reentrant on iOS: NSCondition). `InterruptibleSleep`, +`Latch` and `BlockingQueue` in commonMain build on it. `expect` / `actual` functions select default platform services at compilation. Stateful transports and caches retain reentrant locking; callbacks still run on their caller's thread. The migration does not introduce implicit UI-thread dispatch. diff --git a/native/kotlin/migrated-java-sources.json b/native/kotlin/migrated-java-sources.json index dd32ffa3..6f65dba4 100644 --- a/native/kotlin/migrated-java-sources.json +++ b/native/kotlin/migrated-java-sources.json @@ -409,6 +409,6 @@ }, { "java": "com/faceclaw/app/util/InterruptibleSleep.java", - "kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt" + "kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Concurrency.kt" } ] diff --git a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt new file mode 100644 index 00000000..fa5bb6a0 --- /dev/null +++ b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/PlatformUtil.android.kt @@ -0,0 +1,57 @@ +package com.faceclaw.app + +import java.security.MessageDigest +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +actual class Sha256Digest actual constructor() { + private val digest = MessageDigest.getInstance("SHA-256") + + actual fun update(bytes: ByteArray, offset: Int, length: Int) = digest.update(bytes, offset, length) + + actual fun digest(): ByteArray = digest.digest() +} + +actual fun currentTimeMillis(): Long = System.currentTimeMillis() + +actual fun formatLocalTime(epochMs: Long, pattern: String): String = + SimpleDateFormat(pattern, Locale.US).format(Date(epochMs)) + +actual object PlatformLog { + actual fun i(tag: String, message: String) { + android.util.Log.i(tag, message) + } + + actual fun w(tag: String, message: String) { + android.util.Log.w(tag, message) + } + + actual fun e(tag: String, message: String, error: Throwable?) { + if (error != null) android.util.Log.e(tag, message, error) else android.util.Log.e(tag, message) + } +} + +actual fun startThread(name: String, daemon: Boolean, body: () -> Unit) { + val thread = Thread(body, name) + thread.isDaemon = daemon + thread.start() +} + +actual fun sleepMs(ms: Long) { + try { + Thread.sleep(ms) + } catch (ignored: InterruptedException) { + Thread.currentThread().interrupt() + } +} + +actual fun writeFileData(path: String, bytes: ByteArray) = java.io.File(path).writeBytes(bytes) + +actual fun fileExists(path: String): Boolean = java.io.File(path).exists() + +actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exists()) it.length() else -1L } + +actual fun deleteFile(path: String): Boolean = java.io.File(path).delete() + +actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to)) diff --git a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/g2protocol/AndroidProtocolPlatform.kt b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/g2protocol/AndroidProtocolPlatform.kt index b5667e1d..96c8a4a1 100644 --- a/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/g2protocol/AndroidProtocolPlatform.kt +++ b/native/kotlin/shared/src/androidMain/kotlin/com/faceclaw/app/g2protocol/AndroidProtocolPlatform.kt @@ -20,6 +20,22 @@ object AndroidProtocolPlatform : ProtocolPlatform { override fun createDeflater(): ProtocolDeflater = AndroidDeflater() + override fun createCondition(): ProtocolCondition = + object : ProtocolCondition { + private val lock = ReentrantLock() + private val condition = lock.newCondition() + + override fun lock() = lock.lock() + + override fun unlock() = lock.unlock() + + override fun awaitMs(timeoutMs: Long) { + if (timeoutMs > 0) condition.await(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + } + + override fun signalAll() = condition.signalAll() + } + @JvmStatic fun writeType(mode: GattWriteMode): Int = when (mode) { diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt index 9d0f0dca..739c23e9 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/PlatformServices.kt @@ -8,3 +8,15 @@ internal expect fun deflateData(bytes: ByteArray): ByteArray internal expect fun inflateData(bytes: ByteArray, size: Int): ByteArray internal expect fun readFileData(path: String): ByteArray + +/** Whole-file write (truncating); parent directories must exist. */ +expect fun writeFileData(path: String, bytes: ByteArray) + +expect fun fileExists(path: String): Boolean + +/** Size in bytes, or -1 when the file does not exist. */ +expect fun fileLength(path: String): Long + +expect fun deleteFile(path: String): Boolean + +expect fun renameFile(from: String, to: String): Boolean diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/SettingsDocument.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/SettingsDocument.kt index baf4af2c..ed240a09 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/SettingsDocument.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/SettingsDocument.kt @@ -5,7 +5,7 @@ class SettingsDocument private constructor(private val settings: Map value - is Map<*, *>, is List<*> -> SettingsJson.write(value) + is Map<*, *>, is List<*> -> Json.write(value) else -> fallback } fun getBoolean(key: String, fallback: Boolean): Boolean = settings[key] as? Boolean ?: fallback @@ -17,7 +17,7 @@ class SettingsDocument private constructor(private val settings: Map || parsed is List<*>) parsed else value } // Each entry migrates version N to N+1. Never rewrite unknown future schemas. @@ -45,7 +45,7 @@ class SettingsDocument private constructor(private val settings: Map ?: error("Expected config object") + val root = Json.parse(text) as? Map<*, *> ?: error("Expected config object") var schema = (root["schema"] as? JsonNumber)?.text?.toIntOrNull() ?: error("Missing schema") require(schema in 1..CURRENT_SCHEMA) { "Unsupported settings schema" } val raw = root["settings"] as? Map<*, *> ?: error("Missing settings") @@ -64,103 +64,7 @@ class SettingsDocument private constructor(private val settings: Map)["schema"] as JsonNumber).text != SettingsDocument.CURRENT_SCHEMA.toString() + ((Json.parse(text) as Map<*, *>)["schema"] as JsonNumber).text != SettingsDocument.CURRENT_SCHEMA.toString() }.getOrDefault(false) fun decode(text: String): SettingsDocument? = runCatching { SettingsDocument.decode(text) }.getOrNull() } - -private data class JsonNumber(val text: String) - -/** Strict JSON plus // and block comments and trailing commas, on both platforms. */ -private object SettingsJson { - private val numberPattern = Regex("-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?") - fun parse(text: String): Any? = Parser(text).parse() - fun write(value: Any?, pretty: Boolean = false, depth: Int = 0): String { - fun quoted(text: String): String = buildString { - append('"') - for (c in text) when (c) { - '"' -> append("\\\""); '\\' -> append("\\\\"); '\n' -> append("\\n"); '\r' -> append("\\r"); '\t' -> append("\\t") - else -> if (c < ' ') append("\\u" + c.code.toString(16).padStart(4, '0')) else append(c) - } - append('"') - } - fun collection(open: String, close: String, entries: List): String { - if (entries.isEmpty()) return open + close - return if (pretty) open + "\n" + entries.joinToString(",\n") { " ".repeat(depth + 1) + it } + "\n" + " ".repeat(depth) + close - else open + entries.joinToString(",") + close - } - return when (value) { - null -> "null" - is String -> quoted(value) - is Boolean -> value.toString() - is JsonNumber -> value.text - is List<*> -> collection("[", "]", value.map { write(it, pretty, depth + 1) }) - is Map<*, *> -> collection("{", "}", value.entries.map { quoted(it.key as String) + (if (pretty) ": " else ":") + write(it.value, pretty, depth + 1) }) - else -> error("Unsupported JSON value") - } - } - private class Parser(val text: String) { - var i = 0 - fun parse(): Any? { val result = value(0); space(); require(i == text.length); return result } - fun space() { - while (i < text.length) { - if (text[i] in " \r\n\t\uFEFF") { i++; continue } - if (text.startsWith("//", i)) { while (i < text.length && text[i] != '\n' && text[i] != '\r') i++; continue } - if (text.startsWith("/*", i)) { val end = text.indexOf("*/", i + 2); require(end >= 0); i = end + 2; continue } - break - } - } - fun take(c: Char): Boolean { space(); return if (i < text.length && text[i] == c) { i++; true } else false } - fun value(depth: Int): Any? { - require(depth < 64); space(); require(i < text.length) - return when (text[i]) { - '"' -> string() - '{' -> { - i++; val result = linkedMapOf() - if (!take('}')) while (true) { - space(); val key = string(); require(!result.containsKey(key)); require(take(':')) - result[key] = value(depth + 1) - if (take('}')) break - require(take(',')); if (take('}')) break - } - result - } - '[' -> { - i++; val result = mutableListOf() - if (!take(']')) while (true) { - result.add(value(depth + 1)); if (take(']')) break - require(take(',')); if (take(']')) break - } - result - } - 't' -> literal("true", true) - 'f' -> literal("false", false) - 'n' -> literal("null", null) - else -> { - val match = numberPattern.find(text, i) - require(match != null && match.range.first == i) - require(match.value.toDouble().isFinite()); i += match.value.length; JsonNumber(match.value) - } - } - } - fun literal(word: String, result: Any?): Any? { require(text.startsWith(word, i)); i += word.length; return result } - fun string(): String { - require(i < text.length && text[i++] == '"') - return buildString { - while (true) { - require(i < text.length); val c = text[i++] - if (c == '"') break - require(c >= ' ') - if (c != '\\') { append(c); continue } - require(i < text.length) - append(when (val escaped = text[i++]) { - '"', '\\', '/' -> escaped - 'b' -> '\b'; 'f' -> '\u000c'; 'n' -> '\n'; 'r' -> '\r'; 't' -> '\t' - 'u' -> { require(i + 4 <= text.length); val hex = text.substring(i, i + 4); require(hex.all { it in "0123456789abcdefABCDEF" }); val code = hex.toInt(16); i += 4; code.toChar() } - else -> error("Invalid escape") - }) - } - } - } - } -} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/alarms/AlarmRingingPolicy.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/alarms/AlarmRingingPolicy.kt new file mode 100644 index 00000000..26eb92f2 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/alarms/AlarmRingingPolicy.kt @@ -0,0 +1,275 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField + +/** One item currently ringing. Subclassed by the Android service for its own bookkeeping. */ +open class RingingItem( + @JvmField val id: Long, + title: String?, + text: String?, + kind: String?, + snoozeMinutes: Int, + @JvmField val ringAtMs: Long, +) { + @JvmField val title: String = if (title == null || title.trim().isEmpty()) "Alarm" else title + @JvmField val text: String = text ?: "" + @JvmField val kind: String = kind ?: AlarmSchedule.KIND_ALARM + @JvmField val snoozeMinutes: Int = maxOf(1, snoozeMinutes) + @JvmField var deliveredAtMs: Long = 0 + @JvmField var escalated: Boolean = false + @JvmField var silenced: Boolean = false +} + +/** Delayed callbacks for the escalation timers; the owner's main thread on Android. */ +interface AlarmScheduler { + fun postDelayed(token: Any, delayMs: Long, action: () -> Unit) + + fun cancel(token: Any) + + fun cancelAll() +} + +/** Platform effects of ringing: notifications, sound, the schedule, and the journal. */ +interface RingingSink { + /** Show or refresh the item's notification (title/status wording comes from the item). */ + fun showNotification(item: RingingItem) + + /** The item is gone: drop its notification and any per-item resources. */ + fun itemRemoved(item: RingingItem) + + /** Something started ringing: keep the device awake for the escalation window. */ + fun ringingStarted() + + fun startSound() + + fun stopSound() + + /** Re-arm the item (a snooze) at [atMs]. */ + fun reschedule(item: RingingItem, atMs: Long, minutes: Int) + + /** Forget the item's schedule entry (a dismiss). */ + fun cancelSchedule(id: Long) + + fun recordPhoneAction(id: Long, action: String, minutes: Int) + + fun log(line: String) +} + +/** + * Ringing escalation. Each item starts silent on the phone (a notification only, the glasses do + * the ringing) and escalates to phone sound when the glasses cannot carry it: not connected, + * not on a head, charging, no delivery confirmation from the JS side within [DELIVERY_WAIT_MS], + * or no acknowledgement within [ACK_WAIT_MS] of delivery. Sound stops after [AUTO_SILENCE_MS] + * while the notification stays. The policy is process-wide; the platform host (a service on + * Android) attaches its scheduler and sink while it is alive. + */ +class AlarmRingingPolicy( + private val gate: GlassesStatusGate, + private val clock: () -> Long = ::currentTimeMillis, + platform: ProtocolPlatform = protocolPlatform(), +) { + private val lock = platform.createLock() + /** Ringing items, oldest first. */ + private val ringing = LinkedHashMap() + /** + * Signals that arrived before the ring request was processed (the JS engine fires, launches + * its window and reports delivery on the same thread the host queues behind), keyed by id + * with their arrival time. Consumed by [ring]. + */ + private val deliveredEarly = LinkedHashMap() + private val stoppedEarly = LinkedHashMap() + private var scheduler: AlarmScheduler? = null + private var sink: RingingSink? = null + + fun attach(scheduler: AlarmScheduler, sink: RingingSink) { + lock.withLock { + this.scheduler = scheduler + this.sink = sink + } + } + + fun detach() { + lock.withLock { + scheduler?.cancelAll() + scheduler = null + sink = null + } + } + + fun isRinging(id: Long): Boolean = lock.withLock { ringing.containsKey(id) } + + fun isAnythingRinging(): Boolean = lock.withLock { ringing.isNotEmpty() } + + fun snapshot(): List = lock.withLock { ArrayList(ringing.values) } + + /** + * The glasses report showing [id]. True when the item is ringing (the host should process + * the delivery); false when it is not yet known, in which case the signal is kept for a + * ring request that follows within [EARLY_SIGNAL_MAX_AGE_MS]. + */ + fun deliveredOrDefer(id: Long): Boolean = + lock.withLock { + if (ringing.containsKey(id)) true + else { + deliveredEarly[id] = clock() + false + } + } + + /** As [deliveredOrDefer], for an acknowledgement / cancellation. */ + fun stopOrDefer(id: Long): Boolean = + lock.withLock { + if (ringing.containsKey(id)) true + else { + stoppedEarly[id] = clock() + false + } + } + + /** An item came due (idempotent per id: a duplicate ring only refreshes the notification). */ + fun ring(item: RingingItem) { + if (item.id == 0L) return + lock.withLock { + val sink = sink ?: return + val existing = ringing[item.id] + if (existing != null) { + // The engine's JS timeout and the platform alarm both fired: one ring. + sink.showNotification(existing) + return + } + if (takeEarlySignal(stoppedEarly, item.id)) { + // Acknowledged or cancelled before this request was processed. + sink.log("ring " + item.id + " already acknowledged") + return + } + ringing[item.id] = item + sink.log("ring " + item.kind + " " + item.id + " (" + gate.description() + ")") + sink.ringingStarted() + sink.showNotification(item) + if (!gate.canCarryAlarm()) { + escalate(item, gate.description()) + return + } + if (takeEarlySignal(deliveredEarly, item.id)) { + markDelivered(item.id) + return + } + // The glasses could carry it: give the JS side a moment to confirm they are + // actually showing it, then hold for the acknowledgement. + scheduler?.postDelayed("delivery:" + item.id, DELIVERY_WAIT_MS) { + lock.withLock { + val current = ringing[item.id] + if (current != null && !current.escalated && current.deliveredAtMs == 0L) { + escalate(current, "not delivered to the glasses") + } + } + } + } + } + + fun markDelivered(id: Long) { + lock.withLock { + val item = ringing[id] + if (item == null || item.deliveredAtMs != 0L) return + item.deliveredAtMs = clock() + sink?.log("delivered $id to glasses") + scheduler?.postDelayed("ack:$id", ACK_WAIT_MS) { + lock.withLock { + val current = ringing[id] + if (current != null && !current.escalated) { + escalate(current, "not acknowledged on the glasses") + } + } + } + } + } + + private fun escalate(item: RingingItem, reason: String) { + if (item.escalated) return + item.escalated = true + val sink = sink ?: return + sink.log("phone sound for " + item.id + ": " + reason) + sink.showNotification(item) + sink.startSound() + scheduler?.postDelayed("silence:" + item.id, AUTO_SILENCE_MS) { + lock.withLock { + val current = ringing[item.id] + if (current != null && current.escalated && !current.silenced) { + current.silenced = true + this.sink?.log("auto-silenced " + current.id) + this.sink?.showNotification(current) + syncSound() + } + } + } + } + + /** Dismissed on the phone: journal it, stop it, and drop the schedule entry. */ + fun dismiss(id: Long) { + lock.withLock { + val item = ringing[id] ?: return + val sink = sink ?: return + sink.recordPhoneAction(id, "dismiss", 0) + removeItem(item) + // A one-off is over; the engine re-arms repeats. + sink.cancelSchedule(id) + } + } + + /** Snoozed on the phone: journal it, stop it, and re-arm on the phone directly so the snooze holds even with the JS side gone. */ + fun snooze(id: Long) { + lock.withLock { + val item = ringing[id] ?: return + val sink = sink ?: return + val minutes = item.snoozeMinutes + sink.recordPhoneAction(id, "snooze", minutes) + removeItem(item) + sink.reschedule(item, clock() + minutes * 60_000L, minutes) + } + } + + /** Stop an item without a phone action (acknowledged on the glasses / cancelled by the engine). */ + fun stopQuietly(id: Long, reason: String) { + lock.withLock { + val item = ringing[id] ?: return + sink?.log("$reason $id") + removeItem(item) + } + } + + /** Start or stop the sound to match whether any escalated, not-yet-silenced item exists. */ + fun syncSound() { + lock.withLock { + val sink = sink ?: return + if (ringing.values.any { it.escalated && !it.silenced }) sink.startSound() else sink.stopSound() + } + } + + fun cancelTimers() { + lock.withLock { scheduler?.cancelAll() } + } + + private fun removeItem(item: RingingItem) { + ringing.remove(item.id) + sink?.itemRemoved(item) + syncSound() + } + + /** Consume an early signal for the id; stale entries are dropped. */ + private fun takeEarlySignal(signals: MutableMap, id: Long): Boolean { + val at = signals.remove(id) + val now = clock() + signals.entries.removeAll { now - it.value > EARLY_SIGNAL_MAX_AGE_MS } + return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS + } + + companion object { + /** How long to wait for the JS side to confirm the glasses are showing the item. */ + const val DELIVERY_WAIT_MS = 5_000L + /** How long after delivery to the glasses before the phone joins in. */ + const val ACK_WAIT_MS = 30_000L + /** Sound and vibration stop after this; the notification stays. */ + const val AUTO_SILENCE_MS = 10 * 60_000L + const val EARLY_SIGNAL_MAX_AGE_MS = 15_000L + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/alarms/AlarmSchedule.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/alarms/AlarmSchedule.kt new file mode 100644 index 00000000..6a8dceaf --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/alarms/AlarmSchedule.kt @@ -0,0 +1,235 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic + +/** Key/value persistence for the alarm schedule, journal and log (device-protected prefs on Android). */ +interface AlarmStore { + fun read(key: String): String? + + fun write(key: String, value: String?) +} + +/** One scheduled item: the JS engine's item id, its wall-clock trigger time, and what to show. */ +class AlarmEntry( + @JvmField val id: Long, + @JvmField val at: Long, + title: String?, + text: String?, + kind: String?, + @JvmField val snoozeMinutes: Int, +) { + @JvmField val title: String = title ?: "" + @JvmField val text: String = text ?: "" + @JvmField val kind: String = kind ?: AlarmSchedule.KIND_ALARM + + fun toJson(): Map = + linkedMapOf( + "id" to JsonNumber.of(id), + "at" to JsonNumber.of(at), + "title" to title, + "text" to text, + "kind" to kind, + "snoozeMinutes" to JsonNumber.of(snoozeMinutes), + ) + + companion object { + @JvmStatic + fun fromJson(value: Any?): AlarmEntry? { + val map = value.asObject() ?: return null + return AlarmEntry( + map["id"].asLong() ?: 0L, + map["at"].asLong() ?: 0L, + map["title"].asString() ?: "", + map["text"].asString() ?: "", + map["kind"].asString() ?: AlarmSchedule.KIND_ALARM, + map["snoozeMinutes"].asInt() ?: 10, + ) + } + } +} + +/** + * The persisted alarm schedule, the phone-action journal and the diagnostics log, plus the + * replay / staleness policies. Everything the platform's alarm clock API does not own. + * Compound read-modify-write operations are serialized on one lock; the store itself only + * needs to be atomic per key. + */ +class AlarmSchedule( + private val store: AlarmStore, + private val clock: () -> Long = ::currentTimeMillis, + platform: ProtocolPlatform = protocolPlatform(), +) { + private val lock = platform.createLock() + + /** All entries, in stored order. */ + fun entries(): List = lock.withLock { readEntries() } + + fun find(id: Long): AlarmEntry? = lock.withLock { readEntries().firstOrNull { it.id == id } } + + /** Add or replace the entry with the same id (moved to the end). */ + fun put(entry: AlarmEntry) { + lock.withLock { + val next = readEntries().filter { it.id != entry.id } + entry + writeEntries(next) + } + } + + fun remove(id: Long) { + lock.withLock { writeEntries(readEntries().filter { it.id != id }) } + } + + /** + * Replay the schedule after a reboot, package update or clock change: entries already due + * ring now if inside the grace window, older ones are dropped as missed (and logged), the + * rest are handed to [arm]. Logs the outcome. + */ + fun replay(reason: String?, ring: (AlarmEntry) -> Unit, arm: (AlarmEntry) -> Unit) { + val scheduled = entries() + val now = clock() + var armed = 0 + for (entry in scheduled) { + if (entry.at <= now) { + if (now - entry.at <= LATE_GRACE_MS) { + ring(entry) + } else { + log("missed " + entry.kind + " " + entry.id + " (" + reason + ")") + remove(entry.id) + } + continue + } + arm(entry) + armed++ + } + log("rescheduled $armed ($reason)") + } + + /** The soonest future trigger time, or null when nothing is pending. */ + fun soonestPendingAt(now: Long = clock()): Long? = + entries().filter { it.at > now }.minOfOrNull { it.at } + + /** + * The schedule and the system alarm clock can disagree (a force-stop clears alarms but not + * the file). True when the soonest stored entry is not what the system reports as its next + * alarm clock; another app's sooner alarm is fine, only a later (or absent) one proves ours + * is missing. [systemNextAlarmAt] is null when the system has no alarm clock set. + */ + fun looksStale(systemNextAlarmAt: Long?, now: Long = clock()): Boolean { + val soonest = soonestPendingAt(now) ?: return false + return systemNextAlarmAt == null || systemNextAlarmAt > soonest + 1000 + } + + // ------------------------------------------------------------------ + // Phone-action journal (dismiss / snooze done on the phone, replayed by the JS engine) + + fun appendJournal(id: Long, action: String, minutes: Int) { + val event = linkedMapOf( + "id" to JsonNumber.of(id), + "action" to action, + "minutes" to JsonNumber.of(minutes), + "at" to JsonNumber.of(clock()), + ) + lock.withLock { + val journal = readArray(KEY_JOURNAL) + event + store.write(KEY_JOURNAL, Json.write(journal)) + } + log("phone " + action + " " + id + (if (minutes > 0) " ($minutes min)" else "")) + } + + /** Hand the journal over as a JSON array and clear it. */ + fun drainJournal(): String = + lock.withLock { + val text = Json.write(readArray(KEY_JOURNAL)) + store.write(KEY_JOURNAL, "[]") + text + } + + // ------------------------------------------------------------------ + // Diagnostics log (bounded, newest last) + + fun log(line: String) { + lock.withLock { + val existing = readArray(KEY_LOG).map { it.asString() ?: it?.toString() ?: "" } + val kept = existing.drop(maxOf(0, existing.size - (LOG_LINES - 1))) + store.write(KEY_LOG, Json.write(kept + (clock().toString() + " " + line))) + } + } + + fun readLog(): String = + lock.withLock { readArray(KEY_LOG).joinToString("\n") { it.asString() ?: it?.toString() ?: "" } } + + // ------------------------------------------------------------------ + + private fun readArray(key: String): List { + val raw = store.read(key) ?: return emptyList() + return runCatching { Json.parseArray(raw) }.getOrDefault(emptyList()) + } + + private fun readEntries(): List = readArray(KEY_SCHEDULED).mapNotNull { AlarmEntry.fromJson(it) } + + private fun writeEntries(entries: List) { + store.write(KEY_SCHEDULED, Json.write(entries.map { it.toJson() })) + } + + companion object { + const val KIND_TIMER = "timer" + const val KIND_ALARM = "alarm" + + const val KEY_SCHEDULED = "scheduled" + const val KEY_JOURNAL = "journal" + const val KEY_LOG = "log" + const val LOG_LINES = 60 + + /** + * A schedule entry found already past due (after a reboot, or when the process was + * asleep) rings if it is at most this late; older ones are dropped as missed. Matches + * the JS engine's grace window. + */ + const val LATE_GRACE_MS = 5 * 60_000L + + /** A stable per-item request code for platform alarm/notification identities. */ + @JvmStatic + fun requestCode(id: Long): Int = (id xor (id ushr 32)).toInt() and 0x7fffffff + } +} + +/** + * The last glasses status report pushed by the JS side. In memory only, so a dead process + * reads as "no glasses"; a report older than [MAX_AGE_MS] is not trusted either. + */ +class GlassesStatusGate(private val clock: () -> Long = ::currentTimeMillis) { + @Volatile private var connected = false + @Volatile private var worn = false + @Volatile private var charging = false + @Volatile private var statusAtMs = 0L + + fun set(connected: Boolean, worn: Boolean, charging: Boolean) { + this.connected = connected + this.worn = worn + this.charging = charging + statusAtMs = clock() + } + + /** True only when a fresh report says the glasses are connected, on a head, and not charging. */ + fun canCarryAlarm(): Boolean { + val at = statusAtMs + if (at == 0L || clock() - at > MAX_AGE_MS) return false + return connected && worn && !charging + } + + fun description(): String { + val at = statusAtMs + if (at == 0L) return "no glasses status" + if (clock() - at > MAX_AGE_MS) return "glasses status stale" + if (!connected) return "glasses not connected" + if (charging) return "glasses charging" + if (!worn) return "glasses not worn" + return "glasses worn" + } + + companion object { + /** A glasses status report older than this is not trusted (the JS side is probably gone). */ + const val MAX_AGE_MS = 3 * 60_000L + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/AudioSegmentation.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/AudioSegmentation.kt new file mode 100644 index 00000000..d1f98a4d --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/AudioSegmentation.kt @@ -0,0 +1,127 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic +import kotlin.math.abs +import kotlin.math.min +import kotlin.math.sqrt + +/** + * Sample-level helpers shared by the push-to-talk voice controller and the caption engine + * (previously two private copies each). All PCM is 16 kHz mono S16LE unless stated. + */ +object AudioSegmentation { + /** + * Where to end a decode segment: the center of the quietest [windowSamples]-long window + * within the last [searchSamples] of `samples[offset until offset + length]`, so the cut + * lands between words rather than splitting one. Returns [length] when the region is + * too short to search. + */ + @JvmStatic + fun quietestCutPoint(samples: FloatArray, length: Int, searchSamples: Int, windowSamples: Int): Int = + quietestCutPoint(length, searchSamples, windowSamples) { samples[it].toDouble() } + + @JvmStatic + fun quietestCutPoint(samples: ShortArray, offset: Int, length: Int, searchSamples: Int, windowSamples: Int): Int = + quietestCutPoint(length, searchSamples, windowSamples) { samples[offset + it].toDouble() } + + private inline fun quietestCutPoint(length: Int, searchSamples: Int, windowSamples: Int, sample: (Int) -> Double): Int { + val searchStart = maxOf(0, length - searchSamples) + val win = windowSamples + if (length - searchStart <= win) return length + var sum = 0.0 + for (i in searchStart until searchStart + win) { + val s = sample(i) + sum += s * s + } + var best = sum + var bestStart = searchStart + var start = searchStart + 1 + while (start + win <= length) { + val dropped = sample(start - 1) + val added = sample(start + win - 1) + sum += added * added - dropped * dropped + if (sum < best) { + best = sum + bestStart = start + } + start++ + } + return bestStart + win / 2 + } + + @JvmStatic + fun peakAmplitude(samples: FloatArray): Float = peakAmplitude(samples, samples.size) + + @JvmStatic + fun peakAmplitude(samples: FloatArray, count: Int): Float { + var peak = 0f + for (i in 0 until count) { + val a = abs(samples[i]) + if (a > peak) peak = a + } + return peak + } + + /** + * Scales [samples] in place so the peak reaches [targetPeak], with the gain capped at + * [maxGain]; audio already at or above the target is left untouched. + */ + @JvmStatic + fun normalizePeak(samples: FloatArray, targetPeak: Float, maxGain: Float) { + val peak = peakAmplitude(samples) + if (peak <= 0f) return + val gain = min(targetPeak / peak, maxGain) + if (gain <= 1f) return + for (i in samples.indices) samples[i] *= gain + } + + /** S16 samples to [-1, 1] floats. */ + @JvmStatic + fun pcm16ToFloat(samples: ShortArray, offset: Int, count: Int): FloatArray { + val out = FloatArray(count) + for (i in 0 until count) out[i] = samples[offset + i] / 32768.0f + return out + } + + /** S16LE bytes to [-1, 1] floats (an odd trailing byte is ignored). */ + @JvmStatic + fun pcm16leToFloat(bytes: ByteArray, offset: Int, byteCount: Int): FloatArray { + val count = byteCount / 2 + val out = FloatArray(count) + for (i in 0 until count) { + val base = offset + i * 2 + out[i] = pcm16le(bytes[base], bytes[base + 1]) / 32768.0f + } + return out + } + + @JvmStatic + fun pcm16leToFloat(bytes: ByteArray): FloatArray = pcm16leToFloat(bytes, 0, bytes.size) + + /** One little-endian S16 sample from its two bytes. */ + @JvmStatic + fun pcm16le(low: Byte, high: Byte): Short = ((low.toInt() and 0xff) or (high.toInt() shl 8)).toShort() + + /** S16 samples to S16LE bytes. */ + @JvmStatic + fun pcm16ToLittleEndian(samples: ShortArray, offset: Int, count: Int): ByteArray { + val le = ByteArray(count * 2) + for (i in 0 until count) { + val s = samples[offset + i].toInt() + le[i * 2] = (s and 0xff).toByte() + le[i * 2 + 1] = ((s shr 8) and 0xff).toByte() + } + return le + } + + /** Root-mean-square of `samples[offset until offset + count]` in S16 units. */ + @JvmStatic + fun rms(samples: ShortArray, offset: Int, count: Int): Double { + var sumSquares = 0.0 + for (i in 0 until count) { + val s = samples[offset + i].toDouble() + sumSquares += s * s + } + return sqrt(sumSquares / maxOf(1, count)) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/CaptionEngineCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/CaptionEngineCore.kt new file mode 100644 index 00000000..6d18e7e6 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/CaptionEngineCore.kt @@ -0,0 +1,380 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.math.max +import kotlin.math.min +import kotlin.math.sqrt + +/** Receives segmenter events synchronously on the thread feeding [CaptionSegmenter.processChunk]. */ +interface CaptionSegmentSink { + fun onSpeechStart(startMs: Long) + + /** [samples] is the segmenter's own buffer, valid only for the duration of the call. */ + fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double) +} + +/** + * Splits decoded mic PCM into utterances with an adaptive energy gate: the noise floor is a + * rolling mean tracked while idle, onset fires at ONSET_FACTOR above it (keeping PRE_ROLL_MS of + * idle audio so leading consonants survive), and the utterance ends after [silenceMs] below + * RELEASE_FACTOR or at MAX_UTTERANCE_MS. Boundaries are measured on the 16 kHz sample clock, + * not the wall clock: BLE delivers mic packets in bursts. + */ +class CaptionSegmenter(private val sink: CaptionSegmentSink) { + companion object { + const val SAMPLE_RATE = 16000 + const val ONSET_FACTOR = 3.0 + const val RELEASE_FACTOR = 1.8 + const val MIN_RMS = 220.0 + const val NOISE_EMA_ALPHA = 0.05 + const val PRE_ROLL_MS = 400 + const val DEFAULT_SILENCE_MS = 800 + const val MIN_SILENCE_MS = 200 + const val MAX_SILENCE_MS = 3000 + const val MIN_UTTERANCE_MS = 350 + const val MAX_UTTERANCE_MS = 15000 + } + + @Volatile + var silenceMs = DEFAULT_SILENCE_MS + set(value) { + field = max(MIN_SILENCE_MS, min(MAX_SILENCE_MS, value)) + } + + private var totalSamples: Long = 0 + private var noiseFloor = 0.0 + private var inUtterance = false + private var utteranceStartSample: Long = 0 + private var silenceRunSamples: Long = 0 + private var utterancePeakRms = 0.0 + private val utterance = ShortArray(SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)) + private var utteranceLength = 0 + private val preRoll = ShortArray(SAMPLE_RATE * PRE_ROLL_MS / 1000) + private var preRollLength = 0 + + val isInUtterance: Boolean + get() = inUtterance + + fun reset() { + totalSamples = 0 + noiseFloor = 0.0 + inUtterance = false + utteranceLength = 0 + preRollLength = 0 + silenceRunSamples = 0 + utterancePeakRms = 0.0 + } + + /** Feeds one chunk of S16LE PCM bytes (any chunking). */ + fun processChunk(chunk: ByteArray) { + val count = chunk.size / 2 + val pcm = ShortArray(count) + for (i in 0 until count) { + pcm[i] = AudioSegmentation.pcm16le(chunk[i * 2], chunk[i * 2 + 1]) + } + processChunk(pcm, count) + } + + fun processChunk(pcm: ShortArray, count: Int) { + var sumSquares = 0.0 + for (i in 0 until count) { + val s = pcm[i].toDouble() + sumSquares += s * s + } + val rms = sqrt(sumSquares / max(1, count)) + totalSamples += count.toLong() + + if (!inUtterance) { + // Track the noise floor only while idle so speech doesn't raise it. + noiseFloor = if (noiseFloor == 0.0) rms else noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA + val threshold = max(noiseFloor, MIN_RMS) + if (rms >= threshold * ONSET_FACTOR) { + inUtterance = true + utteranceLength = 0 + silenceRunSamples = 0 + utterancePeakRms = rms + appendUtterance(preRoll, preRollLength) + utteranceStartSample = max(0L, totalSamples - count - preRollLength) + appendUtterance(pcm, count) + sink.onSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE) + } else { + appendPreRoll(pcm, count) + } + return + } + + appendUtterance(pcm, count) + utterancePeakRms = max(utterancePeakRms, rms) + val threshold = max(noiseFloor, MIN_RMS) + if (rms < threshold * RELEASE_FACTOR) { + silenceRunSamples += count.toLong() + } else { + silenceRunSamples = 0 + } + val utteranceMs = utteranceLength.toLong() * 1000 / SAMPLE_RATE + val silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs + if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) { + finalizeUtterance() + inUtterance = false + preRollLength = 0 + } + } + + /** Emits a trailing utterance (if any) so its text isn't lost on stop. */ + fun flush() { + if (inUtterance && utteranceLength > 0) { + finalizeUtterance() + } + inUtterance = false + preRollLength = 0 + } + + private fun appendPreRoll(pcm: ShortArray, count: Int) { + // Keep the last PRE_ROLL_MS of idle audio so onset consonants survive. + if (count >= preRoll.size) { + pcm.copyInto(preRoll, 0, count - preRoll.size, count) + preRollLength = preRoll.size + return + } + val keep = min(preRollLength, preRoll.size - count) + preRoll.copyInto(preRoll, 0, preRollLength - keep, preRollLength) + pcm.copyInto(preRoll, keep, 0, count) + preRollLength = keep + count + } + + private fun appendUtterance(pcm: ShortArray, count: Int) { + val room = utterance.size - utteranceLength + val copied = min(room, count) + if (copied > 0) { + pcm.copyInto(utterance, utteranceLength, 0, copied) + utteranceLength += copied + } + } + + private fun finalizeUtterance() { + val length = utteranceLength + val startMs = utteranceStartSample * 1000 / SAMPLE_RATE + val endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE + if (endMs - startMs < MIN_UTTERANCE_MS) { + return + } + sink.onUtterance(utterance, length, startMs, endMs, utterancePeakRms) + } +} + +/** + * Transcribes one utterance in model-safe segments cut at the quietest window (Moonshine v2 + * fails past ~9.1 s of input) and attaches a voice-print from at most its first 10 s. + */ +class UtteranceDecoder(private val transcriber: OfflineTranscriber?, private val embedder: SpeakerEmbedder?) { + companion object { + const val SAMPLE_RATE = 16000 + const val DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8 + const val CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2 + const val CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000 + const val NORMALIZE_TARGET_PEAK = 0.9f + const val NORMALIZE_MAX_GAIN = 30f + // Voice-prints degrade on very long inputs; embed at most the first 10 s. + const val EMBED_MAX_SAMPLES = SAMPLE_RATE * 10 + } + + val hasTranscriber: Boolean + get() = transcriber != null + + fun recognize(utterance: ShortArray, length: Int): String { + val recognizer = transcriber + if (recognizer == null || length <= 0) { + return "" + } + val joined = StringBuilder() + var offset = 0 + while (offset < length) { + val remaining = length - offset + var segment = min(remaining, DECODE_SEGMENT_MAX_SAMPLES) + if (remaining > DECODE_SEGMENT_MAX_SAMPLES) { + segment = AudioSegmentation.quietestCutPoint(utterance, offset, segment, CUT_SEARCH_SAMPLES, CUT_WINDOW_SAMPLES) + } + val samples = AudioSegmentation.pcm16ToFloat(utterance, offset, segment) + AudioSegmentation.normalizePeak(samples, NORMALIZE_TARGET_PEAK, NORMALIZE_MAX_GAIN) + val part = recognizer.recognize(samples, samples.size).trim() + if (part.isNotEmpty()) { + if (joined.isNotEmpty() && ".,!?;:%)]}".indexOf(part[0]) < 0) { + joined.append(' ') + } + joined.append(part) + } + offset += segment + } + return joined.toString().trim() + } + + fun embed(utterance: ShortArray, length: Int): FloatArray? { + val currentEmbedder = embedder + if (currentEmbedder == null || length <= 0) { + return null + } + val count = min(length, EMBED_MAX_SAMPLES) + return currentEmbedder.embed(AudioSegmentation.pcm16ToLittleEndian(utterance, 0, count), SAMPLE_RATE) + } + + fun release() { + transcriber?.release() + embedder?.release() + } +} + +/** Platform services for [CaptionSession]. */ +interface CaptionHost { + val dispatcher: CallbackDispatcher + + /** Loads the caption recognizer from [modelDir], or null when the model files are absent. */ + fun loadTranscriber(modelDir: String): OfflineTranscriber? + + /** Loads the speaker-embedding model, or null when the file is absent. */ + fun loadEmbedder(modelPath: String): SpeakerEmbedder? +} + +/** + * Continuous captioning over pushed mic PCM: queue, worker thread (owned here, via the shared + * [startThread]), model lifecycle, segmentation and per-utterance decode/embedding, with + * listener callbacks marshalled through the host dispatcher. + */ +class CaptionSession( + private val host: CaptionHost, + private val platform: ProtocolPlatform = protocolPlatform(), + private val threadName: String = "FaceclawCaptionEngine", + private val logError: (String, Throwable) -> Unit = { message, error -> PlatformLog.e(TAG, message, error) }, +) : CaptionSegmentSink { + companion object { + const val TAG = "FaceclawCaptions" + const val MAX_QUEUE_PACKETS = 200 + } + + private val lock = platform.createLock() + private val queue = DroppingPacketQueue(MAX_QUEUE_PACKETS, platform) + private val segmenter = CaptionSegmenter(this) + @Volatile + private var listener: FaceclawCaptionEngineListener? = null + @Volatile + private var asrModelDir: String? = null + @Volatile + private var speakerModelPath: String? = null + @Volatile + private var started = false + private var workerFinished: Latch? = null + @Volatile + private var workerThreadName: String? = null + private var decoder: UtteranceDecoder? = null + + fun setListener(listener: FaceclawCaptionEngineListener?) { + this.listener = listener + } + + fun setAsrModelDir(dir: String?) { + this.asrModelDir = dir + } + + fun setSpeakerModelPath(path: String?) { + this.speakerModelPath = path + } + + fun setSilenceMs(ms: Int) { + segmenter.silenceMs = ms + } + + fun start() { + lock.withLock { + if (started) { + return + } + started = true + val finished = Latch(1, platform) + workerFinished = finished + startThread(threadName, true) { runLoop(finished) } + } + } + + /** Stops and waits up to 2 s for the worker unless called from it. */ + fun stop() { + var finished: Latch? = null + var workerName: String? = null + lock.withLock { + if (!started) { + return + } + started = false + finished = workerFinished + workerName = workerThreadName + } + queue.wake() + if (finished != null && (workerName == null || currentThreadName() != workerName)) { + finished.await(2000) + } + } + + /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ + fun acceptPcm(pcm16le: ByteArray?) { + if (!started || pcm16le == null || pcm16le.size < 2) { + return + } + queue.put(pcm16le) + } + + private fun runLoop(finished: Latch) { + workerThreadName = currentThreadName() + try { + loadModels() + segmenter.reset() + while (started) { + val chunk = queue.take(250) { started } ?: continue + segmenter.processChunk(chunk) + } + // Flush a trailing utterance so its text isn't lost on stop. + segmenter.flush() + } catch (t: Throwable) { + logError("caption engine failed", t) + emitStatus("Captions failed: " + t.message) + } finally { + decoder?.release() + decoder = null + lock.withLock { + workerFinished = null + workerThreadName = null + } + finished.countDown() + } + } + + private fun loadModels() { + val modelDir = asrModelDir + val transcriber = if (modelDir != null) { + emitStatus("Loading caption model...") + host.loadTranscriber(modelDir) + } else { + null + } + val embedModel = speakerModelPath + val embedder = if (embedModel != null) host.loadEmbedder(embedModel) else null + decoder = UtteranceDecoder(transcriber, embedder) + emitStatus(if (transcriber != null) "Captions listening..." else "Captions listening (no ASR model)...") + } + + override fun onSpeechStart(startMs: Long) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onSpeechStart(startMs) } + } + + override fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double) { + val currentDecoder = decoder ?: return + val text = currentDecoder.recognize(samples, length) + val embedding = currentDecoder.embed(samples, length) + val currentListener = listener ?: return + val embeddingCopy = embedding?.copyOf() + host.dispatcher.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) } + } + + private fun emitStatus(status: String?) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onStatus(status) } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Lc3PacketFramer.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Lc3PacketFramer.kt new file mode 100644 index 00000000..71e9e660 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/Lc3PacketFramer.kt @@ -0,0 +1,93 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic + +/** Decodes the LC3 frames of one G2 packet into [pcmOut]; returns samples written, <= 0 on error. */ +interface Lc3Codec { + fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int +} + +/** + * G2 microphone packet framing shared by the Android JNI decoder and the iOS port: length + * check, counter-based duplicate/late/missing accounting, trailer parsing and statistics. + * The codec itself (liblc3) is injected. + * + * Both arms may relay the same packet, so a counter gap of 0 or >= 128 (a late copy, including + * across the 255 -> 0 wrap) is dropped before it can disturb the decoder's history. The + * former Android decoder only dropped gap == 0; this adopts the wrap-safe rule the iOS + * decoder already used. + */ +class Lc3PacketFramer(private val codec: Lc3Codec) { + companion object { + const val SAMPLE_RATE = 16000 + const val FRAME_US = 10000 + const val FRAME_BYTES = 40 + const val FRAMES_PER_PACKET = 5 + const val PACKET_BYTES = 205 + const val COUNTER_OFFSET = 204 + const val SAMPLES_PER_FRAME = 160 + const val SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME + const val LC3_BYTES = FRAME_BYTES * FRAMES_PER_PACKET + // Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the + // glasses DSP appends two signed 16-bit LE values computed on the raw + // stereo capture before the mono downmix — a signal-strength ratio and the + // asin-derived direction-of-arrival in degrees — then the packet counter. + const val SSR_OFFSET = 200 + const val ANGLE_OFFSET = 202 + /** Counter gaps at or beyond this are late copies, not losses. */ + const val LATE_GAP = 128 + + @JvmStatic + fun readTrailerS16(packet: ByteArray, offset: Int): Int = + ((packet[offset].toInt() and 0xff) or (packet[offset + 1].toInt() shl 8)).toShort().toInt() + } + + private var lastCounter = -1 + var realPackets: Long = 0 + private set + var duplicatePackets: Long = 0 + private set + var missingPackets: Long = 0 + private set + var decodeErrors: Long = 0 + private set + /** Firmware signal-strength ratio from the most recent decoded packet. */ + var lastSsr: Int = 0 + private set + /** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */ + var lastAngleDegrees: Int = 0 + private set + + /** + * Returns the number of PCM samples written to [pcmOut], or 0 when the packet was dropped + * (wrong length, duplicate/late, or codec failure; see the counters). + */ + fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int { + if (packet == null || packet.size != PACKET_BYTES) { + decodeErrors++ + return 0 + } + if (pcmOut == null || pcmOut.size < SAMPLES_PER_PACKET) { + throw IllegalArgumentException("pcmOut must hold $SAMPLES_PER_PACKET samples") + } + val counter = packet[COUNTER_OFFSET].toInt() and 0xff + if (lastCounter >= 0) { + val gap = (counter - lastCounter) and 0xff + if (gap == 0 || gap >= LATE_GAP) { + duplicatePackets++ + return 0 + } + missingPackets += (gap - 1).toLong() + } + val decoded = codec.decodeFrames(packet, 0, LC3_BYTES, pcmOut) + if (decoded <= 0) { + decodeErrors++ + return 0 + } + realPackets++ + lastCounter = counter + lastSsr = readTrailerS16(packet, SSR_OFFSET) + lastAngleDegrees = readTrailerS16(packet, ANGLE_OFFSET) + return decoded + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt new file mode 100644 index 00000000..33c81cf1 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoiceCaptureSession.kt @@ -0,0 +1,833 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmStatic +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min + +/** Platform services the voice-capture session needs; supplied by the Android/iOS adapter. */ +interface VoiceCaptureHost { + val dispatcher: CallbackDispatcher + + /** Runs first on the worker thread of every session (Android deletes legacy model files here). */ + fun onSessionStarting() + + /** True when a glasses session can carry microphone packets (false when no communicator is set). */ + fun isG2SessionReady(): Boolean + + /** Whether the glasses are still delivering packets; see [VoiceCaptureSession.isCapturing]. */ + fun isG2AudioCaptureActive(): Boolean + + /** The glasses packet source, or null when no communicator is available. */ + fun g2AudioSource(): PacketAudioSource? + + fun createPacketDecoder(): AudioPacketDecoder + + /** Opens the phone microphone at 16 kHz mono PCM16, or null when it cannot start. */ + fun openPhoneMic(): PcmAudioSource? + + /** Whether the on-device model files for [kind] are present (downloaded). */ + fun hasTranscriberModel(kind: VoiceModelKind): Boolean + + /** Loads the recognizer for [kind]; throws when the model cannot be loaded. */ + fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber + + /** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */ + fun speakerEmbedder(modelPath: String): SpeakerEmbedder + + /** Where a saved recording goes, or null when recordings cannot be stored. */ + fun recordingSink(): RecordingSink? +} + +/** + * Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late + * accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker + * verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with + * REPLACE-semantics partials, and speaker verification at session end. The session owns its + * worker thread (started through the shared [startThread]); listener callbacks go through the + * host's [CallbackDispatcher] so the adapter decides the delivery thread. + */ +class VoiceCaptureSession( + private val host: VoiceCaptureHost, + private val platform: ProtocolPlatform = protocolPlatform(), + private val threadName: String = "FaceclawVoiceController", + private val logInfo: (String) -> Unit = { PlatformLog.i(TAG, it) }, + private val logWarn: (String) -> Unit = { PlatformLog.w(TAG, it) }, + private val logError: (String, Throwable) -> Unit = { message, error -> PlatformLog.e(TAG, message, error) }, +) { + companion object { + const val TAG = "FaceclawVoice" + const val SAMPLE_RATE = 16000 + const val MAX_AUDIO_QUEUE_PACKETS = 80 + const val EXPECTED_PACKET_INTERVAL_MS = 50 + const val LATE_PACKET_INTERVAL_MS = 90 + const val STATS_INTERVAL_MS = 5_000 + // Push-to-talk utterance boundaries come from the button. We re-decode the + // current audio segment in full for each live partial and emit the complete + // utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder + // used here fails once a single input grows past roughly 9.1 seconds, so + // longer utterances are committed in model-safe segments. + const val TRANSCRIPT_DECODE_INTERVAL_MS = 700 + const val TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3 + const val TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8 + // When a segment fills, cut at the quietest window within the last + // TRANSCRIPT_CUT_SEARCH_SAMPLES rather than mid-word at the 8s mark; the + // audio after the cut carries over into the next segment. + const val TRANSCRIPT_CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2 + const val TRANSCRIPT_CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000 + // Glasses-mic PCM peaks around 0.1 full scale, and at that level the + // quantized Moonshine model often returns empty or garbled text. Boost + // each decode window toward this peak, with a gain cap so near-silent + // buffers aren't amplified into pure noise. + const val TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f + const val TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f + const val TRANSCRIPT_LOG_PREVIEW_CHARS = 80 + // Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f + // buffer) a segment is treated as silence and never reaches the Whisper + // recognizer at all, rather than risking a hallucinated non-answer. Picked + // conservatively low (well under typical mic noise floor already seen in + // this pipeline's normalization target) -- UNTESTED on real hardware, tune + // against real glasses captures rather than trusting this number. + const val WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f + // Speaker verification against the enrolled wearer voice-print ("my voice + // only" command gating). Configured before start(); the utterance PCM is + // buffered (capped) and verified once at session end. + const val VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10 + const val VERIFY_MIN_SAMPLES = SAMPLE_RATE + const val DEFAULT_VERIFY_THRESHOLD = 0.8f + /** Samples per G2 packet; the decode buffer the worker reuses. */ + const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET + + /** Joins transcript pieces, attaching leading punctuation to the previous text. */ + @JvmStatic + fun joinTranscript(prefix: String?, suffix: String?): String { + if (prefix.isNullOrEmpty()) { + return suffix ?: "" + } + if (suffix.isNullOrEmpty()) { + return prefix + } + val first = suffix[0] + val attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0 + return prefix + (if (attachesToPrevious) "" else " ") + suffix + } + + @JvmStatic + fun parseModelKind(kind: String?): VoiceModelKind = + if ("whisper" == kind) VoiceModelKind.WHISPER else VoiceModelKind.MOONSHINE + } + + enum class Mode { + ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind) + CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side + } + + private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) { + val arm: String = arm ?: "?" + } + + private val lock = platform.createLock() + private val audioQueue = DroppingPacketQueue(MAX_AUDIO_QUEUE_PACKETS, platform) + @Volatile + private var listener: FaceclawVoiceControllerListener? = null + @Volatile + private var started = false + // Set once the worker has the mic enabled for this session. Read and + // written under `lock`, so it flips with `started` atomically. + private var audioStarted = false + private var workerFinished: Latch? = null + @Volatile + private var workerThreadName: String? = null + // Capture from the phone's own microphone instead of the G2 over BLE + // (preview-only mode). Latched into activePhoneMic at start() (under `lock`) + // so a mid-session setter call can't switch pipelines underneath the worker. + @Volatile + private var usePhoneMic = false + private var activePhoneMic = false + private var mode = Mode.CLOUD + @Volatile + private var onboardModelKind = VoiceModelKind.MOONSHINE + private var recognizer: OfflineTranscriber? = null + private var packetDecoder: AudioPacketDecoder? = null + private val transcriptSamples = FloatArray(TRANSCRIPT_SEGMENT_MAX_SAMPLES) + private var transcriptSampleCount = 0 + private var committedTranscriptSampleCount: Long = 0 + private var committedTranscript = "" + private var currentSegmentTranscript = "" + private var lastTranscriptDecodeAtMs: Long = 0 + private var lastTranscript = "" + @Volatile + private var saveRecordings = false + @Volatile + private var endpointing = false + private val endpointDetector = VoiceEndpointDetector() + private var recordingPcm: ByteSink? = null + @Volatile + private var verifySpeakerModelPath: String? = null + @Volatile + private var verifyWearerEmbedding: FloatArray? = null + @Volatile + private var verifyThreshold = DEFAULT_VERIFY_THRESHOLD + private var verifyBuffer: ShortArray? = null + private var verifyCount = 0 + // Global mic processing (Microphones app config): spectral noise + // suppression and firmware-DoA beam gating, applied to every capture + // session that opts in. The raw tap opts out. + @Volatile + private var suppressionEnabled = false + @Volatile + private var beamFilterEnabled = false + @Volatile + private var beamCenterDeg = 0 + @Volatile + private var beamHalfWidthDeg = 180 + private var suppressor: FaceclawNoiseSuppressor? = null + private var queuedPackets: Long = 0 + private var decodedSamples: Long = 0 + private var latePackets: Long = 0 + private var wrongArmPackets: Long = 0 + private var lastPacketArrivalMs: Long = 0 + private var maxInterPacketMs: Long = 0 + private var lastStatsAtMs: Long = 0 + + fun setListener(listener: FaceclawVoiceControllerListener?) { + this.listener = listener + } + + fun setUsePhoneMic(usePhoneMic: Boolean) { + this.usePhoneMic = usePhoneMic + } + + fun setSaveRecordings(saveRecordings: Boolean) { + this.saveRecordings = saveRecordings + } + + fun setOnboardModelKind(kind: String?) { + this.onboardModelKind = parseModelKind(kind) + } + + fun setEndpointing(endpointing: Boolean) { + this.endpointing = endpointing + } + + fun setSpeakerVerification(speakerModelPath: String?, wearerEmbedding: FloatArray?, threshold: Float) { + this.verifySpeakerModelPath = speakerModelPath + this.verifyWearerEmbedding = wearerEmbedding + this.verifyThreshold = if (threshold > 0) threshold else DEFAULT_VERIFY_THRESHOLD + } + + fun clearSpeakerVerification() { + this.verifySpeakerModelPath = null + this.verifyWearerEmbedding = null + } + + fun setNoiseSuppression(enabled: Boolean) { + this.suppressionEnabled = enabled + } + + fun setBeamFilter(enabled: Boolean, centerDeg: Int, halfWidthDeg: Int) { + this.beamFilterEnabled = enabled + this.beamCenterDeg = centerDeg + this.beamHalfWidthDeg = max(5, min(180, halfWidthDeg)) + } + + /** Direction gating: true when [angleDegrees] lies within centerDeg ± halfWidthDeg (wrap-safe). */ + fun withinBeam(angleDegrees: Int): Boolean { + var delta = angleDegrees - beamCenterDeg + while (delta > 180) delta -= 360 + while (delta < -180) delta += 360 + return abs(delta) <= beamHalfWidthDeg + } + + private fun applySuppression(pcm: ShortArray, count: Int) { + try { + if (suppressor == null) { + suppressor = FaceclawNoiseSuppressor(SAMPLE_RATE) + } + val le = AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count) + val cleaned = suppressor!!.process(le)!! + val cleanedCount = min(count, cleaned.size / 2) + for (i in 0 until cleanedCount) { + pcm[i] = AudioSegmentation.pcm16le(cleaned[i * 2], cleaned[i * 2 + 1]) + } + } catch (t: Throwable) { + logWarn("noise suppression failed; passing audio through (${t.message})") + suppressionEnabled = false + } + } + + fun start(requestedMode: String?) { + start(requestedMode, 0) + } + + fun start(requestedMode: String?, captureId: Int) { + lock.withLock { + if (started) { + emitStatus("Voice control is already listening.") + emitStopped(captureId) + return + } + if (!usePhoneMic && !host.isG2SessionReady()) { + emitStatus("Voice control needs an active G2 connection.") + emitStopped(captureId) + return + } + mode = parseMode(requestedMode) + activePhoneMic = usePhoneMic + started = true + audioStarted = false + val finished = Latch(1, platform) + workerFinished = finished + startThread(threadName, true) { runLoop(captureId, finished) } + } + } + + /** + * Whether mic audio is actually flowing. start() only records intent: the + * enable lives in the glasses' EvenHub session, so a transport drop or a + * session suspend can leave this session started with a worker that will + * never see another packet. + */ + fun isCapturing(): Boolean { + var audioUp = false + var phoneMic = false + lock.withLock { + if (!started) { + return false + } + audioUp = audioStarted + phoneMic = activePhoneMic + } + if (!audioUp) { + // The worker is still bringing the mic up; report it as running so + // a concurrent request shares it instead of restarting it. + return true + } + if (phoneMic) { + // The phone mic has no session to lose the enable to; it runs until stop(). + return true + } + return host.isG2AudioCaptureActive() + } + + /** + * Stops the session and waits up to 1.5 s for the worker to finish unless + * called from the worker itself. The worker checks the stop flag between + * packets (every packet or 250 ms wait) and between phone-mic reads (50 ms). + */ + fun stop() { + var finished: Latch? = null + var workerName: String? = null + lock.withLock { + if (!started) { + return + } + started = false + finished = workerFinished + workerName = workerThreadName + } + stopG2Audio() + audioQueue.wake() + if (finished != null && (workerName == null || currentThreadName() != workerName)) { + finished.await(1500) + } + } + + fun close() { + stop() + } + + private fun parseMode(requestedMode: String?): Mode { + if ("cloud" == requestedMode) { + return Mode.CLOUD + } + return Mode.ONBOARD + } + + private fun runLoop(captureId: Int, finished: Latch) { + workerThreadName = currentThreadName() + try { + host.onSessionStarting() + val currentMode = mode + val currentOnboardKind = onboardModelKind + if (currentMode == Mode.ONBOARD) { + if (!host.hasTranscriberModel(currentOnboardKind)) { + emitStatus("Voice model not downloaded (see Settings > Voice).") + return + } + emitStatus("Loading transcription model...") + recognizer = host.loadTranscriber(currentOnboardKind) + resetTranscriptState() + lastTranscript = "" + } + endpointDetector.reset() + suppressor?.reset() + recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null + val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null + verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null + verifyCount = 0 + if (activePhoneMic) { + val record = host.openPhoneMic() + if (record == null) { + emitStatus("Could not start the phone microphone.") + return + } + lock.withLock { audioStarted = true } + emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...") + try { + processPhoneAudio(record) + } finally { + record.close() + } + } else { + packetDecoder = host.createPacketDecoder() + if (!startG2Audio()) { + emitStatus("Could not start G2 microphone input.") + return + } + lock.withLock { audioStarted = true } + emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...") + processG2Audio() + } + // Verification result must precede the final transcript so the + // TS bridge can suppress a non-wearer command before it is acted + // on (the callbacks are posted in order to the dispatcher). + runSpeakerVerification() + // Button released / stop requested: emit one final full-utterance + // transcript so the UI can freeze it. + if (currentMode == Mode.ONBOARD) { + decodeTranscript(true) + } + } catch (error: Throwable) { + logError("Voice control failed", error) + emitStatus("Voice control failed: " + error.message) + } finally { + stopG2Audio() + writeRecordingIfAny() + releaseRecognizer() + releaseDecoder() + lock.withLock { + started = false + audioStarted = false + workerFinished = null + workerThreadName = null + } + emitStopped(captureId) + finished.countDown() + } + } + + private fun appendRecording(pcm: ShortArray, count: Int) { + val out = recordingPcm ?: return + out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count)) + } + + private fun writeRecordingIfAny() { + val out = recordingPcm + recordingPcm = null + if (out == null || out.size() == 0) { + return + } + val sink = host.recordingSink() ?: return + try { + sink.save(out.toByteArray(), SAMPLE_RATE) + } catch (t: Throwable) { + logWarn("failed to save voice recording (${t.message})") + } + } + + private fun startG2Audio(): Boolean { + val source = host.g2AudioSource() ?: return false + resetAudioStats() + audioQueue.clear() + return source.start(object : FaceclawAudioPacketListener { + override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { + queueAudioPacket(data, arm, arrivalMs) + } + }) + } + + private fun processG2Audio() { + val pcm = ShortArray(PACKET_SAMPLES) + while (started) { + val currentDecoder = packetDecoder ?: return + val packet = takeAudioPacket() + if (packet == null) { + continue + } + val count = currentDecoder.decodePacket(packet.data, pcm) + if (count <= 0) { + maybeEmitAudioStats(false) + continue + } + decodedSamples += count.toLong() + // The beam filter drops packets whose firmware direction-of-arrival + // falls outside the listening wedge (isolating the aimed talker for + // every consumer, recognition included); the spectral noise + // suppressor then cleans what remains before it reaches the + // recognizer, cloud PCM, endpointing, or speaker verification. + val angleDegrees = currentDecoder.lastAngleDegrees + val ssr = currentDecoder.lastSsr + if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { + emitFrameMeta(angleDegrees, ssr) + maybeEmitAudioStats(false) + continue + } + processPcmChunk(pcm, count, angleDegrees, ssr, true) + maybeEmitAudioStats(false) + } + } + + /** + * Per-chunk processing shared by the G2 and phone-mic paths, downstream of + * decode and the beam filter. hasFrameMeta is false for the phone mic, + * which has no firmware DSP metadata to report. + */ + internal fun processPcmChunk(pcm: ShortArray, count: Int, angleDegrees: Int, ssr: Int, hasFrameMeta: Boolean) { + if (suppressionEnabled) { + applySuppression(pcm, count) + } + if (recordingPcm != null) { + appendRecording(pcm, count) + } + val verifyBuffer = this.verifyBuffer + if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { + val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount) + pcm.copyInto(verifyBuffer, verifyCount, 0, copied) + verifyCount += copied + } + if (endpointing && endpointDetector.accept(pcm, count)) { + emitSpeechEnd() + } + // PCM and frame metadata flow in every mode so levels, recording, + // and the Microphones radar keep working alongside onboard ASR. + emitPcm(pcm, count) + if (hasFrameMeta) { + emitFrameMeta(angleDegrees, ssr) + } + if (mode != Mode.CLOUD) { + processRecognizer(AudioSegmentation.pcm16ToFloat(pcm, 0, count)) + } + } + + /** + * Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame + * metadata. The blocking read returns every chunk (50 ms), which bounds how + * long a stop() waits for the loop to notice `started` dropped. + */ + private fun processPhoneAudio(record: PcmAudioSource) { + val pcm = ShortArray(SAMPLE_RATE / 20) + while (started) { + val read = record.read(pcm) + if (read < 0) { + logWarn("phone mic read failed: $read") + return + } + if (read == 0) { + continue + } + decodedSamples += read.toLong() + processPcmChunk(pcm, read, 0, 0, false) + } + } + + private fun processRecognizer(samples: FloatArray) { + appendTranscriptSamples(samples) + // Each Whisper call re-encodes the whole buffer plus ~10s of tail padding, + // so it skips the live-partial redecode Moonshine does on this interval and + // only decodes when a segment commits (8s buffer fill) or the utterance ends + // (decodeTranscript(true) in runLoop()). No live preview text in Whisper + // mode -- status stays "Listening..." until release. Deliberate tradeoff. + if (onboardModelKind == VoiceModelKind.WHISPER) { + return + } + val now = platform.elapsedRealtimeMs() + if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && + now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS + ) { + decodeTranscript(false) + lastTranscriptDecodeAtMs = now + } + } + + private fun appendTranscriptSamples(samples: FloatArray) { + var sourceOffset = 0 + while (sourceOffset < samples.size) { + val available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount + val count = min(available, samples.size - sourceOffset) + samples.copyInto(transcriptSamples, transcriptSampleCount, sourceOffset, sourceOffset + count) + transcriptSampleCount += count + sourceOffset += count + if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) { + commitTranscriptSegment() + } + } + } + + /** + * Decode the current model-safe segment and emit the best transcript of the + * complete utterance (REPLACE semantics — the caller displays it as-is). + */ + private fun decodeTranscript(isFinal: Boolean) { + if (recognizer == null || transcriptSampleCount <= 0) { + if (isFinal) { + emitTranscript(lastTranscript, true) + } + return + } + val segmentSampleCount = transcriptSampleCount + var segmentText = recognizeTranscriptSegment(segmentSampleCount) + if (segmentText.isNotEmpty()) { + currentSegmentTranscript = segmentText + } else { + segmentText = currentSegmentTranscript + } + val text = joinTranscript(committedTranscript, segmentText) + lastTranscript = text + logTranscriptDecode(isFinal, segmentSampleCount, text) + emitTranscript(text, isFinal) + } + + /** + * Finalize a full segment before accepting more audio. This keeps every + * Moonshine invocation below its failing sequence length while retaining + * all earlier text in the replace-semantics preview. + */ + private fun commitTranscriptSegment() { + val cut = AudioSegmentation.quietestCutPoint( + transcriptSamples, transcriptSampleCount, TRANSCRIPT_CUT_SEARCH_SAMPLES, TRANSCRIPT_CUT_WINDOW_SAMPLES, + ) + var segmentText = recognizeTranscriptSegment(cut) + if (segmentText.isEmpty()) { + // Fallback text came from partial decodes of the full buffer, so it + // may include words from the carried-over tail; rare now that decode + // windows are peak-normalized. + segmentText = currentSegmentTranscript + } + committedTranscript = joinTranscript(committedTranscript, segmentText) + currentSegmentTranscript = "" + committedTranscriptSampleCount += cut.toLong() + val tail = transcriptSampleCount - cut + transcriptSamples.copyInto(transcriptSamples, 0, cut, cut + tail) + transcriptSampleCount = tail + lastTranscript = committedTranscript + lastTranscriptDecodeAtMs = platform.elapsedRealtimeMs() + logTranscriptDecode(false, cut, committedTranscript) + emitTranscript(committedTranscript, false) + } + + private fun recognizeTranscriptSegment(sampleCount: Int): String { + val currentRecognizer = recognizer + if (currentRecognizer == null || sampleCount <= 0) { + return "" + } + val segment = transcriptSamples.copyOf(sampleCount) + // Whisper hallucinates text on near-silent input; gate it on the + // PRE-normalization peak, since normalizePeak() below would otherwise + // amplify true silence right up to the target level. Moonshine does not + // share this failure mode in practice, so it is left unchanged. + if (onboardModelKind == VoiceModelKind.WHISPER && + AudioSegmentation.peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD + ) { + return "" + } + AudioSegmentation.normalizePeak(segment, TRANSCRIPT_NORMALIZE_TARGET_PEAK, TRANSCRIPT_NORMALIZE_MAX_GAIN) + return currentRecognizer.recognize(segment, segment.size).trim() + } + + private fun logTranscriptDecode(isFinal: Boolean, segmentSampleCount: Int, text: String) { + val totalAudioSec = (committedTranscriptSampleCount + transcriptSampleCount) / SAMPLE_RATE.toDouble() + val preview = if (text.length <= TRANSCRIPT_LOG_PREVIEW_CHARS) text + else text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..." + logInfo( + (if (onboardModelKind == VoiceModelKind.WHISPER) "Whisper" else "Moonshine") + " decode final=" + isFinal + + " audioSec=" + formatFixed(totalAudioSec, 2) + + " segmentAudioSec=" + formatFixed(segmentSampleCount / SAMPLE_RATE.toDouble(), 2) + + " textLen=" + text.length + " text=\"" + preview + "\"", + ) + } + + private fun resetTranscriptState() { + transcriptSampleCount = 0 + committedTranscriptSampleCount = 0 + committedTranscript = "" + currentSegmentTranscript = "" + lastTranscriptDecodeAtMs = 0 + } + + private fun emitPcm(pcm: ShortArray, count: Int) { + val currentListener = listener + if (currentListener == null || count <= 0) { + return + } + val le = AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count) + host.dispatcher.post { currentListener.onPcm(le) } + } + + /** + * Embed the session's buffered utterance and compare it to the enrolled + * wearer voice-print. Fails open: a session too short to verify, or a + * model that will not load, counts as the wearer rather than silencing + * every command. + */ + private fun runSpeakerVerification() { + val buffer = verifyBuffer + val wearer = verifyWearerEmbedding + val modelPath = verifySpeakerModelPath + verifyBuffer = null + if (buffer == null || wearer == null || modelPath == null) { + return + } + if (verifyCount < VERIFY_MIN_SAMPLES) { + emitSpeakerVerified(true, 0f) + return + } + try { + val embedder = host.speakerEmbedder(modelPath) + val le = AudioSegmentation.pcm16ToLittleEndian(buffer, 0, verifyCount) + val embedding = embedder.embed(le, SAMPLE_RATE) + if (embedding == null || embedding.size != wearer.size) { + emitSpeakerVerified(true, 0f) + return + } + val dot = VoicePrint.dot(embedding, wearer) + val isWearer = dot >= verifyThreshold + logInfo( + "speaker verification similarity=" + formatFixed(dot, 3) + + " threshold=" + verifyThreshold + " isWearer=" + isWearer, + ) + emitSpeakerVerified(isWearer, dot.toFloat()) + } catch (t: Throwable) { + logWarn("speaker verification failed (${t.message})") + emitSpeakerVerified(true, 0f) + } + } + + private fun emitSpeakerVerified(isWearer: Boolean, similarity: Float) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onSpeakerVerified(isWearer, similarity) } + } + + private fun emitFrameMeta(angleDegrees: Int, ssr: Int) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) } + } + + private fun emitSpeechEnd() { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onSpeechEnd() } + } + + private fun emitStopped(captureId: Int) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onStopped(captureId) } + } + + private fun stopG2Audio() { + host.g2AudioSource()?.stop() + maybeEmitAudioStats(true) + } + + private fun releaseRecognizer() { + recognizer?.release() + recognizer = null + } + + private fun releaseDecoder() { + packetDecoder?.close() + packetDecoder = null + } + + /** Called from the BLE notification thread for every glasses mic packet. */ + fun queueAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) { + if (!started || data == null) { + return + } + if ("L" != arm) { + wrongArmPackets++ + } + audioQueue.put(AudioPacket(data, arm, arrivalMs)) { + queuedPackets++ + if (lastPacketArrivalMs > 0) { + val delta = arrivalMs - lastPacketArrivalMs + if (delta > maxInterPacketMs) { + maxInterPacketMs = delta + } + if (delta > LATE_PACKET_INTERVAL_MS) { + latePackets++ + } + } + lastPacketArrivalMs = arrivalMs + } + } + + private fun takeAudioPacket(): AudioPacket? { + val packet = audioQueue.take(250) { started } + if (packet == null) { + maybeEmitAudioStats(false) + } + return packet + } + + private fun resetAudioStats() { + queuedPackets = 0 + decodedSamples = 0 + latePackets = 0 + wrongArmPackets = 0 + lastPacketArrivalMs = 0 + maxInterPacketMs = 0 + lastStatsAtMs = platform.elapsedRealtimeMs() + } + + /** Diagnostic packet accounting for tests and adapters. */ + class AudioStats( + val queuedPackets: Long, + val queueDroppedPackets: Long, + val latePackets: Long, + val wrongArmPackets: Long, + val maxInterPacketMs: Long, + val decodedSamples: Long, + ) + + fun audioStats(): AudioStats = AudioStats( + queuedPackets, audioQueue.droppedCount, latePackets, wrongArmPackets, maxInterPacketMs, decodedSamples, + ) + + private fun maybeEmitAudioStats(force: Boolean) { + val now = platform.elapsedRealtimeMs() + if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) { + return + } + lastStatsAtMs = now + val currentDecoder = packetDecoder + val real = currentDecoder?.realPackets ?: 0L + val duplicate = currentDecoder?.duplicatePackets ?: 0L + val missing = currentDecoder?.missingPackets ?: 0L + val decodeErrors = currentDecoder?.decodeErrors ?: 0L + val status = "G2 mic packets=" + queuedPackets + + " decoded=" + real + + " missing=" + missing + + " duplicate=" + duplicate + + " late=" + latePackets + + " maxGapMs=" + maxInterPacketMs + + " queueDrop=" + audioQueue.droppedCount + + " decodeErrors=" + decodeErrors + + " wrongArm=" + wrongArmPackets + + " audioSec=" + formatFixed(decodedSamples / SAMPLE_RATE.toDouble(), 1) + // Audio-pipeline stats are diagnostic; keep them in the log only, out of + // the on-glasses voice UI. + logInfo(status + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS) + } + + private fun emitStatus(status: String?) { + val currentListener = listener ?: return + host.dispatcher.post { currentListener.onStatus(status) } + } + + private fun emitTranscript(text: String?, isFinal: Boolean) { + val currentListener = listener ?: return + logInfo("Emit transcript final=" + isFinal + " textLen=" + (text?.trim()?.length ?: 0)) + host.dispatcher.post { currentListener.onTranscript(text, isFinal) } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt new file mode 100644 index 00000000..148c68a4 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePorts.kt @@ -0,0 +1,137 @@ +package com.faceclaw.app + +import kotlin.math.abs +import kotlin.math.round + +/** Which on-device model ONBOARD voice input loads. */ +enum class VoiceModelKind { + MOONSHINE, + WHISPER, +} + +/** + * Whole-utterance (offline) recognizer: each call decodes one complete buffer. The Android + * implementation wraps sherpa-onnx; a streaming recognizer (Apple Speech) could satisfy this + * by buffering and re-running, but the session's REPLACE-semantics partials assume re-decoding + * the same buffer is cheap enough on the MOONSHINE cadence. + */ +interface OfflineTranscriber { + /** Returns the trimmed text for [count] float samples at 16 kHz, or "" when nothing was recognized. */ + fun recognize(samples: FloatArray, count: Int): String + + fun release() +} + +/** Speaker voice-print extraction over 16 kHz mono S16LE PCM; null when unavailable or too short. */ +interface SpeakerEmbedder { + fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? + + fun release() +} + +/** Marshals listener callbacks to the platform's UI/JS thread (Android: the main Handler). */ +fun interface CallbackDispatcher { + fun post(action: () -> Unit) +} + +/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */ +fun interface RecordingSink { + fun save(pcm16le: ByteArray, sampleRate: Int) +} + +/** A G2 microphone packet decoder (the shared [Lc3PacketFramer] over a platform LC3 codec). */ +interface AudioPacketDecoder { + fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int + + val lastAngleDegrees: Int + val lastSsr: Int + val realPackets: Long + val duplicatePackets: Long + val missingPackets: Long + val decodeErrors: Long + + fun close() +} + +/** Glasses microphone packets over BLE (Android: FaceclawBleCommunicator's capture API). */ +interface PacketAudioSource { + fun start(listener: FaceclawAudioPacketListener): Boolean + + fun stop() + + /** Whether the glasses are still delivering packets for this capture. */ + fun isActive(): Boolean +} + +/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */ +interface PcmAudioSource { + /** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */ + fun read(pcm: ShortArray): Int + + fun close() +} + +/** Formats [value] with exactly [decimals] fraction digits (the shared stand-in for %.Nf). */ +fun formatFixed(value: Double, decimals: Int): String { + var scale = 1L + repeat(decimals) { scale *= 10 } + val scaled = round(abs(value) * scale).toLong() + val whole = scaled / scale + val fraction = scaled % scale + val sign = if (value < 0 && scaled != 0L) "-" else "" + return if (decimals == 0) "$sign$whole" else "$sign$whole." + fraction.toString().padStart(decimals, '0') +} + +/** + * Bounded FIFO between a producer thread and one consumer: when full the oldest item is + * dropped (counted). [take] waits up to [waitMs] and returns null on timeout or when [open] is + * false, so the consumer can poll a stop flag and emit periodic diagnostics. + */ +class DroppingPacketQueue(private val capacity: Int, platform: ProtocolPlatform = protocolPlatform()) { + private val condition = platform.createCondition() + private val items = ArrayDeque() + var droppedCount: Long = 0 + private set + + fun put(item: T) { + condition.withLock { + if (items.size >= capacity) { + items.removeFirst() + droppedCount++ + } + items.addLast(item) + condition.signalAll() + } + } + + /** Runs [action] while holding the queue lock, after enqueuing (used to update stats atomically). */ + fun put(item: T, action: () -> Unit) { + condition.withLock { + if (items.size >= capacity) { + items.removeFirst() + droppedCount++ + } + items.addLast(item) + action() + condition.signalAll() + } + } + + fun take(waitMs: Long, open: () -> Boolean): T? { + condition.withLock { + if (items.isEmpty() && open()) condition.awaitMs(waitMs) + return items.removeFirstOrNull() + } + } + + fun clear() { + condition.withLock { + items.clear() + droppedCount = 0 + } + } + + fun wake() { + condition.withLock { condition.signalAll() } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePrint.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePrint.kt new file mode 100644 index 00000000..b3a26c9d --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/VoicePrint.kt @@ -0,0 +1,104 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic +import kotlin.math.sqrt + +/** Voice-print (speaker embedding) vector math shared by extraction, verification and storage. */ +object VoicePrint { + /** L2-normalizes [v] in place and returns it; a null or zero vector is returned unchanged. */ + @JvmStatic + fun l2Normalize(v: FloatArray?): FloatArray? { + if (v == null) return null + var sum = 0.0 + for (x in v) sum += x.toDouble() * x + val norm = sqrt(sum) + if (norm <= 0) return v + for (i in v.indices) v[i] = (v[i] / norm).toFloat() + return v + } + + /** Dot product; on L2-normalized vectors this is the cosine similarity. */ + @JvmStatic + fun dot(a: FloatArray, b: FloatArray): Double { + require(a.size == b.size) { "embedding sizes differ: ${a.size} vs ${b.size}" } + var sum = 0.0 + for (i in a.indices) sum += a[i].toDouble() * b[i] + return sum + } + + /** Utterances shorter than this (~0.5 s of S16 mono at [sampleRate]) produce unreliable prints. */ + @JvmStatic + fun isLongEnoughToEmbed(pcm16leBytes: Int, sampleRate: Int): Boolean = pcm16leBytes >= sampleRate +} + +/** Persistence-side voice-print math: float32-LE blobs and centroid updates. */ +object SpeakerEmbeddings { + /** Decodes a float32-LE blob; null for a null, empty or misaligned blob. */ + @JvmStatic + fun blobToFloats(blob: ByteArray?): FloatArray? { + if (blob == null || blob.size < 4 || blob.size % 4 != 0) return null + val out = FloatArray(blob.size / 4) + for (i in out.indices) { + val base = i * 4 + val bits = (blob[base].toInt() and 0xff) or + ((blob[base + 1].toInt() and 0xff) shl 8) or + ((blob[base + 2].toInt() and 0xff) shl 16) or + ((blob[base + 3].toInt() and 0xff) shl 24) + out[i] = Float.fromBits(bits) + } + return out + } + + @JvmStatic + fun floatsToBlob(values: FloatArray): ByteArray { + val out = ByteArray(values.size * 4) + for (i in values.indices) { + val bits = values[i].toRawBits() + val base = i * 4 + out[base] = (bits and 0xff).toByte() + out[base + 1] = ((bits shr 8) and 0xff).toByte() + out[base + 2] = ((bits shr 16) and 0xff).toByte() + out[base + 3] = ((bits shr 24) and 0xff).toByte() + } + return out + } + + class Centroid(val embedding: FloatArray, val count: Int) + + /** + * Running-mean update of a speaker's centroid with one new print, with the existing + * weight capped at [maxCount] so a long session cannot drown the enrolled voice. A + * missing or differently sized current centroid is replaced by [update] (count 1). + */ + @JvmStatic + fun runningMean(current: FloatArray?, currentCount: Int, update: FloatArray, maxCount: Int): Centroid { + if (current == null || current.size != update.size) return Centroid(update, 1) + val effective = minOf(currentCount, maxOf(1, maxCount)) + val merged = FloatArray(current.size) + for (i in merged.indices) merged[i] = (current[i] * effective + update[i]) / (effective + 1) + VoicePrint.l2Normalize(merged) + return Centroid(merged, currentCount + 1) + } + + /** + * Blends two centroids weighted by their sample counts when both exist with the same + * size; otherwise keeps [into], falling back to [from] when [into] is missing. Null when + * neither exists. + */ + @JvmStatic + fun blend(from: FloatArray?, fromCount: Int, into: FloatArray?, intoCount: Int): FloatArray? { + if (from != null && into != null && from.size == into.size) { + val blended = FloatArray(into.size) + val total = maxOf(1, fromCount + intoCount) + for (i in blended.indices) blended[i] = (into[i] * intoCount + from[i] * fromCount) / total + VoicePrint.l2Normalize(blended) + return blended + } + return into ?: from + } + + /** When merging speakers, the target keeps its conversation insights; it adopts the source's only when it has none. */ + @JvmStatic + fun shouldAdoptInsights(targetHasInsights: Boolean, sourceHasInsights: Boolean): Boolean = + !targetHasInsights && sourceHasInsights +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WavPcmReader.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WavPcmReader.kt new file mode 100644 index 00000000..7fb7b2fd --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WavPcmReader.kt @@ -0,0 +1,66 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic + +/** RIFF/WAVE header walk shared by the AAC transcoder and the diarizer (16-bit PCM only). */ +object WavPcmReader { + /** Bytes of a WAV file the header walk needs at most; chunk payloads beyond it are skipped by offset. */ + const val HEADER_PREFIX_BYTES = 65536 + + class WavInfo( + @JvmField val sampleRate: Int, + @JvmField val channels: Int, + @JvmField val bitsPerSample: Int, + /** Byte offset of the first PCM sample. */ + @JvmField val dataOffset: Long, + /** PCM byte count; a zero-length data chunk (header never patched after a crash) is recovered from the file length. */ + @JvmField val dataBytes: Long, + ) + + /** + * Parses the fmt/data chunks from [prefix], the first bytes of a WAV file of [fileLength] + * bytes. Throws IllegalArgumentException for anything that is not 16-bit PCM WAV or whose + * data chunk header lies beyond the prefix. + */ + @JvmStatic + fun parseHeader(prefix: ByteArray, fileLength: Long): WavInfo { + require(prefix.size >= 12 && prefix[0] == 'R'.code.toByte() && prefix[1] == 'I'.code.toByte() && + prefix[2] == 'F'.code.toByte() && prefix[3] == 'F'.code.toByte() && + prefix[8] == 'W'.code.toByte() && prefix[9] == 'A'.code.toByte()) { "not a WAV file" } + var sampleRate = 0 + var channels = 0 + var bits = 0 + var offset = 12L + while (offset + 8 <= prefix.size) { + val at = offset.toInt() + val id = u32(prefix, at) + val size = u32(prefix, at + 4).toLong() and 0xffffffffL + offset += 8 + when (id) { + 0x20746d66 -> { // "fmt " + val available = minOf(size, 16L).toInt() + require(offset + available <= prefix.size) { "truncated fmt chunk" } + val f = offset.toInt() + val audioFormat = u16(prefix, f) + channels = u16(prefix, f + 2) + sampleRate = u32(prefix, f + 4) + bits = u16(prefix, f + 14) + require(audioFormat == 1 && bits == 16) { "only 16-bit PCM WAV is supported" } + offset += size + } + 0x61746164 -> { // "data" + require(sampleRate != 0) { "missing fmt or data chunk" } + val dataBytes = if (size > 0) size else fileLength - offset + return WavInfo(sampleRate, channels, bits, offset, maxOf(0L, dataBytes)) + } + else -> offset += size + } + } + throw IllegalArgumentException("missing fmt or data chunk") + } + + private fun u16(bytes: ByteArray, at: Int): Int = (bytes[at].toInt() and 0xff) or ((bytes[at + 1].toInt() and 0xff) shl 8) + + private fun u32(bytes: ByteArray, at: Int): Int = u16(bytes, at) or (u16(bytes, at + 2) shl 16) +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/evenhub/EvenHubAssetServer.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/evenhub/EvenHubAssetServer.kt new file mode 100644 index 00000000..8125528e --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/evenhub/EvenHubAssetServer.kt @@ -0,0 +1,94 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic + +/** + * Offline asset resolution for an unpacked EvenHub app served from a fake per-app origin. + * Shared by the Android WebViewClient interceptor and the iOS WKURLSchemeHandler so both apply + * the same path rules, MIME table and bridge-shim injection. + */ +object EvenHubAssetServer { + /** A resolved request: [relativePath] is root-relative with no leading slash. */ + data class Asset(val relativePath: String, val mime: String) { + val isHtml: Boolean + get() = mime == "text/html" + } + + /** A loaded response body. */ + class Response(val mime: String, val bytes: ByteArray) + + /** + * Maps a decoded URL path to a root-relative file path, or null when it escapes the root. + * Empty and "/" paths serve index.html. Segments are normalised lexically ("." and ".."); + * platform adapters must still canonicalise the final file to defeat symlinks. + */ + @JvmStatic + fun resolve(urlPath: String?): Asset? { + val path = if (urlPath.isNullOrEmpty() || urlPath == "/") "/index.html" else urlPath + val segments = ArrayList() + for (segment in path.split('/')) { + when (segment) { + "", "." -> continue + ".." -> if (segments.isEmpty()) return null else segments.removeAt(segments.size - 1) + else -> segments.add(segment) + } + } + if (segments.isEmpty()) return null + val relative = segments.joinToString("/") + return Asset(relative, mimeTypeFor(segments.last())) + } + + /** True when [canonicalFile] lies inside [canonicalRoot] (both already canonicalised, no trailing slash). */ + @JvmStatic + fun isInsideRoot(canonicalRoot: String, canonicalFile: String): Boolean = + canonicalFile == canonicalRoot || canonicalFile.startsWith(canonicalRoot.trimEnd('/') + "/") + + /** + * Inserts the bridge shim right after `` when present, else before everything, so + * window.flutter_inappwebview exists before any app script runs. + */ + @JvmStatic + fun injectIntoHtml(html: String, injectScript: String): String { + val tag = "" + val headIndex = html.lowercase().indexOf("= 0) { + val close = html.indexOf('>', headIndex) + if (close >= 0) return html.substring(0, close + 1) + tag + html.substring(close + 1) + } + return tag + html + } + + /** Loads [asset] under [rootDir] with the shim spliced into HTML; null when the file is unreadable. */ + @JvmStatic + fun load(rootDir: String, asset: Asset, injectScript: String): Response? { + val path = rootDir.trimEnd('/') + "/" + asset.relativePath + if (!fileExists(path)) return null + val bytes = try { readFileData(path) } catch (e: Exception) { return null } + if (!asset.isHtml) return Response(asset.mime, bytes) + return Response(asset.mime, injectIntoHtml(bytes.decodeToString(), injectScript).encodeToByteArray()) + } + + /** Extension-based MIME lookup (the union of the Android and iOS tables). */ + @JvmStatic + fun mimeTypeFor(name: String): String { + val lower = name.lowercase() + val dot = lower.lastIndexOf('.') + val ext = if (dot >= 0) lower.substring(dot + 1) else "" + return MIME_TYPES[ext] ?: "application/octet-stream" + } + + private val MIME_TYPES = mapOf( + "html" to "text/html", "htm" to "text/html", + "js" to "text/javascript", "mjs" to "text/javascript", + "css" to "text/css", + "json" to "application/json", + "wasm" to "application/wasm", + "svg" to "image/svg+xml", + "png" to "image/png", "jpg" to "image/jpeg", "jpeg" to "image/jpeg", "gif" to "image/gif", + "webp" to "image/webp", "ico" to "image/x-icon", + "woff" to "font/woff", "woff2" to "font/woff2", "ttf" to "font/ttf", "otf" to "font/otf", + "txt" to "text/plain", "map" to "text/plain", + "xml" to "application/xml", "mp3" to "audio/mpeg", "wav" to "audio/wav", "ogg" to "audio/ogg", + "mp4" to "video/mp4", "webm" to "video/webm", + ) +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/DeviceInfoProbeFlow.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/DeviceInfoProbeFlow.kt new file mode 100644 index 00000000..906ce7e6 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/DeviceInfoProbeFlow.kt @@ -0,0 +1,219 @@ +package com.faceclaw.app + +/** + * Connects to the glasses (stock-firmware compatible), reads the device-info / settings + * response, and reports the firmware versions plus the firmware-extension string (empty on + * stock firmware). Used by onboarding to decide whether to flash. Owns its own [StockLink] and + * runs [run] on the caller's worker thread; listener callbacks are synchronous on that thread + * (the platform adapter marshals them). Shows nothing on the lens. + */ +class DeviceInfoProbeFlow( + link: StockLink, + rightAddress: String, + leftAddress: String, + private val listener: FaceclawDeviceInfoProbeListener, + private val timings: StockFlowTimings = StockFlowTimings(), + platform: ProtocolPlatform = protocolPlatform(), +) { + companion object { + const val TAG = "FaceclawDeviceInfo" + + /** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */ + const val ARM_ATTEMPTS = 3 + } + + private val rightAddress = rightAddress.trim() + private val leftAddress = leftAddress.trim() + private val sleeper = InterruptibleSleep(platform) + private val session = StockLinkSession(link, ::emitLog, timings, traceFrames = true, platform = platform) + + val cancelled: Boolean + get() = session.cancelled + + fun cancel() { + session.cancel() + sleeper.interrupt() + } + + fun run() { + try { + if (rightAddress.isEmpty()) { + emitError("No right-arm address configured.") + return + } + + val haveLeft = leftAddress.isNotEmpty() && !leftAddress.equals(rightAddress, ignoreCase = true) + + // Bring up and authenticate BOTH arms before asking for versions. Each arm is its + // own peripheral with its own bond, so on a fresh phone each raises its own pairing + // prompt; completing both here means the flash step (and the app proper) find both + // bonds in place. Strictly one arm at a time: Android pairs with one device at a + // time, and a second arm connected while the first was still pairing has been seen + // to drop within a second and never pair (2026-09-11 logcat). + var rightAuthenticated = bringUpArm(rightAddress, "right") + var leftAuthenticated = false + if (haveLeft && !cancelled) { + leftAuthenticated = bringUpArm(leftAddress, "left") + } + if (cancelled) { + emitError("Cancelled.") + return + } + if (!session.isConnected(rightAddress)) { + // The right lens may have dropped while the left one paired; it is bonded by + // now, so this comes back quickly. + emitLog("right lens disconnected while the left lens was set up; reconnecting") + rightAuthenticated = bringUpArm(rightAddress, "right") + if (cancelled) { + emitError("Cancelled.") + return + } + } + + var ack: ByteArray? = null + var rightFailure: String? = null + try { + ack = queryArm(rightAddress, "right", rightAuthenticated) + } catch (e: IllegalStateException) { + rightFailure = StockLinkSession.messageOf(e) + emitLog("right-lens query failed: $rightFailure") + } + // The right lens is the documented control endpoint, but a silent right lens has + // been observed on stock 2.2.9 even after a successful security auth — run the same + // probe against the left lens rather than giving up, and log which lens answered. + if (ack == null && !cancelled && haveLeft) { + emitLog("right lens did not answer the settings query; probing the left lens") + try { + ack = queryArm(leftAddress, "left", leftAuthenticated) + } catch (e: Exception) { + emitLog("left-lens probe failed: " + StockLinkSession.messageOf(e)) + } + } + if (cancelled) { + emitError("Cancelled.") + return + } + if (ack == null) { + throw IllegalStateException( + rightFailure + ?: ("no response to the device-info query on either lens — check `adb logcat -s " + TAG + "` for the frame trace") + ) + } + + val info = BleProtocol.parseSettingsFirmwareInfo(ack) + val left = info?.leftVersion ?: "" + val right = info?.rightVersion ?: "" + val extension = info?.extension ?: "" + emitLog("device-info: L=$left R=$right ext=[$extension]") + listener.onResult(left, right, extension) + } catch (e: Exception) { + emitError(if (cancelled) "Cancelled." else StockLinkSession.messageOf(e)) + } finally { + session.closeQuietly() + } + } + + /** + * Connect to one arm and complete its security-auth exchange, retrying when the link drops + * mid-pairing. Returns whether the auth success was seen (false = connected but + * unconfirmed, tolerated because the custom firmware's response to the exchange is not yet + * hardware-verified); throws when the arm cannot be brought up at all. + */ + private fun bringUpArm(address: String, label: String): Boolean { + var lastFailure: String? = null + var attempt = 1 + while (attempt <= ARM_ATTEMPTS && !cancelled) { + if (attempt > 1) { + // A link the OS dropped while it was still pairing comes back once the bond + // settles; reconnecting sooner just fails again. + session.waitForBondToSettle(address, label, sleeper) + if (cancelled) { + break + } + emitLog("retrying the $label lens (attempt $attempt/$ARM_ATTEMPTS): $lastFailure") + sleeper.sleep(timings.armRetryDelayMs.toLong()) + } + listener.onState("connecting", label) + try { + session.bringUp(address) + } catch (e: IllegalStateException) { + lastFailure = StockLinkSession.messageOf(e) + emitLog("$label lens: $lastFailure") + attempt++ + continue + } + if (cancelled) { + break + } + + // Firmware 2.2.9 answers no queries until the security-auth exchange completes over + // an encrypted link; on a phone with no existing bond this is also what triggers SMP + // pairing (and its OS prompt), so it must come before the prelude and query. + listener.onState("authenticating", label) + when (session.authenticate(address, label)) { + StockLinkSession.AuthResult.SUCCESS -> return true + StockLinkSession.AuthResult.UNCONFIRMED -> return false + StockLinkSession.AuthResult.LINK_DROPPED -> { + lastFailure = "link dropped during authentication" + attempt++ + } + } + } + if (cancelled) { + return false + } + throw IllegalStateException( + "could not connect to the $label lens ($address): " + lastFailure + + " — if the phone showed a Bluetooth pairing request, accept it and try again" + ) + } + + /** + * Prelude + settings read on an already connected/authenticated lens. Returns the settings + * ack protobuf, an unsolicited settings push that carried firmware versions, or null when + * the lens never answered the read. Throws on prelude failure; the caller wraps the fallback + * lens's attempt so its failure cannot mask the primary lens's outcome. + */ + private fun queryArm(address: String, label: String, authenticated: Boolean): ByteArray? { + if (cancelled) { + return null + } + listener.onState("querying", label) + // Session prelude, then a settings/device-info read (both arms' versions and the + // firmware-extension string ride back in one response). + session.sendPrelude( + address, + " ($label lens)" + + (if (authenticated) "" else "; authentication did not complete — if the phone shows a Bluetooth pairing request, accept it and try again"), + ) + + // Two attempts: the first read straight after a fresh pairing has been seen to go + // unanswered while a later one succeeds. + var attempt = 0 + while (attempt < 2 && !cancelled) { + val ack = session.readSettings(address, timings.queryTimeoutMs) + if (ack != null) { + return ack + } + // A push with the firmware versions on the device's own magic is as good as the + // ack we asked for. + val pushed = session.unsolicitedSettingsPb + if (pushed != null) { + emitLog("using unsolicited settings push instead of the read ack ($label lens)") + return pushed + } + emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)") + attempt++ + } + return null + } + + // Log lines reach the platform log through the adapter's onLog (Android: logcat under TAG). + private fun emitLog(line: String) { + listener.onLog(line) + } + + private fun emitError(message: String) { + listener.onError(message) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FirmwareImage.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FirmwareImage.kt new file mode 100644 index 00000000..a1c62bce --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FirmwareImage.kt @@ -0,0 +1,167 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic + +/** + * Stock EVENOTA container layout, validation and CRC-32C, shared by the Android flasher and + * (via stage 3) the iOS port of app/g2/firmware-ota.ts. Mirrors g2flash.py: a 0x40-byte + * header (component count at +8), a 16-byte table-of-contents entry per component + * (offset at +4, CRC at +12), and per component a 128-byte subheader (payload size at +8, + * CRC at +12, NUL-terminated name at +48) followed by the payload. + * + * Validation takes the stricter of the two former implementations: the Android checks + * (component count, CRC in both TOC and subheader, main-app preamble/MRAM ceiling) plus the + * TypeScript ones (TOC/subheader/payload bounds, no overlapping components). + */ +object FirmwareImage { + const val HEADER_SIZE = 0x40 + const val TOC_ENTRY_SIZE = 16 + const val SUBHEADER_SIZE = 128 + const val NAME_OFFSET = 48 + const val NAME_MAX = 80 + + // Firmware 2.2.4 has 5 components; 2.2.6 has 6. + const val MIN_SEGMENTS = 5 + const val MAX_SEGMENTS = 6 + const val REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin" + const val APP_LOAD_ADDR = 0x00438000L + const val APP_MAX_END = 0x007F0000L + const val APP_PREAMBLE = 0x20 + + class Segment( + /** Offset of the 128-byte subheader within the image. */ + @JvmField val off: Int, + /** Payload size in bytes (payload starts at off + SUBHEADER_SIZE). */ + @JvmField val ps: Int, + /** CRC-32C recorded in the table of contents. */ + @JvmField val crc: Long, + @JvmField val name: String, + ) { + val payloadStart: Int get() = off + SUBHEADER_SIZE + val end: Int get() = payloadStart + ps + } + + private val CRC32C_TABLE: IntArray = IntArray(256).also { table -> + for (b in 0 until 256) { + var c = b shl 24 + for (i in 0 until 8) { + c = if ((c and 0x80000000.toInt()) != 0) (c shl 1) xor 0x1edc6f41 else c shl 1 + } + table[b] = c + } + } + + /** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */ + @JvmStatic + fun crc32cMsb(data: ByteArray, offset: Int = 0, length: Int = data.size - offset): Int { + require(offset >= 0 && length >= 0 && offset + length <= data.size) + var crc = 0 + for (i in offset until offset + length) { + crc = (crc shl 8) xor CRC32C_TABLE[((crc ushr 24) xor (data[i].toInt() and 0xff)) and 0xff] + } + return crc + } + + /** Unsigned CRC-32C as a Long, for comparison with recorded values. */ + @JvmStatic + fun crc32cUnsigned(data: ByteArray, offset: Int = 0, length: Int = data.size - offset): Long = + crc32cMsb(data, offset, length).toLong() and 0xffffffffL + + @JvmStatic + fun readU32(buf: ByteArray, offset: Int): Long { + if (offset < 0 || offset + 4 > buf.size) throw IllegalStateException("firmware header is truncated") + return (buf[offset].toLong() and 0xffL) or + ((buf[offset + 1].toLong() and 0xffL) shl 8) or + ((buf[offset + 2].toLong() and 0xffL) shl 16) or + ((buf[offset + 3].toLong() and 0xffL) shl 24) + } + + @JvmStatic + fun readCString(buf: ByteArray, offset: Int, maxLen: Int): String { + var end = offset + val limit = minOf(buf.size, offset + maxLen) + while (end < limit && buf[end].toInt() != 0) end++ + return buildString(end - offset) { for (i in offset until end) append((buf[i].toInt() and 0xff).toChar()) } + } + + /** Parses the table of contents without checking CRCs; throws on bounds violations. */ + @JvmStatic + fun parseSegments(img: ByteArray): List { + if (img.size < HEADER_SIZE) throw IllegalStateException("file is too small to be a firmware image") + val n = readU32(img, 8) + if (n <= 0 || n > 64) throw IllegalStateException("implausible component count $n (corrupt image?)") + val count = n.toInt() + val tocEnd = HEADER_SIZE + count * TOC_ENTRY_SIZE + if (tocEnd > img.size) throw IllegalStateException("table of contents runs past end of file") + val segs = ArrayList(count) + for (i in 0 until count) { + val base = HEADER_SIZE + i * TOC_ENTRY_SIZE + val crc = readU32(img, base + 12) + val offLong = readU32(img, base + 4) + if (offLong < tocEnd || offLong + SUBHEADER_SIZE > img.size) { + throw IllegalStateException("segment $i subheader is out of bounds") + } + val off = offLong.toInt() + val psLong = readU32(img, off + 8) + if (psLong <= 0 || off + SUBHEADER_SIZE + psLong > img.size) { + throw IllegalStateException("segment $i payload is truncated") + } + val name = readCString(img, off + NAME_OFFSET, NAME_MAX) + val seg = Segment(off, psLong.toInt(), crc, name) + for (other in segs) { + if (seg.off < other.end && other.off < seg.end) { + throw IllegalStateException("components ${other.name} and ${seg.name} overlap") + } + } + segs.add(seg) + } + return segs + } + + /** Full validation: layout, component count, CRCs, and the main-app preamble/MRAM check. */ + @JvmStatic + fun validate(img: ByteArray): List { + val segs = parseSegments(img) + if (segs.size < MIN_SEGMENTS || segs.size > MAX_SEGMENTS) { + throw IllegalStateException("expected $MIN_SEGMENTS-$MAX_SEGMENTS components, found ${segs.size}") + } + var main: Segment? = null + for (s in segs) { + val calc = crc32cUnsigned(img, s.payloadStart, s.ps) + val subCrc = readU32(img, s.off + 12) + if (calc != s.crc || calc != subCrc) { + throw IllegalStateException("component ${s.name} CRC32C is stale (image not checksum-fixed)") + } + if (REQUIRED_SEGMENT == s.name) { + if (main != null) throw IllegalStateException("required component $REQUIRED_SEGMENT is duplicated") + main = s + } + } + if (main == null) throw IllegalStateException("required component $REQUIRED_SEGMENT not found") + checkMainAppFitsMram(img, main) + return segs + } + + @JvmStatic + fun checkMainAppFitsMram(img: ByteArray, main: Segment) { + if (main.ps < APP_PREAMBLE) throw IllegalStateException("main-app payload is smaller than its preamble") + val loadAddr = readU32(img, main.payloadStart + 0x14) + val preLen = readU32(img, main.payloadStart) and 0xFFFFFFL + if (loadAddr != APP_LOAD_ADDR) { + throw IllegalStateException( + "main-app preamble load address is 0x" + loadAddr.toString(16) + + ", expected 0x" + APP_LOAD_ADDR.toString(16)) + } + if (preLen != main.ps.toLong()) { + throw IllegalStateException("main-app preamble length ($preLen) != staged payload size (${main.ps})") + } + val progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE + if (progEnd > APP_MAX_END) { + val over = progEnd - APP_MAX_END + throw IllegalStateException( + "main-app is too large: programmed region ends at 0x" + progEnd.toString(16) + + ", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)") + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FlashPromptFlow.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FlashPromptFlow.kt new file mode 100644 index 00000000..d1b5a3f8 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/FlashPromptFlow.kt @@ -0,0 +1,353 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile + +/** + * A minimal, stock-firmware-compatible flow used only to show the pre-flash confirmation + * prompt on the glasses and read back the user's Yes/No choice. It speaks only the stock subset + * — session prelude, a text container, a list container, and heartbeats. No images. + * + * Both arms are connected and security-authenticated up front, even though the prompt itself + * only needs the right arm (the lenses relay messages to each other, and acks/events only ever + * come from the right arm): on an unbonded phone the auth exchange is what raises the OS + * pairing prompt, and doing both here means both prompts appear at the start rather than one + * popping up half way through flashing the second lens. While connected, each arm's battery + * level is read (they are independent batteries) so the caller can refuse to flash on a low + * charge. + * + * With `skipPrompt` the on-glasses confirmation is not shown: the flow is just connect + auth + + * battery read + result(approved). Used to re-check the battery after the user has already + * confirmed once. [run] executes on the caller's worker thread; the heartbeat runs on its own + * thread while the prompt is up. + */ +class FlashPromptFlow( + link: StockLink, + rightAddress: String, + leftAddress: String, + warningText: String, + private val skipPrompt: Boolean, + private val listener: FaceclawFlashPromptListener, + private val timings: StockFlowTimings = StockFlowTimings(), + platform: ProtocolPlatform = protocolPlatform(), +) { + companion object { + const val TAG = "FaceclawFlashPrompt" + const val TEXT_NAME = "flashwarn" + const val LIST_NAME = "flashmenu" + const val TEXT_CONTAINER_ID = 1 + const val LIST_CONTAINER_ID = 2 + // Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid. + val ITEMS = arrayOf("No, cancel", "Yes, flash") + } + + private val rightAddress = rightAddress.trim() + private val leftAddress = leftAddress.trim() + private val warningText = warningText + private val platform = platform + private val lock = platform.createLock() + private val sleeper = InterruptibleSleep(platform) + private val selectionLatch = Latch(1, platform) + private val session = StockLinkSession(link, ::emitLog, timings, platform = platform) + + @Volatile + private var finished = false + + @Volatile + private var rightConnected = false + + @Volatile + private var leftConnected = false + + @Volatile + private var rightLost = false + + @Volatile + private var approved: Boolean? = null + + /** The running heartbeat's sleeper; interrupting it ends that thread. */ + @Volatile + private var heartbeat: InterruptibleSleep? = null + + val cancelled: Boolean + get() = session.cancelled + + init { + session.onEvent = { _, frame -> handleEvent(frame) } + session.onDisconnected = ::handleDisconnected + } + + /** Abort the prompt (e.g. the user backed out on the phone). */ + fun cancel() { + session.cancel() + selectionLatch.countDown() + sleeper.interrupt() + stopHeartbeat() + } + + fun run() { + try { + if (rightAddress.isEmpty()) { + listener.onState("error", "No right-arm address configured.") + return + } + + listener.onState("connecting", "") + connectArm(rightAddress, "right") + rightConnected = true + if (cancelled) { + teardown() + return + } + if (leftAddress.isNotEmpty()) { + connectArm(leftAddress, "left") + leftConnected = true + if (cancelled) { + teardown() + return + } + } + + listener.onState("connected", "") + // Auth both arms now (pairing prompts, if any, happen here) before anything else, + // so both bonds exist by the time flashing starts. + authenticateArm(rightAddress, "right") + if (leftConnected) { + authenticateArm(leftAddress, "left") + } + if (cancelled) { + teardown() + return + } + session.sendPrelude(rightAddress, ": $rightAddress") + + if (!skipPrompt) { + showPrompt(rightAddress) + startHeartbeat() + listener.onState("prompting", "") + + selectionLatch.await(timings.selectionTimeoutMs.toLong()) + stopHeartbeat() + + if (cancelled) { + listener.onState("cancelled", "") + teardown() + return + } + if (approved == null) { + if (rightLost) { + listener.onState("disconnected", "Lost connection to the glasses.") + } else { + listener.onState("timeout", "No response from the glasses.") + } + teardown() + return + } + try { + sendShutdown() + } catch (ignored: Exception) { + } + if (approved != true) { + listener.onResult(false) + listener.onState("result", "declined") + teardown() + return + } + } + + // Approved (or prompt skipped): read each arm's battery while the links are still + // up, then report the result. + listener.onState("battery", "") + readBatteries() + if (cancelled) { + listener.onState("cancelled", "") + teardown() + return + } + if (rightLost) { + listener.onState("disconnected", "Lost connection to the glasses.") + teardown() + return + } + finished = true + listener.onResult(true) + listener.onState("result", "approved") + teardown() + } catch (e: Exception) { + stopHeartbeat() + if (!cancelled) { + listener.onState("error", StockLinkSession.messageOf(e)) + } + teardown() + } + } + + private fun connectArm(address: String, arm: String) { + try { + session.bringUp(address) + } catch (e: IllegalStateException) { + throw IllegalStateException(StockLinkSession.messageOf(e) + ": $arm arm ($address)") + } + emitLog("connected $arm arm") + } + + /** + * Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a session (and closes + * the link after ~30 s) without it, and on an unbonded phone this is what triggers SMP + * pairing, so it may sit waiting on an OS pairing prompt. + */ + private fun authenticateArm(address: String, arm: String) { + if (session.authenticate(address, arm) != StockLinkSession.AuthResult.SUCCESS) { + throw IllegalStateException( + "could not authenticate with the $arm arm ($address" + + ") — if the phone shows a Bluetooth pairing request, accept it and try again" + ) + } + emitLog("security auth complete: $arm arm") + } + + private fun showPrompt(address: String) { + // Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. Two attempts: + // on 2.2.9 the first request sent right after the prelude can be dropped while the lens + // emits its own sid-0x80 notifications (observed with the device-info settings read). + var pageAck: ByteArray? = null + var attempt = 0 + while (attempt < 2 && pageAck == null && !cancelled) { + val magic = session.allocMagic() + val page = BleProtocol.buildCreatePromptPage(magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS) + pageAck = session.writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, timings.createAckTimeoutMs) + if (pageAck == null) { + emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address) + } + attempt++ + } + if (pageAck == null) { + throw IllegalStateException("prompt page not acked: $address") + } + emitLog("prompt page shown on $address") + } + + /** + * Read each arm's battery via a sid-0x09 settings read on that arm's own link (the arms + * have independent batteries and each answers with its own). An arm that does not answer + * reports -1; the caller decides what to do with a partial reading. + */ + private fun readBatteries() { + val right = if (rightConnected) readBattery(rightAddress, "right") else -1 + val left = if (leftConnected) readBattery(leftAddress, "left") else -1 + emitLog("battery R=" + (if (right < 0) "?" else "$right%") + " L=" + (if (left < 0) "?" else "$left%")) + listener.onBattery(right, left) + } + + private fun readBattery(address: String, arm: String): Int { + var attempt = 0 + while (attempt < 2 && !cancelled) { + val ack = session.readSettings(address, timings.batteryAckTimeoutMs) + if (ack == null) { + emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm") + attempt++ + continue + } + val snapshot = BleProtocol.parseSettingsBattery(ack) + if (snapshot != null) { + return snapshot.battery + } + emitLog("battery read ack had no battery field: $arm arm") + attempt++ + } + return -1 + } + + private fun startHeartbeat() { + stopHeartbeat() + val ticker = InterruptibleSleep(platform) + heartbeat = ticker + startThread("faceclaw-flash-hb", true) { + while (heartbeat === ticker && ticker.sleep(timings.heartbeatIntervalMs.toLong())) { + if (heartbeat !== ticker) break + sendHeartbeat() + } + } + } + + private fun sendHeartbeat() { + if (finished || cancelled) { + return + } + try { + val heartbeat = BleProtocol.buildHeartbeat(session.allocMagic()) + if (rightConnected) { + session.writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat) + } + } catch (e: Exception) { + emitLog("heartbeat failed: " + e.message) + } + } + + private fun stopHeartbeat() { + val ticker = heartbeat ?: return + heartbeat = null + ticker.interrupt() + } + + private fun sendShutdown() { + val shutdown = BleProtocol.buildShutdown(session.allocMagic(), 0) + if (rightConnected) { + session.writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown) + } + } + + private fun teardown() { + finished = true + stopHeartbeat() + session.closeQuietly() + rightConnected = false + leftConnected = false + } + + private fun handleEvent(frame: BleProtocol.ParsedFrame) { + val selection = BleProtocol.parseListSelection(frame) ?: return + if (LIST_NAME != selection.containerName) { + return + } + // Only a confirmed click is a decision; scroll/highlight changes are ignored. + if (selection.eventType != BleProtocol.EVENT_CLICK) { + return + } + val itemName = selection.itemName + val yes = selection.itemIndex == 1 || (itemName != null && itemName.lowercase().startsWith("yes")) + lock.withLock { + if (finished) { + return + } + finished = true + approved = yes + } + emitLog("selection: " + (if (yes) "flash" else "cancel") + " (index " + selection.itemIndex + ")") + selectionLatch.countDown() + } + + private fun handleDisconnected(address: String) { + if (address.equals(leftAddress, ignoreCase = true)) { + // The prompt itself only needs the right arm; a dropped left link just means no + // left battery reading. + leftConnected = false + emitLog("left arm disconnected") + return + } + if (address.equals(rightAddress, ignoreCase = true)) { + rightConnected = false + lock.withLock { + if (!finished && !cancelled) { + rightLost = true + finished = true + selectionLatch.countDown() + } + } + } + } + + // Log lines reach the platform log through the adapter's onLog (Android: logcat under TAG). + private fun emitLog(line: String) { + listener.onLog(line) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/OtaFlashFlow.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/OtaFlashFlow.kt new file mode 100644 index 00000000..e2f5ee58 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/OtaFlashFlow.kt @@ -0,0 +1,312 @@ +package com.faceclaw.app + +/** + * Ports g2flash.py's OTA flash procedure. Streams the EVENOTA firmware container to each lens + * over the firmware-data service using the aa21 envelope (BleProtocol.framePb) and the c0/c1 + * control/data protocol. + * + * Each lens connection completes the sid-0x80 security-auth exchange before OTA BEGIN — + * firmware 2.2.9 closes unauthenticated links ~30 s in, killing the transfer mid-component. No + * control-channel traffic is injected during the transfer itself: official OTA captures show + * none between BEGIN and the final END, heartbeats demonstrably do not satisfy the 2.2.9 + * deadline, and a concurrent control writer would interleave with the marker/data + * transactions. (See notes/ble-connections-2.2.9.md.) + * + * Lenses are flashed one at a time (left, then right). Finishing either lens reboots BOTH + * lenses, so the second lens is briefly unreachable — the reconnect for it uses a generous + * window rather than failing fast. [run] executes on the caller's worker thread. + */ +class OtaFlashFlow( + link: StockLink, + rightAddress: String, + leftAddress: String, + private val firmwarePath: String, + private val listener: FaceclawFirmwareFlasherListener, + private val timings: StockFlowTimings = StockFlowTimings(), + platform: ProtocolPlatform = protocolPlatform(), + /** Test seam: the firmware container bytes; defaults to reading [firmwarePath]. */ + private val readImage: (String) -> ByteArray = ::readFirmwareFile, +) { + companion object { + const val TAG = "FaceclawFlasher" + + // OTA message types (envelope sid byte) and control opcodes. + const val SID_CTRL = 0xc0 + const val SID_DATA = 0xc1 + const val FLAG_OTA = 0x00 + const val OP_BEGIN = 0x00 + const val OP_FILE_CHECK = 0x01 + const val OP_BLOCK = 0x02 + const val OP_END = 0x03 + + const val BLOCK_SIZE = 4096 + const val BLOCK_NAK_RETRIES = 3 + const val COMPONENT_RETRIES = 3 + + // END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART. + private val END_OK = intArrayOf(0, 8, 9) + private val EMPTY = ByteArray(0) + + fun isEndOk(status: Int): Boolean = END_OK.contains(status) + + private fun readFirmwareFile(path: String): ByteArray = + try { + readFileData(path) + } catch (e: Exception) { + throw IllegalStateException("could not read firmware: " + e.message, e) + } + } + + private val rightAddress = rightAddress.trim() + private val leftAddress = leftAddress.trim() + private val sleeper = InterruptibleSleep(platform) + // The auth exchange uses its own magic range (0x60..0x7f) and the OTA envelope counter + // starts at 1, mirroring the stock sequence. + private val session = StockLinkSession(link, ::emitLog, timings, magicStart = 0x60, magicEnd = 0x7f, seqStart = 1, platform = platform) + + val cancelled: Boolean + get() = session.cancelled + + fun cancel() { + session.cancel() + sleeper.interrupt() + } + + fun run() { + try { + if (leftAddress.isEmpty() || rightAddress.isEmpty()) { + throw IllegalStateException("Both lens addresses are required to flash.") + } + + listener.onState("validating", "") + val img = readImage(firmwarePath) + val segs = FirmwareImage.validate(img) + emitLog("firmware validated: " + segs.size + " components, " + img.size + " bytes") + + flashLens("left", leftAddress, img, segs, timings.firstLensConnectWindowMs) + if (cancelled) { + listener.onState("error", "Cancelled.") + listener.onComplete(false, "Cancelled after the left lens.") + return + } + + listener.onState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens.") + sleeper.sleep(timings.rebootSettleMs.toLong()) + flashLens("right", rightAddress, img, segs, timings.secondLensConnectWindowMs) + + listener.onState("done", "") + listener.onComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware.") + } catch (e: Exception) { + val message = if (cancelled) "Cancelled." else StockLinkSession.messageOf(e) + listener.onState("error", message) + listener.onComplete(false, message) + } finally { + session.closeQuietly() + } + } + + private fun flashLens(lens: String, address: String, img: ByteArray, segs: List, connectWindowMs: Int) { + listener.onState("connecting", lens) + connectLensResilient(lens, address, connectWindowMs) + sleeper.sleep(timings.notifySettleMs.toLong()) + session.otaAcks.clear() + // Fresh envelope counter for the OTA stream (the auth exchange during bring-up used its + // own); mirrors the stock sequence. + session.resetSeq(1) + + listener.onState("flashing", lens) + val beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, timings.ctrlAckTimeoutMs) + if (!isEndOk(beginStatus)) { + emitLog("warning: unexpected begin status $beginStatus; continuing") + } + // Progress is reported in bytes across the whole image: the segments are one large + // firmware blob plus several small ones, so a bar stepped per segment would sit still + // through the big one and then sprint through the rest. + var totalBytes = 0L + for (seg in segs) { + totalBytes += seg.ps + } + var bytesBefore = 0L + for (i in segs.indices) { + if (cancelled) { + throw IllegalStateException("Cancelled.") + } + flashComponentWithRetry(lens, address, i, segs.size, segs[i], img, bytesBefore, totalBytes) + bytesBefore += segs[i].ps + } + emitLog("$lens lens: all components verified") + session.disconnectQuietly(address) + } + + private fun flashComponentWithRetry( + lens: String, + address: String, + index: Int, + count: Int, + seg: FirmwareImage.Segment, + img: ByteArray, + bytesBefore: Long, + totalBytes: Long, + ) { + for (attempt in 0 until COMPONENT_RETRIES) { + if (attempt > 0) { + emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES) + } + var endStatus: Int + try { + endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes) + } catch (e: StockTimeoutException) { + emitLog(seg.name + ": block phase failed: " + e.message) + endStatus = -1 + } catch (e: RuntimeException) { + emitLog(seg.name + ": block phase failed: " + e.message) + endStatus = -1 + } + if (isEndOk(endStatus)) { + emitLog(seg.name + ": END verify OK (status " + endStatus + ")") + return + } + if (endStatus >= 0) { + emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")") + } + session.otaAcks.clear() + sleeper.sleep(timings.componentRetryDelayMs.toLong()) + } + throw IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts") + } + + private fun flashComponent( + lens: String, + address: String, + index: Int, + count: Int, + seg: FirmwareImage.Segment, + img: ByteArray, + bytesBefore: Long, + totalBytes: Long, + ): Int { + val sub = img.copyOfRange(seg.off, seg.off + FirmwareImage.SUBHEADER_SIZE) + val ps = seg.ps + val payloadStart = seg.off + FirmwareImage.SUBHEADER_SIZE + + val checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, timings.ctrlAckTimeoutMs) + if (checkStatus != 0) { + throw IllegalStateException("FILE_CHECK rejected status=$checkStatus") + } + + val blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE + for (b in 0 until blockCount) { + if (cancelled) { + throw IllegalStateException("Cancelled.") + } + val start = payloadStart + b * BLOCK_SIZE + val end = minOf(start + BLOCK_SIZE, payloadStart + ps) + val block = img.copyOfRange(start, end) + + var accepted = false + for (tries in 0 until BLOCK_NAK_RETRIES) { + val status = sendBlock(address, block) // throws StockTimeoutException -> component re-flash + if (status == 0) { + accepted = true + break + } + emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status + " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES) + } + if (!accepted) { + throw IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times") + } + if (b % 20 == 0 || b == blockCount - 1) { + val bytesSent = bytesBefore + minOf((b + 1).toLong() * BLOCK_SIZE, ps.toLong()) + listener.onProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes) + } + } + emitLog(seg.name + ": data phase done; sending END") + return sendCtrlAndWait(address, OP_END, EMPTY, timings.ctrlAckTimeoutMs) + } + + /** Send one 4 KB block as a marker + data pair sharing one envelope seq. */ + private fun sendBlock(address: String, block: ByteArray): Int { + val seq = session.nextSeq() + session.otaAcks.clear() + writeOta(address, SID_CTRL, byteArrayOf(OP_BLOCK.toByte()), seq) + writeOta(address, SID_DATA, block, seq) + return waitAck(OP_BLOCK, timings.blockAckTimeoutMs) + } + + private fun sendCtrlAndWait(address: String, op: Int, data: ByteArray, timeoutMs: Int): Int { + val seq = session.nextSeq() + session.otaAcks.clear() + val payload = ByteArray(1 + data.size) + payload[0] = op.toByte() + data.copyInto(payload, 1) + writeOta(address, SID_CTRL, payload, seq) + return waitAck(op, timeoutMs) + } + + private fun writeOta(address: String, sid: Int, payload: ByteArray, seq: Int): Boolean = + session.writeFrame(address, BleProtocol.OTA_DATA_WRITE_UUID, sid, FLAG_OTA, payload, seq) + + private fun waitAck(wantOp: Int, timeoutMs: Int): Int { + val deadline = session.now() + timeoutMs + while (true) { + val remaining = deadline - session.now() + if (remaining <= 0 || cancelled) { + break + } + val pb = session.otaAcks.poll(remaining) ?: break + if (pb.size >= 2 && (pb[0].toInt() and 0xff) == wantOp) { + return pb[1].toInt() and 0xff + } + // Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting. + } + throw StockTimeoutException("no ack op=0x" + wantOp.toString(16) + " within " + timeoutMs + "ms") + } + + // ---- connection ---------------------------------------------------------- + + private fun connectLensResilient(lens: String, address: String, windowMs: Int) { + val deadline = session.now() + windowMs + var lastError: String? = "unknown" + var attempt = 0 + while (session.now() < deadline && !cancelled) { + attempt++ + try { + if (bringUpLens(address)) { + emitLog("connected $lens lens (attempt $attempt)") + return + } + lastError = "connect/discover incomplete" + } catch (e: Exception) { + lastError = StockLinkSession.messageOf(e) + } + emitLog("$lens connect attempt $attempt failed ($lastError); retrying...") + session.disconnectQuietly(address) + sleeper.sleep(timings.otaRetryDelayMs.toLong()) + } + if (cancelled) { + throw IllegalStateException("Cancelled.") + } + throw IllegalStateException("could not reach $lens lens: $lastError") + } + + private fun bringUpLens(address: String): Boolean { + try { + // Control-channel notify carries the auth response; the OTA notify carries the acks. + session.bringUp(address, otaChannel = true) + } catch (e: IllegalStateException) { + return false + } + // Complete the sid-0x80 security-auth exchange on this connection. On an unbonded + // phone this triggers SMP pairing (possibly with an OS prompt), so the wait is + // generous. Must succeed before any OTA traffic. + if (session.authenticate(address, "ota") != StockLinkSession.AuthResult.SUCCESS) { + throw IllegalStateException("security auth not acknowledged — if the phone shows a Bluetooth pairing request, accept it") + } + return true + } + + // Log lines reach the platform log through the adapter's onLog (Android: logcat under TAG). + private fun emitLog(line: String) { + listener.onLog(line) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ProtocolPlatform.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ProtocolPlatform.kt index ab7691f0..c9189e18 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ProtocolPlatform.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ProtocolPlatform.kt @@ -8,6 +8,34 @@ interface ProtocolPlatform { fun createLock(): ProtocolLock fun createDeflater(): ProtocolDeflater + + /** A mutex plus condition variable for blocking waits (see [ProtocolCondition]). */ + fun createCondition(): ProtocolCondition +} + +/** + * Mutex + condition variable. NOT reentrant: iOS backs it with NSCondition, so a thread that + * already holds the lock must not lock it again (Android's implementation would tolerate it + * and hide the bug). Hold the lock while calling [awaitMs] and [signalAll]. + */ +interface ProtocolCondition { + fun lock() + + fun unlock() + + /** Waits up to [timeoutMs] (spurious wake-ups possible); the lock is held again on return. */ + fun awaitMs(timeoutMs: Long) + + fun signalAll() +} + +inline fun ProtocolCondition.withLock(action: () -> T): T { + lock() + try { + return action() + } finally { + unlock() + } } /** Must be reentrant: transport encoding can reset its history under the lock. */ diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/StockLink.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/StockLink.kt new file mode 100644 index 00000000..b7a200f2 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/StockLink.kt @@ -0,0 +1,91 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField + +/** Android bond state of a peripheral; iOS reports UNKNOWN (pairing is invisible to the app). */ +enum class BondState { + NONE, + BONDING, + BONDED, + UNKNOWN, +} + +/** Inbound side of a [StockLink]; called on whatever thread the platform stack delivers on. */ +interface StockLinkListener { + fun onNotification(address: String, characteristicUuid: String, data: ByteArray) + + fun onConnectionStateChange(address: String, connected: Boolean) +} + +/** + * The GATT boundary used by the stock-firmware flows (device-info probe, flash prompt, OTA + * flasher). Calls block on the caller's thread with explicit timeouts; addresses are opaque + * strings (Android MAC, iOS peripheral identifier). Android wraps FaceclawBleManager; an iOS + * implementation sits on CoreBluetooth. Methods other than [isConnected]/[bondState]/ + * [disconnect]/[close] may throw IllegalStateException when the link is not connected. + */ +interface StockLink { + fun setListener(listener: StockLinkListener?) + + fun connect(address: String, timeoutMs: Int): Boolean + + /** Best-effort link tuning after connect (connection priority, MTU). */ + fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int) + + fun discoverServices(address: String, timeoutMs: Int): Boolean + + fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean + + fun writeFrames( + address: String, + characteristicUuid: String, + frames: List, + mode: GattWriteMode, + timeoutMs: Int, + ): Boolean + + fun isConnected(address: String): Boolean + + fun bondState(address: String): BondState + + fun disconnect(address: String) + + fun close() +} + +/** A wait for a lens reply expired. */ +class StockTimeoutException(message: String) : Exception(message) + +/** + * Timeouts and pacing shared by the stock flows. Production uses the defaults; tests shrink + * them. Values are milliseconds unless named otherwise. + */ +class StockFlowTimings( + @JvmField val connectTimeoutMs: Int = ConnectionOptions.CONNECT_TIMEOUT_MS, + @JvmField val servicesTimeoutMs: Int = ConnectionOptions.SERVICES_TIMEOUT_MS, + @JvmField val descriptorTimeoutMs: Int = ConnectionOptions.DESCRIPTOR_TIMEOUT_MS, + @JvmField val writeTimeoutMs: Int = ConnectionOptions.WRITE_TIMEOUT_MS, + @JvmField val preludeTimeoutMs: Int = ConnectionOptions.PRELUDE_TIMEOUT_MS, + @JvmField val securityAuthTimeoutMs: Int = ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS, + @JvmField val securityAuthSoftTimeoutMs: Int = ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS, + /** Upper bound on waiting while the OS reports pairing in progress (a dialog may be up). */ + @JvmField val pairingWaitCapMs: Int = 90_000, + /** After a fresh bond, how long the firmware gets to notify success before the auth request is re-sent. */ + @JvmField val postBondGraceMs: Int = 2_000, + @JvmField val pollMs: Int = 250, + /** Delay between connect/auth attempts on one arm and between auth re-sends. */ + @JvmField val armRetryDelayMs: Int = 1_000, + @JvmField val queryTimeoutMs: Int = 4_000, + @JvmField val heartbeatIntervalMs: Int = 4_000, + @JvmField val selectionTimeoutMs: Int = 120_000, + @JvmField val createAckTimeoutMs: Int = 3_000, + @JvmField val batteryAckTimeoutMs: Int = 3_000, + @JvmField val blockAckTimeoutMs: Int = 4_000, + @JvmField val ctrlAckTimeoutMs: Int = 8_000, + @JvmField val firstLensConnectWindowMs: Int = 30_000, + @JvmField val secondLensConnectWindowMs: Int = 120_000, + @JvmField val rebootSettleMs: Int = 5_000, + @JvmField val notifySettleMs: Int = 2_500, + @JvmField val otaRetryDelayMs: Int = 2_500, + @JvmField val componentRetryDelayMs: Int = 1_500, +) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/StockLinkSession.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/StockLinkSession.kt new file mode 100644 index 00000000..d83adeab --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/StockLinkSession.kt @@ -0,0 +1,443 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile + +/** + * The stock-firmware request/reply plumbing that the device-info probe, the flash prompt and + * the OTA flasher all need on top of a [StockLink]: connect + subscribe, envelope sequence and + * magic allocation, frame demux, ack matching keyed by (sid, magic), the bond-aware security + * auth exchange (only the firmware's SUCCESS satisfies it), the unsolicited settings capture, + * async-event and disconnect hooks, and the OTA ack queue. Blocking calls run on the owning + * flow's worker thread; notifications arrive on the platform's callback thread. + */ +class StockLinkSession( + private val link: StockLink, + private val log: (String) -> Unit, + private val timings: StockFlowTimings = StockFlowTimings(), + magicStart: Int = 100, + private val magicEnd: Int = 255, + seqStart: Int = 0x40, + /** Log every parsed control frame with a payload prefix (the probe's 2.2.9 diagnostics). */ + private val traceFrames: Boolean = false, + private val platform: ProtocolPlatform = protocolPlatform(), +) : StockLinkListener { + enum class AuthResult { + SUCCESS, + UNCONFIRMED, + LINK_DROPPED, + } + + private val lock = platform.createLock() + private val magicStart = magicStart + private var nextMagic = magicStart + private var nextSeq = seqStart + private val pendingAcks = HashMap() + private val ackPayloads = HashMap() + private val authMagics = HashSet() + private var authLatch: Latch? = null + + /** Latest sid-0x09 frame carrying firmware versions, whatever magic it came with. */ + @Volatile + var unsolicitedSettingsPb: ByteArray? = null + private set + + /** OTA acks (first two payload bytes: opcode, status) from the data-notify characteristic. */ + val otaAcks = BlockingQueue(platform) + + /** Async events (flag 0x01/0x06) on the control characteristic, e.g. list selections. */ + @Volatile + var onEvent: ((address: String, frame: BleProtocol.ParsedFrame) -> Unit)? = null + + @Volatile + var onDisconnected: ((address: String) -> Unit)? = null + + @Volatile + var cancelled = false + private set + + init { + link.setListener(this) + } + + val platformServices: ProtocolPlatform + get() = platform + + fun now(): Long = platform.elapsedRealtimeMs() + + /** Wake every pending wait; subsequent waits return immediately as timeouts. */ + fun cancel() { + cancelled = true + lock.withLock { + for (latch in pendingAcks.values) latch.countDown() + authLatch?.countDown() + } + } + + fun allocMagic(): Int = + lock.withLock { + val magic = nextMagic + nextMagic = if (nextMagic >= magicEnd) magicStart else nextMagic + 1 + magic + } + + fun nextSeq(): Int = + lock.withLock { + val seq = nextSeq and 0xff + nextSeq = (nextSeq + 1) and 0xff + seq + } + + fun resetSeq(value: Int) { + lock.withLock { nextSeq = value and 0xff } + } + + /** Connect, tune, discover and subscribe to the control notify (plus the OTA notify when asked). Throws on failure. */ + fun bringUp(address: String, otaChannel: Boolean = false) { + if (!link.connect(address, timings.connectTimeoutMs)) { + throw IllegalStateException("connect failed") + } + link.prepareLink(address, ConnectionOptions.DESIRED_MTU, timings.connectTimeoutMs) + if (!link.discoverServices(address, timings.servicesTimeoutMs)) { + throw IllegalStateException("service discovery failed") + } + if (!link.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, timings.descriptorTimeoutMs)) { + throw IllegalStateException("could not subscribe to notifications") + } + if (otaChannel && !link.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, timings.descriptorTimeoutMs)) { + throw IllegalStateException("could not subscribe to OTA notifications") + } + } + + fun isConnected(address: String): Boolean = link.isConnected(address) + + fun disconnectQuietly(address: String) { + try { + link.disconnect(address) + } catch (ignored: Exception) { + } + } + + fun closeQuietly() { + try { + link.close() + } catch (ignored: Exception) { + } + } + + /** Frame and write one message on the control characteristic; false when the write failed. */ + fun writeFrame(address: String, sid: Int, flag: Int, payload: ByteArray): Boolean = + writeFrame(address, BleProtocol.WRITE_CHAR_UUID, sid, flag, payload, nextSeq()) + + /** @throws IllegalStateException when the link reports "not connected". */ + fun writeFrame(address: String, characteristicUuid: String, sid: Int, flag: Int, payload: ByteArray, seq: Int): Boolean { + val frames = BleProtocol.framePb(payload, sid, flag, seq) + return link.writeFrames(address, characteristicUuid, frames, ConnectionOptions.WRITE_MODE, timings.writeTimeoutMs) + } + + /** + * Write a request and wait for the ack carrying the same sid and magic. Returns the ack's + * protobuf (possibly empty), or null on write failure / timeout / cancel. + */ + fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? { + val key = ackKey(sid, magic) + val latch = Latch(1, platform) + lock.withLock { + pendingAcks[key] = latch + ackPayloads.remove(key) + } + try { + val seq = nextSeq() + log( + "tx " + address + " sid=0x" + hex2(sid) + " flag=0x" + hex2(flag) + " magic=" + magic + + " seq=0x" + hex2(seq) + " len=" + payload.size + ) + val written = + try { + writeFrame(address, BleProtocol.WRITE_CHAR_UUID, sid, flag, payload, seq) + } catch (e: IllegalStateException) { + // "Not connected" — the link went away underneath us. + false + } + if (!written) { + log("tx write FAILED sid=0x" + sid.toString(16) + " (" + address + ")") + return null + } + if (!latch.await(timeoutMs.toLong()) || cancelled) { + return null + } + return lock.withLock { ackPayloads[key] ?: ByteArray(0) } + } finally { + lock.withLock { + if (pendingAcks[key] === latch) pendingAcks.remove(key) + ackPayloads.remove(key) + } + } + } + + /** Mandatory session prelude on sid=0x01; throws when it is not acked. */ + fun sendPrelude(address: String, failureDetail: String = "") { + if (writeAndAwaitAck( + address, + BleProtocol.PRELUDE_ACK_SID, + BleProtocol.FLAG_REQUEST, + BleProtocol.PRELUDE_ACK_MAGIC, + BleProtocol.PRELUDE_F5872_PAYLOAD, + timings.preludeTimeoutMs, + ) == null + ) { + throw IllegalStateException("session prelude not acked$failureDetail") + } + } + + /** One sid-0x09 settings read; the ack protobuf or null when unanswered. */ + fun readSettings(address: String, timeoutMs: Int): ByteArray? { + val magic = allocMagic() + return writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, magic, BleProtocol.buildSettingsQuery(magic), timeoutMs) + } + + /** + * Send the sid-0x80 authentication request and wait for the firmware's SUCCESS notification, + * which only arrives once the link is encrypted. On an unbonded phone that means waiting + * through the OS pairing flow (BOND_BONDING, possibly with a dialog up), so the wait is + * extended while the OS reports pairing in progress and the request is re-sent once after a + * fresh bond in case the pre-pairing write was dropped. Bond state is polled; 250 ms + * granularity is plenty here. Platforms that cannot see bonding (UNKNOWN) get the plain + * timeout. The first reply with a non-success result is expected before encryption and does + * not satisfy the wait. + */ + fun authenticate(address: String, label: String): AuthResult { + val start = now() + var deadline = start + timings.securityAuthTimeoutMs + val hardDeadline = start + timings.pairingWaitCapMs + val latch = Latch(1, platform) + lock.withLock { + authMagics.clear() + authLatch = latch + } + val initialBond = link.bondState(address) + log("security auth: $label lens, bond state " + bondStateName(initialBond)) + try { + var sends = 0 + var lastSendFailed = false + var nextSendAt = start + var bondedAt = -1L + var resentAfterBond = false + var loggedBonding = false + while (!cancelled) { + val now = now() + val bond = link.bondState(address) + + if (bond == BondState.BONDING) { + // OS pairing in progress — the firmware will answer once the link is + // encrypted. Don't write, don't give up, don't move on to the other arm. + if (!loggedBonding) { + log("the OS is pairing with the $label lens; waiting for the user to accept") + loggedBonding = true + } + if (now >= hardDeadline) { + break + } + } else { + if (bond == BondState.BONDED && initialBond != BondState.BONDED && bondedAt < 0) { + bondedAt = now + log("bond established with the $label lens; waiting for the auth success") + deadline = maxOf(deadline, now + timings.securityAuthSoftTimeoutMs) + } + val wantSend = sends == 0 || + (lastSendFailed && sends < 3 && now >= nextSendAt) || + (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= timings.postBondGraceMs) + if (wantSend) { + if (bondedAt >= 0) { + resentAfterBond = true + if (sends > 0) { + log("re-sending the auth request after pairing ($label lens)") + } + } + lastSendFailed = !sendAuthRequest(address) + sends++ + nextSendAt = now() + timings.armRetryDelayMs + if (lastSendFailed && !link.isConnected(address)) { + return AuthResult.LINK_DROPPED + } + } + if (now >= deadline) { + break + } + } + + if (latch.await(timings.pollMs.toLong()) && !cancelled) { + log("security auth complete: $label lens ($address)") + return AuthResult.SUCCESS + } + if (!link.isConnected(address)) { + log( + "security auth: $label lens disconnected while waiting" + + (if (bond == BondState.BONDING) " (the OS was still pairing)" else "") + ) + return AuthResult.LINK_DROPPED + } + } + if (cancelled) { + return AuthResult.UNCONFIRMED + } + log("security auth unconfirmed: $label lens ($address), bond state " + bondStateName(link.bondState(address))) + return AuthResult.UNCONFIRMED + } finally { + lock.withLock { + authLatch = null + authMagics.clear() + } + } + } + + private fun sendAuthRequest(address: String): Boolean { + val magic = allocMagic() + lock.withLock { authMagics.add(magic) } + val seq = nextSeq() + val payload = BleProtocol.buildAuthenticationRequest(magic) + log( + "tx " + address + " sid=0x" + hex2(BleProtocol.SID_SECURITY_AUTH) + " flag=0x" + hex2(BleProtocol.FLAG_SECURITY_AUTH) + + " magic=" + magic + " seq=0x" + hex2(seq) + " len=" + payload.size + ) + val written = + try { + writeFrame(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, payload, seq) + } catch (e: IllegalStateException) { + false + } + if (!written) { + log("tx write FAILED sid=0x80 ($address)") + } + return written + } + + /** Block while the OS reports this arm as BONDING (pairing dialog up / SMP in flight). */ + fun waitForBondToSettle(address: String, label: String, sleeper: InterruptibleSleep) { + val cap = now() + timings.pairingWaitCapMs + var logged = false + while (!cancelled && link.bondState(address) == BondState.BONDING && now() < cap) { + if (!logged) { + log("waiting for the OS to finish pairing with the $label lens before reconnecting") + logged = true + } + sleeper.sleep(timings.pollMs.toLong()) + } + } + + // ---- inbound -------------------------------------------------------------- + + override fun onNotification(address: String, characteristicUuid: String, data: ByteArray) { + if (BleProtocol.OTA_DATA_NOTIFY_UUID.equals(characteristicUuid, ignoreCase = true)) { + val frame = BleProtocol.parseFrame(data) + if (frame.ok && frame.pb.size >= 2) { + otaAcks.put(frame.pb.copyOf(minOf(frame.pb.size, 2))) + } + return + } + if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) { + return + } + // One notification value can carry several envelope frames back to back; reading only + // the first would silently drop the rest. + val frames = BleProtocol.splitFrames(data) + if (traceFrames && frames.size > 1) { + log("rx " + address + " value carries " + frames.size + " frames (raw " + data.size + " bytes)") + } + for (buf in frames) { + handleFrame(address, buf, data.size) + } + } + + private fun handleFrame(address: String, buf: ByteArray, rawValueLength: Int) { + val frame = BleProtocol.parseFrame(buf) + if (!frame.ok) { + if (traceFrames) { + log( + "rx " + address + " unparseable frame len=" + buf.size + " (raw value " + rawValueLength + ")" + + " head=" + BinaryEncoding.bytesToHex(buf.copyOf(minOf(16, buf.size))) + ) + } + return + } + if (traceFrames) { + // Log every control frame while diagnosing 2.2.9: sid/flag/type/magic plus a + // payload prefix is enough to reconstruct what the lens said. + val declared = if (buf.size > 3) buf[3].toInt() and 0xff else 0 + val truncated = if (buf.size < 8 + declared) " TRUNCATED(declared=$declared)" else "" + log( + "rx " + address + " sid=0x" + hex2(frame.sid) + " flag=0x" + hex2(frame.flag) + " type=" + frame.msgType + + " magic=" + frame.msgSeq + " frag=" + (if (buf.size > 5) buf[5].toInt() and 0xff else 0) + "/" + + (if (buf.size > 4) buf[4].toInt() and 0xff else 0) + " len=" + frame.pb.size + truncated + + " pb=" + BinaryEncoding.bytesToHex(frame.pb.copyOf(minOf(48, frame.pb.size))) + ) + } + if (frame.sid == BleProtocol.SID_UI_SETTING && BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) { + // Any settings frame carrying firmware versions answers the probe's question, + // whether or not it matches the magic we asked with. + unsolicitedSettingsPb = frame.pb + } + if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { + // Async event, not an ack. This includes the lens's own periodic sid-0x80 notifies + // (device-chosen magic, non-empty result). + onEvent?.invoke(address, frame) + return + } + if (frame.sid == BleProtocol.SID_SECURITY_AUTH) { + var success = false + var ours = false + lock.withLock { + val currentLatch = authLatch + if (currentLatch != null) { + ours = authMagics.contains(frame.msgSeq) + for (magic in authMagics) { + if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) { + success = true + currentLatch.countDown() + break + } + } + } + } + if (ours && !success) { + // Expected before the link is encrypted: the firmware answers the request + // straight away with a non-success result and sends the real success once + // pairing/encryption completes. + log("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting") + } + return + } + if (frame.msgSeq >= 0) { + val key = ackKey(frame.sid, frame.msgSeq) + lock.withLock { + val latch = pendingAcks[key] + if (latch != null) { + ackPayloads[key] = frame.pb + latch.countDown() + } + } + } + } + + override fun onConnectionStateChange(address: String, connected: Boolean) { + if (!connected) { + log("disconnected: $address") + onDisconnected?.invoke(address) + } + } + + companion object { + private fun ackKey(sid: Int, magic: Int): String = "$sid:$magic" + + internal fun hex2(value: Int): String = (value and 0xff).toString(16).padStart(2, '0') + + internal fun bondStateName(state: BondState): String = + when (state) { + BondState.NONE -> "none" + BondState.BONDING -> "bonding" + BondState.BONDED -> "bonded" + BondState.UNKNOWN -> "unknown" + } + + internal fun messageOf(e: Throwable): String = e.message ?: e.toString() + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionControls.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionControls.kt new file mode 100644 index 00000000..7694eb3a --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionControls.kt @@ -0,0 +1,695 @@ +package com.faceclaw.app + +import com.faceclaw.app.GlassesSessionCore.Companion.DASHBOARD_TILE +import com.faceclaw.app.GlassesSessionCore.Companion.FACECLAW_WAKE_CONTROL_WAIT_MS +import com.faceclaw.app.GlassesSessionCore.Companion.TAG + +/* + * GlassesSessionCore, part 3: CFW lease controls, EvenHub shutdown, the bandwidth + * benchmark stream, transport-failure handling and the listener emits (all + * marshalled through SessionHost.postToMain, or a subscriber's own dispatcher + * for compass events). + */ + +// --------------------------------------------------------------------------- +// Bandwidth benchmark + +/** Pending + in-flight benchmark no-op messages. */ +internal fun GlassesSessionCore.benchmarkOutstandingLocked(): Int { + var count = 0 + for (message in pendingMessages) { + if ("bandwidth" == message.kind) count++ + } + for (message in inFlightMessages) { + if ("bandwidth" == message.kind) count++ + } + return count +} + +/** + * Keep the benchmark stream fed: top the pending queue up so the send + * window never starves, stop enqueueing once the run expires (or a message + * times out), and finish the run when the last outstanding message drains. + */ +internal fun GlassesSessionCore.maintainBenchmarkLocked(now: Long) { + if (!sessionReady || !fixedLayoutCreated || shutdownRequested) { + finishBenchmarkLocked(true, "session no longer ready") + return + } + if (benchmarkLinkPending || now < benchmarkReadyAtMs) { + return + } + if (benchmarkAborted || now >= benchmarkDeadlineAtMs) { + // The run is over: drop queued-but-unsent no-ops (sending them + // would stretch the run past its deadline) and finish once the + // in-flight tail has acked or timed out. + val pendingIterator = pendingMessages.iterator() + while (pendingIterator.hasNext()) { + val queued = pendingIterator.next() + if ("bandwidth" == queued.kind) { + pendingIterator.remove() + magicPool.release(queued.sid, queued.magic, queued.label, "benchmark over") + } + } + if (benchmarkOutstandingLocked() == 0) { + finishBenchmarkLocked(false, "complete") + } + return + } + // One more than the window so a fresh message is always ready to write + // the moment an ack frees a slot. + var outstanding = benchmarkOutstandingLocked() + while (outstanding <= benchmarkWindowSize) { + enqueueBenchmarkMessageLocked() + outstanding++ + } +} + +internal fun GlassesSessionCore.finishBenchmarkLocked(aborted: Boolean, reason: String) { + if (!benchmarkActive) { + return + } + benchmarkActive = false + benchmarkAborted = benchmarkAborted or aborted + // Prefer the last ack as the end time so drain lag after the deadline + // doesn't dilute the throughput figure. + benchmarkEndAtMs = if (benchmarkLastAckAtMs != 0L) benchmarkLastAckAtMs else now() + logLine("bandwidth benchmark " + (if (benchmarkAborted) "aborted" else "finished") + " (" + reason + "): " + + benchmarkMessagesAcked + "/" + benchmarkMessagesSent + " acked, " + + benchmarkPayloadBytesAcked + "B payload, " + benchmarkTimeouts + " timeouts") +} + +internal fun GlassesSessionCore.enqueueBenchmarkMessageLocked() { + // Fresh bytes per message: the transport's persistent compression history + // would compress even a random payload if we reused it across the run. + val payload = ByteArray(benchmarkMessageSize) + kotlin.random.Random.nextBytes(payload) + payload[0] = CFW_MSG_DIAGNOSTICS.toByte() // CFW diagnostic-control mode... + payload[1] = 0x7f.toByte() // ...with an unused sub-op: acked, no effect + val message = messageBuilder.imagePayload( + "bandwidth", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "bandwidth no-op " + payload.size + "B", + connectionOptions.sendImagesToLeft) + val payloadBytes = payload.size + // Logical custom-message bytes; excludes length tag and packet framing. + val wireBytes = message.message.size + message.onSent = MessageCallback { + benchmarkMessagesSent++ + if (benchmarkStartAtMs == 0L) { + benchmarkStartAtMs = now() + benchmarkDeadlineAtMs = benchmarkStartAtMs + benchmarkDurationMs + } + } + message.onAck = MessageCallback { + val ackedAtMs = now() + benchmarkMessagesAcked++ + benchmarkPayloadBytesAcked += payloadBytes + benchmarkWireBytesAcked += wireBytes + benchmarkLastAckAtMs = ackedAtMs + // No-op payloads ride the image path, so the firmware resets its + // heartbeat timer on them just like real image messages. + lastHeartbeatAckedAtMs = ackedAtMs + } + message.onTimeout = MessageCallback { + benchmarkTimeouts++ + benchmarkAborted = true + logLine("bandwidth benchmark ack timeout") + } + pendingMessages.addLast(message) +} + +// --------------------------------------------------------------------------- +// EvenHub shutdown and audio control waits + +internal fun GlassesSessionCore.sendShutdownInternal(exitMode: Int, reconnectOnTimeout: Boolean): Boolean { + val magic: Int + val startedAtMs = now() + monitor.withLock { + if (!running || !sessionReady) { + logLine("skip shutdown; session not ready") + return false + } + if (shutdownRequested) { + return true + } + shutdownRequested = true + // Magic values wrap, so an ACK from a much older suspend must not + // satisfy this request after enough sleep/wake cycles. + lastShutdownAckMagic = 0 + clearPendingMessagesLocked("shutdown requested") + val message = messageBuilder.shutdown(exitMode) + magic = message.magic + message.onAck = MessageCallback { + lastShutdownAckMagic = message.magic + fixedLayoutCreated = false + displayedFingerprint = "" + } + message.onTimeout = MessageCallback { + if (reconnectOnTimeout) { + handleTransportFailure("shutdown ack timeout") + } else { + logLine("EvenHub shutdown ack timeout; keeping BLE connected") + } + } + pendingMessages.addFirst(message) + logLine("queue shutdown") + // The stock compass keeps the magnetometer sampling independently of + // the plugin task, so ending the page does not stop it. Force a + // disable ahead of the shutdown command whenever it may be running: + // this also covers a disable that was wiped by the queue flush above + // or whose ack was lost, and the charging-mode/exit paths where the + // Compass window never got a chance to release it. + if (compassMaybeOn && fixedLayoutCreated) { + enqueueCompassControlLocked(true, false) + } + } + interruptibleSleep.interrupt() + + val ackDeadline = now() + ConnectionOptions.ACK_TIMEOUT_MS + 500 + monitor.withLock { + while (running + && sessionReady + && lastShutdownAckMagic != magic + && lastShutdownExitAtMs < startedAtMs + && hasPendingOrInflightMagicLocked(magic)) { + val remaining = ackDeadline - now() + if (remaining <= 0) { + break + } + monitor.awaitMs(minOf(remaining, 100L)) + } + val acked = lastShutdownAckMagic == magic + if (acked) { + val exitDeadline = now() + ConnectionOptions.ACK_TIMEOUT_MS + 500 + while (running && sessionReady && lastShutdownExitAtMs < startedAtMs) { + val remaining = exitDeadline - now() + if (remaining <= 0) { + break + } + monitor.awaitMs(minOf(remaining, 100L)) + } + } + return acked || lastShutdownExitAtMs >= startedAtMs + } +} + +internal fun GlassesSessionCore.waitForAudioControlAck(magic: Int, operation: String): Boolean { + val deadline = now() + ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 500 + monitor.withLock { + while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) { + val remaining = deadline - now() + if (remaining <= 0) { + break + } + monitor.awaitMs(minOf(remaining, 100L)) + } + val acked = lastAudioControlAckMagic == magic + if (!acked) { + if ("enable" == operation) { + audioPacketListener = null + audioCaptureActive = false + } + logLine("G2 mic $operation ack timeout") + } + return acked + } +} + +// --------------------------------------------------------------------------- +// CFW wake and framebuffer leases + +/** + * Replace any stale lease control with one right-arm and one left-arm + * fire-and-forget write. With priority=true the right arm is sent first so + * CLAIM reaches the lens that originated the deferred wake immediately. + */ +internal fun GlassesSessionCore.enqueueFaceclawWakeControlLocked(operation: Int, nonce: Int, priority: Boolean): Int { + clearMessagesOfKindLocked("wake-lease-control") + val generation = ++faceclawWakeControlGeneration + faceclawWakeControlSentCount = 0 + if (operation == BleProtocol.FACECLAW_WAKE_OP_ACQUIRE) { + lastFaceclawWakeLeaseQueuedAtMs = now() + } + val onSent = MessageCallback { + if (faceclawWakeControlGeneration == generation) { + faceclawWakeControlSentCount += 1 + } + } + val right = messageBuilder.faceclawWakeControl(operation, nonce, false) + val left = messageBuilder.faceclawWakeControl(operation, nonce, true) + right.onSent = onSent + left.onSent = onSent + if (priority) { + pendingMessages.addFirst(left) + pendingMessages.addFirst(right) + } else { + pendingMessages.addLast(right) + pendingMessages.addLast(left) + } + logLine("queue " + right.label + " + L") + return generation +} + +internal fun GlassesSessionCore.waitForFaceclawWakeControlDelivery(generation: Int, timeoutMs: Long): Boolean { + val deadline = now() + maxOf(0L, timeoutMs) + monitor.withLock { + while (running + && sessionReady + && faceclawWakeControlGeneration == generation + && faceclawWakeControlSentCount < 2) { + val remaining = deadline - now() + if (remaining <= 0) { + break + } + monitor.awaitMs(minOf(remaining, 100L)) + } + return faceclawWakeControlGeneration == generation + && faceclawWakeControlSentCount >= 2 + } +} + +/** + * Acquire/renew or release CFW's independent direct-framebuffer repaint + * guard on both arms. It is separate from the optional idle-wake lease: + * every Faceclaw display session needs this guard while its EvenHub layout + * contains swipe-capturing stock widgets. + */ +internal fun GlassesSessionCore.enqueueFaceclawFramebufferControlLocked(operation: Int, priority: Boolean): Int { + clearMessagesOfKindLocked("framebuffer-lease-control") + val generation = ++faceclawFramebufferControlGeneration + faceclawFramebufferControlSentCount = 0 + if (operation == BleProtocol.FACECLAW_FB_OP_ACQUIRE) { + lastFaceclawFramebufferLeaseQueuedAtMs = now() + } + val onSent = MessageCallback { + if (faceclawFramebufferControlGeneration == generation) { + faceclawFramebufferControlSentCount += 1 + } + } + val right = messageBuilder.faceclawFramebufferControl(operation, false) + val left = messageBuilder.faceclawFramebufferControl(operation, true) + right.onSent = onSent + left.onSent = onSent + if (priority) { + pendingMessages.addFirst(left) + pendingMessages.addFirst(right) + } else { + pendingMessages.addLast(right) + pendingMessages.addLast(left) + } + logLine("queue " + right.label + " + L") + return generation +} + +internal fun GlassesSessionCore.waitForFaceclawFramebufferControlDelivery(generation: Int, timeoutMs: Long): Boolean { + val deadline = now() + maxOf(0L, timeoutMs) + monitor.withLock { + while (running + && sessionReady + && faceclawFramebufferControlGeneration == generation + && faceclawFramebufferControlSentCount < 2) { + val remaining = deadline - now() + if (remaining <= 0) { + break + } + monitor.awaitMs(minOf(remaining, 100L)) + } + return faceclawFramebufferControlGeneration == generation + && faceclawFramebufferControlSentCount >= 2 + } +} + +internal fun GlassesSessionCore.releaseFaceclawFramebufferLease(): Boolean { + val generation: Int + monitor.withLock { + if (!running || !sessionReady) { + return true + } + generation = enqueueFaceclawFramebufferControlLocked( + BleProtocol.FACECLAW_FB_OP_RELEASE, + true + ) + } + interruptibleSleep.interrupt() + return waitForFaceclawFramebufferControlDelivery( + generation, + FACECLAW_WAKE_CONTROL_WAIT_MS + ) +} + +// --------------------------------------------------------------------------- +// Failure handling + +/** The address of a configured arm the OS no longer holds a bond for, or null. */ +internal fun GlassesSessionCore.firstUnpairedArm(): String? { + if (!link.isBonded(rightAddress)) return rightAddress + if (!link.isBonded(leftAddress)) return leftAddress + return null +} + +/** + * A connect failure while an arm's bond is missing (the pairing was + * forgotten in the OS settings, or the address was typed in by hand and + * never paired) will repeat forever, so instead of scheduling a retry, + * park the worker loop and tell the user to re-pair. Only an explicit + * connect (which builds a fresh communicator) starts a new attempt. + */ +internal fun GlassesSessionCore.handleUnpairedFailure(address: String) { + logError("Connect failed and $address is not paired; suspending reconnect") + monitor.withLock { + reconnectHalted = true + sessionReady = false + fixedLayoutCreated = false + startupProbePending = false + shutdownRequested = false + chargingMode = false + imageRetryAfterMs = 0 + displayedFingerprint = "" + faceclawWakePendingNonce = -1 + lastFaceclawWakeLeaseQueuedAtMs = 0 + faceclawWakeControlSentCount = 0 + clearAllMessagesLocked("arm not paired: $address") + reconnectAfterMs = Long.MAX_VALUE + link.disconnect(rightAddress) + link.disconnect(leftAddress) + } + if (!userDisconnectRequested) { + setStateDisplay( + "unpaired", + "The glasses (" + address + ") are not paired with this phone." + + " Use \"Pair glasses\" to pair them again, then connect." + ) + } + interruptibleSleep.interrupt() +} + +internal fun GlassesSessionCore.handleTransportFailure(reason: String?) { + logError("Transport failure: $reason") + monitor.withLock { + maybeEmitEvenAppConflictLocked(reason) + finishBenchmarkLocked(true, "transport failure") + sessionReady = false + fixedLayoutCreated = false + startupProbePending = false + shutdownRequested = false + chargingMode = false + imageRetryAfterMs = 0 + displayedFingerprint = "" + faceclawWakePendingNonce = -1 + lastFaceclawWakeLeaseQueuedAtMs = 0 + faceclawWakeControlSentCount = 0 + clearAllMessagesLocked("transport failure: $reason") + reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS + link.disconnect(rightAddress) + link.disconnect(leftAddress) + } + if (!userDisconnectRequested) { + setStateDisplay("retrying", if (reason == null || reason.isEmpty()) "Reconnecting..." else "Reconnecting after $reason") + } + interruptibleSleep.interrupt() +} + +internal fun GlassesSessionCore.resetSessionStateLocked() { + ringBattery = -1 + ringCharging = -1 + sessionReady = false + shutdownRequested = false + fixedLayoutCreated = false + chargingMode = false + rightConnected = false + leftConnected = false + ringConnected = false + ringNotificationsReady = false + reconnectAfterMs = 0 + reconnectHalted = false + ringReconnectAfterMs = 0 + lastAckAtMs = 0 + lastIncomingAtMs = 0 + lastHeartbeatSentAtMs = 0 + lastSessionReadyAtMs = 0 + consecutiveAckTimeouts = 0 + lastAudioControlAckMagic = 0 + audioCaptureActive = false + audioPacketListener = null + compassControlLastSent = -1 + // A dead transport orphans any glasses-side compass state; the fresh + // session re-asserts the desired state once its layout is ready. + compassMaybeOn = false + faceclawWakePendingNonce = -1 + lastFaceclawWakeLeaseQueuedAtMs = 0 + faceclawWakeControlSentCount = 0 + cfwCleanupDelivered = false + for (transport in cfwTransports) transport.reset() + lastCfwCleanupAckMagic = 0 + wearState = -1 + displayedFingerprint = "" + // Deliberately not clearing silentMode: it is a property of the glasses, + // not of our session, and silent mode blocks app launches, so it can be + // the very cause of the session teardown that got us here. +} + +// --------------------------------------------------------------------------- +// Listener emits (main thread via the host) + +internal fun GlassesSessionCore.emitAmbientLight(body: ByteArray) { + if (ambientLightListeners.isEmpty()) { + return + } + val copy = body.copyOf() + host.postToMain { + for (alsListener in ambientLightListeners) { + try { + alsListener.onAmbientLight(copy) + } catch (t: Throwable) { + logWarn("ambient light listener failed", t) + } + } + } +} + +internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String) { + if (micStatusListeners.isEmpty()) { + return + } + val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" + else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?" + val copy = body.copyOf() + host.postToMain { + for (micListener in micStatusListeners) { + try { + micListener.onMicStatus(copy, arm) + } catch (t: Throwable) { + logWarn("mic status listener failed", t) + } + } + } +} + +internal fun GlassesSessionCore.emitRingEvent(event: G2Event, frameId: Int) { + val current = listener + if (current == null) { + frameTimings.finishFrame(frameId, "discarded: no listener attached") + return + } + val containerNameSnapshot = event.containerName + host.postToMain { + frameTimings.log(frameId, "dispatching input event on main thread") + try { + current.onRingEvent(event.kind, containerNameSnapshot, event.eventType, event.eventSource, event.systemExitReasonCode, frameId, + event.ringTick, event.ringType, event.ringAux, event.ringSpeed) + } catch (t: Throwable) { + logWarn("listener onRingEvent failed", t) + frameTimings.finishFrame(frameId, "discarded: listener onRingEvent failed") + } + } +} + +/** Finish a frame owned by the communicator and tell the TS side, which may be awaiting it. */ +internal fun GlassesSessionCore.finishFrame(frameId: Int, outcome: String) { + if (frameId <= 0) { + return + } + frameTimings.finishFrame(frameId, outcome) + val current = listener ?: return + host.postToMain { + try { + current.onFrameFinished(frameId, outcome) + } catch (t: Throwable) { + logWarn("listener onFrameFinished failed", t) + } + } +} + +internal fun GlassesSessionCore.emitImuData(x: Double, y: Double, z: Double, eventSource: Int) { + if (imuListeners.isEmpty()) { + return + } + host.postToMain { + for (imuListener in imuListeners) { + try { + imuListener.onImuData(x, y, z, eventSource) + } catch (t: Throwable) { + logWarn("listener onImuData failed", t) + } + } + } +} + +internal fun GlassesSessionCore.emitCompassEvent(event: BleProtocol.CompassEvent) { + if (event.diagnosticFlags >= 0) { + val sources = arrayOf("unknown", "GRV", "GMRV", "RV") + host.log(SessionLogLevel.INFO, "FaceclawCompass", "heading=" + event.headingDegrees + + " magneticAccuracy=" + event.magneticAccuracy + + " magneticAnomalies=" + event.magneticAnomalies + + " orientationSource=" + sources[event.orientationSource] + + " flags=0x" + event.diagnosticFlags.toString(16) + + " sampleTimeMs=" + event.sampleTimeMs, null) + } + for (subscription in compassSubscriptions) { + subscription.dispatcher.post { + try { + subscription.listener.onCompassEvent(event.command, event.headingDegrees, + event.magneticAccuracy, event.magneticAnomalies, event.orientationSource, + event.diagnosticFlags, event.sampleTimeMs) + } catch (t: Throwable) { + logWarn("listener onCompassEvent failed", t) + } + } + } +} + +internal fun GlassesSessionCore.emitSilentMode(silent: Boolean) { + val current = listener ?: return + host.postToMain { + try { + current.onSilentMode(silent) + } catch (t: Throwable) { + logWarn("listener onSilentMode failed", t) + } + } +} + +internal fun GlassesSessionCore.emitWearState(wearing: Boolean) { + val current = listener ?: return + host.postToMain { + try { + current.onWearState(wearing) + } catch (t: Throwable) { + logWarn("listener onWearState failed", t) + } + } +} + +internal fun GlassesSessionCore.emitPhoneLockStateIfChanged(force: Boolean) { + val locked: Boolean + monitor.withLock { + val now = now() + if (!force && now - lastPhoneLockCheckAtMs < 1_000) return + lastPhoneLockCheckAtMs = now + locked = host.isPhoneLocked() + val value = if (locked) 1 else 0 + if (!force && value == phoneLockState) return + phoneLockState = value + } + val current = listener ?: return + host.postToMain { + try { + current.onPhoneLockState(locked) + } catch (t: Throwable) { + logWarn("listener onPhoneLockState failed", t) + } + } +} + +internal fun GlassesSessionCore.emitBatteryState(headsetBattery: Int, headsetCharging: Int) { + val reportedRingBattery = ringBattery + val reportedRingCharging = ringCharging + val current = listener ?: return + host.postToMain { + try { + current.onBatteryState(headsetBattery, headsetCharging, reportedRingBattery, reportedRingCharging) + } catch (t: Throwable) { + logWarn("listener onBatteryState failed", t) + } + } +} + +internal fun GlassesSessionCore.emitFirmwareInfo(info: BleProtocol.FirmwareInfo) { + val current = listener ?: return + host.postToMain { + try { + current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.extension) + } catch (t: Throwable) { + logWarn("listener onFirmwareInfo failed", t) + } + } +} + +internal fun GlassesSessionCore.emitFrameMetrics(paintMs: Int, transmitMs: Int, tileCount: Int) { + val current = listener ?: return + host.postToMain { + try { + current.onFrameMetrics(paintMs, transmitMs, tileCount) + } catch (t: Throwable) { + logWarn("listener onFrameMetrics failed", t) + } + } +} + +internal fun GlassesSessionCore.maybeEmitEvenAppConflictLocked(reason: String?) { + if ("write failed" != reason) { + return + } + val now = now() + if (lastSessionReadyAtMs <= 0 || now - lastSessionReadyAtMs > ConnectionOptions.EVEN_APP_WRITE_FAILURE_WINDOW_MS) { + return + } + if (lastEvenAppConflictAtMs > 0 && now - lastEvenAppConflictAtMs < 60_000) { + return + } + if (!host.isEvenAppActive()) { + return + } + lastEvenAppConflictAtMs = now + emitEvenAppConflict("The Even Realities app still appears to be running. It can hold the glasses BLE link and cause Faceclaw write failures. Open its app settings and force stop it, then reconnect Faceclaw.") +} + +internal fun GlassesSessionCore.emitEvenAppConflict(message: String?) { + val current = listener ?: return + val messageSnapshot = message ?: "" + host.postToMain { + try { + current.onEvenAppConflict(messageSnapshot) + } catch (t: Throwable) { + logWarn("listener onEvenAppConflict failed", t) + } + } +} + +internal fun GlassesSessionCore.setStateDisplay(nextPhase: String, nextStatus: String) { + monitor.withLock { + phase = nextPhase + status = nextStatus + } + emitState() +} + +internal fun GlassesSessionCore.emitState() { + val current = listener ?: return + val phaseSnapshot: String + val statusSnapshot: String + monitor.withLock { + phaseSnapshot = phase + statusSnapshot = status + } + host.postToMain { + try { + current.onStateChange(phaseSnapshot, statusSnapshot) + } catch (t: Throwable) { + logWarn("listener onStateChange failed", t) + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt new file mode 100644 index 00000000..8175ccf9 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -0,0 +1,1917 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmField + +/** + * The Faceclaw glasses session: connect/auth/prelude, the send scheduler and CFW + * message window, the image pipeline, and every EvenHub/CFW control the app + * drives. Platform-neutral; GATT goes through [SessionLink], side effects + * through [SessionHost], clocks/locks through [ProtocolPlatform]. The worker + * thread that runs [run] is started by the host from [start]. + * + * The class body holds the state and the public API; the scheduler, image + * pipeline and control helpers are extension functions in the sibling files + * (GlassesSessionSend.kt, GlassesSessionControls.kt), which is why the state is + * `internal` rather than private. + * + * Locking: [monitor] guards session state (reentrant); its wait sites are the + * public blocking waits and the connect settling sleep, each of which holds the + * lock exactly once. [desiredTilesLock] guards the desired-frame slot and is + * taken inside the monitor where both are needed. + */ +class GlassesSessionCore( + internal val link: SessionLink, + internal val host: SessionHost, + internal val frameTimings: FrameTimingsCore, + internal val platform: ProtocolPlatform, + rightAddress: String?, + leftAddress: String?, + ringAddress: String?, +) : FaceclawBleListener { + companion object { + const val TAG = "FaceclawComm" + + // Local metadata for custom-command bookkeeping, not an EvenHub container. + // Submitted frames supply pixel geometry; the stock layout only captures input. + internal val DASHBOARD_TILE: BleProtocol.ImageTileOptions = + BleProtocol.ImageTileOptions("img00", 10, 0, 0, 576, 288) + + internal const val FACECLAW_WAKE_LEASE_RENEW_MS = 45_000L + internal const val FACECLAW_WAKE_CONTROL_WAIT_MS = 1_500L + internal const val CFW_CLEANUP_WAIT_MS = 4_000L + internal const val COMPASS_REPORT_INTERVAL_MS = 100 + internal const val COMPASS_MIN_CHANGE_DEGREES = 0 + + // A timed-out benchmark message aborts the run, but its already-in-flight + // peers still time out one by one; keep the window comfortably below + // MAX_CONSECUTIVE_ACK_TIMEOUTS so a dead run can't escalate into a + // transport-failure reconnect all by itself. + internal const val BENCHMARK_MAX_WINDOW = 6 + + internal fun chargingStatusText(battery: Int): String { + return if (battery >= 0) + "Glasses charging. Battery $battery%." + else + "Glasses charging." + } + + internal fun timestamp(elapsedMs: Long): String { + val wallMs = currentTimeMillis() + return formatLocalTime(wallMs, "yyyy-MM-dd HH:mm:ss.SSS") + " elapsed=${elapsedMs}ms" + } + + private fun requireAddress(name: String, address: String?): String { + if (address == null || address.trim().isEmpty()) { + throw IllegalArgumentException("$name is required") + } + return address.trim() + } + + internal fun hex(data: ByteArray?): String { + if (data == null || data.isEmpty()) { + return "" + } + val out = CharArray(data.size * 2) + val digits = "0123456789abcdef".toCharArray() + for (i in data.indices) { + val value = data[i].toInt() and 0xff + out[i * 2] = digits[value ushr 4] + out[i * 2 + 1] = digits[value and 0x0f] + } + return out.concatToString() + } + + internal fun safeMessage(t: Throwable?): String { + if (t == null) { + return "unknown" + } + val trace = t.stackTraceToString() + if (trace.trim().isNotEmpty()) { + return trace + } + val message = t.message + return if (message == null || message.trim().isEmpty()) t.toString() else message + } + } + + internal val interruptibleSleep = InterruptibleSleep(platform) + internal val monitor = SessionMonitor(platform) + internal val rightAddress: String = requireAddress("rightAddress", rightAddress) + internal val leftAddress: String = requireAddress("leftAddress", leftAddress) + internal val ringAddress: String = ringAddress?.trim() ?: "" + + @Volatile internal var listener: FaceclawBleCommunicatorListener? = null + internal val imuListeners = CopyOnWriteList(platform) + /** A compass subscriber plus the dispatcher it registered from (see addCompassListener). */ + internal class CompassSubscription( + @JvmField val listener: FaceclawCompassListener, + @JvmField val dispatcher: SessionDispatcher, + ) + + internal val compassSubscriptions = CopyOnWriteList(platform) + internal val ambientLightListeners = CopyOnWriteList(platform) + internal val micStatusListeners = CopyOnWriteList(platform) + @Volatile internal var running = false + @Volatile internal var userDisconnectRequested = false + // Set when a connect attempt failed while an arm's OS bond is gone: + // retrying is pointless until the user re-pairs, so the worker loop parks + // instead of redialing. Cleared by start() (a fresh explicit connect). + @Volatile internal var reconnectHalted = false + + internal var phase = "disconnected" + internal var status = "Disconnected." + + internal var rightConnected = false + internal var leftConnected = false + internal var ringConnected = false + internal var ringNotificationsReady = false + internal var sessionReady = false + internal var fixedLayoutCreated = false + internal var shutdownRequested = false + // CFW firmware-debug-flags overlay (mode 7). Desired value pushed from TS; the + // sub-op last sent this session (-1 = not yet), reset on (re)connect so the + // overlay state is re-asserted on every reconnect and whenever the value changes. + @Volatile internal var firmwareDebugFlagsEnabled = false + internal var firmwareDebugFlagsLastSent = -1 + // Desired CFW mode-10 compass state. It survives reconnects; lastSent is + // reset with each session so an open Compass window is re-asserted. + /** + * Who currently wants the stock compass running (the Compass window, the + * Navigate worker, ...). The magnetometer is one shared resource, so it + * stays on while any owner holds it and is released when the last lets go; + * this keeps one app's release from silently switching off another's feed. + */ + internal val compassOwners: MutableSet = HashSet() + internal var compassControlLastSent = -1 + // Whether the glasses-side compass may still be running: set when an enable + // is enqueued, cleared only when a disable is acked. Drives the forced + // disable sent ahead of an EvenHub shutdown/suspend, since a pending + // disable can be wiped by the shutdown's queue flush and the retry loop + // does not run while shutdownRequested (magnetometer left on = battery drain). + internal var compassMaybeOn = false + internal var startupProbePending = false + // Desired ownership of CFW's fail-open stock-wake lease (dashboard launch + // and Even AI foreground takeover). This survives a transport reconnect; + // the lease itself is volatile firmware state and is re-acquired once both + // arms are ready. + internal var faceclawWakeLeaseEnabled = false + internal var lastFaceclawWakeLeaseQueuedAtMs = 0L + internal var faceclawWakeControlGeneration = 0 + internal var faceclawWakeControlSentCount = 0 + internal var lastFaceclawFramebufferLeaseQueuedAtMs = 0L + internal var faceclawFramebufferControlGeneration = 0 + internal var faceclawFramebufferControlSentCount = 0 + internal var faceclawWakePendingNonce = -1 + /** + * The last firmware-info read said the glasses run Faceclaw's custom + * firmware. Gates the private modes (cleanup, resource cache, ...) so stock + * or third-party firmware never sees them; the TS side checks the actual + * revision and disconnects on a mismatch, so no per-feature gating is + * needed here. + */ + internal var customFirmwareDetected = false + internal var cfwCleanupDelivered = false + internal var lastCfwCleanupAckMagic = 0 + + internal var reconnectAfterMs = 0L + internal var ringReconnectAfterMs = 0L + internal var lastAckAtMs = 0L + internal var lastIncomingAtMs = 0L + internal var lastHeartbeatSentAtMs = 0L + internal var lastHeartbeatAckedAtMs = 0L + internal var lastConnectionOrInputAtMs = 0L + internal var lastBatteryRefreshAtMs = 0L + internal var imageRetryAfterMs = 0L + internal var lastSessionReadyAtMs = 0L + internal var lastEvenAppConflictAtMs = 0L + internal var consecutiveAckTimeouts = 0 + internal var lastAudioControlAckMagic = 0 + + internal val connectionOptions = ConnectionOptions() + internal val magicPool = BleMagicPool(platform) + internal val messageBuilder = MessageBuilder(magicPool) + internal var nextTransportSeq = 0x40 + internal var nextMapSessionIdValue = 0 + internal var nextImageUpdateId = 1 + // Wire frame id for mode-3 deltas (CFW reorder/skip/dup diagnostic). uint16, + // advanced by 1 per emitted delta; kept in [1, 0xfffe] to avoid the CFW's + // 0xffff "empty" sentinel. + internal var nextImageFrameId = 1 + internal var lastShutdownAckMagic = 0 + internal var lastShutdownExitAtMs = 0L + internal var headsetBattery = -1 + internal var headsetCharging = -1 + internal var ringBattery = -1 + internal var ringCharging = -1 + // Silent mode: 1 = on, 0 = off, -1 = not yet known. See updateSilentModeLocked. + internal var silentMode = -1 + internal var wearState = -1 + internal var phoneLockState = -1 + internal var lastPhoneLockCheckAtMs = 0L + internal var audioCaptureActive = false + internal var firmwareInfoQueried = false + // Glasses are in the charging case: nobody is wearing them, so display + // communication pauses and only battery polls flow (see driveSession). + internal var chargingMode = false + @Volatile internal var audioPacketListener: FaceclawAudioPacketListener? = null + + // BLE bandwidth benchmark (Developer app). Streams no-op image payloads + // (CFW mode 7 with an unused sub-op: parsed, acked, and discarded — stock + // firmware likewise ignores unknown image modes) for a fixed duration with + // a selectable message size and pipeline window, then reports throughput. + // While active, desired-frame sends are held back and heartbeats are + // satisfied by the benchmark's own acks, so the stream is the only image + // traffic. All state below is guarded by `monitor`; results are read with + // getBandwidthBenchmarkStatus() and survive until the next run starts. + internal var benchmarkActive = false + internal var benchmarkAborted = false + internal var benchmarkMessageSize = 0 + internal var benchmarkWindowSize = 0 + internal var benchmarkDurationMs = 0 + internal var benchmarkLinkMode = 0 + internal var benchmarkLinkPending = false + internal var benchmarkReadyAtMs = 0L + internal var benchmarkStartAtMs = 0L // first benchmark write; 0 until then + internal var benchmarkDeadlineAtMs = 0L // start + duration; MAX_VALUE until first write + internal var benchmarkLastAckAtMs = 0L + internal var benchmarkEndAtMs = 0L // 0 while running; set when the run drains + internal var benchmarkMessagesSent = 0 + internal var benchmarkMessagesAcked = 0 + internal var benchmarkTimeouts = 0 + internal var benchmarkPayloadBytesAcked = 0L + internal var benchmarkWireBytesAcked = 0L + + internal var displayedFingerprint = "" + // The frame the firmware shadow will hold once the current image pipeline + // drains: the most recently ENQUEUED image (headerless packed 4bpp, see + // BmpUtil.pack4bppFromGray8), which is the correct base for the next delta + // when frames are pipelined. Set at enqueue; cleared whenever the image + // pipeline is cleared (clearAllMessagesLocked / clearMessagesOfKindLocked + // "image"), so it is only ever read while it holds a valid current-session base. + internal var lastEnqueuedPacked: ByteArray = ByteArray(0) + internal var lastEnqueuedWidth = 0 + internal var lastEnqueuedHeight = 0 + internal var lastEnqueuedFingerprint = "" + internal val imageUpdateStats: MutableMap = HashMap() + + internal val desiredTilesLock: ProtocolLock = platform.createLock() + internal var desiredFingerprint = "" + // Headerless packed 4bpp frame (see BmpUtil.pack4bppFromGray8) plus its + // pixel dimensions. + internal var desiredPacked: ByteArray? = ByteArray(0) + internal var desiredWidth = 0 + internal var desiredHeight = 0 + internal var desiredPaintMs = 0 + internal var desiredFrameId = 0 + // Screen-space deferred draws (glyphs + images) whose pixels are baked + // into desiredPacked; the resource-cache planner may replay them as + // on-glasses cached draws. + internal var desiredDraws: Array? = arrayOf() + // (frame, reason) of the last "waiting to send" line, so a frame that + // stalls for seconds records one line per state change (see + // noteImageStallLocked). Send-loop thread only. + internal var stallFrameId = 0 + internal var stallReason = "" + // Highest compositor sequence stored as the desired frame; composites that + // lost a store race to a newer one are discarded (their content is already + // included in the newer composite). + internal var lastStoredCompositeSeq = 0L + + // Wire submissions need screenGray + the shell scene, never preview pixels. + // Preview/screenshot/recording callers render those explicitly on demand. + internal val compositor = SurfaceCompositor(false) + + // Phone-side model of the CFW's 192 KiB resource cache. + // Reset whenever the image pipeline / EvenHub session is torn down: the + // firmware frees the cache with the fb lease, and after any resync the + // cheap safe assumption is an empty cache (glyphs re-upload lazily). + internal val resourceCache = ResourceCacheState() + internal val scenePlanner = ScenePlanner(resourceCache) + internal var desiredShellScene: ShellScene = ShellScene.EMPTY + + internal val pendingMessages = ArrayDeque() + internal val cfwTransports = arrayOf(CfwTransport(platform), CfwTransport(platform)) + internal val inFlightMessages = ArrayDeque() + internal var prewrittenMessage: OutboundMessage? = null + internal var prewrittenFrames: List = emptyList() + + internal fun now(): Long = platform.elapsedRealtimeMs() + + internal fun logLine(line: String) = host.log(SessionLogLevel.INFO, TAG, line, null) + + internal fun logDebug(line: String) = host.log(SessionLogLevel.DEBUG, TAG, line, null) + + internal fun logWarn(line: String, error: Throwable? = null) = host.log(SessionLogLevel.WARN, TAG, line, error) + + internal fun logError(line: String) = host.log(SessionLogLevel.ERROR, TAG, line, null) + + // --------------------------------------------------------------------- + // Lifecycle + + fun setListener(listener: FaceclawBleCommunicatorListener?) { + this.listener = listener + emitState() + emitPhoneLockStateIfChanged(true) + } + + /** Start the worker (through the host); false when it was already running. */ + fun start(): Boolean { + monitor.withLock { + if (running) { + return false + } + running = true + userDisconnectRequested = false + reconnectHalted = false + shutdownRequested = false + host.startWorker { run() } + return true + } + } + + fun disconnect() { + /* On the normal path DashboardController already sent mode 11 after + * quiescing its producers. Also cover direct/early close callers here; + * a successful cleanup must remain the final BLE message. Older CFWs + * fall back to the standalone framebuffer-lease release. */ + if (!cfwCleanupDelivered && !sendCfwCleanup()) { + releaseFaceclawFramebufferLease() + } + monitor.withLock { + userDisconnectRequested = true + running = false + audioCaptureActive = false + audioPacketListener = null + } + setStateDisplay("disconnecting", "Disconnecting...") + interruptibleSleep.interrupt() + host.joinWorker(5_000) + monitor.withLock { + resetSessionStateLocked() + clearAllMessagesLocked("disconnect") + // Unknown until the next connection's first push or settings poll. + silentMode = -1 + } + link.disconnect(rightAddress) + link.disconnect(leftAddress) + if (hasRingAddress()) { + link.disconnect(ringAddress) + } + link.close() + host.setScreenWakeLock(false) + setStateDisplay("disconnected", "Disconnected.") + } + + fun close() { + disconnect() + for (transport in cfwTransports) transport.close() + } + + /** The platform's screen-on/off/user-present signal: refresh the lock state and wake the worker. */ + fun onPhoneLockSignal() { + emitPhoneLockStateIfChanged(true) + interruptibleSleep.interrupt() + } + + fun setG2ScreenOn(screenOn: Boolean) { + host.postToMain { host.setScreenWakeLock(screenOn) } + } + + fun setFirmwareDebugFlags(enabled: Boolean) { + // Just record it; the drive loop emits the mode-7 control message when the + // display path is ready and idle, and re-emits when this value changes. + firmwareDebugFlagsEnabled = enabled + } + + fun startG2AudioCapture(listener: FaceclawAudioPacketListener?): Boolean { + if (listener == null) { + throw IllegalArgumentException("listener is required") + } + var magic = 0 + monitor.withLock { + if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { + logLine("skip G2 mic enable; EvenHub display path not ready") + return false + } + audioPacketListener = listener + val message = createAudioControlMessageLocked(true) + magic = message.magic + pendingMessages.addFirst(message) + logLine("queue G2 mic enable") + } + interruptibleSleep.interrupt() + return waitForAudioControlAck(magic, "enable") + } + + fun stopG2AudioCapture() { + var magic = 0 + monitor.withLock { + audioPacketListener = null + audioCaptureActive = false + clearMessagesOfKindLocked("audio-control") + if (running && sessionReady) { + val message = createAudioControlMessageLocked(false) + magic = message.magic + pendingMessages.addFirst(message) + logLine("queue G2 mic disable") + } + } + interruptibleSleep.interrupt() + if (magic != 0) { + waitForAudioControlAck(magic, "disable") + } + } + + fun isSessionReady(): Boolean { + monitor.withLock { + return running && sessionReady + } + } + + /** + * Whether the glasses mic is enabled right now. The enable lives in the + * current EvenHub session, so it dies with a transport drop, the charging + * case, or a suspend — silently, from the phone's point of view. Callers + * that track a capture across those events must check this rather than + * assume their earlier enable still holds. + */ + fun isAudioCaptureActive(): Boolean { + monitor.withLock { + return running && sessionReady && !shutdownRequested && audioCaptureActive + } + } + + /** + * Acquire/renew or release CFW's volatile wake-takeover lease on both + * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure + * the fail-open firmware policy is installed before relying on wakeword + * interception or suspending EvenHub. + */ + fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean { + val generation: Int + monitor.withLock { + faceclawWakeLeaseEnabled = enabled + if (!running || !sessionReady) { + return !enabled + } + generation = enqueueFaceclawWakeControlLocked( + if (enabled) BleProtocol.FACECLAW_WAKE_OP_ACQUIRE else BleProtocol.FACECLAW_WAKE_OP_RELEASE, + 0, + true + ) + if (!enabled) { + faceclawWakePendingNonce = -1 + } + } + interruptibleSleep.interrupt() + return waitForFaceclawWakeControlDelivery(generation, FACECLAW_WAKE_CONTROL_WAIT_MS) + } + + /** + * Wait until the recreated layout and retained compositor frame have both + * landed. If this wake came from CFW's deferred double tap, READY + * is then sent to both arms to cancel their stock-dashboard fallback. + */ + fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean { + val deadline = now() + maxOf(0, timeoutMs) + var readyGeneration = 0 + monitor.withLock { + while (running && sessionReady) { + var frameReady = false + desiredTilesLock.locked { + frameReady = desiredFingerprint.isNotEmpty() + && desiredFingerprint == displayedFingerprint + } + if (!shutdownRequested && fixedLayoutCreated && frameReady) { + if (faceclawWakePendingNonce >= 0) { + readyGeneration = enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_READY, + faceclawWakePendingNonce, + true + ) + faceclawWakePendingNonce = -1 + } + break + } + val remaining = deadline - now() + if (remaining <= 0) { + return false + } + monitor.awaitMs(minOf(remaining, 100L)) + } + if (!running || !sessionReady) { + return false + } + } + if (readyGeneration != 0) { + interruptibleSleep.interrupt() + if (!waitForFaceclawWakeControlDelivery(readyGeneration, FACECLAW_WAKE_CONTROL_WAIT_MS)) { + logLine("wake READY delivery not confirmed before fallback deadline") + } + } + return true + } + + /** + * Enable or disable the IMU (accelerometer) report stream. Fire-and-forget: + * the control message is queued ahead of other traffic; readings arrive via + * registered FaceclawImuListeners. reportFrq is the requested sample rate + * (ignored on disable). + */ + fun setImuReportEnabled(enable: Boolean, reportFrq: Int) { + monitor.withLock { + if (!running || !sessionReady) { + logLine("skip IMU " + (if (enable) "enable" else "disable") + "; session not ready") + return + } + clearMessagesOfKindLocked("imu-control") + val message = messageBuilder.enableOrDisableImu(enable, reportFrq) + message.onTimeout = MessageCallback { logLine("IMU control ack timeout") } + pendingMessages.addFirst(message) + logLine("queue IMU " + (if (enable) "enable freq=$reportFrq" else "disable")) + } + interruptibleSleep.interrupt() + } + + /** + * Enable/disable the stock compass through CFW image-handler mode 10. The + * desired state is retained across reconnects; headings arrive through + * stock sid-0x08 navigation notifications and FaceclawCompassListeners. + */ + fun setCompassEnabled(enable: Boolean) { + setCompassEnabled("compass", enable) + } + + /** + * As above, on behalf of a named owner. The compass runs while at least + * one owner has enabled it; an owner disabling it only takes effect once + * no other owner still wants it. + */ + fun setCompassEnabled(owner: String?, enable: Boolean) { + monitor.withLock { + val before = compassOwners.isNotEmpty() + if (enable) { + compassOwners.add(owner) + } else { + compassOwners.remove(owner) + } + val wanted = compassOwners.isNotEmpty() + if (wanted == before && compassControlLastSent == (if (wanted) 1 else 0)) { + logLine("compass " + (if (enable) "enable" else "disable") + " by " + owner + + "; state unchanged (owners=" + compassOwners + ")") + return + } + compassControlLastSent = -1 + clearMessagesOfKindLocked("compass-control") + if (running && sessionReady && !shutdownRequested && fixedLayoutCreated) { + enqueueCompassControlLocked(true, wanted) + } else { + logLine("defer compass " + (if (enable) "enable" else "disable") + "; display path not ready") + } + } + interruptibleSleep.interrupt() + } + + /** + * Set the lens brightness. Fire-and-forget, like the IMU control: the + * message is queued ahead of other traffic and any not-yet-sent brightness + * message is superseded. When autoAdjust is true the ambient-light sensor + * drives brightness and brightnessLevel is ignored; otherwise + * brightnessLevel (0-100) is applied directly. + */ + fun setBrightness(autoAdjust: Boolean, brightnessLevel: Int) { + monitor.withLock { + if (!running || !sessionReady) { + logLine("skip brightness set; session not ready") + return + } + clearMessagesOfKindLocked("brightness-control") + val message = messageBuilder.setBrightness(autoAdjust, brightnessLevel) + message.onTimeout = MessageCallback { logLine("brightness control ack timeout") } + pendingMessages.addFirst(message) + logLine("queue brightness " + (if (autoAdjust) "auto" else "level=$brightnessLevel")) + } + interruptibleSleep.interrupt() + } + + /** + * Enable the stock wear detector, then ask CFW to emit its current cached + * state. The queue order matters: the query must run after the setting is + * applied, including on a fresh install where wear detection was disabled. + */ + fun enableWearDetectionAndRequestState() { + monitor.withLock { + if (!running || !sessionReady) { + logLine("skip wear detector setup; session not ready") + return + } + clearMessagesOfKindLocked("wear-detection-control") + clearMessagesOfKindLocked("wear-query-control") + val queryLeft = messageBuilder.faceclawWearQuery(true) + val queryRight = messageBuilder.faceclawWearQuery(false) + val enable = messageBuilder.setWearDetection(true) + enable.onTimeout = MessageCallback { logLine("wear detection enable ack timeout") } + pendingMessages.addFirst(queryLeft) + pendingMessages.addFirst(queryRight) + pendingMessages.addFirst(enable) + logLine("queue wear detection enable + current-state query") + } + interruptibleSleep.interrupt() + } + + // --------------------------------------------------------------------- + // Bandwidth benchmark + + /** + * Start the BLE bandwidth benchmark: stream messageSize-byte no-op image + * payloads for durationMs, keeping up to windowSize messages awaiting ack + * at once, then leave the results for getBandwidthBenchmarkStatus(). + * Returns false when a run is already active or the image path is not + * ready. The duration clock starts at the first benchmark write, so + * traffic already queued ahead of the run doesn't count against it. + */ + fun startBandwidthBenchmark(messageSize: Int, windowSize: Int, durationMs: Int): Boolean { + return startBandwidthBenchmarkWithLinkMode(messageSize, windowSize, durationMs, 0) + } + + // 0: current link; 1: re-request HIGH; 2: request 2M; 3: both. + fun startBandwidthBenchmarkWithLinkMode(messageSize: Int, windowSize: Int, + durationMs: Int, linkMode: Int): Boolean { + monitor.withLock { + if (benchmarkActive || !running || !sessionReady || !fixedLayoutCreated + || shutdownRequested || chargingMode) { + logLine("skip bandwidth benchmark; already running or image path not ready") + return false + } + benchmarkMessageSize = maxOf(2, minOf(messageSize, ConnectionOptions.IMAGE_FRAGMENT_SIZE)) + benchmarkWindowSize = maxOf(1, minOf(windowSize, BENCHMARK_MAX_WINDOW)) + benchmarkDurationMs = maxOf(1_000, durationMs) + benchmarkLinkMode = linkMode and 3 + benchmarkLinkPending = true + benchmarkReadyAtMs = Long.MAX_VALUE + benchmarkStartAtMs = 0 + benchmarkDeadlineAtMs = Long.MAX_VALUE + benchmarkLastAckAtMs = 0 + benchmarkEndAtMs = 0 + benchmarkMessagesSent = 0 + benchmarkMessagesAcked = 0 + benchmarkTimeouts = 0 + benchmarkPayloadBytesAcked = 0 + benchmarkWireBytesAcked = 0 + benchmarkAborted = false + benchmarkActive = true + logLine("bandwidth benchmark start: size=" + benchmarkMessageSize + + "B window=" + benchmarkWindowSize + " duration=" + benchmarkDurationMs + + "ms linkMode=" + benchmarkLinkMode) + } + interruptibleSleep.interrupt() + return true + } + + /** + * Cancel an in-progress benchmark (benchmark page closed). Queued no-op + * messages are dropped; in-flight ones drain through their normal acks. + */ + fun cancelBandwidthBenchmark() { + monitor.withLock { + if (!benchmarkActive) { + return + } + clearMessagesOfKindLocked("bandwidth") + finishBenchmarkLocked(true, "cancelled") + } + interruptibleSleep.interrupt() + } + + /** Status/results of the current or most recent benchmark run, as JSON. */ + fun getBandwidthBenchmarkStatus(): String { + monitor.withLock { + val state = if (benchmarkActive) + (if (benchmarkStartAtMs == 0L) "starting" else "running") + else + (if (benchmarkEndAtMs != 0L) "done" else "idle") + val end = if (benchmarkActive) now() else benchmarkEndAtMs + val elapsed = if (benchmarkStartAtMs == 0L) 0L else maxOf(0L, end - benchmarkStartAtMs) + try { + val status = linkedMapOf() + status["state"] = state + status["messageSize"] = benchmarkMessageSize + status["windowSize"] = benchmarkWindowSize + status["linkMode"] = benchmarkLinkMode + status["elapsedMs"] = elapsed + status["messagesSent"] = benchmarkMessagesSent + status["messagesAcked"] = benchmarkMessagesAcked + status["timeouts"] = benchmarkTimeouts + status["payloadBytesAcked"] = benchmarkPayloadBytesAcked + status["wireBytesAcked"] = benchmarkWireBytesAcked + status["aborted"] = benchmarkAborted + return Json.write(status) + } catch (e: Exception) { + return "{\"state\":\"idle\"}" + } + } + } + + // --------------------------------------------------------------------- + // Sensor and status listeners + + fun addImuListener(listener: FaceclawImuListener?) { + if (listener != null) { + imuListeners.add(listener) + } + } + + fun removeImuListener(listener: FaceclawImuListener?) { + if (listener != null) { + imuListeners.remove(listener) + } + } + + fun addAmbientLightListener(listener: FaceclawAmbientLightListener?) { + if (listener != null) { + ambientLightListeners.add(listener) + } + } + + fun removeAmbientLightListener(listener: FaceclawAmbientLightListener?) { + if (listener != null) { + ambientLightListeners.remove(listener) + } + } + + /** Request one CFW ambient-light report (image-handler mode 16 op 0). */ + fun queryAmbientLight() { + monitor.withLock { + enqueueAmbientLightControlLocked(byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 0.toByte()), "als query", false) + } + interruptibleSleep.interrupt() + } + + /** + * Start or stop CFW passive light-sensor polling (image-handler mode 16 + * ops 1/2). While polling, the firmware reads its OPT3001 every intervalMs + * (clamped 100..5000 by the firmware), never steps the panel brightness + * itself, and pushes a field-105 report when the reading moved by at least + * minDelta or heartbeatMs elapsed. bindToLease stops polling automatically + * when the Faceclaw framebuffer lease is released or lapses. Starting is + * idempotent and re-opens the sensor if the stock firmware closed it. + */ + fun setAmbientLightPolling(enable: Boolean, intervalMs: Int, minDelta: Int, + heartbeatMs: Int, bindToLease: Boolean) { + val payload = if (enable) + byteArrayOf( + CFW_MSG_AMBIENT_LIGHT.toByte(), + 1.toByte(), + (if (bindToLease) 1 else 0).toByte(), + (intervalMs and 0xff).toByte(), + ((intervalMs shr 8) and 0xff).toByte(), + (minDelta and 0xff).toByte(), + ((minDelta shr 8) and 0xff).toByte(), + (heartbeatMs and 0xff).toByte(), + ((heartbeatMs shr 8) and 0xff).toByte(), + ) + else + byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 2.toByte()) + monitor.withLock { + clearMessagesOfKindLocked("als-control") + enqueueAmbientLightControlLocked(payload, "als polling " + (if (enable) "start" else "stop"), true) + } + interruptibleSleep.interrupt() + } + + fun addMicStatusListener(listener: FaceclawMicStatusListener?) { + if (listener != null) { + micStatusListeners.add(listener) + } + } + + fun removeMicStatusListener(listener: FaceclawMicStatusListener?) { + if (listener != null) { + micStatusListeners.remove(listener) + } + } + + /** + * Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings + * field-103 write to both temples, or to a single one. Fire-and-forget: + * the firmware answers with a field-104 status notify per temple, which + * arrives through addMicStatusListener. + */ + fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) { + if (record == null || record.size < 4) { + return + } + monitor.withLock { + if (!running || !sessionReady) { + logLine("skip mic control ($label); session not ready") + return + } + if (rightTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false)) + } + if (leftTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true)) + } + logLine("queue mic control $label") + } + interruptibleSleep.interrupt() + } + + /** + * Forward render-characteristic audio packets to the listener WITHOUT + * sending the stock EvenHub audio-control enable. Used for the CFW + * mic_control streaming path, where capture is armed through settings + * field 103 and the temples emit 'SM' frames on the same characteristic + * that stock mono LC3 uses. Returns false when no session is up. + */ + fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean { + if (listener == null) { + throw IllegalArgumentException("listener is required") + } + monitor.withLock { + if (!running || !sessionReady || shutdownRequested) { + logLine("skip G2 audio forwarding; session not ready") + return false + } + audioPacketListener = listener + audioCaptureActive = true + logLine("G2 audio forwarding enabled") + } + return true + } + + fun stopG2AudioForwarding() { + monitor.withLock { + audioPacketListener = null + audioCaptureActive = false + logLine("G2 audio forwarding disabled") + } + } + + /** + * Subscribe to compass events. Callbacks are delivered on the dispatcher of + * the thread that registered (falling back to the main thread), so app + * worker isolates can listen without a cross-thread hop into their JS. + */ + fun addCompassListener(listener: FaceclawCompassListener?) { + if (listener == null) { + return + } + compassSubscriptions.add(CompassSubscription(listener, host.currentThreadDispatcher())) + } + + fun removeCompassListener(listener: FaceclawCompassListener?) { + if (listener == null) { + return + } + for (subscription in compassSubscriptions) { + if (subscription.listener === listener) { + compassSubscriptions.remove(subscription) + } + } + } + + // --------------------------------------------------------------------- + // Compositor entry points (pixels arrive as ByteReaders; the platform + // adapter wraps its native buffers without copying) + + /** Set the compositor's output frame size. Call before configuring surfaces. */ + fun configureCompositorScreen(width: Int, height: Int) { + compositor.configureScreen(width, height) + } + + /** The current composite for previews/screenshots, or null before any surface exists. */ + fun previewComposite(): SurfaceCompositor.Composite? = compositor.previewComposite() + + /** + * Show or hide a compositor surface, immediately submitting the resulting + * frame. Recompositing here (rather than waiting for the next surface + * update) is what makes a just-foregrounded window's retained frame + * actually appear — otherwise a static window (e.g. the terminal hub) whose + * frame landed while briefly hidden would stay blank until its next repaint. + */ + fun setSurfaceVisible(id: String, visible: Boolean) { + // Its own frame: this recomposite is a real screen update with real + // latency, and without one it would show up in other frames' logs only + // as an anonymous "superseded by frame#0". + val frameId = frameTimings.startFrame( + "compositor:visible $id=$visible") + compositor.setSurfaceVisible(id, visible) + val composite = compositor.composite() + val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) + storeDesiredComposite(composite, packed, 0, frameId) + } + + /** + * Blank (screen off) or unblank the composited output, immediately + * submitting the resulting frame. Retained surface state is untouched, so + * unblanking restores the previous screen content without repaints. + */ + fun setScreenBlanked(blanked: Boolean) { + val frameId = frameTimings.startFrame( + "compositor:" + (if (blanked) "blank" else "unblank")) + compositor.setBlanked(blanked) + val composite = compositor.composite() + val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) + storeDesiredComposite(composite, packed, 0, frameId) + } + + /** + * Create or reconfigure a compositor surface. transparency is one of the + * SurfaceCompositor.TRANSPARENCY_* constants. Geometry changes take effect + * when the next frame is submitted. + */ + fun configureSurface(id: String?, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) { + compositor.configureSurface(id, x, y, width, height, zOrder, transparency) + } + + fun removeSurface(id: String) { + compositor.removeSurface(id) + } + + /** Stage an immutable shell snapshot alongside the retained app screen. */ + fun submitShellScene(bytes: ByteReader, paintMs: Int, frameId: Int) { + compositor.setShellScene(bytes) + val composite = compositor.composite() + storeDesiredComposite(composite, BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height), paintMs, frameId) + } + + /** Legacy compositor dimming for callers without a shell scene. */ + fun setUnderlayDim(belowZOrder: Int, factor256: Int) { + compositor.setUnderlayDim(belowZOrder, factor256) + } + + /** + * Apply an update to one compositor surface and submit the recomposited + * screen as the desired frame. The update covers the rect (rectX, rectY, + * rectWidth, rectHeight) in surface-local coordinates; contentFingerprint + * identifies the surface's full content after the update. glyphs carries + * the frame's glyph draws (see SurfaceCompositor's glyph overload for the + * buffer format), or null when the submitter has no glyph metadata; the + * pixels alone remain fully correct. + */ + fun submitSurfaceFrame( + pixels8bpp: ByteReader, + surfaceId: String, + rectX: Int, + rectY: Int, + rectWidth: Int, + rectHeight: Int, + contentFingerprint: String?, + paintMs: Int, + frameId: Int, + glyphs: ByteReader? + ) { + logLine("Received an updated frame for surface $surfaceId") + frameTimings.log(frameId, "surface " + surfaceId + " updated rect=" + + rectWidth + "x" + rectHeight + "+" + rectX + "+" + rectY + + (if (glyphs == null) " (no glyph draws)" else "")) + frameTimings.spanStart(frameId, "composite") + val composite = compositor.applyAndComposite( + surfaceId, pixels8bpp, rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs) + frameTimings.spanEnd(frameId, "composite") + // Pack the composited 8bpp buffer down to the headerless 4bpp frame + // format the wire planners consume; BMP framing is added later only for + // the uncompressed fallback. + frameTimings.spanStart(frameId, "pack-4bpp") + val packed = BmpUtil.pack4bppFromGray8(composite.screenGray, composite.width, composite.height) + frameTimings.spanEnd(frameId, "pack-4bpp") + storeDesiredComposite(composite, packed, paintMs, frameId) + } + + /** Store a composite as the desired frame unless a newer one won the race. */ + internal fun storeDesiredComposite(composite: SurfaceCompositor.Composite, packed: ByteArray, paintMs: Int, frameId: Int) { + var supersededFrameId = 0 + var stale = false + desiredTilesLock.locked { + if (composite.seq <= lastStoredCompositeSeq) { + // A concurrent submission composited after us and stored first; + // its composite already includes this surface update. + stale = true + } else { + lastStoredCompositeSeq = composite.seq + supersededFrameId = desiredFrameId + desiredPacked = packed + desiredWidth = composite.width + desiredHeight = composite.height + desiredFingerprint = composite.fingerprint + desiredPaintMs = paintMs + desiredFrameId = frameId + desiredDraws = composite.draws + desiredShellScene = composite.shellScene + } + } + if (stale) { + finishFrame(frameId, "discarded: composite superseded before store") + return + } + if (supersededFrameId != 0 && supersededFrameId != frameId) { + finishFrame(supersededFrameId, "discarded: superseded by frame#$frameId before send") + } + frameTimings.log(frameId, "image submitted as desired frame") + interruptibleSleep.interrupt() + } + + /** + * Play a tone sequence via CFW load_image_z mode 5 kind 4. The payload is + * the complete wire buffer ([5][4][nSteps][freqLo,freqHi,duty,msLo,msHi]*n, + * up to 48 steps), built on the TS side; it rides the arbitrary-payload + * image path like the other mode-5 controls. + */ + fun playBuzzerSequence(bytes: ByteArray?) { + monitor.withLock { + if (!running || !sessionReady || !fixedLayoutCreated) { + logLine("skip buzzer sequence; session not ready") + return + } + if (bytes == null || bytes.size < 3) { + logLine("skip buzzer sequence; empty payload") + return + } + val message = messageBuilder.imagePayload( + DASHBOARD_TILE, + nextMapSessionId(), + bytes, + "buzzer sequence " + bytes.size + "B", + connectionOptions.sendImagesToLeft + ) + message.onTimeout = MessageCallback { + handleTransportFailure("buzzer sequence ack timeout") + } + pendingMessages.addLast(message) + logLine("queue " + message.label) + } + interruptibleSleep.interrupt() + } + + // --------------------------------------------------------------------- + // EvenHub page lifecycle + + fun sendShutdown(exitMode: Int): Boolean { + return sendShutdownInternal(exitMode, true) + } + + /** + * Send CFW image-handler mode 11 after quiescing normal traffic. A successful + * return means the cleanup was ACKed and no later Faceclaw message should be + * emitted before closing BLE. Unsupported/older CFWs return false so callers + * can use the legacy shutdown-and-lease-release path. + */ + fun sendCfwCleanup(): Boolean { + val magic: Int + monitor.withLock { + if (cfwCleanupDelivered) { + return true + } + if (!customFirmwareDetected || !running || !sessionReady + || shutdownRequested || !fixedLayoutCreated) { + logLine("skip CFW cleanup; mode 11 unavailable or image path not ready") + return false + } + + /* Stop auto-renewals, heartbeats, image generation, and control + * retries, then discard everything that has not reached BLE yet. */ + shutdownRequested = true + lastCfwCleanupAckMagic = 0 + clearPendingMessagesLocked("CFW cleanup requested") + logLine("quiescing transport for CFW cleanup") + } + interruptibleSleep.interrupt() + + /* WINDOW_SIZE can exceed one, so merely appending cleanup would allow it + * to overlap previously-written image fragments. Wait until all of those + * ACK or time out; shutdownRequested prevents the drive loop from adding + * any fresh automatic traffic meanwhile. */ + val drainDeadline = now() + CFW_CLEANUP_WAIT_MS + monitor.withLock { + while (running && sessionReady && !inFlightMessages.isEmpty()) { + val remaining = drainDeadline - now() + if (remaining <= 0) break + monitor.awaitMs(minOf(remaining, 100L)) + } + if (!running || !sessionReady || !inFlightMessages.isEmpty()) { + shutdownRequested = false + logLine("CFW cleanup could not drain prior traffic") + return false + } + + /* External producers are expected to be stopped by the caller, but + * clear once more at the barrier so cleanup is definitely last. */ + clearPendingMessagesLocked("CFW cleanup barrier") + val message = messageBuilder.cfwCleanup( + DASHBOARD_TILE, + nextMapSessionId(), + connectionOptions.sendImagesToLeft + ) + magic = message.magic + message.onAck = MessageCallback { + lastCfwCleanupAckMagic = message.magic + cfwCleanupDelivered = true + compassMaybeOn = false + faceclawWakeLeaseEnabled = false + logLine("CFW cleanup completed") + } + message.onTimeout = MessageCallback { logLine("CFW cleanup ack timeout") } + pendingMessages.addLast(message) + logLine("queue CFW cleanup") + } + interruptibleSleep.interrupt() + + val deadline = now() + CFW_CLEANUP_WAIT_MS + monitor.withLock { + while (running + && sessionReady + && lastCfwCleanupAckMagic != magic + && hasPendingOrInflightMagicLocked(magic)) { + val remaining = deadline - now() + if (remaining <= 0) break + monitor.awaitMs(minOf(remaining, 100L)) + } + val acked = lastCfwCleanupAckMagic == magic + if (!acked) shutdownRequested = false + return acked + } + } + + /** + * End the EvenHub page while retaining both arm GATT connections and all + * notification subscriptions. A missed ACK is intentionally non-fatal: + * reconnecting Bluetooth here would defeat the power-saving mode. + */ + fun suspendEvenHubSession(): Boolean { + // Ending the plugin task releases the fb lease, which frees the + // on-glasses resource cache; forget it phone-side either way (a lost + // ack may still have taken effect). + monitor.withLock { + resourceCache.reset() + } + if (sendShutdownInternal(0, false)) { + return true + } + monitor.withLock { + // A shutdown ACK can be lost even though the command took effect. + // If the transport remains intentionally quiesced, callers still + // need to remember to run the resume path on the next wake. + return running && sessionReady && shutdownRequested + } + } + + /** + * Start a fresh EvenHub plugin task on the existing BLE transport, then let + * the session driver create the layout, warm up the image path, and send + * the desired frame. + */ + fun resumeEvenHubSession(): Boolean { + var claimGeneration = 0 + monitor.withLock { + if (!running || !sessionReady || chargingMode) { + logLine("skip EvenHub resume; transport not ready") + return false + } + if (!shutdownRequested) { + return true + } + if (faceclawWakePendingNonce >= 0 + && hasPendingOrInflightKindLocked("wake-lease-control")) { + claimGeneration = faceclawWakeControlGeneration + } + logLine("replaying session prelude for EvenHub resume") + } + + // A custom double-tap wake has only a short unclaimed fail-open + // deadline. Let the worker put CLAIM on both arms before the direct + // prelude write begins. + if (claimGeneration != 0) { + waitForFaceclawWakeControlDelivery(claimGeneration, 500) + } + + try { + // Empty-name Cmd=9 tears down the whole plugin task, not just its + // image container. Re-run the launch prelude before Cmd=0 CREATE. + sendPrelude(true) + } catch (t: Throwable) { + logLine("EvenHub resume prelude failed: " + safeMessage(t)) + handleTransportFailure("EvenHub resume prelude failed") + return false + } + + monitor.withLock { + if (!running || !sessionReady || chargingMode) { + return false + } + shutdownRequested = false + fixedLayoutCreated = false + startupProbePending = false + audioCaptureActive = false + clearAllMessagesPreservingWakeLeaseLocked("EvenHub resume") + displayedFingerprint = "" + imageRetryAfterMs = 0 + lastHeartbeatSentAtMs = 0 + lastHeartbeatAckedAtMs = 0 + logLine("EvenHub session resume requested") + } + interruptibleSleep.interrupt() + return true + } + + // --------------------------------------------------------------------- + // Worker loop and inbound traffic + + /** The worker loop; runs on the thread the host started from [start]. */ + fun run() { + logLine("communicator start R=$rightAddress L=$leftAddress ring=$ringAddress") + while (true) { + try { + if (!running) { + logWarn("Exiting event looop") + break + } + emitPhoneLockStateIfChanged(false) + if (!sessionReady) { + if (reconnectHalted) { + interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS.toLong()) + continue + } + val now = now() + if (now < reconnectAfterMs) { + interruptibleSleep.sleep(minOf(ConnectionOptions.IDLE_SLEEP_MS.toLong(), reconnectAfterMs - now)) + continue + } + logWarn("Attempting to connect") + connectLoopOnce() + continue + } + + if (shouldAttemptRingConnect()) { + tryConnectRing("retry") + continue + } + + val sleepMs = driveSession() + if (sleepMs > 0) { + interruptibleSleep.sleep(sleepMs) + } + } catch (t: Throwable) { + if (!running) { + // The host interrupted the worker to stop it (JVM thread + // interrupt surfacing from a wait); nothing to recover. + logLine("communicator loop interrupted while stopping: " + (t.message ?: t.toString())) + break + } + logLine("communicator loop error: " + safeMessage(t)) + handleTransportFailure("loop error") + } + } + logLine("communicator stop") + } + + override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) { + if (address == null || characteristicUuid == null || data == null) { + return + } + val uuid = characteristicUuid.lowercase() + if (isDirectRingNotification(address, uuid)) { + handleDirectRingNotification(uuid, data) + return + } + if (BleProtocol.RENDER_NOTIFY_UUID == uuid) { + handleRenderNotification(address, data) + return + } + if (BleProtocol.NOTIFY_CHAR_UUID != uuid) { + return + } + if (data.size >= 7 && (data[6].toInt() and 255) == CfwTransport.SID) { + val acks = CfwTransport.parseAcks(data) ?: return + monitor.withLock { + lastIncomingAtMs = now() + for (ack in acks) { + for (message in inFlightMessages) { + val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress + if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) { + message.acceptCfwAck(ack) + logLine("CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId + + " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2].toInt() and 255) + + " redundant=" + (ack !== acks[0]) + + " size=" + ack.size + " crc=" + ack.checksum + + " expected=" + message.message.size + "/" + message.cfwChecksum + + " ackedLenses=" + message.cfwAckLenses + + " ageMs=" + (lastIncomingAtMs - message.sentAtMs)) + message.ackPayload = data.copyOf() + break + } + } + } + drainCfwAcknowledgementsLocked() + } + interruptibleSleep.interrupt() + return + } + logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size) + val frame = BleProtocol.parseFrame(data) + val decodedWearState = BleProtocol.parseWearState(frame) + val compassEvent = if (address.equals(rightAddress, ignoreCase = true)) + BleProtocol.parseCompassEvent(frame) + else + null + var emitWearState = false + var event: G2Event? = null + monitor.withLock { + lastIncomingAtMs = now() + if (decodedWearState >= 0 && decodedWearState != wearState) { + wearState = decodedWearState + emitWearState = true + } + var faceclawWakeNotification = false + if (shutdownRequested + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING + && address.equals(rightAddress, ignoreCase = true)) { + val wakeNonce = BleProtocol.parseFaceclawWakeEvent(frame.pb) + if (wakeNonce >= 0) { + faceclawWakePendingNonce = wakeNonce + enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_CLAIM, + wakeNonce, + true + ) + faceclawWakeNotification = true + lastConnectionOrInputAtMs = lastIncomingAtMs + // The event type says which gesture woke the glasses: TS + // gives a head-up the Glanceboard and a double tap the + // regular UI. The claim handshake is the same for both. + val headUp = BleProtocol.parseFaceclawWakeEventCode(frame.pb) == + BleProtocol.FACECLAW_WAKE_EVENT_HEAD_UP + event = G2Event( + "display-wake", + "", + if (headUp) BleProtocol.EVENT_HEAD_UP else BleProtocol.EVENT_DOUBLE_CLICK, + 0, + 0 + ) + logLine("claimed deferred dashboard wake nonce=" + wakeNonce + + (if (headUp) " (head-up)" else "")) + } + } + if (!faceclawWakeNotification + && shutdownRequested + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING + && address.equals(rightAddress, ignoreCase = true)) { + // CFW idle-gesture forwarding (firmware revision 2+): with no + // EvenHub page the stock display thread drops taps, long + // presses and releases; the CFW reports them on the settings + // sid while our wake lease is held. Deliver them as the + // sys-events a live page would have produced, so TS treats a + // sleep-time tap or hold exactly like one during soft sleep + // (the Glanceboard). + val gesture = BleProtocol.parseFaceclawGestureEvent(frame.pb) + if (gesture != null) { + lastConnectionOrInputAtMs = lastIncomingAtMs + event = G2Event("sys-event", "", gesture.eventType, gesture.eventSource, 0) + logLine("idle gesture forwarded by CFW: type=" + gesture.eventType + + " source=" + gesture.eventSource) + } + } + if (!faceclawWakeNotification + && event == null + && shutdownRequested + && address.equals(rightAddress, ignoreCase = true) + && BleProtocol.isDisplayWakeStateChange(frame)) { + // With no EvenHub page, ring/arm double-taps are handled by the + // stock display lifecycle and surface only as this state ping. + // Translate it back into an input event so TS can wake the shell + // and recreate the page. + event = G2Event( + "display-wake", + "", + BleProtocol.EVENT_DOUBLE_CLICK, + 0, + 0 + ) + } + if (!faceclawWakeNotification + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // Mode-17 query notifications; settings READs are handled by + // createBatteryQueryMessageLocked so headset/ring update together. + if (address.equals(rightAddress, ignoreCase = true) + && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { + val ring = BleProtocol.parseRingBattery(frame.pb) + if (ring != null) { + ringBattery = ring.battery + ringCharging = ring.charging + emitBatteryState(headsetBattery, headsetCharging) + } + } + // CFW mic status (field 104) rides both standalone pushes and + // settings read acks, from each temple on its own link. + val micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb) + if (micStatus != null) { + emitMicStatus(micStatus, address) + } + } + if (!faceclawWakeNotification + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // CFW ambient-light report (field 105) from the master temple. + val alsReport = BleProtocol.parseFaceclawAlsReport(frame.pb) + if (alsReport != null) { + emitAmbientLight(alsReport) + } + } + if (!faceclawWakeNotification + && decodedWearState < 0 + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // Device-initiated settings push. It carries a magic the glasses + // chose, so it would otherwise fall through to resolveAckLocked, + // match nothing, and be logged as an unexpected ack. + val pushedSilentMode = BleProtocol.parseSilentModePush(frame.pb) + if (pushedSilentMode >= 0) { + updateSilentModeLocked(pushedSilentMode > 0) + return + } + } + if (!faceclawWakeNotification + && decodedWearState < 0 + && frame.ok + && frame.msgSeq >= 0 + && frame.flag != BleProtocol.FLAG_NOTIFY + && frame.flag != BleProtocol.FLAG_NOTIFY_ALT) { + lastAckAtMs = lastIncomingAtMs + resolveAckLocked(frame.sid, frame.msgSeq, frame.pb) + } + if (!faceclawWakeNotification + && event == null + && frame.ok + && address.equals(rightAddress, ignoreCase = true) + && (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT)) { + var decoded = G2Event.decode(frame) + if (decoded != null + && "sys-event" == decoded.kind + && decoded.eventType == BleProtocol.EVENT_HEAD_UP) { + // CFW forwards the IMU head-up while our page is on screen + // (soft sleep). Surface it as the same wake-only input the + // deferred head-up wake produces from a dark display. + decoded = G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, decoded.eventSource, 0) + logLine("head-up forwarded by CFW while page on screen") + } + event = decoded + if (decoded != null) { + // Pure IMU samples arrive continuously; don't let them count + // as user input (which would starve battery polling). + val pureImuSample = "sys-event" == decoded.kind + && decoded.eventType == BleProtocol.EVENT_IMU_DATA_REPORT + if (!pureImuSample) { + lastConnectionOrInputAtMs = lastIncomingAtMs + } + if ("list-click" == decoded.kind || "text-click" == decoded.kind) { + // Container-routed touchpad input reached us, so the + // firmware is dispatching input: silent mode is off, + // whether or not its end-of-silent push arrived. + updateSilentModeLocked(false) + } + if ("sys-event" == decoded.kind) { + if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT || decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) { + if (shutdownRequested) { + lastShutdownExitAtMs = now() + } + fixedLayoutCreated = false + displayedFingerprint = "" + clearAllMessagesLocked("firmware exit event") + } + } + } + } + } + interruptibleSleep.interrupt() + if (emitWearState) { + logLine(if (decodedWearState > 0) "wear state ON_HEAD" else "wear state OFF_HEAD") + emitWearState(decodedWearState > 0) + } + if (compassEvent != null) { + emitCompassEvent(compassEvent) + } + val finalEvent = event + if (finalEvent != null) { + if (finalEvent.hasImu) { + emitImuData(finalEvent.imuX, finalEvent.imuY, finalEvent.imuZ, finalEvent.eventSource) + } + // A standalone IMU_DATA_REPORT is a sensor sample, not a gesture: + // deliver it only to IMU listeners, skipping the input pipeline (and + // its per-frame latency bookkeeping) to avoid flooding it. + val pureImuSample = "sys-event" == finalEvent.kind + && finalEvent.eventType == BleProtocol.EVENT_IMU_DATA_REPORT + if (!pureImuSample) { + val frameId = frameTimings.startFrame( + "input:" + finalEvent.kind + " type=" + finalEvent.eventType + " src=" + finalEvent.eventSource) + frameTimings.log(frameId, "input event decoded from BLE notification") + emitRingEvent(finalEvent, frameId) + } + } + } + + internal fun handleDirectRingNotification(characteristicUuid: String, data: ByteArray) { + val decoded = FaceclawRingEventDecoder.decode(data) + if (decoded == null) { + logDebug("direct ring notify ignored: characteristicUuid=" + characteristicUuid + " raw=" + hex(data)) + return + } + + val event = decoded.event + val arrivalMs = now() + monitor.withLock { + lastIncomingAtMs = arrivalMs + lastConnectionOrInputAtMs = arrivalMs + } + logLine("direct ring " + decoded.label + " " + decoded.detail + " raw=" + hex(data)) + val frameId = frameTimings.startFrame("input:ring:" + decoded.label) + frameTimings.log(frameId, "input event decoded from direct ring notification") + emitRingEvent(event, frameId) + interruptibleSleep.interrupt() + } + + internal fun handleRenderNotification(address: String, data: ByteArray) { + val listenerToCall: FaceclawAudioPacketListener? + val arrivalMs = now() + monitor.withLock { + lastIncomingAtMs = arrivalMs + listenerToCall = if (audioCaptureActive) audioPacketListener else null + } + if (listenerToCall == null) { + return + } + val arm = if (address.equals(leftAddress, ignoreCase = true)) "L" else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?" + try { + listenerToCall.onAudioPacket(data.copyOf(), arm, arrivalMs) + } catch (t: Throwable) { + logLine("G2 mic packet listener failed: " + safeMessage(t)) + } + } + + override fun onConnectionStateChange(address: String?, connected: Boolean) { + monitor.withLock { + if (address == null) { + return + } + if (isConfiguredRingAddress(address)) { + ringConnected = connected + ringNotificationsReady = false + if (!connected) { + ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS + } + logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected") + return + } + if (address.equals(rightAddress, ignoreCase = true)) { + rightConnected = connected + } else if (address.equals(leftAddress, ignoreCase = true)) { + leftConnected = connected + } else { + return + } + if (!connected) { + sessionReady = false + fixedLayoutCreated = false + startupProbePending = false + chargingMode = false + audioCaptureActive = false + audioPacketListener = null + clearAllMessagesLocked("connection lost") + displayedFingerprint = "" + if (!reconnectHalted) { + reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS + } + } + } + interruptibleSleep.interrupt() + if (connected) { + setStateDisplay("connected", "Connected.") + } else if (!reconnectHalted) { + // While parked on a missing bond, keep the "unpaired" display: this + // callback is just the teardown of the arm that did connect. + setStateDisplay("connecting", "Connecting to the glasses...") + } + } + + // --------------------------------------------------------------------- + // Connect sequence (worker thread) + + internal fun connectLoopOnce() { + setStateDisplay("connecting", "Connecting to the glasses...") + try { + connectArm(rightAddress, true) + connectArm(leftAddress, true) + if (!sleepDuringConnectSettling(800)) { + return + } + authenticateArms() + sendPrelude() + + monitor.withLock { + sessionReady = true + // A fresh transport prelude always starts an active EvenHub + // lifecycle, even if the previous connection dropped while + // its page was intentionally suspended. + shutdownRequested = false + fixedLayoutCreated = false + clearAllMessagesLocked("session ready") + displayedFingerprint = "" + lastAckAtMs = now() + lastIncomingAtMs = lastAckAtMs + lastConnectionOrInputAtMs = lastAckAtMs + lastSessionReadyAtMs = lastAckAtMs + lastBatteryRefreshAtMs = 0 + imageRetryAfterMs = 0 + lastHeartbeatSentAtMs = 0 + lastHeartbeatAckedAtMs = 0 + consecutiveAckTimeouts = 0 + lastAudioControlAckMagic = 0 + audioCaptureActive = false + faceclawWakePendingNonce = -1 + cfwCleanupDelivered = false + for (transport in cfwTransports) transport.reset() + lastCfwCleanupAckMagic = 0 + lastFaceclawWakeLeaseQueuedAtMs = 0 + lastFaceclawFramebufferLeaseQueuedAtMs = 0 + enqueueFaceclawFramebufferControlLocked( + BleProtocol.FACECLAW_FB_OP_ACQUIRE, + true + ) + if (faceclawWakeLeaseEnabled) { + enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, + 0, + true + ) + } + } + setStateDisplay("connected", "Connected.") + logLine("session ready") + monitor.withLock { + // Query settings promptly on the first session so firmware + // version/extension (and battery) arrive without waiting for + // the input-quiet battery poll. The settings response doubles as + // the firmware-compatibility check surfaced during onboarding. + if (!firmwareInfoQueried) { + firmwareInfoQueried = true + lastBatteryRefreshAtMs = now() + pendingMessages.addLast(createBatteryQueryMessageLocked()) + logLine("queue settings query for firmware info") + } + } + tryConnectRing("initial") + } catch (t: Throwable) { + logLine("connect failed: " + safeMessage(t)) + val unpairedArm = firstUnpairedArm() + if (unpairedArm != null) { + handleUnpairedFailure(unpairedArm) + } else { + handleTransportFailure("connect failed") + } + } + } + + internal fun sleepDuringConnectSettling(delayMs: Long): Boolean { + val deadline = now() + delayMs + monitor.withLock { + while (running && !userDisconnectRequested) { + val remaining = deadline - now() + if (remaining <= 0) { + return true + } + monitor.awaitMs(minOf(remaining, 100L)) + } + return false + } + } + + internal fun connectArm(address: String, enableRenderNotify: Boolean) { + if (!link.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + throw IllegalStateException("connect failed: $address") + } + // requestConnectionPriority has no callback in this Android compile target, so there is + // no reliable completion point to keep it in the global GATT operation pipeline. But it's + // important enough for performance that we call it anyways. + link.requestHighPriority(address) + + link.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + + if (!link.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + throw IllegalStateException("discoverServices failed: $address") + } + if (!link.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + throw IllegalStateException("enableNotifications failed: " + address + " " + BleProtocol.NOTIFY_CHAR_UUID) + } + if (enableRenderNotify) { + link.enableNotifications(address, BleProtocol.RENDER_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS) + } + monitor.withLock { + if (address.equals(rightAddress, ignoreCase = true)) { + rightConnected = true + } else if (address.equals(leftAddress, ignoreCase = true)) { + leftConnected = true + } + } + } + + internal fun shouldAttemptRingConnect(): Boolean { + if (!hasRingAddress()) { + return false + } + val now = now() + monitor.withLock { + return running + && sessionReady + && !ringNotificationsReady + && now >= ringReconnectAfterMs + && pendingMessages.isEmpty() + && inFlightMessages.isEmpty() + } + } + + internal fun tryConnectRing(reason: String) { + if (!hasRingAddress()) { + return + } + try { + connectRing() + } catch (t: Throwable) { + monitor.withLock { + ringConnected = false + ringNotificationsReady = false + ringReconnectAfterMs = now() + ConnectionOptions.RING_RECONNECT_DELAY_MS + } + logLine("direct ring connect failed (" + reason + "): " + safeMessage(t)) + } + } + + internal fun connectRing() { + logLine("connecting direct ring $ringAddress") + if (!link.connect(ringAddress, ConnectionOptions.CONNECT_TIMEOUT_MS)) { + throw IllegalStateException("connect failed: $ringAddress") + } + + link.requestHighPriority(ringAddress) + link.requestMtu(ringAddress, ConnectionOptions.RING_DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS) + + if (!link.discoverServices(ringAddress, ConnectionOptions.SERVICES_TIMEOUT_MS)) { + throw IllegalStateException("discoverServices failed: $ringAddress") + } + + val phoneNotify = enableRingNotification(BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID) + val dataNotify = enableRingNotification(BleProtocol.R1_NOTIFY_CHAR_UUID) + if (!phoneNotify && !dataNotify) { + throw IllegalStateException("no R1 notify characteristic subscribed") + } + + monitor.withLock { + ringConnected = true + ringNotificationsReady = true + ringReconnectAfterMs = 0 + } + logLine("direct ring ready phoneNotify=$phoneNotify dataNotify=$dataNotify") + } + + internal fun enableRingNotification(characteristicUuid: String): Boolean { + try { + return link.enableNotifications( + ringAddress, + characteristicUuid, + true, + ConnectionOptions.DESCRIPTOR_TIMEOUT_MS + ) + } catch (t: Throwable) { + logDebug("direct ring notify subscribe skipped: " + characteristicUuid + " " + safeMessage(t)) + return false + } + } + + /** + * Complete the sid-0x80 security-auth exchange on both freshly opened arm + * connections. Firmware 2.2.9 answers no queries until it completes over an + * encrypted link and closes unauthenticated links after ~30 s (see + * ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is + * also what triggers SMP pairing. Deliberately soft: on timeout we log and + * continue rather than fail the connect — the custom firmware's response + * behavior is not yet hardware-verified, and on stock firmware an + * unanswered auth just means the prelude fails exactly as it did before. + * A pairing prompt accepted after our window still bonds at the OS level, + * so the next reconnect attempt authenticates promptly. + */ + internal fun authenticateArms() { + val right = messageBuilder.securityAuth(false) + val left = messageBuilder.securityAuth(true) + val now = now() + for (message in arrayOf(right, left)) { + message.onAck = MessageCallback { + } + message.onTimeout = MessageCallback { + } + message.sentAtMs = now + writeMessage(message) + } + val deadline = now() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS + while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { + val done = monitor.withLock { + !running || userDisconnectRequested || inFlightMessages.isEmpty() + } + if (done) { + break + } + val remaining = deadline - now() + if (remaining <= 0) { + break + } + interruptibleSleep.sleep(minOf(remaining, 100L)) + } + monitor.withLock { + if (!inFlightMessages.isEmpty()) { + clearInFlightMessagesLocked("security auth timeout") + logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)") + return + } + } + val rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic) + val leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic) + logLine("security auth R=" + (if (rightOk) "ok" else "unconfirmed") + " L=" + (if (leftOk) "ok" else "unconfirmed")) + } + + internal fun sendPrelude() { + sendPrelude(false) + } + + internal fun sendPrelude(preserveWakeLeaseControls: Boolean) { + monitor.withLock { + if (preserveWakeLeaseControls) { + clearAllMessagesPreservingWakeLeaseLocked("prelude") + } else { + clearAllMessagesLocked("prelude") + } + } + val now = now() + val prelude = messageBuilder.prelude() + prelude.onAck = MessageCallback { + } + prelude.onTimeout = MessageCallback { + handleTransportFailure("ack timeout") + } + prelude.sentAtMs = now + writeMessage(prelude) + + val deadline = now() + ConnectionOptions.PRELUDE_TIMEOUT_MS + while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { + val done = monitor.withLock { + !running || userDisconnectRequested || inFlightMessages.isEmpty() + } + if (done) { + break + } + val remaining = deadline - now() + if (remaining <= 0) { + break + } + interruptibleSleep.sleep(minOf(remaining, 100L)) + } + monitor.withLock { + if (!inFlightMessages.isEmpty()) { + clearInFlightMessagesLocked("prelude timeout") + throw IllegalStateException("prelude ack timeout") + } + } + } + + // --------------------------------------------------------------------- + // Small helpers shared by the extension files + + internal fun nextMapSessionId(): Int { + val id = nextMapSessionIdValue + val increment = if (connectionOptions.skipSessionIds) 2 else 1 + nextMapSessionIdValue = (nextMapSessionIdValue + increment) and 0xff + return id + } + + internal fun hasRingAddress(): Boolean { + return ringAddress.trim().isNotEmpty() + } + + internal fun isConfiguredRingAddress(address: String?): Boolean { + return hasRingAddress() && address != null && address.equals(ringAddress, ignoreCase = true) + } + + internal fun isDirectRingNotification(address: String?, characteristicUuid: String?): Boolean { + if (!isConfiguredRingAddress(address) || characteristicUuid == null) { + return false + } + return BleProtocol.R1_PHONE_NOTIFY_CHAR_UUID == characteristicUuid + || BleProtocol.R1_NOTIFY_CHAR_UUID == characteristicUuid + } + + internal fun desiredFingerprintSnapshot(): String { + desiredTilesLock.locked { + return desiredFingerprint + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt new file mode 100644 index 00000000..8411cb3b --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt @@ -0,0 +1,1301 @@ +package com.faceclaw.app + +import com.faceclaw.app.GlassesSessionCore.Companion.DASHBOARD_TILE +import com.faceclaw.app.GlassesSessionCore.Companion.COMPASS_MIN_CHANGE_DEGREES +import com.faceclaw.app.GlassesSessionCore.Companion.COMPASS_REPORT_INTERVAL_MS +import com.faceclaw.app.GlassesSessionCore.Companion.FACECLAW_WAKE_LEASE_RENEW_MS +import com.faceclaw.app.GlassesSessionCore.Companion.safeMessage +import com.faceclaw.app.GlassesSessionCore.Companion.timestamp + +/* + * GlassesSessionCore, part 2: the send scheduler (driveSession), the CFW/EvenHub + * ack window, the desired-image pipeline and the message-queue bookkeeping. + * Everything here runs on the worker thread or under `monitor` (functions + * suffixed Locked require the caller to hold it). + */ + +internal fun GlassesSessionCore.driveSession(): Long { + while (true) { + // Run on the sender thread, outside `monitor`: Bluetooth API calls can + // wait on the global GATT lock, and callbacks need the session lock. + // Keep negotiation outside the timed stream. A fixed settling period + // is only an experiment boundary; HCI must confirm actual parameters. + var linkMode = -1 + var benchmarkAddresses: Array? = null + monitor.withLock { + if (benchmarkActive && benchmarkLinkPending) { + benchmarkLinkPending = false + linkMode = benchmarkLinkMode + benchmarkAddresses = arrayOf(leftAddress, rightAddress) + } + } + val addressesToPrepare = benchmarkAddresses + if (addressesToPrepare != null) { + for (address in addressesToPrepare) { + try { + link.prepareBenchmarkLink(address, linkMode) + } catch (error: RuntimeException) { + logLine("benchmark link request failed: " + safeMessage(error)) + } + } + monitor.withLock { + // Cancellation/new-run races leave the new run pending; its + // own preparation will replace this deadline before sending. + benchmarkReadyAtMs = now() + 1_000 + } + } + var messageToWrite: OutboundMessage? = null + var messageToPrewrite: OutboundMessage? = null + val now = now() + + maybeFinishNoChangeDesiredFrame() + + monitor.withLock { + drainCfwAcknowledgementsLocked() + if (cfwCleanupDelivered) { + /* Successful mode 11 must be the last Faceclaw write. Drop + * anything a late external producer attempted to enqueue + * while DashboardController was closing the transport. */ + clearPendingMessagesLocked("after CFW cleanup") + return 250 + } + if (sessionReady + && now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS + && !hasPendingOrInflightKindLocked("framebuffer-lease-control")) { + enqueueFaceclawFramebufferControlLocked( + BleProtocol.FACECLAW_FB_OP_ACQUIRE, + false + ) + } + if (faceclawWakeLeaseEnabled + && sessionReady + && now - lastFaceclawWakeLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS + && !hasPendingOrInflightKindLocked("wake-lease-control")) { + enqueueFaceclawWakeControlLocked( + BleProtocol.FACECLAW_WAKE_OP_ACQUIRE, + 0, + false + ) + } + if (!inFlightMessages.isEmpty()) { + val oldest = inFlightMessages.firstOrNull() + val replay = CfwMessageWindow.replayWindow(inFlightMessages, now) + if (!replay.isEmpty()) { + logLine("CFW recovery: replay " + replay.size + + " unresolved message(s) from id=" + replay[0].magic) + for (candidate in replay) { + if (candidate.cfwRetries >= CfwMessageWindow.MAX_RETRIES) { + handleTransportFailure("CFW recovery retry limit") + return 0 + } + logLine("CFW replay id=" + candidate.magic + " label=" + candidate.label + + " cause=" + (if (candidate.cfwRetryPending) "NACK" + else if (candidate.cfwAckLenses == CfwTransport.BOTH) "ordered tail" else "missing ACK") + + " ackedLenses=" + candidate.cfwAckLenses + + " ageMs=" + (now - candidate.sentAtMs)) + inFlightMessages.remove(candidate) + magicPool.release(candidate.sid, candidate.magic, candidate.label, "CFW replay") + candidate.prepareCfwReplay(magicPool.allocate()) + } + for (i in replay.indices.reversed()) pendingMessages.addFirst(replay[i]) + return 0 + } + if (oldest != null && oldest.ackDeadlineAtMs <= now) { + logLine("message timed out: " + oldest.label) + inFlightMessages.removeFirst() + logLine("message timed out: " + oldest.label + " sid=" + oldest.sid + + " id=" + oldest.magic + " ackedLenses=" + oldest.cfwAckLenses + + " ageMs=" + (now - oldest.sentAtMs)) + magicPool.release(oldest.sid, oldest.magic, oldest.label, "timeout") + if (oldest.sid == CfwTransport.SID) + cfwTransports[if (oldest.isLeftArmMessage) 0 else 1].reset() + handleAckTimeoutLocked(oldest) + return 0 + } + if (connectionOptions.WINDOW_SIZE <= 1 + && sessionReady + && prewrittenMessage == null + && !pendingMessages.isEmpty() + && canPrewriteCandidate(pendingMessages.firstOrNull()) + && !shouldBlockPrewriteForHeartbeatLocked(now)) { + // Only the serial (window==1) path pre-sends the all-but-last + // packet; with a real window we just send the next message fully. + messageToPrewrite = pendingMessages.firstOrNull() + } + } + + if (chargingMode) { + // Glasses are in the case: no display traffic, only battery + // polls (which also detect the end of charging). + finishDesiredFrameLocked("discarded: glasses charging") + if (sessionReady && inFlightMessages.isEmpty() && !pendingMessages.isEmpty()) { + messageToWrite = pendingMessages.removeFirst() + logLine("sending pending message (charging): " + messageToWrite!!.label) + } else if (sessionReady && pendingMessages.isEmpty() && inFlightMessages.isEmpty() + && now - lastBatteryRefreshAtMs >= ConnectionOptions.CHARGING_BATTERY_POLL_MS) { + logLine("Writing charging-mode battery poll") + messageToWrite = createBatteryQueryMessageLocked() + lastBatteryRefreshAtMs = now + } else { + return 1_000 + } + } else { + if (messageToPrewrite == null + && !shutdownRequested + && !fixedLayoutCreated + && pendingMessages.isEmpty() + && inFlightMessages.isEmpty()) { + logLine("enqueueing create layout") + enqueueCreateLayoutLocked() + } else if (messageToPrewrite != null) { + // Prewrite outside the lock; the logical message remains pending until + // its final BLE frame is sent after the current protocol ACK. + } + if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated + && (if (firmwareDebugFlagsEnabled) 2 else 1) != firmwareDebugFlagsLastSent + && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { + logLine("enqueueing firmware debug flags " + (if (firmwareDebugFlagsEnabled) "show" else "hide")) + enqueueFirmwareDebugFlagsLocked() + } + + if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated + && (if (compassOwners.isEmpty()) 0 else 1) != compassControlLastSent + && pendingMessages.isEmpty() && inFlightMessages.isEmpty()) { + val wanted = compassOwners.isNotEmpty() + logLine("enqueueing compass " + (if (wanted) "enable" else "disable")) + enqueueCompassControlLocked(false, wanted) + } + + if (benchmarkActive) { + maintainBenchmarkLocked(now) + } + + // Up to WINDOW_SIZE messages may be in flight at once (full + // pipelining); a slot frees when an ack arrives. An active + // bandwidth benchmark measures its own selected window instead. + val windowHasRoom = inFlightMessages.size < + maxOf(1, if (benchmarkActive) benchmarkWindowSize else connectionOptions.WINDOW_SIZE) + // A frame ready to send right now: don't inject a fresh + // heartbeat in front of it (the image's own ack resets the + // firmware heartbeat timer, so the heartbeat is redundant). + // "Ready" covers both a fresh composite still to be planned + // and a planned image already queued but not yet written -- + // enqueueing sets lastEnqueuedFingerprint, so testing only + // the desired fingerprint left the queued-but-unwritten + // window unprotected and a heartbeat that came due there + // cost the frame a full ack round trip (measured 124ms on + // frame#232 of the 2026-08-20 02:27 capture). + // A benchmark run counts as image traffic here for the same + // reason: its acks reset the firmware heartbeat timer, so a + // fresh heartbeat in front of it is redundant. + val imageWaiting = !shutdownRequested && fixedLayoutCreated + && !hasPendingOrInflightKindLocked("heartbeat") + && (benchmarkActive + || (now >= imageRetryAfterMs + && (hasPendingImageLocked() + || desiredFingerprintSnapshot() != lastEnqueuedFingerprint))) + noteImageStallLocked(now, windowHasRoom) + if (messageToPrewrite == null && handleHeartbeat(imageWaiting)) { + return ConnectionOptions.IDLE_SLEEP_MS.toLong() + } + + if (messageToPrewrite == null && sessionReady && windowHasRoom && !pendingMessages.isEmpty() + && CfwMessageWindow.canSend(inFlightMessages, pendingMessages.firstOrNull()!!)) { + messageToWrite = pendingMessages.removeFirst() + logLine("sending pending message: " + messageToWrite!!.label) + } else if (messageToPrewrite == null && !shutdownRequested && !benchmarkActive + && fixedLayoutCreated + && windowHasRoom && !hasPendingImageLocked() + && now >= imageRetryAfterMs + && desiredFingerprintSnapshot() != lastEnqueuedFingerprint) { + // Enqueue the next frame's delta against lastEnqueuedPacked + // (what the shadow will be), so it can pipeline behind an + // image still awaiting its ack. + logLine("Enqueued image update") + enqueueDesiredImageLocked() + return 0 + } else if (messageToPrewrite == null && shouldPollBatteryLocked(now)) { + logLine("Writing battery query") + messageToWrite = createBatteryQueryMessageLocked() + lastBatteryRefreshAtMs = now + } else if (messageToPrewrite == null && (!pendingMessages.isEmpty() || !inFlightMessages.isEmpty())) { + return ConnectionOptions.IDLE_SLEEP_MS.toLong() + } else if (messageToPrewrite == null) { + return 250 + } + } + } + + val prewrite = messageToPrewrite + if (prewrite != null) { + if (prewriteMessage(prewrite)) { + return 0 + } + return ConnectionOptions.IDLE_SLEEP_MS.toLong() + } + + val write = messageToWrite!! + if (!writeMessage(write)) { + monitor.withLock { + if (removePreparedMessageLocked(write) || write.magic == 0) { + handleTransportFailure("write failed") + } + } + return 0 + } + } +} + +/** + * If the desired image already matches what the glasses display, nothing will + * ever be enqueued for it, so finish its frame now (otherwise the TS side + * would block on it until its backpressure timeout). + */ +internal fun GlassesSessionCore.maybeFinishNoChangeDesiredFrame() { + var frameIdToFinish = 0 + monitor.withLock { + desiredTilesLock.locked { + if (desiredFrameId != 0 && lastEnqueuedFingerprint.isNotEmpty() && desiredFingerprint == lastEnqueuedFingerprint) { + frameIdToFinish = desiredFrameId + desiredFrameId = 0 + } + } + } + if (frameIdToFinish != 0) { + finishFrame(frameIdToFinish, "discarded: no change from displayed image") + } +} + +internal fun GlassesSessionCore.shouldBlockPrewriteForHeartbeatLocked(now: Long): Boolean { + if (shutdownRequested || !fixedLayoutCreated) { + return false + } + return hasPendingOrInflightKindLocked("heartbeat") + || now - lastHeartbeatAckedAtMs >= ConnectionOptions.HEARTBEAT_READY_MS +} + +internal fun GlassesSessionCore.canPrewriteCandidate(message: OutboundMessage?): Boolean { + if (message == null || message.sid == CfwTransport.SID || !message.isLeftArmMessage) { + return false + } + if ("image" != message.kind) { + return false + } + return message.message.size + 2 > 232 +} + +internal fun GlassesSessionCore.handleHeartbeat(imageWaiting: Boolean): Boolean { + val now = now() + val heartbeatEligible = !shutdownRequested && fixedLayoutCreated + val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat") + val heartbeatElapsedMs = now - lastHeartbeatAckedAtMs + val heartbeatReady = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS + val heartbeatUrgent = heartbeatEligible && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_URGENT_MS + + if (heartbeatReady && !heartbeatPending && inFlightMessages.isEmpty()) { + if (imageWaiting && !heartbeatUrgent) { + // Defer to the waiting frame: sending it now satisfies the + // firmware heartbeat deadline (its ack resets the timer), and + // the heartbeat still fires once we reach the URGENT threshold + // if rendering goes quiet again. Preserves the pending-heartbeat + // inter-lens-sync invariant below (that path is untouched). + return false + } + logLine("Writing heartbeat") + val heartbeatMessage = createHeartbeatMessage() + lastHeartbeatSentAtMs = now + writeMessage(heartbeatMessage) + return true + } else if (heartbeatUrgent) { + return true + } else if (heartbeatPending) { + // Don't send other message types while a heartbeat is pending because that + // can lead to inter-lens sync issues + return true + } + + return false +} + +internal fun GlassesSessionCore.createHeartbeatMessage(): OutboundMessage { + val message = messageBuilder.heartbeat() + message.onAck = MessageCallback { + monitor.withLock { + lastHeartbeatAckedAtMs = now() + } + } + message.onTimeout = MessageCallback { + // If a heartbeat fails to ack and we're over the heartbeat deadline, assume the connection is failed and reconnect. + // Otherwise ignore it, which will cause a retransmission attempt. + val isPastDeadline: Boolean + monitor.withLock { + isPastDeadline = now() - lastHeartbeatSentAtMs >= ConnectionOptions.HEARTBEAT_FAILURE_DEADLINE_MS + } + if (isPastDeadline) { + handleTransportFailure("heartbeat ack timeout") + } + } + return message +} + +internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean { + val now = now() + message.writeStartedAtMs = now + message.sentAtMs = now + message.ackDeadlineAtMs = now + message.ackTimeoutMs + ConnectionOptions.WRITE_TIMEOUT_MS + if (message.magic != 0) { + monitor.withLock { + inFlightMessages.addLast(message) + } + } + if (message.imageUpdateId > 0 && message.imageMessageNumber == 1) { + monitor.withLock { + val stats = imageUpdateStats[message.imageUpdateId] + if (stats != null && stats.firstWriteStartedAtMs <= 0) { + stats.firstWriteStartedAtMs = now + } + } + } + + val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress + val frames: List + if (prewrittenMessage != null && prewrittenMessage !== message) { + if (!spoilPrewrittenMessage("before " + message.label)) { + return false + } + } + if (prewrittenMessage === message) { + frames = listOf(prewrittenFrames[prewrittenFrames.size - 1]) + prewrittenMessage = null + prewrittenFrames = emptyList() + } else if (message.sid == CfwTransport.SID) { + if (message.cfwRetries > 0) cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset() + frames = cfwTransports[if (message.isLeftArmMessage) 0 else 1].encode( + message.message, message.magic, CfwTransport.BOTH, + link.negotiatedMtu(writeAddress)) + } else { + frames = BleProtocol.framePb( + message.message, + message.sid, + message.flag, + nextTransportSeq++ + ) + } + val result = link.writeFrames( + writeAddress, + BleProtocol.WRITE_CHAR_UUID, + frames, + ConnectionOptions.WRITE_MODE, + ConnectionOptions.WRITE_TIMEOUT_MS + ) + + monitor.withLock { + val sentAtMs = now() + message.sentAtMs = sentAtMs + message.ackDeadlineAtMs = sentAtMs + message.ackTimeoutMs + logImageUpdateSendLandmarkLocked(message) + val onSent = message.onSent + if (result && onSent != null) { + onSent.run() + monitor.signalAll() + } + } + + if (!result && message.sid == CfwTransport.SID) + cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset() + return result +} + +internal fun GlassesSessionCore.prewriteMessage(message: OutboundMessage): Boolean { + if (prewrittenMessage === message) { + return true + } + if (prewrittenMessage != null && !spoilPrewrittenMessage("before prewrite " + message.label)) { + return false + } + if (!canPrewriteCandidate(message)) { + return false + } + + val frames: List = BleProtocol.framePb( + message.message, + message.sid, + message.flag, + nextTransportSeq++ + ) + if (frames.size <= 1) { + return false + } + + val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress + val prefixFrames = frames.subList(0, frames.size - 1) + val result = link.writeFrames( + writeAddress, + BleProtocol.WRITE_CHAR_UUID, + prefixFrames, + ConnectionOptions.WRITE_MODE, + ConnectionOptions.WRITE_TIMEOUT_MS + ) + if (!result) { + return false + } + + prewrittenMessage = message + prewrittenFrames = frames.toList() + logLine("prewrote " + message.label + " frames=" + prefixFrames.size + "/" + frames.size) + return true +} + +internal fun GlassesSessionCore.spoilPrewrittenMessage(reason: String): Boolean { + val message = prewrittenMessage + if (message == null || prewrittenFrames.isEmpty()) { + prewrittenMessage = null + prewrittenFrames = emptyList() + return true + } + val finalFrame = prewrittenFrames[prewrittenFrames.size - 1].copyOf() + if (finalFrame.size > 8) { + finalFrame[finalFrame.size - 1] = (finalFrame[finalFrame.size - 1].toInt() xor 0xff).toByte() + } + prewrittenMessage = null + prewrittenFrames = emptyList() + + val writeAddress = if (message.isLeftArmMessage) leftAddress else rightAddress + logLine("spoiling prewritten " + message.label + ": " + reason) + return link.writeFrames( + writeAddress, + BleProtocol.WRITE_CHAR_UUID, + listOf(finalFrame), + ConnectionOptions.WRITE_MODE, + ConnectionOptions.WRITE_TIMEOUT_MS + ) +} + +internal fun GlassesSessionCore.removePreparedMessageLocked(message: OutboundMessage?): Boolean { + if (message == null || message.magic == 0) { + return false + } + val iterator = inFlightMessages.iterator() + while (iterator.hasNext()) { + if (iterator.next() === message) { + iterator.remove() + magicPool.release(message.sid, message.magic, message.label, "write failed") + return true + } + } + return false +} + +/** Also drain on the worker: a clear/timeout may have removed a blocking + * head since the last notification. Fully ACKed controls must never reach + * the generic timeout path just because no further BLE reply arrived. */ +internal fun GlassesSessionCore.drainCfwAcknowledgementsLocked() { + while (true) { + val first = CfwMessageWindow.acknowledgedHead(inFlightMessages) ?: return + lastAckAtMs = now() + resolveAckLocked(first, first.ackPayload) + } +} + +internal fun GlassesSessionCore.resolveAckLocked(sid: Int, magic: Int, pb: ByteArray?) { + val iterator = inFlightMessages.iterator() + while (iterator.hasNext()) { + val message = iterator.next() + if (message.sid == sid && message.magic == magic) { + resolveAckLocked(message, pb) + return + } + } + recordUnexpectedAckLocked(sid, magic) +} + +internal fun GlassesSessionCore.resolveAckLocked(message: OutboundMessage, pb: ByteArray?) { + logLine("Got ACK for " + message.label + "(sid=" + message.sid + ", id=" + message.magic + ")") + inFlightMessages.remove(message) + message.ackPayload = pb?.copyOf() ?: ByteArray(0) + magicPool.release(message.sid, message.magic, message.label, "ack") + val onAck = message.onAck + if (onAck != null) { + onAck.run() + } + consecutiveAckTimeouts = 0 + // onAck may have just satisfied a waiter blocked on the monitor (e.g. + // awaitEvenHubSessionReady polling fixedLayoutCreated/displayedFingerprint + // after a create-layout or image ack). Without this, that waiter only + // notices on its own up-to-100ms poll tick, adding avoidable latency to + // every EvenHub wake. Always called with the monitor held (see call site). + monitor.signalAll() +} + +internal fun GlassesSessionCore.logImageUpdateSendLandmarkLocked(message: OutboundMessage) { + if (message.imageUpdateId <= 0) { + return + } + val stats = imageUpdateStats[message.imageUpdateId] + val frameId = if (stats == null) 0 else stats.frameId + if (message.imageMessageNumber == 1) { + if (stats != null && stats.firstWriteStartedAtMs <= 0) { + stats.firstWriteStartedAtMs = if (message.writeStartedAtMs > 0) message.writeStartedAtMs else message.sentAtMs + } + frameTimings.log(frameId, "first bluetooth packet sent") + logImageUpdateLandmarkLocked("first bluetooth message sent", message, message.sentAtMs) + } + if (message.imageMessageNumber == message.imageMessageCount) { + frameTimings.log(frameId, + "last bluetooth packet sent (message " + message.imageMessageNumber + "/" + message.imageMessageCount + ")") + logImageUpdateLandmarkLocked("last bluetooth message sent", message, message.sentAtMs) + } +} + +internal fun GlassesSessionCore.logImageUpdateAckLandmarkLocked(message: OutboundMessage) { + if (message.imageUpdateId <= 0 || message.imageMessageNumber != message.imageMessageCount) { + return + } + val ackedAtMs = now() + link.recordDisplayFrameSent() + val stats = imageUpdateStats.remove(message.imageUpdateId) + if (stats != null && stats.firstWriteStartedAtMs > 0) { + emitFrameMetrics(stats.paintMs, maxOf(0L, ackedAtMs - stats.firstWriteStartedAtMs).toInt(), stats.tileCount) + } + if (stats != null) { + finishFrame(stats.frameId, "sent") + } + logImageUpdateLandmarkLocked("last bluetooth message acked", message, ackedAtMs) +} + +/** Remove the stats entry for an image update that will not complete, finishing its frame. */ +internal fun GlassesSessionCore.discardImageUpdateStatsLocked(imageUpdateId: Int, reason: String) { + if (imageUpdateId <= 0) { + return + } + val stats = imageUpdateStats.remove(imageUpdateId) + if (stats != null) { + finishFrame(stats.frameId, "discarded: $reason") + } +} + +internal fun GlassesSessionCore.logImageUpdateLandmarkLocked(event: String, message: OutboundMessage, elapsedMs: Long) { + logLine("image update#" + message.imageUpdateId + " " + event + + " at " + timestamp(elapsedMs) + + " message=" + message.imageMessageNumber + "/" + message.imageMessageCount + + " label=" + message.label) +} + +internal fun GlassesSessionCore.enqueueCreateLayoutLocked() { + // New session: re-assert the firmware-debug-flags overlay once + // the layout is ready (the mode-7 send is gated on this having reset). + firmwareDebugFlagsLastSent = -1 + val message = messageBuilder.createLayout() + message.onAck = MessageCallback { + startupProbePending = false + clearMessagesOfKindLocked("startup-text-probe") + fixedLayoutCreated = true + displayedFingerprint = "" + } + message.onTimeout = MessageCallback { + if (startupProbePending) { + logLine("create layout timed out while startup text probe is pending") + if (hasPendingOrInflightKindLocked("startup-text-probe")) { + return@MessageCallback + } + startupProbePending = false + } + handleTransportFailure("ack timeout") + } + pendingMessages.addLast(message) + logLine("queue create layout") +} + +internal fun GlassesSessionCore.enqueueStartupProbeLocked() { + enqueueCreateLayoutLocked() + + val message = messageBuilder.startupTextProbe() + message.onAck = MessageCallback { + startupProbePending = false + clearMessagesOfKindLocked("create-layout") + fixedLayoutCreated = true + displayedFingerprint = "" + logLine("existing dashboard layout accepted text probe") + } + message.onTimeout = MessageCallback { + startupProbePending = false + if (hasPendingOrInflightKindLocked("create-layout")) { + return@MessageCallback + } + handleTransportFailure("ack timeout") + } + pendingMessages.addLast(message) + startupProbePending = true + logLine("queue startup text probe") +} + +/** + * Send the CFW mode-7 diagnostic-flag control op through the private stream: + * [7][2] to show the on-glasses debug-flag overlay, [7][1] to hide it. Uses the + * arbitrary-payload custom path (no bmp/dedup/frame-timing interaction). + */ +internal fun GlassesSessionCore.enqueueFirmwareDebugFlagsLocked() { + val show = firmwareDebugFlagsEnabled + val sub = if (show) 2 else 1 + val payload = byteArrayOf(CFW_MSG_DIAGNOSTICS.toByte(), sub.toByte()) + val message = messageBuilder.imagePayload( + DASHBOARD_TILE, nextMapSessionId(), payload, + "fw-debug-flags " + (if (show) "show" else "hide"), + connectionOptions.sendImagesToLeft) + pendingMessages.addLast(message) + firmwareDebugFlagsLastSent = sub + logLine("queue firmware debug flags " + (if (show) "show" else "hide")) +} + +/** + * Send CFW image-handler mode 10. Enable uses the configurable form + * [10][2][interval-ms LE16][minimum-change-degrees LE16]; disable remains [10][0]. + */ +internal fun GlassesSessionCore.enqueueCompassControlLocked(priority: Boolean, enable: Boolean) { + val sentState = if (enable) 1 else 0 + val payload = if (enable) + byteArrayOf( + CFW_MSG_COMPASS.toByte(), + 2.toByte(), + (COMPASS_REPORT_INTERVAL_MS and 0xff).toByte(), + ((COMPASS_REPORT_INTERVAL_MS shr 8) and 0xff).toByte(), + (COMPASS_MIN_CHANGE_DEGREES and 0xff).toByte(), + ((COMPASS_MIN_CHANGE_DEGREES shr 8) and 0xff).toByte(), + ) + else + byteArrayOf(CFW_MSG_COMPASS.toByte(), 0.toByte()) + val message = messageBuilder.imagePayload( + "compass-control", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "compass " + (if (enable) "enable" else "disable"), + connectionOptions.sendImagesToLeft) + message.onTimeout = MessageCallback { + compassControlLastSent = -1 + logLine("compass control ack timeout") + } + if (enable) { + compassMaybeOn = true + } else { + message.onAck = MessageCallback { compassMaybeOn = false } + } + if (priority) pendingMessages.addFirst(message) + else pendingMessages.addLast(message) + compassControlLastSent = sentState + logLine("queue " + message.label) +} + +internal fun GlassesSessionCore.enqueueAmbientLightControlLocked(payload: ByteArray, label: String, priority: Boolean) { + if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) { + logLine("skip $label; display path not ready") + return + } + val message = messageBuilder.imagePayload( + "als-control", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + label, + connectionOptions.sendImagesToLeft) + message.onTimeout = MessageCallback { logLine("$label ack timeout") } + if (priority) pendingMessages.addFirst(message) + else pendingMessages.addLast(message) + logLine("queue $label") +} + +internal fun GlassesSessionCore.enqueueDesiredImageLocked() { + val fingerprint = desiredFingerprintSnapshot() + val packedSnapshot: ByteArray? + val width: Int + val height: Int + val paintMs: Int + val frameId: Int + val draws: Array? + val scene: ShellScene + desiredTilesLock.locked { + packedSnapshot = desiredPacked + width = desiredWidth + height = desiredHeight + paintMs = desiredPaintMs + frameId = desiredFrameId + draws = desiredDraws + scene = desiredShellScene + desiredFrameId = 0 + } + val packedFrame: ByteArray = packedSnapshot ?: ByteArray(0) + if (customFirmwareDetected && packedFrame.size > 0) { + val rendered = scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId, + connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES, + connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS) + nextImageFrameId = rendered.nextFid + val commands = rendered.commands + for (i in 0 until commands.size - 1) enqueueResourceCommandLocked(commands[i]) + val plan = BleImageOptimizer.TileImagePlan( + 0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId(), commands[commands.size - 1]) + finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId) + return + } + + frameTimings.spanStart(frameId, "compress-and-plan") + // Incremental (mode 3 bounding box) update against the last ENQUEUED frame + // (the base the firmware shadow will hold when this update is applied). + // lastEnqueuedPacked is cleared whenever the image pipeline is cleared, so + // a non-empty value means the display base is trusted. + var incrementalPayload: ByteArray? = null + var incrementalLog: String? = null + if (connectionOptions.INCREMENTAL_FRAMES && lastEnqueuedPacked.size > 0 + && lastEnqueuedWidth == width && lastEnqueuedHeight == height) { + val baseFid = nextImageFrameId + val single = + BleImageOptimizer.buildIncrementalImagePayload(lastEnqueuedPacked, packedFrame, width, height, baseFid) + if (single != null) { + incrementalPayload = single.payload + // advance only when a delta is actually emitted, so consecutive + // deltas carry consecutive ids (CFW skip/reorder detection) + nextImageFrameId = if (nextImageFrameId >= 0xfffe) 1 else nextImageFrameId + 1 + incrementalLog = "incremental update bbox=" + + ((single.payload[3].toInt() and 0xff) * 4) + "x" + ((single.payload[4].toInt() and 0xff) * 2) + + "+" + ((single.payload[1].toInt() and 0xff) * 4) + "+" + ((single.payload[2].toInt() and 0xff) * 2) + + " changed=" + single.changedBytes + "/" + single.boxBytes + "B" + + " clusters=" + single.clusterCount + + // When the bounding box spans multiple clusters or is sizeable, try + // splitting into tight rects (CFW mode-8). Only replace the single + // box if the multi-rect message is actually smaller on the wire. + if (connectionOptions.MULTI_RECT_FRAMES + && (single.clusterCount > 1 || single.payload.size > ConnectionOptions.MULTI_RECT_MIN_PAYLOAD)) { + val multi = BleImageOptimizer.buildMultiRectImagePayload( + lastEnqueuedPacked, packedFrame, width, height, baseFid, ConnectionOptions.MULTI_RECT_MAX_RECTS) + if (multi != null && multi.payload.size < single.payload.size) { + incrementalPayload = multi.payload + nextImageFrameId = multi.nextFid // rectCount fids consumed + incrementalLog = "multi-rect update n=" + multi.rectCount + + " covered=" + multi.coveredBytes + "B" + + " payload=" + multi.payload.size + "B (vs bbox " + single.payload.size + "B)" + } + } + } + } + val plan = if (incrementalPayload != null) + BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId(), incrementalPayload) + else + BleImageOptimizer.TileImagePlan(0, DASHBOARD_TILE, packedFrame, width, height, nextMapSessionId()) + frameTimings.spanEnd(frameId, "compress-and-plan") + if (incrementalLog != null) { + frameTimings.log(frameId, incrementalLog) + } + finishEnqueueDesiredImageLocked(plan, fingerprint, paintMs, frameId) +} + +/** Queue complete private commands, advance the delta base, and retain frame/ACK bookkeeping. */ +internal fun GlassesSessionCore.finishEnqueueDesiredImageLocked( + plan: BleImageOptimizer.TileImagePlan, fingerprint: String, paintMs: Int, frameId: Int) { + val updateId = nextImageUpdateId++ + val commands: MutableList = ArrayList() + if (plan.payload.size <= CfwTransport.MAX_MESSAGE) { + commands.add(plan.payload) + } else { + // A noisy full frame can exceed the stream record limit. Repaint it + // with independently decodable RLE bands below the decoded-message limit. + commands.addAll(BleImageOptimizer.encodeFullFrameBands( + plan.packed!!, plan.width, plan.height, nextImageFrameId)) + for (i in 0 until commands.size) { + nextImageFrameId = if (nextImageFrameId >= 0xfffe) 1 else nextImageFrameId + 1 + } + } + val messageCount = commands.size + imageUpdateStats[updateId] = BleImageOptimizer.ImageUpdateStats(paintMs, 1, frameId) + for (i in 0 until commands.size) { + enqueueCustomImageLocked(plan, commands[i], fingerprint, updateId, i + 1, messageCount) + } + // This frame is now the base for the next delta (it will be the firmware + // shadow once applied), even though it hasn't been acked yet — that is what + // lets the next frame pipeline behind it. plan.packed is the full frame; + // frames are immutable by convention, so referencing it is safe. + lastEnqueuedPacked = plan.packed!! + lastEnqueuedWidth = plan.width + lastEnqueuedHeight = plan.height + lastEnqueuedFingerprint = fingerprint + + frameTimings.log(frameId, "queued image update#" + updateId + + " messages=" + messageCount + " payload=" + plan.payload.size + "B") + logLine("queue image update#" + updateId + " fingerprint=" + fingerprint + + " messages=" + messageCount) +} + +/** + * Enqueue one resource eviction/upload command ahead of the image message that + * references its glyphs (the transport is FIFO, so no ack round trip is + * needed before use). A timeout means the on-glasses cache state is + * unknown; forget everything phone-side (glyphs re-upload lazily) and let + * the accompanying image update's own timeout drive the frame resync. + */ +internal fun GlassesSessionCore.enqueueResourceCommandLocked(payload: ByteArray) { + val message = messageBuilder.imagePayload( + "resources", + DASHBOARD_TILE, + nextMapSessionId(), + payload, + "resource command " + payload.size + "B", + connectionOptions.sendImagesToLeft) + message.onTimeout = MessageCallback { + resourceCache.reset() + logLine("resource command ack timeout; resource cache state reset") + } + pendingMessages.addLast(message) + logLine("queue " + message.label) +} + +internal fun GlassesSessionCore.enqueueCustomImageLocked( + plan: BleImageOptimizer.TileImagePlan, + payload: ByteArray, + fingerprint: String, + updateId: Int, + messageNumber: Int, + messageCount: Int +) { + val message = messageBuilder.customMessage("image", payload, + "image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft) + message.setImageUpdatePosition(updateId, messageNumber, messageCount) + message.onAck = MessageCallback { + imageRetryAfterMs = 0 + // Firmware >= 2.2.4.34 resets its heartbeat timer when it receives + // image messages (not just heartbeats), so an acked image fragment + // satisfies the heartbeat deadline and heartbeats stop contending + // with active rendering. + lastHeartbeatAckedAtMs = now() + logImageUpdateAckLandmarkLocked(message) + var imageStillInFlight = false + for (inFlight in inFlightMessages) { + if ("image" == inFlight.kind) { + imageStillInFlight = true + break + } + } + if (!imageStillInFlight) { + var imageStillQueued = false + for (queued in pendingMessages) { + if ("image" == queued.kind) { + imageStillQueued = true + break + } + } + if (!imageStillQueued) { + displayedFingerprint = fingerprint + } + } + } + message.onTimeout = MessageCallback { + discardImageUpdateStatsLocked(message.imageUpdateId, "image ack timeout (will retry)") + clearMessagesOfKindLocked("image") + displayedFingerprint = "" + imageRetryAfterMs = now() + ConnectionOptions.IMAGE_RETRY_DELAY_MS + } + pendingMessages.addLast(message) +} + +internal fun GlassesSessionCore.createAudioControlMessageLocked(enable: Boolean): OutboundMessage { + val message = messageBuilder.enableOrDisableMic(enable) + message.onAck = MessageCallback { + lastAudioControlAckMagic = message.magic + audioCaptureActive = message.label?.contains("enable") == true + logLine(if (audioCaptureActive) "G2 mic enabled" else "G2 mic disabled") + } + message.onTimeout = MessageCallback { + handleTransportFailure("audio control ack timeout") + } + return message +} + +internal fun GlassesSessionCore.shouldPollBatteryLocked(now: Long): Boolean { + return !shutdownRequested + && sessionReady + && pendingMessages.isEmpty() + && inFlightMessages.isEmpty() + && now - lastConnectionOrInputAtMs >= ConnectionOptions.BATTERY_INPUT_QUIET_MS + && (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS) +} + +internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessage { + val message = messageBuilder.batteryQuery() + message.onAck = MessageCallback { + val ring = BleProtocol.parseRingBattery(message.ackPayload) + // Old firmware and missing/malformed extensions must clear any prior reading. + ringBattery = if (ring == null) -1 else ring.battery + ringCharging = if (ring == null) -1 else ring.charging + val snapshot = BleProtocol.parseSettingsBattery(message.ackPayload) + if (snapshot != null) { + headsetBattery = snapshot.battery + headsetCharging = snapshot.charging + emitBatteryState(headsetBattery, headsetCharging) + if (snapshot.silentMode >= 0) { + // Backstop for the push in onNotification: the firmware is + // confirmed to push silent-mode-on, but the off transition is + // not, so re-read the authoritative value on every poll. + updateSilentModeLocked(snapshot.silentMode > 0) + } + updateChargingModeLocked(snapshot.charging > 0, snapshot.battery) + } + val firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload) + if (firmwareInfo != null) { + customFirmwareDetected = firmwareInfo.isFaceclawFirmware() + emitFirmwareInfo(firmwareInfo) + } + } + message.onTimeout = MessageCallback { + logLine("Battery query timed out") + } + return message +} + +/** + * Track silent mode, which the wearer toggles by long-pressing both + * touchpads at once. While it is on the firmware refuses input events and + * app launches and powers the display down, so the glasses look dead even + * though the BLE session is healthy; the phone UI says so explicitly. + */ +internal fun GlassesSessionCore.updateSilentModeLocked(silent: Boolean) { + val next = if (silent) 1 else 0 + if (silentMode == next) { + return + } + silentMode = next + logLine(if (silent) "glasses entered silent mode" else "glasses left silent mode") + emitSilentMode(silent) +} + +/** + * Track whether the glasses are in the charging case. Charging means nobody + * is wearing them: display communication pauses (no heartbeats, so the + * firmware tears down its EvenHub context on its own) and only battery polls + * continue. When charging stops, tear the transport down and let the normal + * reconnect loop rebuild the session, layout, and first frame. + */ +internal fun GlassesSessionCore.updateChargingModeLocked(charging: Boolean, battery: Int) { + if (charging == chargingMode) { + if (chargingMode) { + setStateDisplay("charging", GlassesSessionCore.chargingStatusText(battery)) + } + return + } + if (charging) { + chargingMode = true + clearAllMessagesLocked("glasses charging") + fixedLayoutCreated = false + startupProbePending = false + displayedFingerprint = "" + finishDesiredFrameLocked("discarded: glasses charging") + logLine("glasses are charging; pausing display communication") + setStateDisplay("charging", GlassesSessionCore.chargingStatusText(battery)) + } else { + chargingMode = false + logLine("glasses removed from charger; reconnecting") + handleTransportFailure("charging ended") + } +} + +/** + * Record, into the frame that is waiting, why it did not go out on this + * pass of the send loop. Without this the export shows a bare multi-second + * jump between "image submitted as desired frame" and the first BLE + * packet, with no hint whether we were blocked on a heartbeat, the + * BLE window, or another message queued ahead. Deduped on (frame, reason), + * so a frame stalled for seconds gets one line per state change rather + * than one per loop pass. + */ +internal fun GlassesSessionCore.noteImageStallLocked(now: Long, windowHasRoom: Boolean) { + var frameId: Int + desiredTilesLock.locked { + frameId = desiredFrameId + } + val reason: String? + if (frameId != 0 && desiredFingerprintSnapshot() != lastEnqueuedFingerprint) { + reason = describeEnqueueBlockerLocked(now, windowHasRoom) + } else { + // Nothing waiting to be planned; an already-planned image may still + // be queued behind other traffic. + val queuedImage = firstPendingImageLocked() + frameId = if (queuedImage == null) 0 else imageUpdateFrameIdLocked(queuedImage.imageUpdateId) + reason = if (queuedImage == null) null else describeWriteBlockerLocked(now, windowHasRoom, queuedImage) + } + if (frameId == 0 || reason == null) { + stallFrameId = 0 + stallReason = "" + return + } + if (frameId == stallFrameId && reason == stallReason) { + return + } + stallFrameId = frameId + stallReason = reason + frameTimings.log(frameId, "waiting to send: $reason") +} + +/** Why the desired composite has not been turned into wire messages yet, or null. */ +internal fun GlassesSessionCore.describeEnqueueBlockerLocked(now: Long, windowHasRoom: Boolean): String? { + if (shutdownRequested) { + return "shutdown requested" + } + if (!sessionReady) { + return "BLE session not ready" + } + if (!fixedLayoutCreated) { + return "display layout not created yet" + } + if (now < imageRetryAfterMs) { + return "image retry backoff (" + (imageRetryAfterMs - now) + "ms left)" + } + if (hasPendingImageLocked()) { + return "an earlier image is still queued" + } + if (!windowHasRoom) { + return "BLE window full (" + inFlightMessages.size + " message(s) in flight)" + } + if (hasPendingOrInflightKindLocked("heartbeat")) { + return "heartbeat in flight" + } + if (!pendingMessages.isEmpty()) { + return pendingMessages.size.toString() + " message(s) queued ahead, next " + + pendingMessages.firstOrNull()?.label + } + return null +} + +/** Why a planned image message has not been written to BLE yet, or null. */ +internal fun GlassesSessionCore.describeWriteBlockerLocked(now: Long, windowHasRoom: Boolean, queuedImage: OutboundMessage): String? { + if (!sessionReady) { + return "BLE session not ready" + } + if (!windowHasRoom) { + return "BLE window full (" + inFlightMessages.size + " message(s) in flight)" + } + if (hasPendingOrInflightKindLocked("heartbeat")) { + return "heartbeat in flight" + } + // handleHeartbeat is a barrier: while one is due it holds back every + // other write, so a frame queued at the wrong moment waits a heartbeat + // round trip. Reported explicitly because it is otherwise invisible -- + // heartbeats belong to no frame. + val heartbeatElapsedMs = now - lastHeartbeatAckedAtMs + if (fixedLayoutCreated && !shutdownRequested + && heartbeatElapsedMs >= ConnectionOptions.HEARTBEAT_READY_MS) { + return "heartbeat due (" + heartbeatElapsedMs + "ms since the last one acked)" + } + val head = pendingMessages.firstOrNull() + if (head != null && head !== queuedImage) { + return "queued behind " + head.label + } + return null +} + +internal fun GlassesSessionCore.firstPendingImageLocked(): OutboundMessage? { + for (message in pendingMessages) { + if (message.imageUpdateId > 0) { + return message + } + } + return null +} + +internal fun GlassesSessionCore.imageUpdateFrameIdLocked(imageUpdateId: Int): Int { + val stats = imageUpdateStats[imageUpdateId] + return if (stats == null) 0 else stats.frameId +} + +internal fun GlassesSessionCore.finishDesiredFrameLocked(outcome: String) { + val frameIdToFinish: Int + desiredTilesLock.locked { + frameIdToFinish = desiredFrameId + desiredFrameId = 0 + } + finishFrame(frameIdToFinish, outcome) +} + +internal fun GlassesSessionCore.hasPendingOrInflightKindLocked(kind: String): Boolean { + for (queued in pendingMessages) { + if (kind == queued.kind) { + return true + } + } + for (inFlight in inFlightMessages) { + if (kind == inFlight.kind) { + return true + } + } + return false +} + +internal fun GlassesSessionCore.hasPendingOrInflightMagicLocked(magic: Int): Boolean { + for (queued in pendingMessages) { + if (queued.magic == magic) { + return true + } + } + for (inFlight in inFlightMessages) { + if (inFlight.magic == magic) { + return true + } + } + return false +} + +internal fun GlassesSessionCore.hasPendingMagicLocked(sid: Int, magic: Int): Boolean { + for (queued in pendingMessages) { + if (queued.sid == sid && queued.magic == magic) { + return true + } + } + return false +} + +internal fun GlassesSessionCore.handleAckTimeoutLocked(message: OutboundMessage) { + consecutiveAckTimeouts += 1 + + val onTimeout = message.onTimeout + if (onTimeout != null) { + onTimeout.run() + } + + if (consecutiveAckTimeouts > ConnectionOptions.MAX_CONSECUTIVE_ACK_TIMEOUTS) { + handleTransportFailure("too many ack timeouts") + } +} + +internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) { + val pendingIterator = pendingMessages.iterator() + while (pendingIterator.hasNext()) { + val message = pendingIterator.next() + if (kind == message.kind) { + pendingIterator.remove() + discardPrewriteIfMatchesLocked(message) + if ("image" == kind) { + discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared ($kind)") + } + magicPool.release(message.sid, message.magic, message.label, "cleared pending $kind") + } + } + val inFlightIterator = inFlightMessages.iterator() + while (inFlightIterator.hasNext()) { + val message = inFlightIterator.next() + if (kind == message.kind) { + inFlightIterator.remove() + if ("image" == kind) { + discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared ($kind)") + } + magicPool.release(message.sid, message.magic, message.label, "cleared inflight $kind") + } + } + if ("image" == kind) { + // The image pipeline was flushed (e.g. ack timeout -> keyframe resync): + // drop the pipelined delta base so the next image is a full keyframe. + lastEnqueuedPacked = ByteArray(0) + lastEnqueuedFingerprint = "" + } +} + +internal fun GlassesSessionCore.clearAllMessagesLocked(reason: String) { + clearPendingMessagesLocked(reason) + clearInFlightMessagesLocked(reason) + // The image pipeline is gone: drop the pipelined delta base so the next + // image is a full keyframe rather than a delta onto a stale base. + lastEnqueuedPacked = ByteArray(0) + lastEnqueuedFingerprint = "" + // Queued resource commands (if any) were dropped with the rest, and the + // session churn behind a full clear may have freed the on-glasses + // cache; forget it phone-side so glyphs re-upload lazily. + resourceCache.reset() +} + +/** + * A custom wake queues CLAIM before NativeScript asks for a resume. Keep + * that private control while flushing stale EvenHub traffic around the + * direct prelude write. + */ +internal fun GlassesSessionCore.clearAllMessagesPreservingWakeLeaseLocked(reason: String) { + val pendingIterator = pendingMessages.iterator() + while (pendingIterator.hasNext()) { + val message = pendingIterator.next() + if ("wake-lease-control" == message.kind) { + continue + } + pendingIterator.remove() + discardPrewriteIfMatchesLocked(message) + discardImageUpdateStatsLocked( + message.imageUpdateId, + "pending messages cleared: $reason" + ) + magicPool.release( + message.sid, + message.magic, + message.label, + "cleared pending: $reason" + ) + } + clearInFlightMessagesLocked(reason) + lastEnqueuedPacked = ByteArray(0) + lastEnqueuedFingerprint = "" + resourceCache.reset() +} + +/** Any image update whose fragments are still queued (not yet sent). */ +internal fun GlassesSessionCore.hasPendingImageLocked(): Boolean { + for (message in pendingMessages) { + if ("image" == message.kind) { + return true + } + } + return false +} + +internal fun GlassesSessionCore.clearPendingMessagesLocked(reason: String) { + while (!pendingMessages.isEmpty()) { + val message = pendingMessages.removeFirst() + discardPrewriteIfMatchesLocked(message) + discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared: $reason") + magicPool.release(message.sid, message.magic, message.label, "cleared pending: $reason") + } +} + +internal fun GlassesSessionCore.clearInFlightMessagesLocked(reason: String) { + while (!inFlightMessages.isEmpty()) { + val message = inFlightMessages.removeFirst() + discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared: $reason") + magicPool.release(message.sid, message.magic, message.label, "cleared inflight: $reason") + } +} + +internal fun GlassesSessionCore.discardPrewriteIfMatchesLocked(message: OutboundMessage?) { + if (message != null && message === prewrittenMessage) { + prewrittenMessage = null + prewrittenFrames = emptyList() + } +} + +internal fun GlassesSessionCore.recordUnexpectedAckLocked(sid: Int, magic: Int) { + if (magic < BleMagicPool.MIN_MAGIC || magic > BleMagicPool.MAX_MAGIC) { + return + } + val previous = magicPool.getReleaseRecord(sid, magic) + if (previous == null) { + val pendingNote = if (hasPendingMagicLocked(sid, magic)) " while that magic is only pending locally" else "" + logLine("unexpected ACK sid=" + sid + " magic=" + magic + pendingNote + + "; possible Even app BLE contention") + return + } + if ("timeout" == previous.reason) { + logLine("late ACK after timeout sid=" + sid + " magic=" + magic + + " label=" + previous.label + + "; ACK timeout may be too short") + return + } + if ("ack" == previous.reason) { + logLine("duplicate ACK for already-acked message sid=" + sid + " magic=" + magic + + " label=" + previous.label + + "; possible Even app BLE contention") + return + } + logLine("late ACK for released message sid=" + sid + " magic=" + magic + + " label=" + previous.label + + " release=" + previous.reason) +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/SessionPorts.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/SessionPorts.kt new file mode 100644 index 00000000..1630beb4 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/SessionPorts.kt @@ -0,0 +1,165 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.InvocationKind +import kotlin.contracts.contract + +/** + * GATT transport used by [GlassesSessionCore]. Addresses are opaque strings (Android MAC + * addresses, CoreBluetooth identifiers on iOS); every call blocks the calling thread until the + * operation completes or [timeoutMs] elapses, exactly like Android's FaceclawBleManager. Inbound + * traffic reaches the core through its FaceclawBleListener methods, which the implementation + * must call from whatever thread the platform stack delivers on. + */ +interface SessionLink { + fun connect(address: String, timeoutMs: Int): Boolean + + /** Best effort; no completion point (see connectArm). */ + fun requestHighPriority(address: String) + + fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean + + fun negotiatedMtu(address: String): Int + + fun discoverServices(address: String, timeoutMs: Int): Boolean + + fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean + + fun writeFrames( + address: String, + characteristicUuid: String, + frames: List, + mode: GattWriteMode, + timeoutMs: Int, + ): Boolean + + fun disconnect(address: String) + + fun close() + + /** Whether the OS still holds a pairing bond for the address (iOS: always true). */ + fun isBonded(address: String): Boolean + + /** Bandwidth-benchmark link tuning; 0: current link, 1: HIGH priority, 2: 2M PHY, 3: both. */ + fun prepareBenchmarkLink(address: String, mode: Int) + + /** Traffic counter hook: one display frame fully acknowledged. */ + fun recordDisplayFrameSent() +} + +/** Delivers an action on a specific thread (a Looper, a dispatch queue, ...). */ +fun interface SessionDispatcher { + fun post(action: () -> Unit) +} + +enum class SessionLogLevel { + DEBUG, + INFO, + WARN, + ERROR, +} + +/** + * Platform side effects the session core needs. Listener callbacks are marshalled through + * [postToMain]; everything else is a synchronous query or command on the calling thread. + */ +interface SessionHost { + fun postToMain(action: () -> Unit) + + /** A dispatcher for the calling thread (its Looper/queue), falling back to the main thread. */ + fun currentThreadDispatcher(): SessionDispatcher + + fun isPhoneLocked(): Boolean + + /** Hold (true) or release (false) the "glasses screen on" wake lock; idempotent. */ + fun setScreenWakeLock(on: Boolean) + + /** Whether the stock Even app appears active (it can hold the BLE link). */ + fun isEvenAppActive(): Boolean + + /** Start the worker thread that runs [body] (the core's run loop) once. */ + fun startWorker(body: () -> Unit) + + /** Interrupt (where supported) and wait up to [timeoutMs] for the worker started by [startWorker]. */ + fun joinWorker(timeoutMs: Long) + + fun log(level: SessionLogLevel, tag: String, message: String, error: Throwable?) +} + +/** + * Java-monitor replacement: a reentrant state lock plus a separate condition used only at the + * blocking wait sites. Waiters must hold [lock] exactly once when calling [awaitMs]; notifiers + * may hold it (they normally do). Ordering is always lock -> signal, and the waiter never holds + * the signal while acquiring the lock, so the pair cannot deadlock and cannot lose a wake-up: + * the predicate is checked under [lock], the signal is taken before [lock] is released, and + * [ProtocolCondition.awaitMs] releases the signal atomically. + */ +internal class SessionMonitor(platform: ProtocolPlatform) { + val lock: ProtocolLock = platform.createLock() + private val signal: ProtocolCondition = platform.createCondition() + + fun signalAll() { + signal.lock() + try { + signal.signalAll() + } finally { + signal.unlock() + } + } + + /** Caller holds [lock] once; returns with it held again. Spurious wake-ups are possible. */ + fun awaitMs(timeoutMs: Long) { + signal.lock() + lock.unlock() + try { + signal.awaitMs(timeoutMs) + } finally { + signal.unlock() + } + lock.lock() + } +} + +/** Snapshot-iterated listener list (java.util.concurrent.CopyOnWriteArrayList stand-in). */ +internal class CopyOnWriteList(platform: ProtocolPlatform) : Iterable { + private val lock = platform.createLock() + + @Volatile private var items: List = emptyList() + + fun add(item: T) { + lock.withLock { items = items + item } + } + + fun remove(item: T) { + lock.withLock { items = items - item } + } + + fun isEmpty(): Boolean = items.isEmpty() + + override fun iterator(): Iterator = items.iterator() +} + +/** Runs [action] under the monitor's state lock; the contract lets `val`s be assigned inside, like `synchronized`. */ +@OptIn(ExperimentalContracts::class) +internal inline fun SessionMonitor.withLock(action: () -> T): T { + contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) } + lock.lock() + try { + return action() + } finally { + lock.unlock() + } +} + +/** [ProtocolLock.withLock] with a call-in-place contract. */ +@OptIn(ExperimentalContracts::class) +internal inline fun ProtocolLock.locked(action: () -> T): T { + contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) } + lock() + try { + return action() + } finally { + unlock() + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GlyphPacket.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GlyphPacket.kt new file mode 100644 index 00000000..5d719a5a --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GlyphPacket.kt @@ -0,0 +1,168 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic +import kotlin.math.pow +import kotlin.math.roundToInt + +/** + * Wire packets produced by the platform text rasterizers (Android FontFileRenderer over + * minikin/HarfBuzz, iOS over CoreText) and consumed by the TypeScript side + * (app/native/font-files.ts, app/graphics/ttf-font.ts). Rasterization stays per platform; + * the packet layouts, gamma mapping and ink-box cropping live here so both platforms agree. + * + * Image packet (renderText / renderWrapped): [widthLo, widthHi, heightLo, heightHi, pixels...], + * one gamma-mapped coverage byte per pixel, row-major. + * + * Glyph-cell packet (renderGlyphCell), little-endian: + * [advance u16, 26.6 fixed point] + * [bearingX s16] ink left relative to the pen position + * [inkTop s16] ink top relative to the line top (ascent above baseline) + * [width u16][height u16] + * [width*height coverage bytes, gamma-mapped] + * An ink-free glyph (space) has width = height = 0 but a valid advance. + */ +object GlyphPacket { + const val IMAGE_HEADER_BYTES = 4 + const val GLYPH_CELL_HEADER_BYTES = 10 + + /** out = 255 * (coverage/255)^gamma; non-positive gamma means linear. */ + @JvmStatic + fun gammaLut(gammaIn: Double): ByteArray { + val gamma = if (gammaIn > 0 && gammaIn.isFinite()) gammaIn else 1.0 + val lut = ByteArray(256) + for (i in 0 until 256) { + lut[i] = (255.0 * (i / 255.0).pow(gamma)).roundToInt().coerceIn(0, 255).toByte() + } + return lut + } + + /** [first, last] painted (nonzero) column indexes, or null if blank. */ + @JvmStatic + fun paintedColumnRange(pixels: ByteArray, width: Int, height: Int): IntArray? { + var first = width + var last = -1 + for (y in 0 until height) { + val row = y * width + var x = 0 + while (x < first) { + if (pixels[row + x].toInt() != 0) { + first = x + break + } + x++ + } + x = width - 1 + while (x > last) { + if (pixels[row + x].toInt() != 0) { + last = x + break + } + x-- + } + } + return if (last >= first) intArrayOf(first, last) else null + } + + /** Tight [x0, y0, x1, y1] (inclusive) box of nonzero pixels, or null if blank. */ + @JvmStatic + fun inkBounds(pixels: ByteArray, width: Int, height: Int): IntArray? { + var x0 = width + var x1 = -1 + var y0 = height + var y1 = -1 + for (y in 0 until height) { + val row = y * width + for (x in 0 until width) { + if (pixels[row + x].toInt() != 0) { + if (x < x0) x0 = x + if (x > x1) x1 = x + if (y < y0) y0 = y + if (y > y1) y1 = y + } + } + } + return if (x1 >= x0) intArrayOf(x0, y0, x1, y1) else null + } + + /** Image packet of the whole width*height alpha raster. */ + @JvmStatic + fun packImage(alpha8: ByteArray, width: Int, height: Int, gamma: Double): ByteArray = + packImageRegion(alpha8, width, 0, 0, width, height, gamma) + + /** Image packet of the [x, y, width, height] region of a raster with row [stride]. */ + @JvmStatic + fun packImageRegion(alpha8: ByteArray, stride: Int, x: Int, y: Int, width: Int, height: Int, gamma: Double): ByteArray { + require(width >= 0 && height >= 0 && x >= 0 && y >= 0 && x + width <= stride) + val lut = gammaLut(gamma) + val out = ByteArray(IMAGE_HEADER_BYTES + width * height) + putU16(out, 0, width) + putU16(out, 2, height) + for (row in 0 until height) { + val src = (y + row) * stride + x + val dst = IMAGE_HEADER_BYTES + row * width + for (col in 0 until width) { + out[dst + col] = lut[alpha8[src + col].toInt() and 0xff] + } + } + return out + } + + /** 26.6 fixed-point advance clamped to u16. */ + @JvmStatic + fun advanceFixed(advancePx: Double): Int = (advancePx * 64.0).roundToInt().coerceIn(0, 0xffff) + + /** Glyph cell with no ink (space): valid advance, zero-size box. */ + @JvmStatic + fun emptyGlyphCell(advanceFixed: Int): ByteArray = glyphCellPacket(advanceFixed, 0, 0, 0, 0, null) + + /** Glyph cell from already gamma-mapped coverage. */ + @JvmStatic + fun glyphCellPacket(advanceFixed: Int, bearingX: Int, inkTop: Int, width: Int, height: Int, coverage: ByteArray?): ByteArray { + val out = ByteArray(GLYPH_CELL_HEADER_BYTES + (coverage?.size ?: 0)) + putU16(out, 0, advanceFixed) + putU16(out, 2, bearingX) + putU16(out, 4, inkTop) + putU16(out, 6, width) + putU16(out, 8, height) + coverage?.copyInto(out, GLYPH_CELL_HEADER_BYTES) + return out + } + + /** + * Glyph cell cropped to the tight ink box of a padded raster. [penX]/[baselineY] are the + * pen position inside the raster and [ascent] the font ascent above the baseline; the + * returned bearing/inkTop are relative to those, as the wire format requires. + */ + @JvmStatic + fun packGlyphCell( + advanceFixed: Int, + alpha8: ByteArray, + rasterWidth: Int, + rasterHeight: Int, + penX: Int, + baselineY: Int, + ascent: Int, + gamma: Double, + ): ByteArray { + val box = inkBounds(alpha8, rasterWidth, rasterHeight) ?: return emptyGlyphCell(advanceFixed) + val x0 = box[0] + val y0 = box[1] + val width = box[2] - x0 + 1 + val height = box[3] - y0 + 1 + val lut = gammaLut(gamma) + val coverage = ByteArray(width * height) + for (y in 0 until height) { + val src = (y0 + y) * rasterWidth + x0 + val dst = y * width + for (x in 0 until width) { + coverage[dst + x] = lut[alpha8[src + x].toInt() and 0xff] + } + } + return glyphCellPacket(advanceFixed, x0 - penX, ascent + (y0 - baselineY), width, height, coverage) + } + + private fun putU16(out: ByteArray, offset: Int, value: Int) { + out[offset] = (value and 0xff).toByte() + out[offset + 1] = ((value shr 8) and 0xff).toByte() + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt new file mode 100644 index 00000000..f11f1af9 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/GrayPacket.kt @@ -0,0 +1,117 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic +import kotlin.math.pow +import kotlin.math.roundToInt + +/** + * Converts ARGB pixels to the grayscale packet format used across the TS bridge: + * [widthLo, widthHi, heightLo, heightHi, pixels...] with one byte per pixel, row-major. + * Transparent pixels darken toward black (the on-glasses background). Platform code owns + * decoding and scaling (Bitmap / ImageIO); this object owns the pixel math so both platforms + * produce identical tone and dither. + */ +object GrayPacket { + const val HEADER_SIZE = 4 + + /** + * gamma > 1 darkens midtones (out = 255 * (in/255)^gamma). Source pixels are sRGB-encoded + * but the G2 drives its 16 levels roughly linearly, so mid-gray otherwise displays far + * brighter than intended; 2.2 undoes the sRGB encoding outright. + * + * dither error-diffuses against the 16 levels the frame is eventually packed to + * (BmpUtil.GRAY_TO_NIBBLE) instead of leaving each pixel to round on its own, which turns + * smooth gradients into visible bands. Dithered output is already quantized: every byte is + * a level times 16, which BmpUtil re-quantizes back to exactly that level. + */ + @JvmStatic + fun fromArgb(pixels: IntArray, width: Int, height: Int, gamma: Float, dither: Boolean): ByteArray { + require(width > 0 && height > 0 && pixels.size >= width * height) { "bad pixel plane" } + val out = ByteArray(HEADER_SIZE + width * height) + writeHeader(out, width, height) + val tone = toneCurve(gamma) + for (i in 0 until width * height) { + val p = pixels[i] + val a = (p ushr 24) and 0xff + val r = (p shr 16) and 0xff + val g = (p shr 8) and 0xff + val b = p and 0xff + out[HEADER_SIZE + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte() + } + if (dither) { + ditherToDisplayLevels(out, HEADER_SIZE, width, height) + } + return out + } + + @JvmStatic + fun writeHeader(out: ByteArray, width: Int, height: Int) { + out[0] = (width and 0xff).toByte() + out[1] = ((width shr 8) and 0xff).toByte() + out[2] = (height and 0xff).toByte() + out[3] = ((height shr 8) and 0xff).toByte() + } + + /** 256-entry map of source gray to displayed gray for out = in^gamma. */ + @JvmStatic + fun toneCurve(gamma: Float): IntArray { + val curve = IntArray(256) + for (v in 0 until 256) { + curve[v] = if (gamma == 1f) v else (255f * (v / 255f).pow(gamma)).roundToInt() + } + return curve + } + + /** + * Floyd-Steinberg error diffusion onto the display's 16 gray levels, in place over an 8bpp + * plane. Serpentine scanning keeps the diffusion from building up a directional texture + * across wide flat areas. + * + * Level n is written back as n * 16 rather than the n * 17 it stands for so that BmpUtil's + * (v + 8) >> 4 reproduces n exactly; level 0 is written as 1, not 0, because 0 is the shell's + * color-key for transparent. + */ + @JvmStatic + fun ditherToDisplayLevels(plane: ByteArray, offset: Int, width: Int, height: Int) { + var curr = FloatArray(width) + var next = FloatArray(width) + for (x in 0 until width) { + curr[x] = (plane[offset + x].toInt() and 0xff).toFloat() + } + for (y in 0 until height) { + val rowStart = offset + y * width + val nextStart = rowStart + width + val hasNext = y + 1 < height + for (x in 0 until width) { + next[x] = if (hasNext) (plane[nextStart + x].toInt() and 0xff).toFloat() else 0f + } + val leftToRight = (y and 1) == 0 + val start = if (leftToRight) 0 else width - 1 + val step = if (leftToRight) 1 else -1 + for (i in 0 until width) { + val x = start + i * step + val wanted = curr[x] + val level = (minOf(255f, maxOf(0f, wanted)) / 17f).roundToInt() + val error = wanted - level * 17f + plane[rowStart + x] = (if (level == 0) 1 else level * 16).toByte() + val ahead = x + step + if (ahead >= 0 && ahead < width) { + curr[ahead] += error * (7f / 16f) + } + if (hasNext) { + if (ahead >= 0 && ahead < width) { + next[ahead] += error * (1f / 16f) + } + next[x] += error * (5f / 16f) + val behind = x - step + if (behind >= 0 && behind < width) { + next[behind] += error * (3f / 16f) + } + } + } + val swap = curr + curr = next + next = swap + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/OpenTypeNames.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/OpenTypeNames.kt new file mode 100644 index 00000000..97a3b7c3 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/OpenTypeNames.kt @@ -0,0 +1,110 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic + +/** Random access over font bytes, so large files are not read whole just for a name. */ +interface RandomAccessBytes { + val length: Long + + /** Reads exactly [count] bytes at [offset] into [into]; throws when out of range. */ + fun read(offset: Long, into: ByteArray, count: Int) +} + +class ArrayRandomAccessBytes(private val bytes: ByteArray) : RandomAccessBytes { + override val length: Long + get() = bytes.size.toLong() + + override fun read(offset: Long, into: ByteArray, count: Int) { + require(offset >= 0 && count >= 0 && offset + count <= bytes.size) { "read out of range" } + bytes.copyInto(into, 0, offset.toInt(), offset.toInt() + count) + } +} + +/** + * Minimal OpenType 'name' table reader (TTF/OTF/TTC, first face) for font titles. + * Returns "Family\nStyle" ("Style" may be empty), preferring the typographic names + * (ids 16/17) over the legacy ones (1/2), or "" when the table cannot be parsed. + */ +object OpenTypeNames { + private const val TTCF = 0x74746366 + private const val NAME = 0x6e616d65 + + @JvmStatic + fun parseFile(path: String): String = parse(ArrayRandomAccessBytes(readFileData(path))) + + @JvmStatic + fun parse(bytes: ByteArray): String = parse(ArrayRandomAccessBytes(bytes)) + + @JvmStatic + fun parse(source: RandomAccessBytes): String { + val fileSize = source.length + if (fileSize < 12) return "" + var offset = 0L + if (readU32(source, 0) == TTCF) { // 'ttcf': use the first face + if (readU32(source, 8) < 1) return "" + offset = readU32(source, 12).toLong() and 0xffffffffL + } + val numTables = readU16(source, offset + 4) + var nameOffset = -1L + for (i in 0 until numTables) { + val rec = offset + 12 + i * 16L + if (rec + 16 > fileSize) return "" + if (readU32(source, rec) == NAME) { + nameOffset = readU32(source, rec + 8).toLong() and 0xffffffffL + break + } + } + if (nameOffset < 0 || nameOffset + 6 > fileSize) return "" + val count = readU16(source, nameOffset + 2) + val stringStorage = nameOffset + readU16(source, nameOffset + 4) + var family: String? = null + var style: String? = null + var preferredFamily: String? = null + var preferredStyle: String? = null + for (i in 0 until count) { + val rec = nameOffset + 6 + i * 12L + if (rec + 12 > fileSize) break + val platform = readU16(source, rec) + val nameId = readU16(source, rec + 6) + if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) continue + val length = readU16(source, rec + 8) + val strOffset = stringStorage + readU16(source, rec + 10) + if (strOffset + length > fileSize || length <= 0 || length > 512) continue + val data = ByteArray(length) + source.read(strOffset, data, length) + // Platform 0 (Unicode) and 3 (Windows) store UTF-16BE; + // platform 1 (Mac) is close enough to Latin-1 for names. + val value = (if (platform == 1) latin1(data) else utf16be(data)).trim() + if (value.isEmpty()) continue + if (nameId == 1 && family == null) family = value + if (nameId == 2 && style == null) style = value + if (nameId == 16 && preferredFamily == null) preferredFamily = value + if (nameId == 17 && preferredStyle == null) preferredStyle = value + } + val outFamily = preferredFamily ?: family ?: return "" + return outFamily + "\n" + (preferredStyle ?: style ?: "") + } + + private fun latin1(data: ByteArray): String = + CharArray(data.size) { (data[it].toInt() and 0xff).toChar() }.concatToString() + + private fun utf16be(data: ByteArray): String { + val chars = CharArray(data.size / 2) { + (((data[2 * it].toInt() and 0xff) shl 8) or (data[2 * it + 1].toInt() and 0xff)).toChar() + } + return chars.concatToString() + } + + private fun readU16(source: RandomAccessBytes, offset: Long): Int { + val b = ByteArray(2) + source.read(offset, b, 2) + return ((b[0].toInt() and 0xff) shl 8) or (b[1].toInt() and 0xff) + } + + private fun readU32(source: RandomAccessBytes, offset: Long): Int { + val b = ByteArray(4) + source.read(offset, b, 4) + return ((b[0].toInt() and 0xff) shl 24) or ((b[1].toInt() and 0xff) shl 16) or + ((b[2].toInt() and 0xff) shl 8) or (b[3].toInt() and 0xff) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/PreviewPalette.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/PreviewPalette.kt new file mode 100644 index 00000000..134c750b --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/PreviewPalette.kt @@ -0,0 +1,52 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic +import kotlin.math.pow +import kotlin.math.roundToLong + +/** + * Palette for the phone-UI preview of an 8bpp grayscale frame: gray -> ARGB with a brightening + * gamma, either grayscale or green-on-black (matching the physical glasses). Both platforms + * should build their preview images from this table so they brighten identically. + */ +object PreviewPalette { + private var cachedLut: IntArray? = null + private var cachedGammaBits: Long = 0 + private var cachedGreen: Boolean = false + private val lock = protocolPlatform().createLock() + + /** ARGB value for [gray] under [brightenGamma]: 255 * (gray/255)^gamma per channel. */ + @JvmStatic + fun argb(gray: Int, brightenGamma: Double, green: Boolean): Int { + val v = (255 * (gray / 255.0).pow(brightenGamma)).roundToLong().coerceIn(0L, 255L).toInt() + return if (green) (0xff000000.toInt() or (v shl 8)) else (0xff000000.toInt() or (v shl 16) or (v shl 8) or v) + } + + /** Reuse the current preview palette; rebuild only when gamma or color changes. */ + @JvmStatic + fun lookupTable(brightenGamma: Double, green: Boolean): IntArray { + val gammaBits = brightenGamma.toBits() + lock.withLock { + val cached = cachedLut + if (cached != null && cachedGammaBits == gammaBits && cachedGreen == green) { + return cached + } + val lut = IntArray(256) { argb(it, brightenGamma, green) } + cachedGammaBits = gammaBits + cachedGreen = green + cachedLut = lut + return lut + } + } + + /** Expands [count] gray bytes starting at [offset] to ARGB pixels through the palette. */ + @JvmStatic + fun expand(gray: ByteArray, offset: Int, count: Int, brightenGamma: Double, green: Boolean): IntArray { + val lut = lookupTable(brightenGamma, green) + val colors = IntArray(count) + for (i in 0 until count) { + colors[i] = lut[gray[offset + i].toInt() and 0xff] + } + return colors + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/StatusIconArt.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/StatusIconArt.kt new file mode 100644 index 00000000..625d9326 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/StatusIconArt.kt @@ -0,0 +1,98 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic +import kotlin.math.abs + +/** + * Pure 24x24 8bpp icons for the system status strip (Wi-Fi level, cellular level, hotspot). + * Platform code reads the levels; this object only draws, so both platforms render the same + * artwork. + */ +object StatusIconArt { + const val ICON_SIZE = 24 + + @JvmStatic + fun wifi(level: Int): ByteArray { + val icon = ByteArray(ICON_SIZE * ICON_SIZE) + val cx = 12 + val cy = 18 + fillRect(icon, cx - 1, cy - 1, 3, 3, 230) + val radii = intArrayOf(5, 9, 13, 17) + for (index in radii.indices) { + if (level <= index) { + continue + } + drawWifiArc(icon, cx, cy, radii[index], 210) + } + return icon + } + + @JvmStatic + fun cell(level: Int): ByteArray { + val icon = ByteArray(ICON_SIZE * ICON_SIZE) + for (bar in 0 until 4) { + val x = 5 + bar * 4 + val height = 5 + bar * 4 + val value = if (level > bar) 220 else 55 + fillRect(icon, x, 20 - height, 3, height, value) + } + return icon + } + + @JvmStatic + fun hotspot(): ByteArray { + val icon = ByteArray(ICON_SIZE * ICON_SIZE) + fillRect(icon, 11, 17, 3, 3, 230) + drawWifiArc(icon, 12, 19, 7, 210) + drawWifiArc(icon, 12, 19, 12, 210) + fillRect(icon, 6, 5, 12, 2, 170) + return icon + } + + /** Nearest-neighbour resample of a 24x24 icon to size x size. */ + @JvmStatic + fun scale(source: ByteArray, size: Int): ByteArray { + if (size == ICON_SIZE) { + return source + } + val scaled = ByteArray(size * size) + for (y in 0 until size) { + val sy = minOf(ICON_SIZE - 1, (y * ICON_SIZE) / size) + for (x in 0 until size) { + val sx = minOf(ICON_SIZE - 1, (x * ICON_SIZE) / size) + scaled[y * size + x] = source[sy * ICON_SIZE + sx] + } + } + return scaled + } + + private fun drawWifiArc(icon: ByteArray, cx: Int, cy: Int, radius: Int, value: Int) { + val inner = (radius - 1) * (radius - 1) + val outer = (radius + 1) * (radius + 1) + for (y in 0 until ICON_SIZE) { + for (x in 0 until ICON_SIZE) { + val dx = x - cx + val dy = y - cy + val dist = dx * dx + dy * dy + if (dy <= 0 && dist >= inner && dist <= outer && abs(dx) <= radius) { + setPixel(icon, x, y, value) + } + } + } + } + + private fun fillRect(icon: ByteArray, x: Int, y: Int, width: Int, height: Int, value: Int) { + for (row in y until y + height) { + for (col in x until x + width) { + setPixel(icon, col, row, value) + } + } + } + + private fun setPixel(icon: ByteArray, x: Int, y: Int, value: Int) { + if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) { + return + } + icon[y * ICON_SIZE + x] = value.coerceIn(0, 255).toByte() + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SvgIconSource.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SvgIconSource.kt new file mode 100644 index 00000000..676b4b31 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SvgIconSource.kt @@ -0,0 +1,150 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic +import kotlin.math.min + +/** One drawable element of a parsed icon; coordinates are in viewBox units. */ +sealed class SvgShape { + class PathData(@JvmField val d: String) : SvgShape() + + class Circle(@JvmField val cx: Float, @JvmField val cy: Float, @JvmField val r: Float) : SvgShape() + + class Ellipse(@JvmField val cx: Float, @JvmField val cy: Float, @JvmField val rx: Float, @JvmField val ry: Float) : SvgShape() + + /** rx/ry are both 0 for a sharp rectangle, otherwise both positive. */ + class Rect( + @JvmField val x: Float, + @JvmField val y: Float, + @JvmField val width: Float, + @JvmField val height: Float, + @JvmField val rx: Float, + @JvmField val ry: Float, + ) : SvgShape() + + class Line(@JvmField val x1: Float, @JvmField val y1: Float, @JvmField val x2: Float, @JvmField val y2: Float) : SvgShape() + + /** [points] is x0,y0,x1,y1,... with at least one pair; [closed] for polygon. */ + class Polyline(@JvmField val points: FloatArray, @JvmField val closed: Boolean) : SvgShape() +} + +class SvgIcon( + @JvmField val minX: Float, + @JvmField val minY: Float, + @JvmField val viewWidth: Float, + @JvmField val viewHeight: Float, + /** Lucide-style outline icons (fill="none") are stroked; everything else is filled. */ + @JvmField val stroked: Boolean, + @JvmField val shapes: List, +) { + /** + * Mapping from viewBox to a size*size raster: [translateX, translateY, scale]. Apply as + * translate(tx, ty), scale(s), translate(-minX, -minY), i.e. the viewBox centered in the box. + */ + fun placement(size: Int): FloatArray { + val scale = min(size / viewWidth, size / viewHeight) + return floatArrayOf((size - viewWidth * scale) / 2f, (size - viewHeight * scale) / 2f, scale) + } +} + +/** + * Parses the small subset of SVG used by Lucide / simple Noun Project icons (path, circle, + * ellipse, rect, line, polyline, polygon) into shapes the platform path builders draw. + * Also owns the 3-level coverage quantization applied after rasterization on both platforms. + */ +object SvgIconSource { + private val TAG_RE = Regex("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>") + private val VIEWBOX_RE = Regex("viewBox\\s*=\\s*\"([^\"]*)\"") + private val SEPARATOR_RE = Regex("[\\s,]+") + + /** Null when nothing drawable was found. */ + @JvmStatic + fun parse(svg: String): SvgIcon? { + var minX = 0f + var minY = 0f + var viewW = 24f + var viewH = 24f + val vb = VIEWBOX_RE.find(svg) + if (vb != null) { + val parts = SEPARATOR_RE.split(vb.groupValues[1].trim()) + if (parts.size == 4) { + minX = parts[0].toFloat() + minY = parts[1].toFloat() + viewW = parts[2].toFloat() + viewH = parts[3].toFloat() + } + } + val stroked = svg.contains("fill=\"none\"") + val shapes = ArrayList() + for (m in TAG_RE.findAll(svg)) { + parseElement(m.groupValues[1], m.groupValues[2])?.let { shapes.add(it) } + } + if (shapes.isEmpty()) return null + return SvgIcon(minX, minY, viewW, viewH, stroked, shapes) + } + + /** + * Snap antialiased coverage toward hard edges so the 4bpp frames sent over BLE deflate + * better: mostly-transparent and mostly-opaque pixels become exactly 0/255 (long runs, and + * 0 stays color-key transparent for bitBlt), and only genuinely half-covered edge pixels + * keep one mid gray. At icon sizes this is visually near-identical. + */ + @JvmStatic + fun quantizeCoverage(alpha: Int): Int { + if (alpha < 64) return 0 + if (alpha >= 192) return 255 + return 128 + } + + /** Quantized coverage bytes from ARGB pixels (alpha channel = coverage). */ + @JvmStatic + fun quantizeArgb(pixels: IntArray): ByteArray { + val gray = ByteArray(pixels.size) + for (i in pixels.indices) gray[i] = quantizeCoverage((pixels[i] ushr 24) and 0xff).toByte() + return gray + } + + private fun parseElement(tag: String, attrs: String): SvgShape? = + when (tag) { + "path" -> attr(attrs, "d")?.let { SvgShape.PathData(it) } + "circle" -> { + val r = num(attrs, "r", 0f) + if (r > 0) SvgShape.Circle(num(attrs, "cx", 0f), num(attrs, "cy", 0f), r) else null + } + "ellipse" -> { + val rx = num(attrs, "rx", 0f) + val ry = num(attrs, "ry", 0f) + if (rx > 0 && ry > 0) SvgShape.Ellipse(num(attrs, "cx", 0f), num(attrs, "cy", 0f), rx, ry) else null + } + "rect" -> { + val w = num(attrs, "width", 0f) + val h = num(attrs, "height", 0f) + var rx = num(attrs, "rx", 0f) + var ry = num(attrs, "ry", if (rx > 0) rx else 0f) + if (rx <= 0) rx = ry + if (ry <= 0) ry = rx + if (w > 0 && h > 0) { + val rounded = rx > 0 || ry > 0 + SvgShape.Rect(num(attrs, "x", 0f), num(attrs, "y", 0f), w, h, if (rounded) rx else 0f, if (rounded) ry else 0f) + } else null + } + "line" -> SvgShape.Line(num(attrs, "x1", 0f), num(attrs, "y1", 0f), num(attrs, "x2", 0f), num(attrs, "y2", 0f)) + "polyline", "polygon" -> { + val points = attr(attrs, "points") ?: return null + val nums = SEPARATOR_RE.split(points.trim()) + val pairs = nums.size / 2 + if (pairs == 0) return null + val values = FloatArray(pairs * 2) { nums[it].toFloat() } + SvgShape.Polyline(values, tag == "polygon") + } + else -> null + } + + private fun attr(attrs: String, name: String): String? = + Regex("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").find(attrs)?.groupValues?.get(1) + + private fun num(attrs: String, name: String, fallback: Float): Float { + val value = attr(attrs, name) ?: return fallback + return value.trim().toFloatOrNull() ?: fallback + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/HttpIdentity.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/HttpIdentity.kt new file mode 100644 index 00000000..f39b17f0 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/HttpIdentity.kt @@ -0,0 +1,26 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.jvm.JvmStatic + +/** + * Shared HTTP identity for the native networking helpers on both platforms. The version + * lives in app/version.ts, so the TypeScript side pushes the real string in at startup + * (see installNativeUserAgent in app/util/http.ts); the default is only a fallback for + * requests that somehow beat that call. + */ +object HttpIdentity { + const val DEFAULT_USER_AGENT = "Faceclaw" + + @Volatile + private var userAgent = DEFAULT_USER_AGENT + + @JvmStatic + fun setUserAgent(value: String?) { + val trimmed = value?.trim() + if (!trimmed.isNullOrEmpty()) userAgent = trimmed + } + + @JvmStatic + fun getUserAgent(): String = userAgent +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/RemoteInputSession.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/RemoteInputSession.kt new file mode 100644 index 00000000..40d9bbe0 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/RemoteInputSession.kt @@ -0,0 +1,229 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmField +import kotlin.jvm.JvmStatic + +/** One accepted client connection of the local input port. */ +interface RemoteInputConnection { + /** Reads one byte, blocking at most [timeoutMs]; returns -1 on end of stream, timeout or error. */ + fun readByte(timeoutMs: Long): Int + + fun write(bytes: ByteArray) + + fun close() +} + +/** A bound, listening socket. */ +interface RemoteInputListener { + /** Blocks for the next client; returns null when the listener is closed or accept fails. */ + fun accept(): RemoteInputConnection? + + val isClosed: Boolean + + fun close() +} + +/** OS socket access for [RemoteInputSession]; a POSIX actual is the iOS counterpart. */ +interface LocalServerPort { + /** True when [address] is a unicast address assigned to a local interface (not any/multicast). */ + fun isLocalUnicastAddress(address: String): Boolean + + /** Binds and listens on [address]:[port]; throws when the port cannot be opened. */ + fun bind(address: String, port: Int): RemoteInputListener +} + +interface RandomSource { + fun nextBytes(count: Int): ByteArray +} + +/** One network address of a local interface, as enumerated by the platform. */ +data class InterfaceAddress( + val interfaceName: String, + val address: String, + val up: Boolean, + val pointToPoint: Boolean, + val loopback: Boolean, + val linkLocal: Boolean, + val anyLocal: Boolean, +) + +/** + * Explicit-address, bounded local input port shared by Android (java.net) and iOS (POSIX). + * Exactly one request is in flight: a client sends one newline-terminated UTF-8 frame of at most + * 64 KiB, the session publishes it as a pending JSON request, notifies the app through + * [dispatch] (which must run the callback on the app's UI thread, outside the lock), and blocks + * the serving thread until [complete] supplies the reply or the request deadline passes. + */ +class RemoteInputSession( + private val port: LocalServerPort, + private val random: RandomSource, + private val dispatch: (() -> Unit) -> Unit, + platform: ProtocolPlatform = protocolPlatform(), + private val wallClock: () -> Long = ::currentTimeMillis, + private val requestTimeoutMs: Long = REQUEST_TIMEOUT_MS, + private val readTimeoutMs: Long = READ_TIMEOUT_MS, +) { + private val condition = platform.createCondition() + private var server: RemoteInputListener? = null + private var client: RemoteInputConnection? = null + private var pending: String? = null + private var reply: String? = null + private var nextId = 0L + private var requestId = 0L + private var deadline = 0L + private var requestListener: (() -> Unit)? = null + + fun setRequestListener(listener: (() -> Unit)?) { + condition.withLock { requestListener = listener } + } + + /** Returns "" on success or a user-facing error. Never resolves hostnames or binds a wildcard. */ + fun start(port: Int, address: String): String { + condition.withLock { + if (server != null) return "" + if (!isNumericLocalAddress(address)) return "Invalid local address." + if (!this.port.isLocalUnicastAddress(address)) return "Address is not assigned to a local interface." + val listener = try { + this.port.bind(address, port) + } catch (e: Exception) { + return "Could not open local input port $port." + } + server = listener + startThread("FaceclawInput", true) { serve(listener) } + return "" + } + } + + fun stop() { + condition.withLock { + try { server?.close() } catch (ignored: Exception) {} + try { client?.close() } catch (ignored: Exception) {} + server = null + client = null + pending = null + requestId = 0 + reply = null + condition.signalAll() + } + } + + /** The pending request JSON (id, expiresAt, body), consumed once; null when none or expired. */ + fun nextRequest(): String? { + condition.withLock { + if (wallClock() >= deadline) { + pending = null + return null + } + val result = pending + pending = null + return result + } + } + + fun complete(id: Long, value: String?) { + condition.withLock { + if (id == requestId && wallClock() < deadline) { + reply = value + condition.signalAll() + } + } + } + + fun randomSecret(): String = BinaryEncoding.bytesToHex(random.nextBytes(32)) + + private fun notifyRequestReady(id: Long) { + // Called with the condition held; dispatching keeps JS off the socket thread and outside the lock. + dispatch { + val callback = condition.withLock { + if (requestId != id || pending == null || wallClock() >= deadline) null else requestListener + } + callback?.invoke() + } + } + + private fun serve(listener: RemoteInputListener) { + while (!listener.isClosed) { + val connection = listener.accept() ?: continue + try { + condition.withLock { + if (server !== listener) return + client = connection + } + val text = readFrame(connection) ?: continue + val response: String = condition.withLock { + if (server !== listener) return + requestId = ++nextId + deadline = wallClock() + requestTimeoutMs + reply = null + pending = Json.write(linkedMapOf("id" to requestId, "expiresAt" to deadline, "body" to text)) + notifyRequestReady(requestId) + while (reply == null && server === listener && wallClock() < deadline) { + condition.awaitMs(maxOf(1L, deadline - wallClock())) + } + val result = reply ?: TIMEOUT_RESPONSE + if (server === listener) { + pending = null + reply = null + requestId = 0 + } + result + } + connection.write((response + "\n").encodeToByteArray()) + } catch (ignored: Exception) { + // Never log request bodies: they contain credentials and possibly private text. + } finally { + try { connection.close() } catch (ignored: Exception) {} + condition.withLock { if (server === listener) client = null } + } + } + } + + /** One newline-terminated frame, or null when the client hung up, stalled or overflowed. */ + private fun readFrame(connection: RemoteInputConnection): String? { + val readDeadline = wallClock() + readTimeoutMs + val body = ByteSink() + while (true) { + val remaining = readDeadline - wallClock() + if (remaining <= 0) return null + val b = connection.readByte(remaining) + if (b == -1) return null + if (b == '\n'.code) break + if (body.size() >= MAX_FRAME_BYTES) return null + body.write(b) + } + return try { + body.toByteArray().decodeToString(throwOnInvalidSequence = true) + } catch (e: Exception) { + null + } + } + + companion object { + const val MAX_FRAME_BYTES = 65536 + const val REQUEST_TIMEOUT_MS = 5000L + const val READ_TIMEOUT_MS = 5000L + + @JvmField + val TIMEOUT_RESPONSE = "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}" + + private val numericAddress = Regex("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?") + + /** Numeric IPv4/IPv6 literals only (optionally zone-scoped); hostnames are rejected. */ + @JvmStatic fun isNumericLocalAddress(address: String): Boolean = numericAddress.matches(address) + + /** Addresses a remote client could reach: up interfaces, minus loopback, link-local and wildcard. */ + @JvmStatic + fun reachableAddresses(all: List): List = + all.filter { it.up && !it.loopback && !it.linkLocal && !it.anyLocal } + + /** JSON array of {name, address, pointToPoint} consumed by app/remote/listeners.ts. */ + @JvmStatic + fun interfacesJson(all: List): String = + Json.write(reachableAddresses(all).map { + linkedMapOf("name" to it.interfaceName, "address" to it.address, "pointToPoint" to it.pointToPoint) + }) + + /** Lowercase hex SHA-256 of the UTF-8 token; the stored form of remote-input tokens. */ + @JvmStatic fun tokenHash(value: String): String = sha256Hex(value.encodeToByteArray()) + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/ResumableDownload.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/ResumableDownload.kt new file mode 100644 index 00000000..f1fce7f6 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/ResumableDownload.kt @@ -0,0 +1,178 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile + +/** One open HTTP response body; [read] returns bytes read or -1 at end. */ +interface DownloadResponse { + val code: Int + + /** Body length if known, else -1. */ + val contentLength: Long + + fun read(buffer: ByteArray): Int + + fun close() +} + +/** Blocking GET; `rangeFrom > 0` asks for `Range: bytes=-`. */ +interface DownloadTransport { + fun open(url: String, rangeFrom: Long): DownloadResponse +} + +interface FileSink { + fun write(bytes: ByteArray, offset: Int, length: Int) + + fun close() +} + +/** Storage port; paths are platform file paths. Multi-GB files must stream, hence no whole-file reads. */ +interface DownloadStorage { + /** Size in bytes, or -1 when missing. */ + fun length(path: String): Long + + fun prepareParent(path: String) + + fun openWrite(path: String, append: Boolean): FileSink + + /** Streams an existing file through [consumer] (bytes, count); stops early when it returns false. Returns bytes consumed. */ + fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long + + fun rename(from: String, to: String): Boolean + + fun delete(path: String): Boolean +} + +/** + * Large-file download to ".part" with HTTP Range resume across app restarts, a pinned + * sha256 hashed incrementally (including the resumed prefix), then a rename into place. + * [run] blocks on the calling thread; listener callbacks are invoked synchronously on it, so + * the platform adapter marshals them (Android posts to the constructing Looper). + */ +class ResumableDownload( + private val url: String, + private val destPath: String, + expectedSha256: String?, + private val expectedTotalBytes: Long, + private val transport: DownloadTransport, + private val storage: DownloadStorage, + private val listener: FaceclawModelDownloaderListener, + private val now: () -> Long = { currentTimeMillis() }, +) { + private val partPath = "$destPath.part" + private val expectedSha256 = expectedSha256?.lowercase() ?: "" + + @Volatile + private var cancelled = false + + @Volatile + private var response: DownloadResponse? = null + + /** Stops the download but keeps the .part file so a later run resumes. */ + fun cancel() { + cancelled = true + try { + response?.close() + } catch (ignored: Exception) { + } + } + + fun run() { + try { + if (storage.length(destPath) > 0) { + listener.onDone(destPath) + return + } + storage.prepareParent(destPath) + + var digest = Sha256Digest() + var offset = 0L + if (storage.length(partPath) >= 0) { + offset = hashExistingPrefix(digest) + if (cancelled) return + } + + val opened = transport.open(url, offset) + response = opened + var done: Long + try { + if (opened.code < 200 || opened.code >= 300) { + listener.onError("Model download failed: HTTP " + opened.code) + return + } + val resumed = opened.code == 206 + if (offset > 0 && !resumed) { + // Server ignored the Range header; start over. + offset = 0L + digest = Sha256Digest() + } + val total = + if (expectedTotalBytes > 0) expectedTotalBytes + else if (opened.contentLength > 0) offset + opened.contentLength + else -1L + + val out = storage.openWrite(partPath, resumed) + try { + val buffer = ByteArray(1 shl 16) + done = offset + var lastProgressAt = 0L + while (true) { + val read = opened.read(buffer) + if (read < 0) break + if (cancelled) return + out.write(buffer, 0, read) + digest.update(buffer, 0, read) + done += read + val time = now() + if (time - lastProgressAt >= PROGRESS_INTERVAL_MS) { + lastProgressAt = time + listener.onProgress(done, total) + } + } + } finally { + out.close() + } + } finally { + try { + opened.close() + } catch (ignored: Exception) { + } + } + + val partLength = storage.length(partPath) + if (expectedTotalBytes > 0 && partLength != expectedTotalBytes) { + listener.onError("Model download ended early (" + partLength + " of " + expectedTotalBytes + " bytes); try again to resume") + return + } + val actualSha = BinaryEncoding.bytesToHex(digest.digest()) + if (expectedSha256.isNotEmpty() && actualSha != expectedSha256) { + storage.delete(partPath) + listener.onError("Model download was corrupted (checksum mismatch); download it again") + return + } + if (!storage.rename(partPath, destPath)) { + listener.onError("Could not move the downloaded model into place") + return + } + listener.onDone(destPath) + } catch (e: Exception) { + if (cancelled) return + PlatformLog.w(TAG, "download failed: $e") + listener.onError("Model download failed: $e") + } + } + + /** Hash the already-downloaded prefix so the final digest covers the whole file. */ + private fun hashExistingPrefix(digest: Sha256Digest): Long = + storage.readAll(partPath) { bytes, count -> + if (cancelled) false + else { + digest.update(bytes, 0, count) + true + } + } + + companion object { + private const val TAG = "FaceclawModelDl" + const val PROGRESS_INTERVAL_MS = 500L + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/SseStream.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/SseStream.kt new file mode 100644 index 00000000..39b5ec13 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/net/SseStream.kt @@ -0,0 +1,209 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic + +/** Streaming POST transport; the platform (okhttp, NSURLSession) implements it. */ +interface StreamingHttp { + fun post(url: String, body: String, headers: List>, sink: ChunkSink): Cancellable +} + +interface Cancellable { + fun cancel() +} + +/** Raw response feed from a [StreamingHttp]; may be called from any thread, one call at a time. */ +interface ChunkSink { + fun onStatus(code: Int) + + fun onChunk(bytes: ByteArray, offset: Int, length: Int) + + fun onComplete() + + fun onFailure(message: String) +} + +sealed class SseEvent { + data class Line(val text: String) : SseEvent() + + data class HttpError(val code: Int, val body: String) : SseEvent() + + object Complete : SseEvent() + + data class Failure(val message: String) : SseEvent() +} + +/** + * Server-sent-events response policy shared by both platforms. Feeds raw bytes in through + * [ChunkSink] and queues [SseEvent]s for [takeEvents]. Behavior (matching the stricter iOS + * implementation): a UTF-8 BOM is stripped from the first line; lines split on LF, CR or CRLF; + * non-2xx responses buffer at most [ERROR_BODY_LIMIT] bytes of body and end in [SseEvent.HttpError]; + * a single line over [LINE_LIMIT] bytes, invalid UTF-8, or more than [QUEUE_BYTE_LIMIT] bytes / + * [QUEUE_EVENT_LIMIT] events waiting to be taken end the stream with a [SseEvent.Failure]. + * After the terminal event nothing further is queued; [cancel] drops everything. + */ +class SseStream : ChunkSink { + private val lock = protocolPlatform().createLock() + private val events = ArrayList() + private var queuedBytes = 0L + private var buffer = ByteArray(4096) + private var bufferLength = 0 + private var status = 0 + private var firstLine = true + private var finished = false + private var cancelled = false + + val isFinished: Boolean + get() = lock.withLock { finished } + + override fun onStatus(code: Int) { + lock.withLock { status = code } + } + + override fun onChunk(bytes: ByteArray, offset: Int, length: Int) { + lock.withLock { + if (finished || cancelled || length <= 0) return + if (status < 200 || status >= 300) { + val remaining = ERROR_BODY_LIMIT - bufferLength + append(bytes, offset, minOf(remaining, length)) + return + } + append(bytes, offset, length) + var start = 0 + var i = 0 + while (i < bufferLength) { + val c = buffer[i] + if (c != CR && c != LF) { + i++ + continue + } + if (c == CR && i + 1 == bufferLength) break + line(start, i - start) + if (finished) return + if (c == CR && buffer[i + 1] == LF) i++ + start = i + 1 + i++ + } + if (start > 0) { + buffer.copyInto(buffer, 0, start, bufferLength) + bufferLength -= start + } + if (bufferLength > LINE_LIMIT) finish(SseEvent.Failure("Streaming response line too large")) + } + } + + override fun onComplete() { + lock.withLock { + if (finished || cancelled) return + if (status < 200 || status >= 300) { + finish(SseEvent.HttpError(status, decode(0, bufferLength) ?: "")) + return + } + if (bufferLength > 0) { + if (buffer[bufferLength - 1] == CR) bufferLength-- + line(0, bufferLength) + if (finished) return + } + finish(SseEvent.Complete) + } + } + + override fun onFailure(message: String) { + lock.withLock { finish(SseEvent.Failure(message)) } + } + + /** Drains queued events in order. Empty once cancelled. */ + fun takeEvents(): List = + lock.withLock { + if (events.isEmpty()) return emptyList() + val taken = ArrayList(events) + events.clear() + queuedBytes = 0 + taken + } + + fun cancel() { + lock.withLock { + cancelled = true + events.clear() + queuedBytes = 0 + bufferLength = 0 + } + } + + private fun line(start: Int, length: Int) { + var text = decode(start, length) + if (text == null) { + finish(SseEvent.Failure("Invalid UTF-8 in streaming response")) + return + } + if (firstLine && text.startsWith('')) text = text.substring(1) + firstLine = false + queuedBytes += length + if (queuedBytes > QUEUE_BYTE_LIMIT || events.size >= QUEUE_EVENT_LIMIT) { + events.clear() + finish(SseEvent.Failure("Streaming response queue overflow")) + return + } + events.add(SseEvent.Line(text)) + } + + private fun decode(start: Int, length: Int): String? = + try { + buffer.decodeToString(start, start + length, throwOnInvalidSequence = true) + } catch (e: CharacterCodingException) { + null + } + + private fun finish(event: SseEvent) { + if (finished || cancelled) return + finished = true + events.add(event) + bufferLength = 0 + } + + private fun append(bytes: ByteArray, offset: Int, length: Int) { + if (length <= 0) return + if (bufferLength + length > buffer.size) buffer = buffer.copyOf(maxOf(bufferLength + length, buffer.size * 2)) + bytes.copyInto(buffer, bufferLength, offset, offset + length) + bufferLength += length + } + + companion object { + const val ERROR_BODY_LIMIT = 64 * 1024 + const val LINE_LIMIT = 2 * 1024 * 1024 + const val QUEUE_BYTE_LIMIT = 8L * 1024 * 1024 + const val QUEUE_EVENT_LIMIT = 16384 + private const val CR: Byte = '\r'.code.toByte() + private const val LF: Byte = '\n'.code.toByte() + + /** + * Unpacks the flat alternating [name, value, name, value, ...] header array the + * TypeScript side builds (okhttp Headers cannot cross the JS bridge). Entries with a + * null/empty name or a null value are skipped, as before. + */ + @JvmStatic + fun unpackHeaders(flat: Array?): List> { + if (flat == null) return emptyList() + val result = ArrayList>() + var i = 0 + while (i + 1 < flat.size) { + val name = flat[i] + val value = flat[i + 1] + if (!name.isNullOrEmpty() && value != null) result.add(name to value) + i += 2 + } + return result + } + + /** Delivers one event to a [FaceclawSseListener]. */ + @JvmStatic + fun deliver(event: SseEvent, listener: FaceclawSseListener) { + when (event) { + is SseEvent.Line -> listener.onLine(event.text) + is SseEvent.HttpError -> listener.onHttpError(event.code, event.body) + SseEvent.Complete -> listener.onComplete() + is SseEvent.Failure -> listener.onFailure(event.message) + } + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/settings/SettingsChangeHub.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/settings/SettingsChangeHub.kt new file mode 100644 index 00000000..f7c03152 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/settings/SettingsChangeHub.kt @@ -0,0 +1,71 @@ +package com.faceclaw.app + +/** Key/value backing store for app settings (SharedPreferences on Android, NSUserDefaults on iOS). */ +interface SettingsStorage { + fun getString(key: String?, defaultValue: String?): String? + + fun putString(key: String?, value: String?) + + fun getBoolean(key: String?, defaultValue: Boolean): Boolean + + fun putBoolean(key: String?, value: Boolean) +} + +/** Runs an action on the thread that registered a listener (a Looper Handler, a dispatch queue). */ +fun interface SettingsDispatcher { + fun post(action: () -> Unit) +} + +/** + * Settings reads/writes plus change fan-out. Each JS isolate registers one listener together + * with a dispatcher for its own thread; every write notifies every listener through its own + * dispatcher, so callbacks never run on a foreign thread. A listener registered without a + * dispatcher is accepted but never notified (it can still read fresh values on demand). + */ +class SettingsChangeHub( + private val storage: SettingsStorage, + private val warn: (String) -> Unit = { PlatformLog.w("FaceclawSettings", it) }, + platform: ProtocolPlatform = protocolPlatform(), +) { + private class Entry(val listener: FaceclawSettingsListener, val dispatcher: SettingsDispatcher?) + + private val lock = platform.createLock() + private var listeners: List = emptyList() + + fun getString(key: String?, defaultValue: String?): String? = storage.getString(key, defaultValue) + + fun setString(key: String?, value: String?) { + storage.putString(key, value) + notifyChanged(key) + } + + fun getBoolean(key: String?, defaultValue: Boolean): Boolean = storage.getBoolean(key, defaultValue) + + fun setBoolean(key: String?, value: Boolean) { + storage.putBoolean(key, value) + notifyChanged(key) + } + + fun register(listener: FaceclawSettingsListener, dispatcher: SettingsDispatcher?) { + if (dispatcher == null) warn("settings listener registered from a thread without a dispatcher; it will never be notified") + lock.withLock { listeners = listeners + Entry(listener, dispatcher) } + } + + fun unregister(listener: FaceclawSettingsListener?) { + lock.withLock { listeners = listeners.filter { it.listener !== listener } } + } + + fun notifyChanged(key: String?) { + val current = lock.withLock { listeners } + for (entry in current) { + val dispatcher = entry.dispatcher ?: continue + dispatcher.post { + try { + entry.listener.onSettingChanged(key) + } catch (error: Exception) { + warn("settings listener failed for key $key: ${error.message}") + } + } + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Concurrency.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Concurrency.kt new file mode 100644 index 00000000..02b82a99 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Concurrency.kt @@ -0,0 +1,105 @@ +package com.faceclaw.app + +/** + * Sleep that another thread can cut short. An interrupt delivered before the sleep begins is + * consumed by the next sleep (returns false immediately) rather than lost: interrupt() runs on + * other threads with no lock held by the sleeper between two work passes, so clearing the flag + * at entry would force a full sleep interval. Returns true when the full duration elapsed. + */ +class InterruptibleSleep(private val platform: ProtocolPlatform = protocolPlatform()) { + private val condition = platform.createCondition() + private var interrupted = false + + fun sleep(ms: Long): Boolean { + val deadline = platform.elapsedRealtimeMs() + maxOf(1L, ms) + condition.lock() + try { + if (interrupted) { + interrupted = false + return false + } + while (true) { + val remaining = deadline - platform.elapsedRealtimeMs() + if (remaining <= 0) return true + condition.awaitMs(remaining) + if (interrupted) { + interrupted = false + return false + } + } + } finally { + condition.unlock() + } + } + + fun interrupt() { + condition.withLock { + interrupted = true + condition.signalAll() + } + } +} + +/** One-shot countdown; await returns true once the count reaches zero, false on timeout. */ +class Latch(count: Int, private val platform: ProtocolPlatform = protocolPlatform()) { + private val condition = platform.createCondition() + private var remaining = count + + fun countDown() { + condition.withLock { + if (remaining > 0) remaining-- + if (remaining == 0) condition.signalAll() + } + } + + fun await(timeoutMs: Long): Boolean { + val deadline = platform.elapsedRealtimeMs() + timeoutMs + condition.withLock { + while (remaining > 0) { + val left = deadline - platform.elapsedRealtimeMs() + if (left <= 0) return false + condition.awaitMs(left) + } + return true + } + } +} + +/** Unbounded FIFO handoff between threads with a timed poll. */ +class BlockingQueue(private val platform: ProtocolPlatform = protocolPlatform()) { + private val condition = platform.createCondition() + private val items = ArrayDeque() + + fun put(item: T) { + condition.withLock { + items.addLast(item) + condition.signalAll() + } + } + + fun poll(timeoutMs: Long): T? { + val deadline = platform.elapsedRealtimeMs() + timeoutMs + condition.withLock { + while (items.isEmpty()) { + val left = deadline - platform.elapsedRealtimeMs() + if (left <= 0) return null + condition.awaitMs(left) + } + return items.removeFirst() + } + } + + fun clear() { + condition.withLock { items.clear() } + } + + val size: Int + get() = condition.withLock { items.size } +} + +/** Starts a platform thread running [body]; daemon threads never block process exit on the JVM. */ +expect fun startThread(name: String, daemon: Boolean, body: () -> Unit) + +expect fun sleepMs(ms: Long) + +fun currentThreadName(): String = com.faceclaw.shared.PlatformInfo.threadName() diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt new file mode 100644 index 00000000..74ee8e57 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/FrameTimingsCore.kt @@ -0,0 +1,451 @@ +package com.faceclaw.app + +import kotlin.math.abs +import kotlin.math.ceil +import kotlin.math.max +import kotlin.math.min + +/** + * Per-frame timing instrumentation, shared between the native BLE layer and the + * Typescript UI layer. A "frame" starts when we receive an input event (or a + * timer fires and we decide to redraw), and finishes when the resulting screen + * update has been fully transmitted to the glasses, is discarded unsent, or + * times out because nobody reported finishing it. + * + * Frames form a tree. One input event can fan out into several renders (the + * focused app's window plus the shell chrome, say), and each of those is its + * own frame linked back to the input frame that caused it via + * [startFrame]. The export nests descendants under their + * root and renders every line against the root's clock, so a single block + * shows the whole input-to-pixels story. Latency percentiles are measured over + * roots: root start to the first descendant that actually reached the glasses. + * + * All public methods are thread-safe and cheap enough to call from the BLE + * worker thread, the platform main thread, the JS thread, and app worker + * threads. Frame IDs are positive ints; 0 means "no frame" and is silently + * ignored everywhere, so callers do not need to null-check. + * + * This core owns the bookkeeping and the export text; the platform facade owns + * the export thread and file write (see the Android `FrameTimings`). The clock + * is [ProtocolPlatform.elapsedRealtimeMs] (monotonic, includes sleep); wall-clock, + * thread names and logging are injectable for tests. + */ +class FrameTimingsCore( + private val platform: ProtocolPlatform = protocolPlatform(), + private val wallClockMs: () -> Long = { currentTimeMillis() }, + private val threadName: () -> String = { currentThreadName() }, + private val formatWallClock: (Long) -> String = { formatLocalTime(it, "yyyy-MM-dd HH:mm:ss.SSS") }, + private val logInfo: (String) -> Unit = { PlatformLog.i(TAG, it) }, + private val logWarn: (String) -> Unit = { PlatformLog.w(TAG, it) }, +) { + companion object { + const val TAG = "FrameTimings" + + /** Frames still open after this long are finished as "timeout". */ + const val FRAME_TIMEOUT_MS = 30_000L + const val EXPORT_INTERVAL_MS = 15_000L + const val SWEEP_INTERVAL_MS = 5_000L + private const val RECENT_ROOTS_KEPT = 40 + private const val SLOWEST_ROOTS_KEPT = 20 + private const val SENT_DURATIONS_WINDOW = 512 + private const val MAX_LINES_PER_FRAME = 200 + /** + * Frames retained for ID lookup. Well above the number any export can + * reference, so a late log/span/finish call always finds its frame; frames + * older than this are unreachable and get collected. + */ + private const val FRAMES_RETAINED = 1024 + + private fun percentileOfSorted(sorted: List, percentile: Int): Long { + val index = ceil(percentile / 100.0 * sorted.size).toInt() - 1 + return sorted[max(0, min(sorted.size - 1, index))] + } + + /** java.lang.String.format("%+7d"): explicit sign, right-aligned in 7 columns. */ + private fun signedWidth7(value: Long): String { + val text = (if (value < 0) "-" else "+") + abs(value).toString() + return text.padStart(7) + } + } + + private class Line( + val atMs: Long, // elapsedRealtimeMs + val thread: String, + val message: String? + ) + + private class Frame( + val id: Int, + var reason: String, + val startedAtMs: Long, // elapsedRealtimeMs + val startedWallClockMs: Long, + val parent: Frame? + ) { + val children: MutableList = ArrayList(0) + val lines: MutableList = ArrayList() + val openSpans: MutableMap = LinkedHashMap() + var finishedAtMs: Long = 0 + var outcome: String? = null // null while open; "sent", "discarded: ...", "timeout: ..." + /** Set on the root once some frame in its subtree reached the glasses. */ + var subtreeSentRecorded = false + + fun root(): Frame { + var frame: Frame = this + while (frame.parent != null) { + frame = frame.parent!! + } + return frame + } + + fun isOpen(): Boolean = outcome == null + + fun durationMs(): Long = finishedAtMs - startedAtMs + + fun wasSent(): Boolean { + val outcome = this.outcome + return outcome != null && outcome.startsWith("sent") + } + + /** Last moment anything in this subtree happened, for whole-tree duration. */ + fun subtreeEndAtMs(nowMs: Long): Long { + var end = if (isOpen()) nowMs else finishedAtMs + for (child in children) { + end = max(end, child.subtreeEndAtMs(nowMs)) + } + return end + } + + fun subtreeHasOpenFrame(): Boolean { + if (isOpen()) return true + for (child in children) { + if (child.subtreeHasOpenFrame()) return true + } + return false + } + } + + private val lock = platform.createLock() + /** + * Every frame we still retain, keyed by ID, in insertion order: open frames + * plus enough history that late calls and the export can still resolve one. + */ + private val framesById: LinkedHashMap = LinkedHashMap() + private val recentRoots = ArrayDeque() + private val slowestRoots: MutableList = ArrayList() + private val inputLatencies = ArrayDeque() + private val renderLatencies = ArrayDeque() + private var nextFrameId = 1 + private var framesStarted: Long = 0 + private var framesSent: Long = 0 + private var framesDiscarded: Long = 0 + private var framesTimedOut: Long = 0 + private var exportDirty = false + + /** Begin a root frame; reason is a short label like "input:sys-event type=0". */ + fun startFrame(reason: String?): Int = startFrame(reason, 0) + + /** + * Begin a frame caused by an existing one (parentFrameId; 0 for a root). + * The child is nested under its root in the export and its latency counts + * against the root, so an input event that fans out into an app render and + * a shell-chrome render reads as one timeline instead of three. + */ + fun startFrame(reason: String?, parentFrameId: Int): Int { + val now = platform.elapsedRealtimeMs() + val frame = lock.withLock { + val parent = framesById[parentFrameId] + val frame = Frame(nextFrameId++, reason ?: "", now, wallClockMs(), parent) + framesById[frame.id] = frame + // Like LinkedHashMap.removeEldestEntry: drop at most the single eldest + // entry per insert, and only when it is already finished. + if (framesById.size > FRAMES_RETAINED) { + val eldest = framesById.entries.first() + if (!eldest.value.isOpen()) framesById.remove(eldest.key) + } + framesStarted++ + if (parent != null) { + parent.children.add(frame) + addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")") + } + frame + } + return frame.id + } + + /** + * Append to a frame's label, for facts only known after it started (which + * app is in the foreground, which window a render is for). Shows up in the + * frame's header line, so the export is scannable without reading bodies. + */ + fun annotate(frameId: Int, text: String?) { + if (text == null || text.isEmpty()) return + lock.withLock { + val frame = framesById[frameId] ?: return + frame.reason = if (frame.reason.isEmpty()) text else frame.reason + " " + text + } + } + + fun log(frameId: Int, message: String?) { + val now = platform.elapsedRealtimeMs() + lock.withLock { + val frame = openFrameLocked(frameId) ?: return + addLineLocked(frame, now, message) + } + } + + fun spanStart(frameId: Int, name: String) { + val now = platform.elapsedRealtimeMs() + lock.withLock { + val frame = openFrameLocked(frameId) ?: return + frame.openSpans[name] = now + addLineLocked(frame, now, "span $name start") + } + } + + fun spanEnd(frameId: Int, name: String) { + val now = platform.elapsedRealtimeMs() + lock.withLock { + val frame = openFrameLocked(frameId) ?: return + val startedAt = frame.openSpans.remove(name) + if (startedAt == null) { + addLineLocked(frame, now, "span $name end (start not recorded)") + } else { + addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)") + } + } + } + + /** + * Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything + * starting with "discarded", or "timeout". First finish wins; later calls for + * the same frame are ignored, so racing completion paths are safe. + */ + fun finishFrame(frameId: Int, outcome: String?) { + val now = platform.elapsedRealtimeMs() + val finished = lock.withLock { + val frame = openFrameLocked(frameId) ?: return + finishFrameLocked(frame, now, outcome ?: "discarded: no outcome given") + frame + } + logInfo("frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome + + " in " + finished.durationMs() + "ms") + } + + /** One-line stats summary, e.g. for showing in the phone UI. */ + fun statsSummary(): String { + lock.withLock { + val input = percentilesLocked(inputLatencies) + return "frames started=" + framesStarted + " sent=" + framesSent + + " discarded=" + framesDiscarded + " timeout=" + framesTimedOut + + (if (input == null) + "" + else " | input-to-display p50=" + input[0] + "ms p90=" + input[1] + + "ms p99=" + input[2] + "ms max=" + input[3] + "ms") + } + } + + /** Finish every frame open longer than [FRAME_TIMEOUT_MS] as a timeout; returns how many. */ + fun sweepTimedOut(): Int { + val now = platform.elapsedRealtimeMs() + val timedOut: MutableList = ArrayList() + lock.withLock { + for (frame in ArrayList(framesById.values)) { + if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) { + finishFrameLocked(frame, now, "timeout: never reported finished") + timedOut.add(frame) + } + } + } + for (frame in timedOut) { + logWarn("frame#" + frame.id + " [" + frame.reason + "] timed out after " + + frame.durationMs() + "ms without being finished") + } + return timedOut.size + } + + /** True once a frame finished since the last [takeExport]/[buildExport] with clear. */ + val isExportDirty: Boolean + get() = lock.withLock { exportDirty } + + /** The full export text (does not clear the dirty flag). */ + fun buildExport(): String = lock.withLock { buildExportLocked() } + + /** The export text if anything changed since the last take, else null; clears the dirty flag. */ + fun takeExport(): String? = lock.withLock { + if (!exportDirty) return null + exportDirty = false + buildExportLocked() + } + + // --------------------------------------------------------------------- + + /** The frame with this ID if it exists and is still open, else null. */ + private fun openFrameLocked(frameId: Int): Frame? { + if (frameId <= 0) return null + val frame = framesById[frameId] + return if (frame != null && frame.isOpen()) frame else null + } + + private fun addLineLocked(frame: Frame, now: Long, message: String?) { + if (frame.lines.size >= MAX_LINES_PER_FRAME) { + if (frame.lines.size == MAX_LINES_PER_FRAME) { + frame.lines.add(Line(now, threadName(), "... line cap reached")) + } + return + } + frame.lines.add(Line(now, threadName(), message)) + } + + private fun finishFrameLocked(frame: Frame, now: Long, outcome: String) { + frame.finishedAtMs = now + frame.outcome = outcome + for (open in frame.openSpans.entries) { + frame.lines.add(Line(now, threadName(), + "span " + open.key + " never ended (started at +" + + (open.value - frame.startedAtMs) + "ms)")) + } + frame.openSpans.clear() + addLineLocked(frame, now, "finished: $outcome") + + val root = frame.root() + if (frame.wasSent()) { + framesSent++ + // The user-visible latency is input (or timer) to the first pixels + // that actually reached the glasses, wherever in the tree that was. + if (!root.subtreeSentRecorded) { + root.subtreeSentRecorded = true + recordLatencyLocked(root, now - root.startedAtMs) + insertSlowestLocked(root) + } + } else if (outcome.startsWith("timeout")) { + framesTimedOut++ + } else { + framesDiscarded++ + } + + if (root === frame) { + recentRoots.addLast(frame) + while (recentRoots.size > RECENT_ROOTS_KEPT) { + recentRoots.removeFirst() + } + } + exportDirty = true + } + + private fun recordLatencyLocked(root: Frame, latencyMs: Long) { + val bucket = if (root.reason.startsWith("input:")) inputLatencies else renderLatencies + bucket.addLast(latencyMs) + while (bucket.size > SENT_DURATIONS_WINDOW) { + bucket.removeFirst() + } + } + + private fun insertSlowestLocked(root: Frame) { + if (!slowestRoots.contains(root)) { + slowestRoots.add(root) + } + // Sorted and trimmed at export time, when every subtree duration is final. + } + + /** {p50, p90, p99, max} over a rolling latency window, or null if empty. */ + private fun percentilesLocked(window: ArrayDeque): LongArray? { + if (window.isEmpty()) return null + val sorted: MutableList = ArrayList(window) + sorted.sort() + return longArrayOf( + percentileOfSorted(sorted, 50), + percentileOfSorted(sorted, 90), + percentileOfSorted(sorted, 99), + sorted[sorted.size - 1], + ) + } + + private fun buildExportLocked(): String { + val now = platform.elapsedRealtimeMs() + val out = StringBuilder(64 * 1024) + out.append("FrameTimings export at ").append(formatWallClock(wallClockMs())).append('\n') + out.append(statsSummaryLocked()).append('\n') + out.append('\n') + out.append("Frames are trees: an input event's own frame is the root, and the renders it\n") + out.append("caused are indented under it with offsets measured from the root's start.\n") + out.append('\n') + + // Stable sort, descending by whole-tree duration (matches Collections.sort). + slowestRoots.sortWith(compareByDescending { it.subtreeEndAtMs(now) - it.startedAtMs }) + while (slowestRoots.size > SLOWEST_ROOTS_KEPT) { + slowestRoots.removeAt(slowestRoots.size - 1) + } + + out.append("=== slowest frames that reached the glasses ===\n") + for (frame in slowestRoots) { + appendTreeLocked(out, frame, frame, now) + } + + out.append("=== recent frames (oldest first) ===\n") + for (frame in recentRoots) { + appendTreeLocked(out, frame, frame, now) + } + return out.toString() + } + + private fun statsSummaryLocked(): String { + val input = percentilesLocked(inputLatencies) + val render = percentilesLocked(renderLatencies) + val out = StringBuilder() + out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent) + .append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut) + .append(" open=").append(openFrameCountLocked()) + appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size, input) + appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size, render) + return out.toString() + } + + private fun appendPercentilesLocked(out: StringBuilder, label: String, count: Int, percentiles: LongArray?) { + if (percentiles == null) return + out.append('\n').append(label).append(" (last ").append(count).append("): p50=") + .append(percentiles[0]).append("ms p90=").append(percentiles[1]) + .append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms") + } + + private fun openFrameCountLocked(): Int { + var open = 0 + for (frame in framesById.values) { + if (frame.isOpen()) open++ + } + return open + } + + /** Print a frame and its descendants, all timed against root's start. */ + private fun appendTreeLocked(out: StringBuilder, frame: Frame, root: Frame, now: Long) { + var depth = 0 + var walk: Frame? = frame + while (walk !== root) { + depth++ + walk = walk!!.parent + } + val indent = " ".repeat(depth) + val startOffsetMs = frame.startedAtMs - root.startedAtMs + out.append(indent).append("frame#").append(frame.id) + .append(" [").append(frame.reason).append(']') + if (frame === root) { + out.append(" started ").append(formatWallClock(frame.startedWallClockMs)) + val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs + out.append(" duration ").append(frame.durationMs()).append("ms") + if (frame.children.isNotEmpty()) { + out.append(" (tree ").append(totalMs).append("ms") + .append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')') + } + } else { + out.append(" started +").append(startOffsetMs).append("ms") + .append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms")) + } + out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n') + for (line in frame.lines) { + out.append(indent).append(" ").append(signedWidth7(line.atMs - root.startedAtMs)) + .append("ms [").append(line.thread).append("] ").append(line.message).append('\n') + } + for (child in frame.children) { + appendTreeLocked(out, child, root, now) + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Json.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Json.kt new file mode 100644 index 00000000..147aa751 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Json.kt @@ -0,0 +1,161 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic + +/** A JSON number kept as its source text so integers and decimals survive a round trip. */ +data class JsonNumber(val text: String) { + fun toLong(): Long = text.toLongOrNull() ?: text.toDouble().toLong() + + fun toInt(): Int = toLong().toInt() + + fun toDouble(): Double = text.toDouble() + + companion object { + @JvmStatic fun of(value: Long): JsonNumber = JsonNumber(value.toString()) + + @JvmStatic fun of(value: Int): JsonNumber = JsonNumber(value.toString()) + + @JvmStatic + fun of(value: Double): JsonNumber { + require(value.isFinite()) { "JSON numbers must be finite" } + val whole = value.toLong() + return JsonNumber(if (whole.toDouble() == value) whole.toString() else value.toString()) + } + } +} + +/** + * Minimal JSON codec shared by both platforms (org.json is JVM-only). Values are + * `Map` (insertion ordered), `List`, `String`, `Boolean`, `JsonNumber` + * and `null`. `parse` also accepts `//` and block comments plus trailing commas, which the + * settings documents rely on; `write` always emits strict JSON. + */ +object Json { + private val numberPattern = Regex("-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?") + + @JvmStatic fun parse(text: String): Any? = Parser(text).parse() + + @JvmStatic fun parseObject(text: String): Map = parse(text).asObject() ?: error("JSON object expected") + + @JvmStatic fun parseArray(text: String): List = parse(text).asArray() ?: error("JSON array expected") + + @JvmStatic + fun write(value: Any?, pretty: Boolean = false): String = write(value, pretty, 0) + + @JvmStatic + fun quote(text: String): String = buildString { + append('"') + for (c in text) when (c) { + '"' -> append("\\\"") + '\\' -> append("\\\\") + '\n' -> append("\\n") + '\r' -> append("\\r") + '\t' -> append("\\t") + else -> if (c < ' ') append("\\u" + c.code.toString(16).padStart(4, '0')) else append(c) + } + append('"') + } + + private fun write(value: Any?, pretty: Boolean, depth: Int): String { + fun collection(open: String, close: String, entries: List): String { + if (entries.isEmpty()) return open + close + return if (pretty) open + "\n" + entries.joinToString(",\n") { " ".repeat(depth + 1) + it } + "\n" + " ".repeat(depth) + close + else open + entries.joinToString(",") + close + } + return when (value) { + null -> "null" + is String -> quote(value) + is Boolean -> value.toString() + is JsonNumber -> value.text + is Int -> value.toString() + is Long -> value.toString() + is Float -> JsonNumber.of(value.toDouble()).text + is Double -> JsonNumber.of(value).text + is List<*> -> collection("[", "]", value.map { write(it, pretty, depth + 1) }) + is Array<*> -> collection("[", "]", value.map { write(it, pretty, depth + 1) }) + is Map<*, *> -> collection("{", "}", value.entries.map { quote(it.key as String) + (if (pretty) ": " else ":") + write(it.value, pretty, depth + 1) }) + else -> error("Unsupported JSON value: ${value::class.simpleName}") + } + } + + private class Parser(val text: String) { + var i = 0 + fun parse(): Any? { val result = value(0); space(); require(i == text.length) { "Trailing characters in JSON" }; return result } + fun space() { + while (i < text.length) { + if (text[i] in " \r\n\t") { i++; continue } + if (text.startsWith("//", i)) { while (i < text.length && text[i] != '\n' && text[i] != '\r') i++; continue } + if (text.startsWith("/*", i)) { val end = text.indexOf("*/", i + 2); require(end >= 0); i = end + 2; continue } + break + } + } + fun take(c: Char): Boolean { space(); return if (i < text.length && text[i] == c) { i++; true } else false } + fun value(depth: Int): Any? { + require(depth < 64); space(); require(i < text.length) { "Unexpected end of JSON" } + return when (text[i]) { + '"' -> string() + '{' -> { + i++; val result = linkedMapOf() + if (!take('}')) while (true) { + space(); val key = string(); require(!result.containsKey(key)) { "Duplicate key $key" }; require(take(':')) + result[key] = value(depth + 1) + if (take('}')) break + require(take(',')); if (take('}')) break + } + result + } + '[' -> { + i++; val result = mutableListOf() + if (!take(']')) while (true) { + result.add(value(depth + 1)); if (take(']')) break + require(take(',')); if (take(']')) break + } + result + } + 't' -> literal("true", true) + 'f' -> literal("false", false) + 'n' -> literal("null", null) + else -> { + val match = numberPattern.find(text, i) + require(match != null && match.range.first == i) { "Invalid JSON value at $i" } + require(match.value.toDouble().isFinite()); i += match.value.length; JsonNumber(match.value) + } + } + } + fun literal(word: String, result: Any?): Any? { require(text.startsWith(word, i)); i += word.length; return result } + fun string(): String { + require(i < text.length && text[i++] == '"') + return buildString { + while (true) { + require(i < text.length); val c = text[i++] + if (c == '"') break + require(c >= ' ') + if (c != '\\') { append(c); continue } + require(i < text.length) + append(when (val escaped = text[i++]) { + '"', '\\', '/' -> escaped + 'b' -> '\b'; 'f' -> '\u000c'; 'n' -> '\n'; 'r' -> '\r'; 't' -> '\t' + 'u' -> { require(i + 4 <= text.length); val hex = text.substring(i, i + 4); require(hex.all { it in "0123456789abcdefABCDEF" }); val code = hex.toInt(16); i += 4; code.toChar() } + else -> error("Invalid escape") + }) + } + } + } + } +} + +// Typed accessors for parsed values; each returns null when the value has another type. +@Suppress("UNCHECKED_CAST") +fun Any?.asObject(): Map? = this as? Map + +fun Any?.asArray(): List? = this as? List + +fun Any?.asString(): String? = this as? String + +fun Any?.asBoolean(): Boolean? = this as? Boolean + +fun Any?.asLong(): Long? = (this as? JsonNumber)?.toLong() ?: (this as? Long) ?: (this as? Int)?.toLong() + +fun Any?.asInt(): Int? = asLong()?.toInt() + +fun Any?.asDouble(): Double? = (this as? JsonNumber)?.toDouble() ?: (this as? Double) ?: (this as? Int)?.toDouble() ?: (this as? Long)?.toDouble() diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/PlatformUtil.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/PlatformUtil.kt new file mode 100644 index 00000000..a7813a56 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/PlatformUtil.kt @@ -0,0 +1,30 @@ +package com.faceclaw.app + +/** Incremental SHA-256; [digest] finishes the hash. */ +expect class Sha256Digest() { + fun update(bytes: ByteArray, offset: Int, length: Int) + + fun digest(): ByteArray +} + +fun Sha256Digest.update(bytes: ByteArray) = update(bytes, 0, bytes.size) + +fun sha256(bytes: ByteArray): ByteArray = Sha256Digest().also { it.update(bytes) }.digest() + +fun sha256Hex(bytes: ByteArray): String = BinaryEncoding.bytesToHex(sha256(bytes)) + +/** Wall-clock milliseconds since the Unix epoch (may jump; use ProtocolPlatform for intervals). */ +expect fun currentTimeMillis(): Long + +/** Formats [epochMs] in local time with a Unicode/SimpleDateFormat-style [pattern] (e.g. "yyyyMMdd-HHmmss-SSS"). */ +expect fun formatLocalTime(epochMs: Long, pattern: String): String + +expect object PlatformLog { + fun i(tag: String, message: String) + + fun w(tag: String, message: String) + + fun e(tag: String, message: String, error: Throwable?) +} + +fun PlatformLog.e(tag: String, message: String) = e(tag, message, null) diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt new file mode 100644 index 00000000..27f43702 --- /dev/null +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/PlatformUtil.ios.kt @@ -0,0 +1,105 @@ +@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class) + +package com.faceclaw.app + +import kotlinx.cinterop.addressOf +import kotlinx.cinterop.alloc +import kotlinx.cinterop.convert +import kotlinx.cinterop.free +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.nativeHeap +import kotlinx.cinterop.ptr +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.usePinned +import platform.CoreCrypto.CC_SHA256_CTX +import platform.CoreCrypto.CC_SHA256_DIGEST_LENGTH +import platform.CoreCrypto.CC_SHA256_Final +import platform.CoreCrypto.CC_SHA256_Init +import platform.CoreCrypto.CC_SHA256_Update +import platform.Foundation.NSDate +import platform.Foundation.NSDateFormatter +import platform.Foundation.NSLocale +import platform.Foundation.NSLog +import platform.Foundation.NSThread +import platform.Foundation.dateWithTimeIntervalSince1970 +import platform.Foundation.timeIntervalSince1970 +import platform.posix.F_OK +import platform.posix.access +import platform.posix.fclose +import platform.posix.fopen +import platform.posix.fwrite +import platform.posix.rename +import platform.posix.stat +import platform.posix.unlink +import platform.posix.usleep + +actual class Sha256Digest actual constructor() { + private var context = nativeHeap.alloc().also { CC_SHA256_Init(it.ptr) } + private var finished = false + + actual fun update(bytes: ByteArray, offset: Int, length: Int) { + check(!finished) { "digest already finished" } + if (length == 0) return + bytes.usePinned { CC_SHA256_Update(context.ptr, it.addressOf(offset), length.convert()) } + } + + actual fun digest(): ByteArray { + check(!finished) { "digest already finished" } + val out = ByteArray(CC_SHA256_DIGEST_LENGTH) + out.usePinned { CC_SHA256_Final(it.addressOf(0).reinterpret(), context.ptr) } + nativeHeap.free(context.ptr) + finished = true + return out + } +} + +actual fun currentTimeMillis(): Long = (NSDate().timeIntervalSince1970 * 1000.0).toLong() + +actual fun formatLocalTime(epochMs: Long, pattern: String): String { + val formatter = NSDateFormatter() + formatter.locale = NSLocale("en_US_POSIX") + formatter.dateFormat = pattern + return formatter.stringFromDate(NSDate.dateWithTimeIntervalSince1970(epochMs / 1000.0)) +} + +actual object PlatformLog { + actual fun i(tag: String, message: String) = NSLog("%@", "$tag: $message") + + actual fun w(tag: String, message: String) = NSLog("%@", "$tag: [warn] $message") + + actual fun e(tag: String, message: String, error: Throwable?) = + NSLog("%@", "$tag: [error] $message" + (error?.let { " (${it.message})" } ?: "")) +} + +actual fun startThread(name: String, daemon: Boolean, body: () -> Unit) { + val thread = NSThread(block = body) + thread.name = name + thread.start() +} + +actual fun sleepMs(ms: Long) { + if (ms > 0) usleep((ms * 1000).convert()) +} + +actual fun writeFileData(path: String, bytes: ByteArray) { + val file = fopen(path, "wb") ?: error("Cannot write $path") + try { + if (bytes.isNotEmpty()) + bytes.usePinned { + check(fwrite(it.addressOf(0), 1u, bytes.size.convert(), file) == bytes.size.convert()) { "Short write to $path" } + } + } finally { + fclose(file) + } +} + +actual fun fileExists(path: String): Boolean = access(path, F_OK) == 0 + +actual fun fileLength(path: String): Long = memScoped { + val info = alloc() + if (stat(path, info.ptr) != 0) -1L else info.st_size +} + +actual fun deleteFile(path: String): Boolean = unlink(path) == 0 + +actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0 diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/g2protocol/IosProtocolPlatform.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/g2protocol/IosProtocolPlatform.kt index 3b24fcca..4db40e9f 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/g2protocol/IosProtocolPlatform.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/g2protocol/IosProtocolPlatform.kt @@ -10,7 +10,10 @@ import kotlinx.cinterop.* import platform.CoreBluetooth.CBCharacteristicWriteType import platform.CoreBluetooth.CBCharacteristicWriteWithResponse import platform.CoreBluetooth.CBCharacteristicWriteWithoutResponse +import platform.Foundation.NSCondition +import platform.Foundation.NSDate import platform.Foundation.NSRecursiveLock +import platform.Foundation.dateWithTimeIntervalSinceNow import platform.posix.CLOCK_MONOTONIC_RAW import platform.posix.clock_gettime_nsec_np import platform.posix.memset @@ -32,6 +35,21 @@ object IosProtocolPlatform : ProtocolPlatform { override fun createDeflater(): ProtocolDeflater = IosDeflater() + override fun createCondition(): ProtocolCondition = + object : ProtocolCondition { + private val condition = NSCondition() + + override fun lock() = condition.lock() + + override fun unlock() = condition.unlock() + + override fun awaitMs(timeoutMs: Long) { + if (timeoutMs > 0) condition.waitUntilDate(NSDate.dateWithTimeIntervalSinceNow(timeoutMs / 1000.0)) + } + + override fun signalAll() = condition.broadcast() + } + fun writeType(mode: GattWriteMode): CBCharacteristicWriteType = when (mode) { GattWriteMode.WITH_RESPONSE -> CBCharacteristicWriteWithResponse diff --git a/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/FrameTimingsFormatTest.kt b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/FrameTimingsFormatTest.kt new file mode 100644 index 00000000..312d43bf --- /dev/null +++ b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/FrameTimingsFormatTest.kt @@ -0,0 +1,33 @@ +package com.faceclaw.app + +import java.util.Locale +import kotlin.test.Test +import kotlin.test.assertEquals + +/** The shared export must match what the Java-era String.format("%+7d") produced. */ +class FrameTimingsFormatTest { + private class FakeClock : ProtocolPlatform by AndroidProtocolPlatform { + var nowMs = 5_000L + override fun elapsedRealtimeMs(): Long = nowMs + } + + @Test + fun lineOffsetsMatchJavaFormat() { + val clock = FakeClock() + val core = FrameTimingsCore(clock, wallClockMs = { 0L }, threadName = { "worker" }, formatWallClock = { "w" }, logInfo = {}, logWarn = {}) + val offsets = listOf(-1_234_567L, -12L, -1L, 0L, 1L, 42L, 999L, 1_000L, 12_345L, 1_234_567L, 12_345_678L) + val root = core.startFrame("input:fmt") + for (offset in offsets) { + clock.nowMs = 5_000L + offset + core.log(root, "o=$offset") + } + clock.nowMs = 5_000L + core.finishFrame(root, "sent") + val export = core.buildExport() + for (offset in offsets) { + val javaLine = String.format(Locale.US, " %+7dms [%s] %s", offset, "worker", "o=$offset") + assert(export.contains("\n$javaLine\n")) { "missing <$javaLine> in\n$export" } + } + assertEquals(2, export.split("\n").count { it.contains("finished: sent") }) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AlarmRingingPolicyTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AlarmRingingPolicyTest.kt new file mode 100644 index 00000000..0dda856b --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AlarmRingingPolicyTest.kt @@ -0,0 +1,123 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +private class FakeScheduler : AlarmScheduler { + val pending = LinkedHashMap Unit>>() + override fun postDelayed(token: Any, delayMs: Long, action: () -> Unit) { pending[token] = delayMs to action } + override fun cancel(token: Any) { pending.remove(token) } + override fun cancelAll() { pending.clear() } + fun fire(token: Any) { val entry = pending.remove(token) ?: error("no timer $token"); entry.second() } +} + +private class FakeSink : RingingSink { + val events = ArrayList() + var sounding = false + override fun showNotification(item: RingingItem) { events.add("notify ${item.id} " + (if (item.silenced) "silenced" else if (item.escalated) "ringing" else "glasses")) } + override fun itemRemoved(item: RingingItem) { events.add("removed ${item.id}") } + override fun ringingStarted() { events.add("wake") } + override fun startSound() { if (!sounding) { sounding = true; events.add("sound on") } } + override fun stopSound() { if (sounding) { sounding = false; events.add("sound off") } } + override fun reschedule(item: RingingItem, atMs: Long, minutes: Int) { events.add("reschedule ${item.id} $atMs $minutes") } + override fun cancelSchedule(id: Long) { events.add("cancel $id") } + override fun recordPhoneAction(id: Long, action: String, minutes: Int) { events.add("action $id $action $minutes") } + override fun log(line: String) { events.add("log $line") } +} + +class AlarmRingingPolicyTest { + private var now = 5_000_000L + private val gate = GlassesStatusGate { now } + private val scheduler = FakeScheduler() + private val sink = FakeSink() + private val policy = AlarmRingingPolicy(gate, { now }, testPlatform()).also { it.attach(scheduler, sink) } + + private fun item(id: Long, kind: String = AlarmSchedule.KIND_ALARM) = RingingItem(id, "Wake", "", kind, 9, now) + + @Test + fun glassesUnavailableEscalatesImmediately() { + policy.ring(item(1)) + assertEquals(listOf("log ring alarm 1 (no glasses status)", "wake", "notify 1 glasses", "log phone sound for 1: no glasses status", "notify 1 ringing", "sound on"), sink.events) + assertTrue(policy.isRinging(1)) + assertEquals(setOf("silence:1"), scheduler.pending.keys) + scheduler.fire("silence:1") + assertEquals(listOf("log auto-silenced 1", "notify 1 silenced", "sound off"), sink.events.drop(6)) + } + + @Test + fun deliveryAndAcknowledgementTimersEscalate() { + gate.set(connected = true, worn = true, charging = false) + policy.ring(item(2)) + assertEquals(listOf("log ring alarm 2 (glasses worn)", "wake", "notify 2 glasses"), sink.events) + assertEquals(setOf("delivery:2"), scheduler.pending.keys) + policy.markDelivered(2) + assertEquals("log delivered 2 to glasses", sink.events.last()) + assertEquals(setOf("delivery:2", "ack:2"), scheduler.pending.keys) + scheduler.fire("delivery:2") // delivered in time: no escalation + assertFalse(policy.snapshot().single().escalated) + scheduler.fire("ack:2") + assertTrue(policy.snapshot().single().escalated) + assertTrue(sink.sounding) + // Acknowledged on the glasses afterwards: quiet stop, sound off. + policy.stopQuietly(2, "acknowledged") + assertEquals(listOf("log acknowledged 2", "removed 2", "sound off"), sink.events.takeLast(3)) + assertFalse(policy.isAnythingRinging()) + } + + @Test + fun undeliveredRingEscalatesAfterDeliveryWait() { + gate.set(connected = true, worn = true, charging = false) + policy.ring(item(3)) + scheduler.fire("delivery:3") + assertEquals("log phone sound for 3: not delivered to the glasses", sink.events[3]) + assertTrue(sink.sounding) + } + + @Test + fun snoozeAndDismissRecordActionsAndRearm() { + policy.ring(item(4, AlarmSchedule.KIND_TIMER)) + policy.snooze(4) + assertEquals(listOf("action 4 snooze 9", "removed 4", "sound off", "reschedule 4 ${now + 9 * 60_000L} 9"), sink.events.takeLast(4)) + policy.ring(item(5)) + policy.dismiss(5) + assertEquals(listOf("action 5 dismiss 0", "removed 5", "sound off", "cancel 5"), sink.events.takeLast(4)) + policy.dismiss(99) // unknown: no-op + assertEquals("cancel 5", sink.events.last()) + } + + @Test + fun earlySignalsAreConsumedOrExpire() { + gate.set(connected = true, worn = true, charging = false) + assertFalse(policy.deliveredOrDefer(6)) + policy.ring(item(6)) + assertEquals("log delivered 6 to glasses", sink.events.last()) + assertEquals(setOf("ack:6"), scheduler.pending.keys) + + assertFalse(policy.stopOrDefer(7)) + policy.ring(item(7)) + assertEquals("log ring 7 already acknowledged", sink.events.last()) + assertFalse(policy.isRinging(7)) + + assertFalse(policy.stopOrDefer(8)) + now += AlarmRingingPolicy.EARLY_SIGNAL_MAX_AGE_MS + 1 + policy.ring(item(8)) + assertTrue(policy.isRinging(8)) + assertTrue(policy.stopOrDefer(8)) + assertTrue(policy.deliveredOrDefer(8)) + } + + @Test + fun duplicateRingOnlyRefreshesNotificationAndDetachDropsTimers() { + policy.ring(item(9)) + val before = sink.events.size + policy.ring(item(9)) + assertEquals(listOf("notify 9 ringing"), sink.events.drop(before)) + assertEquals(1, policy.snapshot().size) + policy.detach() + assertTrue(scheduler.pending.isEmpty()) + policy.ring(item(10)) // no host: ignored + assertFalse(policy.isRinging(10)) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AlarmScheduleTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AlarmScheduleTest.kt new file mode 100644 index 00000000..d8cd493b --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AlarmScheduleTest.kt @@ -0,0 +1,109 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +private class MemoryStore : AlarmStore { + val values = HashMap() + override fun read(key: String): String? = values[key] + override fun write(key: String, value: String?) { values[key] = value } +} + +class AlarmScheduleTest { + private val store = MemoryStore() + private var now = 1_700_000_000_000L + private val schedule = AlarmSchedule(store, { now }, testPlatform()) + + @Test + fun persistsEntriesAsJsonAndReplacesById() { + schedule.put(AlarmEntry(7, now + 60_000, "Tea", "steep", AlarmSchedule.KIND_TIMER, 5)) + schedule.put(AlarmEntry(8, now + 120_000, null, null, null, 10)) + assertEquals("""[{"id":7,"at":${now + 60_000},"title":"Tea","text":"steep","kind":"timer","snoozeMinutes":5},{"id":8,"at":${now + 120_000},"title":"","text":"","kind":"alarm","snoozeMinutes":10}]""", store.values[AlarmSchedule.KEY_SCHEDULED]) + schedule.put(AlarmEntry(7, now + 90_000, "Tea", "steep", AlarmSchedule.KIND_TIMER, 5)) + assertEquals(listOf(8L, 7L), schedule.entries().map { it.id }) + assertEquals(now + 90_000, schedule.find(7)?.at) + schedule.remove(8) + assertNull(schedule.find(8)) + // A corrupt file reads as empty rather than failing. + store.values[AlarmSchedule.KEY_SCHEDULED] = "{nope" + assertTrue(schedule.entries().isEmpty()) + } + + @Test + fun replayRingsRecentDropsMissedArmsFuture() { + schedule.put(AlarmEntry(1, now - 60_000, "recent", "", null, 10)) + schedule.put(AlarmEntry(2, now - AlarmSchedule.LATE_GRACE_MS - 1, "old", "", AlarmSchedule.KIND_TIMER, 10)) + schedule.put(AlarmEntry(3, now + 60_000, "future", "", null, 10)) + val rang = ArrayList() + val armed = ArrayList() + schedule.replay("boot", { rang.add(it.id) }, { armed.add(it.id) }) + assertEquals(listOf(1L), rang) + assertEquals(listOf(3L), armed) + assertEquals(listOf(1L, 3L), schedule.entries().map { it.id }) + val log = schedule.readLog() + assertTrue(log.contains("$now missed timer 2 (boot)"), log) + assertTrue(log.endsWith("$now rescheduled 1 (boot)"), log) + } + + @Test + fun stalenessComparesSoonestPendingWithSystemAlarm() { + assertFalse(schedule.looksStale(null)) + schedule.put(AlarmEntry(1, now - 1, "past", "", null, 10)) + assertNull(schedule.soonestPendingAt()) + schedule.put(AlarmEntry(2, now + 50_000, "", "", null, 10)) + schedule.put(AlarmEntry(3, now + 20_000, "", "", null, 10)) + assertEquals(now + 20_000, schedule.soonestPendingAt()) + assertTrue(schedule.looksStale(null)) + assertTrue(schedule.looksStale(now + 30_000)) + assertFalse(schedule.looksStale(now + 20_500)) + assertFalse(schedule.looksStale(now + 5_000)) + } + + @Test + fun journalAppendsAndDrains() { + schedule.appendJournal(5, "snooze", 10) + now += 1000 + schedule.appendJournal(6, "dismiss", 0) + assertEquals("""[{"id":5,"action":"snooze","minutes":10,"at":${now - 1000}},{"id":6,"action":"dismiss","minutes":0,"at":$now}]""", schedule.drainJournal()) + assertEquals("[]", schedule.drainJournal()) + val log = schedule.readLog().lines() + assertEquals("${now - 1000} phone snooze 5 (10 min)", log[0]) + assertEquals("$now phone dismiss 6", log[1]) + } + + @Test + fun logKeepsOnlyTheLastLines() { + for (index in 0 until AlarmSchedule.LOG_LINES + 5) schedule.log("line $index") + val lines = schedule.readLog().lines() + assertEquals(AlarmSchedule.LOG_LINES, lines.size) + assertEquals("$now line 5", lines.first()) + assertEquals("$now line ${AlarmSchedule.LOG_LINES + 4}", lines.last()) + } + + @Test + fun glassesGateRequiresFreshWornConnectedNotCharging() { + var clock = 10_000L + val gate = GlassesStatusGate { clock } + assertFalse(gate.canCarryAlarm()) + assertEquals("no glasses status", gate.description()) + gate.set(connected = true, worn = true, charging = false) + assertTrue(gate.canCarryAlarm()) + assertEquals("glasses worn", gate.description()) + gate.set(connected = true, worn = false, charging = false) + assertEquals("glasses not worn", gate.description()) + gate.set(connected = true, worn = true, charging = true) + assertEquals("glasses charging", gate.description()) + gate.set(connected = false, worn = true, charging = false) + assertEquals("glasses not connected", gate.description()) + gate.set(connected = true, worn = true, charging = false) + clock += GlassesStatusGate.MAX_AGE_MS + 1 + assertFalse(gate.canCarryAlarm()) + assertEquals("glasses status stale", gate.description()) + assertEquals(0x7fffffff and (0x1_0000_0002L xor 1L).toInt(), AlarmSchedule.requestCode(0x1_0000_0002L)) + assertNotNull(AlarmEntry.fromJson(Json.parse("""{"id":1,"at":2}"""))) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AudioHelpersTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AudioHelpersTest.kt new file mode 100644 index 00000000..8c4d7a0a --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/AudioHelpersTest.kt @@ -0,0 +1,130 @@ +package com.faceclaw.app + +import kotlin.math.abs +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class AudioHelpersTest { + @Test + fun cutPointLandsInQuietestWindowForFloatsAndShorts() { + // 1000 loud samples with a silent gap at 700..760; search the last 500 with a 40 window. + val floats = FloatArray(1000) { if (it in 700 until 760) 0f else 0.5f } + val cut = AudioSegmentation.quietestCutPoint(floats, floats.size, 500, 40) + assertTrue(cut in 700..760, "cut=$cut") + val shorts = ShortArray(1100) { if (it - 100 in 700 until 760) 0 else 16000 } + assertEquals(cut, AudioSegmentation.quietestCutPoint(shorts, 100, 1000, 500, 40)) + // Too short to search: returns the length unchanged. + assertEquals(30, AudioSegmentation.quietestCutPoint(FloatArray(30) { 0.1f }, 30, 500, 40)) + } + + @Test + fun normalizePeakScalesUpButNeverDown() { + val quiet = floatArrayOf(0.01f, -0.02f, 0.005f) + AudioSegmentation.normalizePeak(quiet, 0.9f, 30f) + assertTrue(abs(quiet[1] + 0.6f) < 1e-6f, "capped at maxGain 30: ${quiet[1]}") + val loud = floatArrayOf(0.95f, -0.5f) + AudioSegmentation.normalizePeak(loud, 0.9f, 30f) + assertEquals(0.95f, loud[0]) + val silent = FloatArray(4) + AudioSegmentation.normalizePeak(silent, 0.9f, 30f) + assertEquals(0f, AudioSegmentation.peakAmplitude(silent)) + } + + @Test + fun pcmConversionsRoundTrip() { + val samples = shortArrayOf(0, 1, -1, 32767, -32768, 256) + val le = AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size) + assertEquals(listOf(0, 0, 1, 0, 0xff, 0xff, 0xff, 0x7f, 0, 0x80, 0, 1), le.map { it.toInt() and 0xff }) + val floats = AudioSegmentation.pcm16leToFloat(le) + assertEquals(AudioSegmentation.pcm16ToFloat(samples, 0, samples.size).toList(), floats.toList()) + assertEquals(32767 / 32768.0f, floats[3]) + assertEquals(-1f, floats[4]) + assertEquals(2.0, AudioSegmentation.rms(shortArrayOf(2, -2, 2, -2), 0, 4)) + } + + @Test + fun voicePrintMathAndBlobCodec() { + val v = floatArrayOf(3f, 4f) + VoicePrint.l2Normalize(v) + assertEquals(listOf(0.6f, 0.8f), v.toList()) + assertTrue(abs(VoicePrint.dot(v, v) - 1.0) < 1e-6) + assertNull(VoicePrint.l2Normalize(null)) + assertTrue(VoicePrint.isLongEnoughToEmbed(16000, 16000)) + assertTrue(!VoicePrint.isLongEnoughToEmbed(15999, 16000)) + + val blob = SpeakerEmbeddings.floatsToBlob(floatArrayOf(1.5f, -2f)) + assertEquals(listOf(0, 0, 0xc0, 0x3f, 0, 0, 0, 0xc0), blob.map { it.toInt() and 0xff }) + assertEquals(listOf(1.5f, -2f), SpeakerEmbeddings.blobToFloats(blob)!!.toList()) + assertNull(SpeakerEmbeddings.blobToFloats(byteArrayOf(1, 2, 3))) + assertNull(SpeakerEmbeddings.blobToFloats(null)) + } + + @Test + fun centroidUpdatesAndBlends() { + val first = SpeakerEmbeddings.runningMean(null, 0, floatArrayOf(1f, 0f), 5) + assertEquals(1, first.count) + assertEquals(listOf(1f, 0f), first.embedding.toList()) + val second = SpeakerEmbeddings.runningMean(first.embedding, first.count, floatArrayOf(0f, 1f), 5) + assertEquals(2, second.count) + assertTrue(abs(second.embedding[0] - second.embedding[1]) < 1e-6f) + assertTrue(abs(VoicePrint.dot(second.embedding, second.embedding) - 1.0) < 1e-6) + // The cap limits the existing weight: count 100 capped at 1 behaves like an equal blend. + val capped = SpeakerEmbeddings.runningMean(floatArrayOf(1f, 0f), 100, floatArrayOf(0f, 1f), 1) + assertTrue(abs(capped.embedding[0] - capped.embedding[1]) < 1e-6f) + assertEquals(101, capped.count) + // Mismatched sizes restart the centroid. + assertEquals(1, SpeakerEmbeddings.runningMean(floatArrayOf(1f), 4, floatArrayOf(0f, 1f), 5).count) + + val blended = SpeakerEmbeddings.blend(floatArrayOf(1f, 0f), 1, floatArrayOf(0f, 1f), 3) + assertNotNull(blended) + assertTrue(blended[1] > blended[0]) + assertEquals(listOf(0f, 1f), SpeakerEmbeddings.blend(null, 0, floatArrayOf(0f, 1f), 3)!!.toList()) + assertEquals(listOf(1f, 0f), SpeakerEmbeddings.blend(floatArrayOf(1f, 0f), 2, null, 0)!!.toList()) + assertNull(SpeakerEmbeddings.blend(null, 0, null, 0)) + assertTrue(SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights = false, sourceHasInsights = true)) + assertTrue(!SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights = true, sourceHasInsights = true)) + } + + private fun wav(dataSize: Int, extraChunk: Boolean = false, channels: Int = 1, bits: Int = 16, format: Int = 1): ByteArray { + val out = ByteSink() + fun u16(v: Int) { out.write(v); out.write(v ushr 8) } + fun u32(v: Int) { u16(v); u16(v ushr 16) } + out.write("RIFF".encodeToByteArray()); u32(0); out.write("WAVE".encodeToByteArray()) + if (extraChunk) { out.write("LIST".encodeToByteArray()); u32(4); u32(0) } + out.write("fmt ".encodeToByteArray()); u32(18) + u16(format); u16(channels); u32(16000); u32(16000 * channels * 2); u16(channels * 2); u16(bits); u16(0) + out.write("data".encodeToByteArray()); u32(dataSize) + return out.toByteArray() + } + + @Test + fun wavHeaderParsesChunksAndRecoversZeroSize() { + val header = wav(4000) + val info = WavPcmReader.parseHeader(header, header.size + 4000L) + assertEquals(16000, info.sampleRate) + assertEquals(1, info.channels) + assertEquals(16, info.bitsPerSample) + assertEquals(header.size.toLong(), info.dataOffset) + assertEquals(4000L, info.dataBytes) + + val withList = wav(100, extraChunk = true) + assertEquals(withList.size.toLong(), WavPcmReader.parseHeader(withList, withList.size + 100L).dataOffset) + + val unpatched = wav(0) + assertEquals(1234L, WavPcmReader.parseHeader(unpatched, unpatched.size + 1234L).dataBytes) + + assertFailsWith { WavPcmReader.parseHeader("RIFX".encodeToByteArray() + ByteArray(20), 24) } + assertFailsWith { WavPcmReader.parseHeader(wav(10, bits = 8), 100) } + assertFailsWith { WavPcmReader.parseHeader(wav(10, format = 3), 100) } + assertFailsWith { WavPcmReader.parseHeader(wav(10).copyOf(20), 100) } + // The shared header matches the shared writer. + val written = BinaryEncoding.wavHeader(640, 16000, 1) + val parsed = WavPcmReader.parseHeader(written, 44 + 640L) + assertEquals(44L, parsed.dataOffset) + assertEquals(640L, parsed.dataBytes) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/CaptionEngineCoreTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/CaptionEngineCoreTest.kt new file mode 100644 index 00000000..486d4130 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/CaptionEngineCoreTest.kt @@ -0,0 +1,127 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class CaptionEngineCoreTest { + private class Sink : CaptionSegmentSink { + val starts = mutableListOf() + val utterances = mutableListOf>() + override fun onSpeechStart(startMs: Long) { starts.add(startMs) } + override fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double) { + utterances.add(listOf(length, startMs, endMs, samples[0].toInt())) + } + } + + private fun chunk(amplitude: Int, samples: Int = 800): ShortArray = + ShortArray(samples) { if (it % 2 == 0) amplitude.toShort() else (-amplitude).toShort() } + + @Test + fun segmenterUsesPreRollOnsetReleaseAndSilence() { + val sink = Sink() + val segmenter = CaptionSegmenter(sink) + repeat(10) { segmenter.processChunk(chunk(100), 800) } // 500 ms idle, noise floor ~100 -> threshold 220 + repeat(10) { segmenter.processChunk(chunk(3000), 800) } // 500 ms speech + assertEquals(listOf(100L), sink.starts) // 500 ms - 400 ms pre-roll + repeat(15) { segmenter.processChunk(chunk(100), 800) } // 750 ms < 800 ms silence: still open + assertTrue(sink.utterances.isEmpty()) + segmenter.processChunk(chunk(100), 800) // 800 ms of silence closes it + assertEquals(1, sink.utterances.size) + val (length, startMs, endMs, first) = sink.utterances[0] + assertEquals(100L, startMs) + assertEquals(6400 + 10 * 800 + 16 * 800, length) // pre-roll + speech + trailing silence + assertEquals(startMs as Long + (length as Int) * 1000L / 16000, endMs) + assertEquals(100, first) // the pre-roll audio leads the utterance + assertTrue(segmenter.isInUtterance.not()) + } + + @Test + fun segmenterDropsTooShortUtterancesAndCapsLength() { + val sink = Sink() + val segmenter = CaptionSegmenter(sink) + segmenter.silenceMs = 100 // clamps to 200 + segmenter.processChunk(chunk(10), 800) // seeds the noise floor; becomes the 50 ms pre-roll + segmenter.processChunk(chunk(3000), 800) // onset + repeat(4) { segmenter.processChunk(chunk(0), 800) } // 200 ms silence + assertEquals(1, sink.starts.size) + assertTrue(sink.utterances.isEmpty()) // 300 ms < 350 ms minimum + repeat(400) { segmenter.processChunk(chunk(3000), 800) } // 20 s of speech: capped at 15 s + assertEquals(1, sink.utterances.size) + assertEquals(16000 * 15, sink.utterances[0][0]) + segmenter.flush() + assertEquals(2, sink.utterances.size) // the remainder is flushed on stop + } + + @Test + fun utteranceDecoderSplitsLongAudioAndEmbedsTenSeconds() { + val calls = mutableListOf() + val transcriber = object : OfflineTranscriber { + override fun recognize(samples: FloatArray, count: Int): String { calls.add(count); return if (calls.size == 2) ", b" else "a" } + override fun release() {} + } + var embedded = 0 + val embedder = object : SpeakerEmbedder { + override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? { embedded = pcm16le.size; return floatArrayOf(1f) } + override fun release() {} + } + val decoder = UtteranceDecoder(transcriber, embedder) + val utterance = ShortArray(16000 * 12) { if (it % 2 == 0) 2000 else -2000 } + assertEquals("a, b", decoder.recognize(utterance, utterance.size)) + assertEquals(2, calls.size) + assertTrue(calls[0] <= UtteranceDecoder.DECODE_SEGMENT_MAX_SAMPLES) + assertEquals(utterance.size, calls.sum()) + assertNotNull(decoder.embed(utterance, utterance.size)) + assertEquals(16000 * 10 * 2, embedded) + assertEquals("", UtteranceDecoder(null, null).recognize(utterance, utterance.size)) + assertNull(UtteranceDecoder(null, null).embed(utterance, utterance.size)) + } + + @Test + fun captionSessionRunsQueueThroughSegmenterAndDecoder() { + val startLatch = Latch(1, testPlatform()) + val utteranceLatch = Latch(1, testPlatform()) + val events = mutableListOf() + val listener = object : FaceclawCaptionEngineListener { + override fun onUtterance(text: String?, embedding: FloatArray?, startMs: Long, endMs: Long, peakRms: Double) { + events.add("utterance:$text:${embedding?.size}:$startMs"); utteranceLatch.countDown() + } + override fun onSpeechStart(startMs: Long) { events.add("start:$startMs"); startLatch.countDown() } + override fun onStatus(status: String?) { events.add("status:$status") } + } + val host = object : CaptionHost { + override val dispatcher = CallbackDispatcher { it() } + override fun loadTranscriber(modelDir: String): OfflineTranscriber? = if (modelDir == "models") object : OfflineTranscriber { + override fun recognize(samples: FloatArray, count: Int): String = "hi" + override fun release() {} + } else null + override fun loadEmbedder(modelPath: String): SpeakerEmbedder? = object : SpeakerEmbedder { + override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = floatArrayOf(0.5f, 0.5f) + override fun release() {} + } + } + val session = CaptionSession(host, testPlatform(), "caption-test") { m, e -> error("$m: $e") } + session.setListener(listener) + session.setAsrModelDir("models") + session.setSpeakerModelPath("speaker") + session.setSilenceMs(300) + session.acceptPcm(ByteArray(1600)) // ignored before start + session.start() + val loud = AudioSegmentation.pcm16ToLittleEndian(chunk(3000), 0, 800) + val quiet = AudioSegmentation.pcm16ToLittleEndian(chunk(0), 0, 800) + val idle = AudioSegmentation.pcm16ToLittleEndian(chunk(10), 0, 800) + repeat(2) { session.acceptPcm(idle) } // seeds the noise floor and fills 100 ms of pre-roll + repeat(10) { session.acceptPcm(loud) } + assertTrue(startLatch.await(5000)) + repeat(8) { session.acceptPcm(quiet) } + assertTrue(utteranceLatch.await(5000)) + session.stop() + session.stop() // idempotent + assertEquals("status:Loading caption model...", events[0]) + assertEquals("status:Captions listening...", events[1]) + assertEquals("start:0", events[2]) + assertEquals("utterance:hi:2:0", events[3]) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/EvenHubAssetServerTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/EvenHubAssetServerTest.kt new file mode 100644 index 00000000..b064efb2 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/EvenHubAssetServerTest.kt @@ -0,0 +1,47 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class EvenHubAssetServerTest { + @Test + fun resolvesPathsWithIndexDefaultAndEscapeChecks() { + assertEquals("index.html", EvenHubAssetServer.resolve(null)!!.relativePath) + assertEquals("index.html", EvenHubAssetServer.resolve("")!!.relativePath) + assertEquals("index.html", EvenHubAssetServer.resolve("/")!!.relativePath) + assertEquals("assets/app.js", EvenHubAssetServer.resolve("/assets/app.js")!!.relativePath) + assertEquals("assets/app.js", EvenHubAssetServer.resolve("/assets/./x/../app.js")!!.relativePath) + assertNull(EvenHubAssetServer.resolve("/../secret")) + assertNull(EvenHubAssetServer.resolve("/a/../../secret")) + assertNull(EvenHubAssetServer.resolve("/..")) + assertTrue(EvenHubAssetServer.resolve("/index.html")!!.isHtml) + assertTrue(EvenHubAssetServer.isInsideRoot("/apps/x", "/apps/x/index.html")) + assertTrue(EvenHubAssetServer.isInsideRoot("/apps/x/", "/apps/x/a/b")) + assertEquals(false, EvenHubAssetServer.isInsideRoot("/apps/x", "/apps/xy/index.html")) + assertEquals(false, EvenHubAssetServer.isInsideRoot("/apps/x", "/apps/index.html")) + } + + @Test + fun mimeTableCoversBothPlatformsTables() { + assertEquals("text/html", EvenHubAssetServer.mimeTypeFor("Index.HTM")) + assertEquals("text/javascript", EvenHubAssetServer.mimeTypeFor("bundle.mjs")) + assertEquals("application/wasm", EvenHubAssetServer.mimeTypeFor("core.wasm")) + assertEquals("font/woff2", EvenHubAssetServer.mimeTypeFor("a.woff2")) + assertEquals("image/svg+xml", EvenHubAssetServer.mimeTypeFor("icon.svg")) + assertEquals("text/plain", EvenHubAssetServer.mimeTypeFor("bundle.js.map")) + assertEquals("application/octet-stream", EvenHubAssetServer.mimeTypeFor("blob")) + assertEquals("application/octet-stream", EvenHubAssetServer.mimeTypeFor("weird.xyz")) + } + + @Test + fun injectsShimAfterHeadOrAtStart() { + assertEquals( + "t", + EvenHubAssetServer.injectIntoHtml("t", "S"), + ) + assertEquals("x", EvenHubAssetServer.injectIntoHtml("x", "S")) + assertEquals(", tamper: (ByteArray) -> Unit = {}): ByteArray { + val count = components.size + val toc = FirmwareImage.HEADER_SIZE + count * FirmwareImage.TOC_ENTRY_SIZE + var offset = toc + val offsets = components.map { c -> offset.also { offset += FirmwareImage.SUBHEADER_SIZE + c.payload.size } } + val img = ByteArray(offset) + putU32(img, 8, count.toLong()) + components.forEachIndexed { i, c -> + val off = offsets[i] + putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 4, off.toLong()) + putU32(img, off + 8, c.payload.size.toLong()) + c.name.encodeToByteArray().copyInto(img, off + FirmwareImage.NAME_OFFSET) + c.payload.copyInto(img, off + FirmwareImage.SUBHEADER_SIZE) + } + tamper(img) + components.forEachIndexed { i, c -> + val off = offsets[i] + val crc = FirmwareImage.crc32cUnsigned(img, off + FirmwareImage.SUBHEADER_SIZE, c.payload.size) + putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 12, crc) + putU32(img, off + 12, crc) + } + return img + } + + private fun mainApp(size: Int): ByteArray { + val payload = ByteArray(size) { (it * 7).toByte() } + putU32(payload, 0, size.toLong()) + putU32(payload, 0x14, FirmwareImage.APP_LOAD_ADDR) + return payload + } + + private fun validComponents(): List = + listOf( + Component("ota/bootloader.bin", ByteArray(300) { it.toByte() }), + Component("ota/a.bin", ByteArray(64) { 1 }), + Component("ota/b.bin", ByteArray(128) { 2 }), + Component("ota/c.bin", ByteArray(32) { 3 }), + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(600)), + ) + + @Test + fun crc32cMatchesKnownVector() { + // Castagnoli polynomial, MSB-first, init 0, no reflection or final xor (g2flash.py + // crc32c_msb); vectors from an independent Python implementation of that definition. + assertEquals(0xc052a8c8L, FirmwareImage.crc32cUnsigned("123456789".encodeToByteArray())) + assertEquals(0xd7b91914L, FirmwareImage.crc32cUnsigned(ByteArray(256) { it.toByte() })) + assertEquals(0L, FirmwareImage.crc32cUnsigned(ByteArray(0))) + assertEquals(FirmwareImage.crc32cUnsigned("23456789".encodeToByteArray()), FirmwareImage.crc32cUnsigned("123456789".encodeToByteArray(), 1, 8)) + } + + @Test + fun validImageParsesAllComponents() { + val segs = FirmwareImage.validate(container(validComponents())) + assertEquals(5, segs.size) + assertEquals(listOf(300, 64, 128, 32, 600), segs.map { it.ps }) + assertEquals(FirmwareImage.REQUIRED_SEGMENT, segs[4].name) + assertTrue(segs.zipWithNext().all { (a, b) -> a.end <= b.off }) + } + + @Test + fun rejectsStaleCrc() { + val img = container(validComponents()) + img[img.size - 1] = (img[img.size - 1] + 1).toByte() + assertTrue(assertFailsWith { FirmwareImage.validate(img) }.message!!.contains("CRC32C")) + } + + @Test + fun rejectsOverlappingComponents() { + val img = container(validComponents()) { buf -> + // Point component 1's TOC entry at component 0, so both claim the same bytes. + val off0 = FirmwareImage.readU32(buf, FirmwareImage.HEADER_SIZE + 4) + putU32(buf, FirmwareImage.HEADER_SIZE + 16 + 4, off0) + } + assertTrue(assertFailsWith { FirmwareImage.parseSegments(img) }.message!!.contains("overlap")) + } + + @Test + fun rejectsTruncatedPayload() { + val img = container(validComponents()) + val truncated = img.copyOf(img.size - 10) + assertTrue(assertFailsWith { FirmwareImage.parseSegments(truncated) }.message!!.contains("truncated")) + assertFailsWith { FirmwareImage.parseSegments(ByteArray(0x10)) } + } + + @Test + fun rejectsWrongComponentCountAndMissingMain() { + val four = validComponents().dropLast(1) + assertTrue(assertFailsWith { FirmwareImage.validate(container(four)) }.message!!.contains("components")) + val noMain = validComponents().dropLast(1) + Component("ota/other.bin", ByteArray(40)) + assertTrue(assertFailsWith { FirmwareImage.validate(container(noMain)) }.message!!.contains("not found")) + } + + @Test + fun rejectsMainAppBeyondMramCeiling() { + val ceiling = (FirmwareImage.APP_MAX_END - FirmwareImage.APP_LOAD_ADDR + FirmwareImage.APP_PREAMBLE).toInt() + val fits = validComponents().dropLast(1) + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(ceiling)) + FirmwareImage.validate(container(fits)) + val tooBig = validComponents().dropLast(1) + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(ceiling + 1)) + assertTrue(assertFailsWith { FirmwareImage.validate(container(tooBig)) }.message!!.contains("MRAM")) + val badAddr = validComponents().dropLast(1) + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(600).also { putU32(it, 0x14, 0x1234L) }) + assertTrue(assertFailsWith { FirmwareImage.validate(container(badAddr)) }.message!!.contains("load address")) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FoundationTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FoundationTest.kt new file mode 100644 index 00000000..621153c0 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FoundationTest.kt @@ -0,0 +1,69 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class FoundationTest { + @Test + fun jsonRoundTripsObjectsArraysAndNumbers() { + val text = """{"a":1,"b":[true,null,"x\ny"],"c":{"d":2.5,"e":-3}}""" + val value = Json.parseObject(text) + assertEquals(1, value["a"].asInt()) + assertEquals(listOf(true, null, "x\ny"), value["b"].asArray()) + assertEquals(2.5, value["c"].asObject()!!["d"].asDouble()) + assertEquals(-3L, value["c"].asObject()!!["e"].asLong()) + assertEquals(text, Json.write(value)) + assertEquals("[1,2,\"q\\\"\"]", Json.write(listOf(1, 2L, "q\""))) + assertEquals("1", JsonNumber.of(1.0).text) + assertEquals("1.5", JsonNumber.of(1.5).text) + assertNull("x".asInt()) + } + + @Test + fun sha256MatchesKnownVector() { + assertEquals("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", sha256Hex("abc".encodeToByteArray())) + val incremental = Sha256Digest() + incremental.update("a".encodeToByteArray()) + incremental.update("bc".encodeToByteArray()) + assertEquals(sha256("abc".encodeToByteArray()).toList(), incremental.digest().toList()) + } + + @Test + fun latchAndQueueHandOffAcrossThreads() { + val platform = testPlatform() + val latch = Latch(1, platform) + val queue = BlockingQueue(platform) + assertFalse(latch.await(20)) + startThread("foundation-test", true) { + sleepMs(20) + queue.put("hello") + latch.countDown() + } + assertEquals("hello", queue.poll(2000)) + assertTrue(latch.await(2000)) + assertNull(queue.poll(10)) + } + + @Test + fun interruptibleSleepConsumesEarlyInterrupt() { + val platform = testPlatform() + val sleep = InterruptibleSleep(platform) + sleep.interrupt() + val start = platform.elapsedRealtimeMs() + assertFalse(sleep.sleep(500)) + assertTrue(platform.elapsedRealtimeMs() - start < 400) + assertTrue(sleep.sleep(5)) + startThread("foundation-wake", true) { sleepMs(30); sleep.interrupt() } + assertFalse(sleep.sleep(5000)) + } + + @Test + fun wallClockFormatsPattern() { + val text = formatLocalTime(0L, "yyyy") + assertTrue(text == "1970" || text == "1969", text) + assertTrue(currentTimeMillis() > 1_600_000_000_000L) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameTimingsCoreTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameTimingsCoreTest.kt new file mode 100644 index 00000000..ee9dc077 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameTimingsCoreTest.kt @@ -0,0 +1,139 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class FrameTimingsCoreTest { + private class FakeClock : ProtocolPlatform by testPlatform() { + var nowMs = 1_000L + override fun elapsedRealtimeMs(): Long = nowMs + } + + private fun core(clock: FakeClock, logs: MutableList = ArrayList()) = FrameTimingsCore( + platform = clock, + wallClockMs = { 1_700_000_000_000L + clock.nowMs }, + threadName = { "t" }, + formatWallClock = { "wall@$it" }, + logInfo = { logs.add("I $it") }, + logWarn = { logs.add("W $it") }, + ) + + @Test + fun exportRendersTreeAgainstRootClockByteForByte() { + val clock = FakeClock() + val logs = ArrayList() + val core = core(clock, logs) + val root = core.startFrame("input:sys-event type=0") + clock.nowMs += 5 + core.annotate(root, "fg=terminal") + core.log(root, "input event received on JS side") + clock.nowMs += 1 + core.spanStart(root, "handle-input") + clock.nowMs += 10 + val child = core.startFrame("render:shell", root) + clock.nowMs += 4 + core.spanEnd(root, "handle-input") + core.spanEnd(root, "missing") + core.spanStart(child, "paint") + clock.nowMs += 30 + core.finishFrame(child, "discarded: superseded") + clock.nowMs += 50 + core.finishFrame(root, "sent") + core.finishFrame(root, "sent") // ignored: first finish wins + core.log(root, "late line is dropped") + clock.nowMs += 100 + + assertEquals(listOf( + "I frame#2 [render:shell] -> discarded: superseded in 34ms", + "I frame#1 [input:sys-event type=0 fg=terminal] -> sent in 100ms", + ), logs) + assertEquals("frames started=2 sent=1 discarded=1 timeout=0 | input-to-display p50=100ms p90=100ms p99=100ms max=100ms", core.statsSummary()) + + val expected = """ + |FrameTimings export at wall@1700000001200 + |frames started=2 sent=1 discarded=1 timeout=0 open=0 + |input-to-display latency (last 1): p50=100ms p90=100ms p99=100ms max=100ms + | + |Frames are trees: an input event's own frame is the root, and the renders it + |caused are indented under it with offsets measured from the root's start. + | + |=== slowest frames that reached the glasses === + |frame#1 [input:sys-event type=0 fg=terminal] started wall@1700000001000 duration 100ms (tree 100ms) outcome sent + | +5ms [t] input event received on JS side + | +6ms [t] span handle-input start + | +16ms [t] spawned frame#2 (render:shell) + | +20ms [t] span handle-input end (14ms) + | +20ms [t] span missing end (start not recorded) + | +100ms [t] finished: sent + | frame#2 [render:shell] started +16ms duration 34ms outcome discarded: superseded + | +20ms [t] span paint start + | +50ms [t] span paint never ended (started at +4ms) + | +50ms [t] finished: discarded: superseded + |=== recent frames (oldest first) === + |frame#1 [input:sys-event type=0 fg=terminal] started wall@1700000001000 duration 100ms (tree 100ms) outcome sent + | +5ms [t] input event received on JS side + | +6ms [t] span handle-input start + | +16ms [t] spawned frame#2 (render:shell) + | +20ms [t] span handle-input end (14ms) + | +20ms [t] span missing end (start not recorded) + | +100ms [t] finished: sent + | frame#2 [render:shell] started +16ms duration 34ms outcome discarded: superseded + | +20ms [t] span paint start + | +50ms [t] span paint never ended (started at +4ms) + | +50ms [t] finished: discarded: superseded + |""".trimMargin() + assertTrue(core.isExportDirty) + assertEquals(expected, core.takeExport()) + assertFalse(core.isExportDirty) + assertEquals(null, core.takeExport()) + assertEquals(expected, core.buildExport()) + } + + @Test + fun sweepFinishesStaleFramesAsTimeoutsAndIgnoresZeroAndUnknownIds() { + val clock = FakeClock() + val logs = ArrayList() + val core = core(clock, logs) + val stale = core.startFrame("render:app") + clock.nowMs += FrameTimingsCore.FRAME_TIMEOUT_MS - 1 + val fresh = core.startFrame("render:late") + assertEquals(0, core.sweepTimedOut()) + clock.nowMs += 1 + assertEquals(1, core.sweepTimedOut()) + assertEquals(listOf("W frame#$stale [render:app] timed out after 30000ms without being finished"), logs) + core.log(0, "ignored") + core.log(999, "ignored") + core.spanStart(0, "x") + core.finishFrame(0, "sent") + core.finishFrame(fresh, "sent") + assertEquals("frames started=2 sent=1 discarded=0 timeout=1", core.statsSummary()) + val export = core.buildExport() + assertTrue(export.contains("frame#$stale [render:app] started wall@")) + assertTrue(export.contains("outcome timeout: never reported finished\n")) + assertTrue(export.contains("render-to-display latency (last 1): p50=1ms p90=1ms p99=1ms max=1ms")) + assertTrue(export.contains("open=0")) + } + + @Test + fun lineCapAndNegativeOffsetsFormatLikeJava() { + val clock = FakeClock() + val core = core(clock) + val root = core.startFrame("input:x") + clock.nowMs -= 12 // a line stamped before the root's start still formats with a sign + core.log(root, "early") + clock.nowMs += 12 + repeat(250) { core.log(root, "line $it") } + core.finishFrame(root, "sent") + val export = core.buildExport() + assertTrue(export.contains("\n -12ms [t] early\n"), export) + // Printed once under "slowest" and once under "recent". + assertEquals(2, Regex("\\.\\.\\. line cap reached").findAll(export).count()) + assertTrue(export.contains("line 198")) + assertFalse(export.contains("line 199")) + // Like the Java original, lines past the cap (including the finish line) are dropped. + assertFalse(export.contains("finished: sent")) + assertTrue(export.contains("outcome sent\n")) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt new file mode 100644 index 00000000..63b1a0fd --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlassesSessionCoreTest.kt @@ -0,0 +1,421 @@ +package com.faceclaw.app + +import kotlin.concurrent.Volatile +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +private const val RIGHT = "AA:00:00:00:00:01" +private const val LEFT = "AA:00:00:00:00:02" + +/** The test platform with a clock the tests can jump forward (the real waits keep real time). */ +private class ClockPlatform(private val base: ProtocolPlatform) : ProtocolPlatform by base { + @Volatile var offsetMs = 0L + + override fun elapsedRealtimeMs(): Long = base.elapsedRealtimeMs() + offsetMs +} + +private fun waitUntil(timeoutMs: Long, condition: () -> Boolean): Boolean { + val platform = testPlatform() + val deadline = platform.elapsedRealtimeMs() + timeoutMs + while (platform.elapsedRealtimeMs() < deadline) { + if (condition()) return true + sleepMs(10) + } + return condition() +} + +private fun ByteArray.containsSequence(needle: ByteArray): Boolean { + if (needle.isEmpty() || size < needle.size) return false + outer@ for (start in 0..size - needle.size) { + for (i in needle.indices) if (this[start + i] != needle[i]) continue@outer + return true + } + return false +} + +/** One logical write as the fake link saw it: the frames, the decoded pb, and the in-flight message (if any). */ +private class Write( + val address: String, + val sid: Int, + val frames: List, + val pb: ByteArray, + val kind: String?, + val label: String?, + val magic: Int, + val message: ByteArray?, +) + +/** + * A GATT link that records every write and, unless a kind is muted, answers it the way the + * firmware would: EvenHub/settings/auth requests get a matching ack frame, CFW stream + * messages get one ACK per lens. + */ +private class FakeLink(private val platform: ProtocolPlatform) : SessionLink { + lateinit var core: GlassesSessionCore + private val lock = platform.createLock() + private val writesList = ArrayList() + val muted = HashSet() + val connected = HashSet() + val disconnected = ArrayList() + @Volatile var closed = false + var seq = 0 + + val writes: List + get() = lock.withLock { writesList.toList() } + + /** Decoded CFW image messages (the drive loop's debug-flags control rides the same stream and is excluded). */ + fun cfwMessages(): List = writes.filter { it.sid == CfwTransport.SID && it.kind == "image" }.mapNotNull { it.message } + + override fun connect(address: String, timeoutMs: Int): Boolean { + connected.add(address) + return true + } + + override fun requestHighPriority(address: String) {} + + override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean = true + + override fun negotiatedMtu(address: String): Int = 247 + + override fun discoverServices(address: String, timeoutMs: Int): Boolean = true + + override fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean = true + + override fun writeFrames(address: String, characteristicUuid: String, frames: List, mode: GattWriteMode, timeoutMs: Int): Boolean { + val sid = frames[0][6].toInt() and 255 + val magic = if (sid == CfwTransport.SID) frames[0][2].toInt() and 255 else BleProtocol.parseFrame(frames[0]).msgSeq + // The in-flight entry gives the label/kind and the decoded CFW message. + val inFlight = core.monitor.withLock { core.inFlightMessages.firstOrNull { it.sid == sid && it.magic == magic && magic != 0 } } + val pb = if (sid == CfwTransport.SID) ByteArray(0) else { + val joined = frames.flatMap { frame -> frame.slice(8 until 8 + (frame[3].toInt() and 255)) }.toByteArray() + joined.copyOf(maxOf(0, joined.size - 2)) + } + val write = Write(address, sid, frames, pb, inFlight?.kind, inFlight?.label, magic, inFlight?.message?.copyOf()) + lock.withLock { writesList.add(write) } + if (inFlight != null && inFlight.kind !in muted) { + if (sid == CfwTransport.SID) { + for (lens in 1..2) core.onNotification(address, BleProtocol.NOTIFY_CHAR_UUID, cfwAck(magic, lens, inFlight.message.size, inFlight.cfwChecksum)) + } else { + core.onNotification(address, BleProtocol.NOTIFY_CHAR_UUID, ackFrame(sid, magic, pb)) + } + } + return true + } + + private fun ackFrame(sid: Int, magic: Int, request: ByteArray): ByteArray { + val body = if (sid == BleProtocol.SID_SECURITY_AUTH) + BleProtocol.encodeVarintField(1, 4) + BleProtocol.encodeVarintField(2, magic) + byteArrayOf(0x1a, 0x00) + else + BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, magic) + return BleProtocol.framePb(body, sid, 0x00, seq++)[0] + } + + override fun disconnect(address: String) { + disconnected.add(address) + connected.remove(address) + } + + override fun close() { + closed = true + } + + override fun isBonded(address: String): Boolean = true + + override fun prepareBenchmarkLink(address: String, mode: Int) {} + + override fun recordDisplayFrameSent() {} +} + +private fun cfwAck(streamId: Int, lens: Int, size: Int, checksum: Int): ByteArray { + val packet = ByteArray(19) + packet[0] = 0xaa.toByte() + packet[1] = 0x12 + packet[3] = 11 + packet[4] = 1 + packet[5] = 1 + packet[6] = CfwTransport.SID.toByte() + packet[8] = 1 + packet[9] = streamId.toByte() + packet[12] = lens.toByte() + packet[13] = size.toByte() + packet[14] = (size ushr 8).toByte() + packet[15] = checksum.toByte() + packet[16] = (checksum ushr 8).toByte() + val crc = CfwTransport.crc(packet.copyOfRange(8, 17)) + packet[17] = crc.toByte() + packet[18] = (crc ushr 8).toByte() + return packet +} + +private class FakeHost(private val platform: ProtocolPlatform) : SessionHost { + private val lock = platform.createLock() + private val lines = ArrayList() + private var done: Latch? = null + var wakeLock = false + + fun logs(): List = lock.withLock { lines.toList() } + + fun hasLog(fragment: String): Boolean = logs().any { it.contains(fragment) } + + override fun postToMain(action: () -> Unit) = action() + + override fun currentThreadDispatcher(): SessionDispatcher = SessionDispatcher { it() } + + override fun isPhoneLocked(): Boolean = false + + override fun setScreenWakeLock(on: Boolean) { + wakeLock = on + } + + override fun isEvenAppActive(): Boolean = false + + override fun startWorker(body: () -> Unit) { + val latch = Latch(1, platform) + done = latch + startThread("test-session", true) { + try { + body() + } finally { + latch.countDown() + } + } + } + + override fun joinWorker(timeoutMs: Long) { + done?.await(timeoutMs) + } + + override fun log(level: SessionLogLevel, tag: String, message: String, error: Throwable?) { + lock.withLock { lines.add("$level $tag: $message") } + } +} + +private class FakeListener : FaceclawBleCommunicatorListener { + val phases = ArrayList() + val events = ArrayList() + val finished = ArrayList() + val batteries = ArrayList() + + override fun onStateChange(phase: String?, status: String?) { + phases.add(phase ?: "") + } + + override fun onRingEvent(kind: String?, containerName: String?, eventType: Int, eventSource: Int, systemExitReasonCode: Int, frameId: Int, ringTick: Long, ringType: Int, ringAux: Int, ringSpeed: Int) { + events.add("$kind:$eventType") + } + + override fun onBatteryState(headsetBattery: Int, headsetCharging: Int, ringBattery: Int, ringCharging: Int) { + batteries.add("$headsetBattery/$headsetCharging/$ringBattery/$ringCharging") + } + + override fun onSilentMode(silent: Boolean) {} + + override fun onWearState(wearing: Boolean) {} + + override fun onPhoneLockState(locked: Boolean) {} + + override fun onEvenAppConflict(message: String?) {} + + override fun onFrameMetrics(paintMs: Int, transmitMs: Int, tileCount: Int) {} + + override fun onFrameFinished(frameId: Int, outcome: String?) { + finished.add("$frameId:$outcome") + } + + override fun onFirmwareInfo(leftVersion: String?, rightVersion: String?, extension: String?) {} +} + +private class Session { + val platform = ClockPlatform(testPlatform()) + val link = FakeLink(platform) + val host = FakeHost(platform) + val listener = FakeListener() + val frameTimings = FrameTimingsCore(platform) + val core = GlassesSessionCore(link, host, frameTimings, platform, RIGHT, LEFT, "") + + init { + link.core = core + core.setListener(listener) + } + + fun layoutCreated(): Boolean = core.monitor.withLock { core.fixedLayoutCreated } + + fun startAndAwaitLayout() { + assertTrue(core.start()) + assertTrue(waitUntil(5_000) { layoutCreated() }, "layout never created; logs=${host.logs().takeLast(20)}") + } + + /** A 64x32 surface frame with a marker byte at [marker]; returns the expected composite fingerprint. */ + fun submitFrame(marker: Int, value: Byte, frameId: Int): Unit { + val pixels = ByteArray(64 * 32) + pixels[marker] = value + core.submitSurfaceFrame(ArrayByteReader(pixels), "s", 0, 0, 64, 32, "content-$marker-$value", 0, frameId, null) + } + + fun displayedMatchesDesired(): Boolean = core.monitor.withLock { + core.desiredTilesLock.locked { core.desiredFingerprint.isNotEmpty() && core.desiredFingerprint == core.displayedFingerprint } + } +} + +class GlassesSessionCoreTest { + @Test + fun connectsAuthenticatesPreludesAndCreatesTheLayout() { + val s = Session() + s.startAndAwaitLayout() + val writes = s.link.writes + assertEquals(setOf(RIGHT, LEFT), writes.filter { it.sid == BleProtocol.SID_SECURITY_AUTH }.map { it.address }.toSet()) + assertTrue(s.host.hasLog("security auth R=ok L=ok"), s.host.logs().toString()) + assertEquals(1, writes.count { it.kind == "prelude" }) + // The framebuffer lease is acquired on both arms (fire-and-forget settings writes, magic 0). + val fbAcquire = byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_FB_OP_ACQUIRE.toByte()) + assertEquals(setOf(RIGHT, LEFT), writes.filter { it.sid == BleProtocol.SID_UI_SETTING && it.pb.containsSequence(fbAcquire) }.map { it.address }.toSet()) + assertEquals(1, writes.count { it.kind == "battery" }, "settings query for firmware info") + assertEquals(1, writes.count { it.kind == "create-layout" }) + assertTrue(s.core.isSessionReady()) + assertEquals(listOf("disconnected", "connecting", "connected"), s.listener.phases.distinct()) + s.core.disconnect() + assertFalse(s.core.isSessionReady()) + assertEquals("disconnected", s.listener.phases.last()) + assertTrue(s.listener.phases.contains("disconnecting")) + assertTrue(s.link.closed) + assertEquals(listOf(RIGHT, LEFT), s.link.disconnected.takeLast(2)) + assertFalse(s.host.wakeLock) + assertTrue(s.host.hasLog("communicator stop")) + } + + @Test + fun imagePipelineSendsAKeyframeThenABoundingBoxDelta() { + val s = Session() + s.startAndAwaitLayout() + s.core.configureCompositorScreen(64, 32) + s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE) + s.submitFrame(100, 0x80.toByte(), 1) + assertTrue(waitUntil(5_000) { s.link.cfwMessages().size >= 1 && s.displayedMatchesDesired() }, s.host.logs().takeLast(20).toString()) + assertTrue(s.listener.finished.contains("1:sent"), s.listener.finished.toString()) + s.submitFrame(101, 0x80.toByte(), 2) + assertTrue(waitUntil(5_000) { s.link.cfwMessages().size >= 2 && s.displayedMatchesDesired() }, s.host.logs().takeLast(20).toString()) + val messages = s.link.cfwMessages() + assertEquals(CFW_MSG_FULL_FRAME, messages[0][0].toInt() and CFW_MSG_TYPE_MASK, "first frame is a keyframe") + assertEquals(CFW_MSG_BOUNDING_BOX, messages[1][0].toInt() and CFW_MSG_TYPE_MASK, "second frame is a delta") + assertTrue(s.listener.finished.contains("2:sent")) + // Identical content again: nothing is sent and the frame is finished as a no-change. + s.submitFrame(101, 0x80.toByte(), 3) + assertTrue(waitUntil(2_000) { s.listener.finished.any { it.startsWith("3:") } }) + assertTrue(s.listener.finished.any { it == "3:discarded: no change from displayed image" }, s.listener.finished.toString()) + assertEquals(2, s.link.cfwMessages().size) + s.core.disconnect() + } + + @Test + fun heartbeatAndLeaseRenewalFollowTheClock() { + val s = Session() + s.startAndAwaitLayout() + // The first heartbeat follows the layout at once (nothing has been acked yet); the + // next one waits for HEARTBEAT_READY_MS since that ack. + assertTrue(waitUntil(3_000) { s.link.writes.count { it.kind == "heartbeat" } == 1 }, s.host.logs().takeLast(10).toString()) + assertTrue(s.host.hasLog("Got ACK for heartbeat")) + sleepMs(300) + assertEquals(1, s.link.writes.count { it.kind == "heartbeat" }) + s.platform.offsetMs += ConnectionOptions.HEARTBEAT_READY_MS + 100L + s.core.interruptibleSleep.interrupt() + assertTrue(waitUntil(3_000) { s.link.writes.count { it.kind == "heartbeat" } == 2 }, s.host.logs().takeLast(10).toString()) + val leaseWritesBefore = s.link.writes.count { it.sid == BleProtocol.SID_UI_SETTING && it.pb.containsSequence(byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_FB_OP_ACQUIRE.toByte())) } + s.platform.offsetMs += GlassesSessionCore.FACECLAW_WAKE_LEASE_RENEW_MS + 100L + s.core.interruptibleSleep.interrupt() + assertTrue(waitUntil(3_000) { + s.link.writes.count { it.sid == BleProtocol.SID_UI_SETTING && it.pb.containsSequence(byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_FB_OP_ACQUIRE.toByte())) } == leaseWritesBefore + 2 + }, "framebuffer lease renewed on both arms") + s.core.disconnect() + } + + @Test + fun deferredWakeIsClaimedAndReadyIsSentAfterResume() { + val s = Session() + s.startAndAwaitLayout() + s.core.configureCompositorScreen(64, 32) + s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE) + s.submitFrame(7, 0x40, 1) + assertTrue(waitUntil(5_000) { s.displayedMatchesDesired() }) + + assertTrue(s.core.suspendEvenHubSession()) + assertTrue(s.core.monitor.withLock { s.core.shutdownRequested }) + assertEquals(1, s.link.writes.count { it.kind == "shutdown" }) + + // CFW deferred double-tap wake on the right arm: field 102 [F,C,ver,event=1,nonce LE16]. + val nonce = 0x1234 + val event = byteArrayOf(70, 67, 1, 1, (nonce and 0xff).toByte(), (nonce ushr 8).toByte()) + val pb = BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, 0) + BleProtocol.encodeBytesField(BleProtocol.FACECLAW_WAKE_EVENT_FIELD, event) + s.core.onNotification(RIGHT, BleProtocol.NOTIFY_CHAR_UUID, BleProtocol.framePb(pb, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_NOTIFY, 9)[0]) + assertTrue(s.listener.events.contains("display-wake:${BleProtocol.EVENT_DOUBLE_CLICK}"), s.listener.events.toString()) + val claim = byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_WAKE_OP_CLAIM.toByte(), (nonce and 0xff).toByte(), (nonce ushr 8).toByte()) + assertTrue(waitUntil(3_000) { s.link.writes.count { it.pb.containsSequence(claim) } == 2 }, "CLAIM written to both arms") + assertEquals(RIGHT, s.link.writes.first { it.pb.containsSequence(claim) }.address, "right arm first") + + assertTrue(s.core.resumeEvenHubSession()) + assertEquals(2, s.link.writes.count { it.kind == "prelude" }) + assertTrue(waitUntil(5_000) { s.layoutCreated() && s.displayedMatchesDesired() }, s.host.logs().takeLast(20).toString()) + assertTrue(s.core.awaitEvenHubSessionReady(5_000)) + val ready = byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_WAKE_OP_READY.toByte(), (nonce and 0xff).toByte(), (nonce ushr 8).toByte()) + assertEquals(2, s.link.writes.count { it.pb.containsSequence(ready) }, "READY written to both arms") + s.core.disconnect() + } + + @Test + fun unexpectedAndDuplicateAcksAreLoggedNotApplied() { + val s = Session() + s.startAndAwaitLayout() + val unknown = BleProtocol.framePb(BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, 150), BleProtocol.SID_EVENHUB, 0, 5)[0] + s.core.onNotification(RIGHT, BleProtocol.NOTIFY_CHAR_UUID, unknown) + assertTrue(s.host.hasLog("unexpected ACK sid=${BleProtocol.SID_EVENHUB} magic=150"), s.host.logs().takeLast(5).toString()) + val layout = s.link.writes.first { it.kind == "create-layout" } + val duplicate = BleProtocol.framePb(BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, layout.magic), layout.sid, 0, 6)[0] + s.core.onNotification(RIGHT, BleProtocol.NOTIFY_CHAR_UUID, duplicate) + assertTrue(s.host.hasLog("duplicate ACK for already-acked message sid=${layout.sid} magic=${layout.magic}"), s.host.logs().takeLast(5).toString()) + assertTrue(s.layoutCreated()) + s.core.disconnect() + } + + @Test + fun ackTimeoutOnTheLayoutReconnects() { + val s = Session() + s.link.muted.add("create-layout") + assertTrue(s.core.start()) + assertTrue(waitUntil(5_000) { s.link.writes.any { it.kind == "create-layout" } }) + assertFalse(s.layoutCreated()) + s.platform.offsetMs += ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 1_000L + s.core.interruptibleSleep.interrupt() + assertTrue(waitUntil(3_000) { s.listener.phases.contains("retrying") }, s.host.logs().takeLast(10).toString()) + assertTrue(s.host.hasLog("message timed out: create-layout")) + assertTrue(s.host.hasLog("Transport failure: ack timeout")) + assertTrue(s.link.disconnected.containsAll(listOf(RIGHT, LEFT))) + s.core.disconnect() + assertEquals("disconnected", s.listener.phases.last()) + } + + @Test + fun cfwNackTriggersReplayWithAFreshStreamId() { + val s = Session() + s.link.muted.add("image") + s.startAndAwaitLayout() + s.core.configureCompositorScreen(64, 32) + s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE) + s.submitFrame(3, 0x20, 1) + assertTrue(waitUntil(5_000) { s.link.cfwMessages().size == 1 }) + val first = s.link.writes.first { it.sid == CfwTransport.SID && it.kind == "image" } + val message = assertNotNull(s.core.monitor.withLock { s.core.inFlightMessages.firstOrNull { it.magic == first.magic } }) + // A NACK from lens 1 marks the message for replay; the window re-sends it under a new id. + val nack = cfwAck(first.magic, 1, 0, 0).also { it[8] = 3; val crc = CfwTransport.crc(it.copyOfRange(8, 17)); it[17] = crc.toByte(); it[18] = (crc ushr 8).toByte() } + s.core.onNotification(LEFT, BleProtocol.NOTIFY_CHAR_UUID, nack) + assertTrue(waitUntil(3_000) { s.link.cfwMessages().size == 2 }, s.host.logs().takeLast(10).toString()) + assertTrue(s.host.hasLog("CFW replay id=${first.magic}")) + val second = s.link.writes.filter { it.sid == CfwTransport.SID && it.kind == "image" }[1] + assertTrue(second.magic != first.magic) + assertEquals(1, message.cfwRetries) + assertContentEquals(first.message, second.message) + s.core.disconnect() + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlyphAndIconSourceTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlyphAndIconSourceTest.kt new file mode 100644 index 00000000..46b27dd9 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GlyphAndIconSourceTest.kt @@ -0,0 +1,128 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class GlyphAndIconSourceTest { + private fun u16(bytes: ByteArray, at: Int) = (bytes[at].toInt() and 0xff) or ((bytes[at + 1].toInt() and 0xff) shl 8) + private fun s16(bytes: ByteArray, at: Int) = u16(bytes, at).toShort().toInt() + + // 6x4 raster: ink in columns 2..4, rows 1..2, one 128 edge pixel. + private val raster = byteArrayOf( + 0, 0, 0, 0, 0, 0, + 0, 0, -1, -1, 0, 0, + 0, 0, 0, -1, 0x7f, 0, + 0, 0, 0, 0, 0, 0, + ) + + @Test + fun gammaLutIsIdentityAtOneAndDarkensAboveOne() { + val linear = GlyphPacket.gammaLut(1.0) + for (i in 0 until 256) assertEquals(i, linear[i].toInt() and 0xff) + assertContentEquals(linear, GlyphPacket.gammaLut(0.0)) + val squared = GlyphPacket.gammaLut(2.0) + assertEquals(64, squared[128].toInt() and 0xff) + assertEquals(255, squared[255].toInt() and 0xff) + } + + @Test + fun inkScansFindPaintedBoxes() { + assertContentEquals(intArrayOf(2, 4), GlyphPacket.paintedColumnRange(raster, 6, 4)) + assertContentEquals(intArrayOf(2, 1, 4, 2), GlyphPacket.inkBounds(raster, 6, 4)) + assertNull(GlyphPacket.paintedColumnRange(ByteArray(12), 4, 3)) + assertNull(GlyphPacket.inkBounds(ByteArray(12), 4, 3)) + } + + @Test + fun imagePacketsCarryLittleEndianHeaderAndGammaMappedRegion() { + val whole = GlyphPacket.packImage(raster, 6, 4, 1.0) + assertEquals(4 + 24, whole.size) + assertEquals(6, u16(whole, 0)) + assertEquals(4, u16(whole, 2)) + assertContentEquals(raster, whole.copyOfRange(4, whole.size)) + val region = GlyphPacket.packImageRegion(raster, 6, 2, 1, 3, 2, 2.0) + assertEquals(3, u16(region, 0)) + assertEquals(2, u16(region, 2)) + assertContentEquals(byteArrayOf(-1, -1, 0, 0, -1, 63), region.copyOfRange(4, region.size)) + } + + @Test + fun glyphCellPacketsCropToInkAndEncodeSignedOffsets() { + val cell = GlyphPacket.packGlyphCell(GlyphPacket.advanceFixed(5.5), raster, 6, 4, penX = 3, baselineY = 3, ascent = 10, gamma = 1.0) + assertEquals(352, u16(cell, 0)) + assertEquals(-1, s16(cell, 2)) // ink starts one pixel left of the pen + assertEquals(8, s16(cell, 4)) // ascent 10, ink top two rows above the baseline + assertEquals(3, u16(cell, 6)) + assertEquals(2, u16(cell, 8)) + assertContentEquals(byteArrayOf(-1, -1, 0, 0, -1, 0x7f), cell.copyOfRange(10, cell.size)) + val empty = GlyphPacket.packGlyphCell(64, ByteArray(12), 4, 3, 0, 0, 10, 1.0) + assertContentEquals(byteArrayOf(64, 0, 0, 0, 0, 0, 0, 0, 0, 0), empty) + assertEquals(0xffff, GlyphPacket.advanceFixed(5000.0)) + } + + private fun nameTable(vararg records: Triple): ByteArray { + // records: (platformId, nameId, value); platform 1 stored Latin-1, others UTF-16BE + val strings = ArrayList() + for ((platform, _, value) in records) { + strings.add(if (platform == 1) ByteArray(value.length) { value[it].code.toByte() } + else ByteArray(value.length * 2) { if (it % 2 == 0) (value[it / 2].code shr 8).toByte() else value[it / 2].code.toByte() }) + } + val out = ByteSink() + fun u16(v: Int) { out.write(v ushr 8); out.write(v) } + fun u32(v: Int) { u16(v ushr 16); u16(v and 0xffff) } + val nameOffset = 12 + 16 + val nameLength = 6 + records.size * 12 + strings.sumOf { it.size } + u32(0x00010000); u16(1); u16(0); u16(0); u16(0) // offset table, 1 table + u32(0x6e616d65); u32(0); u32(nameOffset); u32(nameLength) // 'name' record + u16(0); u16(records.size); u16(6 + records.size * 12) + var offset = 0 + for ((i, record) in records.withIndex()) { + u16(record.first); u16(0); u16(0); u16(record.second); u16(strings[i].size); u16(offset) + offset += strings[i].size + } + for (s in strings) out.write(s) + return out.toByteArray() + } + + @Test + fun openTypeNamesPreferTypographicFamilyAndStyle() { + assertEquals("Roboto\nLight", OpenTypeNames.parse(nameTable(Triple(3, 1, "Roboto Light"), Triple(3, 2, "Regular"), Triple(3, 16, "Roboto"), Triple(1, 17, "Light")))) + assertEquals("Mono\n", OpenTypeNames.parse(nameTable(Triple(1, 1, " Mono ")))) + assertEquals("", OpenTypeNames.parse(nameTable(Triple(3, 2, "Bold")))) + assertEquals("", OpenTypeNames.parse(ByteArray(8))) + } + + @Test + fun svgIconsParseShapesStrokeRuleAndPlacement() { + val svg = """ + """ + val icon = SvgIconSource.parse(svg)!! + assertTrue(icon.stroked) + assertEquals(listOf("Rect", "Circle", "PathData", "Polyline", "Line"), icon.shapes.map { it::class.simpleName }) + val rect = icon.shapes[0] as SvgShape.Rect + assertEquals(2f, rect.rx); assertEquals(2f, rect.ry) + assertEquals(3f, (icon.shapes[1] as SvgShape.Circle).r) + assertEquals("M4 4h16", (icon.shapes[2] as SvgShape.PathData).d) + val polygon = icon.shapes[3] as SvgShape.Polyline + assertContentEquals(floatArrayOf(1f, 2f, 3f, 4f), polygon.points) + assertTrue(polygon.closed) + assertContentEquals(floatArrayOf(0f, 0f, 2f), icon.placement(48)) + val filled = SvgIconSource.parse("""""")!! + assertTrue(!filled.stroked) + assertContentEquals(floatArrayOf(0f, 5f, 0.5f), filled.placement(20)) + assertEquals(-2f, filled.minX) + assertNull(SvgIconSource.parse("""""")) + } + + @Test + fun coverageQuantizesToThreeLevels() { + assertEquals(0, SvgIconSource.quantizeCoverage(63)) + assertEquals(128, SvgIconSource.quantizeCoverage(64)) + assertEquals(128, SvgIconSource.quantizeCoverage(191)) + assertEquals(255, SvgIconSource.quantizeCoverage(192)) + assertContentEquals(byteArrayOf(0, -128, -1), SvgIconSource.quantizeArgb(intArrayOf(0x10ffffff, 0x80000000.toInt(), 0xff000000.toInt()))) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt new file mode 100644 index 00000000..1fc78bbd --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/GraphicsCoresTest.kt @@ -0,0 +1,99 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertSame +import kotlin.test.assertTrue + +class GraphicsCoresTest { + private fun checksum(bytes: ByteArray): Long { + var sum = 7L + for (b in bytes) sum = sum * 31 + (b.toInt() and 0xff) + return sum + } + + @Test + fun grayPacketHeaderAndFlatConversion() { + // 3x2: opaque white, opaque mid, transparent white, half-alpha red, opaque black, opaque green + val pixels = intArrayOf( + 0xffffffff.toInt(), 0xff808080.toInt(), 0x00ffffff, + 0x80ff0000.toInt(), 0xff000000.toInt(), 0xff00ff00.toInt(), + ) + val packet = GrayPacket.fromArgb(pixels, 3, 2, 1f, false) + assertEquals(4 + 6, packet.size) + assertContentEquals(byteArrayOf(3, 0, 2, 0), packet.copyOfRange(0, 4)) + assertEquals(255, packet[4].toInt() and 0xff) + assertEquals(128, packet[5].toInt() and 0xff) + assertEquals(0, packet[6].toInt() and 0xff) // alpha 0 darkens to black + assertEquals(76 * 128 / 255, packet[7].toInt() and 0xff) // luma 76 at half alpha + assertEquals(0, packet[8].toInt() and 0xff) + assertEquals(149, packet[9].toInt() and 0xff) + val wide = GrayPacket.fromArgb(IntArray(300 * 2), 300, 2, 1f, false) + assertContentEquals(byteArrayOf(0x2c, 1, 2, 0), wide.copyOfRange(0, 4)) + } + + @Test + fun toneCurveDarkensMidtones() { + val identity = GrayPacket.toneCurve(1f) + assertEquals((0..255).toList(), identity.toList()) + val curve = GrayPacket.toneCurve(2.2f) + assertEquals(0, curve[0]) + assertEquals(255, curve[255]) + assertEquals(56, curve[128]) + assertTrue(curve.toList() == curve.sorted()) + } + + @Test + fun ditherIsDeterministicAndQuantized() { + // A horizontal ramp over 16x4 so the diffusion has somewhere to go. + val pixels = IntArray(16 * 4) { i -> val v = (i % 16) * 17; 0xff000000.toInt() or (v shl 16) or (v shl 8) or v } + val a = GrayPacket.fromArgb(pixels, 16, 4, 1f, true) + val b = GrayPacket.fromArgb(pixels, 16, 4, 1f, true) + assertContentEquals(a, b) + for (i in 4 until a.size) { + val v = a[i].toInt() and 0xff + assertTrue(v == 1 || (v % 16 == 0 && v in 16..240), "byte $i = $v") + } + // Mid-gray flat area dithers to a mix of two adjacent levels, never a solid. + val flat = GrayPacket.fromArgb(IntArray(8 * 8) { 0xff7f7f7f.toInt() }, 8, 8, 1f, true) + val levels = flat.drop(4).map { it.toInt() and 0xff }.toSet() + assertEquals(setOf(112, 128), levels) + assertEquals(2001783152304756847L, checksum(a)) + } + + @Test + fun previewPaletteValues() { + assertEquals(0xff000000.toInt(), PreviewPalette.argb(0, 0.7, false)) + assertEquals(0xffffffff.toInt(), PreviewPalette.argb(255, 0.7, false)) + assertEquals(0xff00ff00.toInt(), PreviewPalette.argb(255, 0.7, true)) + val gray = PreviewPalette.argb(128, 0.7, false) + val v = gray and 0xff + assertEquals(157, v) + assertEquals(0xff000000.toInt() or (v shl 16) or (v shl 8) or v, gray) + assertEquals(0xff000000.toInt() or (v shl 8), PreviewPalette.argb(128, 0.7, true)) + assertEquals(0xff808080.toInt(), PreviewPalette.argb(128, 1.0, false)) + val lut = PreviewPalette.lookupTable(0.7, false) + assertSame(lut, PreviewPalette.lookupTable(0.7, false)) + assertEquals(256, lut.size) + val expanded = PreviewPalette.expand(byteArrayOf(0, 0x80.toByte(), 0xff.toByte()), 0, 3, 0.7, false) + assertContentEquals(intArrayOf(lut[0], lut[128], lut[255]), expanded) + } + + @Test + fun statusIconsAreStable() { + val wifi = StatusIconArt.wifi(4) + val cell = StatusIconArt.cell(2) + val hotspot = StatusIconArt.hotspot() + for (icon in listOf(wifi, cell, hotspot)) { + assertEquals(24 * 24, icon.size) + assertTrue(icon.any { it.toInt() != 0 }) + } + assertTrue(StatusIconArt.wifi(0).count { it.toInt() != 0 } < wifi.count { it.toInt() != 0 }) + assertEquals(StatusIconArt.wifi(2).toList(), StatusIconArt.wifi(2).toList()) + assertEquals(listOf(-7515428168691714401L, -7643893777543336767L, 7675840471157348143L), listOf(checksum(wifi), checksum(cell), checksum(hotspot))) + val scaled = StatusIconArt.scale(wifi, 12) + assertEquals(144, scaled.size) + assertSame(wifi, StatusIconArt.scale(wifi, 24)) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/Lc3PacketFramerTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/Lc3PacketFramerTest.kt new file mode 100644 index 00000000..274d54b4 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/Lc3PacketFramerTest.kt @@ -0,0 +1,67 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class Lc3PacketFramerTest { + private class FakeCodec(var fail: Boolean = false) : Lc3Codec { + var calls = 0 + override fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int { + calls++ + assertEquals(0, offset) + assertEquals(Lc3PacketFramer.LC3_BYTES, length) + return if (fail) -1 else Lc3PacketFramer.SAMPLES_PER_PACKET + } + } + + private fun packet(counter: Int, ssr: Int = 0, angle: Int = 0): ByteArray { + val p = ByteArray(Lc3PacketFramer.PACKET_BYTES) + p[Lc3PacketFramer.COUNTER_OFFSET] = counter.toByte() + p[Lc3PacketFramer.SSR_OFFSET] = (ssr and 0xff).toByte() + p[Lc3PacketFramer.SSR_OFFSET + 1] = ((ssr shr 8) and 0xff).toByte() + p[Lc3PacketFramer.ANGLE_OFFSET] = (angle and 0xff).toByte() + p[Lc3PacketFramer.ANGLE_OFFSET + 1] = ((angle shr 8) and 0xff).toByte() + return p + } + + @Test + fun countsRealDuplicateLateAndMissingPackets() { + val codec = FakeCodec() + val framer = Lc3PacketFramer(codec) + val pcm = ShortArray(Lc3PacketFramer.SAMPLES_PER_PACKET) + assertEquals(800, framer.decodePacket(packet(250), pcm)) + assertEquals(800, framer.decodePacket(packet(251), pcm)) + assertEquals(0, framer.decodePacket(packet(251), pcm)) // relayed by the other arm + assertEquals(0, framer.decodePacket(packet(250), pcm)) // late copy (gap 255) + assertEquals(800, framer.decodePacket(packet(254), pcm)) // 252, 253 lost + assertEquals(800, framer.decodePacket(packet(1), pcm)) // wraps past 255 -> 0; 255, 0 lost + assertEquals(0, framer.decodePacket(packet(129), pcm)) // gap 128 is late, not 127 losses + assertEquals(4, framer.realPackets) + assertEquals(3, framer.duplicatePackets) + assertEquals(4, framer.missingPackets) + assertEquals(0, framer.decodeErrors) + assertEquals(4, codec.calls) + } + + @Test + fun parsesSignedTrailerAndRejectsBadInput() { + val codec = FakeCodec() + val framer = Lc3PacketFramer(codec) + val pcm = ShortArray(Lc3PacketFramer.SAMPLES_PER_PACKET) + framer.decodePacket(packet(7, ssr = -1234, angle = -45), pcm) + assertEquals(-1234, framer.lastSsr) + assertEquals(-45, framer.lastAngleDegrees) + assertEquals(0, framer.decodePacket(ByteArray(10), pcm)) + assertEquals(0, framer.decodePacket(null, pcm)) + assertEquals(2, framer.decodeErrors) + assertFailsWith { framer.decodePacket(packet(8), ShortArray(10)) } + codec.fail = true + assertEquals(0, framer.decodePacket(packet(8), pcm)) + assertEquals(3, framer.decodeErrors) + // A failed decode does not advance the counter, so 8 is still accepted afterwards. + codec.fail = false + assertEquals(800, framer.decodePacket(packet(8), pcm)) + assertEquals(2, framer.realPackets) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RemoteInputSessionTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RemoteInputSessionTest.kt new file mode 100644 index 00000000..a77c68ba --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RemoteInputSessionTest.kt @@ -0,0 +1,178 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +private class FakeConnection(input: String, private val bytesOverride: ByteArray? = null) : RemoteInputConnection { + private val bytes = bytesOverride ?: input.encodeToByteArray() + private var index = 0 + private val written = ByteSink() + var closed = false + val output: String get() = written.toByteArray().decodeToString() + + override fun readByte(timeoutMs: Long): Int = if (index < bytes.size) bytes[index++].toInt() and 255 else -1 + + override fun write(bytes: ByteArray) = written.write(bytes) + + override fun close() { closed = true } +} + +private class FakePort(platform: ProtocolPlatform) : LocalServerPort { + val connections = BlockingQueue(platform) + var localAddresses = setOf("127.0.0.1", "192.168.1.5") + var bindFails = false + var listener: FakeListener? = null + + inner class FakeListener : RemoteInputListener { + override var isClosed = false + override fun accept(): RemoteInputConnection? { + while (!isClosed) { + val next = connections.poll(20) + if (next != null) return next + } + return null + } + override fun close() { isClosed = true } + } + + override fun isLocalUnicastAddress(address: String): Boolean = address in localAddresses + + override fun bind(address: String, port: Int): RemoteInputListener { + if (bindFails) throw IllegalStateException("in use") + return FakeListener().also { listener = it } + } +} + +private fun waitFor(timeoutMs: Long = 3000, condition: () -> Boolean): Boolean { + val platform = testPlatform() + val deadline = platform.elapsedRealtimeMs() + timeoutMs + while (platform.elapsedRealtimeMs() < deadline) { + if (condition()) return true + sleepMs(5) + } + return condition() +} + +class RemoteInputSessionTest { + private val platform = testPlatform() + private val random = object : RandomSource { override fun nextBytes(count: Int) = ByteArray(count) { it.toByte() } } + private fun dispatchOnThread(action: () -> Unit) = startThread("remote-input-dispatch", true, action) + + private fun session(port: FakePort, requestTimeoutMs: Long = 5000, readTimeoutMs: Long = 5000) = + RemoteInputSession(port, random, ::dispatchOnThread, platform, requestTimeoutMs = requestTimeoutMs, readTimeoutMs = readTimeoutMs) + + @Test + fun startValidatesAddressesAndBindFailures() { + val port = FakePort(platform) + val session = session(port) + assertEquals("Invalid local address.", session.start(7000, "localhost")) + assertEquals("Address is not assigned to a local interface.", session.start(7000, "10.0.0.9")) + port.bindFails = true + assertEquals("Could not open local input port 7000.", session.start(7000, "127.0.0.1")) + port.bindFails = false + assertEquals("", session.start(7000, "127.0.0.1")) + assertEquals("", session.start(7000, "127.0.0.1"), "second start is a no-op") + session.stop() + assertTrue(waitFor { port.listener!!.isClosed }) + assertTrue(RemoteInputSession.isNumericLocalAddress("fe80::1%en0")) + assertFalse(RemoteInputSession.isNumericLocalAddress("example.com")) + } + + @Test + fun requestIsPublishedAnsweredAndWrittenBack() { + val port = FakePort(platform) + val session = session(port) + var notified = 0 + session.setRequestListener { notified++ } + assertEquals("", session.start(7000, "127.0.0.1")) + val connection = FakeConnection("{\"token\":\"t\",\"text\":\"hi\"}\n") + port.connections.put(connection) + assertTrue(waitFor { notified == 1 }) + val pending = Json.parseObject(session.nextRequest()!!) + assertEquals(1L, pending["id"].asLong()) + assertEquals("{\"token\":\"t\",\"text\":\"hi\"}", pending["body"].asString()) + assertTrue(pending["expiresAt"].asLong()!! > currentTimeMillis()) + assertNull(session.nextRequest(), "consumed once") + session.complete(99L, "{\"ok\":false}") + assertFalse(waitFor(100) { connection.output.isNotEmpty() }, "wrong id is ignored") + session.complete(1L, "{\"ok\":true}") + assertTrue(waitFor { connection.output == "{\"ok\":true}\n" }) + assertTrue(waitFor { connection.closed }) + session.stop() + } + + @Test + fun timeoutRepliesWithTheTimeoutResponse() { + val port = FakePort(platform) + val session = session(port, requestTimeoutMs = 150) + assertEquals("", session.start(7000, "127.0.0.1")) + val connection = FakeConnection("slow\n") + port.connections.put(connection) + assertTrue(waitFor { connection.output == RemoteInputSession.TIMEOUT_RESPONSE + "\n" }) + assertNull(session.nextRequest(), "expired request is dropped") + session.stop() + } + + @Test + fun oversizedAndIncompleteFramesGetNoReply() { + val port = FakePort(platform) + val session = session(port) + var notified = 0 + session.setRequestListener { notified++ } + assertEquals("", session.start(7000, "127.0.0.1")) + val huge = FakeConnection("", ByteArray(RemoteInputSession.MAX_FRAME_BYTES + 2) { 'a'.code.toByte() }.also { it[it.size - 1] = '\n'.code.toByte() }) + port.connections.put(huge) + assertTrue(waitFor { huge.closed }) + assertEquals("", huge.output) + val partial = FakeConnection("no newline") + port.connections.put(partial) + assertTrue(waitFor { partial.closed }) + assertEquals("", partial.output) + val invalidUtf8 = FakeConnection("", byteArrayOf(0xff.toByte(), 0xfe.toByte(), '\n'.code.toByte())) + port.connections.put(invalidUtf8) + assertTrue(waitFor { invalidUtf8.closed }) + assertEquals("", invalidUtf8.output) + assertEquals(0, notified) + assertNull(session.nextRequest()) + session.stop() + } + + @Test + fun stopWhilePendingReleasesTheServingThread() { + val port = FakePort(platform) + val session = session(port) + var notified = 0 + session.setRequestListener { notified++ } + assertEquals("", session.start(7000, "127.0.0.1")) + val connection = FakeConnection("pending\n") + port.connections.put(connection) + assertTrue(waitFor { notified == 1 }) + session.stop() + assertTrue(waitFor { connection.closed }) + assertNull(session.nextRequest()) + assertEquals("", session.start(7001, "127.0.0.1"), "restart after stop") + session.stop() + } + + @Test + fun helpersFilterInterfacesAndHashTokens() { + val all = listOf( + InterfaceAddress("lo0", "127.0.0.1", up = true, pointToPoint = false, loopback = true, linkLocal = false, anyLocal = false), + InterfaceAddress("wlan0", "192.168.1.5", up = true, pointToPoint = false, loopback = false, linkLocal = false, anyLocal = false), + InterfaceAddress("wlan0", "fe80::1%wlan0", up = true, pointToPoint = false, loopback = false, linkLocal = true, anyLocal = false), + InterfaceAddress("tun0", "100.64.0.2", up = true, pointToPoint = true, loopback = false, linkLocal = false, anyLocal = false), + InterfaceAddress("eth0", "10.0.0.2", up = false, pointToPoint = false, loopback = false, linkLocal = false, anyLocal = false), + InterfaceAddress("any", "0.0.0.0", up = true, pointToPoint = false, loopback = false, linkLocal = false, anyLocal = true), + ) + assertEquals( + "[{\"name\":\"wlan0\",\"address\":\"192.168.1.5\",\"pointToPoint\":false},{\"name\":\"tun0\",\"address\":\"100.64.0.2\",\"pointToPoint\":true}]", + RemoteInputSession.interfacesJson(all), + ) + assertEquals("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", RemoteInputSession.tokenHash("abc")) + val session = session(FakePort(platform)) + assertEquals("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", session.randomSecret()) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResumableDownloadTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResumableDownloadTest.kt new file mode 100644 index 00000000..be68ffa3 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResumableDownloadTest.kt @@ -0,0 +1,149 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class ResumableDownloadTest { + private class Storage : DownloadStorage { + val files = HashMap() + override fun length(path: String): Long = files[path]?.size?.toLong() ?: -1L + override fun prepareParent(path: String) {} + override fun openWrite(path: String, append: Boolean): FileSink { + if (!append) files[path] = ByteArray(0) + return object : FileSink { + override fun write(bytes: ByteArray, offset: Int, length: Int) { + files[path] = (files[path] ?: ByteArray(0)) + bytes.copyOfRange(offset, offset + length) + } + override fun close() {} + } + } + override fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long { + val data = files[path] ?: return 0 + var at = 0L + for (chunk in data.toList().chunked(7)) { + val bytes = chunk.toByteArray() + if (!consumer(bytes, bytes.size)) break + at += bytes.size + } + return at + } + override fun rename(from: String, to: String): Boolean { files[to] = files.remove(from) ?: return false; return true } + override fun delete(path: String): Boolean = files.remove(path) != null + } + + private class Transport(val content: ByteArray, val honorRange: Boolean = true, val truncateAt: Int = -1) : DownloadTransport { + var lastRange = -1L + override fun open(url: String, rangeFrom: Long): DownloadResponse { + lastRange = rangeFrom + val start = if (honorRange) rangeFrom.toInt() else 0 + val end = if (truncateAt >= 0) minOf(truncateAt, content.size) else content.size + val body = content.copyOfRange(start, end) + var at = 0 + return object : DownloadResponse { + override val code: Int = if (honorRange && rangeFrom > 0) 206 else 200 + override val contentLength: Long = body.size.toLong() + override fun read(buffer: ByteArray): Int { + if (at >= body.size) return -1 + val n = minOf(5, body.size - at) + body.copyInto(buffer, 0, at, at + n) + at += n + return n + } + override fun close() {} + } + } + } + + private class Events : FaceclawModelDownloaderListener { + val progress = ArrayList>() + var done: String? = null + var error: String? = null + override fun onProgress(bytesDownloaded: Long, totalBytes: Long) { progress.add(bytesDownloaded to totalBytes) } + override fun onDone(path: String?) { done = path } + override fun onError(message: String?) { error = message } + } + + private val content = ByteArray(40) { (it * 3).toByte() } + private val sha = sha256Hex(content) + + private fun run(storage: Storage, transport: Transport, expectedSha: String? = sha, total: Long = content.size.toLong()): Events { + val events = Events() + var clock = 0L + ResumableDownload("http://x/m", "/dest/model", expectedSha, total, transport, storage, events) { clock += 600; clock }.run() + return events + } + + @Test + fun freshDownloadVerifiesAndRenames() { + val storage = Storage() + val transport = Transport(content) + val events = run(storage, transport) + assertNull(events.error) + assertEquals("/dest/model", events.done) + assertEquals(content.toList(), storage.files["/dest/model"]!!.toList()) + assertEquals(0L, transport.lastRange) + assertTrue(events.progress.isNotEmpty() && events.progress.last().second == 40L) + val again = run(storage, transport) + assertEquals("/dest/model", again.done) + } + + @Test + fun resumesFromPartWhenServerHonorsRange() { + val storage = Storage() + storage.files["/dest/model.part"] = content.copyOfRange(0, 17) + val transport = Transport(content) + val events = run(storage, transport) + assertEquals(17L, transport.lastRange) + assertNull(events.error) + assertEquals(content.toList(), storage.files["/dest/model"]!!.toList()) + } + + @Test + fun restartsWhenServerIgnoresRange() { + val storage = Storage() + storage.files["/dest/model.part"] = ByteArray(17) { 1 } + val transport = Transport(content, honorRange = false) + val events = run(storage, transport) + assertEquals(17L, transport.lastRange) + assertNull(events.error) + assertEquals(content.toList(), storage.files["/dest/model"]!!.toList()) + } + + @Test + fun checksumMismatchDeletesPart() { + val storage = Storage() + val events = run(storage, Transport(content), expectedSha = "00".repeat(32)) + assertEquals("Model download was corrupted (checksum mismatch); download it again", events.error) + assertNull(storage.files["/dest/model.part"]) + assertNull(storage.files["/dest/model"]) + } + + @Test + fun shortBodyKeepsPartForResume() { + val storage = Storage() + val events = run(storage, Transport(content, truncateAt = 30)) + assertEquals("Model download ended early (30 of 40 bytes); try again to resume", events.error) + assertEquals(30, storage.files["/dest/model.part"]!!.size) + val resumed = run(storage, Transport(content)) + assertNull(resumed.error) + assertEquals(content.toList(), storage.files["/dest/model"]!!.toList()) + } + + @Test + fun httpErrorIsReported() { + val storage = Storage() + val transport = object : DownloadTransport { + override fun open(url: String, rangeFrom: Long): DownloadResponse = object : DownloadResponse { + override val code = 404 + override val contentLength = -1L + override fun read(buffer: ByteArray) = -1 + override fun close() {} + } + } + val events = Events() + ResumableDownload("u", "/dest/model", null, 0, transport, storage, events).run() + assertEquals("Model download failed: HTTP 404", events.error) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SettingsChangeHubTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SettingsChangeHubTest.kt new file mode 100644 index 00000000..42a019d3 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SettingsChangeHubTest.kt @@ -0,0 +1,46 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SettingsChangeHubTest { + private class MemorySettings : SettingsStorage { + val values = HashMap() + override fun getString(key: String?, defaultValue: String?): String? = values[key] as? String ?: defaultValue + override fun putString(key: String?, value: String?) { values[key] = value } + override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = values[key] as? Boolean ?: defaultValue + override fun putBoolean(key: String?, value: Boolean) { values[key] = value } + } + + @Test + fun writesNotifyEachListenerThroughItsOwnDispatcher() { + val warnings = ArrayList() + val hub = SettingsChangeHub(MemorySettings(), { warnings.add(it) }, testPlatform()) + val queued = ArrayList<() -> Unit>() + val seen = ArrayList() + val listener = object : FaceclawSettingsListener { override fun onSettingChanged(key: String?) { seen.add(key) } } + hub.register(listener) { queued.add(it) } + val silent = object : FaceclawSettingsListener { override fun onSettingChanged(key: String?) { seen.add("silent") } } + hub.register(silent, null) + assertEquals(1, warnings.size) + + hub.setString("font", "Inter") + hub.setBoolean("dark", true) + assertEquals("Inter", hub.getString("font", null)) + assertTrue(hub.getBoolean("dark", false)) + assertTrue(seen.isEmpty()) // delivered only when the dispatcher runs + queued.forEach { it() } + assertEquals(listOf("font", "dark"), seen) + + val failing = object : FaceclawSettingsListener { override fun onSettingChanged(key: String?) { throw IllegalStateException("boom") } } + hub.register(failing) { it() } + hub.setString("x", null) + assertTrue(warnings.last().contains("boom")) + hub.unregister(listener) + hub.unregister(failing) + queued.clear() + hub.setBoolean("dark", false) + assertTrue(queued.isEmpty()) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SseStreamTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SseStreamTest.kt new file mode 100644 index 00000000..901ba60d --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/SseStreamTest.kt @@ -0,0 +1,87 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class SseStreamTest { + private fun feed(stream: SseStream, vararg chunks: String) { + for (chunk in chunks) { + val bytes = chunk.encodeToByteArray() + stream.onChunk(bytes, 0, bytes.size) + } + } + + @Test + fun splitsLinesAcrossChunksWithBomAndCrlf() { + val stream = SseStream() + stream.onStatus(200) + feed(stream, "event: a\r", "\ndata: {\"x\":1}\n\nda", "ta: tail") + assertEquals(listOf(SseEvent.Line("event: a"), SseEvent.Line("data: {\"x\":1}"), SseEvent.Line("")), stream.takeEvents()) + stream.onComplete() + assertEquals(listOf(SseEvent.Line("data: tail"), SseEvent.Complete), stream.takeEvents()) + assertTrue(stream.isFinished) + feed(stream, "late\n") + assertEquals(emptyList(), stream.takeEvents()) + } + + @Test + fun errorBodyIsBufferedAndCapped() { + val stream = SseStream() + stream.onStatus(429) + feed(stream, "{\"error\":", "\"slow down\"}\n") + assertEquals(emptyList(), stream.takeEvents()) + stream.onComplete() + assertEquals(listOf(SseEvent.HttpError(429, "{\"error\":\"slow down\"}\n")), stream.takeEvents()) + + val capped = SseStream() + capped.onStatus(500) + val big = ByteArray(SseStream.ERROR_BODY_LIMIT + 100) { 'x'.code.toByte() } + capped.onChunk(big, 0, big.size) + capped.onComplete() + val event = capped.takeEvents().single() as SseEvent.HttpError + assertEquals(SseStream.ERROR_BODY_LIMIT, event.body.length) + } + + @Test + fun queueOverflowAndOversizedLinesFail() { + val stream = SseStream() + stream.onStatus(200) + val line = "x".repeat(1023) + "\n" + val block = line.repeat(64).encodeToByteArray() + var fed = 0L + while (fed <= SseStream.QUEUE_BYTE_LIMIT + block.size) { + stream.onChunk(block, 0, block.size) + fed += block.size + } + val events = stream.takeEvents() + assertEquals(SseEvent.Failure("Streaming response queue overflow"), events.single()) + + val long = SseStream() + long.onStatus(200) + val huge = ByteArray(SseStream.LINE_LIMIT + 1) { 'y'.code.toByte() } + long.onChunk(huge, 0, huge.size) + assertEquals(SseEvent.Failure("Streaming response line too large"), long.takeEvents().single()) + } + + @Test + fun failureAndCancelAreTerminal() { + val stream = SseStream() + stream.onStatus(200) + feed(stream, "a\n") + stream.onFailure("boom") + assertEquals(listOf(SseEvent.Line("a"), SseEvent.Failure("boom")), stream.takeEvents()) + val cancelled = SseStream() + cancelled.onStatus(200) + feed(cancelled, "a\n") + cancelled.cancel() + cancelled.onComplete() + assertEquals(emptyList(), cancelled.takeEvents()) + } + + @Test + fun unpacksFlatHeaderArrays() { + assertEquals(listOf("A" to "1", "C" to "3"), SseStream.unpackHeaders(arrayOf("A", "1", "", "2", "C", "3", "D"))) + assertEquals(emptyList(), SseStream.unpackHeaders(null)) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/StockFlowsTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/StockFlowsTest.kt new file mode 100644 index 00000000..9573d705 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/StockFlowsTest.kt @@ -0,0 +1,382 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** A stock-firmware lens pair behind the StockLink port; replies are scripted per written message. */ +private class FakeStockLink : StockLink { + class Written(val address: String, val uuid: String, val sid: Int, val flag: Int, val seq: Int, val pb: ByteArray) { + /** Protobuf magic (field 2) of a control message; -1 for OTA payloads. */ + val magic: Int get() = if (sid == 0xc0 || sid == 0xc1) -1 else BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(pb), 2, -1) + } + + var inbound: StockLinkListener? = null + val connected = HashSet() + val bond = HashMap() + val writes = ArrayList() + var connectResult: (String) -> Boolean = { true } + var responder: (Written) -> Unit = {} + var closed = false + + override fun setListener(listener: StockLinkListener?) { inbound = listener } + override fun connect(address: String, timeoutMs: Int): Boolean = connectResult(address).also { if (it) connected.add(address) } + override fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int) {} + override fun discoverServices(address: String, timeoutMs: Int): Boolean = address in connected + override fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean = address in connected + override fun writeFrames(address: String, characteristicUuid: String, frames: List, mode: GattWriteMode, timeoutMs: Int): Boolean { + if (address !in connected) throw IllegalStateException("Not connected: $address") + val head = frames.first() + val body = ByteArray(frames.sumOf { it.size - 8 }) + var at = 0 + for (f in frames) { f.copyInto(body, at, 8); at += f.size - 8 } + val written = Written(address, characteristicUuid, head[6].toInt() and 0xff, head[7].toInt() and 0xff, head[2].toInt() and 0xff, body) + writes.add(written) + responder(written) + return true + } + override fun isConnected(address: String): Boolean = address in connected + override fun bondState(address: String): BondState = bond[address] ?: BondState.BONDED + override fun disconnect(address: String) { connected.remove(address) } + override fun close() { closed = true; connected.clear() } + + /** Deliver a lens frame on the control notify characteristic. */ + fun notify(address: String, pb: ByteArray, sid: Int, flag: Int = 0) { + for (frame in BleProtocol.framePb(pb, sid, flag, 7)) inbound!!.onNotification(address, BleProtocol.NOTIFY_CHAR_UUID, frame) + } + fun otaAck(address: String, op: Int, status: Int) { + inbound!!.onNotification(address, BleProtocol.OTA_DATA_NOTIFY_UUID, BleProtocol.framePb(byteArrayOf(op.toByte(), status.toByte()), 0xc1, 0, 1).single()) + } + fun drop(address: String) { connected.remove(address); inbound!!.onConnectionStateChange(address, false) } +} + +private fun varint(value: Int): ByteArray { var v = value; val out = ArrayList(); do { var b = v and 0x7f; v = v ushr 7; if (v != 0) b = b or 0x80; out.add(b.toByte()) } while (v != 0); return out.toByteArray() } +private fun field(number: Int, bytes: ByteArray): ByteArray = varint((number shl 3) or 2) + varint(bytes.size) + bytes +private fun field(number: Int, text: String): ByteArray = field(number, text.encodeToByteArray()) +private fun authSuccess(magic: Int) = BleProtocol.encodeVarintField(1, 4) + BleProtocol.encodeVarintField(2, magic) + field(3, ByteArray(0)) +private fun authPending(magic: Int) = BleProtocol.encodeVarintField(1, 4) + BleProtocol.encodeVarintField(2, magic) + field(3, BleProtocol.encodeVarintField(1, 1)) +private fun ack(magic: Int) = BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, magic) +private fun settingsAck(magic: Int, battery: Int = -1, left: String = "", right: String = "", extension: String = ""): ByteArray { + var request = ByteArray(0) + if (left.isNotEmpty()) request += field(5, left) + if (right.isNotEmpty()) request += field(6, right) + if (battery >= 0) request += BleProtocol.encodeVarintField(12, battery) + var pb = BleProtocol.encodeVarintField(1, 2) + BleProtocol.encodeVarintField(2, magic) + field(4, request) + if (extension.isNotEmpty()) pb += field(100, extension) + return pb +} +private fun listSelection(container: String, item: String, index: Int, eventType: Int): ByteArray = + field(13, field(1, field(2, container) + field(3, item) + BleProtocol.encodeVarintField(4, index) + BleProtocol.encodeVarintField(5, eventType))) + +private val fast = StockFlowTimings( + securityAuthTimeoutMs = 300, securityAuthSoftTimeoutMs = 100, pairingWaitCapMs = 2_000, postBondGraceMs = 20, pollMs = 5, + armRetryDelayMs = 5, queryTimeoutMs = 40, preludeTimeoutMs = 40, heartbeatIntervalMs = 15, selectionTimeoutMs = 80, createAckTimeoutMs = 40, + batteryAckTimeoutMs = 40, blockAckTimeoutMs = 60, ctrlAckTimeoutMs = 60, firstLensConnectWindowMs = 200, secondLensConnectWindowMs = 200, + rebootSettleMs = 5, notifySettleMs = 5, otaRetryDelayMs = 5, componentRetryDelayMs = 5, +) + +private const val R = "AA:00:00:00:00:01" +private const val L = "AA:00:00:00:00:02" + +private class ProbeEvents : FaceclawDeviceInfoProbeListener { + val logs = ArrayList(); val states = ArrayList(); var result: List? = null; var error: String? = null + override fun onLog(line: String?) { logs.add(line ?: "") } + override fun onState(state: String?, detail: String?) { states.add("$state:$detail") } + override fun onResult(leftVersion: String?, rightVersion: String?, extension: String?) { result = listOf(leftVersion ?: "", rightVersion ?: "", extension ?: "") } + override fun onError(message: String?) { error = message } +} + +/** Standard lens behaviour: auth succeeds, prelude and settings are acked with the given versions/battery. */ +private fun stockResponder(link: FakeStockLink, left: String = "2.2.9", right: String = "2.2.9", extension: String = "", battery: Int = 85): (FakeStockLink.Written) -> Unit = { w -> + when (w.sid) { + BleProtocol.SID_SECURITY_AUTH -> link.notify(w.address, authSuccess(w.magic), w.sid) + BleProtocol.PRELUDE_ACK_SID -> link.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid) + BleProtocol.SID_UI_SETTING -> link.notify(w.address, settingsAck(w.magic, battery, left, right, extension), w.sid) + BleProtocol.SID_EVENHUB -> link.notify(w.address, ack(w.magic), w.sid) + } +} + +class StockFlowsTest { + @Test + fun probeAuthenticatesBothArmsThenReadsVersions() { + val link = FakeStockLink() + link.responder = stockResponder(link, "L1", "R1", "Faceclaw/3") + val events = ProbeEvents() + DeviceInfoProbeFlow(link, R, L, events, fast, testPlatform()).run() + assertEquals(listOf("L1", "R1", "Faceclaw/3"), events.result) + assertEquals(null, events.error) + assertEquals(listOf("connecting:right", "authenticating:right", "connecting:left", "authenticating:left", "querying:right"), events.states) + // Right arm then left arm auth, then prelude + one settings read on the right arm only. + assertEquals(listOf(R, L), link.writes.filter { it.sid == BleProtocol.SID_SECURITY_AUTH }.map { it.address }) + assertEquals(listOf(R), link.writes.filter { it.sid == BleProtocol.SID_UI_SETTING }.map { it.address }) + assertTrue(link.closed) + } + + @Test + fun probeUsesUnsolicitedSettingsPushAndFallsBackToLeftLens() { + // Right lens: auth ok, but the settings read is never acked; instead a push on a + // device-chosen magic carries the versions. + val pushed = FakeStockLink() + pushed.responder = { w -> + when (w.sid) { + BleProtocol.SID_SECURITY_AUTH -> pushed.notify(w.address, authSuccess(w.magic), w.sid) + BleProtocol.PRELUDE_ACK_SID -> pushed.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid) + BleProtocol.SID_UI_SETTING -> pushed.notify(w.address, settingsAck(250, -1, "P1", "P2", ""), w.sid) + } + } + var events = ProbeEvents() + DeviceInfoProbeFlow(pushed, R, "", events, fast, testPlatform()).run() + assertEquals(listOf("P1", "P2", ""), events.result) + assertTrue(events.logs.any { it.contains("unsolicited settings push") }) + + // Right lens silent on settings (both attempts); the left lens answers. + val silentRight = FakeStockLink() + silentRight.responder = { w -> + when (w.sid) { + BleProtocol.SID_SECURITY_AUTH -> silentRight.notify(w.address, authSuccess(w.magic), w.sid) + BleProtocol.PRELUDE_ACK_SID -> silentRight.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid) + BleProtocol.SID_UI_SETTING -> if (w.address == L) silentRight.notify(w.address, settingsAck(w.magic, -1, "LL", "RR", ""), w.sid) + } + } + events = ProbeEvents() + DeviceInfoProbeFlow(silentRight, R, L, events, fast, testPlatform()).run() + assertEquals(listOf("LL", "RR", ""), events.result) + assertTrue(events.logs.any { it.contains("probing the left lens") }) + assertEquals(2, silentRight.writes.count { it.sid == BleProtocol.SID_UI_SETTING && it.address == R }) + assertContains(events.states, "querying:left") + } + + @Test + fun probeWaitsThroughPairingAndResendsAfterBond() { + // Unbonded phone: the first auth request is answered with the pre-encryption + // non-success result, the OS then pairs (no writes while BONDING), and once bonded the + // request is re-sent after the grace period and finally succeeds. + val link = FakeStockLink() + link.bond[R] = BondState.NONE + val authReplies = ArrayList() + link.responder = { w -> + when (w.sid) { + BleProtocol.SID_SECURITY_AUTH -> { authReplies.add(w.magic); link.notify(w.address, if (authReplies.size == 1) authPending(w.magic) else authSuccess(w.magic), w.sid) } + BleProtocol.PRELUDE_ACK_SID -> link.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid) + BleProtocol.SID_UI_SETTING -> link.notify(w.address, settingsAck(w.magic, -1, "a", "b", ""), w.sid) + } + } + startThread("bond-flip", true) { sleepMs(20); link.bond[R] = BondState.BONDING; sleepMs(40); link.bond[R] = BondState.BONDED } + val events = ProbeEvents() + DeviceInfoProbeFlow(link, R, "", events, fast, testPlatform()).run() + assertEquals(listOf("a", "b", ""), events.result, events.logs.joinToString("\n")) + assertTrue(events.logs.any { it.contains("is not the success result") }) + assertTrue(events.logs.any { it.contains("is pairing with the right lens") }) + assertTrue(events.logs.any { it.contains("bond established") }) + assertTrue(events.logs.any { it.contains("re-sending the auth request after pairing") }) + assertEquals(2, authReplies.size) + assertTrue(events.logs.any { it.contains("security auth complete: right lens") }) + } + + @Test + fun probeReportsUnreachableArmAndCancel() { + val link = FakeStockLink() + link.connectResult = { false } + val events = ProbeEvents() + DeviceInfoProbeFlow(link, R, L, events, fast, testPlatform()).run() + assertNotNull(events.error).let { assertTrue(it.contains("could not connect to the right lens"), it) } + assertEquals(3, events.states.count { it == "connecting:right" }) + + val cancelled = FakeStockLink() + cancelled.responder = { w -> if (w.sid == BleProtocol.SID_SECURITY_AUTH) cancelled.notify(w.address, authPending(w.magic), w.sid) } + val slow = StockFlowTimings(securityAuthTimeoutMs = 5_000, pollMs = 5) + val flow = DeviceInfoProbeFlow(cancelled, R, "", ProbeEvents().also { events.logs.clear() }, slow, testPlatform()) + startThread("cancel", true) { sleepMs(30); flow.cancel() } + val start = testPlatform().elapsedRealtimeMs() + flow.run() + assertTrue(testPlatform().elapsedRealtimeMs() - start < 2_000) + assertTrue(flow.cancelled) + } + + private class PromptEvents : FaceclawFlashPromptListener { + val logs = ArrayList(); val states = ArrayList(); var battery: Pair? = null; var result: Boolean? = null + override fun onLog(line: String?) { logs.add(line ?: "") } + override fun onState(state: String?, detail: String?) { states.add("$state:$detail") } + override fun onBattery(rightPercent: Int, leftPercent: Int) { battery = rightPercent to leftPercent } + override fun onResult(approved: Boolean) { result = approved } + } + + private fun promptResponder(link: FakeStockLink, selection: Int?, batteries: Map): (FakeStockLink.Written) -> Unit = { w -> + when (w.sid) { + BleProtocol.SID_SECURITY_AUTH -> link.notify(w.address, authSuccess(w.magic), w.sid) + BleProtocol.PRELUDE_ACK_SID -> link.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid) + BleProtocol.SID_UI_SETTING -> link.notify(w.address, settingsAck(w.magic, batteries[w.address] ?: -1), w.sid) + BleProtocol.SID_EVENHUB -> { + val cmd = BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(w.pb), 1, -1) + link.notify(w.address, ack(w.magic), w.sid) + // Cmd 0 = create page: the user taps a row right away (a scroll first, which must be ignored). + if (cmd == 0 && selection != null) { + link.notify(w.address, listSelection(FlashPromptFlow.LIST_NAME, "other", 0, BleProtocol.EVENT_SCROLL_TOP), BleProtocol.SID_EVENHUB, BleProtocol.FLAG_NOTIFY) + link.notify(w.address, listSelection(FlashPromptFlow.LIST_NAME, FlashPromptFlow.ITEMS[selection], selection, BleProtocol.EVENT_CLICK), BleProtocol.SID_EVENHUB, BleProtocol.FLAG_NOTIFY) + } + } + } + } + + @Test + fun promptApprovalReadsBothBatteriesAndShutsDown() { + val link = FakeStockLink() + link.responder = promptResponder(link, 1, mapOf(R to 85, L to 70)) + val events = PromptEvents() + FlashPromptFlow(link, R, L, "warning", false, events, fast, testPlatform()).run() + assertEquals(true, events.result, events.logs.joinToString("\n")) + assertEquals(85 to 70, events.battery) + assertEquals(listOf("connecting:", "connected:", "prompting:", "battery:", "result:approved"), events.states) + val hub = link.writes.filter { it.sid == BleProtocol.SID_EVENHUB }.map { BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(it.pb), 1, -1) } + assertEquals(0, hub.first()) // create page + assertEquals(9, hub.last()) // shutdown + assertTrue(link.writes.all { it.sid != BleProtocol.SID_EVENHUB || it.address == R }) + assertTrue(link.closed) + } + + @Test + fun promptDeclineTimeoutAndSkip() { + val declined = FakeStockLink() + declined.responder = promptResponder(declined, 0, mapOf()) + var events = PromptEvents() + FlashPromptFlow(declined, R, "", "w", false, events, fast, testPlatform()).run() + assertEquals(false, events.result) + assertEquals("result:declined", events.states.last()) + assertEquals(null, events.battery) + + val silent = FakeStockLink() + silent.responder = promptResponder(silent, null, mapOf()) + events = PromptEvents() + FlashPromptFlow(silent, R, "", "w", false, events, fast, testPlatform()).run() + assertEquals(null, events.result) + assertEquals("timeout:No response from the glasses.", events.states.last()) + // Heartbeats (EvenHub cmd 12) kept the page alive while waiting (interval 15 ms, wait 80 ms). + assertTrue(silent.writes.count { it.sid == BleProtocol.SID_EVENHUB && BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(it.pb), 1, -1) == 12 } >= 2) + + val skipped = FakeStockLink() + skipped.responder = promptResponder(skipped, null, mapOf(R to 50)) + events = PromptEvents() + FlashPromptFlow(skipped, R, L, "w", true, events, fast, testPlatform()).run() + assertEquals(true, events.result) + assertEquals(50 to -1, events.battery) + assertFalse(skipped.writes.any { it.sid == BleProtocol.SID_EVENHUB }) + + val lost = FakeStockLink() + lost.responder = { w -> + promptResponder(lost, null, mapOf())(w) + if (w.sid == BleProtocol.SID_EVENHUB) lost.drop(R) + } + events = PromptEvents() + FlashPromptFlow(lost, R, "", "w", false, events, fast, testPlatform()).run() + assertEquals("disconnected:Lost connection to the glasses.", events.states.last()) + } + + // ---- OTA --------------------------------------------------------------- + + private fun putU32(buf: ByteArray, offset: Int, value: Long) { for (i in 0 until 4) buf[offset + i] = ((value ushr (8 * i)) and 0xff).toByte() } + + private fun container(): ByteArray { + val mainApp = ByteArray(6000) { (it * 7).toByte() }.also { putU32(it, 0, 6000L); putU32(it, 0x14, FirmwareImage.APP_LOAD_ADDR) } + val components = listOf("ota/bootloader.bin" to ByteArray(300) { it.toByte() }, "ota/a.bin" to ByteArray(64) { 1 }, "ota/b.bin" to ByteArray(128) { 2 }, "ota/c.bin" to ByteArray(32) { 3 }, FirmwareImage.REQUIRED_SEGMENT to mainApp) + val toc = FirmwareImage.HEADER_SIZE + components.size * FirmwareImage.TOC_ENTRY_SIZE + var offset = toc + val offsets = components.map { c -> offset.also { offset += FirmwareImage.SUBHEADER_SIZE + c.second.size } } + val img = ByteArray(offset) + putU32(img, 8, components.size.toLong()) + components.forEachIndexed { i, c -> + val off = offsets[i] + putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 4, off.toLong()) + putU32(img, off + 8, c.second.size.toLong()) + c.first.encodeToByteArray().copyInto(img, off + FirmwareImage.NAME_OFFSET) + c.second.copyInto(img, off + FirmwareImage.SUBHEADER_SIZE) + val crc = FirmwareImage.crc32cUnsigned(c.second) + putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 12, crc) + putU32(img, off + 12, crc) + } + return img + } + + private class FlashEvents : FaceclawFirmwareFlasherListener { + val logs = ArrayList(); val states = ArrayList(); val progress = ArrayList(); var complete: Pair? = null + override fun onLog(line: String?) { logs.add(line ?: "") } + override fun onProgress(lens: String?, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int, bytesSent: Long, bytesTotal: Long) { progress.add("$lens $componentIndex/$componentCount $blockIndex/$blockCount $bytesSent/$bytesTotal") } + override fun onState(state: String?, detail: String?) { states.add("$state:$detail") } + override fun onComplete(success: Boolean, detail: String?) { complete = success to (detail ?: "") } + } + + /** OTA lens: NAKs one block once and fails one END once, exercising both retry paths. */ + private class OtaLens(val link: FakeStockLink) { + val received = HashMap() + var nakOnce = true + var failEndOnce = true + var component = ByteArray(0) + fun respond(w: FakeStockLink.Written) { + when (w.sid) { + BleProtocol.SID_SECURITY_AUTH -> link.notify(w.address, authSuccess(w.magic), w.sid) + OtaFlashFlow.SID_CTRL -> { + val pb = BleProtocol.stripTrailingCrc(w.pb) + when (pb[0].toInt()) { + OtaFlashFlow.OP_BEGIN -> link.otaAck(w.address, OtaFlashFlow.OP_BEGIN, 0) + OtaFlashFlow.OP_FILE_CHECK -> { component = ByteArray(0); link.otaAck(w.address, OtaFlashFlow.OP_FILE_CHECK, 0) } + OtaFlashFlow.OP_END -> { + val status = if (failEndOnce) { failEndOnce = false; 5 } else { received[w.address] = (received[w.address] ?: ByteArray(0)) + component; 9 } + link.otaAck(w.address, OtaFlashFlow.OP_END, status) + } + OtaFlashFlow.OP_BLOCK -> {} + } + } + OtaFlashFlow.SID_DATA -> { + val block = BleProtocol.stripTrailingCrc(w.pb) + if (nakOnce) { nakOnce = false; link.otaAck(w.address, OtaFlashFlow.OP_BLOCK, 1) } else { component += block; link.otaAck(w.address, OtaFlashFlow.OP_BLOCK, 0) } + } + } + } + } + + @Test + fun otaFlashesBothLensesWithBlockNakAndEndRetry() { + val img = container() + val segs = FirmwareImage.validate(img) + val link = FakeStockLink() + val lens = OtaLens(link) + link.responder = lens::respond + val events = FlashEvents() + OtaFlashFlow(link, R, L, "fw.bin", events, fast, testPlatform()) { img }.run() + assertEquals(true to "Both lenses flashed. The glasses are rebooting into the custom firmware.", events.complete, events.logs.joinToString("\n")) + assertEquals(listOf("validating:", "connecting:left", "flashing:left", "rebooting:Left lens done. Both lenses reboot briefly; reconnecting for the right lens.", "connecting:right", "flashing:right", "done:"), events.states) + // Every component payload arrived intact on both lenses, in order. + val expected = segs.fold(ByteArray(0)) { acc, s -> acc + img.copyOfRange(s.payloadStart, s.payloadStart + s.ps) } + assertContentEquals(expected, lens.received[L]!!) + assertContentEquals(expected, lens.received[R]!!) + assertTrue(events.logs.any { it.contains("NAK=1 resend 1/3") }) + assertTrue(events.logs.any { it.contains("END verify FAILED (status 5)") }) + assertTrue(events.logs.any { it.contains("re-flash attempt 2/3") }) + assertEquals("left 5/5 2/2 6524/6524", events.progress.first { it.startsWith("left 5/5 2") }) + // Marker + data share one envelope seq; the OTA stream restarts at 1 per lens. + val leftOta = link.writes.filter { it.address == L && it.uuid == BleProtocol.OTA_DATA_WRITE_UUID } + assertEquals(1, leftOta.first().seq) + val firstBlock = leftOta.indexOfFirst { it.sid == OtaFlashFlow.SID_DATA } + assertEquals(leftOta[firstBlock - 1].seq, leftOta[firstBlock].seq) + assertEquals(setOf(BleProtocol.WRITE_CHAR_UUID), link.writes.filter { it.sid == BleProtocol.SID_SECURITY_AUTH }.map { it.uuid }.toSet()) + } + + @Test + fun otaGivesUpWhenLensConnectWindowExpires() { + val link = FakeStockLink() + link.connectResult = { false } + val events = FlashEvents() + OtaFlashFlow(link, R, L, "fw.bin", events, fast, testPlatform()) { container() }.run() + assertEquals(false to "could not reach left lens: connect/discover incomplete", events.complete) + assertEquals("error:could not reach left lens: connect/discover incomplete", events.states.last()) + assertTrue(events.logs.count { it.contains("left connect attempt") } >= 2) + assertTrue(link.closed) + + val unreadable = FlashEvents() + OtaFlashFlow(FakeStockLink(), R, L, "/nonexistent/fw.bin", unreadable, fast, testPlatform()).run() + assertTrue(unreadable.complete!!.second.startsWith("could not read firmware"), unreadable.complete!!.second) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt new file mode 100644 index 00000000..17711f98 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VoiceCaptureSessionTest.kt @@ -0,0 +1,273 @@ +package com.faceclaw.app + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class VoiceCaptureSessionTest { + /** Real clock plus an offset the test can advance to force decode cadences. */ + private class ShiftedPlatform(private val base: ProtocolPlatform) : ProtocolPlatform by base { + var offsetMs = 0L + override fun elapsedRealtimeMs(): Long = base.elapsedRealtimeMs() + offsetMs + } + + private class Events : FaceclawVoiceControllerListener { + val statuses = mutableListOf() + val transcripts = mutableListOf>() + val pcmChunks = mutableListOf() + val frameMeta = mutableListOf>() + var speechEnds = 0 + var verified: Pair? = null + val stopped = mutableListOf() + val stoppedLatch = Latch(1, testPlatform()) + override fun onStatus(status: String?) { statuses.add(status) } + override fun onTranscript(text: String?, isFinal: Boolean) { transcripts.add(text to isFinal) } + override fun onStopped(captureId: Int) { stopped.add(captureId); stoppedLatch.countDown() } + override fun onPcm(pcm16le: ByteArray?) { pcmChunks.add(pcm16le!!) } + override fun onFrameMeta(angleDegrees: Int, ssr: Int) { frameMeta.add(angleDegrees to ssr) } + override fun onSpeechEnd() { speechEnds++ } + override fun onSpeakerVerified(isWearer: Boolean, similarity: Float) { verified = isWearer to similarity } + } + + /** Scripted phone mic: each entry is a chunk amplitude; after the script the read fails (-1). */ + private class ScriptedMic(private val amplitudes: List, private val onRead: () -> Unit = {}) : PcmAudioSource { + var index = 0 + var closed = false + override fun read(pcm: ShortArray): Int { + onRead() + if (index >= amplitudes.size) return -1 + val amplitude = amplitudes[index++] + for (i in pcm.indices) pcm[i] = (if (i % 2 == 0) amplitude else -amplitude).toShort() + return pcm.size + } + override fun close() { closed = true } + } + + private class FakeTranscriber(private val text: (Int) -> String) : OfflineTranscriber { + val calls = mutableListOf() + var released = false + override fun recognize(samples: FloatArray, count: Int): String { calls.add(count); return text(count) } + override fun release() { released = true } + } + + private class FakeHost( + val platform: ProtocolPlatform, + var mic: PcmAudioSource? = null, + var transcriber: OfflineTranscriber? = null, + var hasModel: Boolean = true, + var embedding: FloatArray? = null, + var g2Ready: Boolean = false, + ) : VoiceCaptureHost { + override val dispatcher = CallbackDispatcher { it() } + var packetListener: FaceclawAudioPacketListener? = null + var g2Stops = 0 + val recordings = mutableListOf() + override fun onSessionStarting() {} + override fun isG2SessionReady(): Boolean = g2Ready + override fun isG2AudioCaptureActive(): Boolean = packetListener != null + override fun g2AudioSource(): PacketAudioSource? = if (!g2Ready) null else object : PacketAudioSource { + override fun start(listener: FaceclawAudioPacketListener): Boolean { packetListener = listener; return true } + override fun stop() { g2Stops++ } + override fun isActive(): Boolean = packetListener != null + } + override fun createPacketDecoder(): AudioPacketDecoder = object : AudioPacketDecoder { + override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int { + if (packet == null || packet.size != Lc3PacketFramer.PACKET_BYTES) return 0 + pcmOut!!.fill(packet[0].toShort()) + lastAngleDegrees = Lc3PacketFramer.readTrailerS16(packet, Lc3PacketFramer.ANGLE_OFFSET) + lastSsr = Lc3PacketFramer.readTrailerS16(packet, Lc3PacketFramer.SSR_OFFSET) + realPackets++ + return Lc3PacketFramer.SAMPLES_PER_PACKET + } + override var lastAngleDegrees = 0 + override var lastSsr = 0 + override var realPackets = 0L + override val duplicatePackets = 0L + override val missingPackets = 0L + override val decodeErrors = 0L + override fun close() {} + } + override fun openPhoneMic(): PcmAudioSource? = mic + override fun hasTranscriberModel(kind: VoiceModelKind): Boolean = hasModel + override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber = transcriber ?: error("no transcriber") + override fun speakerEmbedder(modelPath: String): SpeakerEmbedder = object : SpeakerEmbedder { + override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding + override fun release() {} + } + override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) } + } + + private val logs = mutableListOf() + + private fun session(host: VoiceCaptureHost, platform: ProtocolPlatform, name: String) = + VoiceCaptureSession(host, platform, name, { logs.add("I $it") }, { logs.add("W $it") }, { m, e -> logs.add("E $m ${e.message}") }) + + private fun packet(value: Int, angle: Int, ssr: Int): ByteArray { + val bytes = ByteArray(Lc3PacketFramer.PACKET_BYTES) + bytes[0] = value.toByte() + bytes[Lc3PacketFramer.SSR_OFFSET] = (ssr and 0xff).toByte() + bytes[Lc3PacketFramer.SSR_OFFSET + 1] = (ssr shr 8).toByte() + bytes[Lc3PacketFramer.ANGLE_OFFSET] = (angle and 0xff).toByte() + bytes[Lc3PacketFramer.ANGLE_OFFSET + 1] = (angle shr 8).toByte() + return bytes + } + + @Test + fun beamGateWrapsAroundAndClampsHalfWidth() { + val session = session(FakeHost(testPlatform()), testPlatform(), "voice-test-beam") + session.setBeamFilter(true, 170, 30) + assertTrue(session.withinBeam(170)) + assertTrue(session.withinBeam(-170)) // 20 degrees away across the wrap + assertFalse(session.withinBeam(120)) + session.setBeamFilter(true, 0, 1) // clamps to 5 + assertTrue(session.withinBeam(5)) + assertFalse(session.withinBeam(6)) + } + + @Test + fun droppingQueueKeepsNewestAndCountsDrops() { + val queue = DroppingPacketQueue(3, testPlatform()) + for (i in 1..5) queue.put(i) + assertEquals(2L, queue.droppedCount) + assertEquals(3, queue.take(10) { true }) + assertEquals(4, queue.take(10) { true }) + assertEquals(5, queue.take(10) { true }) + assertNull(queue.take(10, { true })) + assertNull(queue.take(10, { false })) + } + + @Test + fun g2PacketsAreCountedGatedAndForwarded() { + val platform = ShiftedPlatform(testPlatform()) + val host = FakeHost(platform, g2Ready = true) + val events = Events() + val session = session(host, platform, "voice-test-g2") + session.setListener(events) + session.setBeamFilter(true, 0, 30) + session.start("cloud", 7) + val deadline = platform.elapsedRealtimeMs() + 3000 + while (host.packetListener == null && platform.elapsedRealtimeMs() < deadline) sleepMs(5) + val listener = host.packetListener!! + listener.onAudioPacket(packet(1, 10, 5), "L", 1000) + listener.onAudioPacket(packet(2, 90, 5), "L", 1050) // outside the beam: meta only + listener.onAudioPacket(packet(3, 0, 0), "R", 1200) // wrong arm, late (150 ms gap), ssr 0 passes the gate + listener.onAudioPacket(ByteArray(3), "L", 1230) // bad length: dropped by the decoder + val waitUntil = platform.elapsedRealtimeMs() + 3000 + while (events.pcmChunks.size < 2 && platform.elapsedRealtimeMs() < waitUntil) sleepMs(5) + session.stop() + assertTrue(events.stoppedLatch.await(3000)) + val stats = session.audioStats() + assertEquals(4L, stats.queuedPackets) + assertEquals(1L, stats.wrongArmPackets) + assertEquals(1L, stats.latePackets) + assertEquals(150L, stats.maxInterPacketMs) + assertEquals(2, events.pcmChunks.size) + assertEquals(Lc3PacketFramer.SAMPLES_PER_PACKET * 2, events.pcmChunks[0].size) + assertEquals(listOf(10 to 5, 90 to 5, 0 to 0), events.frameMeta) + assertEquals(listOf(7), events.stopped) + assertTrue(host.g2Stops >= 1) + assertTrue(events.statuses.contains("Listening (cloud)...")) + assertTrue(events.transcripts.isEmpty()) + assertTrue(logs.any { it.startsWith("I G2 mic packets=4 decoded=3") && it.contains("wrongArm=1") }, logs.toString()) + } + + @Test + fun startRefusesWithoutGlassesAndWhenAlreadyRunning() { + val host = FakeHost(testPlatform(), g2Ready = false) + val events = Events() + val session = session(host, testPlatform(), "voice-test-refuse") + session.setListener(events) + session.start("onboard", 3) + assertEquals>(listOf("Voice control needs an active G2 connection."), events.statuses) + assertEquals(listOf(3), events.stopped) + assertFalse(session.isCapturing()) + } + + @Test + fun onboardTranscriptCommitsSegmentsAndFinalizesWithReplaceSemantics() { + val platform = ShiftedPlatform(testPlatform()) + // 170 chunks of 800 samples = 8.5 s: one 8 s segment commit plus a tail. + val mic = ScriptedMic(List(170) { 3000 }) { platform.offsetMs += 1000 } + val transcriber = FakeTranscriber { "x" } + val host = FakeHost(platform, mic = mic, transcriber = transcriber) + val events = Events() + val session = session(host, platform, "voice-test-onboard") + session.setListener(events) + session.setUsePhoneMic(true) + session.setSaveRecordings(true) + session.start("onboard", 1) + assertTrue(events.stoppedLatch.await(10000)) + assertTrue(mic.closed) + assertTrue(transcriber.released) + assertEquals("Loading transcription model...", events.statuses[0]) + assertEquals("Listening...", events.statuses[1]) + val finals = events.transcripts.filter { it.second } + assertEquals(1, finals.size) + assertEquals("x x", finals.single().first) + assertEquals("x" to false, events.transcripts.first()) + assertTrue(events.transcripts.any { it == ("x x" to false) }) + // Every segment handed to the model is at most 8 s and was peak-normalized. + assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES }) + assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString()) + assertEquals(1, host.recordings.size) + assertEquals(170 * 800 * 2, host.recordings[0].size) + assertEquals(170, events.pcmChunks.size) + assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x")) + assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y")) + } + + @Test + fun endpointingFiresOnceAfterSpeechThenSilence() { + val platform = ShiftedPlatform(testPlatform()) + val script = List(8) { 0 } + List(10) { 3000 } + List(24) { 0 } + val host = FakeHost(platform, mic = ScriptedMic(script)) + val events = Events() + val session = session(host, platform, "voice-test-endpoint") + session.setListener(events) + session.setUsePhoneMic(true) + session.setEndpointing(true) + session.start("cloud", 2) + assertTrue(events.stoppedLatch.await(10000)) + assertEquals(1, events.speechEnds) + assertNull(events.verified) + } + + @Test + fun speakerVerificationFailsOpenAndComparesEmbeddings() { + fun run(embedding: FloatArray?, chunks: Int): Pair? { + val platform = ShiftedPlatform(testPlatform()) + val host = FakeHost(platform, mic = ScriptedMic(List(chunks) { 1000 }), embedding = embedding) + val events = Events() + val session = session(host, platform, "voice-test-verify") + session.setListener(events) + session.setUsePhoneMic(true) + session.setSpeakerVerification("model", floatArrayOf(1f, 0f), 0.8f) + session.start("cloud", 4) + assertTrue(events.stoppedLatch.await(10000)) + return events.verified + } + assertEquals?>(true to 0f, run(floatArrayOf(1f, 0f), 10)) // under 1 s: too short, fails open + assertEquals?>(true to 0f, run(null, 30)) // no embedding: fails open + assertEquals?>(true to 0f, run(floatArrayOf(1f), 30)) // dimension mismatch: fails open + assertEquals?>(true to 1f, run(floatArrayOf(1f, 0f), 30)) + assertEquals?>(false to 0f, run(floatArrayOf(0f, 1f), 30)) + } + + @Test + fun missingModelReportsAndStops() { + val host = FakeHost(testPlatform(), mic = ScriptedMic(emptyList()), hasModel = false) + val events = Events() + val session = session(host, testPlatform(), "voice-test-nomodel") + session.setListener(events) + session.setUsePhoneMic(true) + session.start("onboard", 9) + assertTrue(events.stoppedLatch.await(5000)) + assertEquals>(listOf("Voice model not downloaded (see Settings > Voice)."), events.statuses) + assertEquals(listOf(9), events.stopped) + assertEquals("1.50", formatFixed(1.5, 2)) + assertEquals("-0.333", formatFixed(-1.0 / 3, 3)) + assertEquals("0", formatFixed(0.4, 0)) + } +}