Convert most Android-specific kotlin to shared or shareable

This commit is contained in:
James Babcock committed 2026-09-22 18:43:55 -07:00
1 parent f9724ddcab
commit 587364e5e6
92 files changed
+14347 -9193

No files matched your search

@@ -0,0 +1,41 @@
package com.faceclaw.app
import android.bluetooth.BluetoothGatt
/** [SessionLink] over the Android GATT boundary ([FaceclawBleManager]). */
class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLink {
override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs)
override fun requestHighPriority(address: String) {
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
}
override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean =
bleManager.requestMtu(address, mtu, timeoutMs)
override fun negotiatedMtu(address: String): Int = bleManager.getNegotiatedMtu(address)
override fun discoverServices(address: String, timeoutMs: Int): Boolean =
bleManager.discoverServices(address, timeoutMs)
override fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean =
bleManager.enableNotifications(address, characteristicUuid, enable, timeoutMs)
override fun writeFrames(
address: String,
characteristicUuid: String,
frames: List<ByteArray>,
mode: GattWriteMode,
timeoutMs: Int,
): Boolean = bleManager.writeFrames(address, characteristicUuid, frames, AndroidProtocolPlatform.writeType(mode), timeoutMs)
override fun disconnect(address: String) = bleManager.disconnect(address)
override fun close() = bleManager.close()
override fun isBonded(address: String): Boolean = bleManager.isBonded(address)
override fun prepareBenchmarkLink(address: String, mode: Int) = bleManager.prepareBenchmarkLink(address, mode)
override fun recordDisplayFrameSent() = FaceclawBleManager.recordDisplayFrameSent()
}
@@ -0,0 +1,147 @@
package com.faceclaw.app
import android.util.Log
import com.k2fsa.sherpa.onnx.FeatureConfig
import com.k2fsa.sherpa.onnx.OfflineModelConfig
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizer
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult
import com.k2fsa.sherpa.onnx.OfflineStream
import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig
import java.io.File
/**
* Android implementations of the shared voice ports: the sherpa-onnx offline recognizer as an
* [OfflineTranscriber], the phone microphone as a [PcmAudioSource], and the JNI LC3 decoder as
* an [AudioPacketDecoder]. The pipelines themselves live in the shared VoiceCaptureSession and
* CaptionSession.
*/
internal object AndroidSpeechEngines {
private const val TAG = "FaceclawVoice"
private const val SAMPLE_RATE = 16000
private const val FEATURE_DIM = 80
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
val modelConfig = OfflineModelConfig.builder()
.setNumThreads(1)
if (kind == VoiceModelKind.WHISPER) {
modelConfig
.setWhisper(OfflineWhisperModelConfig.builder()
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
.setLanguage("en")
.setTask("transcribe")
.build())
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
} else {
modelConfig
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(File(modelDir, "encoder_model.ort").absolutePath)
.setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath)
.build())
.setTokens(File(modelDir, "tokens.txt").absolutePath)
}
return OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(modelConfig.build())
.build()
}
class SherpaTranscriber(config: OfflineRecognizerConfig) : OfflineTranscriber {
private var recognizer: OfflineRecognizer? = OfflineRecognizer(config)
override fun recognize(samples: FloatArray, count: Int): String {
val currentRecognizer = recognizer ?: return ""
val segment = if (count == samples.size) samples else samples.copyOf(count)
val offlineStream: OfflineStream = currentRecognizer.createStream()
try {
offlineStream.acceptWaveform(segment, SAMPLE_RATE)
currentRecognizer.decode(offlineStream)
val result: OfflineRecognizerResult? = currentRecognizer.getResult(offlineStream)
val raw: String? = if (result == null) "" else result.text
return raw?.trim() ?: ""
} finally {
offlineStream.release()
}
}
override fun release() {
recognizer?.release()
recognizer = null
}
}
/** The JNI LC3 decoder behind the shared packet-decoder port. */
class Lc3DecoderAdapter : AudioPacketDecoder {
private val decoder = FaceclawLc3Decoder()
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int = decoder.decodePacket(packet, pcmOut)
override val lastAngleDegrees: Int get() = decoder.getLastAngleDegrees()
override val lastSsr: Int get() = decoder.getLastSsr()
override val realPackets: Long get() = decoder.getRealPackets()
override val duplicatePackets: Long get() = decoder.getDuplicatePackets()
override val missingPackets: Long get() = decoder.getMissingPackets()
override val decodeErrors: Long get() = decoder.getDecodeErrors()
override fun close() = decoder.close()
}
/**
* Open the phone's own microphone at the pipeline's native format (16 kHz
* mono PCM16), or null when it cannot start — the permission is missing
* (SecurityException) or the device refuses the configuration.
*/
fun openPhoneMic(chunkSamples: Int): PcmAudioSource? {
var record: android.media.AudioRecord? = null
try {
val minBytes = android.media.AudioRecord.getMinBufferSize(
SAMPLE_RATE,
android.media.AudioFormat.CHANNEL_IN_MONO,
android.media.AudioFormat.ENCODING_PCM_16BIT)
val bufferBytes = Math.max(minBytes, chunkSamples * 2 * 4)
record = android.media.AudioRecord(
android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION,
SAMPLE_RATE,
android.media.AudioFormat.CHANNEL_IN_MONO,
android.media.AudioFormat.ENCODING_PCM_16BIT,
bufferBytes)
if (record.state != android.media.AudioRecord.STATE_INITIALIZED) {
record.release()
return null
}
record.startRecording()
if (record.recordingState != android.media.AudioRecord.RECORDSTATE_RECORDING) {
record.release()
return null
}
return AudioRecordSource(record)
} catch (t: Throwable) {
Log.w(TAG, "phone mic open failed", t)
if (record != null) {
record.release()
}
return null
}
}
private class AudioRecordSource(private val record: android.media.AudioRecord) : PcmAudioSource {
override fun read(pcm: ShortArray): Int = record.read(pcm, 0, pcm.size)
override fun close() {
try {
record.stop()
} catch (ignored: Throwable) {
// Already stopped or never recording; release below either way.
}
record.release()
}
}
}
@@ -0,0 +1,64 @@
package com.faceclaw.app
import android.bluetooth.BluetoothDevice
import android.bluetooth.BluetoothGatt
import android.content.Context
/**
* Android [StockLink]: a 1:1 wrapper over FaceclawBleManager for the stock-firmware flows
* (device-info probe, flash prompt, OTA flasher). Owns its own GATT manager, exactly as each
* flow did before the shared cores existed.
*/
class AndroidStockLink(context: Context) : StockLink, FaceclawBleListener {
private val bleManager = FaceclawBleManager(context.applicationContext)
@Volatile
private var listener: StockLinkListener? = null
init {
bleManager.setListener(this)
}
override fun setListener(listener: StockLinkListener?) {
this.listener = listener
}
override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs)
override fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int) {
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, desiredMtu, timeoutMs)
}
override fun discoverServices(address: String, timeoutMs: Int): Boolean = bleManager.discoverServices(address, timeoutMs)
override fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean =
bleManager.enableNotifications(address, characteristicUuid, true, timeoutMs)
override fun writeFrames(address: String, characteristicUuid: String, frames: List<ByteArray>, mode: GattWriteMode, timeoutMs: Int): Boolean =
bleManager.writeFrames(address, characteristicUuid, frames, AndroidProtocolPlatform.writeType(mode), timeoutMs)
override fun isConnected(address: String): Boolean = bleManager.isConnected(address)
override fun bondState(address: String): BondState =
when (bleManager.getBondState(address)) {
BluetoothDevice.BOND_NONE -> BondState.NONE
BluetoothDevice.BOND_BONDING -> BondState.BONDING
BluetoothDevice.BOND_BONDED -> BondState.BONDED
else -> BondState.UNKNOWN
}
override fun disconnect(address: String) = bleManager.disconnect(address)
override fun close() = bleManager.close()
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (address == null || characteristicUuid == null || data == null) return
listener?.onNotification(address, characteristicUuid, data)
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (address == null) return
listener?.onConnectionStateChange(address, connected)
}
}
@@ -22,9 +22,6 @@ import android.os.PowerManager
import android.os.VibrationEffect
import android.os.Vibrator
import java.util.ArrayList
import java.util.Collections
import java.util.LinkedHashMap
/**
* The ringing foreground service. One instance carries every item currently
@@ -37,22 +34,14 @@ import java.util.LinkedHashMap
* to total silence) Do Not Disturb do not silence it.
*/
class FaceclawAlarmService : Service() {
/** The shared item model; kept as a nested subclass so the lock-screen activity's references hold. */
internal class Ringing(
@JvmField val id: Long,
id: Long,
title: String?,
text: String?,
kind: String?,
snoozeMinutes: Int
) {
@JvmField val title: String = if (title == null || title.trim().isEmpty()) "Alarm" else title
@JvmField val text: String = text ?: ""
@JvmField val kind: String = kind ?: FaceclawAlarms.KIND_ALARM
@JvmField val snoozeMinutes: Int = Math.max(1, snoozeMinutes)
@JvmField val ringAtMs: Long = System.currentTimeMillis()
@JvmField var deliveredAtMs: Long = 0
@JvmField var escalated: Boolean = false
@JvmField var silenced: Boolean = false
}
) : RingingItem(id, title, text, kind, snoozeMinutes, System.currentTimeMillis())
companion object {
const val CHANNEL_ID = "faceclaw-alarms"
@@ -72,17 +61,11 @@ class FaceclawAlarmService : Service() {
/** Alarm-stream level (fraction of max) used when the wearer has it muted. */
internal const val MUTED_VOLUME_FRACTION = 0.6f
/** Ringing items, oldest first; static so the activity can read them. */
private val ringing: MutableMap<Long, Ringing> = Collections.synchronizedMap(LinkedHashMap())
/**
* Signals that arrived before the ring intent was processed (the JS
* engine fires, launches its window and reports delivery on the same
* main thread the service's onStartCommand queues behind), keyed by id
* with their arrival time. Consumed by startRinging.
* The escalation policy is process-wide (the activity reads it, the static entry points
* consult it before the service exists); the live service attaches its timers/effects.
*/
private val deliveredEarly: MutableMap<Long, Long> = Collections.synchronizedMap(LinkedHashMap())
private val stoppedEarly: MutableMap<Long, Long> = Collections.synchronizedMap(LinkedHashMap())
private const val EARLY_SIGNAL_MAX_AGE_MS = 15_000L
private val policy = AlarmRingingPolicy(FaceclawAlarms.glassesGate)
// ------------------------------------------------------------------
// Static entry points
@@ -102,8 +85,7 @@ class FaceclawAlarmService : Service() {
@JvmStatic
fun delivered(context: Context, id: Long) {
if (!ringing.containsKey(id)) {
deliveredEarly[id] = System.currentTimeMillis()
if (!policy.deliveredOrDefer(id)) {
return
}
val appContext = context.applicationContext
@@ -114,8 +96,7 @@ class FaceclawAlarmService : Service() {
@JvmStatic
fun stopItem(context: Context, id: Long) {
if (!ringing.containsKey(id)) {
stoppedEarly[id] = System.currentTimeMillis()
if (!policy.stopOrDefer(id)) {
return
}
val appContext = context.applicationContext
@@ -125,21 +106,13 @@ class FaceclawAlarmService : Service() {
}
@JvmStatic
fun isRinging(id: Long): Boolean {
return ringing.containsKey(id)
}
fun isRinging(id: Long): Boolean = policy.isRinging(id)
@JvmStatic
fun isAnythingRinging(): Boolean {
return !ringing.isEmpty()
}
fun isAnythingRinging(): Boolean = policy.isAnythingRinging()
@JvmStatic
internal fun snapshot(): List<Ringing> {
synchronized(ringing) {
return ArrayList(ringing.values)
}
}
internal fun snapshot(): List<Ringing> = policy.snapshot().map { it as Ringing }
@JvmStatic
internal fun actionIntent(appContext: Context, action: String, id: Long): PendingIntent {
@@ -157,16 +130,6 @@ class FaceclawAlarmService : Service() {
return PendingIntent.getService(appContext, code, intent, flags)
}
/** Consume an early signal for the id; stale entries are dropped. */
private fun takeEarlySignal(signals: MutableMap<Long, Long>, id: Long): Boolean {
val at = signals.remove(id)
val now = System.currentTimeMillis()
synchronized(signals) {
signals.values.removeIf { stamp -> now - stamp > EARLY_SIGNAL_MAX_AGE_MS }
}
return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS
}
private fun notificationId(id: Long): Int {
return 0x41000000 or (FaceclawAlarms.requestCode(id) and 0x00ffffff)
}
@@ -188,6 +151,11 @@ class FaceclawAlarmService : Service() {
return null
}
override fun onCreate() {
super.onCreate()
policy.attach(timers, effects)
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
val action = intent?.action
val id = if (intent == null) 0L else intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0)
@@ -206,13 +174,13 @@ class FaceclawAlarmService : Service() {
intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10)
))
ACTION_DELIVERED ->
markDelivered(id)
policy.markDelivered(id)
ACTION_DISMISS ->
dismiss(id)
policy.dismiss(id)
ACTION_SNOOZE ->
snooze(id)
policy.snooze(id)
ACTION_STOP_ITEM ->
stopItemQuietly(id, "acknowledged")
policy.stopQuietly(id, "acknowledged")
else -> {}
}
finishIfIdle()
@@ -220,6 +188,7 @@ class FaceclawAlarmService : Service() {
}
override fun onDestroy() {
policy.detach()
handler.removeCallbacksAndMessages(null)
stopSound()
releaseWakeLock()
@@ -234,120 +203,69 @@ class FaceclawAlarmService : Service() {
return
}
ensureChannel()
val existing = ringing[item.id]
if (existing != null) {
// The engine's JS timeout and the AlarmManager alarm both fired: one ring.
showNotification(existing)
return
}
if (takeEarlySignal(stoppedEarly, item.id)) {
// Acknowledged or cancelled before this intent was processed.
FaceclawAlarms.log(this, "ring " + item.id + " already acknowledged")
return
}
ringing[item.id] = item
FaceclawAlarms.log(this, "ring " + item.kind + " " + item.id + " (" + FaceclawAlarms.glassesStatusDescription() + ")")
acquireWakeLock()
showNotification(item)
if (!FaceclawAlarms.glassesCanCarryAlarm()) {
escalate(item, FaceclawAlarms.glassesStatusDescription())
return
}
if (takeEarlySignal(deliveredEarly, item.id)) {
markDelivered(item.id)
return
}
// The glasses could carry it: give the JS side a moment to confirm
// they are actually showing it, then hold for the acknowledgement.
handler.postDelayed({
val current = ringing[item.id]
if (current != null && !current.escalated && current.deliveredAtMs == 0L) {
escalate(current, "not delivered to the glasses")
policy.ring(item)
}
/** Escalation timers on the service's main-thread Handler. */
private val timers = object : AlarmScheduler {
private val pending = HashMap<Any, Runnable>()
override fun postDelayed(token: Any, delayMs: Long, action: () -> Unit) {
pending.remove(token)?.let { handler.removeCallbacks(it) }
val runnable = Runnable {
pending.remove(token)
action()
}
}, DELIVERY_WAIT_MS)
pending[token] = runnable
handler.postDelayed(runnable, delayMs)
}
override fun cancel(token: Any) {
pending.remove(token)?.let { handler.removeCallbacks(it) }
}
override fun cancelAll() {
pending.clear()
handler.removeCallbacksAndMessages(null)
}
}
private fun markDelivered(id: Long) {
val item = ringing[id]
if (item == null || item.deliveredAtMs != 0L) {
return
}
item.deliveredAtMs = System.currentTimeMillis()
FaceclawAlarms.log(this, "delivered $id to glasses")
handler.postDelayed({
val current = ringing[id]
if (current != null && !current.escalated) {
escalate(current, "not acknowledged on the glasses")
/** The phone-side effects of the shared policy: notifications, sound, schedule, journal. */
private val effects = object : RingingSink {
override fun showNotification(item: RingingItem) = this@FaceclawAlarmService.showNotification(item as Ringing)
override fun itemRemoved(item: RingingItem) {
val manager = getSystemService(NotificationManager::class.java)
manager?.cancel(notificationId(item.id))
if (item.id == foregroundId) {
promoteAnotherToForeground()
}
}, ACK_WAIT_MS)
}
private fun escalate(item: Ringing, reason: String) {
if (item.escalated) {
return
}
item.escalated = true
FaceclawAlarms.log(this, "phone sound for " + item.id + ": " + reason)
showNotification(item)
startSound()
handler.postDelayed({
val current = ringing[item.id]
if (current != null && current.escalated && !current.silenced) {
current.silenced = true
FaceclawAlarms.log(this, "auto-silenced " + current.id)
showNotification(current)
syncSound()
}
}, AUTO_SILENCE_MS)
}
private fun dismiss(id: Long) {
val item = ringing[id] ?: return
FaceclawAlarms.recordPhoneAction(this, id, "dismiss", 0)
removeItem(item)
// Also drop the schedule entry (a one-off is over; the engine re-arms repeats).
FaceclawAlarms.cancel(this, id)
}
override fun ringingStarted() = acquireWakeLock()
private fun snooze(id: Long) {
val item = ringing[id] ?: return
val minutes = item.snoozeMinutes
FaceclawAlarms.recordPhoneAction(this, id, "snooze", minutes)
removeItem(item)
// Re-arm on the phone directly so the snooze holds even with the JS
// side gone; the engine replays the journal and lands on the same id.
FaceclawAlarms.schedule(
this,
id,
System.currentTimeMillis() + minutes * 60_000L,
item.title,
item.text,
item.kind,
minutes
)
}
override fun startSound() = this@FaceclawAlarmService.startSound()
private fun stopItemQuietly(id: Long, reason: String) {
val item = ringing[id] ?: return
FaceclawAlarms.log(this, "$reason $id")
removeItem(item)
}
override fun stopSound() = this@FaceclawAlarmService.stopSound()
private fun removeItem(item: Ringing) {
ringing.remove(item.id)
val manager = getSystemService(NotificationManager::class.java)
manager?.cancel(notificationId(item.id))
syncSound()
if (item.id == foregroundId) {
promoteAnotherToForeground()
override fun reschedule(item: RingingItem, atMs: Long, minutes: Int) {
// The engine replays the journal and lands on the same id.
FaceclawAlarms.schedule(this@FaceclawAlarmService, item.id, atMs, item.title, item.text, item.kind, minutes)
}
override fun cancelSchedule(id: Long) = FaceclawAlarms.cancel(this@FaceclawAlarmService, id)
override fun recordPhoneAction(id: Long, action: String, minutes: Int) =
FaceclawAlarms.recordPhoneAction(this@FaceclawAlarmService, id, action, minutes)
override fun log(line: String) = FaceclawAlarms.log(this@FaceclawAlarmService, line)
}
private fun finishIfIdle() {
if (!ringing.isEmpty()) {
if (policy.isAnythingRinging()) {
return
}
handler.removeCallbacksAndMessages(null)
policy.cancelTimers()
stopSound()
releaseWakeLock()
stopForeground(STOP_FOREGROUND_REMOVE)
@@ -359,7 +277,7 @@ class FaceclawAlarmService : Service() {
private fun showNotification(item: Ringing) {
val notification = buildNotification(item)
if (foregroundId == 0L || foregroundId == item.id || !ringing.containsKey(foregroundId)) {
if (foregroundId == 0L || foregroundId == item.id || !policy.isRinging(foregroundId)) {
foregroundId = item.id
startForegroundCompat(notificationId(item.id), notification)
} else {
@@ -456,21 +374,6 @@ class FaceclawAlarmService : Service() {
// ------------------------------------------------------------------
// Sound and vibration (only while some item is escalated and not yet silenced)
private fun syncSound() {
var wanted = false
for (item in snapshot()) {
if (item.escalated && !item.silenced) {
wanted = true
break
}
}
if (wanted) {
startSound()
} else {
stopSound()
}
}
private fun startSound() {
if (player != null) {
return
@@ -47,32 +47,19 @@ class FaceclawAlarms private constructor() {
internal const val EXTRA_KIND = "kind"
internal const val EXTRA_SNOOZE_MINUTES = "snoozeMinutes"
const val KIND_TIMER = "timer"
const val KIND_ALARM = "alarm"
const val KIND_TIMER = AlarmSchedule.KIND_TIMER
const val KIND_ALARM = AlarmSchedule.KIND_ALARM
private const val PREFS_NAME = "faceclaw-alarms"
private const val KEY_SCHEDULED = "scheduled"
private const val KEY_JOURNAL = "journal"
private const val KEY_LOG = "log"
private const val LOG_LINES = 60
/**
* A schedule entry found already past due (after a reboot, or when the
* process was asleep) rings if it is at most this late; older ones are
* dropped as missed. Matches the JS engine's grace window.
*/
internal const val LATE_GRACE_MS = 5 * 60_000L
/** A glasses status report older than this is not trusted (the JS side is probably gone). */
internal const val GLASSES_STATUS_MAX_AGE_MS = 3 * 60_000L
/** Grace window for past-due entries found on replay (shared policy). */
internal const val LATE_GRACE_MS = AlarmSchedule.LATE_GRACE_MS
/** A glasses status report older than this is not trusted (shared policy). */
internal const val GLASSES_STATUS_MAX_AGE_MS = GlassesStatusGate.MAX_AGE_MS
@Volatile
private var glassesConnected = false
@Volatile
private var glassesWorn = false
@Volatile
private var glassesCharging = false
@Volatile
private var glassesStatusAtMs = 0L
/** The last glasses status pushed by the JS side; read by the ringing service. */
@JvmField
internal val glassesGate = GlassesStatusGate()
@Volatile
private var listener: FaceclawAlarmListener? = null
@@ -95,66 +82,30 @@ class FaceclawAlarms private constructor() {
return base.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
}
// The Java statics were `synchronized`, i.e. locked on the class object;
// the explicit blocks below keep that single monitor.
private fun readArray(context: Context, key: String): JSONArray {
synchronized(FaceclawAlarms::class.java) {
val raw = prefs(context).getString(key, "[]")
return try {
JSONArray(raw)
} catch (error: JSONException) {
JSONArray()
}
}
}
@Volatile
private var scheduleInstance: AlarmSchedule? = null
private fun writeArray(context: Context, key: String, array: JSONArray) {
/** The shared schedule/journal/log core over the device-protected prefs (one per process). */
@JvmStatic
internal fun scheduleCore(context: Context): AlarmSchedule {
scheduleInstance?.let { return it }
synchronized(FaceclawAlarms::class.java) {
prefs(context).edit().putString(key, array.toString()).apply()
scheduleInstance?.let { return it }
val prefs = prefs(context)
val created = AlarmSchedule(object : AlarmStore {
override fun read(key: String): String? = prefs.getString(key, null)
override fun write(key: String, value: String?) {
prefs.edit().putString(key, value).apply()
}
})
scheduleInstance = created
return created
}
}
@JvmStatic
internal fun findScheduled(context: Context, id: Long): JSONObject? {
val scheduled = readArray(context, KEY_SCHEDULED)
for (index in 0 until scheduled.length()) {
val entry = scheduled.optJSONObject(index)
if (entry != null && entry.optLong("id") == id) {
return entry
}
}
return null
}
private fun putScheduled(context: Context, entry: JSONObject) {
synchronized(FaceclawAlarms::class.java) {
val scheduled = readArray(context, KEY_SCHEDULED)
val next = JSONArray()
val id = entry.optLong("id")
for (index in 0 until scheduled.length()) {
val existing = scheduled.optJSONObject(index)
if (existing != null && existing.optLong("id") != id) {
next.put(existing)
}
}
next.put(entry)
writeArray(context, KEY_SCHEDULED, next)
}
}
private fun removeScheduled(context: Context, id: Long) {
synchronized(FaceclawAlarms::class.java) {
val scheduled = readArray(context, KEY_SCHEDULED)
val next = JSONArray()
for (index in 0 until scheduled.length()) {
val existing = scheduled.optJSONObject(index)
if (existing != null && existing.optLong("id") != id) {
next.put(existing)
}
}
writeArray(context, KEY_SCHEDULED, next)
}
}
internal fun findScheduled(context: Context, id: Long): AlarmEntry? = scheduleCore(context).find(id)
// ------------------------------------------------------------------
// Scheduling
@@ -177,26 +128,16 @@ class FaceclawAlarms private constructor() {
snoozeMinutes: Int
) {
val appContext = context.applicationContext
val entry = JSONObject()
try {
entry.put("id", id)
entry.put("at", triggerAtMs)
entry.put("title", title ?: "")
entry.put("text", text ?: "")
entry.put("kind", kind ?: KIND_ALARM)
entry.put("snoozeMinutes", snoozeMinutes)
} catch (ignored: JSONException) {
return
}
putScheduled(appContext, entry)
val entry = AlarmEntry(id, triggerAtMs, title, text, kind ?: KIND_ALARM, snoozeMinutes)
scheduleCore(appContext).put(entry)
armAlarm(appContext, entry)
}
private fun armAlarm(appContext: Context, entry: JSONObject) {
private fun armAlarm(appContext: Context, entry: AlarmEntry) {
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
?: return
val id = entry.optLong("id")
val triggerAt = Math.max(System.currentTimeMillis(), entry.optLong("at"))
val id = entry.id
val triggerAt = Math.max(System.currentTimeMillis(), entry.at)
val fire = expiryPendingIntent(appContext, entry)
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
@@ -219,15 +160,9 @@ class FaceclawAlarms private constructor() {
val appContext = context.applicationContext
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
if (manager != null) {
val entry = JSONObject()
try {
entry.put("id", id)
} catch (ignored: JSONException) {
// unreachable for a numeric put
}
manager.cancel(expiryPendingIntent(appContext, entry))
manager.cancel(expiryPendingIntent(appContext, AlarmEntry(id, 0L, null, null, null, 10)))
}
removeScheduled(appContext, id)
scheduleCore(appContext).remove(id)
FaceclawAlarmService.stopItem(appContext, id)
}
@@ -239,25 +174,7 @@ class FaceclawAlarms private constructor() {
@JvmStatic
fun rescheduleAll(context: Context, reason: String?) {
val appContext = context.applicationContext
val scheduled = readArray(appContext, KEY_SCHEDULED)
val now = System.currentTimeMillis()
var armed = 0
for (index in 0 until scheduled.length()) {
val entry = scheduled.optJSONObject(index) ?: continue
val at = entry.optLong("at")
if (at <= now) {
if (now - at <= LATE_GRACE_MS) {
ringEntry(appContext, entry)
} else {
log(appContext, "missed " + entry.optString("kind") + " " + entry.optLong("id") + " (" + reason + ")")
removeScheduled(appContext, entry.optLong("id"))
}
continue
}
armAlarm(appContext, entry)
armed++
}
log(appContext, "rescheduled $armed ($reason)")
scheduleCore(appContext).replay(reason, { ringEntry(appContext, it) }, { armAlarm(appContext, it) })
}
/**
@@ -271,24 +188,13 @@ class FaceclawAlarms private constructor() {
return false
}
val appContext = context.applicationContext
val scheduled = readArray(appContext, KEY_SCHEDULED)
val now = System.currentTimeMillis()
var soonest = Long.MAX_VALUE
for (index in 0 until scheduled.length()) {
val entry = scheduled.optJSONObject(index)
if (entry != null && entry.optLong("at") > now) {
soonest = Math.min(soonest, entry.optLong("at"))
}
}
if (soonest == Long.MAX_VALUE) {
val core = scheduleCore(appContext)
if (core.soonestPendingAt() == null) {
return false
}
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
?: return false
val next = manager.nextAlarmClock
// Another app's alarm may legitimately be sooner; only a later (or
// absent) system alarm proves ours is missing.
return next == null || next.triggerTime > soonest + 1000
return core.looksStale(manager.nextAlarmClock?.triggerTime)
}
// ------------------------------------------------------------------
@@ -296,15 +202,8 @@ class FaceclawAlarms private constructor() {
/** An item came due: hand it to the ringing service (idempotent per id). */
@JvmStatic
internal fun ringEntry(appContext: Context, entry: JSONObject) {
FaceclawAlarmService.ring(
appContext,
entry.optLong("id"),
entry.optString("title"),
entry.optString("text"),
entry.optString("kind", KIND_ALARM),
entry.optInt("snoozeMinutes", 10)
)
internal fun ringEntry(appContext: Context, entry: AlarmEntry) {
FaceclawAlarmService.ring(appContext, entry.id, entry.title, entry.text, entry.kind, entry.snoozeMinutes)
}
/** The JS engine's own expiry path; the service deduplicates against the alarm's. */
@@ -313,13 +212,7 @@ class FaceclawAlarms private constructor() {
val appContext = context.applicationContext
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
if (manager != null) {
val probe = JSONObject()
try {
probe.put("id", id)
} catch (ignored: JSONException) {
// unreachable for a numeric put
}
manager.cancel(expiryPendingIntent(appContext, probe))
manager.cancel(expiryPendingIntent(appContext, AlarmEntry(id, 0L, null, null, null, 10)))
}
FaceclawAlarmService.ring(appContext, id, title, text, kind, snoozeMinutes)
}
@@ -334,7 +227,7 @@ class FaceclawAlarms private constructor() {
@JvmStatic
fun acknowledge(context: Context, id: Long) {
val appContext = context.applicationContext
removeScheduled(appContext, id)
scheduleCore(appContext).remove(id)
FaceclawAlarmService.stopItem(appContext, id)
}
@@ -343,42 +236,15 @@ class FaceclawAlarms private constructor() {
@JvmStatic
fun setGlassesStatus(connected: Boolean, worn: Boolean, charging: Boolean) {
glassesConnected = connected
glassesWorn = worn
glassesCharging = charging
glassesStatusAtMs = System.currentTimeMillis()
glassesGate.set(connected, worn, charging)
}
/** True only when a fresh report says the glasses are connected, on a head, and not charging. */
@JvmStatic
internal fun glassesCanCarryAlarm(): Boolean {
val age = System.currentTimeMillis() - glassesStatusAtMs
if (glassesStatusAtMs == 0L || age > GLASSES_STATUS_MAX_AGE_MS) {
return false
}
return glassesConnected && glassesWorn && !glassesCharging
}
internal fun glassesCanCarryAlarm(): Boolean = glassesGate.canCarryAlarm()
@JvmStatic
internal fun glassesStatusDescription(): String {
if (glassesStatusAtMs == 0L) {
return "no glasses status"
}
val age = System.currentTimeMillis() - glassesStatusAtMs
if (age > GLASSES_STATUS_MAX_AGE_MS) {
return "glasses status stale"
}
if (!glassesConnected) {
return "glasses not connected"
}
if (glassesCharging) {
return "glasses charging"
}
if (!glassesWorn) {
return "glasses not worn"
}
return "glasses worn"
}
internal fun glassesStatusDescription(): String = glassesGate.description()
// ------------------------------------------------------------------
// Phone actions -> JS
@@ -396,21 +262,7 @@ class FaceclawAlarms private constructor() {
@JvmStatic
internal fun recordPhoneAction(context: Context, id: Long, action: String, minutes: Int) {
val appContext = context.applicationContext
val event = JSONObject()
try {
event.put("id", id)
event.put("action", action)
event.put("minutes", minutes)
event.put("at", System.currentTimeMillis())
} catch (ignored: JSONException) {
return
}
synchronized(FaceclawAlarms::class.java) {
val journal = readArray(appContext, KEY_JOURNAL)
journal.put(event)
writeArray(appContext, KEY_JOURNAL, journal)
}
log(appContext, "phone " + action + " " + id + (if (minutes > 0) " ($minutes min)" else ""))
scheduleCore(appContext).appendJournal(id, action, minutes)
val current = listener
if (current != null) {
mainHandler.post {
@@ -425,14 +277,7 @@ class FaceclawAlarms private constructor() {
/** Hand the journal to the JS side (as a JSON array) and clear it. */
@JvmStatic
fun drainJournal(context: Context): String {
synchronized(FaceclawAlarms::class.java) {
val appContext = context.applicationContext
val journal = readArray(appContext, KEY_JOURNAL)
writeArray(appContext, KEY_JOURNAL, JSONArray())
return journal.toString()
}
}
fun drainJournal(context: Context): String = scheduleCore(context.applicationContext).drainJournal()
// ------------------------------------------------------------------
// Reliability self-check
@@ -577,45 +422,25 @@ class FaceclawAlarms private constructor() {
@JvmStatic
internal fun log(context: Context, line: String) {
synchronized(FaceclawAlarms::class.java) {
val appContext = context.applicationContext
val log = readArray(appContext, KEY_LOG)
val next = JSONArray()
val start = Math.max(0, log.length() - (LOG_LINES - 1))
for (index in start until log.length()) {
next.put(log.optString(index))
}
next.put(System.currentTimeMillis().toString() + " " + line)
writeArray(appContext, KEY_LOG, next)
}
scheduleCore(context.applicationContext).log(line)
}
/** The ring / miss / action log, newest last, one line per entry. */
@JvmStatic
fun readLog(context: Context): String {
val log = readArray(context.applicationContext, KEY_LOG)
val out = StringBuilder()
for (index in 0 until log.length()) {
if (index > 0) {
out.append('\n')
}
out.append(log.optString(index))
}
return out.toString()
}
fun readLog(context: Context): String = scheduleCore(context.applicationContext).readLog()
// ------------------------------------------------------------------
// Intents
private fun expiryPendingIntent(appContext: Context, entry: JSONObject): PendingIntent {
private fun expiryPendingIntent(appContext: Context, entry: AlarmEntry): PendingIntent {
val intent = Intent(appContext, FaceclawAlarmReceiver::class.java)
intent.action = ACTION_EXPIRE
val id = entry.optLong("id")
val id = entry.id
intent.putExtra(EXTRA_ID, id)
intent.putExtra(EXTRA_TITLE, entry.optString("title"))
intent.putExtra(EXTRA_TEXT, entry.optString("text"))
intent.putExtra(EXTRA_KIND, entry.optString("kind", KIND_ALARM))
intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.optInt("snoozeMinutes", 10))
intent.putExtra(EXTRA_TITLE, entry.title)
intent.putExtra(EXTRA_TEXT, entry.text)
intent.putExtra(EXTRA_KIND, entry.kind)
intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.snoozeMinutes)
var flags = PendingIntent.FLAG_UPDATE_CURRENT
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
flags = flags or PendingIntent.FLAG_IMMUTABLE
@@ -635,9 +460,7 @@ class FaceclawAlarms private constructor() {
}
@JvmStatic
internal fun requestCode(id: Long): Int {
return (id xor (id ushr 32)).toInt() and 0x7fffffff
}
internal fun requestCode(id: Long): Int = AlarmSchedule.requestCode(id)
@JvmStatic
internal fun startServiceCompat(appContext: Context, intent: Intent) {
@@ -9,8 +9,6 @@ import android.util.Log
import java.io.File
import java.io.FileInputStream
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.ByteOrder
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
@@ -28,69 +26,25 @@ class FaceclawAudioTranscoder(
private const val TAG = "FaceclawTranscode"
private const val CODEC_TIMEOUT_US: Long = 10_000
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
/** Shared RIFF walk over the file's first bytes: locates fmt and data chunks, 16-bit PCM only. */
@JvmStatic
@Throws(IOException::class)
private fun readWavHeader(path: String?): WavInfo {
FileInputStream(path).use { `in` ->
val head = ByteArray(12)
if (`in`.read(head) != 12 || head[0] != 'R'.code.toByte() || head[1] != 'I'.code.toByte() || head[8] != 'W'.code.toByte()) {
throw IOException("not a WAV file")
private fun readWavHeader(path: String?): WavPcmReader.WavInfo {
val file = File(path)
val fileLength = file.length()
FileInputStream(file).use { `in` ->
val prefix = ByteArray(Math.min(fileLength, WavPcmReader.HEADER_PREFIX_BYTES.toLong()).toInt())
var filled = 0
while (filled < prefix.size) {
val read = `in`.read(prefix, filled, prefix.size - filled)
if (read <= 0) break
filled += read
}
val info = WavInfo()
var offset: Long = 12
val chunkHead = ByteArray(8)
while (`in`.read(chunkHead) == 8) {
val b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN)
val id = b.getInt()
val size = b.getInt().toLong() and 0xffffffffL
offset += 8
if (id == 0x20746d66) { // "fmt "
val fmt = ByteArray(Math.min(size, 16L).toInt())
if (`in`.read(fmt) != fmt.size) {
throw IOException("truncated fmt chunk")
}
val f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN)
val audioFormat = f.getShort().toInt()
info.channels = f.getShort().toInt()
info.sampleRate = f.getInt()
f.getInt()
f.getShort()
val bits = f.getShort().toInt()
if (audioFormat != 1 || bits != 16) {
throw IOException("only 16-bit PCM WAV is supported")
}
var skip = size - fmt.size
while (skip > 0) {
val step = `in`.skip(skip)
if (step <= 0) {
break
}
skip -= step
}
offset += size
} else if (id == 0x61746164) { // "data"
info.dataOffset = offset
// A zero size means the header was never patched (crash
// mid-recording); recover the length from the file size.
info.dataBytes = if (size > 0) size else File(path).length() - offset
break
} else {
var skip = size
while (skip > 0) {
val step = `in`.skip(skip)
if (step <= 0) {
break
}
skip -= step
}
offset += size
}
try {
return WavPcmReader.parseHeader(if (filled == prefix.size) prefix else prefix.copyOf(filled), fileLength)
} catch (e: IllegalArgumentException) {
throw IOException(e.message ?: "not a WAV file")
}
if (info.sampleRate == 0 || info.dataOffset == 0L) {
throw IOException("missing fmt or data chunk")
}
return info
}
}
}
@@ -226,11 +180,4 @@ class FaceclawAudioTranscoder(
}
}
}
private class WavInfo {
var sampleRate = 0
var channels = 0
var dataOffset: Long = 0
var dataBytes: Long = 0
}
}
File diff suppressed because it is too large. Load diff
@@ -2,455 +2,74 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import android.util.Log
import com.k2fsa.sherpa.onnx.FeatureConfig
import com.k2fsa.sherpa.onnx.OfflineModelConfig
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizer
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult
import com.k2fsa.sherpa.onnx.OfflineStream
import java.io.File
import java.util.ArrayDeque
import java.util.Arrays
/**
* Continuous captioning over the decoded mic PCM: segments speech into
* utterances with an adaptive energy gate, transcribes each utterance with
* the on-device Moonshine model, and attaches a speaker voice-print per
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
* and any beam-direction gating), so this engine has no BLE dependencies.
* Continuous captioning over the decoded mic PCM: the shared [CaptionSession] segments speech
* into utterances with an adaptive energy gate, transcribes each with the on-device Moonshine
* model and attaches a speaker voice-print. This Android class only supplies the sherpa-onnx
* recognizer and speaker model and delivers callbacks on the main thread. PCM is pushed in
* from the TS side (which owns mic arbitration and any beam-direction gating), so the engine
* has no BLE dependencies.
*
* Utterance boundaries are measured on the sample clock, not the wall clock:
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
* audio heard. The TS side converts startMs/endMs to wall-clock times using
* the engine start timestamp.
* Utterance boundaries are measured on the sample clock, not the wall clock; the TS side
* converts startMs/endMs to wall-clock times using the engine start timestamp.
*/
class FaceclawCaptionEngine {
companion object {
private const val TAG = "FaceclawCaptions"
private const val SAMPLE_RATE = 16000
private const val FEATURE_DIM = 80
private const val MAX_QUEUE_PACKETS = 200
private val mainHandler = Handler(Looper.getMainLooper())
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
private const val ONSET_FACTOR = 3.0
private const val RELEASE_FACTOR = 1.8
private const val MIN_RMS = 220.0
private const val NOISE_EMA_ALPHA = 0.05
private const val PRE_ROLL_MS = 400
private const val DEFAULT_SILENCE_MS = 800
private const val MIN_UTTERANCE_MS = 350
private const val MAX_UTTERANCE_MS = 15000
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
// segments cut at the quietest window (mirrors FaceclawVoiceController).
private const val DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8
private const val CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2
private const val CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000
private const val NORMALIZE_TARGET_PEAK = 0.9f
private const val NORMALIZE_MAX_GAIN = 30f
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
private const val EMBED_MAX_SAMPLES = SAMPLE_RATE * 10
private val host = object : CaptionHost {
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
override fun loadTranscriber(modelDir: String): OfflineTranscriber? {
if (!File(modelDir, "tokens.txt").exists()) {
return null
}
return AndroidSpeechEngines.SherpaTranscriber(
AndroidSpeechEngines.recognizerConfig(File(modelDir), VoiceModelKind.MOONSHINE))
}
override fun loadEmbedder(modelPath: String): SpeakerEmbedder? {
if (!File(modelPath).exists()) {
return null
}
val speakerId = FaceclawSpeakerId(modelPath)
speakerId.ensureLoaded()
return speakerId
}
}
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private val queueLock = Any()
private val queue = ArrayDeque<ByteArray>()
@Volatile
private var listener: FaceclawCaptionEngineListener? = null
@Volatile
private var asrModelDir: String? = null
@Volatile
private var speakerModelPath: String? = null
@Volatile
private var silenceMs = DEFAULT_SILENCE_MS
private var workerThread: Thread? = null
@Volatile
private var started = false
private var recognizer: OfflineRecognizer? = null
private var speakerId: FaceclawSpeakerId? = null
// Segmentation state (worker thread only).
private var totalSamples: Long = 0
private var noiseFloor = 0.0
private var inUtterance = false
private var utteranceStartSample: Long = 0
private var silenceRunSamples: Long = 0
private var utterancePeakRms = 0.0
private var utterance = ShortArray(SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000))
private var utteranceLength = 0
private val preRoll = ShortArray(SAMPLE_RATE * PRE_ROLL_MS / 1000)
private var preRollLength = 0
private val session = CaptionSession(host)
fun setListener(listener: FaceclawCaptionEngineListener?) {
this.listener = listener
session.setListener(listener)
}
/** Directory holding the Moonshine model files, or null to disable ASR. */
fun setAsrModelDir(dir: String?) {
this.asrModelDir = dir
session.setAsrModelDir(dir)
}
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
fun setSpeakerModelPath(path: String?) {
this.speakerModelPath = path
session.setSpeakerModelPath(path)
}
fun setSilenceMs(ms: Int) {
this.silenceMs = Math.max(200, Math.min(3000, ms))
session.setSilenceMs(ms)
}
fun start() {
synchronized(lock) {
if (started) {
return
}
started = true
workerThread = Thread(Runnable { runLoop() }, "FaceclawCaptionEngine")
workerThread!!.start()
}
session.start()
}
fun stop() {
val threadToJoin: Thread?
synchronized(lock) {
if (!started) {
return
}
started = false
threadToJoin = workerThread
workerThread = null
}
synchronized(queueLock) {
(queueLock as java.lang.Object).notifyAll()
}
if (threadToJoin != null) {
threadToJoin.interrupt()
if (Thread.currentThread() !== threadToJoin) {
try {
threadToJoin.join(2000)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
session.stop()
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
fun acceptPcm(pcm16le: ByteArray?) {
if (!started || pcm16le == null || pcm16le.size < 2) {
return
}
synchronized(queueLock) {
if (queue.size >= MAX_QUEUE_PACKETS) {
queue.removeFirst()
}
queue.addLast(pcm16le)
(queueLock as java.lang.Object).notifyAll()
}
}
private fun runLoop() {
try {
loadModels()
resetSegmentation()
while (started && !Thread.currentThread().isInterrupted) {
val chunk = takeChunk()
if (chunk == null) {
continue
}
processChunk(chunk)
}
// Flush a trailing utterance so its text isn't lost on stop.
if (inUtterance && utteranceLength > 0) {
finalizeUtterance()
}
} catch (t: Throwable) {
Log.e(TAG, "caption engine failed", t)
emitStatus("Captions failed: " + t.message)
} finally {
releaseModels()
}
}
private fun loadModels() {
val modelDir = asrModelDir
if (modelDir != null && File(modelDir, "tokens.txt").exists()) {
emitStatus("Loading caption model...")
recognizer = OfflineRecognizer(OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(File(modelDir, "encoder_model.ort").absolutePath)
.setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath)
.build())
.setTokens(File(modelDir, "tokens.txt").absolutePath)
.setNumThreads(1)
.build())
.build())
} else {
recognizer = null
}
val embedModel = speakerModelPath
if (embedModel != null && File(embedModel).exists()) {
speakerId = FaceclawSpeakerId(embedModel)
speakerId!!.ensureLoaded()
} else {
speakerId = null
}
emitStatus(if (recognizer != null) "Captions listening..." else "Captions listening (no ASR model)...")
}
private fun releaseModels() {
if (recognizer != null) {
recognizer!!.release()
recognizer = null
}
if (speakerId != null) {
speakerId!!.close()
speakerId = null
}
}
private fun takeChunk(): ByteArray? {
synchronized(queueLock) {
while (started && queue.isEmpty()) {
try {
(queueLock as java.lang.Object).wait(250)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return null
}
}
return queue.pollFirst()
}
}
private fun resetSegmentation() {
totalSamples = 0
noiseFloor = 0.0
inUtterance = false
utteranceLength = 0
preRollLength = 0
silenceRunSamples = 0
utterancePeakRms = 0.0
}
private fun processChunk(chunk: ByteArray) {
val count = chunk.size / 2
val pcm = ShortArray(count)
var sumSquares = 0.0
for (i in 0 until count) {
val s = ((chunk[i * 2].toInt() and 0xff) or (chunk[i * 2 + 1].toInt() shl 8)).toShort()
pcm[i] = s
sumSquares += s.toDouble() * s
}
val rms = Math.sqrt(sumSquares / Math.max(1, count))
totalSamples += count.toLong()
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = if (noiseFloor == 0.0) rms else noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA
val threshold = Math.max(noiseFloor, MIN_RMS)
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true
utteranceLength = 0
silenceRunSamples = 0
utterancePeakRms = rms
appendUtterance(preRoll, preRollLength)
utteranceStartSample = Math.max(0L, totalSamples - count - preRollLength)
appendUtterance(pcm, count)
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE)
} else {
appendPreRoll(pcm, count)
}
return
}
appendUtterance(pcm, count)
utterancePeakRms = Math.max(utterancePeakRms, rms)
val threshold = Math.max(noiseFloor, MIN_RMS)
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count.toLong()
} else {
silenceRunSamples = 0
}
val utteranceMs = utteranceLength.toLong() * 1000 / SAMPLE_RATE
val silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance()
inUtterance = false
preRollLength = 0
}
}
private fun appendPreRoll(pcm: ShortArray, count: Int) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.size) {
System.arraycopy(pcm, count - preRoll.size, preRoll, 0, preRoll.size)
preRollLength = preRoll.size
return
}
val keep = Math.min(preRollLength, preRoll.size - count)
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep)
System.arraycopy(pcm, 0, preRoll, keep, count)
preRollLength = keep + count
}
private fun appendUtterance(pcm: ShortArray, count: Int) {
val room = utterance.size - utteranceLength
val copied = Math.min(room, count)
if (copied > 0) {
System.arraycopy(pcm, 0, utterance, utteranceLength, copied)
utteranceLength += copied
}
}
private fun finalizeUtterance() {
val length = utteranceLength
val startMs = utteranceStartSample * 1000 / SAMPLE_RATE
val endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE
if (endMs - startMs < MIN_UTTERANCE_MS) {
return
}
val text = recognizeUtterance(length)
val embedding = embedUtterance(length)
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms)
}
private fun recognizeUtterance(length: Int): String {
if (recognizer == null || length <= 0) {
return ""
}
val joined = StringBuilder()
var offset = 0
while (offset < length) {
val remaining = length - offset
var segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES)
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = findQuietCut(offset, segment)
}
val part = recognizeRange(offset, segment)
if (part.length > 0) {
if (joined.length > 0 && ".,!?;:%)]}".indexOf(part[0]) < 0) {
joined.append(' ')
}
joined.append(part)
}
offset += segment
}
return joined.toString().trim()
}
/**
* End a decode segment at the center of the quietest window near its end
* so the cut lands between words (same approach as the PTT controller).
*/
private fun findQuietCut(offset: Int, segment: Int): Int {
val searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES)
val win = CUT_WINDOW_SAMPLES
if (segment - searchStart <= win) {
return segment
}
var sum = 0.0
for (i in searchStart until searchStart + win) {
val s = utterance[offset + i].toDouble()
sum += s * s
}
var best = sum
var bestStart = searchStart
var start = searchStart + 1
while (start + win <= segment) {
val dropped = utterance[offset + start - 1].toDouble()
val added = utterance[offset + start + win - 1].toDouble()
sum += added * added - dropped * dropped
if (sum < best) {
best = sum
bestStart = start
}
start++
}
return bestStart + win / 2
}
private fun recognizeRange(offset: Int, count: Int): String {
val samples = FloatArray(count)
var peak = 0f
for (i in 0 until count) {
val v = utterance[offset + i] / 32768.0f
samples[i] = v
val a = Math.abs(v)
if (a > peak) {
peak = a
}
}
if (peak > 0f) {
val gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN)
if (gain > 1f) {
for (i in 0 until count) {
samples[i] *= gain
}
}
}
val recognizer = this.recognizer!!
val stream: OfflineStream = recognizer.createStream()
try {
stream.acceptWaveform(samples, SAMPLE_RATE)
recognizer.decode(stream)
val result: OfflineRecognizerResult? = recognizer.getResult(stream)
val raw: String? = if (result == null) "" else result.text
return raw?.trim() ?: ""
} finally {
stream.release()
}
}
private fun embedUtterance(length: Int): FloatArray? {
val currentSpeakerId = speakerId
if (currentSpeakerId == null || length <= 0) {
return null
}
val count = Math.min(length, EMBED_MAX_SAMPLES)
val le = ByteArray(count * 2)
for (i in 0 until count) {
val s = utterance[i].toInt()
le[i * 2] = (s and 0xff).toByte()
le[i * 2 + 1] = ((s shr 8) and 0xff).toByte()
}
return currentSpeakerId.embed(le, SAMPLE_RATE)
}
private fun emitUtterance(text: String?, embedding: FloatArray?, startMs: Long, endMs: Long, peakRms: Double) {
val currentListener = listener
if (currentListener == null) {
return
}
val embeddingCopy = if (embedding == null) null else Arrays.copyOf(embedding, embedding.size)
mainHandler.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) }
}
private fun emitSpeechStart(startMs: Long) {
val currentListener = listener
if (currentListener == null) {
return
}
mainHandler.post { currentListener.onSpeechStart(startMs) }
}
private fun emitStatus(status: String?) {
val currentListener = listener
if (currentListener == null) {
return
}
mainHandler.post { currentListener.onStatus(status) }
session.acceptPcm(pcm16le)
}
}
@@ -13,8 +13,6 @@ import org.json.JSONException
import org.json.JSONObject
import java.io.File
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.ArrayList
/**
@@ -47,41 +45,6 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
}
}
// ---- blob helpers ----
private fun blobToFloats(blob: ByteArray?): FloatArray? {
if (blob == null || blob.size < 4 || blob.size % 4 != 0) {
return null
}
val buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN)
val out = FloatArray(blob.size / 4)
for (i in out.indices) {
out[i] = buffer.getFloat()
}
return out
}
private fun floatsToBlob(values: FloatArray): ByteArray {
val buffer = ByteBuffer.allocate(values.size * 4).order(ByteOrder.LITTLE_ENDIAN)
for (v in values) {
buffer.putFloat(v)
}
return buffer.array()
}
private fun l2NormalizeInPlace(v: FloatArray) {
var sum = 0.0
for (x in v) {
sum += x.toDouble() * x
}
val norm = Math.sqrt(sum)
if (norm <= 0) {
return
}
for (i in v.indices) {
v[i] = (v[i] / norm).toFloat()
}
}
}
override fun onCreate(db: SQLiteDatabase) {
@@ -567,7 +530,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
fun updateSpeakerEmbedding(speakerId: Long, embeddingBase64: String?, maxCount: Int) {
try {
val incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP)
val update = blobToFloats(incoming)
val update = SpeakerEmbeddings.blobToFloats(incoming)
if (update == null) {
return
}
@@ -578,28 +541,16 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
arrayOf(speakerId.toString()))
try {
if (c.moveToFirst()) {
current = if (c.isNull(0)) null else blobToFloats(c.getBlob(0))
current = if (c.isNull(0)) null else SpeakerEmbeddings.blobToFloats(c.getBlob(0))
count = c.getInt(1)
}
} finally {
c.close()
}
val merged: FloatArray
if (current == null || current.size != update.size) {
merged = update
count = 1
} else {
val effective = Math.min(count, Math.max(1, maxCount))
merged = FloatArray(current.size)
for (i in merged.indices) {
merged[i] = (current[i] * effective + update[i]) / (effective + 1)
}
l2NormalizeInPlace(merged)
count = count + 1
}
val centroid = SpeakerEmbeddings.runningMean(current, count, update, maxCount)
val values = ContentValues()
values.put("embedding", floatsToBlob(merged))
values.put("embedding_count", count)
values.put("embedding", SpeakerEmbeddings.floatsToBlob(centroid.embedding))
values.put("embedding_count", centroid.count)
values.put("last_heard_at", System.currentTimeMillis())
writableDatabase.update("speakers", values, "id=?",
arrayOf(speakerId.toString()))
@@ -630,7 +581,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
try {
while (c.moveToNext()) {
val id = c.getLong(0)
val embedding = if (c.isNull(1)) null else blobToFloats(c.getBlob(1))
val embedding = if (c.isNull(1)) null else SpeakerEmbeddings.blobToFloats(c.getBlob(1))
val count = c.getInt(2)
if (id == fromId) {
from = embedding
@@ -644,21 +595,10 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
} finally {
c.close()
}
var merged: FloatArray? = into
if (from != null && into != null && from.size == into.size) {
val blended = FloatArray(into.size)
val total = Math.max(1, fromCount + intoCount)
for (i in blended.indices) {
blended[i] = (into[i] * intoCount + from[i] * fromCount) / total
}
l2NormalizeInPlace(blended)
merged = blended
} else if (into == null) {
merged = from
}
val merged = SpeakerEmbeddings.blend(from, fromCount, into, intoCount)
val values = ContentValues()
if (merged != null) {
values.put("embedding", floatsToBlob(merged))
values.put("embedding", SpeakerEmbeddings.floatsToBlob(merged))
values.put("embedding_count", fromCount + intoCount)
}
if (fromWearer) {
@@ -677,7 +617,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
val hasInsights = !c2.isNull(0) && !c2.getString(0).isEmpty()
if (c2.position == 0) {
targetHasInsights = hasInsights
} else if (!targetHasInsights && hasInsights) {
} else if (SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights, hasInsights)) {
values.put("last_recap", c2.getString(0))
values.put("action_items", if (c2.isNull(1)) "" else c2.getString(1))
values.put("facts", if (c2.isNull(2)) "" else c2.getString(2))
@@ -1,99 +1,44 @@
package com.faceclaw.app
import android.bluetooth.BluetoothDevice
import android.bluetooth.BluetoothGatt
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import android.util.Log
import java.util.HashSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
/**
* Connects to the glasses (stock-firmware compatible), reads the device-info /
* settings response, and reports the firmware versions plus the firmware-
* extension string (empty on stock firmware). Used by onboarding to decide whether to
* flash. Reuses FaceclawBleManager + BleProtocol; owns its own connection and
* runs on a single worker thread. Shows nothing on the lens.
* Android host for the shared [DeviceInfoProbeFlow]: owns the GATT link (AndroidStockLink),
* the worker thread the flow runs on, and delivers listener callbacks on the main thread.
* Public API unchanged for the TypeScript wrapper (app/native/device-info-probe.ts).
*/
class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) : FaceclawBleListener {
companion object {
private const val TAG = "FaceclawDeviceInfo"
private const val QUERY_TIMEOUT_MS = 4_000
/** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */
private const val ARM_ATTEMPTS = 3
private const val ARM_RETRY_DELAY_MS = 1_000
/**
* Upper bound on waiting for the auth success while Android reports the arm
* as BOND_BONDING — i.e. an OS pairing dialog may be sitting there waiting
* for the user, which can take a lot longer than the normal 30 s window.
*/
private const val PAIRING_WAIT_CAP_MS = 90_000
/** After a fresh bond, how long the firmware gets to notify success on its own before the request is re-sent. */
private const val POST_BOND_GRACE_MS = 2_000
private const val POLL_MS = 250
private fun bondStateName(state: Int): String {
return when (state) {
BluetoothDevice.BOND_NONE -> "none"
BluetoothDevice.BOND_BONDING -> "bonding"
BluetoothDevice.BOND_BONDED -> "bonded"
else -> "unknown"
}
}
}
private val context: Context
private val rightAddress: String
private val leftAddress: String
private val bleManager: FaceclawBleManager
class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private var nextSeq = 0x40
private var nextMagic = 100
private var awaitSid = -1
private var awaitMagic = -1
private var awaitPb: ByteArray? = null
private var awaitLatch: CountDownLatch? = null
/**
* Security-auth wait, kept separate from the generic ack wait: only the
* firmware's SUCCESS result (empty result message) may satisfy it. The
* firmware answers the request as soon as it arrives, but with a
* non-success result until the link is encrypted; accepting that first
* reply as "the ack" is what let the probe move on to the other arm while
* Android's pairing dialog for this one was still up.
*/
private val authMagics: MutableSet<Int> = HashSet()
private var authLatch: CountDownLatch? = null
/**
* A sid-0x09 frame that carried firmware versions without matching the
* awaited magic — some firmware pushes the settings snapshot on a magic it
* picked itself instead of (or in addition to) acking the read.
*/
@Volatile
private var unsolicitedSettingsPb: ByteArray? = null
@Volatile
private var listener: FaceclawDeviceInfoProbeListener? = null
@Volatile
private var worker: Thread? = null
@Volatile
private var cancelled = false
init {
this.context = context.applicationContext
this.rightAddress = rightAddress ?: ""
this.leftAddress = leftAddress ?: ""
this.bleManager = FaceclawBleManager(this.context)
this.bleManager.setListener(this)
}
private val flow =
DeviceInfoProbeFlow(
link,
rightAddress ?: "",
leftAddress ?: "",
object : FaceclawDeviceInfoProbeListener {
override fun onLog(line: String?) {
android.util.Log.i(DeviceInfoProbeFlow.TAG, line ?: "")
post { it.onLog(line) }
}
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onResult(leftVersion: String?, rightVersion: String?, extension: String?) =
post { it.onResult(leftVersion, rightVersion, extension) }
override fun onError(message: String?) = post { it.onError(message ?: "") }
},
)
fun setListener(listener: FaceclawDeviceInfoProbeListener?) {
this.listener = listener
@@ -104,525 +49,29 @@ class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddre
if (worker != null) {
return
}
val w = Thread({ run() }, "faceclaw-device-info")
val w = Thread({ flow.run() }, "faceclaw-device-info")
worker = w
w.start()
}
}
fun cancel() {
cancelled = true
val w = worker
if (w != null) {
w.interrupt()
}
flow.cancel()
}
fun close() {
cancel()
try {
bleManager.close()
link.close()
} catch (ignored: Exception) {
}
}
private fun run() {
try {
if (rightAddress.trim().isEmpty()) {
emitError("No right-arm address configured.")
return
}
val haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equals(rightAddress, ignoreCase = true)
// Bring up and authenticate BOTH arms before asking for versions.
// Each arm is its own peripheral with its own bond, so on a fresh
// phone each raises its own pairing prompt; completing both here
// means the flash step (and the app proper) find both bonds in
// place. Strictly one arm at a time: Android pairs with one device
// at a time, and a second arm connected while the first was still
// pairing has been seen to drop within a second and never pair
// (2026-09-11 logcat).
var rightAuthenticated = bringUpArm(rightAddress, "right")
var leftAuthenticated = false
if (haveLeft && !cancelled) {
leftAuthenticated = bringUpArm(leftAddress, "left")
}
if (cancelled) {
emitError("Cancelled.")
return
}
if (!bleManager.isConnected(rightAddress)) {
// The right lens may have dropped while the left one paired;
// it is bonded by now, so this comes back quickly.
emitLog("right lens disconnected while the left lens was set up; reconnecting")
rightAuthenticated = bringUpArm(rightAddress, "right")
if (cancelled) {
emitError("Cancelled.")
return
}
}
var ack: ByteArray? = null
var rightFailure: String? = null
try {
ack = queryArm(rightAddress, "right", rightAuthenticated)
} catch (e: IllegalStateException) {
rightFailure = if (e.message == null) e.toString() else e.message
emitLog("right-lens query failed: $rightFailure")
}
// The right lens is the documented control endpoint, but a silent
// right lens has been observed on stock 2.2.9 even after a
// successful security auth — run the same probe against the left
// lens rather than giving up, and log which lens answered.
if (ack == null && !cancelled && haveLeft) {
emitLog("right lens did not answer the settings query; probing the left lens")
try {
ack = queryArm(leftAddress, "left", leftAuthenticated)
} catch (e: Exception) {
emitLog("left-lens probe failed: " + (if (e.message == null) e.toString() else e.message))
}
}
if (cancelled) {
emitError("Cancelled.")
return
}
if (ack == null) {
throw IllegalStateException(if (rightFailure != null)
rightFailure
else "no response to the device-info query on either lens — check `adb logcat -s "
+ TAG + "` for the frame trace")
}
val info: BleProtocol.FirmwareInfo? = BleProtocol.parseSettingsFirmwareInfo(ack)
val left = if (info == null) "" else info.leftVersion
val right = if (info == null) "" else info.rightVersion
val extension = if (info == null) "" else info.extension
emitLog("device-info: L=$left R=$right ext=[$extension]")
emitResult(left, right, extension)
} catch (e: Exception) {
val message = if (cancelled) "Cancelled." else (if (e.message == null) e.toString() else e.message)
emitError(message)
} finally {
try {
bleManager.close()
} catch (ignored: Exception) {
}
}
}
/**
* Connect to one arm and complete its security-auth exchange, retrying
* when the link drops mid-pairing. Returns whether the auth success was
* seen (false = connected but unconfirmed, tolerated because the custom
* firmware's response to the exchange is not yet hardware-verified);
* throws when the arm cannot be brought up at all.
*/
@Throws(InterruptedException::class)
private fun bringUpArm(address: String, label: String): Boolean {
var lastFailure: String? = null
var attempt = 1
while (attempt <= ARM_ATTEMPTS && !cancelled) {
if (attempt > 1) {
// A link Android dropped while it was still pairing comes back
// once the bond settles; reconnecting sooner just fails again.
waitForBondToSettle(address, label)
if (cancelled) {
break
}
emitLog("retrying the $label lens (attempt $attempt/$ARM_ATTEMPTS): $lastFailure")
Thread.sleep(ARM_RETRY_DELAY_MS.toLong())
}
emitState("connecting", label)
try {
connectArm(address)
} catch (e: IllegalStateException) {
lastFailure = if (e.message == null) e.toString() else e.message
emitLog("$label lens: $lastFailure")
attempt++
continue
}
if (cancelled) {
break
}
// Firmware 2.2.9 answers no queries until the security-auth
// exchange completes over an encrypted link; on a phone with no
// existing bond this is also what triggers SMP pairing (and its OS
// prompt), so it must come before the prelude and query.
emitState("authenticating", label)
val result = authenticate(address, label)
if (result == AuthResult.SUCCESS) {
return true
}
if (result == AuthResult.UNCONFIRMED) {
return false
}
lastFailure = "link dropped during authentication"
attempt++
}
if (cancelled) {
return false
}
throw IllegalStateException("could not connect to the $label lens ($address): "
+ lastFailure + " — if Android showed a Bluetooth pairing request, accept it and try again")
}
private enum class AuthResult { SUCCESS, UNCONFIRMED, LINK_DROPPED }
/**
* Send the sid-0x80 authentication request and wait for the firmware's
* SUCCESS notification, which only arrives once the link is encrypted.
* On an unbonded phone that means waiting through Android's pairing flow
* (BOND_BONDING, possibly with a dialog up), so the wait is extended while
* the OS reports pairing in progress and the request is re-sent once
* after a fresh bond in case the pre-pairing write was dropped. Bond
* state is polled rather than awaited via broadcast; 250 ms granularity
* is plenty here.
*/
@Throws(InterruptedException::class)
private fun authenticate(address: String, label: String): AuthResult {
val start = SystemClock.elapsedRealtime()
var deadline = start + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS
val hardDeadline = start + PAIRING_WAIT_CAP_MS
val latch = CountDownLatch(1)
synchronized(lock) {
authMagics.clear()
authLatch = latch
}
val initialBond = bleManager.getBondState(address)
emitLog("security auth: $label lens, Android bond state " + bondStateName(initialBond))
try {
var sends = 0
var lastSendFailed = false
var nextSendAt = start
var bondedAt = -1L
var resentAfterBond = false
var loggedBonding = false
while (!cancelled) {
val now = SystemClock.elapsedRealtime()
val bond = bleManager.getBondState(address)
if (bond == BluetoothDevice.BOND_BONDING) {
// OS pairing in progress — the firmware will answer once the
// link is encrypted. Don't write, don't give up, don't move
// on to the other arm.
if (!loggedBonding) {
emitLog("Android is pairing with the $label lens; waiting for the user to accept")
loggedBonding = true
}
if (now >= hardDeadline) {
break
}
} else {
if (bond == BluetoothDevice.BOND_BONDED && initialBond != BluetoothDevice.BOND_BONDED
&& bondedAt < 0) {
bondedAt = now
emitLog("Android bond established with the $label lens; waiting for the auth success")
deadline = Math.max(deadline, now + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS)
}
val wantSend = sends == 0
|| (lastSendFailed && sends < 3 && now >= nextSendAt)
|| (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= POST_BOND_GRACE_MS)
if (wantSend) {
if (bondedAt >= 0) {
resentAfterBond = true
if (sends > 0) {
emitLog("re-sending the auth request after pairing ($label lens)")
}
}
lastSendFailed = !sendAuthRequest(address)
sends++
nextSendAt = SystemClock.elapsedRealtime() + ARM_RETRY_DELAY_MS
if (lastSendFailed && !bleManager.isConnected(address)) {
return AuthResult.LINK_DROPPED
}
}
if (now >= deadline) {
break
}
}
if (latch.await(POLL_MS.toLong(), TimeUnit.MILLISECONDS)) {
emitLog("security auth complete: $label lens ($address)")
return AuthResult.SUCCESS
}
if (!bleManager.isConnected(address)) {
emitLog("security auth: $label lens disconnected while waiting"
+ (if (bond == BluetoothDevice.BOND_BONDING) " (Android was still pairing)" else ""))
return AuthResult.LINK_DROPPED
}
}
if (cancelled) {
return AuthResult.UNCONFIRMED
}
emitLog("security auth unconfirmed: $label lens ($address), Android bond state "
+ bondStateName(bleManager.getBondState(address)))
return AuthResult.UNCONFIRMED
} finally {
synchronized(lock) {
authLatch = null
authMagics.clear()
}
}
}
private fun sendAuthRequest(address: String): Boolean {
val magic = allocMagic()
val seq: Int
synchronized(lock) {
authMagics.add(magic)
seq = nextSeq++ and 0xff
}
val payload = BleProtocol.buildAuthenticationRequest(magic)
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, magic, seq, payload.size))
val frames = BleProtocol.framePb(payload, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, seq)
var written: Boolean
try {
written = bleManager.writeFrames(
address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS)
} catch (e: IllegalStateException) {
// "Not connected" — the link went away underneath us.
written = false
}
if (!written) {
emitLog("tx write FAILED sid=0x80 ($address)")
}
return written
}
/** Block while Android reports this arm as BOND_BONDING (pairing dialog up / SMP in flight). */
@Throws(InterruptedException::class)
private fun waitForBondToSettle(address: String, label: String) {
val cap = SystemClock.elapsedRealtime() + PAIRING_WAIT_CAP_MS
var logged = false
while (!cancelled && bleManager.getBondState(address) == BluetoothDevice.BOND_BONDING
&& SystemClock.elapsedRealtime() < cap) {
if (!logged) {
emitLog("waiting for Android to finish pairing with the $label lens before reconnecting")
logged = true
}
Thread.sleep(POLL_MS.toLong())
}
}
/**
* Prelude + settings read on an already connected/authenticated lens.
* Returns the settings ack protobuf, an unsolicited settings push that
* carried firmware versions, or null when the lens never answered the read.
* Throws on prelude failure; the caller wraps the fallback lens's attempt
* so its failure cannot mask the primary lens's outcome.
*/
@Throws(InterruptedException::class)
private fun queryArm(address: String, label: String, authenticated: Boolean): ByteArray? {
if (cancelled) {
return null
}
emitState("querying", label)
// Session prelude, then a settings/device-info read (both arms'
// versions and the firmware-extension string ride back in one response).
if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw IllegalStateException("session prelude not acked ($label lens)"
+ (if (authenticated) "" else "; authentication did not complete —"
+ " if Android shows a Bluetooth pairing request, accept it and try again"))
}
// Two attempts: the first read straight after a fresh pairing has been
// seen to go unanswered while a later one succeeds.
var attempt = 0
while (attempt < 2 && !cancelled) {
val magic = allocMagic()
val ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST,
magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS)
if (ack != null) {
return ack
}
// A push with the firmware versions on the device's own magic is
// as good as the ack we asked for.
val pushed = unsolicitedSettingsPb
if (pushed != null) {
emitLog("using unsolicited settings push instead of the read ack ($label lens)")
return pushed
}
emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)")
attempt++
}
return null
}
private fun connectArm(address: String) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw IllegalStateException("connect failed")
}
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
throw IllegalStateException("service discovery failed")
}
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw IllegalStateException("could not subscribe to notifications")
}
}
@Throws(InterruptedException::class)
private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? {
val latch = CountDownLatch(1)
synchronized(lock) {
awaitSid = sid
awaitMagic = magic
awaitPb = null
awaitLatch = latch
}
val seq: Int
synchronized(lock) {
seq = nextSeq++ and 0xff
}
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
address, sid, flag, magic, seq, payload.size))
val frames = BleProtocol.framePb(payload, sid, flag, seq)
val written = bleManager.writeFrames(
address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS)
if (!written) {
emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid))
synchronized(lock) {
awaitLatch = null
}
return null
}
val acked = latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS)
synchronized(lock) {
awaitLatch = null
return if (acked) awaitPb else null
}
}
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
return
}
// One notification value can carry several envelope frames back to
// back; reading only the first would silently drop the rest.
val frames = BleProtocol.splitFrames(data)
if (frames.size > 1) {
emitLog("rx " + address + " value carries " + frames.size + " frames (raw " + data!!.size + " bytes)")
}
for (buf in frames) {
handleFrame(address, buf, data!!.size)
}
}
private fun handleFrame(address: String?, buf: ByteArray, rawValueLength: Int) {
val frame = BleProtocol.parseFrame(buf)
if (!frame.ok) {
emitLog("rx " + address + " unparseable frame len=" + buf.size + " (raw value " + rawValueLength + ")"
+ " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.size))))
return
}
// Log every control frame while diagnosing 2.2.9: sid/flag/type/magic
// plus a payload prefix is enough to reconstruct what the lens said.
val declared = if (buf.size > 3) buf[3].toInt() and 0xff else 0
val truncated = if (buf.size < 8 + declared) " TRUNCATED(declared=$declared)" else ""
emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s",
address, frame.sid, frame.flag, frame.msgType, frame.msgSeq,
if (buf.size > 5) buf[5].toInt() and 0xff else 0, if (buf.size > 4) buf[4].toInt() and 0xff else 0, frame.pb.size, truncated,
FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.size)))))
if (frame.sid == BleProtocol.SID_UI_SETTING
&& BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) {
// Any settings frame carrying firmware versions answers the probe's
// question, whether or not it matches the magic we asked with.
unsolicitedSettingsPb = frame.pb
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
// Async event, not an ack. This includes the lens's own periodic
// sid-0x80 notifies (device-chosen magic, non-empty result).
return
}
if (frame.sid == BleProtocol.SID_SECURITY_AUTH) {
var success = false
var ours = false
synchronized(lock) {
val currentLatch = authLatch
if (currentLatch != null) {
ours = authMagics.contains(frame.msgSeq)
for (magic in authMagics) {
if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) {
success = true
currentLatch.countDown()
break
}
}
}
}
if (ours && !success) {
// Expected before the link is encrypted: the firmware answers
// the request straight away with a non-success result and
// sends the real success once pairing/encryption completes.
emitLog("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting")
}
return
}
synchronized(lock) {
val currentLatch = awaitLatch
if (currentLatch != null && frame.sid == awaitSid && frame.msgSeq == awaitMagic) {
awaitPb = frame.pb
currentLatch.countDown()
}
}
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (!connected) {
Log.i(TAG, "disconnected: $address")
}
}
private fun allocMagic(): Int {
synchronized(lock) {
val magic = nextMagic
nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1
return magic
}
}
private fun emitLog(line: String) {
Log.i(TAG, line)
private fun post(action: (FaceclawDeviceInfoProbeListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
current.onLog(line)
}
}
}
private fun emitState(state: String, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onState(state, safeDetail)
}
}
}
private fun emitResult(left: String, right: String, extension: String) {
mainHandler.post {
val current = listener
if (current != null) {
current.onResult(left, right, extension)
}
}
}
private fun emitError(message: String?) {
val safeMessage = message ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onError(safeMessage)
action(current)
}
}
}
@@ -129,17 +129,19 @@ class FaceclawDiarizer(
val file = File(path)
val fileLength = file.length()
DataInputStream(FileInputStream(file)).use { `in` ->
val header = ByteArray(44)
`in`.readFully(header)
if (header[0] != 'R'.code.toByte() || header[1] != 'I'.code.toByte() || header[8] != 'W'.code.toByte()) {
throw IOException("not a WAV file")
// Shared RIFF walk (handles extra chunks and unpatched data sizes); the
// stream is then positioned at the first PCM sample.
val prefix = ByteArray(Math.min(fileLength, WavPcmReader.HEADER_PREFIX_BYTES.toLong()).toInt())
`in`.readFully(prefix)
val info = try {
WavPcmReader.parseHeader(prefix, fileLength)
} catch (e: IllegalArgumentException) {
throw IOException(e.message ?: "not a WAV file")
}
val channels = (header[22].toInt() and 0xff) or (header[23].toInt() shl 8)
val bits = (header[34].toInt() and 0xff) or (header[35].toInt() shl 8)
if (channels != 1 || bits != 16) {
if (info.channels != 1) {
throw IOException("only 16-bit mono WAV is supported")
}
val dataBytes = fileLength - 44
val dataBytes = info.dataBytes
val count = Math.min(dataBytes / 2, Int.MAX_VALUE.toLong()).toInt()
val samples = FloatArray(count)
val chunk = ByteArray(65536)
@@ -147,6 +149,21 @@ class FaceclawDiarizer(
var carried = 0
var carriedByte: Byte = 0
var read = 0
// The prefix already holds the start of the PCM data; consume it first.
val prefixData = prefix.size - info.dataOffset.toInt()
if (prefixData > 0) {
val pairs = prefixData / 2
var i = 0
while (i < pairs && sampleIndex < count) {
val base = info.dataOffset.toInt() + i * 2
samples[sampleIndex++] = AudioSegmentation.pcm16le(prefix[base], prefix[base + 1]) / 32768.0f
i++
}
if ((prefixData and 1) == 1) {
carried = 1
carriedByte = prefix[prefix.size - 1]
}
}
while (sampleIndex < count && `in`.read(chunk).also { read = it } > 0) {
var offset = 0
if (carried == 1) {
@@ -15,7 +15,6 @@ import java.io.FileInputStream
import java.io.IOException
import java.io.InputStream
import java.nio.charset.StandardCharsets
import java.util.Locale
/**
* Serves an unpacked EvenHub app's files to its WebView from a fake per-app
@@ -52,25 +51,25 @@ class FaceclawEvenHubWebViewClient(
if (url == null || host != url.host) {
return null // External hosts: normal network handling.
}
var path = url.path
if (path == null || path.isEmpty() || path == "/") {
path = "/index.html"
}
val path = url.path
try {
// Path rules, MIME table and shim injection are shared (EvenHubAssetServer);
// the canonical-file check below additionally defeats symlinks.
val asset = EvenHubAssetServer.resolve(path)
val root = File(rootDir).canonicalFile
val target = File(root, path.substring(1)).canonicalFile
if (!target.path.startsWith(root.path) || !target.isFile) {
val target = asset?.let { File(root, it.relativePath).canonicalFile }
if (asset == null || target == null || !EvenHubAssetServer.isInsideRoot(root.path, target.path) || !target.isFile) {
Log.w(TAG, "404 $path")
return WebResourceResponse(
"text/plain", "utf-8",
ByteArrayInputStream("not found".toByteArray(StandardCharsets.UTF_8)))
}
val mime = mimeTypeFor(target.name)
if (mime == "text/html") {
return WebResourceResponse(mime, "utf-8", ByteArrayInputStream(injectIntoHtml(target)))
if (asset.isHtml) {
val html = EvenHubAssetServer.injectIntoHtml(String(readAll(target), StandardCharsets.UTF_8), injectScript)
return WebResourceResponse(asset.mime, "utf-8", ByteArrayInputStream(html.toByteArray(StandardCharsets.UTF_8)))
}
val stream: InputStream = FileInputStream(target)
return WebResourceResponse(mime, null, stream)
return WebResourceResponse(asset.mime, null, stream)
} catch (e: IOException) {
Log.e(TAG, "serve failed for $path", e)
return WebResourceResponse(
@@ -94,30 +93,9 @@ class FaceclawEvenHubWebViewClient(
}
}
@Throws(IOException::class)
private fun injectIntoHtml(file: File): ByteArray {
val raw = readAll(file)
val html = String(raw, StandardCharsets.UTF_8)
val tag = "<script>$injectScript</script>"
// After <head...> if present, else before everything.
val headIndex = indexOfIgnoreCase(html, "<head")
if (headIndex >= 0) {
val close = html.indexOf('>', headIndex)
if (close >= 0) {
return (html.substring(0, close + 1) + tag + html.substring(close + 1))
.toByteArray(StandardCharsets.UTF_8)
}
}
return (tag + html).toByteArray(StandardCharsets.UTF_8)
}
companion object {
private const val TAG = "FaceclawEvenHub"
private fun indexOfIgnoreCase(haystack: String, needle: String): Int {
return haystack.lowercase(Locale.ROOT).indexOf(needle)
}
@Throws(IOException::class)
private fun readAll(file: File): ByteArray {
val input = FileInputStream(file)
@@ -134,25 +112,5 @@ class FaceclawEvenHubWebViewClient(
input.close()
}
}
private fun mimeTypeFor(name: String): String {
val lower = name.lowercase(Locale.ROOT)
if (lower.endsWith(".html") || lower.endsWith(".htm")) return "text/html"
if (lower.endsWith(".js") || lower.endsWith(".mjs")) return "application/javascript"
if (lower.endsWith(".css")) return "text/css"
if (lower.endsWith(".json")) return "application/json"
if (lower.endsWith(".png")) return "image/png"
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"
if (lower.endsWith(".gif")) return "image/gif"
if (lower.endsWith(".svg")) return "image/svg+xml"
if (lower.endsWith(".webp")) return "image/webp"
if (lower.endsWith(".ico")) return "image/x-icon"
if (lower.endsWith(".wasm")) return "application/wasm"
if (lower.endsWith(".woff")) return "font/woff"
if (lower.endsWith(".woff2")) return "font/woff2"
if (lower.endsWith(".ttf")) return "font/ttf"
if (lower.endsWith(".txt") || lower.endsWith(".map")) return "text/plain"
return "application/octet-stream"
}
}
}
@@ -1,185 +1,54 @@
package com.faceclaw.app
import android.bluetooth.BluetoothGatt
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import android.util.Log
import java.io.ByteArrayOutputStream
import java.io.FileInputStream
import java.io.IOException
import java.nio.charset.StandardCharsets
import java.util.ArrayList
import java.util.Arrays
import java.util.concurrent.CountDownLatch
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
/**
* Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA
* firmware container to each lens over the firmware-data service using the
* aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol.
*
* Each lens connection completes the sid-0x80 security-auth exchange before
* OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing
* the transfer mid-component. No control-channel traffic is injected during
* the transfer itself: official OTA captures show none between BEGIN and the
* final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a
* concurrent control writer would interleave with the marker/data
* transactions. (See ../notes/ble-connections-2.2.9.md.)
*
* Lenses are flashed one at a time (left, then right). Finishing either lens
* reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect
* for it uses a generous window rather than failing fast.
*
* Separate from FaceclawBleCommunicator on purpose: this speaks the stock OTA
* protocol and must run before the custom firmware exists. It reuses the GATT
* wrapper (FaceclawBleManager) and framing (BleProtocol).
* Android host for the shared [OtaFlashFlow] (g2flash.py's OTA procedure over the stock
* firmware-data service): owns the GATT link (AndroidStockLink) and the worker thread, and
* delivers listener callbacks on the main thread. Separate from FaceclawBleCommunicator on
* purpose: this speaks the stock OTA protocol and must run before the custom firmware exists.
* Public API unchanged for app/native/firmware-flasher.ts.
*/
class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) : FaceclawBleListener {
companion object {
private const val TAG = "FaceclawFlasher"
// OTA message types (envelope sid byte) and control opcodes.
private const val SID_CTRL = 0xc0
private const val SID_DATA = 0xc1
private const val FLAG_OTA = 0x00
private const val OP_BEGIN = 0x00
private const val OP_FILE_CHECK = 0x01
private const val OP_BLOCK = 0x02
private const val OP_END = 0x03
private const val BLOCK_SIZE = 4096
private const val BLOCK_ACK_TIMEOUT_MS = 4_000
private const val CTRL_ACK_TIMEOUT_MS = 8_000
private const val BLOCK_NAK_RETRIES = 3
private const val COMPONENT_RETRIES = 3
// Reconnect windows. The first lens connects from an idle device; the second
// must ride out the post-first-lens reboot of both lenses.
private const val FIRST_LENS_CONNECT_WINDOW_MS = 30_000
private const val SECOND_LENS_CONNECT_WINDOW_MS = 120_000
private const val REBOOT_SETTLE_MS = 5_000
private const val NOTIFY_SETTLE_MS = 2_500
private const val RETRY_DELAY_MS = 2_500
// END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART.
private val END_OK = intArrayOf(0, 8, 9)
// Firmware layout expectations (mirrors g2flash.py). Firmware 2.2.4 has 5
// components; 2.2.6 has 6.
private const val MIN_SEGMENTS = 5
private const val MAX_SEGMENTS = 6
private const val REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin"
private const val APP_LOAD_ADDR = 0x00438000L
private const val APP_MAX_END = 0x007F0000L
private const val APP_PREAMBLE = 0x20
// ---- firmware container parsing / validation -----------------------------
private val EMPTY = ByteArray(0)
// ---- helpers -------------------------------------------------------------
private fun isEndOk(status: Int): Boolean {
for (ok in END_OK) {
if (ok == status) {
return true
}
}
return false
}
private fun readFile(path: String): ByteArray {
try {
FileInputStream(path).use { fis ->
val out = ByteArrayOutputStream()
val buf = ByteArray(65536)
var read: Int
while (fis.read(buf).also { read = it } >= 0) {
out.write(buf, 0, read)
}
return out.toByteArray()
}
} catch (e: IOException) {
throw IllegalStateException("could not read firmware: " + e.message, e)
}
}
private fun readU32(buf: ByteArray, offset: Int): Long {
return (buf[offset].toLong() and 0xffL) or
((buf[offset + 1].toLong() and 0xffL) shl 8) or
((buf[offset + 2].toLong() and 0xffL) shl 16) or
((buf[offset + 3].toLong() and 0xffL) shl 24)
}
private fun readCString(buf: ByteArray, offset: Int, maxLen: Int): String {
var end = offset
val limit = Math.min(buf.size, offset + maxLen)
while (end < limit && buf[end].toInt() != 0) {
end++
}
return String(buf, offset, end - offset, StandardCharsets.ISO_8859_1)
}
private val CRC32C_TABLE: IntArray = buildCrc32cTable()
private fun buildCrc32cTable(): IntArray {
val table = IntArray(256)
for (b in 0 until 256) {
var c = b shl 24
for (i in 0 until 8) {
c = if ((c and 0x80000000.toInt()) != 0) (c shl 1) xor 0x1edc6f41 else c shl 1
}
table[b] = c
}
return table
}
/** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */
private fun crc32cMsb(data: ByteArray): Int {
var crc = 0
for (value in data) {
crc = (crc shl 8) xor CRC32C_TABLE[((crc ushr 24) xor (value.toInt() and 0xff)) and 0xff]
}
return crc
}
}
private val context: Context
private val leftAddress: String
private val rightAddress: String
private val firmwarePath: String
private val bleManager: FaceclawBleManager
class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private val dataAcks = LinkedBlockingQueue<ByteArray>()
@Volatile
private var listener: FaceclawFirmwareFlasherListener? = null
@Volatile
private var worker: Thread? = null
@Volatile
private var cancelled = false
private var nextSeq = 0
private var nextAuthMagic = 0x60
private var pendingAuthMagic = -1
private var authLatch: CountDownLatch? = null
private var authAckPb: ByteArray? = null
private val flow =
OtaFlashFlow(
link,
rightAddress ?: "",
leftAddress ?: "",
firmwarePath ?: "",
object : FaceclawFirmwareFlasherListener {
override fun onLog(line: String?) {
android.util.Log.i(OtaFlashFlow.TAG, line ?: "")
post { it.onLog(line) }
}
init {
this.context = context.applicationContext
this.rightAddress = rightAddress ?: ""
this.leftAddress = leftAddress ?: ""
this.firmwarePath = firmwarePath ?: ""
this.bleManager = FaceclawBleManager(this.context)
this.bleManager.setListener(this)
}
override fun onProgress(
lens: String?,
componentIndex: Int,
componentCount: Int,
blockIndex: Int,
blockCount: Int,
bytesSent: Long,
bytesTotal: Long,
) = post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
},
)
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
this.listener = listener
@@ -190,517 +59,29 @@ class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddre
if (worker != null) {
return
}
val w = Thread({ run() }, "faceclaw-flasher")
val w = Thread({ flow.run() }, "faceclaw-flasher")
worker = w
w.start()
}
}
fun cancel() {
cancelled = true
val w = worker
if (w != null) {
w.interrupt()
}
flow.cancel()
}
fun close() {
cancel()
try {
bleManager.close()
link.close()
} catch (ignored: Exception) {
}
}
private fun run() {
try {
if (leftAddress.trim().isEmpty() || rightAddress.trim().isEmpty()) {
throw IllegalStateException("Both lens addresses are required to flash.")
}
emitState("validating", "")
val img = readFile(firmwarePath)
val segs = validate(img)
emitLog("firmware validated: " + segs.size + " components, " + img.size + " bytes")
flashLens("left", leftAddress, img, segs, FIRST_LENS_CONNECT_WINDOW_MS)
if (cancelled) {
emitState("error", "Cancelled.")
emitComplete(false, "Cancelled after the left lens.")
return
}
emitState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens.")
sleepInterruptibly(REBOOT_SETTLE_MS)
flashLens("right", rightAddress, img, segs, SECOND_LENS_CONNECT_WINDOW_MS)
emitState("done", "")
emitComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware.")
} catch (e: Exception) {
var message = if (e.message == null) e.toString() else e.message
if (cancelled) {
message = "Cancelled."
}
emitState("error", message)
emitComplete(false, message)
} finally {
teardown()
}
}
@Throws(TimeoutException::class)
private fun flashLens(lens: String, address: String, img: ByteArray, segs: List<Segment>, connectWindowMs: Int) {
emitState("connecting", lens)
connectLensResilient(lens, address, connectWindowMs)
sleepInterruptibly(NOTIFY_SETTLE_MS)
drainAcks()
// Fresh envelope counter for the OTA stream (the auth exchange during
// bring-up used its own); mirrors the stock sequence.
resetSeq()
emitState("flashing", lens)
val beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS)
if (!isEndOk(beginStatus)) {
emitLog("warning: unexpected begin status $beginStatus; continuing")
}
// Progress is reported in bytes across the whole image: the segments
// are one large firmware blob plus several small ones, so a bar
// stepped per segment would sit still through the big one and then
// sprint through the rest.
var totalBytes = 0L
for (seg in segs) {
totalBytes += seg.ps
}
var bytesBefore = 0L
for (i in segs.indices) {
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
flashComponentWithRetry(lens, address, i, segs.size, segs[i], img, bytesBefore, totalBytes)
bytesBefore += segs[i].ps
}
emitLog("$lens lens: all components verified")
try {
bleManager.disconnect(address)
} catch (ignored: Exception) {
}
}
private fun flashComponentWithRetry(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray,
bytesBefore: Long, totalBytes: Long) {
for (attempt in 0 until COMPONENT_RETRIES) {
if (attempt > 0) {
emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES)
}
var endStatus: Int
try {
endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes)
} catch (e: TimeoutException) {
emitLog(seg.name + ": block phase failed: " + e.message)
endStatus = -1
} catch (e: RuntimeException) {
emitLog(seg.name + ": block phase failed: " + e.message)
endStatus = -1
}
if (isEndOk(endStatus)) {
emitLog(seg.name + ": END verify OK (status " + endStatus + ")")
return
}
if (endStatus >= 0) {
emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")")
}
drainAcks()
sleepInterruptibly(1_500)
}
throw IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts")
}
@Throws(TimeoutException::class)
private fun flashComponent(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray,
bytesBefore: Long, totalBytes: Long): Int {
val sub = Arrays.copyOfRange(img, seg.off, seg.off + 128)
val ps = seg.ps
val payloadStart = seg.off + 128
val checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, CTRL_ACK_TIMEOUT_MS)
if (checkStatus != 0) {
throw IllegalStateException("FILE_CHECK rejected status=$checkStatus")
}
val blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE
for (b in 0 until blockCount) {
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
val start = payloadStart + b * BLOCK_SIZE
val end = Math.min(start + BLOCK_SIZE, payloadStart + ps)
val block = Arrays.copyOfRange(img, start, end)
var accepted = false
for (tries in 0 until BLOCK_NAK_RETRIES) {
val status = sendBlock(address, block) // throws TimeoutException -> component re-flash
if (status == 0) {
accepted = true
break
}
emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status
+ " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES)
}
if (!accepted) {
throw IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times")
}
if (b % 20 == 0 || b == blockCount - 1) {
val bytesSent = bytesBefore + Math.min((b + 1).toLong() * BLOCK_SIZE, ps.toLong())
emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes)
}
}
emitLog(seg.name + ": data phase done; sending END")
return sendCtrlAndWait(address, OP_END, EMPTY, CTRL_ACK_TIMEOUT_MS)
}
/** Send one 4 KB block as a marker + data pair sharing one envelope seq. */
@Throws(TimeoutException::class)
private fun sendBlock(address: String, block: ByteArray): Int {
val seq = nextSeq()
drainAcks()
writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, byteArrayOf(OP_BLOCK.toByte()), seq)
writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_DATA, block, seq)
return waitAck(OP_BLOCK, BLOCK_ACK_TIMEOUT_MS)
}
@Throws(TimeoutException::class)
private fun sendCtrlAndWait(address: String, op: Int, data: ByteArray, timeoutMs: Int): Int {
val seq = nextSeq()
drainAcks()
val payload = ByteArray(1 + data.size)
payload[0] = op.toByte()
System.arraycopy(data, 0, payload, 1, data.size)
writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, payload, seq)
return waitAck(op, timeoutMs)
}
private fun writeOta(address: String, writeChar: String, sid: Int, payload: ByteArray, seq: Int): Boolean {
val frames = BleProtocol.framePb(payload, sid, FLAG_OTA, seq)
return bleManager.writeFrames(
address, writeChar, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS)
}
@Throws(TimeoutException::class)
private fun waitAck(wantOp: Int, timeoutMs: Int): Int {
val deadline = SystemClock.elapsedRealtime() + timeoutMs
while (true) {
val remaining = deadline - SystemClock.elapsedRealtime()
if (remaining <= 0) {
break
}
val pb: ByteArray?
try {
pb = dataAcks.poll(remaining, TimeUnit.MILLISECONDS)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
throw TimeoutException("interrupted waiting for ack op=$wantOp")
}
if (pb == null) {
break
}
if (pb.size >= 2 && (pb[0].toInt() and 0xff) == wantOp) {
return pb[1].toInt() and 0xff
}
// Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting.
}
throw TimeoutException("no ack op=0x" + Integer.toHexString(wantOp) + " within " + timeoutMs + "ms")
}
// ---- connection ----------------------------------------------------------
private fun connectLensResilient(lens: String, address: String, windowMs: Int) {
val deadline = SystemClock.elapsedRealtime() + windowMs
var lastError: String? = "unknown"
var attempt = 0
while (SystemClock.elapsedRealtime() < deadline && !cancelled) {
attempt++
try {
if (bringUpLens(address)) {
emitLog("connected $lens lens (attempt $attempt)")
return
}
lastError = "connect/discover incomplete"
} catch (e: Exception) {
lastError = if (e.message == null) e.toString() else e.message
}
emitLog("$lens connect attempt $attempt failed ($lastError); retrying...")
try {
bleManager.disconnect(address)
} catch (ignored: Exception) {
}
sleepInterruptibly(RETRY_DELAY_MS)
}
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
throw IllegalStateException("could not reach $lens lens: $lastError")
}
private fun bringUpLens(address: String): Boolean {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
return false
}
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
return false
}
// Control-channel notify carries the auth response; mandatory.
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
return false
}
if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
return false
}
if (!authenticate(address)) {
throw IllegalStateException(
"security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it")
}
return true
}
/**
* Complete the sid-0x80 security-auth exchange on this connection. On an
* unbonded phone this triggers SMP pairing (possibly with an OS prompt),
* so the wait is generous. Must succeed before any OTA traffic.
*/
private fun authenticate(address: String): Boolean {
val magic: Int
val latch = CountDownLatch(1)
synchronized(lock) {
magic = nextAuthMagic
nextAuthMagic = if (nextAuthMagic >= 0x7f) 0x60 else nextAuthMagic + 1
pendingAuthMagic = magic
authAckPb = null
authLatch = latch
}
try {
if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH,
BleProtocol.buildAuthenticationRequest(magic), nextSeq())) {
emitLog("auth write failed: $address")
return false
}
val signalled: Boolean
try {
signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return false
}
val pb: ByteArray?
synchronized(lock) {
pb = authAckPb
}
if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) {
emitLog("security auth not acknowledged (" + address
+ ") — if Android shows a Bluetooth pairing request, accept it")
return false
}
emitLog("security auth complete: $address")
return true
} finally {
synchronized(lock) {
pendingAuthMagic = -1
authLatch = null
}
}
}
// ---- firmware container parsing / validation -----------------------------
private class Segment(
@JvmField val off: Int,
@JvmField val ps: Int,
@JvmField val crc: Long,
@JvmField val name: String
)
private fun parseSegments(img: ByteArray): List<Segment> {
if (img.size < 0x40) {
throw IllegalStateException("file is too small to be a firmware image")
}
val n = readU32(img, 8)
if (n <= 0 || n > 64) {
throw IllegalStateException("implausible component count $n (corrupt image?)")
}
val segs = ArrayList<Segment>()
var i = 0
while (i < n) {
val base = 0x40 + i * 16
val crc = readU32(img, base + 12)
val off = readU32(img, base + 4).toInt()
if (off + 128 > img.size) {
throw IllegalStateException("segment $i subheader runs past end of file")
}
val ps = readU32(img, off + 8).toInt()
val name = readCString(img, off + 48, 80)
segs.add(Segment(off, ps, crc, name))
i++
}
return segs
}
private fun validate(img: ByteArray): List<Segment> {
val segs = parseSegments(img)
if (segs.size < MIN_SEGMENTS || segs.size > MAX_SEGMENTS) {
throw IllegalStateException(
"expected " + MIN_SEGMENTS + "-" + MAX_SEGMENTS + " components, found " + segs.size)
}
var main: Segment? = null
for (s in segs) {
val payload = Arrays.copyOfRange(img, s.off + 128, s.off + 128 + s.ps)
val calc = crc32cMsb(payload).toLong() and 0xffffffffL
val subCrc = readU32(img, s.off + 12)
if (calc != s.crc || calc != subCrc) {
throw IllegalStateException("component " + s.name + " CRC32C is stale (image not checksum-fixed)")
}
if (REQUIRED_SEGMENT == s.name) {
main = s
}
}
if (main == null) {
throw IllegalStateException("required component $REQUIRED_SEGMENT not found")
}
checkMainAppFitsMram(img, main)
return segs
}
private fun checkMainAppFitsMram(img: ByteArray, main: Segment) {
if (main.ps < APP_PREAMBLE) {
throw IllegalStateException("main-app payload is smaller than its preamble")
}
val loadAddr = readU32(img, main.off + 128 + 0x14)
val preLen = readU32(img, main.off + 128) and 0xFFFFFFL
if (loadAddr != APP_LOAD_ADDR) {
throw IllegalStateException(
"main-app preamble load address is 0x" + java.lang.Long.toHexString(loadAddr)
+ ", expected 0x" + java.lang.Long.toHexString(APP_LOAD_ADDR))
}
if (preLen != main.ps.toLong()) {
throw IllegalStateException(
"main-app preamble length (" + preLen + ") != staged payload size (" + main.ps + ")")
}
val progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE
if (progEnd > APP_MAX_END) {
val over = progEnd - APP_MAX_END
throw IllegalStateException(
"main-app is too large: programmed region ends at 0x" + java.lang.Long.toHexString(progEnd)
+ ", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)")
}
}
// ---- listener callbacks --------------------------------------------------
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
// Control channel: only the security-auth response is awaited here.
// A value can carry several envelope frames back to back (e.g. the
// response packed with one of the lens's own sid-0x80 notifies).
for (buf in BleProtocol.splitFrames(data)) {
val frame = BleProtocol.parseFrame(buf)
if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) {
synchronized(lock) {
val currentLatch = authLatch
if (currentLatch != null && frame.msgSeq == pendingAuthMagic) {
authAckPb = frame.pb
currentLatch.countDown()
}
}
}
}
return
}
if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equals(characteristicUuid, ignoreCase = true)) {
return // OTA acks arrive on the data-notify char
}
val frame = BleProtocol.parseFrame(data)
if (!frame.ok || frame.pb.size < 2) {
return
}
dataAcks.add(Arrays.copyOf(frame.pb, Math.min(frame.pb.size, 2)))
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (!connected) {
Log.i(TAG, "disconnected: $address")
}
}
// ---- helpers -------------------------------------------------------------
private fun teardown() {
try {
bleManager.close()
} catch (ignored: Exception) {
}
}
private fun resetSeq() {
synchronized(lock) {
nextSeq = 0
}
}
private fun nextSeq(): Int {
synchronized(lock) {
nextSeq = (nextSeq + 1) and 0xff
return nextSeq
}
}
private fun drainAcks() {
dataAcks.clear()
}
private fun sleepInterruptibly(ms: Int) {
try {
Thread.sleep(ms.toLong())
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
}
}
private fun emitLog(line: String) {
Log.i(TAG, line)
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
current.onLog(line)
}
}
}
private fun emitProgress(lens: String, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int,
bytesSent: Long, bytesTotal: Long) {
mainHandler.post {
val current = listener
if (current != null) {
current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal)
}
}
}
private fun emitState(state: String, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onState(state, safeDetail)
}
}
}
private fun emitComplete(success: Boolean, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onComplete(success, safeDetail)
action(current)
}
}
}
@@ -1,110 +1,53 @@
package com.faceclaw.app
import android.bluetooth.BluetoothGatt
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.TimeUnit
/**
* A minimal, stock-firmware-compatible BLE communicator used only to show the
* pre-flash confirmation prompt on the glasses and read back the user's Yes/No
* choice. It is deliberately separate from FaceclawBleCommunicator: the main
* app assumes the custom firmware (image containers, compositor, etc.), whereas
* this must work on unmodified glasses, so it speaks only the stock subset —
* session prelude, a text container, a list container, and heartbeats. No
* images.
*
* It reuses the low-level GATT wrapper (FaceclawBleManager) and the framing /
* protobuf helpers (BleProtocol), but owns its own GATT connection and runs its
* whole lifecycle on a single worker thread. It expects the main app to be
* disconnected while it runs (onboarding, before flashing).
*
* Both arms are connected and security-authenticated up front, even though the
* prompt itself only needs the right arm (the lenses relay messages to each
* other, and acks/events only ever come from the right arm): on an unbonded
* phone the auth exchange is what raises the OS pairing prompt, and doing both
* here means both prompts appear at the start rather than one popping up half
* way through flashing the second lens. While connected, each arm's battery
* level is read (they are independent batteries) so the caller can refuse to
* flash on a low charge.
*
* With `skipPrompt` the on-glasses confirmation is not shown: the flow is just
* connect + auth + battery read + result(approved). Used to re-check the
* battery after the user has already confirmed once.
* Android host for the shared [FlashPromptFlow] (the stock-firmware pre-flash Yes/No prompt
* plus battery read): owns the GATT link (AndroidStockLink) and the worker thread, and
* delivers listener callbacks on the main thread. It expects the main app to be disconnected
* while it runs (onboarding, before flashing). Public API unchanged for
* app/native/flash-prompt-communicator.ts.
*/
class FaceclawFlashPromptCommunicator(
context: Context, rightAddress: String?, leftAddress: String?, warningText: String?, skipPrompt: Boolean) : FaceclawBleListener {
companion object {
private const val TAG = "FaceclawFlashPrompt"
private const val TEXT_NAME = "flashwarn"
private const val LIST_NAME = "flashmenu"
private const val TEXT_CONTAINER_ID = 1
private const val LIST_CONTAINER_ID = 2
// Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid.
private val ITEMS = arrayOf("No, cancel", "Yes, flash")
private const val HEARTBEAT_INTERVAL_MS = 4_000
private const val SELECTION_TIMEOUT_MS = 120_000
private const val CREATE_ACK_TIMEOUT_MS = 3_000
private const val BATTERY_ACK_TIMEOUT_MS = 3_000
private fun ackKey(sid: Int, magic: Int): String {
return "$sid:$magic"
}
}
private val context: Context
private val rightAddress: String
private val leftAddress: String
private val warningText: String
private val skipPrompt: Boolean
private val bleManager: FaceclawBleManager
context: Context,
rightAddress: String?,
leftAddress: String?,
warningText: String?,
skipPrompt: Boolean,
) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private val pendingAcks = ConcurrentHashMap<String, CountDownLatch>()
private val ackPayloads = ConcurrentHashMap<String, ByteArray>()
private val selectionLatch = CountDownLatch(1)
@Volatile
private var listener: FaceclawFlashPromptListener? = null
@Volatile
private var worker: Thread? = null
@Volatile
private var cancelled = false
@Volatile
private var finished = false
@Volatile
private var rightConnected = false
@Volatile
private var leftConnected = false
@Volatile
private var rightLost = false
@Volatile
private var approved: Boolean? = null
@Volatile
private var heartbeatExecutor: ScheduledExecutorService? = null
private var nextMagic = 100
private var nextSeq = 0x40
private val flow =
FlashPromptFlow(
link,
rightAddress ?: "",
leftAddress ?: "",
warningText ?: "",
skipPrompt,
object : FaceclawFlashPromptListener {
override fun onLog(line: String?) {
android.util.Log.i(FlashPromptFlow.TAG, line ?: "")
post { it.onLog(line) }
}
init {
this.context = context.applicationContext
this.rightAddress = rightAddress ?: ""
this.leftAddress = leftAddress ?: ""
this.warningText = warningText ?: ""
this.skipPrompt = skipPrompt
this.bleManager = FaceclawBleManager(this.context)
this.bleManager.setListener(this)
}
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onBattery(rightPercent: Int, leftPercent: Int) = post { it.onBattery(rightPercent, leftPercent) }
override fun onResult(approved: Boolean) = post { it.onResult(approved) }
},
)
fun setListener(listener: FaceclawFlashPromptListener?) {
this.listener = listener
@@ -115,7 +58,7 @@ class FaceclawFlashPromptCommunicator(
if (worker != null) {
return
}
val w = Thread({ run() }, "faceclaw-flash-prompt")
val w = Thread({ flow.run() }, "faceclaw-flash-prompt")
worker = w
w.start()
}
@@ -123,443 +66,22 @@ class FaceclawFlashPromptCommunicator(
/** Abort the prompt (e.g. the user backed out on the phone). */
fun cancel() {
cancelled = true
selectionLatch.countDown()
val w = worker
if (w != null) {
w.interrupt()
}
flow.cancel()
}
fun close() {
cancel()
stopHeartbeat()
try {
bleManager.close()
link.close()
} catch (ignored: Exception) {
}
}
private fun run() {
try {
if (rightAddress.trim().isEmpty()) {
emitState("error", "No right-arm address configured.")
return
}
emitState("connecting", "")
connectArm(rightAddress, "right")
rightConnected = true
if (cancelled) {
teardown()
return
}
if (!leftAddress.trim().isEmpty()) {
connectArm(leftAddress, "left")
leftConnected = true
if (cancelled) {
teardown()
return
}
}
emitState("connected", "")
// Auth both arms now (pairing prompts, if any, happen here) before
// anything else, so both bonds exist by the time flashing starts.
authenticateArm(rightAddress, "right")
if (leftConnected) {
authenticateArm(leftAddress, "left")
}
if (cancelled) {
teardown()
return
}
sendPrelude(rightAddress)
if (!skipPrompt) {
showPrompt(rightAddress)
startHeartbeat()
emitState("prompting", "")
selectionLatch.await(SELECTION_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS)
stopHeartbeat()
if (cancelled) {
emitState("cancelled", "")
teardown()
return
}
if (approved == null) {
if (rightLost) {
emitState("disconnected", "Lost connection to the glasses.")
} else {
emitState("timeout", "No response from the glasses.")
}
teardown()
return
}
try {
sendShutdown()
} catch (ignored: Exception) {
}
if (approved != true) {
emitResult(false)
emitState("result", "declined")
teardown()
return
}
}
// Approved (or prompt skipped): read each arm's battery while the
// links are still up, then report the result.
emitState("battery", "")
readBatteries()
if (cancelled) {
emitState("cancelled", "")
teardown()
return
}
if (rightLost) {
emitState("disconnected", "Lost connection to the glasses.")
teardown()
return
}
finished = true
emitResult(true)
emitState("result", "approved")
teardown()
} catch (e: Exception) {
stopHeartbeat()
if (!cancelled) {
val message = if (e.message == null) e.toString() else e.message
emitState("error", message)
}
teardown()
}
}
private fun connectArm(address: String, arm: String) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw IllegalStateException("could not connect to the $arm arm ($address)")
}
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
throw IllegalStateException("discoverServices failed: $arm arm ($address)")
}
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw IllegalStateException("enableNotifications failed: $arm arm ($address)")
}
emitLog("connected $arm arm")
}
/**
* Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a
* session (and closes the link after ~30 s) without it, and on an
* unbonded phone this is what triggers SMP pairing, so it may sit
* waiting on an OS pairing prompt.
*/
@Throws(InterruptedException::class)
private fun authenticateArm(address: String, arm: String) {
val authMagic = allocMagic()
val authAck = writeAndAwaitAck(
address,
BleProtocol.SID_SECURITY_AUTH,
BleProtocol.FLAG_SECURITY_AUTH,
authMagic,
BleProtocol.buildAuthenticationRequest(authMagic),
ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS)
if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) {
throw IllegalStateException(
"could not authenticate with the $arm arm ($address"
+ ") — if Android shows a Bluetooth pairing request, accept it and try again")
}
emitLog("security auth complete: $arm arm")
}
/** Mandatory session prelude on sid=0x01 (app-launch). Right arm only; it relays. */
@Throws(InterruptedException::class)
private fun sendPrelude(address: String) {
if (writeAndAwaitAck(
address,
BleProtocol.PRELUDE_ACK_SID,
BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC,
BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw IllegalStateException("session prelude not acked: $address")
}
}
@Throws(InterruptedException::class)
private fun showPrompt(address: String) {
// Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0.
// Two attempts: on 2.2.9 the first request sent right after the
// prelude can be dropped while the lens emits its own sid-0x80
// notifications (observed with the device-info settings read).
var pageAck: ByteArray? = null
var attempt = 0
while (attempt < 2 && pageAck == null && !cancelled) {
val magic = allocMagic()
val page = BleProtocol.buildCreatePromptPage(
magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS)
pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS)
if (pageAck == null) {
emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address)
}
attempt++
}
if (pageAck == null) {
throw IllegalStateException("prompt page not acked: $address")
}
emitLog("prompt page shown on $address")
}
/**
* Read each arm's battery via a sid-0x09 settings read on that arm's own
* link (the arms have independent batteries and each answers with its
* own). An arm that does not answer reports -1; the caller decides what
* to do with a partial reading.
*/
@Throws(InterruptedException::class)
private fun readBatteries() {
val right = if (rightConnected) readBattery(rightAddress, "right") else -1
val left = if (leftConnected) readBattery(leftAddress, "left") else -1
emitLog("battery R=" + (if (right < 0) "?" else "$right%") + " L=" + (if (left < 0) "?" else "$left%"))
emitBattery(right, left)
}
@Throws(InterruptedException::class)
private fun readBattery(address: String, arm: String): Int {
var attempt = 0
while (attempt < 2 && !cancelled) {
val magic = allocMagic()
val ack = writeAndAwaitAck(
address,
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
magic,
BleProtocol.buildSettingsQuery(magic),
BATTERY_ACK_TIMEOUT_MS)
if (ack == null) {
emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm")
attempt++
continue
}
val snapshot: BleProtocol.BatterySnapshot? = BleProtocol.parseSettingsBattery(ack)
if (snapshot != null) {
return snapshot.battery
}
emitLog("battery read ack had no battery field: $arm arm")
attempt++
}
return -1
}
/** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */
@Throws(InterruptedException::class)
private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? {
val latch = CountDownLatch(1)
val key = ackKey(sid, magic)
pendingAcks[key] = latch
try {
if (!writeFrame(address, sid, flag, payload)) {
return null
}
if (!latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS)) {
return null
}
val pb = ackPayloads[key]
return pb ?: ByteArray(0)
} finally {
pendingAcks.remove(key, latch)
ackPayloads.remove(key)
}
}
private fun writeFrame(address: String, sid: Int, flag: Int, payload: ByteArray): Boolean {
val seq: Int
synchronized(lock) {
seq = nextSeq++ and 0xff
}
val frames = BleProtocol.framePb(payload, sid, flag, seq)
return bleManager.writeFrames(
address,
BleProtocol.WRITE_CHAR_UUID,
frames,
AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE),
ConnectionOptions.WRITE_TIMEOUT_MS)
}
private fun startHeartbeat() {
stopHeartbeat()
val executor = Executors.newSingleThreadScheduledExecutor { runnable ->
val t = Thread(runnable, "faceclaw-flash-hb")
t.isDaemon = true
t
}
heartbeatExecutor = executor
executor.scheduleWithFixedDelay(
{ sendHeartbeat() }, HEARTBEAT_INTERVAL_MS.toLong(), HEARTBEAT_INTERVAL_MS.toLong(), TimeUnit.MILLISECONDS)
}
private fun sendHeartbeat() {
if (finished || cancelled) {
return
}
try {
val heartbeat = BleProtocol.buildHeartbeat(allocMagic())
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat)
}
} catch (e: Exception) {
Log.w(TAG, "heartbeat failed: " + e.message)
}
}
private fun stopHeartbeat() {
val executor = heartbeatExecutor
heartbeatExecutor = null
if (executor != null) {
executor.shutdownNow()
}
}
private fun sendShutdown() {
val shutdown = BleProtocol.buildShutdown(allocMagic(), 0)
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown)
}
}
private fun teardown() {
finished = true
stopHeartbeat()
try {
bleManager.close()
} catch (ignored: Exception) {
}
rightConnected = false
leftConnected = false
}
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
return
}
// A value can carry several envelope frames back to back.
for (buf in BleProtocol.splitFrames(data)) {
handleFrame(address, buf)
}
}
private fun handleFrame(address: String?, buf: ByteArray) {
val frame = BleProtocol.parseFrame(buf)
if (!frame.ok) {
return
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
handleEvent(frame)
return
}
if (frame.msgSeq >= 0) {
val key = ackKey(frame.sid, frame.msgSeq)
val latch = pendingAcks[key]
if (latch != null) {
ackPayloads[key] = frame.pb
latch.countDown()
}
}
}
private fun handleEvent(frame: BleProtocol.ParsedFrame) {
val selection: BleProtocol.ListSelection? = BleProtocol.parseListSelection(frame)
if (selection == null || LIST_NAME != selection.containerName) {
return
}
// Only a confirmed click is a decision; scroll/highlight changes are ignored.
if (selection.eventType != BleProtocol.EVENT_CLICK) {
return
}
val itemName = selection.itemName
val yes = selection.itemIndex == 1
|| (itemName != null && itemName.lowercase().startsWith("yes"))
synchronized(lock) {
if (finished) {
return
}
finished = true
approved = yes
}
emitLog("selection: " + (if (yes) "flash" else "cancel") + " (index " + selection.itemIndex + ")")
selectionLatch.countDown()
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (connected) {
return
}
if (address!!.equals(leftAddress, ignoreCase = true)) {
// The prompt itself only needs the right arm; a dropped left link
// just means no left battery reading.
leftConnected = false
emitLog("left arm disconnected")
return
}
if (address.equals(rightAddress, ignoreCase = true)) {
rightConnected = false
synchronized(lock) {
if (!finished && !cancelled) {
rightLost = true
finished = true
selectionLatch.countDown()
}
}
}
}
private fun allocMagic(): Int {
synchronized(lock) {
val magic = nextMagic
nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1
return magic
}
}
private fun emitLog(line: String) {
Log.i(TAG, line)
private fun post(action: (FaceclawFlashPromptListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
current.onLog(line)
}
}
}
private fun emitState(state: String, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onState(state, safeDetail)
}
}
}
private fun emitBattery(rightPercent: Int, leftPercent: Int) {
mainHandler.post {
val current = listener
if (current != null) {
current.onBattery(rightPercent, leftPercent)
}
}
}
private fun emitResult(approvedResult: Boolean) {
mainHandler.post {
val current = listener
if (current != null) {
current.onResult(approvedResult)
action(current)
}
}
}
@@ -7,27 +7,18 @@ import okhttp3.Request
import okhttp3.Response
/**
* Shared HTTP identity for the Java networking helpers. The version lives in
* Android okhttp face of the shared HTTP identity. The version lives in
* app/version.ts, so the TypeScript side pushes the real string in at startup
* (see installNativeUserAgent in app/util/http.ts); the default below is only
* a fallback for requests that somehow beat that call.
* (see installNativeUserAgent in app/util/http.ts); the value itself is held
* by the shared HttpIdentity so every platform's networking reads one string.
*/
class FaceclawHttp private constructor() {
companion object {
@Volatile
private var userAgent = "Faceclaw"
@JvmStatic
fun setUserAgent(value: String?) = HttpIdentity.setUserAgent(value)
@JvmStatic
fun setUserAgent(value: String?) {
if (value != null && !value.trim().isEmpty()) {
userAgent = value.trim()
}
}
@JvmStatic
fun getUserAgent(): String {
return userAgent
}
fun getUserAgent(): String = HttpIdentity.getUserAgent()
/** Stamps our User-Agent on any request that doesn't already carry one. */
@JvmStatic
@@ -39,7 +30,7 @@ class FaceclawHttp private constructor() {
if (request.header("User-Agent") != null) {
return chain.proceed(request)
}
return chain.proceed(request.newBuilder().header("User-Agent", userAgent).build())
return chain.proceed(request.newBuilder().header("User-Agent", HttpIdentity.getUserAgent()).build())
}
}
}
@@ -2,23 +2,23 @@ package com.faceclaw.app
import android.util.Log
/**
* Android LC3 decoder for G2 microphone packets: the liblc3 JNI codec behind the shared
* [Lc3PacketFramer], which owns the counter/duplicate/trailer logic and statistics.
*/
class FaceclawLc3Decoder {
companion object {
private const val TAG = "FaceclawLc3"
const val SAMPLE_RATE = 16000
const val FRAME_US = 10000
const val FRAME_BYTES = 40
const val FRAMES_PER_PACKET = 5
const val PACKET_BYTES = 205
const val COUNTER_OFFSET = 204
const val SAMPLES_PER_FRAME = 160
const val SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
const val SSR_OFFSET = 200
const val ANGLE_OFFSET = 202
const val SAMPLE_RATE = Lc3PacketFramer.SAMPLE_RATE
const val FRAME_US = Lc3PacketFramer.FRAME_US
const val FRAME_BYTES = Lc3PacketFramer.FRAME_BYTES
const val FRAMES_PER_PACKET = Lc3PacketFramer.FRAMES_PER_PACKET
const val PACKET_BYTES = Lc3PacketFramer.PACKET_BYTES
const val COUNTER_OFFSET = Lc3PacketFramer.COUNTER_OFFSET
const val SAMPLES_PER_FRAME = Lc3PacketFramer.SAMPLES_PER_FRAME
const val SAMPLES_PER_PACKET = Lc3PacketFramer.SAMPLES_PER_PACKET
const val SSR_OFFSET = Lc3PacketFramer.SSR_OFFSET
const val ANGLE_OFFSET = Lc3PacketFramer.ANGLE_OFFSET
init {
System.loadLibrary("faceclaw_lc3")
@@ -32,17 +32,15 @@ class FaceclawLc3Decoder {
private external fun nativeDestroy(handle: Long)
}
private var nativeHandle: Long
private var lastCounter = -1
private var realPackets: Long = 0
private var duplicatePackets: Long = 0
private var missingPackets: Long = 0
private var decodeErrors: Long = 0
private var lastSsr = 0
private var lastAngleDegrees = 0
private var nativeHandle: Long = nativeCreate(FRAME_US, SAMPLE_RATE)
private val framer = Lc3PacketFramer(object : Lc3Codec {
override fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int {
// The JNI entry decodes the five 40-byte frames at the start of the packet.
return nativeDecodePacket(nativeHandle, packet, pcmOut)
}
})
init {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE)
if (nativeHandle == 0L) {
throw IllegalStateException("Could not create LC3 decoder")
}
@@ -54,70 +52,30 @@ class FaceclawLc3Decoder {
throw IllegalStateException("LC3 decoder is closed")
}
if (packet == null || packet.size != PACKET_BYTES) {
decodeErrors++
Log.w(TAG, "unexpected G2 audio packet length=" + (packet?.size ?: -1))
return 0
}
if (pcmOut == null || pcmOut.size < SAMPLES_PER_PACKET) {
throw IllegalArgumentException("pcmOut must hold $SAMPLES_PER_PACKET samples")
}
val counter = packet[COUNTER_OFFSET].toInt() and 0xff
if (lastCounter >= 0) {
val gap = (counter - lastCounter) and 0xff
if (gap == 0) {
duplicatePackets++
return 0
}
val missing = gap - 1
if (missing > 0) {
missingPackets += missing.toLong()
}
}
val decoded = nativeDecodePacket(nativeHandle, packet, pcmOut)
if (decoded <= 0) {
decodeErrors++
return 0
}
realPackets++
lastCounter = counter
lastSsr = ((packet[SSR_OFFSET].toInt() and 0xff) or (packet[SSR_OFFSET + 1].toInt() shl 8)).toShort().toInt()
lastAngleDegrees = ((packet[ANGLE_OFFSET].toInt() and 0xff) or (packet[ANGLE_OFFSET + 1].toInt() shl 8)).toShort().toInt()
return decoded
return framer.decodePacket(packet, pcmOut)
}
/** Firmware signal-strength ratio from the most recent decoded packet. */
@Synchronized
fun getLastSsr(): Int {
return lastSsr
}
fun getLastSsr(): Int = framer.lastSsr
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
@Synchronized
fun getLastAngleDegrees(): Int {
return lastAngleDegrees
}
fun getLastAngleDegrees(): Int = framer.lastAngleDegrees
@Synchronized
fun getRealPackets(): Long {
return realPackets
}
fun getRealPackets(): Long = framer.realPackets
@Synchronized
fun getDuplicatePackets(): Long {
return duplicatePackets
}
fun getDuplicatePackets(): Long = framer.duplicatePackets
@Synchronized
fun getMissingPackets(): Long {
return missingPackets
}
fun getMissingPackets(): Long = framer.missingPackets
@Synchronized
fun getDecodeErrors(): Long {
return decodeErrors
}
fun getDecodeErrors(): Long = framer.decodeErrors
@Synchronized
fun close() {
@@ -2,13 +2,11 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.security.MessageDigest
import java.io.InputStream
import java.util.concurrent.TimeUnit
import okhttp3.Call
@@ -17,10 +15,10 @@ import okhttp3.Request
/**
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
* restarts, verifies a pinned sha256 (hashed incrementally, including the
* resumed prefix), then renames into place. Listener callbacks are posted to
* the constructing thread's Looper, like the other Faceclaw bridges.
* Face). The resume/hash/verify/rename policy is the shared ResumableDownload;
* this class supplies okhttp, java.io storage, the worker thread, and posts
* listener callbacks to the constructing thread's Looper, like the other
* Faceclaw bridges.
*/
class FaceclawModelDownloader(
url: String,
@@ -30,35 +28,31 @@ class FaceclawModelDownloader(
listener: FaceclawModelDownloaderListener?
) {
private val callbackHandler: Handler
private val url: String
private val destFile: File
private val partFile: File
private val expectedSha256: String
private val expectedTotalBytes: Long
private val listener: FaceclawModelDownloaderListener
@Volatile
private var cancelled = false
@Volatile
private var call: Call? = null
private val transport = OkHttpTransport()
private val download: ResumableDownload
private var worker: Thread? = null
init {
if (listener == null) throw IllegalArgumentException("listener is required")
val looper = Looper.myLooper()
this.callbackHandler = Handler(looper ?: Looper.getMainLooper())
this.url = url
this.destFile = File(destPath)
this.partFile = File("$destPath.part")
this.expectedSha256 = expectedSha256?.lowercase() ?: ""
this.expectedTotalBytes = expectedTotalBytes
this.listener = listener
val posted = object : FaceclawModelDownloaderListener {
override fun onProgress(bytesDownloaded: Long, totalBytes: Long) {
callbackHandler.post { listener.onProgress(bytesDownloaded, totalBytes) }
}
override fun onDone(path: String?) {
callbackHandler.post { listener.onDone(path) }
}
override fun onError(message: String?) {
callbackHandler.post { listener.onError(message) }
}
}
download = ResumableDownload(url, File(destPath).absolutePath, expectedSha256, expectedTotalBytes, transport, FileStorage, posted)
}
companion object {
private const val TAG = "FaceclawModelDl"
private const val PROGRESS_INTERVAL_MS = 500L
@Volatile
private var sharedClient: OkHttpClient? = null
@@ -79,120 +73,76 @@ class FaceclawModelDownloader(
}
fun start() {
val thread = Thread({ run() }, "FaceclawModelDl")
val thread = Thread({ download.run() }, "FaceclawModelDl")
worker = thread
thread.start()
}
/** Stops the download but keeps the .part file so a later start() resumes. */
fun cancel() {
cancelled = true
val current = call
current?.cancel()
download.cancel()
transport.call?.cancel()
}
private fun run() {
try {
if (destFile.exists() && destFile.length() > 0) {
post { listener.onDone(destFile.absolutePath) }
return
}
destFile.parentFile.mkdirs()
var digest = MessageDigest.getInstance("SHA-256")
var offset = 0L
if (partFile.exists()) {
offset = hashExistingPrefix(digest)
if (cancelled) return
}
/** One call at a time per downloader, so cancel() can abort a connect in progress. */
private class OkHttpTransport : DownloadTransport {
@Volatile
var call: Call? = null
override fun open(url: String, rangeFrom: Long): DownloadResponse {
val builder = Request.Builder().url(url)
if (offset > 0) builder.header("Range", "bytes=$offset-")
val newCall = getClient().newCall(builder.build())
call = newCall
if (rangeFrom > 0) builder.header("Range", "bytes=$rangeFrom-")
val call: Call = getClient().newCall(builder.build())
this.call = call
val response = call.execute()
val body = response.body
val input: InputStream? = body?.byteStream()
return object : DownloadResponse {
override val code: Int = response.code
override val contentLength: Long = body?.contentLength() ?: -1L
newCall.execute().use { response ->
val body = response.body
if (!response.isSuccessful || body == null) {
postError("Model download failed: HTTP " + response.code)
return
}
val resumed = response.code == 206
if (offset > 0 && !resumed) {
// Server ignored the Range header; start over.
offset = 0L
digest = MessageDigest.getInstance("SHA-256")
}
val total = if (expectedTotalBytes > 0) expectedTotalBytes
else (if (body.contentLength() > 0) offset + body.contentLength() else -1L)
override fun read(buffer: ByteArray): Int = input?.read(buffer) ?: -1
FileOutputStream(partFile, resumed).use { out ->
body.byteStream().use { input ->
val buffer = ByteArray(1 shl 16)
var done = offset
var lastProgressAt = 0L
var read: Int
while (input.read(buffer).also { read = it } != -1) {
if (cancelled) return
out.write(buffer, 0, read)
digest.update(buffer, 0, read)
done += read
val now = System.currentTimeMillis()
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
lastProgressAt = now
val doneFinal = done
val totalFinal = total
post { listener.onProgress(doneFinal, totalFinal) }
}
}
}
override fun close() {
call.cancel()
response.close()
}
}
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
postError("Model download ended early (" + partFile.length() + " of "
+ expectedTotalBytes + " bytes); try again to resume")
return
}
val actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest())
if (!expectedSha256.isEmpty() && actualSha != expectedSha256) {
partFile.delete()
postError("Model download was corrupted (checksum mismatch); download it again")
return
}
if (!partFile.renameTo(destFile)) {
postError("Could not move the downloaded model into place")
return
}
post { listener.onDone(destFile.absolutePath) }
} catch (e: Exception) {
if (cancelled) return
Log.w(TAG, "download failed", e)
postError("Model download failed: $e")
}
}
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
@Throws(IOException::class)
private fun hashExistingPrefix(digest: MessageDigest): Long {
var hashed = 0L
FileInputStream(partFile).use { input ->
val buffer = ByteArray(1 shl 16)
var read: Int
while (input.read(buffer).also { read = it } != -1) {
if (cancelled) return hashed
digest.update(buffer, 0, read)
hashed += read
private object FileStorage : DownloadStorage {
override fun length(path: String): Long = File(path).let { if (it.exists()) it.length() else -1L }
override fun prepareParent(path: String) {
File(path).parentFile?.mkdirs()
}
override fun openWrite(path: String, append: Boolean): FileSink {
val out = FileOutputStream(path, append)
return object : FileSink {
override fun write(bytes: ByteArray, offset: Int, length: Int) = out.write(bytes, offset, length)
override fun close() = out.close()
}
}
return hashed
}
private fun post(runnable: Runnable) {
callbackHandler.post(runnable)
}
override fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long {
var consumed = 0L
FileInputStream(path).use { input ->
val buffer = ByteArray(1 shl 16)
while (true) {
val read = input.read(buffer)
if (read < 0) break
if (!consumer(buffer, read)) break
consumed += read
}
}
return consumed
}
private fun postError(message: String) {
post { listener.onError(message) }
override fun rename(from: String, to: String): Boolean = File(from).renameTo(File(to))
override fun delete(path: String): Boolean = File(path).delete()
}
}
@@ -2,20 +2,19 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.net.InetAddress
import java.net.NetworkInterface
import java.util.Collections
import org.json.JSONArray
import java.net.ServerSocket
import java.net.Socket
import java.nio.charset.StandardCharsets
import java.nio.ByteBuffer
import java.security.MessageDigest
import java.security.SecureRandom
import org.json.JSONObject
import java.util.Collections
/** Explicit-address, bounded input port. Complete requests notify JS on the main thread. */
/**
* Explicit-address, bounded input port. The request state machine lives in the shared
* [RemoteInputSession]; this adapter supplies java.net sockets, SecureRandom and main-thread
* delivery, and keeps the JVM API TypeScript calls (app/native/remote-input.ts).
*/
class FaceclawRemoteInput {
companion object {
private val INSTANCE = FaceclawRemoteInput()
@@ -23,130 +22,91 @@ class FaceclawRemoteInput {
@JvmStatic
fun interfaces(): String {
val result = JSONArray()
val all = ArrayList<InterfaceAddress>()
try {
for (device in Collections.list(NetworkInterface.getNetworkInterfaces())) {
if (!device.isUp) continue
for (address in Collections.list(device.inetAddresses)) {
if (address.isLoopbackAddress || address.isLinkLocalAddress || address.isAnyLocalAddress) continue
result.put(JSONObject().put("name", device.name).put("address", address.hostAddress)
.put("pointToPoint", device.isPointToPoint))
all.add(InterfaceAddress(
interfaceName = device.name,
address = address.hostAddress ?: continue,
up = device.isUp,
pointToPoint = device.isPointToPoint,
loopback = address.isLoopbackAddress,
linkLocal = address.isLinkLocalAddress,
anyLocal = address.isAnyLocalAddress,
))
}
}
} catch (ignored: Exception) {}
return result.toString()
}
private fun hex(bytes: ByteArray): String {
val result = StringBuilder(bytes.size * 2)
for (b in bytes) result.append(String.format(java.util.Locale.ROOT, "%02x", b.toInt() and 255))
return result.toString()
return RemoteInputSession.interfacesJson(all)
}
}
private var server: ServerSocket? = null
private var client: Socket? = null
private var pending: String? = null
private var reply: String? = null
private var nextId = 0L
private var requestId = 0L
private var deadline = 0L
private val mainHandler = Handler(Looper.getMainLooper())
private var requestListener: Runnable? = null
private val session = RemoteInputSession(
port = AndroidLocalServerPort,
random = object : RandomSource {
private val secureRandom = SecureRandom()
override fun nextBytes(count: Int): ByteArray = ByteArray(count).also { secureRandom.nextBytes(it) }
},
dispatch = { action -> mainHandler.post(action) },
platform = AndroidProtocolPlatform,
)
@Synchronized fun setRequestListener(listener: Runnable?) { requestListener = listener }
fun setRequestListener(listener: Runnable?) { session.setRequestListener(listener?.let { { it.run() } }) }
private fun notifyRequestReady(id: Long) {
// Called with this monitor held; posting avoids entering JS on the socket thread.
mainHandler.post {
val callback: Runnable?
synchronized(this) {
if (requestId != id || pending == null || System.currentTimeMillis() >= deadline) return@post
callback = requestListener
}
callback?.run()
}
}
fun start(port: Int): String { return startAddress(port, "127.0.0.1") }
fun startAddress(port: Int, address: String): String = session.start(port, address)
fun stop() = session.stop()
fun nextRequest(): String? = session.nextRequest()
fun complete(id: Long, value: String?) = session.complete(id, value)
fun randomSecret(): String = session.randomSecret()
fun hash(value: String): String = RemoteInputSession.tokenHash(value)
}
@Synchronized fun start(port: Int): String { return startAddress(port, "127.0.0.1") }
@Synchronized fun startAddress(port: Int, address: String): String {
if (server != null) return ""
try {
// Numeric local addresses only; never resolve hostnames or bind a wildcard.
if (!address.matches(Regex("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?"))) return "Invalid local address."
private object AndroidLocalServerPort : LocalServerPort {
override fun isLocalUnicastAddress(address: String): Boolean {
return try {
// Numeric literals only reach here, so getByName never resolves a hostname.
val local = InetAddress.getByName(address)
if (local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null)
return "Address is not assigned to a local interface."
val listener = ServerSocket(port, 8, local)
server = listener
val thread = Thread({ serve(listener) }, "FaceclawInput")
thread.isDaemon = true
thread.start()
return ""
} catch (e: Exception) { return "Could not open local input port $port." }
}
@Synchronized fun stop() {
try { server?.close() } catch (ignored: Exception) {}
try { client?.close() } catch (ignored: Exception) {}
server = null; client = null; pending = null; requestId = 0; reply = null
(this as java.lang.Object).notifyAll()
}
@Synchronized fun nextRequest(): String? {
if (System.currentTimeMillis() >= deadline) { pending = null; return null }
val result = pending; pending = null; return result
}
@Synchronized fun complete(id: Long, value: String?) {
if (id == requestId && System.currentTimeMillis() < deadline) { reply = value; (this as java.lang.Object).notifyAll() }
}
private fun serve(listener: ServerSocket) {
while (!listener.isClosed) {
try {
listener.accept().use { socket ->
synchronized(this) { if (server !== listener) return; client = socket }
socket.soTimeout = 5000
socket.tcpNoDelay = true
val readDeadline = System.currentTimeMillis() + 5000
val body = ByteArrayOutputStream()
var b = -1
while (true) {
val remaining = readDeadline - System.currentTimeMillis()
if (remaining <= 0) throw Exception("Read timeout")
socket.soTimeout = remaining.toInt()
b = socket.getInputStream().read()
if (b == -1 || b == '\n'.code) break
if (body.size() >= 65536) throw Exception("Invalid frame")
body.write(b)
}
if (b != '\n'.code) return@use
val text = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(body.toByteArray())).toString()
val response: String
synchronized(this) {
if (server !== listener) return
requestId = ++nextId
deadline = System.currentTimeMillis() + 5000
reply = null
pending = JSONObject().put("id", requestId).put("expiresAt", deadline)
.put("body", text).toString()
notifyRequestReady(requestId)
while (reply == null && server === listener && System.currentTimeMillis() < deadline) {
(this as java.lang.Object).wait(Math.max(1L, deadline - System.currentTimeMillis()))
}
response = reply ?: "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"
if (server === listener) { pending = null; reply = null; requestId = 0 }
}
socket.getOutputStream().write((response + "\n").toByteArray(StandardCharsets.UTF_8))
}
} catch (ignored: Exception) {
// Never log request bodies: they contain credentials and possibly private text.
} finally {
synchronized(this) { if (server === listener) client = null }
}
!(local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null)
} catch (e: Exception) {
false
}
}
fun randomSecret(): String {
val bytes = ByteArray(32); SecureRandom().nextBytes(bytes); return hex(bytes)
}
fun hash(value: String): String {
try { return hex(MessageDigest.getInstance("SHA-256").digest(value.toByteArray(StandardCharsets.UTF_8))) }
catch (e: Exception) { throw IllegalStateException("SHA-256 unavailable", e) }
override fun bind(address: String, port: Int): RemoteInputListener {
val server = ServerSocket(port, 8, InetAddress.getByName(address))
return object : RemoteInputListener {
override fun accept(): RemoteInputConnection? {
val socket = try { server.accept() } catch (e: IOException) { return null }
return AndroidConnection(socket)
}
override val isClosed: Boolean get() = server.isClosed
override fun close() = server.close()
}
}
}
private class AndroidConnection(private val socket: Socket) : RemoteInputConnection {
init {
socket.soTimeout = 5000
socket.tcpNoDelay = true
}
override fun readByte(timeoutMs: Long): Int {
return try {
socket.soTimeout = timeoutMs.coerceIn(1L, Int.MAX_VALUE.toLong()).toInt()
socket.getInputStream().read()
} catch (e: IOException) {
-1
}
}
override fun write(bytes: ByteArray) {
socket.getOutputStream().write(bytes)
socket.getOutputStream().flush()
}
override fun close() = socket.close()
}
@@ -4,9 +4,6 @@ import android.content.Context
import android.content.SharedPreferences
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.util.concurrent.CopyOnWriteArrayList
/**
* App settings store shared by every JS isolate (main thread and app
@@ -22,16 +19,26 @@ import java.util.concurrent.CopyOnWriteArrayList
* NativeScript worker threads run a message loop, so both the main thread
* and workers have a Looper; a listener registered from a Looper-less thread
* is accepted but never notified (it can still read fresh values on demand).
* The reads/writes and the fan-out live in the shared SettingsChangeHub.
*/
class FaceclawSettings private constructor(context: Context) {
private val prefs: SharedPreferences = context.applicationContext
.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
private val listeners = CopyOnWriteArrayList<ListenerEntry>()
private val hub = SettingsChangeHub(object : SettingsStorage {
override fun getString(key: String?, defaultValue: String?): String? = prefs.getString(key, defaultValue)
private class ListenerEntry(val listener: FaceclawSettingsListener, val handler: Handler?)
override fun putString(key: String?, value: String?) {
prefs.edit().putString(key, value).apply()
}
override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = prefs.getBoolean(key, defaultValue)
override fun putBoolean(key: String?, value: Boolean) {
prefs.edit().putBoolean(key, value).apply()
}
})
companion object {
private const val TAG = "FaceclawSettings"
private const val PREFS_NAME = "faceclaw_settings"
@Volatile
private var instance: FaceclawSettings? = null
@@ -58,23 +65,13 @@ class FaceclawSettings private constructor(context: Context) {
}
}
fun getString(key: String?, defaultValue: String?): String? {
return prefs.getString(key, defaultValue)
}
fun getString(key: String?, defaultValue: String?): String? = hub.getString(key, defaultValue)
fun setString(key: String?, value: String?) {
prefs.edit().putString(key, value).apply()
notifyChanged(key)
}
fun setString(key: String?, value: String?) = hub.setString(key, value)
fun getBoolean(key: String?, defaultValue: Boolean): Boolean {
return prefs.getBoolean(key, defaultValue)
}
fun getBoolean(key: String?, defaultValue: Boolean): Boolean = hub.getBoolean(key, defaultValue)
fun setBoolean(key: String?, value: Boolean) {
prefs.edit().putBoolean(key, value).apply()
notifyChanged(key)
}
fun setBoolean(key: String?, value: Boolean) = hub.setBoolean(key, value)
/**
* Register a change listener. Must be called from the thread whose
@@ -83,30 +80,9 @@ class FaceclawSettings private constructor(context: Context) {
*/
fun registerListener(listener: FaceclawSettingsListener) {
val looper = Looper.myLooper()
if (looper == null) {
Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified")
}
listeners.add(ListenerEntry(listener, if (looper != null) Handler(looper) else null))
val handler = if (looper != null) Handler(looper) else null
hub.register(listener, if (handler != null) SettingsDispatcher { action -> handler.post(action) } else null)
}
fun unregisterListener(listener: FaceclawSettingsListener?) {
for (entry in listeners) {
if (entry.listener === listener) {
listeners.remove(entry)
}
}
}
private fun notifyChanged(key: String?) {
for (entry in listeners) {
val handler = entry.handler ?: continue
handler.post {
try {
entry.listener.onSettingChanged(key)
} catch (e: Exception) {
Log.w(TAG, "settings listener failed for key $key", e)
}
}
}
}
fun unregisterListener(listener: FaceclawSettingsListener?) = hub.unregister(listener)
}
@@ -11,28 +11,9 @@ import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig
* utterance; matching/enrollment happens on the TS side (cosine similarity
* against the stored speaker profiles).
*/
class FaceclawSpeakerId(private val modelPath: String?) {
class FaceclawSpeakerId(private val modelPath: String?) : SpeakerEmbedder {
companion object {
private const val TAG = "FaceclawSpeakerId"
@JvmStatic
private fun l2Normalize(v: FloatArray?): FloatArray? {
if (v == null) {
return null
}
var sum = 0.0
for (x in v) {
sum += x.toDouble() * x
}
val norm = Math.sqrt(sum)
if (norm <= 0) {
return v
}
for (i in v.indices) {
v[i] = (v[i] / norm).toFloat()
}
return v
}
}
private var extractor: SpeakerEmbeddingExtractor? = null
@@ -71,20 +52,18 @@ class FaceclawSpeakerId(private val modelPath: String?) {
* is too short to embed (under ~0.5 s).
*/
@Synchronized
fun embed(pcm16le: ByteArray?, sampleRate: Int): FloatArray? {
if (pcm16le == null || pcm16le.size < sampleRate) {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedNullable(pcm16le, sampleRate)
@Synchronized
fun embedNullable(pcm16le: ByteArray?, sampleRate: Int): FloatArray? {
if (pcm16le == null || !VoicePrint.isLongEnoughToEmbed(pcm16le.size, sampleRate)) {
return null
}
if (!ensureLoaded()) {
return null
}
val extractor = this.extractor!!
val count = pcm16le.size / 2
val samples = FloatArray(count)
for (i in 0 until count) {
val s = ((pcm16le[i * 2].toInt() and 0xff) or (pcm16le[i * 2 + 1].toInt() shl 8)).toShort()
samples[i] = s / 32768.0f
}
val samples = AudioSegmentation.pcm16leToFloat(pcm16le)
var stream: OnlineStream? = null
try {
stream = extractor.createStream()
@@ -94,7 +73,7 @@ class FaceclawSpeakerId(private val modelPath: String?) {
return null
}
val embedding = extractor.compute(stream)
return l2Normalize(embedding)
return VoicePrint.l2Normalize(embedding)
} catch (t: Throwable) {
Log.w(TAG, "embedding failed", t)
return null
@@ -105,6 +84,8 @@ class FaceclawSpeakerId(private val modelPath: String?) {
}
}
override fun release() = close()
@Synchronized
fun close() {
val extractor = this.extractor
@@ -4,7 +4,6 @@ import android.os.Handler
import android.os.Looper
import android.util.Log
import java.io.BufferedReader
import java.io.IOException
import java.util.concurrent.TimeUnit
@@ -23,6 +22,10 @@ import okhttp3.Response
* is delivered line by line as it arrives, so the JS side can parse SSE
* events incrementally. Like FaceclawWebSocket, listener callbacks are posted
* to the Looper of the thread that constructed this object.
*
* The line splitting, error-body and overflow policy live in the shared
* SseStream; this class only supplies okhttp as the transport and the Handler
* delivery.
*/
class FaceclawSseRequest
/**
@@ -32,7 +35,8 @@ class FaceclawSseRequest
*/
constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener: FaceclawSseListener?) {
private val callbackHandler: Handler
private val call: Call
private val stream = SseStream()
private val transfer: Cancellable
@Volatile
private var cancelled = false
@@ -45,63 +49,49 @@ constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener:
}
val looper = Looper.myLooper()
callbackHandler = Handler(looper ?: Looper.getMainLooper())
val builder = Request.Builder()
.url(url.trim())
.post((jsonBody ?: "").toRequestBody(JSON))
if (headers != null) {
var i = 0
while (i + 1 < headers.size) {
val name = headers[i]
val value = headers[i + 1]
if (name != null && !name.isEmpty() && value != null) {
builder.addHeader(name, value)
}
i += 2
}
}
call = getClient().newCall(builder.build())
call.enqueue(object : Callback {
override fun onFailure(call: Call, e: IOException) {
postFailure(listener, e.toString())
val sink = object : ChunkSink {
override fun onStatus(code: Int) {
stream.onStatus(code)
}
override fun onResponse(call: Call, response: Response) {
try {
response.body.use { body ->
if (!response.isSuccessful) {
val code = response.code
var errorBody = ""
try {
errorBody = body?.string() ?: ""
} catch (e: IOException) {
Log.w(TAG, "error body read failed", e)
}
val finalBody = errorBody
post { listener.onHttpError(code, finalBody) }
return
}
if (body == null) {
post { listener.onComplete() }
return
}
val reader = BufferedReader(body.charStream())
var line: String?
while (reader.readLine().also { line = it } != null) {
val finalLine = line
post { listener.onLine(finalLine) }
}
post { listener.onComplete() }
}
} catch (e: IOException) {
postFailure(listener, e.toString())
}
override fun onChunk(bytes: ByteArray, offset: Int, length: Int) {
stream.onChunk(bytes, offset, length)
drain(listener)
}
})
override fun onComplete() {
stream.onComplete()
drain(listener)
}
override fun onFailure(message: String) {
stream.onFailure(message)
drain(listener)
}
}
transfer = OkHttpStreaming.post(url.trim(), jsonBody ?: "", SseStream.unpackHeaders(headers), sink)
}
fun cancel() {
cancelled = true
call.cancel()
stream.cancel()
transfer.cancel()
}
/** Hands every queued event to the listener on the constructing thread, in order. */
private fun drain(listener: FaceclawSseListener) {
val events = stream.takeEvents()
if (events.isEmpty()) return
post {
for (event in events) {
if (cancelled) return@post
try {
SseStream.deliver(event, listener)
} catch (t: Throwable) {
Log.w(TAG, "listener callback failed", t)
}
}
}
}
private fun post(runnable: Runnable) {
@@ -112,19 +102,47 @@ constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener:
if (cancelled) {
return@post
}
try {
runnable.run()
} catch (t: Throwable) {
Log.w(TAG, "listener callback failed", t)
}
runnable.run()
}
}
private fun postFailure(listener: FaceclawSseListener, message: String) {
if (cancelled) {
return
/** okhttp transport: enqueues the call and feeds the body to the sink in chunks. */
private object OkHttpStreaming : StreamingHttp {
override fun post(url: String, body: String, headers: List<Pair<String, String>>, sink: ChunkSink): Cancellable {
val builder = Request.Builder()
.url(url)
.post(body.toRequestBody(JSON))
for ((name, value) in headers) builder.addHeader(name, value)
val call = getClient().newCall(builder.build())
call.enqueue(object : Callback {
override fun onFailure(call: Call, e: IOException) {
sink.onFailure(e.toString())
}
override fun onResponse(call: Call, response: Response) {
try {
response.use {
sink.onStatus(response.code)
val input = response.body?.byteStream()
if (input != null) {
val buffer = ByteArray(8192)
while (true) {
val read = input.read(buffer)
if (read < 0) break
if (read > 0) sink.onChunk(buffer, 0, read)
}
}
sink.onComplete()
}
} catch (e: IOException) {
sink.onFailure(e.toString())
}
}
})
return object : Cancellable {
override fun cancel() = call.cancel()
}
}
post { listener.onFailure(message) }
}
companion object {
@@ -141,9 +159,14 @@ constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener:
// Streaming responses can pause between events (e.g. while
// the model thinks), so the read timeout is generous. The
// Anthropic API sends periodic ping events well within it.
// Redirects are not followed: provider credentials in the
// headers must never be forwarded to another endpoint (a
// 3xx surfaces to the listener as an HTTP error instead).
sharedClient = OkHttpClient.Builder()
.connectTimeout(15, TimeUnit.SECONDS)
.readTimeout(180, TimeUnit.SECONDS)
.followRedirects(false)
.followSslRedirects(false)
.addInterceptor(FaceclawHttp.userAgentInterceptor())
.build()
}
@@ -14,10 +14,10 @@ import android.util.Log
import java.io.ByteArrayOutputStream
import java.lang.reflect.Method
/** Reads Wi-Fi/cell/hotspot state; the icon artwork is the shared StatusIconArt. */
class FaceclawSystemStatusIconProvider private constructor() {
companion object {
private const val TAG = "FaceclawStatusIcons"
private const val ICON_SIZE = 24
@JvmStatic
fun getSystemStatusIconGrays(context: Context?, iconSize: Int): ByteArray {
@@ -30,16 +30,16 @@ class FaceclawSystemStatusIconProvider private constructor() {
val wifiLevel = getWifiLevel(appContext)
if (wifiLevel >= 0) {
append(out, scaleIcon(drawWifiIcon(wifiLevel), size))
append(out, StatusIconArt.scale(StatusIconArt.wifi(wifiLevel), size))
}
val cellLevel = getCellLevel(appContext)
if (cellLevel >= 0) {
append(out, scaleIcon(drawCellIcon(cellLevel), size))
append(out, StatusIconArt.scale(StatusIconArt.cell(cellLevel), size))
}
if (isHotspotEnabled(appContext)) {
append(out, scaleIcon(drawHotspotIcon(), size))
append(out, StatusIconArt.scale(StatusIconArt.hotspot(), size))
}
return out.toByteArray()
@@ -106,86 +106,6 @@ class FaceclawSystemStatusIconProvider private constructor() {
return rssi > -127 && rssi < 0
}
private fun drawWifiIcon(level: Int): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
val cx = 12
val cy = 18
fillRect(icon, cx - 1, cy - 1, 3, 3, 230)
val radii = intArrayOf(5, 9, 13, 17)
for (index in radii.indices) {
if (level <= index) {
continue
}
drawWifiArc(icon, cx, cy, radii[index], 210)
}
return icon
}
private fun drawWifiArc(icon: ByteArray, cx: Int, cy: Int, radius: Int, value: Int) {
val inner = (radius - 1) * (radius - 1)
val outer = (radius + 1) * (radius + 1)
for (y in 0 until ICON_SIZE) {
for (x in 0 until ICON_SIZE) {
val dx = x - cx
val dy = y - cy
val dist = dx * dx + dy * dy
if (dy <= 0 && dist >= inner && dist <= outer && Math.abs(dx) <= radius) {
setPixel(icon, x, y, value)
}
}
}
}
private fun drawCellIcon(level: Int): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
for (bar in 0 until 4) {
val x = 5 + bar * 4
val height = 5 + bar * 4
val value = if (level > bar) 220 else 55
fillRect(icon, x, 20 - height, 3, height, value)
}
return icon
}
private fun drawHotspotIcon(): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
fillRect(icon, 11, 17, 3, 3, 230)
drawWifiArc(icon, 12, 19, 7, 210)
drawWifiArc(icon, 12, 19, 12, 210)
fillRect(icon, 6, 5, 12, 2, 170)
return icon
}
private fun scaleIcon(source: ByteArray, size: Int): ByteArray {
if (size == ICON_SIZE) {
return source
}
val scaled = ByteArray(size * size)
for (y in 0 until size) {
val sy = Math.min(ICON_SIZE - 1, (y * ICON_SIZE) / size)
for (x in 0 until size) {
val sx = Math.min(ICON_SIZE - 1, (x * ICON_SIZE) / size)
scaled[y * size + x] = source[sy * ICON_SIZE + sx]
}
}
return scaled
}
private fun fillRect(icon: ByteArray, x: Int, y: Int, width: Int, height: Int, value: Int) {
for (row in y until y + height) {
for (col in x until x + width) {
setPixel(icon, col, row, value)
}
}
}
private fun setPixel(icon: ByteArray, x: Int, y: Int, value: Int) {
if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) {
return
}
icon[y * ICON_SIZE + x] = Math.max(0, Math.min(255, value)).toByte()
}
private fun append(out: ByteArrayOutputStream, bytes: ByteArray) {
out.write(bytes, 0, bytes.size)
}
File diff suppressed because it is too large. Load diff
@@ -11,7 +11,6 @@ import android.util.Log
import java.io.File
import java.io.RandomAccessFile
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.LinkedHashMap
/**
@@ -20,7 +19,9 @@ import java.util.LinkedHashMap
* minikin/HarfBuzz. Output uses the same grayscale packet format as
* ImageFileLoader ([widthLo, widthHi, heightLo, heightHi, pixels...], one
* byte per pixel, row-major), which the TS side turns into a GrayImage and
* the compositor later quantizes to the 4bpp the firmware wants.
* the compositor later quantizes to the 4bpp the firmware wants. The packet
* layouts, gamma mapping and ink cropping are shared with iOS in GlyphPacket;
* only rasterization lives here.
*
* The gamma parameter maps antialiased coverage to output shade:
* out = 255 * (coverage/255)^gamma. The G2 display response looks roughly
@@ -88,27 +89,13 @@ class FontFileRenderer private constructor() {
// keep at least the advance width so spacing stays truthful.
var left = pad
var right = pad + advance
val painted = paintedColumnRange(pixels, width, height)
val painted = GlyphPacket.paintedColumnRange(pixels, width, height)
if (painted != null) {
left = Math.min(left, painted[0])
right = Math.max(right, painted[1] + 1)
}
val outWidth = Math.max(1, right - left)
val lut = gammaLut(gamma)
val out = ByteArray(4 + outWidth * height)
out[0] = (outWidth and 0xff).toByte()
out[1] = ((outWidth shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
for (y in 0 until height) {
val srcRow = y * width
val dstRow = 4 + y * outWidth
for (x in 0 until outWidth) {
out[dstRow + x] = lut[pixels[srcRow + left + x].toInt() and 0xff]
}
}
return out
return GlyphPacket.packImageRegion(pixels, width, left, 0, outWidth, height, gamma)
} catch (e: Exception) {
Log.w(TAG, "text render failed: $path", e)
return ByteArray(0)
@@ -159,16 +146,7 @@ class FontFileRenderer private constructor() {
val pixels = alpha8Pixels(bitmap, maxWidth, height)
bitmap.recycle()
val lut = gammaLut(gamma)
val out = ByteArray(4 + pixels.size)
out[0] = (maxWidth and 0xff).toByte()
out[1] = ((maxWidth shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
for (i in pixels.indices) {
out[4 + i] = lut[pixels[i].toInt() and 0xff]
}
return out
return GlyphPacket.packImage(pixels, maxWidth, height, gamma)
} catch (e: Exception) {
Log.w(TAG, "wrapped render failed: $path", e)
return ByteArray(0)
@@ -206,7 +184,7 @@ class FontFileRenderer private constructor() {
val bounds = android.graphics.Rect()
paint.getTextBounds(text, 0, text.length, bounds)
if (bounds.isEmpty) {
return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null)
return GlyphPacket.emptyGlyphCell(advanceFixed)
}
// getTextBounds can be off by a hair on AA edges; render with
// padding, then trim to the actually painted box.
@@ -224,40 +202,11 @@ class FontFileRenderer private constructor() {
val pixels = alpha8Pixels(bitmap, bw, bh)
bitmap.recycle()
var x0 = bw
var x1 = -1
var y0 = bh
var y1 = -1
for (y in 0 until bh) {
val row = y * bw
for (x in 0 until bw) {
if (pixels[row + x].toInt() != 0) {
if (x < x0) x0 = x
if (x > x1) x1 = x
if (y < y0) y0 = y
if (y > y1) y1 = y
}
}
}
if (x1 < x0) {
return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null)
}
val width = x1 - x0 + 1
val height = y1 - y0 + 1
// Trim to the tight ink box; bearing/inkTop become pen- and
// line-relative inside the shared packer.
val fm = paint.fontMetricsInt
val bearingX = x0 - penX
val inkTop = (-fm.ascent) + (y0 - baselineY)
val lut = gammaLut(gamma)
val coverage = ByteArray(width * height)
for (y in 0 until height) {
val src = (y0 + y) * bw + x0
val dst = y * width
for (x in 0 until width) {
coverage[dst + x] = lut[pixels[src + x].toInt() and 0xff]
}
}
return glyphCellPacket(advanceFixed, bearingX, inkTop, width, height, coverage)
return GlyphPacket.packGlyphCell(
advanceFixed, pixels, bw, bh, penX, baselineY, -fm.ascent, gamma)
} catch (e: Exception) {
Log.w(TAG, "glyph render failed: " + path + " U+" + Integer.toHexString(codePoint), e)
return ByteArray(0)
@@ -294,26 +243,6 @@ class FontFileRenderer private constructor() {
return paint
}
@JvmStatic
private fun glyphCellPacket(
advanceFixed: Int, bearingX: Int, inkTop: Int, width: Int, height: Int, coverage: ByteArray?): ByteArray {
val out = ByteArray(10 + (coverage?.size ?: 0))
out[0] = (advanceFixed and 0xff).toByte()
out[1] = ((advanceFixed shr 8) and 0xff).toByte()
out[2] = (bearingX and 0xff).toByte()
out[3] = ((bearingX shr 8) and 0xff).toByte()
out[4] = (inkTop and 0xff).toByte()
out[5] = ((inkTop shr 8) and 0xff).toByte()
out[6] = (width and 0xff).toByte()
out[7] = ((width shr 8) and 0xff).toByte()
out[8] = (height and 0xff).toByte()
out[9] = ((height shr 8) and 0xff).toByte()
if (coverage != null) {
System.arraycopy(coverage, 0, out, 10, coverage.size)
}
return out
}
/**
* Line metrics for a font at a pixel size, as "ascent descent lineGap"
* (integers, pixels). Empty string if the font cannot be loaded.
@@ -361,8 +290,21 @@ class FontFileRenderer private constructor() {
*/
@JvmStatic
fun getFontName(path: String?): String {
if (path == null) {
return ""
}
try {
return parseNameTable(path)
RandomAccessFile(path, "r").use { raf ->
return OpenTypeNames.parse(object : RandomAccessBytes {
override val length: Long
get() = raf.length()
override fun read(offset: Long, into: ByteArray, count: Int) {
raf.seek(offset)
raf.readFully(into, 0, count)
}
})
}
} catch (e: Exception) {
Log.w(TAG, "name table parse failed: $path", e)
return ""
@@ -431,144 +373,5 @@ class FontFileRenderer private constructor() {
}
return pixels
}
/** [first, last] painted (nonzero) column indexes, or null if blank. */
@JvmStatic
private fun paintedColumnRange(pixels: ByteArray, width: Int, height: Int): IntArray? {
var first = width
var last = -1
for (y in 0 until height) {
val row = y * width
var x = 0
while (x < first) {
if (pixels[row + x].toInt() != 0) {
first = x
break
}
x++
}
x = width - 1
while (x > last) {
if (pixels[row + x].toInt() != 0) {
last = x
break
}
x--
}
}
return if (last >= first) intArrayOf(first, last) else null
}
@JvmStatic
private fun gammaLut(gammaIn: Double): ByteArray {
val lut = ByteArray(256)
var gamma = gammaIn
if (gamma <= 0) {
gamma = 1.0
}
for (i in 0 until 256) {
lut[i] = Math.max(0, Math.min(255,
Math.round(255.0 * Math.pow(i / 255.0, gamma)).toInt())).toByte()
}
return lut
}
// --- Minimal 'name' table reader (TTF/OTF/TTC) for preview titles ---
@JvmStatic
@Throws(Exception::class)
private fun parseNameTable(path: String?): String {
RandomAccessFile(path, "r").use { raf ->
val fileSize = raf.length()
if (fileSize < 12) {
return ""
}
var offset = 0L
val tag = readU32(raf, 0)
if (tag == 0x74746366) { // 'ttcf': use the first face
if (readU32(raf, 8) < 1) {
return ""
}
offset = readU32(raf, 12).toLong() and 0xffffffffL
}
val numTables = readU16(raf, offset + 4)
var nameOffset = -1L
for (i in 0 until numTables) {
val rec = offset + 12 + i * 16L
if (rec + 16 > fileSize) {
return ""
}
if (readU32(raf, rec) == 0x6e616d65) { // 'name'
nameOffset = readU32(raf, rec + 8).toLong() and 0xffffffffL
break
}
}
if (nameOffset < 0 || nameOffset + 6 > fileSize) {
return ""
}
val count = readU16(raf, nameOffset + 2)
val stringStorage = nameOffset + readU16(raf, nameOffset + 4)
var family: String? = null
var style: String? = null
var preferredFamily: String? = null
var preferredStyle: String? = null
for (i in 0 until count) {
val rec = nameOffset + 6 + i * 12L
if (rec + 12 > fileSize) {
break
}
val platform = readU16(raf, rec)
val nameId = readU16(raf, rec + 6)
if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) {
continue
}
val length = readU16(raf, rec + 8)
val strOffset = stringStorage + readU16(raf, rec + 10)
if (strOffset + length > fileSize || length <= 0 || length > 512) {
continue
}
val data = ByteArray(length)
raf.seek(strOffset)
raf.readFully(data)
// Platform 0 (Unicode) and 3 (Windows) store UTF-16BE;
// platform 1 (Mac) is close enough to Latin-1 for names.
var value = if (platform == 1)
String(data, java.nio.charset.StandardCharsets.ISO_8859_1)
else String(data, java.nio.charset.StandardCharsets.UTF_16BE)
value = value.trim()
if (value.isEmpty()) {
continue
}
if (nameId == 1 && family == null) family = value
if (nameId == 2 && style == null) style = value
if (nameId == 16 && preferredFamily == null) preferredFamily = value
if (nameId == 17 && preferredStyle == null) preferredStyle = value
}
val outFamily = preferredFamily ?: family
val outStyle = preferredStyle ?: style
if (outFamily == null) {
return ""
}
return outFamily + "\n" + (outStyle ?: "")
}
}
@JvmStatic
@Throws(Exception::class)
private fun readU16(raf: RandomAccessFile, offset: Long): Int {
raf.seek(offset)
val b = ByteArray(2)
raf.readFully(b)
return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getShort().toInt() and 0xffff
}
@JvmStatic
@Throws(Exception::class)
private fun readU32(raf: RandomAccessFile, offset: Long): Int {
raf.seek(offset)
val b = ByteArray(4)
raf.readFully(b)
return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getInt()
}
}
}
@@ -9,33 +9,13 @@ import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.io.Writer
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.ArrayDeque
import java.util.ArrayList
import java.util.Collections
import java.util.Date
import java.util.LinkedHashMap
import java.util.Locale
/**
* Per-frame timing instrumentation, shared between the Java BLE layer and the
* Typescript UI layer. A "frame" starts when we receive an input event (or a
* timer fires and we decide to redraw), and finishes when the resulting screen
* update has been fully transmitted to the glasses, is discarded unsent, or
* times out because nobody reported finishing it.
*
* Frames form a tree. One input event can fan out into several renders (the
* focused app's window plus the shell chrome, say), and each of those is its
* own frame linked back to the input frame that caused it via
* [startFrame]. The export nests descendants under their
* root and renders every line against the root's clock, so a single block
* shows the whole input-to-pixels story. Latency percentiles are measured over
* roots: root start to the first descendant that actually reached the glasses.
*
* All public methods are thread-safe and cheap enough to call from the BLE
* worker thread, the Android main thread, the JS thread, and app worker
* threads. Frame IDs are positive ints; 0 means "no frame" and is silently
* ignored everywhere, so callers do not need to null-check.
* Android facade over the shared [FrameTimingsCore] (per-frame latency
* instrumentation; see that class for the frame-tree model). This class keeps
* the process-wide singleton the BLE layer and TypeScript call by name, and owns
* the Android-only parts: the export directory derived from the Context and the
* daemon thread that sweeps timed-out frames and writes the export file.
*
* Statistics and full log lines for recent and slowest frames are periodically
* exported to getExternalFilesDir()/frame-timings.txt, retrievable via
@@ -43,131 +23,19 @@ import java.util.Locale
*/
class FrameTimings private constructor() {
companion object {
private const val TAG = "FrameTimings"
private const val TAG = FrameTimingsCore.TAG
private val INSTANCE = FrameTimings()
/** Frames still open after this long are finished as "timeout". */
private const val FRAME_TIMEOUT_MS = 30_000L
private const val EXPORT_INTERVAL_MS = 15_000L
private const val SWEEP_INTERVAL_MS = 5_000L
private const val RECENT_ROOTS_KEPT = 40
private const val SLOWEST_ROOTS_KEPT = 20
private const val SENT_DURATIONS_WINDOW = 512
private const val MAX_LINES_PER_FRAME = 200
/**
* Frames retained for ID lookup. Well above the number any export can
* reference, so a late log/span/finish call always finds its frame; frames
* older than this are unreachable and get collected.
*/
private const val FRAMES_RETAINED = 1024
private const val EXPORT_FILE_NAME = "frame-timings.txt"
@JvmStatic
fun getInstance(): FrameTimings {
return INSTANCE
}
@JvmStatic
private fun percentileOfSorted(sorted: List<Long>, percentile: Int): Long {
val index = Math.ceil(percentile / 100.0 * sorted.size).toInt() - 1
return sorted[Math.max(0, Math.min(sorted.size - 1, index))]
}
@JvmStatic
private fun repeat(unit: String, times: Int): String {
val out = StringBuilder(unit.length * times)
for (i in 0 until times) {
out.append(unit)
}
return out.toString()
}
}
private class Line(
val atMs: Long, // SystemClock.elapsedRealtime()
val thread: String,
val message: String?
)
private class Frame(
val id: Int,
var reason: String,
val startedAtMs: Long, // SystemClock.elapsedRealtime()
val startedWallClockMs: Long,
val parent: Frame?
) {
val children: MutableList<Frame> = ArrayList(0)
val lines: MutableList<Line> = ArrayList()
val openSpans: MutableMap<String, Long> = LinkedHashMap()
var finishedAtMs: Long = 0
var outcome: String? = null // null while open; "sent", "discarded: ...", "timeout: ..."
/** Set on the root once some frame in its subtree reached the glasses. */
var subtreeSentRecorded = false
fun root(): Frame {
var frame: Frame = this
while (frame.parent != null) {
frame = frame.parent!!
}
return frame
}
fun isOpen(): Boolean {
return outcome == null
}
fun durationMs(): Long {
return finishedAtMs - startedAtMs
}
fun wasSent(): Boolean {
val outcome = this.outcome
return outcome != null && outcome.startsWith("sent")
}
/** Last moment anything in this subtree happened, for whole-tree duration. */
fun subtreeEndAtMs(nowMs: Long): Long {
var end = if (isOpen()) nowMs else finishedAtMs
for (child in children) {
end = Math.max(end, child.subtreeEndAtMs(nowMs))
}
return end
}
fun subtreeHasOpenFrame(): Boolean {
if (isOpen()) {
return true
}
for (child in children) {
if (child.subtreeHasOpenFrame()) {
return true
}
}
return false
}
}
/** The shared core; the glasses session core records frames into it directly. */
val core = FrameTimingsCore(AndroidProtocolPlatform)
private val lock = Any()
/**
* Every frame we still retain, keyed by ID: open frames plus enough
* history that late calls and the export can still resolve one.
*/
private val framesById: LinkedHashMap<Int, Frame> =
object : LinkedHashMap<Int, Frame>(64, 0.75f, false) {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Int, Frame>): Boolean {
return size > FRAMES_RETAINED && !eldest.value.isOpen()
}
}
private val recentRoots = ArrayDeque<Frame>()
private val slowestRoots: MutableList<Frame> = ArrayList()
private val inputLatencies = ArrayDeque<Long>()
private val renderLatencies = ArrayDeque<Long>()
private var nextFrameId = 1
private var framesStarted: Long = 0
private var framesSent: Long = 0
private var framesDiscarded: Long = 0
private var framesTimedOut: Long = 0
private var exportDirty = false
private var exportDir: File? = null
private var exportThread: Thread? = null
@@ -189,245 +57,47 @@ class FrameTimings private constructor() {
}
/** Begin a root frame; reason is a short label like "input:sys-event type=0". */
fun startFrame(reason: String?): Int {
return startFrame(reason, 0)
}
fun startFrame(reason: String?): Int = core.startFrame(reason)
/**
* Begin a frame caused by an existing one (parentFrameId; 0 for a root).
* The child is nested under its root in the export and its latency counts
* against the root, so an input event that fans out into an app render and
* a shell-chrome render reads as one timeline instead of three.
*/
fun startFrame(reason: String?, parentFrameId: Int): Int {
val now = SystemClock.elapsedRealtime()
val frame: Frame
synchronized(lock) {
val parent = framesById[parentFrameId]
frame = Frame(nextFrameId++, reason ?: "", now,
System.currentTimeMillis(), parent)
framesById[frame.id] = frame
framesStarted++
if (parent != null) {
parent.children.add(frame)
addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")")
}
}
return frame.id
}
/** Begin a frame caused by an existing one (parentFrameId; 0 for a root). */
fun startFrame(reason: String?, parentFrameId: Int): Int = core.startFrame(reason, parentFrameId)
/**
* Append to a frame's label, for facts only known after it started (which
* app is in the foreground, which window a render is for). Shows up in the
* frame's header line, so the export is scannable without reading bodies.
*/
fun annotate(frameId: Int, text: String?) {
if (text == null || text.isEmpty()) {
return
}
synchronized(lock) {
val frame = framesById[frameId]
if (frame == null) {
return
}
frame.reason = if (frame.reason.isEmpty()) text else frame.reason + " " + text
}
}
/** Append to a frame's label, for facts only known after it started. */
fun annotate(frameId: Int, text: String?) = core.annotate(frameId, text)
fun log(frameId: Int, message: String?) {
val now = SystemClock.elapsedRealtime()
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
addLineLocked(frame, now, message)
}
}
fun log(frameId: Int, message: String?) = core.log(frameId, message)
fun spanStart(frameId: Int, name: String) {
val now = SystemClock.elapsedRealtime()
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
frame.openSpans[name] = now
addLineLocked(frame, now, "span $name start")
}
}
fun spanStart(frameId: Int, name: String) = core.spanStart(frameId, name)
fun spanEnd(frameId: Int, name: String) {
val now = SystemClock.elapsedRealtime()
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
val startedAt = frame.openSpans.remove(name)
if (startedAt == null) {
addLineLocked(frame, now, "span $name end (start not recorded)")
} else {
addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)")
}
}
}
fun spanEnd(frameId: Int, name: String) = core.spanEnd(frameId, name)
/**
* Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything
* starting with "discarded", or "timeout". First finish wins; later calls for
* the same frame are ignored, so racing completion paths are safe.
* starting with "discarded", or "timeout". First finish wins.
*/
fun finishFrame(frameId: Int, outcome: String?) {
val now = SystemClock.elapsedRealtime()
val finished: Frame
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
finishFrameLocked(frame, now, outcome ?: "discarded: no outcome given")
finished = frame
}
Log.i(TAG, "frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome
+ " in " + finished.durationMs() + "ms")
}
fun finishFrame(frameId: Int, outcome: String?) = core.finishFrame(frameId, outcome)
/** One-line stats summary, e.g. for showing in the phone UI. */
fun statsSummary(): String {
synchronized(lock) {
val input = percentilesLocked(inputLatencies)
return "frames started=" + framesStarted + " sent=" + framesSent +
" discarded=" + framesDiscarded + " timeout=" + framesTimedOut +
(if (input == null)
""
else " | input-to-display p50=" + input[0] + "ms p90=" + input[1] +
"ms p99=" + input[2] + "ms max=" + input[3] + "ms")
}
}
fun statsSummary(): String = core.statsSummary()
// ---------------------------------------------------------------------
/** The frame with this ID if it exists and is still open, else null. */
private fun openFrameLocked(frameId: Int): Frame? {
if (frameId <= 0) {
return null
}
val frame = framesById[frameId]
return if (frame != null && frame.isOpen()) frame else null
}
private fun addLineLocked(frame: Frame, now: Long, message: String?) {
if (frame.lines.size >= MAX_LINES_PER_FRAME) {
if (frame.lines.size == MAX_LINES_PER_FRAME) {
frame.lines.add(Line(now, Thread.currentThread().name, "... line cap reached"))
}
return
}
frame.lines.add(Line(now, Thread.currentThread().name, message))
}
private fun finishFrameLocked(frame: Frame, now: Long, outcome: String) {
frame.finishedAtMs = now
frame.outcome = outcome
for (open in frame.openSpans.entries) {
frame.lines.add(Line(now, Thread.currentThread().name,
"span " + open.key + " never ended (started at +" +
(open.value - frame.startedAtMs) + "ms)"))
}
frame.openSpans.clear()
addLineLocked(frame, now, "finished: $outcome")
val root = frame.root()
if (frame.wasSent()) {
framesSent++
// The user-visible latency is input (or timer) to the first pixels
// that actually reached the glasses, wherever in the tree that was.
if (!root.subtreeSentRecorded) {
root.subtreeSentRecorded = true
recordLatencyLocked(root, now - root.startedAtMs)
insertSlowestLocked(root)
}
} else if (outcome.startsWith("timeout")) {
framesTimedOut++
} else {
framesDiscarded++
}
if (root === frame) {
recentRoots.addLast(frame)
while (recentRoots.size > RECENT_ROOTS_KEPT) {
recentRoots.removeFirst()
}
}
exportDirty = true
}
private fun recordLatencyLocked(root: Frame, latencyMs: Long) {
val bucket = if (root.reason.startsWith("input:")) inputLatencies else renderLatencies
bucket.addLast(latencyMs)
while (bucket.size > SENT_DURATIONS_WINDOW) {
bucket.removeFirst()
}
}
private fun insertSlowestLocked(root: Frame) {
if (!slowestRoots.contains(root)) {
slowestRoots.add(root)
}
// Sorted and trimmed at export time, when every subtree duration is final.
}
/** {p50, p90, p99, max} over a rolling latency window, or null if empty. */
private fun percentilesLocked(window: ArrayDeque<Long>): LongArray? {
if (window.isEmpty()) {
return null
}
val sorted: MutableList<Long> = ArrayList(window)
Collections.sort(sorted)
return longArrayOf(
percentileOfSorted(sorted, 50),
percentileOfSorted(sorted, 90),
percentileOfSorted(sorted, 99),
sorted[sorted.size - 1],
)
}
private fun sweepTimedOutFrames() {
val now = SystemClock.elapsedRealtime()
val timedOut: MutableList<Frame> = ArrayList()
synchronized(lock) {
for (frame in ArrayList(framesById.values)) {
if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) {
finishFrameLocked(frame, now, "timeout: never reported finished")
timedOut.add(frame)
}
}
}
for (frame in timedOut) {
Log.w(TAG, "frame#" + frame.id + " [" + frame.reason + "] timed out after " +
frame.durationMs() + "ms without being finished")
}
}
private fun exportLoop() {
var lastExportAtMs = 0L
while (true) {
try {
Thread.sleep(SWEEP_INTERVAL_MS)
Thread.sleep(FrameTimingsCore.SWEEP_INTERVAL_MS)
} catch (e: InterruptedException) {
return
}
try {
sweepTimedOutFrames()
core.sweepTimedOut()
val now = SystemClock.elapsedRealtime()
val shouldExport: Boolean
synchronized(lock) {
shouldExport = exportDirty && exportDir != null && now - lastExportAtMs >= EXPORT_INTERVAL_MS
}
if (shouldExport) {
val dir: File?
synchronized(lock) { dir = exportDir }
if (dir != null && core.isExportDirty && now - lastExportAtMs >= FrameTimingsCore.EXPORT_INTERVAL_MS) {
lastExportAtMs = now
exportToFile()
exportToFile(dir)
}
} catch (t: Throwable) {
Log.w(TAG, "export loop error", t)
@@ -435,14 +105,8 @@ class FrameTimings private constructor() {
}
}
private fun exportToFile() {
val content: String
val dir: File
synchronized(lock) {
dir = exportDir ?: return
content = buildExportLocked()
exportDirty = false
}
private fun exportToFile(dir: File) {
val content = core.takeExport() ?: return
val target = File(dir, EXPORT_FILE_NAME)
val temp = File(dir, "$EXPORT_FILE_NAME.tmp")
try {
@@ -456,99 +120,4 @@ class FrameTimings private constructor() {
Log.w(TAG, "failed to export frame timings", t)
}
}
private fun buildExportLocked(): String {
val now = SystemClock.elapsedRealtime()
val wallClockFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
val out = StringBuilder(64 * 1024)
out.append("FrameTimings export at ").append(wallClockFormat.format(Date())).append('\n')
out.append(statsSummaryLocked()).append('\n')
out.append('\n')
out.append("Frames are trees: an input event's own frame is the root, and the renders it\n")
out.append("caused are indented under it with offsets measured from the root's start.\n")
out.append('\n')
Collections.sort(slowestRoots, Comparator { a, b ->
java.lang.Long.compare(b.subtreeEndAtMs(now) - b.startedAtMs, a.subtreeEndAtMs(now) - a.startedAtMs) })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
out.append("=== slowest frames that reached the glasses ===\n")
for (frame in slowestRoots) {
appendTreeLocked(out, frame, frame, wallClockFormat, now)
}
out.append("=== recent frames (oldest first) ===\n")
for (frame in recentRoots) {
appendTreeLocked(out, frame, frame, wallClockFormat, now)
}
return out.toString()
}
private fun statsSummaryLocked(): String {
val input = percentilesLocked(inputLatencies)
val render = percentilesLocked(renderLatencies)
val out = StringBuilder()
out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent)
.append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut)
.append(" open=").append(openFrameCountLocked())
appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size, input)
appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size, render)
return out.toString()
}
private fun appendPercentilesLocked(out: StringBuilder, label: String, count: Int, percentiles: LongArray?) {
if (percentiles == null) {
return
}
out.append('\n').append(label).append(" (last ").append(count).append("): p50=")
.append(percentiles[0]).append("ms p90=").append(percentiles[1])
.append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms")
}
private fun openFrameCountLocked(): Int {
var open = 0
for (frame in framesById.values) {
if (frame.isOpen()) {
open++
}
}
return open
}
/** Print a frame and its descendants, all timed against root's start. */
private fun appendTreeLocked(
out: StringBuilder, frame: Frame, root: Frame, wallClockFormat: SimpleDateFormat, now: Long) {
var depth = 0
var walk: Frame? = frame
while (walk !== root) {
depth++
walk = walk!!.parent
}
val indent = repeat(" ", depth)
val startOffsetMs = frame.startedAtMs - root.startedAtMs
out.append(indent).append("frame#").append(frame.id)
.append(" [").append(frame.reason).append(']')
if (frame === root) {
out.append(" started ").append(wallClockFormat.format(Date(frame.startedWallClockMs)))
val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs
out.append(" duration ").append(frame.durationMs()).append("ms")
if (!frame.children.isEmpty()) {
out.append(" (tree ").append(totalMs).append("ms")
.append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')')
}
} else {
out.append(" started +").append(startOffsetMs).append("ms")
.append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms"))
}
out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n')
for (line in frame.lines) {
out.append(indent).append(String.format(Locale.US, " %+7dms [%s] %s",
line.atMs - root.startedAtMs, line.thread, line.message)).append('\n')
}
for (child in frame.children) {
appendTreeLocked(out, child, root, wallClockFormat, now)
}
}
}
@@ -8,9 +8,6 @@ import android.util.Log
import androidx.core.graphics.PathParser
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* Renders a small subset of SVG (the shapes used by Lucide / simple Noun
* Project icons: path, circle, ellipse, rect, line, polyline, polygon) to a
@@ -18,13 +15,14 @@ import java.util.regex.Pattern
* (Lucide style: fill="none", 2px stroke, round caps/joins); the returned
* bytes are one coverage value per pixel (0=transparent .. 255=opaque white),
* row-major, size*size. Called once per icon and cached on the TS side.
*
* Parsing, placement math and the coverage quantization are shared with iOS
* (SvgIconSource); this class only maps shapes onto android.graphics.Path and
* rasterizes them.
*/
class IconRenderer private constructor() {
companion object {
private const val TAG = "IconRenderer"
private val TAG_RE: Pattern = Pattern.compile("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>")
private val VIEWBOX_RE: Pattern = Pattern.compile("viewBox\\s*=\\s*\"([^\"]*)\"")
private val SEPARATOR_RE: Pattern = Pattern.compile("[\\s,]+")
@JvmStatic
fun renderSvgGray(svg: String?, size: Int, strokeWidth: Float): ByteArray {
@@ -32,27 +30,10 @@ class IconRenderer private constructor() {
return ByteArray(0)
}
try {
var minX = 0f
var minY = 0f
var viewW = 24f
var viewH = 24f
val vb = VIEWBOX_RE.matcher(svg)
if (vb.find()) {
val parts = SEPARATOR_RE.split(vb.group(1)!!.trim())
if (parts.size == 4) {
minX = java.lang.Float.parseFloat(parts[0])
minY = java.lang.Float.parseFloat(parts[1])
viewW = java.lang.Float.parseFloat(parts[2])
viewH = java.lang.Float.parseFloat(parts[3])
}
}
// Lucide-style icons are stroked outlines (fill="none"); everything
// else (Noun Project glyphs, brand logos) is a filled shape.
val stroked = svg.contains("fill=\"none\"")
val icon = SvgIconSource.parse(svg) ?: return ByteArray(0)
val path = Path()
val m = TAG_RE.matcher(svg)
while (m.find()) {
appendElement(path, m.group(1)!!, m.group(2)!!)
for (shape in icon.shapes) {
appendShape(path, shape)
}
if (path.isEmpty) {
return ByteArray(0)
@@ -60,14 +41,14 @@ class IconRenderer private constructor() {
val bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888)
val canvas = Canvas(bitmap)
val scale = Math.min(size / viewW, size / viewH)
// Center the (square) viewBox in the bitmap, then map its origin.
canvas.translate((size - viewW * scale) / 2f, (size - viewH * scale) / 2f)
canvas.scale(scale, scale)
canvas.translate(-minX, -minY)
val placement = icon.placement(size)
canvas.translate(placement[0], placement[1])
canvas.scale(placement[2], placement[2])
canvas.translate(-icon.minX, -icon.minY)
val paint = Paint(Paint.ANTI_ALIAS_FLAG)
paint.color = 0xffffffff.toInt()
if (stroked) {
if (icon.stroked) {
paint.style = Paint.Style.STROKE
paint.strokeWidth = strokeWidth
paint.strokeCap = Paint.Cap.ROUND
@@ -79,117 +60,47 @@ class IconRenderer private constructor() {
val pixels = IntArray(size * size)
bitmap.getPixels(pixels, 0, size, 0, 0, size, size)
val gray = ByteArray(size * size)
for (i in pixels.indices) {
gray[i] = quantizeCoverage((pixels[i] ushr 24) and 0xff).toByte() // alpha = coverage
}
return gray
return SvgIconSource.quantizeArgb(pixels) // alpha = coverage
} catch (e: Exception) {
Log.w(TAG, "icon render failed", e)
return ByteArray(0)
}
}
/**
* Snap antialiased coverage toward hard edges so the 4bpp frames sent
* over BLE deflate better: mostly-transparent and mostly-opaque pixels
* become exactly 0/255 (long runs, and 0 stays color-key transparent for
* bitBlt), and only genuinely half-covered edge pixels keep one mid gray.
* The packed frame then uses a 3-symbol alphabet instead of all 16
* levels, at icon sizes this is visually near-identical.
*/
@JvmStatic
private fun quantizeCoverage(alpha: Int): Int {
if (alpha < 64) return 0
if (alpha >= 192) return 255
return 128
}
@JvmStatic
private fun appendElement(path: Path, tag: String, attrs: String) {
when (tag) {
"path" -> {
val d = attr(attrs, "d")
if (d != null) {
val sub = PathParser.createPathFromPathData(d)
if (sub != null) {
path.addPath(sub)
}
private fun appendShape(path: Path, shape: SvgShape) {
when (shape) {
is SvgShape.PathData -> {
val sub = PathParser.createPathFromPathData(shape.d)
if (sub != null) {
path.addPath(sub)
}
}
"circle" -> {
val cx = num(attrs, "cx", 0f)
val cy = num(attrs, "cy", 0f)
val r = num(attrs, "r", 0f)
if (r > 0) path.addCircle(cx, cy, r, Path.Direction.CW)
}
"ellipse" -> {
val cx = num(attrs, "cx", 0f)
val cy = num(attrs, "cy", 0f)
val rx = num(attrs, "rx", 0f)
val ry = num(attrs, "ry", 0f)
if (rx > 0 && ry > 0) path.addOval(cx - rx, cy - ry, cx + rx, cy + ry, Path.Direction.CW)
}
"rect" -> {
val x = num(attrs, "x", 0f)
val y = num(attrs, "y", 0f)
val w = num(attrs, "width", 0f)
val h = num(attrs, "height", 0f)
var rx = num(attrs, "rx", 0f)
var ry = num(attrs, "ry", if (rx > 0) rx else 0f)
if (rx <= 0) rx = ry
if (ry <= 0) ry = rx
if (w > 0 && h > 0) {
if (rx > 0 || ry > 0) {
path.addRoundRect(x, y, x + w, y + h, rx, ry, Path.Direction.CW)
} else {
path.addRect(x, y, x + w, y + h, Path.Direction.CW)
}
is SvgShape.Circle -> path.addCircle(shape.cx, shape.cy, shape.r, Path.Direction.CW)
is SvgShape.Ellipse -> path.addOval(
shape.cx - shape.rx, shape.cy - shape.ry, shape.cx + shape.rx, shape.cy + shape.ry, Path.Direction.CW)
is SvgShape.Rect -> {
if (shape.rx > 0 || shape.ry > 0) {
path.addRoundRect(shape.x, shape.y, shape.x + shape.width, shape.y + shape.height,
shape.rx, shape.ry, Path.Direction.CW)
} else {
path.addRect(shape.x, shape.y, shape.x + shape.width, shape.y + shape.height, Path.Direction.CW)
}
}
"line" -> {
path.moveTo(num(attrs, "x1", 0f), num(attrs, "y1", 0f))
path.lineTo(num(attrs, "x2", 0f), num(attrs, "y2", 0f))
is SvgShape.Line -> {
path.moveTo(shape.x1, shape.y1)
path.lineTo(shape.x2, shape.y2)
}
"polyline", "polygon" -> {
val points = attr(attrs, "points")
if (points != null) {
val nums = SEPARATOR_RE.split(points.trim())
var started = false
var i = 0
while (i + 1 < nums.size) {
val px = java.lang.Float.parseFloat(nums[i])
val py = java.lang.Float.parseFloat(nums[i + 1])
if (!started) {
path.moveTo(px, py)
started = true
} else {
path.lineTo(px, py)
}
i += 2
}
if (tag == "polygon") path.close()
is SvgShape.Polyline -> {
val p = shape.points
path.moveTo(p[0], p[1])
var i = 2
while (i + 1 < p.size) {
path.lineTo(p[i], p[i + 1])
i += 2
}
if (shape.closed) path.close()
}
else -> {
}
}
}
@JvmStatic
private fun attr(attrs: String, name: String): String? {
val m: Matcher = Pattern.compile("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").matcher(attrs)
return if (m.find()) m.group(1) else null
}
@JvmStatic
private fun num(attrs: String, name: String, fallback: Float): Float {
val value = attr(attrs, name)
if (value == null) return fallback
try {
return java.lang.Float.parseFloat(value.trim())
} catch (e: NumberFormatException) {
return fallback
}
}
}
@@ -10,7 +10,8 @@ import android.util.Log
* pixels...] with one byte per pixel, row-major, or an empty array on
* failure. FaceclawMediaController's album art shares bitmapToGrayPacket;
* the TS side (app/native/image-files.ts) owns the shared photo-tone preset
* passed as (gamma, dither).
* passed as (gamma, dither). The pixel math (tone curve, alpha darkening,
* dither) lives in the shared GrayPacket so iOS produces identical bytes.
*/
class ImageFileLoader private constructor() {
companion object {
@@ -151,24 +152,7 @@ class ImageFileLoader private constructor() {
}
val pixels = IntArray(width * height)
scaled.getPixels(pixels, 0, width, 0, 0, width, height)
val out = ByteArray(4 + width * height)
out[0] = (width and 0xff).toByte()
out[1] = ((width shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
val tone = toneCurve(gamma)
for (i in pixels.indices) {
val p = pixels[i]
val a = (p ushr 24) and 0xff
val r = (p shr 16) and 0xff
val g = (p shr 8) and 0xff
val b = p and 0xff
out[4 + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte()
}
if (dither) {
ditherToDisplayLevels(out, 4, width, height)
}
return out
return GrayPacket.fromArgb(pixels, width, height, gamma, dither)
} catch (e: Exception) {
Log.w(TAG, "bitmap conversion failed", e)
return ByteArray(0)
@@ -177,70 +161,5 @@ class ImageFileLoader private constructor() {
return ByteArray(0)
}
}
/** 256-entry map of source gray to displayed gray for out = in^gamma. */
@JvmStatic
private fun toneCurve(gamma: Float): IntArray {
val curve = IntArray(256)
for (v in 0 until 256) {
curve[v] = if (gamma == 1f)
v
else Math.round(255f * Math.pow((v / 255f).toDouble(), gamma.toDouble()).toFloat())
}
return curve
}
/**
* Floyd-Steinberg error diffusion onto the display's 16 gray levels, in
* place over an 8bpp plane. Serpentine scanning keeps the diffusion from
* building up a directional texture across wide flat areas.
*
* Level n is written back as n * 16 rather than the n * 17 it stands for
* so that BmpUtil's (v + 8) >> 4 reproduces n exactly; level 0 is written
* as 1, not 0, because 0 is the shell's color-key for transparent.
*/
@JvmStatic
private fun ditherToDisplayLevels(plane: ByteArray, offset: Int, width: Int, height: Int) {
var curr = FloatArray(width)
var next = FloatArray(width)
for (x in 0 until width) {
curr[x] = (plane[offset + x].toInt() and 0xff).toFloat()
}
for (y in 0 until height) {
val rowStart = offset + y * width
val nextStart = rowStart + width
val hasNext = y + 1 < height
for (x in 0 until width) {
next[x] = if (hasNext) (plane[nextStart + x].toInt() and 0xff).toFloat() else 0f
}
val leftToRight = (y and 1) == 0
val start = if (leftToRight) 0 else width - 1
val step = if (leftToRight) 1 else -1
for (i in 0 until width) {
val x = start + i * step
val wanted = curr[x]
val level = Math.round(Math.min(255f, Math.max(0f, wanted)) / 17f)
val error = wanted - level * 17f
plane[rowStart + x] = (if (level == 0) 1 else level * 16).toByte()
val ahead = x + step
if (ahead >= 0 && ahead < width) {
curr[ahead] += error * (7f / 16f)
}
if (hasNext) {
if (ahead >= 0 && ahead < width) {
next[ahead] += error * (1f / 16f)
}
next[x] += error * (5f / 16f)
val behind = x - step
if (behind >= 0 && behind < width) {
next[behind] += error * (3f / 16f)
}
}
}
val swap = curr
curr = next
next = swap
}
}
}
}
@@ -9,34 +9,11 @@ import java.nio.ByteBuffer
* grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS
* ArrayBuffer to one without copying element-by-element); doing the
* gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the
* JS/Java bridge, where it used to cost ~150ms per preview.
* JS/Java bridge, where it used to cost ~150ms per preview. The palette
* itself is the shared PreviewPalette.
*/
class PreviewBitmapUtil private constructor() {
companion object {
private var cachedLut: IntArray? = null
private var cachedGammaBits: Long = 0
private var cachedGreen: Boolean = false
/** Reuse the current preview palette; rebuild only when gamma or color changes. */
@JvmStatic
@Synchronized
private fun lookupTable(brightenGamma: Double, green: Boolean): IntArray {
val gammaBits = java.lang.Double.doubleToLongBits(brightenGamma)
val cached = cachedLut
if (cached != null && cachedGammaBits == gammaBits && cachedGreen == green) {
return cached
}
val lut = IntArray(256)
for (g in 0 until 256) {
val v = Math.max(0L, Math.min(255L, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))).toInt()
lut[g] = if (green) (0xff000000.toInt() or (v shl 8)) else (0xff000000.toInt() or (v shl 16) or (v shl 8) or v)
}
cachedGammaBits = gammaBits
cachedGreen = green
cachedLut = lut
return lut
}
@JvmStatic
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double): Bitmap {
return fromGray(gray, width, height, brightenGamma, false)
@@ -48,13 +25,9 @@ class PreviewBitmapUtil private constructor() {
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
throw IllegalArgumentException("invalid gray preview buffer")
}
val lut = lookupTable(brightenGamma, green)
val bytes = ByteArray(width * height)
gray.get(bytes)
val colors = IntArray(width * height)
for (i in colors.indices) {
colors[i] = lut[bytes[i].toInt() and 0xff]
}
val colors = PreviewPalette.expand(bytes, 0, width * height, brightenGamma, green)
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
}
}
@@ -1,48 +0,0 @@
package com.faceclaw.app
import android.os.SystemClock
class InterruptibleSleep {
private val lock = java.lang.Object()
private var interrupted = false
fun sleep(ms: Long): Boolean {
val delayMs = Math.max(1L, ms)
val deadline = SystemClock.elapsedRealtime() + delayMs
synchronized(lock) {
// Consume (do not clear-and-ignore) an interrupt delivered before this
// sleep began. interrupt() runs on other threads with no lock held by
// the sleeper between driveSession() returning and sleep() starting, so
// clearing the flag at entry lost that wake and forced the worker to
// sleep the full interval — up to 250ms per idle frame. Honoring it
// costs at most one extra (cheap) driveSession pass.
if (interrupted) {
interrupted = false
return false
}
while (true) {
val remaining = deadline - SystemClock.elapsedRealtime()
if (remaining <= 0) {
return true
}
try {
lock.wait(remaining)
} catch (ignored: InterruptedException) {
interrupted = false
return false
}
if (interrupted) {
interrupted = false
return false
}
}
}
}
fun interrupt() {
synchronized(lock) {
interrupted = true
lock.notifyAll()
}
}
}