Merge branch 'main' into glanceboard-tilt-angle-and-glance-down

This commit is contained in:
James Babcock committed 2026-11-15 22:38:39 -08:00
commit 37d81d0308
416 files changed
+34629 -3885

No files matched your search

+3 -2
View File
@@ -11,9 +11,10 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
`alarms/` (AlarmSchedule, AlarmRingingPolicy), `settings/SettingsChangeHub`,
the stock-firmware flows over a `StockLink` GATT port (`g2protocol/StockLinkSession`,
+1 -1
View File
@@ -16,7 +16,7 @@ kotlin {
}
android {
namespace = "com.faceclaw.shared"
compileSdk = 35
compileSdk = 36
minSdk = 24
compilerOptions { jvmTarget.set(JvmTarget.JVM_17) }
withHostTestBuilder {}.configure {}
@@ -40,6 +40,10 @@ constructor(
private var file: RandomAccessFile?
private var dataBytes = 0L
// The PCM bytes the header on disk accounts for. It is rewritten about once a second of
// audio, so a process death mid-recording still leaves a playable file.
private var headerDataBytes = 0L
init {
val parent = File(path).parentFile
if (parent != null && !parent.exists() && !parent.mkdirs())
@@ -58,11 +62,26 @@ constructor(
try {
current.write(pcm16le)
dataBytes += pcm16le.size
if (dataBytes - headerDataBytes >= sampleRate * channels * 2L) writeHeader(current)
} catch (e: IOException) {
Log.w("FaceclawWavRecorder", "append failed", e)
}
}
/** Rewrite the header to cover the PCM written so far, leaving the position at the end. */
private fun writeHeader(target: RandomAccessFile) {
val header =
BinaryEncoding.wavHeader(
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
sampleRate,
channels,
)
target.seek(0)
target.write(header)
target.seek(header.size + dataBytes)
headerDataBytes = dataBytes
}
@Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L)
fun getPath(): String = path
@@ -72,16 +91,7 @@ constructor(
val current = file ?: return dataBytes
file = null
try {
current.use {
it.seek(0)
it.write(
BinaryEncoding.wavHeader(
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
sampleRate,
channels,
)
)
}
current.use { writeHeader(it) }
} catch (e: IOException) {
Log.w("FaceclawWavRecorder", "finish failed", e)
}
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
override var size = 0L
private set
override fun write(bytes: ByteArray, offset: Int, length: Int) {
file.write(bytes, offset, length)
size += length
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
file.seek(position)
file.write(bytes)
file.seek(size)
}
override fun close() = file.close()
}
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
private val file = java.io.RandomAccessFile(path, "r")
override val length = file.length()
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
file.seek(position)
var total = 0
while (total < length) {
val read = file.read(buffer, offset + total, length - total)
if (read <= 0) break
total += read
}
return total
}
override fun close() = file.close()
}
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
expect fun deleteFile(path: String): Boolean
expect fun renameFile(from: String, to: String): Boolean
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
interface SeekableOutput {
/** Bytes appended so far. */
val size: Long
/** Appends [length] bytes of [bytes] from [offset]. */
fun write(bytes: ByteArray, offset: Int, length: Int)
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
fun writeAt(position: Long, bytes: ByteArray)
fun close()
}
/** A file read at arbitrary positions without loading it whole. */
interface SeekableInput {
val length: Long
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
fun close()
}
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
expect fun openFileOutput(path: String): SeekableOutput
/** Opens [path] for reading. Throws when it cannot. */
expect fun openFileInput(path: String): SeekableInput
@@ -0,0 +1,764 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/**
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
*
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
* about a second each, cues and the segment size written when the recording finishes. A
* recording cut short by a crash is still a valid live-style stream of everything up to its last
* complete cluster, and the reader takes it as such.
*/
internal object MatroskaIds {
const val EBML = 0x1A45DFA3
const val EBML_VERSION = 0x4286
const val EBML_READ_VERSION = 0x42F7
const val EBML_MAX_ID_LENGTH = 0x42F2
const val EBML_MAX_SIZE_LENGTH = 0x42F3
const val DOC_TYPE = 0x4282
const val DOC_TYPE_VERSION = 0x4287
const val DOC_TYPE_READ_VERSION = 0x4285
const val VOID = 0xEC
const val SEGMENT = 0x18538067
const val SEEK_HEAD = 0x114D9B74
const val SEEK = 0x4DBB
const val SEEK_ID = 0x53AB
const val SEEK_POSITION = 0x53AC
const val INFO = 0x1549A966
const val TIMESTAMP_SCALE = 0x2AD7B1
const val DURATION = 0x4489
const val DATE_UTC = 0x4461
const val TITLE = 0x7BA9
const val MUXING_APP = 0x4D80
const val WRITING_APP = 0x5741
const val TRACKS = 0x1654AE6B
const val TRACK_ENTRY = 0xAE
const val TRACK_NUMBER = 0xD7
const val TRACK_UID = 0x73C5
const val TRACK_TYPE = 0x83
const val FLAG_DEFAULT = 0x88
const val FLAG_LACING = 0x9C
const val NAME = 0x536E
const val LANGUAGE = 0x22B59C
const val CODEC_ID = 0x86
const val AUDIO = 0xE1
const val SAMPLING_FREQUENCY = 0xB5
const val CHANNELS = 0x9F
const val BIT_DEPTH = 0x6264
const val ATTACHMENTS = 0x1941A469
const val ATTACHED_FILE = 0x61A7
const val FILE_NAME = 0x466E
const val FILE_MIME_TYPE = 0x4660
const val FILE_DATA = 0x465C
const val FILE_UID = 0x46AE
const val CLUSTER = 0x1F43B675
const val TIMESTAMP = 0xE7
const val SIMPLE_BLOCK = 0xA3
const val BLOCK_GROUP = 0xA0
const val BLOCK = 0xA1
const val BLOCK_DURATION = 0x9B
const val CUES = 0x1C53BB6B
const val CUE_POINT = 0xBB
const val CUE_TIME = 0xB3
const val CUE_TRACK_POSITIONS = 0xB7
const val CUE_TRACK = 0xF7
const val CUE_CLUSTER_POSITION = 0xF1
}
/** Builds EBML elements into a byte buffer. */
internal class EbmlWriter(capacity: Int = 256) {
private val sink = ByteSink(capacity)
fun size(): Int = sink.size()
fun bytes(): ByteArray = sink.toByteArray()
/** An element ID, written as given: its length marker is part of the value. */
fun id(id: Int) {
if (id > 0xFFFFFF) sink.write(id ushr 24)
if (id > 0xFFFF) sink.write(id ushr 16)
if (id > 0xFF) sink.write(id ushr 8)
sink.write(id)
}
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
fun dataSize(size: Long, width: Int = 0) {
var bytes = width
if (bytes == 0) {
bytes = 1
// The all-ones value of each width means "unknown size".
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
}
val marked = size or (1L shl (7 * bytes))
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
}
fun uint(id: Int, value: Long) {
var bytes = 1
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
id(id)
dataSize(bytes.toLong())
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
/** A fixed eight-byte integer (dates). */
fun int64(id: Int, value: Long) {
id(id)
dataSize(8)
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
fun string(id: Int, text: String) {
val bytes = text.encodeToByteArray()
binary(id, bytes, 0, bytes.size)
}
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
id(id)
dataSize(length.toLong())
sink.write(bytes, offset, length)
}
fun raw(bytes: ByteArray) = sink.write(bytes)
fun raw(value: Int) = sink.write(value)
fun master(id: Int, build: EbmlWriter.() -> Unit) {
val child = EbmlWriter()
child.build()
id(id)
dataSize(child.size().toLong())
sink.write(child.bytes())
}
companion object {
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
fun padded(element: ByteArray, reserved: Int): ByteArray? {
val filler = reserved - element.size
// A Void element is at least two bytes.
if (filler < 0 || filler == 1) return null
val out = element.copyOf(reserved)
if (filler == 0) return out
val at = element.size
out[at] = MatroskaIds.VOID.toByte()
if (filler < 9) {
out[at + 1] = (0x80 or (filler - 2)).toByte()
} else {
val size = (filler - 9).toLong()
out[at + 1] = 0x01
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
}
return out
}
}
}
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
/**
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
* milliseconds from the start of the file. Not thread-safe.
*/
class MkaWriter(
private val output: SeekableOutput,
title: String?,
/** When the recording began (Unix epoch ms), for the file's date. */
dateUtcMs: Long,
attachments: List<MkaAttachment>,
) {
companion object {
const val CODEC_PCM = "A_PCM/INT/LIT"
const val CODEC_TEXT = "S_TEXT/UTF8"
const val TRACK_AUDIO = 2
const val TRACK_SUBTITLE = 0x11
const val APP_NAME = "Faceclaw"
private const val SEEK_RESERVED = 128
private const val TRACKS_RESERVED = 4096
private const val CLUSTER_SPAN_MS = 1000L
private const val CLUSTER_MAX_BYTES = 1 shl 20
// Block timestamps are signed 16-bit offsets from their cluster's.
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
private const val DURATION_PATCH_CLUSTERS = 5
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
private const val DATE_EPOCH_MS = 978_307_200_000L
}
private class Track(
val number: Int,
val type: Int,
val name: String,
val sampleRate: Int,
val channels: Int,
val preferred: Boolean,
)
private class Cue(val clusterPosition: Long, val timeMs: Long)
private val tracks = ArrayList<Track>()
private val cues = ArrayList<Cue>()
private val segmentSizePosition: Long
private val segmentStart: Long
private val seekPosition: Long
private val infoPosition: Long
private val durationPosition: Long
private val tracksPosition: Long
private val attachmentsPosition: Long
private var cluster: EbmlWriter? = null
private var clusterBaseMs = 0L
private var clustersSincePatch = 0
private var finished = false
private val hasAttachments = attachments.isNotEmpty()
/** The end of the latest audio or text written, in ms from the start of the file. */
var durationMs = 0L
private set
init {
val head = EbmlWriter()
head.master(MatroskaIds.EBML) {
uint(MatroskaIds.EBML_VERSION, 1)
uint(MatroskaIds.EBML_READ_VERSION, 1)
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
string(MatroskaIds.DOC_TYPE, "matroska")
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
}
head.id(MatroskaIds.SEGMENT)
segmentSizePosition = head.size().toLong()
// Unknown size (a live stream) until finish() knows it.
head.raw(0x01)
repeat(7) { head.raw(0xFF) }
segmentStart = head.size().toLong()
append(head.bytes())
seekPosition = output.size
infoPosition = seekPosition + SEEK_RESERVED
val info = EbmlWriter()
info.master(MatroskaIds.INFO) {
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
string(MatroskaIds.MUXING_APP, APP_NAME)
string(MatroskaIds.WRITING_APP, APP_NAME)
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
// Last, so its eight bytes are the element's final ones.
float64(MatroskaIds.DURATION, 0.0)
}
durationPosition = infoPosition + info.size() - 8
tracksPosition = infoPosition + info.size()
attachmentsPosition = tracksPosition + TRACKS_RESERVED
append(seekHead(-1))
append(info.bytes())
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
if (attachments.isNotEmpty()) {
val files = EbmlWriter()
files.master(MatroskaIds.ATTACHMENTS) {
attachments.forEachIndexed { index, file ->
master(MatroskaIds.ATTACHED_FILE) {
string(MatroskaIds.FILE_NAME, file.name)
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
uint(MatroskaIds.FILE_UID, index + 1L)
}
}
}
append(files.bytes())
}
}
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
private fun seekHead(cuesPosition: Long): ByteArray {
val head = EbmlWriter()
head.master(MatroskaIds.SEEK_HEAD) {
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
val idBytes = EbmlWriter()
idBytes.id(id)
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
}
entry(MatroskaIds.INFO, infoPosition)
entry(MatroskaIds.TRACKS, tracksPosition)
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
}
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
}
/**
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
* the one players pick by default.
*/
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
private fun addTrack(track: Track): Int {
check(!finished) { "recording already finished" }
// A block's track number is written in one byte.
if (track.number > 126) return -1
tracks.add(track)
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
if (region == null) {
tracks.remove(track)
return -1
}
output.writeAt(tracksPosition, region)
return track.number
}
private fun listedTracks(): List<Track> =
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
tracks.filter { it.type != TRACK_AUDIO }
private fun tracksElement(): ByteArray {
val listed = listedTracks()
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
val out = EbmlWriter(1024)
out.master(MatroskaIds.TRACKS) {
for (track in listed) {
master(MatroskaIds.TRACK_ENTRY) {
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
uint(MatroskaIds.TRACK_UID, track.number.toLong())
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
uint(MatroskaIds.FLAG_LACING, 0)
string(MatroskaIds.NAME, track.name)
string(MatroskaIds.LANGUAGE, "und")
if (track.type == TRACK_AUDIO) {
string(MatroskaIds.CODEC_ID, CODEC_PCM)
master(MatroskaIds.AUDIO) {
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
uint(MatroskaIds.CHANNELS, track.channels.toLong())
uint(MatroskaIds.BIT_DEPTH, 16)
}
} else {
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
}
}
}
}
return out.bytes()
}
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
val track = track(trackNumber)
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
if (length <= 0) return
val block = openCluster(timeMs)
block.id(MatroskaIds.SIMPLE_BLOCK)
block.dataSize(4L + length)
blockHeader(block, trackNumber, timeMs, 0x80)
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
val frames = length / (2 * track.channels)
noteEnd(timeMs + frames * 1000L / track.sampleRate)
}
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
val bytes = text.encodeToByteArray()
val block = openCluster(timeMs)
block.master(MatroskaIds.BLOCK_GROUP) {
id(MatroskaIds.BLOCK)
dataSize(4L + bytes.size)
blockHeader(this, trackNumber, timeMs, 0)
raw(bytes)
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
}
noteEnd(timeMs + maxOf(1L, durationMs))
}
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
block.raw(0x80 or trackNumber)
block.raw(relative shr 8)
block.raw(relative)
block.raw(flags)
}
private fun noteEnd(endMs: Long) {
if (endMs > durationMs) durationMs = endMs
}
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
private fun openCluster(timeMs: Long): EbmlWriter {
check(!finished) { "recording already finished" }
val time = maxOf(0L, timeMs)
val open = cluster
if (open != null) {
val relative = time - clusterBaseMs
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
flushCluster()
}
clusterBaseMs = time
return EbmlWriter(64 * 1024).also { cluster = it }
}
private fun flushCluster() {
val body = cluster ?: return
cluster = null
if (body.size() == 0) return
val timestamp = EbmlWriter(16)
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
val header = EbmlWriter(32)
header.id(MatroskaIds.CLUSTER)
header.dataSize(timestamp.size().toLong() + body.size())
header.raw(timestamp.bytes())
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
append(header.bytes())
append(body.bytes())
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
}
private fun patchDuration() {
clustersSincePatch = 0
val value = durationMs.toDouble().toRawBits()
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
}
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
fun finish() {
if (finished) return
try {
flushCluster()
finished = true
val cuesPosition = output.size
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
if (cues.isNotEmpty()) {
val index = EbmlWriter(cues.size * 16 + 16)
index.master(MatroskaIds.CUES) {
for (cue in cues) {
master(MatroskaIds.CUE_POINT) {
uint(MatroskaIds.CUE_TIME, cue.timeMs)
master(MatroskaIds.CUE_TRACK_POSITIONS) {
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
}
}
}
}
append(index.bytes())
output.writeAt(seekPosition, seekHead(cuesPosition))
}
patchDuration()
val size = EbmlWriter(8)
size.dataSize(output.size - segmentStart, 8)
output.writeAt(segmentSizePosition, size.bytes())
} finally {
finished = true
output.close()
}
}
}
class MkaTrackInfo(
@JvmField val number: Int,
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
@JvmField val type: Int,
@JvmField val codecId: String,
@JvmField val name: String,
@JvmField val sampleRate: Int,
@JvmField val channels: Int,
@JvmField val bitDepth: Int,
) {
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
}
class MkaBlock(
@JvmField val track: Int,
@JvmField val timeMs: Long,
@JvmField val data: ByteArray,
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
@JvmField val durationMs: Long,
)
/**
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
* IllegalArgumentException for anything that is not such a file.
*/
class MkaReader(private val input: SeekableInput) {
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
private val clusters = ArrayList<ClusterRef>()
private val trackList = ArrayList<MkaTrackInfo>()
private val attachmentList = ArrayList<MkaAttachment>()
private var timestampScale = 1_000_000L
private var infoDurationMs = 0L
private var nextCluster = 0
private var blocks: List<MkaBlock> = emptyList()
private var blockIndex = 0
private var skipBeforeMs = 0L
val tracks: List<MkaTrackInfo> get() = trackList
val attachments: List<MkaAttachment> get() = attachmentList
/** The end of the last audio or text in the file, in ms. */
val durationMs: Long
init {
scan()
var end = infoDurationMs
if (clusters.isNotEmpty()) {
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
}
durationMs = end
}
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
/** Continue from the first block at or after [timeMs]. */
fun seekTo(timeMs: Long) {
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
nextCluster = maxOf(0, index)
blocks = emptyList()
blockIndex = 0
skipBeforeMs = timeMs
}
/** The next block, or null at the end of the file. */
fun next(): MkaBlock? {
while (true) {
while (blockIndex >= blocks.size) {
if (nextCluster >= clusters.size) return null
blocks = readCluster(clusters[nextCluster++])
blockIndex = 0
}
val block = blocks[blockIndex++]
if (block.timeMs >= skipBeforeMs) return block
}
}
fun close() = input.close()
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
private fun blockSpanMs(block: MkaBlock): Long {
if (block.durationMs >= 0) return block.durationMs
val track = track(block.track) ?: return 0
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
}
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
private class Header(val id: Int, val size: Long, val length: Int)
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
if (at >= end) return null
val first = bytes[at].toInt() and 0xff
val idLength = first.countLeadingZeroBits() - 24 + 1
if (idLength > 4 || at + idLength >= end) return null
var id = 0
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
val sizeAt = at + idLength
val sizeFirst = bytes[sizeAt].toInt() and 0xff
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
var allOnes = size == (0xff shr sizeLength).toLong()
for (index in 1 until sizeLength) {
val byte = bytes[sizeAt + index].toInt() and 0xff
if (byte != 0xff) allOnes = false
size = (size shl 8) or byte.toLong()
}
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
}
private fun readAt(position: Long, length: Int): ByteArray {
val buffer = ByteArray(length)
val read = input.readAt(position, buffer, 0, length)
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
}
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
var value = 0L
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
return value
}
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
8 -> Double.fromBits(uint(bytes, at, 8))
else -> 0.0
}
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
var at = start
while (true) {
val child = header(bytes, at, end) ?: return
val payload = at + child.length
if (child.size < 0 || child.size > end - payload) return
visit(child.id, payload, child.size.toInt())
at = payload + child.size.toInt()
}
}
private fun scan() {
val length = input.length
val start = readAt(0, 64)
val ebml = header(start, 0, start.size)
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
var position = ebml.length + ebml.size
val segment = readAt(position, 12).let { header(it, 0, it.size) }
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
position += segment.length
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
while (position < end) {
val head = readAt(position, 12)
val element = header(head, 0, head.size) ?: break
val payload = position + element.length
if (element.id == MatroskaIds.CLUSTER) {
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
// A recording cut short ends in a partial cluster; its complete blocks still count.
val available = minOf(element.size, end - payload)
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
} else if (element.size < 0) {
break
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
when (element.id) {
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
}
}
if (element.size < 0) break
position = payload + element.size
}
}
private fun clusterTime(position: Long, size: Int): Long {
val head = readAt(position, minOf(size, 16))
val first = header(head, 0, head.size)
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
return uint(head, first.length, first.size.toInt())
}
val whole = readAt(position, size)
var time = 0L
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
return time
}
private fun parseInfo(bytes: ByteArray) {
var duration = 0.0
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
MatroskaIds.DURATION -> duration = float(bytes, at, size)
}
}
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
}
private fun parseTracks(bytes: ByteArray) {
trackList.clear()
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.TRACK_ENTRY) {
var number = 0
var type = 0
var codec = ""
var name = ""
var rate = 8000.0
var channels = 1
var depth = 0
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
when (audio) {
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
}
}
}
}
val audio = type == MkaWriter.TRACK_AUDIO
trackList.add(
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
)
}
}
}
private fun parseAttachments(bytes: ByteArray) {
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.ATTACHED_FILE) {
var name = ""
var mime = ""
var data = ByteArray(0)
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
}
}
attachmentList.add(MkaAttachment(name, mime, data))
}
}
}
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
val bytes = readAt(cluster.position, cluster.size)
val out = ArrayList<MkaBlock>()
var clusterRaw = cluster.rawTime
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
MatroskaIds.BLOCK_GROUP -> {
var blockAt = -1
var blockSize = 0
var duration = -1L
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.BLOCK -> {
blockAt = fieldAt
blockSize = fieldSize
}
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
}
}
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
}
}
}
return out
}
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
if (size < 4) return null
val first = bytes[at].toInt() and 0xff
val trackLength = first.countLeadingZeroBits() - 24 + 1
if (trackLength > 2 || size < trackLength + 3) return null
var track = first and (0xff shr trackLength)
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
val head = at + trackLength
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
val flags = bytes[head + 2].toInt() and 0xff
if (flags and 0x06 != 0) return null
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
}
}
@@ -0,0 +1,93 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/**
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
* contract is the header of g2flash patches/mic_control.c).
*/
object MicArrayProtocol {
const val VERSION = 1
const val HEADER_BYTES = 21
const val OP_CONFIGURE = 1
const val OP_STOP = 3
const val OP_RENEW = 4
/** How long an armed temple streams without a CONFIGURE or RENEW. */
const val LEASE_MS = 90_000L
const val FLAG_TRUNCATED = 0x80
const val CODEC_RAW = 1
const val FORMAT_PCM16 = 0
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
class Frame(
/** Low 16 bits of the temple's frame counter. */
@JvmField val sequence: Int,
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
@JvmField val deviceTickMs: Long,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
@JvmField val format: Int,
@JvmField val codec: Int,
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
@JvmField val angleDegrees: Int,
@JvmField val ssr: Int,
/** The firmware cut the payload short of the samples it captured. */
@JvmField val truncated: Boolean,
@JvmField val payloadOffset: Int,
@JvmField val payloadLength: Int,
)
/** A control record with no payload: ['M','C', version, op]. */
@JvmStatic
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
@JvmStatic
fun isConfigure(record: ByteArray?): Boolean =
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
@JvmStatic
fun decodeFrame(data: ByteArray): Frame? {
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
return null
}
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
return Frame(
sequence = u16(data, 4),
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
channels = data[10].toInt() and 0xff,
sampleRate = u16(data, 11) * 100,
format = data[13].toInt() and 0xff,
codec = data[14].toInt() and 0xff,
angleDegrees = u16(data, 15).toShort().toInt(),
ssr = u16(data, 17).toShort().toInt(),
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
payloadOffset = HEADER_BYTES,
payloadLength = length,
)
}
/**
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
* payload isn't raw 16-bit PCM or holds none.
*/
@JvmStatic
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
val frames = frame.payloadLength / (2 * frame.channels)
if (frames <= 0) return null
val samples = ShortArray(frames * frame.channels)
var at = frame.payloadOffset
for (index in samples.indices) {
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
at += 2
}
return samples
}
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
}
@@ -0,0 +1,484 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
import kotlin.math.abs
/** Which microphones a chunk of capture audio came from. */
object MicStream {
/** One mixed channel: the glasses' stock stream, or the phone's own microphone. */
const val MONO = 0
/** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */
const val LEFT = 1
const val RIGHT = 2
/** The mono feed derived from the temples for speech recognition: recorded, never replayed. */
const val MIX = 3
const val COUNT = 4
/** The stream's name in frame metadata. */
@JvmStatic
fun key(stream: Int): String = when (stream) {
MONO -> "mic"
LEFT -> "left"
RIGHT -> "right"
else -> "mix"
}
/** The name of the stream's audio tracks in a recording. */
@JvmStatic
fun trackName(stream: Int): String = when (stream) {
MONO -> "mic"
LEFT -> "left temple"
RIGHT -> "right temple"
else -> "mix"
}
/** The stream a recording's track or metadata names, or -1. */
@JvmStatic
fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1
}
/**
* One chunk of microphone audio as a capture source delivered it (about 50 ms), before any
* processing. Live captures, recordings and replays all trade in these.
*/
class MicChunk(
@JvmField val stream: Int,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
/** Interleaved PCM16; the first [frames] sample frames are the chunk. */
@JvmField val pcm: ShortArray,
@JvmField val frames: Int,
/** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */
@JvmField val timeMs: Long,
) {
/** Whether the source reported a direction of arrival with the chunk. */
@JvmField var hasDirection = false
@JvmField var angleDegrees = 0
@JvmField var ssr = 0
/** The source's packet counter, or -1 when it has none. */
@JvmField var sequence = -1
/** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */
@JvmField var deviceTickMs = -1L
/** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */
@JvmField var clockStream = stream
/** The source cut the chunk short of what it captured. */
@JvmField var truncated = false
/** Sample frames known to be missing between the previous chunk of this stream and this one. */
@JvmField var lostFramesBefore = 0
val durationMs: Long get() = frames * 1000L / sampleRate
}
/**
* Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for
* array processing (beamforming, noise cancellation): replays feed it the recorded chunks with
* the times they first arrived at, so an implementation that decides from the chunks alone
* behaves the same on a replay as it did live.
*/
interface MicArrayMixer {
/** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */
fun mix(chunk: MicChunk): MicChunk?
fun reset()
}
/**
* The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix).
* The temples sample on independent clocks, so their audio is never summed. The left temple
* feeds whenever it is streaming; the right one stands in while it is not.
*/
class SingleTempleMixer : MicArrayMixer {
companion object {
/** A temple that has delivered nothing for this long no longer counts as streaming. */
const val SILENT_MS = 300L
}
private var leftAtMs = Long.MIN_VALUE
override fun reset() {
leftAtMs = Long.MIN_VALUE
}
override fun mix(chunk: MicChunk): MicChunk? {
if (chunk.stream == MicStream.LEFT) {
leftAtMs = chunk.timeMs
} else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) {
return null
}
val channels = chunk.channels
val mono = ShortArray(chunk.frames)
for (frame in 0 until chunk.frames) {
var sum = 0
for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel]
mono[frame] = (sum / channels).toShort()
}
val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs)
out.hasDirection = chunk.hasDirection
out.angleDegrees = chunk.angleDegrees
out.ssr = chunk.ssr
out.deviceTickMs = chunk.deviceTickMs
out.clockStream = chunk.stream
out.truncated = chunk.truncated
return out
}
}
/**
* Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and
* layout as each first appears, plus a "frames" text track with one JSON object per chunk that
* has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md.
*
* Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick,
* or the count of samples from a source without one) says how far apart its chunks were
* captured; the least-delayed recent arrival places that clock on the file's timeline. So a
* chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap.
*
* Safe to share between the capture thread and whoever finishes the recording.
*/
class MicRecordingWriter(
output: SeekableOutput,
/** A JSON document describing the capture, stored as the file's "capture.json" attachment. */
configJson: String?,
/** When the recording began (Unix epoch ms). */
startedAtMs: Long,
platform: ProtocolPlatform = protocolPlatform(),
private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) },
) {
companion object {
const val CONFIG_ATTACHMENT = "capture.json"
const val META_TRACK = "frames"
/** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */
const val DISCONTINUITY_MS = 5000L
private const val WINDOW_BUCKETS = 8
private const val BUCKET_MS = 1000L
}
/**
* Places one track's chunks on the file's timeline. `source` time is the stream's own clock
* since this clock was anchored; the offset from it to arrival time is taken as the least
* seen over the last few seconds, which is the least-delayed delivery.
*/
private class TrackClock(val number: Int, val sampleRate: Int) {
private var anchored = false
private var clockStream = -1
private var tickBase = -1L
private var lastTick = -1L
private var frames = 0L
private val minima = LongArray(WINDOW_BUCKETS)
private var bucket = 0
private var bucketStartMs = 0L
var endMs = Long.MIN_VALUE
private set
fun reset() {
anchored = false
}
private fun offset(): Long = minima.min()
fun stamp(chunk: MicChunk): Long {
val arrival = chunk.timeMs
val ticked = chunk.deviceTickMs >= 0
if (anchored) {
// Another clock (the feed moved to the other temple), or this one restarted.
if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false
else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false
}
var anchorNow = !anchored
var source = 0L
if (anchored) {
frames += chunk.lostFramesBefore
source = sourceTime(chunk)
if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true
}
if (anchorNow) {
anchored = true
clockStream = chunk.clockStream
tickBase = if (ticked) chunk.deviceTickMs else -1L
frames = 0
minima.fill(Long.MAX_VALUE)
bucket = 0
bucketStartMs = arrival
source = 0
}
if (arrival - bucketStartMs >= BUCKET_MS) {
bucket = (bucket + 1) % WINDOW_BUCKETS
minima[bucket] = Long.MAX_VALUE
bucketStartMs = arrival
}
if (arrival - source < minima[bucket]) minima[bucket] = arrival - source
// Never before the end of the track's previous block.
val time = maxOf(source + offset(), endMs)
frames += chunk.frames
lastTick = chunk.deviceTickMs
endMs = time + chunk.durationMs
return time
}
private fun sourceTime(chunk: MicChunk): Long =
if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate
}
private val lock = platform.createLock()
private val writer: MkaWriter
private val clocks = HashMap<Long, TrackClock>()
private var metaTrack = 0
private var originMs = Long.MIN_VALUE
private var lastTimeMs = 0L
private var closed = false
/** The recording stopped early: the file could not be written. */
@Volatile
var failed = false
private set
init {
val attachments = if (configJson.isNullOrEmpty()) emptyList()
else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray()))
writer = MkaWriter(output, null, startedAtMs, attachments)
}
/** Length of the recording so far, in ms. */
val durationMs: Long get() = lock.withLock { writer.durationMs }
/** Appends [chunk] to its stream's track, adding the track on its first chunk. */
fun write(chunk: MicChunk) {
if (chunk.frames <= 0) return
lock.withLock {
if (closed) return
try {
val clock = clockFor(chunk) ?: return
if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs
val time = clock.stamp(chunk) - originMs
val meta = metadata(chunk)
if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta)
val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels)
writer.writeAudio(clock.number, time, bytes, 0, bytes.size)
lastTimeMs = maxOf(lastTimeMs, time)
} catch (error: Throwable) {
fail(error)
}
}
}
/**
* Records a JSON object as an event at the recording's current position (a capture starting
* over with another configuration, say), and has every stream's clock placed afresh by its
* next chunk: what follows an event is not continuous with what came before.
*/
fun event(json: String) {
lock.withLock {
if (closed) return
try {
for (clock in clocks.values) clock.reset()
if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json)
} catch (error: Throwable) {
fail(error)
}
}
}
/** Completes the file. Returns the recording's length in ms. */
fun finish(): Long {
lock.withLock {
if (!closed) {
closed = true
try {
writer.finish()
} catch (error: Throwable) {
failed = true
logWarn("finishing the recording failed: ${error.message}")
}
}
return writer.durationMs
}
}
private fun fail(error: Throwable) {
failed = true
closed = true
logWarn("recording stopped: ${error.message}")
try {
writer.finish()
} catch (ignored: Throwable) {
// The file is whatever made it to disk.
}
}
private fun ensureMetaTrack(): Boolean {
if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK)
return metaTrack > 0
}
private fun clockFor(chunk: MicChunk): TrackClock? {
val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong()
clocks[key]?.let { return it }
// The frames track goes in with the first audio track, so it is there for that track's first chunk.
ensureMetaTrack()
// The mixed streams are what a player should pick; a temple's raw pair is for analysis.
val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX
val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred)
if (number <= 0) {
logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio")
return null
}
return TrackClock(number, chunk.sampleRate).also { clocks[key] = it }
}
/** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */
private fun metadata(chunk: MicChunk): String? {
if (chunk.stream == MicStream.MIX) return null
if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null
val text = StringBuilder(96)
text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs)
if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence)
if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs)
if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr)
if (chunk.truncated) text.append(",\"trunc\":true")
if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore)
return text.append('}').toString()
}
}
/**
* Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO]
* chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the
* metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A
* Matroska file from elsewhere plays its first PCM16 track as the mono microphone.
*
* Throws IllegalArgumentException when the file has no audio a replay can use.
*/
class MicRecordingReader(input: SeekableInput) {
companion object {
const val SAMPLE_RATE = 16000
/** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */
const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20
}
private class Meta(val timeMs: Long, val fields: Map<String, Any?>)
private val reader = MkaReader(input)
private val streams = HashMap<Int, Int>()
private var metaTrack = -1
private val pendingMeta = arrayOfNulls<Meta>(MicStream.COUNT)
private val queued = ArrayDeque<MicChunk>()
val durationMs: Long get() = reader.durationMs
/** The capture's "capture.json" attachment, or null. */
val configJson: String? =
reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString()
/** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */
val description: String
init {
try {
val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 }
for (track in audio) {
val stream = MicStream.named(track.name)
if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream
}
if (streams.isEmpty()) {
// Not one of ours: its first usable track stands in for the microphone.
val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull()
require(first != null) { "no 16-bit PCM audio track" }
streams[first.number] = MicStream.MONO
}
for (number in streams.keys) {
val rate = reader.track(number)!!.sampleRate
require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" }
}
metaTrack = reader.tracks.firstOrNull {
it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK
}?.number ?: -1
description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) ->
val channels = reader.track(number)!!.channels
MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else ""
}
} catch (error: Throwable) {
reader.close()
throw error
}
}
/** Continue from the first chunk at or after [timeMs]. */
fun seekTo(timeMs: Long) {
queued.clear()
pendingMeta.fill(null)
reader.seekTo(timeMs)
}
/** The next chunk in file order, or null at the end. */
fun next(): MicChunk? {
while (queued.isEmpty()) {
val block = reader.next() ?: return null
if (block.track == metaTrack) {
noteMeta(block)
continue
}
val stream = streams[block.track] ?: continue
val track = reader.track(block.track) ?: continue
queueChunks(block, stream, track)
}
return queued.removeFirst()
}
fun close() = reader.close()
private fun noteMeta(block: MkaBlock) {
try {
val fields = Json.parseObject(block.data.decodeToString())
val stream = MicStream.named(fields["s"].asString() ?: return)
if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields)
} catch (ignored: Throwable) {
// An event or a frame we can't read: the audio plays without it.
}
}
private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) {
val sourceChannels = track.channels
val totalFrames = block.data.size / (2 * sourceChannels)
if (totalFrames <= 0) return
// The mixed microphone is one channel however a foreign file stored it.
val channels = if (stream == MicStream.MONO) 1 else sourceChannels
val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs }
pendingMeta[stream] = null
var start = 0
while (start < totalFrames) {
val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start)
val pcm = ShortArray(frames * channels)
for (frame in 0 until frames) {
val at = (start + frame) * sourceChannels * 2
if (channels == sourceChannels) {
for (channel in 0 until channels) {
pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
}
} else {
var sum = 0
for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
pcm[frame] = (sum / sourceChannels).toShort()
}
}
val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate)
if (meta != null && start == 0) applyMeta(chunk, meta.fields)
queued.addLast(chunk)
start += frames
}
}
private fun applyMeta(chunk: MicChunk, fields: Map<String, Any?>) {
chunk.sequence = fields["seq"].asInt() ?: -1
chunk.deviceTickMs = fields["tick"].asLong() ?: -1L
val angle = fields["angle"].asInt()
if (angle != null) {
chunk.hasDirection = true
chunk.angleDegrees = angle
chunk.ssr = fields["ssr"].asInt() ?: 0
}
chunk.truncated = fields["trunc"].asBoolean() ?: false
chunk.lostFramesBefore = fields["lost"].asInt() ?: 0
}
}
@@ -36,17 +36,26 @@ interface VoiceCaptureHost {
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
/** Where a saved recording goes, or null when recordings cannot be stored. */
fun recordingSink(): RecordingSink?
/** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */
fun micArrayLink(): MicArrayLink? = null
/** Opens a recording to replay; throws when it cannot be read. */
fun openRecording(path: String): SeekableInput = openFileInput(path)
}
/**
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, speaker
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
* worker thread (started through the shared [startThread]); listener callbacks go through the
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
*
* Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the
* phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording
* played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go
* to the recorder ([setRecorder]) as they came, before any processing, which is what lets a
* replay of the recording run the same processing again.
*/
class VoiceCaptureSession(
private val host: VoiceCaptureHost,
@@ -98,6 +107,19 @@ class VoiceCaptureSession(
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
/** Samples per G2 packet; the decode buffer the worker reuses. */
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
// Mic array upkeep: the temples stop streaming unless their lease is renewed, and a
// temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again.
const val ARRAY_CHECK_MS = 1000L
const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3
const val ARRAY_TEMPLE_SILENT_MS = 5000L
const val ARRAY_WAITING_MS = 2000L
const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..."
// A replay skips stretches of the recording with no audio longer than this
// (a recording made across a disconnect) instead of sitting through them.
const val REPLAY_MAX_GAP_MS = 2000L
const val REPLAY_POSITION_INTERVAL_MS = 1000L
const val REPLAY_WAIT_SLICE_MS = 20L
const val REPLAY_FINISHED_STATUS = "Replay finished."
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
@JvmStatic
@@ -123,6 +145,8 @@ class VoiceCaptureSession(
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
}
private enum class Source { G2, PHONE, ARRAY, REPLAY }
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
val arm: String = arm ?: "?"
}
@@ -140,11 +164,33 @@ class VoiceCaptureSession(
@Volatile
private var workerThreadName: String? = null
// Capture from the phone's own microphone instead of the G2 over BLE
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
// (preview-only mode). Latched into activeSource at start() (under `lock`)
// so a mid-session setter call can't switch pipelines underneath the worker.
@Volatile
private var usePhoneMic = false
private var activePhoneMic = false
// What the next capture reads (see start()); latched into activeSource and its
// companions under `lock`, like usePhoneMic.
@Volatile
private var micArrayConfigure: ByteArray? = null
@Volatile
private var replayPath: String? = null
@Volatile
private var replayStartMs = 0L
private var activeSource = Source.G2
private var activeConfigure: ByteArray? = null
private var activeReplayPath: String? = null
private var activeReplayStartMs = 0L
// Replay controls, picked up by the worker between chunks.
@Volatile
private var replaySeekMs = -1L
@Volatile
private var replayPaused = false
@Volatile
private var recorder: MicRecordingWriter? = null
@Volatile
private var mixer: MicArrayMixer = SingleTempleMixer()
private var lastMissingPackets = 0L
private var arrayRateWarned = false
private var mode = Mode.CLOUD
@Volatile
private var onboardModelKind = VoiceModelKind.MOONSHINE
@@ -158,11 +204,8 @@ class VoiceCaptureSession(
private var lastTranscriptDecodeAtMs: Long = 0
private var lastTranscript = ""
@Volatile
private var saveRecordings = false
@Volatile
private var endpointing = false
private val endpointDetector = VoiceEndpointDetector()
private var recordingPcm: ByteSink? = null
@Volatile
private var verifySpeakerModelPath: String? = null
@Volatile
@@ -199,8 +242,45 @@ class VoiceCaptureSession(
this.usePhoneMic = usePhoneMic
}
fun setSaveRecordings(saveRecordings: Boolean) {
this.saveRecordings = saveRecordings
/**
* Capture the next glasses session from the temples' own microphones instead of the stock
* mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples
* (it decides how many microphones each one streams). Null goes back to the mixed stream.
*/
fun setMicArray(configureRecord: ByteArray?) {
this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null
}
/** How the temples' chunks become the mono feed that is transcribed. */
fun setMicArrayMixer(mixer: MicArrayMixer) {
this.mixer = mixer
}
/**
* Have the next capture play the recording at [path] from [startMs] in real time, in place
* of any microphone; null goes back to the microphones. A replay needs no glasses.
*/
fun setReplay(path: String?, startMs: Long) {
this.replayPath = path
this.replayStartMs = maxOf(0L, startMs)
}
/** Continue the running replay from [positionMs] into its recording. */
fun seekReplay(positionMs: Long) {
this.replaySeekMs = maxOf(0L, positionMs)
}
/** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */
fun setReplayPaused(paused: Boolean) {
this.replayPaused = paused
}
/**
* Where live captures' chunks are recorded as they arrive, or null. Takes effect at once,
* also mid-capture; the caller finishes the recording. A replay is never recorded.
*/
fun setRecorder(recorder: MicRecordingWriter?) {
this.recorder = recorder
}
fun setOnboardModelKind(kind: String?) {
@@ -268,13 +348,26 @@ class VoiceCaptureSession(
emitStopped(captureId)
return
}
if (!usePhoneMic && !host.isG2SessionReady()) {
val path = replayPath
val configure = micArrayConfigure
val source = when {
path != null -> Source.REPLAY
usePhoneMic -> Source.PHONE
configure != null -> Source.ARRAY
else -> Source.G2
}
if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) {
emitStatus("Voice control needs an active G2 connection.")
emitStopped(captureId)
return
}
mode = parseMode(requestedMode)
activePhoneMic = usePhoneMic
activeSource = source
activeConfigure = configure
activeReplayPath = path
activeReplayStartMs = replayStartMs
replaySeekMs = -1L
replayPaused = false
started = true
audioStarted = false
val finished = Latch(1, platform)
@@ -291,21 +384,22 @@ class VoiceCaptureSession(
*/
fun isCapturing(): Boolean {
var audioUp = false
var phoneMic = false
var source = Source.G2
lock.withLock {
if (!started) {
return false
}
audioUp = audioStarted
phoneMic = activePhoneMic
source = activeSource
}
if (!audioUp) {
// The worker is still bringing the mic up; report it as running so
// a concurrent request shares it instead of restarting it.
return true
}
if (phoneMic) {
// The phone mic has no session to lose the enable to; it runs until stop().
if (source == Source.PHONE || source == Source.REPLAY) {
// Neither has a glasses session to lose the enable to: the phone mic runs
// until stop(), a replay until its recording ends.
return true
}
return host.isG2AudioCaptureActive()
@@ -327,7 +421,7 @@ class VoiceCaptureSession(
finished = workerFinished
workerName = workerThreadName
}
stopG2Audio()
stopSource()
audioQueue.wake()
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
finished.await(1500)
@@ -363,32 +457,65 @@ class VoiceCaptureSession(
}
endpointDetector.reset()
suppressor?.reset()
recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
verifyCount = 0
if (activePhoneMic) {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
mixer.reset()
lastMissingPackets = 0
val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..."
when (activeSource) {
Source.PHONE -> {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
try {
processPhoneAudio(record)
} finally {
record.close()
}
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
try {
processPhoneAudio(record)
} finally {
record.close()
Source.REPLAY -> {
val reader = try {
MicRecordingReader(host.openRecording(activeReplayPath!!))
} catch (error: Throwable) {
emitStatus("Could not open the recording: " + error.message)
emitReplay("error", 0, 0)
return
}
try {
lock.withLock { audioStarted = true }
logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s")
emitStatus(listening)
processReplay(reader)
} finally {
reader.close()
}
}
} else {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
Source.ARRAY -> {
val link = host.micArrayLink()
val configure = activeConfigure
if (link == null || configure == null || !startArray(link, configure)) {
emitStatus("Could not start the glasses microphone array.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
processArray(link, configure, listening)
}
Source.G2 -> {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
processG2Audio()
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
processG2Audio()
}
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
@@ -403,8 +530,7 @@ class VoiceCaptureSession(
logError("Voice control failed", error)
emitStatus("Voice control failed: " + error.message)
} finally {
stopG2Audio()
writeRecordingIfAny()
stopSource()
releaseRecognizer()
releaseDecoder()
lock.withLock {
@@ -418,25 +544,6 @@ class VoiceCaptureSession(
}
}
private fun appendRecording(pcm: ShortArray, count: Int) {
val out = recordingPcm ?: return
out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count))
}
private fun writeRecordingIfAny() {
val out = recordingPcm
recordingPcm = null
if (out == null || out.size() == 0) {
return
}
val sink = host.recordingSink() ?: return
try {
sink.save(out.toByteArray(), SAMPLE_RATE)
} catch (t: Throwable) {
logWarn("failed to save voice recording (${t.message})")
}
}
private fun startG2Audio(): Boolean {
val source = host.g2AudioSource() ?: return false
resetAudioStats()
@@ -449,9 +556,10 @@ class VoiceCaptureSession(
}
private fun processG2Audio() {
val pcm = ShortArray(PACKET_SAMPLES)
var pcm = ShortArray(PACKET_SAMPLES)
while (started) {
val currentDecoder = packetDecoder ?: return
if (pcm.size < currentDecoder.maxSamplesPerPacket) pcm = ShortArray(currentDecoder.maxSamplesPerPacket)
val packet = takeAudioPacket()
if (packet == null) {
continue
@@ -462,23 +570,214 @@ class VoiceCaptureSession(
continue
}
decodedSamples += count.toLong()
// The beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker for
// every consumer, recognition included); the spectral noise
// suppressor then cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
val angleDegrees = currentDecoder.lastAngleDegrees
val ssr = currentDecoder.lastSsr
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr)
maybeEmitAudioStats(false)
continue
}
processPcmChunk(pcm, count, angleDegrees, ssr, true)
val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs)
chunk.hasDirection = currentDecoder.hasFrameMeta
chunk.angleDegrees = currentDecoder.lastAngleDegrees
chunk.ssr = currentDecoder.lastSsr
// Packets the decoder found missing ahead of this one: a gap in a recording.
val missing = currentDecoder.missingPackets
chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt()
lastMissingPackets = missing
handleMonoChunk(chunk)
maybeEmitAudioStats(false)
}
}
/**
* A mixed-microphone chunk from any source: recorded as it came, then on to processing
* unless the beam filter drops it. The filter drops chunks whose firmware
* direction-of-arrival falls outside the listening wedge (isolating the aimed talker for
* every consumer, recognition included); the spectral noise suppressor then cleans what
* remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification.
*/
private fun handleMonoChunk(chunk: MicChunk) {
recordChunk(chunk)
if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) {
emitFrameMeta(chunk.angleDegrees, chunk.ssr)
return
}
processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection)
}
/**
* A chunk of one temple's own microphones: recorded as it came, then mixed into the mono
* feed, which is recorded too (as what was transcribed) and processed like a mixed chunk.
* The beam filter does not apply: a temple's bearing is along its own pair, not the
* head-frame direction the filter is aimed in.
*/
private fun handleArrayChunk(chunk: MicChunk) {
recordChunk(chunk)
val feed = mixer.mix(chunk) ?: return
recordChunk(feed)
if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) {
if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz")
arrayRateWarned = true
return
}
processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection)
}
private fun recordChunk(chunk: MicChunk) {
if (activeSource != Source.REPLAY) recorder?.write(chunk)
}
private fun handleChunk(chunk: MicChunk) {
if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk)
}
private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean {
resetAudioStats()
audioQueue.clear()
if (!startArrayForwarding(link)) return false
return link.sendControl(configure, "configure", true, true)
}
private fun startArrayForwarding(link: MicArrayLink): Boolean =
link.startForwarding(object : FaceclawAudioPacketListener {
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
queueAudioPacket(data, arm, arrivalMs)
}
})
/**
* Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream.
* Between frames it keeps the streams alive: renews the temples' lease, configures a temple
* again when it has gone quiet, and takes forwarding back when the glasses session dropped it.
*/
private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) {
val startedAt = platform.elapsedRealtimeMs()
var checkedAt = startedAt
var renewedAt = startedAt
// Left, right.
val armedAt = longArrayOf(startedAt, startedAt)
val frameAt = longArrayOf(startedAt, startedAt)
var waiting = false
while (started) {
val packet = takeAudioPacket()
val now = platform.elapsedRealtimeMs()
if (packet != null) {
val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1
val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null
val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null
// Anything else is a stock-format packet from before arming took effect.
if (frame != null && pcm != null) {
frameAt[side] = now
val frames = pcm.size / frame.channels
decodedSamples += frames.toLong()
val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT
val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs)
chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0
chunk.angleDegrees = frame.angleDegrees
chunk.ssr = frame.ssr
chunk.sequence = frame.sequence
chunk.deviceTickMs = frame.deviceTickMs
chunk.truncated = frame.truncated
handleArrayChunk(chunk)
}
}
if (now - checkedAt >= ARRAY_CHECK_MS) {
checkedAt = now
if (!link.isForwarding()) {
if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) {
logInfo("mic array forwarding re-established")
armedAt.fill(now)
renewedAt = now
}
} else {
if (now - renewedAt >= ARRAY_RENEW_MS) {
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true)
renewedAt = now
}
for (side in 0..1) {
if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue
logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again")
link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0)
armedAt[side] = now
}
}
val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS
if (silent != waiting) {
waiting = silent
emitStatus(if (silent) ARRAY_WAITING_STATUS else listening)
}
}
maybeEmitAudioStats(false)
}
}
/**
* Replay loop: hands the recording's chunks to the pipeline at the pace they were captured,
* so recognition (its partials, a cloud provider's stream) sees what a live capture would.
* Jumps and pauses asked for from outside take effect between chunks.
*/
private fun processReplay(reader: MicRecordingReader) {
val duration = reader.durationMs
var position = minOf(activeReplayStartMs, duration)
reader.seekTo(position)
emitReplay("start", position, duration)
var reportedAt = position
// Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap.
var anchorWall = -1L
var anchorFile = 0L
var pending: MicChunk? = null
while (started) {
val seek = replaySeekMs
if (seek >= 0) {
replaySeekMs = -1L
// What played so far is a finished utterance; the rest starts clean.
finishReplayedUtterance()
position = minOf(seek, duration)
reader.seekTo(position)
pending = null
anchorWall = -1L
reportedAt = position
emitReplay("seek", position, duration)
}
if (replayPaused) {
anchorWall = -1L
sleepMs(REPLAY_WAIT_SLICE_MS)
continue
}
val chunk = pending ?: reader.next()
if (chunk == null) {
emitReplay("end", duration, duration)
emitStatus(REPLAY_FINISHED_STATUS)
return
}
pending = chunk
val now = platform.elapsedRealtimeMs()
if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) {
anchorWall = now
anchorFile = chunk.timeMs
}
val wait = anchorWall + (chunk.timeMs - anchorFile) - now
if (wait > 0) {
// Short slices, so a stop, jump or pause is noticed promptly.
sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS))
continue
}
pending = null
position = chunk.timeMs
decodedSamples += chunk.frames.toLong()
handleChunk(chunk)
if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) {
reportedAt = position
emitReplay("position", position, duration)
}
}
}
/** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */
private fun finishReplayedUtterance() {
mixer.reset()
endpointDetector.reset()
suppressor?.reset()
if (mode != Mode.ONBOARD) return
decodeTranscript(true)
resetTranscriptState()
lastTranscript = ""
}
/**
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
* decode and the beam filter. hasFrameMeta is false for the phone mic,
@@ -488,9 +787,6 @@ class VoiceCaptureSession(
if (suppressionEnabled) {
applySuppression(pcm, count)
}
if (recordingPcm != null) {
appendRecording(pcm, count)
}
val verifyBuffer = this.verifyBuffer
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
@@ -528,7 +824,7 @@ class VoiceCaptureSession(
continue
}
decodedSamples += read.toLong()
processPcmChunk(pcm, read, 0, 0, false)
handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs()))
}
}
@@ -714,6 +1010,11 @@ class VoiceCaptureSession(
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
}
private fun emitReplay(event: String, positionMs: Long, durationMs: Long) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) }
}
private fun emitSpeechEnd() {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onSpeechEnd() }
@@ -724,8 +1025,21 @@ class VoiceCaptureSession(
host.dispatcher.post { currentListener.onStopped(captureId) }
}
private fun stopG2Audio() {
host.g2AudioSource()?.stop()
/**
* Release what the running capture's source holds on the glasses. Runs on stop() and again
* as the worker winds down (it may have started the source after stop() looked), so
* everything here is safe to repeat.
*/
private fun stopSource() {
val source = lock.withLock { activeSource }
when (source) {
Source.G2 -> host.g2AudioSource()?.stop()
Source.ARRAY -> host.micArrayLink()?.let { link ->
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true)
link.stopForwarding()
}
Source.PHONE, Source.REPLAY -> {}
}
maybeEmitAudioStats(true)
}
@@ -744,7 +1058,7 @@ class VoiceCaptureSession(
if (!started || data == null) {
return
}
if ("L" != arm) {
if ("L" != arm && activeSource != Source.ARRAY) {
wrongArmPackets++
}
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
@@ -34,15 +34,19 @@ fun interface CallbackDispatcher {
fun post(action: () -> Unit)
}
/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */
fun interface RecordingSink {
fun save(pcm16le: ByteArray, sampleRate: Int)
}
/** A G2 microphone packet decoder (the shared [Lc3PacketFramer] over a platform LC3 codec). */
/**
* A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared
* [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus.
*/
interface AudioPacketDecoder {
fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int
/** The most samples one packet decodes to: the size [decodePacket]'s pcmOut needs. */
val maxSamplesPerPacket: Int get() = Lc3PacketFramer.SAMPLES_PER_PACKET
/** Whether packets carry the G2 firmware's DSP metadata ([lastAngleDegrees], [lastSsr]). */
val hasFrameMeta: Boolean get() = true
val lastAngleDegrees: Int
val lastSsr: Int
val realPackets: Long
@@ -63,6 +67,24 @@ interface PacketAudioSource {
fun isActive(): Boolean
}
/**
* The custom firmware's mic_control channel on a glasses session: each temple streams its own
* microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed
* stream uses, while it holds a configuration that the phone keeps renewing.
*/
interface MicArrayLink {
/** Deliver the audio characteristic's packets to [listener]; false when no session is up. */
fun startForwarding(listener: FaceclawAudioPacketListener): Boolean
fun stopForwarding()
/** Whether forwarding still holds: it lives in the glasses session, which can drop it. */
fun isForwarding(): Boolean
/** Queue a control record for the chosen temples; false when nothing was queued. */
fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean
}
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
interface PcmAudioSource {
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
@@ -0,0 +1,10 @@
package com.faceclaw.app
interface FaceclawKeyboardEventListener {
/**
* One CFW keyboard-bridge record (settings field 131) from the glasses: the raw
* ['K','B',ver,kind,...] bytes, possibly one fragment of a longer record. Decoded and
* reassembled on the TS side (app/g2/keyboard/protocol.ts). Delivered in arrival order.
*/
fun onKeyboardEvent(body: ByteArray?): Unit
}
@@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener {
* speaker model could not be loaded.
*/
fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit
/**
* Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay):
* "start" once the file is open, "position" about once per second of it played, "seek" after
* a jump took effect (the transcript of what played before it has been emitted as final),
* "end" when the file ran out, and "error" when it could not be opened (the status says
* why). Positions are milliseconds into the file.
*/
fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit
}
@@ -0,0 +1,794 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmField
class G1FlashTimings(
@JvmField val connectTimeoutMs: Int = ConnectionOptions.CONNECT_TIMEOUT_MS,
@JvmField val servicesTimeoutMs: Int = ConnectionOptions.SERVICES_TIMEOUT_MS,
@JvmField val descriptorTimeoutMs: Int = ConnectionOptions.DESCRIPTOR_TIMEOUT_MS,
@JvmField val writeTimeoutMs: Int = ConnectionOptions.WRITE_TIMEOUT_MS,
/** The OS may be showing its pairing dialog. */
@JvmField val pairingMs: Int = 60_000,
@JvmField val retryDelayMs: Int = 2_000,
@JvmField val controlReplyMs: Int = 1_500,
@JvmField val controlTries: Int = 4,
/** The first chunk of an upload waits for the arm to erase its slot. */
@JvmField val uploadReplyMs: Int = 30_000,
@JvmField val smpReplyMs: Int = 20_000,
/** The arm may go down before its reply leaves. */
@JvmField val resetReplyMs: Int = 5_000,
@JvmField val heartbeatMs: Int = 8_000,
@JvmField val pollMs: Int = 200,
@JvmField val uploadRetries: Int = 40,
/** How often a store job's state is asked for while its notice hasn't come. */
@JvmField val storePollMs: Int = 2_000,
/** Erasing the store over Even's old fonts and pictures takes minutes. */
@JvmField val wipeMs: Int = 1_800_000,
@JvmField val crcMs: Int = 30_000,
)
/**
* Installs an MCUboot image on one G1 arm through MCUmgr SMP, or asks an arm what it runs; a port of scouter_kit's
* g1_firmware/flash.py. The caller handles one arm at a time and looks for the arm again afterwards: it restarts, and
* coming off firmware older than g1v0 0.5.66 it has a new address.
*
* Nothing changes on the arm until the confirm, which comes only after the arm's image list shows that its second slot
* holds exactly this image. MCUboot then overwrites the running firmware with no way back. [run] blocks; the worker thread
* is the host's.
*
* [Mode.RESTORE] puts Even's firmware back on an arm running g1v0: both of a release's images, the application core's
* ([firmwarePath]) and the network core's ([netCorePath]), as Even's own update does, so the two always match.
*/
class G1ArmFlashFlow(
private val link: StockLink,
private val address: String,
private val side: String,
private val firmwarePath: String,
private val netCorePath: String,
private val mode: Mode,
/** How long to keep trying to reach the arm and pair with it. */
private val connectWindowMs: Int,
private val listener: FaceclawFirmwareFlasherListener,
private val timings: G1FlashTimings = G1FlashTimings(),
platform: ProtocolPlatform = protocolPlatform(),
private val readImage: (String) -> ByteArray = ::readFileData,
) : StockLinkListener {
enum class Mode {
/** The arm runs Even's firmware. */
STOCK,
/** The arm runs g1v0: it answers what it runs, and an arm that already has this version is left alone. */
UPDATE,
/** Only say what the arm runs, and pair with it. */
PROBE,
/** The arm runs g1v0 and gets Even's firmware back. */
RESTORE,
}
companion object {
const val TAG = "FaceclawG1Flash"
private const val MIN_BATTERY_PERCENT = 30
private const val ANY_STORE_JOB = -1
private const val STORE_JOB_INTERRUPTED = -1
fun mode(name: String): Mode = when (name) {
"stock" -> Mode.STOCK
"update" -> Mode.UPDATE
"probe" -> Mode.PROBE
"restore" -> Mode.RESTORE
else -> throw IllegalArgumentException("unknown mode $name")
}
}
/** One image of an install: MCUboot's number for it, its bytes and the hash its slot is checked against. */
private class Part(val imageIndex: Int, val bytes: ByteArray, val label: String) {
val hash: ByteArray = G1Smp.imageHash(bytes)
}
private class LinkLostException(message: String) : Exception(message)
/** The arm has no g1v0 service. */
private class NotG1v0Exception : Exception("this arm doesn't run g1v0")
private val platform = platform
private val lock = platform.createLock()
private val sleeper = InterruptibleSleep(platform)
private val assembler = G1Smp.Assembler()
private var smpSeq = -1
private var smpReply: Map<*, *>? = null
private var smpLatch: Latch? = null
private var nextSmpSeq = 0
private var controlTag = -1
private var controlReply: G1Protocol.Reply? = null
private var controlLatch: Latch? = null
private var nextControlTag = 0
private var stockBattery: Int? = null
private var stockBatteryLatch: Latch? = null
/** `NOTICE` 8s since the store job now waited for was asked for. */
private val storeNotices = ArrayList<G1Protocol.StoreNotice>()
/** The OS's half of a stale pairing goes once per run: a second failure isn't that. */
private var bondRemoved = false
private var linkLost = false
private var heartbeats = 0
private var nextHeartbeatAtMs = 0L
@Volatile
private var cancelled = false
fun cancel() {
cancelled = true
lock.withLock {
smpLatch?.countDown()
controlLatch?.countDown()
}
sleeper.interrupt()
}
fun run() {
link.setListener(this)
try {
when (mode) {
Mode.PROBE -> probe()
Mode.RESTORE -> restore()
else -> install()
}
} catch (e: Exception) {
val message = if (cancelled) "Cancelled." else StockLinkSession.messageOf(e)
log("failed: $message")
listener.onState("error", message)
listener.onComplete(false, message)
} finally {
quietly { link.disconnect(address) }
}
}
private fun probe() {
listener.onState("connecting", side)
try {
bringUp()
} catch (e: NotG1v0Exception) {
listener.onComplete(true, """{"firmware":"stock"}""")
return
}
listener.onState("checking", side)
val hello = hello()
listener.onComplete(true, """{"firmware":"g1v0","version":"${hello.version}","safeMode":${hello.safeMode},"inCase":${hello.inCase},"battery":${batteryPercent()}}""")
}
private fun readFirmware(path: String): ByteArray = try {
readImage(path)
} catch (e: Exception) {
throw IllegalStateException("could not read the firmware: " + StockLinkSession.messageOf(e), e)
}
private fun install() {
listener.onState("validating", side)
val image = readFirmware(firmwarePath)
val version = G1Smp.imageVersion(image)
val hash = G1Smp.imageHash(image)
log("image $version, ${image.size} bytes, hash ${BinaryEncoding.bytesToHex(hash.copyOf(8))}")
listener.onState("connecting", side)
bringUp()
if (mode == Mode.UPDATE) {
listener.onState("checking", side)
val hello = hello()
log("runs g1v0 ${hello.version}, safe mode ${hello.safeMode}, in the case ${hello.inCase}, encrypted ${hello.encrypted}")
if (hello.version == version && !hello.safeMode) {
listener.onComplete(true, """{"flashed":false,"version":"$version"}""")
return
}
requireBattery(batteryPercent())
} else {
val battery = stockBatteryPercent()
log(if (battery == null) "battery unknown; going on, as the arm is in the case" else "battery $battery%")
requireBattery(battery)
}
val slots = retrying { images() }
for (slot in slots) log(slot.toString())
listener.onState("flashing", side)
if (holdsImage(slots, G1Smp.IMAGE_APP, hash)) {
log("the second slot already holds this image")
listener.onProgress(side, 1, 1, image.size, image.size, image.size.toLong(), image.size.toLong())
} else {
val startedAt = now()
upload(image, G1Smp.IMAGE_APP)
log("uploaded in ${(now() - startedAt) / 1000} s")
}
if (!holdsImage(retrying { images() }, G1Smp.IMAGE_APP, hash)) {
throw IllegalStateException("The $side arm's second slot doesn't hold the image after the upload. Nothing was installed.")
}
throwIfCancelled()
listener.onState("committing", side)
val confirm = smp(G1Smp.OP_WRITE, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_STATE, G1Smp.confirmRequest(hash), timings.smpReplyMs)
val code = G1Smp.replyCode(confirm)
if (code != 0) throw IllegalStateException("The $side arm refused the image (error $code). Nothing was installed.")
log("confirmed: MCUboot installs it at the next boot")
reset()
listener.onState("rebooting", side)
listener.onComplete(true, """{"flashed":true,"version":"$version"}""")
}
/**
* Even's firmware back on an arm running g1v0: the store cleared if g1v0 ever wrote to it, then both images uploaded
* and checked, and only then marked for MCUboot, the network core first. MCUboot installs both at the reset, the
* network core through its own path to that core, as at every update from Even.
*/
private fun restore() {
listener.onState("validating", side)
val parts = listOf(
Part(G1Smp.IMAGE_NET_CORE, readFirmware(netCorePath), "network core"),
Part(G1Smp.IMAGE_APP, readFirmware(firmwarePath), "application"),
)
for (part in parts) log("${part.label} image: ${part.bytes.size} bytes, hash ${BinaryEncoding.bytesToHex(part.hash.copyOf(8))}")
listener.onState("connecting", side)
bringUp()
listener.onState("checking", side)
val hello = hello()
log("runs g1v0 ${hello.version}, safe mode ${hello.safeMode}, in the case ${hello.inCase}, encrypted ${hello.encrypted}")
requireBattery(batteryPercent())
val wiped = clearStore(hello)
val slots = retrying { images() }
for (slot in slots) log(slot.toString())
listener.onState("flashing", side)
val total = parts.sumOf { it.bytes.size }
var before = 0
for ((index, part) in parts.withIndex()) {
if (holdsImage(slots, part.imageIndex, part.hash)) {
log("the ${part.label}'s second slot already holds Even's image")
listener.onProgress(side, index + 1, parts.size, part.bytes.size, part.bytes.size, (before + part.bytes.size).toLong(), total.toLong())
} else {
val startedAt = now()
upload(part.bytes, part.imageIndex, index + 1, parts.size, before, total)
log("${part.label} uploaded in ${(now() - startedAt) / 1000} s")
}
before += part.bytes.size
}
val uploaded = retrying { images() }
for (part in parts) {
if (!holdsImage(uploaded, part.imageIndex, part.hash)) {
throw IllegalStateException("The $side arm's slot for its ${part.label} doesn't hold Even's image after the upload. Nothing was installed.")
}
}
throwIfCancelled()
listener.onState("committing", side)
// The network core first. If the second request fails, what's left pending is Even's network core under g1v0,
// which g1v0 works with (its fallback installs this very image); Even's application over the older network core
// is the combination that has never been tried.
for (part in parts) {
val permanent = part.imageIndex == G1Smp.IMAGE_APP
val reply = smp(G1Smp.OP_WRITE, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_STATE, G1Smp.confirmRequest(part.hash, permanent), timings.smpReplyMs)
val code = G1Smp.replyCode(reply)
if (code != 0) {
throw IllegalStateException("The $side arm refused Even's ${part.label} image (error $code). The arm wasn't restarted; retry to try again.")
}
}
val marked = retrying { images() }
for (slot in marked) log(slot.toString())
for (part in parts) {
if (!marked.any { it.image == part.imageIndex && it.slot == 1 && it.hash.contentEquals(part.hash) && "pending" in it.flags }) {
throw IllegalStateException("The $side arm didn't mark Even's ${part.label} image for installing. The arm wasn't restarted; retry to try again.")
}
}
log("both images marked: MCUboot installs them at the next boot")
reset()
listener.onState("rebooting", side)
listener.onComplete(true, """{"flashed":true,"firmware":"stock","wiped":$wiped}""")
}
private fun reset() {
try {
smp(G1Smp.OP_WRITE, G1Smp.GROUP_OS, G1Smp.CMD_OS_RESET, emptyMap(), timings.resetReplyMs)
} catch (e: StockTimeoutException) {
log("no answer to the reset")
} catch (e: LinkLostException) {
log("the link went down with the reset")
}
}
// ---- the asset store ---------------------------------------------------------
/**
* Before Even's firmware goes back: g1v0's asset store took the flash where Even's fonts and pictures were, so if it
* ever wrote there, Even's firmware must meet erased flash (flash.py's `wipe_store`). A store that was never written
* is left alone, and Even's data with it. True when it was erased.
*/
private fun clearStore(hello: G1Protocol.Hello): Boolean {
repeat(3) {
val info = assetInfo()
if (info == null) {
log("no asset store: this firmware never wrote where Even's data is")
return false
}
if (!info.available) {
log("the asset store is unavailable, so nothing was written to it")
return false
}
if (info.running) {
// A retried request can find its own job running, as can a retry after a link lost during a wipe.
log("a storage job is running: waiting for it to end")
clearStoreNotices()
waitStoreJob(ANY_STORE_JOB, timings.wipeMs.toLong())
return@repeat
}
if (!storeWritten(info, hello)) {
log("the asset store was never written: Even's fonts and pictures stay as they are")
return false
}
if (hello.safeMode) throw IllegalStateException(safeModeMessage())
listener.onState("wiping", side)
log("erasing the asset store (${info.assets} assets, commit ${info.seq})")
val startedAt = now()
clearStoreNotices()
when (val status = control(G1Protocol.OP_ASSET_FORMAT, G1Protocol.assetFormatRequest(wipe = true)).status) {
0 -> {
val result = waitStoreJob(G1Protocol.STORE_JOB_FORMAT, timings.wipeMs.toLong())
if (result == STORE_JOB_INTERRUPTED) {
throw IllegalStateException("The $side arm restarted while erasing Faceclaw's storage. Nothing was installed; retry to finish it.")
}
if (result != 0) {
throw IllegalStateException("The $side arm couldn't erase Faceclaw's storage (error $result). Nothing was installed.")
}
log("asset store erased in ${(now() - startedAt) / 1000} s")
return true
}
G1Protocol.STATUS_SAFE_MODE -> throw IllegalStateException(safeModeMessage())
// A job runs: the next round waits for it.
G1Protocol.STATUS_ASSET -> {}
else -> throw IllegalStateException("The $side arm refused to erase Faceclaw's storage (error $status). Nothing was installed.")
}
}
throw IllegalStateException("The $side arm's storage couldn't be erased. Nothing was installed.")
}
private fun safeModeMessage(): String =
"The $side arm is in safe mode (its firmware restarted repeatedly), where it can't erase Faceclaw's storage, and that has to " +
"happen before Even's firmware goes back. Install Faceclaw's firmware again, then uninstall it. Nothing was installed."
/**
* Whether g1v0 ever wrote to the store. It only reads it at boot, so with no assets and no commit nothing was written,
* unless a format erased the directory's two sectors (a format resets the commit count), which their fingerprint shows.
*/
private fun storeWritten(info: G1Protocol.AssetInfo, hello: G1Protocol.Hello): Boolean {
if (info.assets > 0 || info.seq != 0L) return true
if (hello.safeMode) {
log("safe mode: the directory can't be checked")
return true
}
val nonErased = flashNonErased(G1Protocol.STORE_ADDRESS, G1Protocol.STORE_DIRECTORY_BYTES)
log("the store's directory sectors: ${nonErased ?: "unknown"} bytes not erased")
return nonErased == null || nonErased == 0L
}
/** How many bytes of flash [from]…[from] + [n] - 1 aren't 0xFF, or null when the arm didn't say. */
private fun flashNonErased(from: Int, n: Int): Long? {
clearStoreNotices()
val status = try {
control(G1Protocol.OP_FLASH_CRC, G1Protocol.flashCrcRequest(from, n)).status
} catch (e: StockTimeoutException) {
return null
}
// 13: a crc job runs, which a retried request finds when the first one's reply was lost; its notice says the same.
if (status != 0 && status != G1Protocol.STATUS_ASSET) return null
val deadline = now() + timings.crcMs
while (now() < deadline) {
throwIfCancelled()
val notice = lock.withLock {
storeNotices.firstOrNull { it.job == G1Protocol.STORE_JOB_CRC && it.address == from && it.n == n }
}
if (notice != null) return if (notice.result == 0) notice.nonErased else null
if (!link.isConnected(address)) return null
sleeper.sleep(timings.pollMs.toLong())
}
return null
}
private fun assetInfo(): G1Protocol.AssetInfo? {
val reply = control(G1Protocol.OP_ASSET_INFO)
return if (reply.status == 0) G1Protocol.parseAssetInfo(reply.body) else null
}
private fun clearStoreNotices() = lock.withLock { storeNotices.clear() }
/**
* The result of the store job running now: its `NOTICE` 8, or `ASSET_INFO`'s state once it has ended (a notice is lost
* with the link, and the job goes on without it). [STORE_JOB_INTERRUPTED] when the arm restarted meanwhile, which ends
* a job unfinished.
*/
private fun waitStoreJob(job: Int, timeoutMs: Long): Int {
val deadline = now() + timeoutMs
var askAt = now() + timings.storePollMs
while (true) {
throwIfCancelled()
val notice = lock.withLock { storeNotices.firstOrNull { job == ANY_STORE_JOB || it.job == job } }
if (notice != null) return notice.result
if (now() >= deadline) throw StockTimeoutException("the $side arm's storage job didn't end within ${timeoutMs / 60_000} minutes")
if (now() >= askAt) {
askAt = now() + timings.storePollMs
val info = try {
assetInfo()
} catch (e: Exception) {
if (e !is StockTimeoutException && e !is LinkLostException) throw e
log("no answer while the storage job runs (${e.message})")
reconnectIfNeeded()
null
}
when (info?.state) {
G1Protocol.STORE_DONE -> return info.lastError
G1Protocol.STORE_FAILED -> return if (info.lastError != 0) info.lastError else G1Protocol.STATUS_ASSET
// A job leaves the state at done or failed until the next boot.
G1Protocol.STORE_IDLE -> return STORE_JOB_INTERRUPTED
}
}
sleeper.sleep(timings.pollMs.toLong())
}
}
private fun requireBattery(percent: Int?) {
if (percent != null && percent < MIN_BATTERY_PERCENT) {
throw IllegalStateException("The $side arm's battery is at $percent%, too low to install safely. " +
"Charge both arms to at least $MIN_BATTERY_PERCENT% in the case, then try again.")
}
}
private fun holdsImage(slots: List<G1Smp.Slot>, imageIndex: Int, hash: ByteArray): Boolean =
slots.any { it.image == imageIndex && it.slot == 1 && it.hash.contentEquals(hash) }
private fun images(): List<G1Smp.Slot> =
G1Smp.parseSlots(smp(G1Smp.OP_READ, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_STATE, emptyMap(), timings.smpReplyMs))
/**
* MCUmgr answers each chunk with the offset it wants next, so an upload resumes after a lost link. Progress counts
* [before] bytes of earlier parts out of [total].
*/
private fun upload(image: ByteArray, imageIndex: Int, part: Int = 1, parts: Int = 1, before: Int = 0, total: Int = image.size) {
val fileSha = sha256(image)
var offset = 0
var failures = 0
while (offset < image.size) {
throwIfCancelled()
val reply = try {
smp(G1Smp.OP_WRITE, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_UPLOAD, G1Smp.uploadRequest(image, offset, fileSha, imageIndex), timings.uploadReplyMs)
} catch (e: Exception) {
if (e !is StockTimeoutException && e !is LinkLostException) throw e
failures++
if (failures > timings.uploadRetries) throw e
log("chunk at $offset failed (${e.message}), trying again")
reconnectIfNeeded()
continue
}
val code = G1Smp.replyCode(reply)
val next = G1Smp.replyOffset(reply)
if (code != 0 || next == null || next < 0 || next > image.size) {
throw IllegalStateException("The $side arm refused the upload at $offset (error $code). Nothing was installed.")
}
offset = next
listener.onProgress(side, part, parts, offset, image.size, (before + offset).toLong(), total.toLong())
}
}
private fun <T> retrying(request: () -> T): T {
var attempt = 0
while (true) {
try {
return request()
} catch (e: Exception) {
if (e !is StockTimeoutException && e !is LinkLostException) throw e
attempt++
if (attempt >= 10) throw e
log("request failed (${e.message}), trying again")
reconnectIfNeeded()
}
}
}
private fun reconnectIfNeeded() {
throwIfCancelled()
lock.withLock { assembler.reset() }
if (!link.isConnected(address)) bringUp()
}
// ---- the link ------------------------------------------------------------
private fun bringUp() {
val deadline = now() + connectWindowMs
var attempt = 0
while (true) {
throwIfCancelled()
attempt++
val failure = try {
connectOnce()
log("connected (attempt $attempt)")
return
} catch (e: NotG1v0Exception) {
throw e
} catch (e: Exception) {
StockLinkSession.messageOf(e)
}
log("connect attempt $attempt failed: $failure")
quietly { link.disconnect(address) }
if (now() >= deadline) {
throw IllegalStateException("Couldn't reach the $side arm: $failure. Keep the glasses in the case with the lid open; a fully " +
"charged arm can take up to 5 minutes to notice it's in the case. If this phone was paired with these glasses before and " +
"this keeps happening, remove them from its Bluetooth settings and try again.")
}
sleeper.sleep(timings.retryDelayMs.toLong())
}
}
private fun connectOnce() {
if (!link.connect(address, timings.connectTimeoutMs)) throw IllegalStateException("connect failed")
lock.withLock {
linkLost = false
assembler.reset()
}
link.prepareLink(address, G1Ble.REQUESTED_MTU, timings.connectTimeoutMs)
if (!link.discoverServices(address, timings.servicesTimeoutMs)) throw IllegalStateException("service discovery failed")
if (mode == Mode.STOCK) {
// Even's firmware serves its own service only on an encrypted link, and the heartbeat goes there.
settleBond()
if (!haveCharacteristic(G1Smp.NUS_TX_UUID)) throw IllegalStateException("no link to Even's service")
if (!subscribe(G1Smp.NUS_TX_UUID)) {
// Bonded and refused: Even's firmware keeps one bond and has lost this one. This run started in the case.
removeStaleBond()
throw IllegalStateException("no link to Even's service")
}
heartbeats = 0
nextHeartbeatAtMs = 0
} else {
if (!haveCharacteristic(G1Ble.EVT_CHAR_UUID)) {
if (!link.isConnected(address)) throw LinkLostException("the link went down")
throw NotG1v0Exception()
}
if (!subscribe(G1Ble.EVT_CHAR_UUID)) throw IllegalStateException("no events from the arm")
// Out of the case the arm obeys only an encrypted link.
settleBond()
val paired = try {
link.writeFrames(address, G1Ble.PAIR_CHAR_UUID, listOf(byteArrayOf(1)), GattWriteMode.WITH_RESPONSE, timings.pairingMs)
} catch (e: IllegalStateException) {
false
}
if (!paired) {
// The OS says bonded and the arm disagrees. Giving that up is safe only where a new bond can be made.
if (link.isConnected(address) && inCaseWithoutEncryption()) removeStaleBond()
throw IllegalStateException("no encrypted link: an arm pairs only in its case")
}
}
if (mode != Mode.PROBE && (!haveCharacteristic(G1Smp.SMP_CHAR_UUID) || !subscribe(G1Smp.SMP_CHAR_UUID))) {
throw IllegalStateException("no update service")
}
}
/** An arm keeps its address across an install, and the OS may remember the services it had before: look again before believing that. */
private fun haveCharacteristic(uuid: String): Boolean {
if (link.hasCharacteristic(address, uuid)) return true
log("no $uuid in the service table: reading it again")
link.refreshServices(address, timings.servicesTimeoutMs)
return link.hasCharacteristic(address, uuid)
}
private fun subscribe(uuid: String): Boolean = try {
link.enableNotifications(address, uuid, timings.descriptorTimeoutMs)
} catch (e: IllegalStateException) {
false
}
private fun removeStaleBond() {
if (bondRemoved || link.bondState(address) != BondState.BONDED) return
if (link.removeBond(address)) {
bondRemoved = true
log("the arm no longer has this phone's pairing: removed it, pairing again")
}
}
/** An unencrypted link gets an answer only from an arm in the case. */
private fun inCaseWithoutEncryption(): Boolean = try {
val hello = G1Protocol.parseHello(control(G1Protocol.OP_HELLO, tries = 1).body)
hello != null && hello.inCase && !hello.encrypted
} catch (e: Exception) {
false
}
/** Null when no answer came, which firmware that doesn't have the query also gives. */
private fun stockBatteryPercent(): Int? {
repeat(timings.controlTries) {
val latch = Latch(1, platform)
lock.withLock {
stockBattery = null
stockBatteryLatch = latch
}
try {
link.writeFrames(address, G1Smp.NUS_RX_UUID, listOf(G1Smp.STOCK_BATTERY_QUERY), GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
} catch (e: IllegalStateException) {
return null
}
latch.await(timings.controlReplyMs.toLong())
throwIfCancelled()
val answer = lock.withLock {
stockBatteryLatch = null
stockBattery
}
if (answer != null) return answer
}
return null
}
/** Asks the OS to pair if it hasn't, and waits for that to end; a platform that doesn't say pairs by itself. */
private fun settleBond() {
if (link.bondState(address) == BondState.NONE) {
listener.onState("pairing", side)
link.requestBond(address)
}
val deadline = now() + timings.pairingMs
var sawBonding = false
while (true) {
throwIfCancelled()
when (link.bondState(address)) {
BondState.BONDED, BondState.UNKNOWN -> return
BondState.BONDING -> sawBonding = true
BondState.NONE -> if (sawBonding) throw IllegalStateException("pairing was refused")
}
if (!link.isConnected(address)) throw LinkLostException("the link went down while pairing")
if (now() >= deadline) throw IllegalStateException("pairing took too long")
sleeper.sleep(timings.pollMs.toLong())
}
}
private fun heartbeatIfDue() {
if (mode != Mode.STOCK || now() < nextHeartbeatAtMs) return
val seq = heartbeats % 0xff
heartbeats++
nextHeartbeatAtMs = now() + timings.heartbeatMs
try {
link.writeFrames(address, G1Smp.NUS_RX_UUID, listOf(G1Smp.stockHeartbeat(seq)), GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
} catch (e: IllegalStateException) {
// The request waiting now finds out.
}
}
private fun smp(op: Int, group: Int, cmd: Int, payload: Map<String, Any?>, timeoutMs: Int): Map<*, *> {
val seq = nextSmpSeq
nextSmpSeq = (nextSmpSeq + 1) and 0xff
val latch = Latch(1, platform)
lock.withLock {
smpSeq = seq
smpReply = null
smpLatch = latch
}
try {
heartbeatIfDue()
val frames = G1Smp.split(G1Smp.frame(op, group, seq, cmd, payload), link.negotiatedMtu(address))
val written = try {
link.writeFrames(address, G1Smp.SMP_CHAR_UUID, frames, GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
} catch (e: IllegalStateException) {
false
}
if (!written) throw LinkLostException("write failed")
val deadline = now() + timeoutMs
while (true) {
throwIfCancelled()
heartbeatIfDue()
val remaining = deadline - now()
if (remaining <= 0) throw StockTimeoutException("no answer from the $side arm within $timeoutMs ms")
latch.await(minOf(remaining, timings.pollMs.toLong()))
val (reply, lost) = lock.withLock { smpReply to linkLost }
if (reply != null) return reply
if (lost) throw LinkLostException("the link went down")
}
} finally {
lock.withLock {
if (smpLatch === latch) smpLatch = null
}
}
}
private fun control(op: Int, body: ByteArray = ByteArray(0), tries: Int = timings.controlTries): G1Protocol.Reply {
repeat(tries) {
val tag = nextControlTag % 255 + 1
nextControlTag = tag
val latch = Latch(1, platform)
lock.withLock {
controlTag = tag
controlReply = null
controlLatch = latch
}
val written = try {
link.writeFrames(address, G1Ble.CTRL_CHAR_UUID, listOf(G1Protocol.control(op, tag, body)), GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
} catch (e: IllegalStateException) {
false
}
if (!written) throw LinkLostException("write failed")
latch.await(timings.controlReplyMs.toLong())
throwIfCancelled()
val (reply, lost) = lock.withLock {
controlLatch = null
controlReply to linkLost
}
if (reply != null) return reply
if (lost) throw LinkLostException("the link went down")
}
throw StockTimeoutException("no answer to request $op from the $side arm")
}
private fun hello(): G1Protocol.Hello {
val reply = control(G1Protocol.OP_HELLO)
return G1Protocol.parseHello(reply.body) ?: throw IllegalStateException("The $side arm's firmware isn't one this app knows.")
}
/** Null when the arm doesn't know yet, or a firmware doesn't say. */
private fun batteryPercent(): Int? {
val reply = control(G1Protocol.OP_PROPS, byteArrayOf(G1Protocol.PROP_BATT_PCT.toByte()))
if (reply.status != 0) return null
val value = G1Protocol.parseProps(reply.body, 1)?.get(0) ?: return null
return if (value in 0..100) value else null
}
private fun throwIfCancelled() {
if (cancelled) throw IllegalStateException("Cancelled.")
}
private fun now(): Long = platform.elapsedRealtimeMs()
private fun log(line: String) = listener.onLog("$side arm: $line")
private inline fun quietly(action: () -> Unit) {
try {
action()
} catch (ignored: Exception) {
}
}
// ---- inbound -------------------------------------------------------------
override fun onNotification(address: String, characteristicUuid: String, data: ByteArray) {
if (characteristicUuid.equals(G1Smp.SMP_CHAR_UUID, ignoreCase = true)) {
lock.withLock {
for ((seq, body) in assembler.feed(data)) {
if (seq == smpSeq) {
smpReply = body
smpLatch?.countDown()
}
}
}
} else if (characteristicUuid.equals(G1Smp.NUS_TX_UUID, ignoreCase = true)) {
val percent = G1Smp.parseStockBattery(data) ?: return
lock.withLock {
stockBattery = percent
stockBatteryLatch?.countDown()
}
} else if (characteristicUuid.equals(G1Ble.EVT_CHAR_UUID, ignoreCase = true)) {
val notice = G1Protocol.parseStoreNotice(data)
if (notice != null) {
lock.withLock { storeNotices.add(notice) }
return
}
val reply = G1Protocol.parseReply(data) ?: return
lock.withLock {
if (reply.tag == controlTag) {
controlReply = reply
controlLatch?.countDown()
}
}
}
}
override fun onConnectionStateChange(address: String, connected: Boolean) {
if (connected) return
lock.withLock {
linkLost = true
smpLatch?.countDown()
controlLatch?.countDown()
}
}
}
@@ -0,0 +1,57 @@
package com.faceclaw.app
/**
* One lens's update on an Even G1: the rectangles that differ from what the arm's frame holds, as RLE bbox calls
* in draw messages. The transport's deflate runs over these messages, as on the G2; there is no resource cache,
* so everything visible arrives as pixels.
*/
object G1FramePlanner {
/** The firmware's limit on a decoded message is 16384 bytes (scouter_kit fc.c MAX_MESSAGE). */
const val MAX_MESSAGE = 16000
/** A big rectangle goes out in bands, so that one noisy area doesn't need a long binary split. */
private const val BAND_ROWS = 32
private const val MAX_RECTS = ConnectionOptions.MULTI_RECT_MAX_RECTS
/** [previous] null or of another size: a full frame. Empty when nothing changed. */
fun plan(previous: ByteArray?, next: ByteArray, width: Int, height: Int): List<ByteArray> {
val stride = (width + 1) / 2
require(next.size == stride * height)
val rects = changedRects(previous, next, width, height)
if (rects.isEmpty()) return emptyList()
val calls = ArrayList<ByteArray>()
for (r in rects) {
var top = r[1]
val bottom = r[1] + r[3]
while (top < bottom) {
val rows = minOf(BAND_ROWS, bottom - top)
addRect(calls, next, stride, r[0], top, r[2], rows)
top += rows
}
}
return DrawProtocol.messages(calls, MAX_MESSAGE)
}
private fun changedRects(previous: ByteArray?, next: ByteArray, width: Int, height: Int): List<IntArray> {
if (previous == null || previous.size != next.size) return listOf(intArrayOf(0, 0, width, height))
BleImageOptimizer.computeChangedRects(previous, next, width, height, MAX_RECTS)?.let { return it }
return listOfNotNull(BleImageOptimizer.computeChangedBox(previous, next, width, height))
}
/** Splits until the call fits a message; a single pixel always does. */
private fun addRect(calls: MutableList<ByteArray>, packed: ByteArray, stride: Int, x: Int, y: Int, w: Int, h: Int) {
val call = DrawProtocol.bbox(packed, stride, x, y, w, h)
if (call.size + MESSAGE_OVERHEAD <= MAX_MESSAGE) {
calls.add(call)
} else if (h > 1) {
addRect(calls, packed, stride, x, y, w, h / 2)
addRect(calls, packed, stride, x, y + h / 2, w, h - h / 2)
} else {
addRect(calls, packed, stride, x, y, w / 2, h)
addRect(calls, packed, stride, x + w / 2, y, w - w / 2, h)
}
}
/** The message type, the call count and the call's length (DrawProtocol.sequence). */
private const val MESSAGE_OVERHEAD = 5
}
@@ -0,0 +1,336 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/** The GATT service of g1v0, the firmware of an Even G1 (scouter_kit g1_firmware/docs/VM.md §6.1). */
object G1Ble {
const val CTRL_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4402"
const val EVT_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4403"
/** Readable and writable only on an encrypted link: a write with response makes the OS pair, then proves the link is encrypted. */
const val PAIR_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4404"
/** Faceclaw's records (VM.md §6.11). */
const val FC_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4405"
/** The iPhone notification relay (VM.md §6.12): the G2's `AN` commands and frames, with this characteristic to itself. */
const val ANCS_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4406"
const val REQUESTED_MTU = 247
/** The oldest firmware this app works with: `PROP_SET` came in here. */
const val REQUIRED_FIRMWARE = "0.5.63"
/** Dotted versions, component by component. */
fun firmwareAtLeast(version: String, required: String = REQUIRED_FIRMWARE): Boolean {
val have = version.split(".").map { it.toIntOrNull() ?: 0 }
val want = required.split(".").map { it.toIntOrNull() ?: 0 }
for (i in 0 until maxOf(have.size, want.size)) {
val delta = have.getOrElse(i) { 0 } - want.getOrElse(i) { 0 }
if (delta != 0) return delta > 0
}
return true
}
}
/**
* The wire formats of an Even G1 on g1v0: control requests and events (VM.md §6.3, §6.4), which Faceclaw uses as they
* are, and the display path's packets and answers (§6.11). No VM instruction is sent; the records inside the packets
* are [CfwTransport]'s, the same as on the G2.
*/
object G1Protocol {
const val OP_HELLO = 0x01
const val OP_HEALTH_OK = 0x07
const val OP_PROPS = 0x0A
const val OP_MIC = 0x13
const val OP_EVENTS = 0x14
const val OP_ASSET_INFO = 0x17
const val OP_ASSET_FORMAT = 0x1B
const val OP_FLASH_CRC = 0x1C
const val OP_FC_INFO = 0x1D
const val OP_PROP_SET = 0x1E
/** Control statuses (VM.md §1). */
const val STATUS_SAFE_MODE = 7
/** A store request: a job runs, no space, no store. */
const val STATUS_ASSET = 13
const val EV_NOTICE = 0x05
const val EV_MIC_DATA = 0x06
const val EV_INPUT = 0x07
const val EV_FC_ACK = 0x08
private const val EV_REPLY = 0x80
private const val EVENT_HEADER = 9
private const val OPT_START = 0x80
private const val OPT_END = 0x40
private const val MAX_PACKET = 244
private const val PACKET_HEADER = 2
/** The light sensor converts only while something uses it, as this bit of `EVENTS` does; the left arm has none. */
const val TRIGGER_LIGHT = 4
const val TRIGGER_BATTERY = 5
const val TRIGGER_CASE = 6
const val TRIGGER_WEAR = 7
const val TRIGGER_TOUCH = 8
const val EVENTS_MASK = (1 shl TRIGGER_LIGHT) or (1 shl TRIGGER_BATTERY) or (1 shl TRIGGER_CASE) or
(1 shl TRIGGER_WEAR) or (1 shl TRIGGER_TOUCH)
const val NOTICE_MIC_STOPPED = 5
const val MIC_STOPPED_LEASE = 3
const val NOTICE_GUARD = 7
/** A store job ended (VM.md §6.10). */
const val NOTICE_STORE_JOB = 8
const val STORE_JOB_FORMAT = 3
const val STORE_JOB_CRC = 4
/** `ASSET_INFO`'s job states. */
const val STORE_IDLE = 0
const val STORE_DONE = 4
const val STORE_FAILED = 5
/** The asset store's partition in the arm's QSPI flash, where Even's fonts and pictures were; its first two sectors hold its directory. */
const val STORE_ADDRESS = 0x140000
const val STORE_DIRECTORY_BYTES = 2 * 4096
const val PROP_BATT_PCT = 0x11
const val PROP_CHARGE = 0x12
const val PROP_IN_CASE = 0x13
const val PROP_WORN = 0x14
const val PROP_ACC_X = 0x18
const val PROP_ACC_Y = 0x19
const val PROP_ACC_Z = 0x1A
/** 0.01 lux, on the right arm, while the panel is on. */
const val PROP_LIGHT = 0x1C
const val PROP_IMU_RATE = 0x28
const val PROP_BRIGHTNESS_MAX = 0x29
const val IMU_RATE_IDLE = 13
const val IMU_RATE_STREAM = 26
const val BRIGHTNESS_LEVEL_MAX = 42
/** A property without a value yet (VM_UNKNOWN). */
const val UNKNOWN = Int.MIN_VALUE
const val CHARGE_CHARGING = 1
const val GESTURE_TAP = 1
const val GESTURE_DOUBLE_TAP = 2
const val GESTURE_TRIPLE_TAP = 3
const val GESTURE_LONG_START = 4
const val GESTURE_LONG_END = 5
/** The packets of one record: `[0x80 first | 0x40 last][stream][the next bytes]`. */
fun packets(record: ByteArray, stream: Int, mtu: Int): List<ByteArray> {
val chunk = minOf(mtu - 3, MAX_PACKET) - PACKET_HEADER
require(record.isNotEmpty() && chunk >= 5 && stream in 0..255)
val packets = ArrayList<ByteArray>(record.size / chunk + 1)
var offset = 0
while (offset < record.size) {
val count = minOf(chunk, record.size - offset)
val packet = ByteArray(PACKET_HEADER + count)
packet[0] = ((if (offset == 0) OPT_START else 0) or (if (offset + count == record.size) OPT_END else 0)).toByte()
packet[1] = stream.toByte()
record.copyInto(packet, PACKET_HEADER, offset, offset + count)
packets.add(packet)
offset += count
}
return packets
}
fun control(op: Int, tag: Int, body: ByteArray = ByteArray(0)): ByteArray =
byteArrayOf(op.toByte(), tag.toByte()) + body
fun eventsRequest(mask: Int): ByteArray = byteArrayOf(mask.toByte(), (mask ushr 8).toByte())
/** `PROP_SET` (0.5.63+): what SYS_WRITE does, for a host that runs no instructions. */
fun propSet(id: Int, value: Int): ByteArray =
byteArrayOf(id.toByte(), value.toByte(), (value ushr 8).toByte(), (value ushr 16).toByte(), (value ushr 24).toByte())
/** On, or a renewal of the lease; 32 kbps, the rate [Lc3PacketFramer] decodes. */
fun micRequest(on: Boolean): ByteArray = byteArrayOf(if (on) 1 else 0, 0)
/** An event's type, or -1 for bytes too short to be one. */
fun eventType(data: ByteArray): Int = if (data.size >= EVENT_HEADER) data[0].toInt() and 255 else -1
class Reply(@JvmField val op: Int, @JvmField val tag: Int, @JvmField val status: Int, @JvmField val body: ByteArray)
fun parseReply(data: ByteArray): Reply? {
val type = eventType(data)
if (type < EV_REPLY || data.size < EVENT_HEADER + 2) return null
return Reply(type and 0x7f, data[EVENT_HEADER].toInt() and 255, data[EVENT_HEADER + 1].toInt() and 255,
data.copyOfRange(EVENT_HEADER + 2, data.size))
}
/** Same shape as [CfwTransport.parseAcks]'s, so the shared window takes it. */
fun parseFcAck(data: ByteArray): CfwTransport.Ack? {
if (eventType(data) != EV_FC_ACK || data.size < EVENT_HEADER + 9) return null
val kind = data[EVENT_HEADER].toInt() and 255
if (kind != 1 && kind != 3) return null
return CfwTransport.Ack(kind == 3, data[EVENT_HEADER + 1].toInt() and 255, u16(data, EVENT_HEADER + 2),
data[EVENT_HEADER + 4].toInt() and 255, u16(data, EVENT_HEADER + 5), u16(data, EVENT_HEADER + 7))
}
class Input(@JvmField val trigger: Int, @JvmField val value: Int)
fun parseInput(data: ByteArray): Input? {
if (eventType(data) != EV_INPUT || data.size < EVENT_HEADER + 7) return null
return Input(data[EVENT_HEADER + 2].toInt() and 255, u32(data, EVENT_HEADER + 3))
}
class Notice(@JvmField val kind: Int, @JvmField val body: ByteArray)
fun parseNotice(data: ByteArray): Notice? {
if (eventType(data) != EV_NOTICE || data.size < EVENT_HEADER + 1) return null
return Notice(data[EVENT_HEADER].toInt() and 255, data.copyOfRange(EVENT_HEADER + 1, data.size))
}
/** `ASSET_FORMAT` frees the whole store; with [wipe] it also erases every sector that isn't erased already. */
fun assetFormatRequest(wipe: Boolean): ByteArray =
le32(0x534B4153) + if (wipe) le32(0x50570000) else ByteArray(0)
/** `FLASH_CRC`: a read-only fingerprint of flash bytes address…address + n - 1, given in `NOTICE` 8. */
fun flashCrcRequest(address: Int, n: Int): ByteArray = le32(address) + le32(n)
class AssetInfo(
@JvmField val assets: Int,
@JvmField val state: Int,
@JvmField val lastError: Int,
/** Without a usable flash there is no store, and nothing was ever written to it. */
@JvmField val available: Boolean,
/** The directory's commit count: 0 for a store that is empty and never was written, or was formatted since. */
@JvmField val seq: Long,
) {
val running: Boolean get() = state in 1..3
}
fun parseAssetInfo(body: ByteArray): AssetInfo? {
if (body.size < 20) return null
return AssetInfo(body[12].toInt() and 255, body[13].toInt() and 255, body[14].toInt() and 255, body[15].toInt() and 1 != 0,
u32(body, 16).toLong() and 0xffffffffL)
}
class StoreNotice(
@JvmField val job: Int,
@JvmField val result: Int,
/** A crc job's range, crc32 and how many of its bytes are not 0xFF; 0 for the other jobs. */
@JvmField val address: Int,
@JvmField val n: Int,
@JvmField val nonErased: Long,
)
fun parseStoreNotice(data: ByteArray): StoreNotice? {
val notice = parseNotice(data) ?: return null
val body = notice.body
if (notice.kind != NOTICE_STORE_JOB || body.size < 18) return null
return StoreNotice(body[0].toInt() and 255, body[1].toInt() and 255, u32(body, 2), u32(body, 6), u32(body, 14).toLong() and 0xffffffffL)
}
private fun le32(value: Int): ByteArray =
byteArrayOf(value.toByte(), (value ushr 8).toByte(), (value ushr 16).toByte(), (value ushr 24).toByte())
/**
* What the arm's light reading looks like to [BrightnessPolicy] and the Developer app: a live passive poll in the
* G2 custom firmware's report (settings field 105), whose value counts tenths of a lux.
*/
fun alsReport(centilux: Int, brightness: Int, tickMs: Long): ByteArray {
val report = ByteArray(24)
report[0] = 0x41
report[1] = 0x4c
report[2] = 1
report[3] = 2 // a poll
report[4] = 0x0b // opened, passive, read ok
report[5] = 3 // polling
val tenths = centilux / 10
for (i in 0..3) {
report[6 + i] = (tenths ushr (8 * i)).toByte()
report[10 + i] = (tenths ushr (8 * i)).toByte()
report[20 + i] = (tickMs ushr (8 * i)).toByte()
}
report[14] = brightness.toByte()
report[16] = brightness.toByte()
report[18] = 0
report[19] = 4 // scale 1.0 in Q10
return report
}
class Hello(
@JvmField val version: String,
@JvmField val leftArm: Boolean,
@JvmField val safeMode: Boolean,
@JvmField val panelOk: Boolean,
@JvmField val encrypted: Boolean,
@JvmField val inCase: Boolean,
/** `MIC` captures here: the right arm. */
@JvmField val mic: Boolean,
/** The iPhone notification relay answers here (0.5.65+). */
@JvmField val ancs: Boolean,
)
fun parseHello(body: ByteArray): Hello? {
if (body.size < 7 || body[0].toInt() != 5) return null
val flags = body[5].toInt() and 255
return Hello("${body[1].toInt() and 255}.${body[2].toInt() and 255}.${body[3].toInt() and 255}",
body[4].toInt() == 1, flags and 1 != 0, flags and 2 != 0, flags and 64 != 0, flags and 128 != 0, body[6].toInt() and 4 != 0,
body[6].toInt() and 8 != 0)
}
/** The display path's version, or -1 for a body that isn't FC_INFO's. */
fun parseFcVersion(body: ByteArray): Int = if (body.size >= 14) body[0].toInt() and 255 else -1
/** One value per requested property, or null if the reply is short. */
fun parseProps(body: ByteArray, count: Int): IntArray? =
if (body.size < 4 * count) null else IntArray(count) { u32(body, 4 * it) }
/**
* What a TouchBar gesture means to Faceclaw, which expects a G2's temple: a tap, a double tap and two slides. A
* G1's TouchBar cannot slide, but it has one per arm, so a tap moves (left up, right down), a double tap selects
* and a triple tap or long press goes back. Null: no event. With the display [dark] nothing can scroll: a tap is
* the G2's tap, which shows the Glanceboard, and the session reports a double tap as a wake.
*/
fun gestureEventType(gesture: Int, leftArm: Boolean, dark: Boolean = false): Int? = when (gesture) {
GESTURE_TAP -> if (dark) BleProtocol.EVENT_CLICK else if (leftArm) BleProtocol.EVENT_SCROLL_TOP else BleProtocol.EVENT_SCROLL_BOTTOM
GESTURE_DOUBLE_TAP -> if (dark) BleProtocol.EVENT_DOUBLE_CLICK else BleProtocol.EVENT_CLICK
GESTURE_TRIPLE_TAP, GESTURE_LONG_START -> BleProtocol.EVENT_DOUBLE_CLICK
else -> null
}
/** `MIC_DATA`'s frames: the counter of the first one, how many, and where they start in [data]. */
fun micFrames(data: ByteArray): Triple<Int, Int, Int>? {
if (eventType(data) != EV_MIC_DATA || data.size < EVENT_HEADER + 3) return null
val count = data[EVENT_HEADER + 2].toInt() and 255
val start = EVENT_HEADER + 3
return if (data.size - start < count * Lc3PacketFramer.FRAME_BYTES) null else Triple(u16(data, EVENT_HEADER), count, start)
}
private fun u16(b: ByteArray, p: Int): Int = (b[p].toInt() and 255) or ((b[p + 1].toInt() and 255) shl 8)
private fun u32(b: ByteArray, p: Int): Int = u16(b, p) or (u16(b, p + 2) shl 16)
}
/**
* A G1's microphone frames as the packets [Lc3PacketFramer] takes from a G2: five 10 ms LC3 frames, then the two values
* of the G2's stereo capture (a G1 has one mic: zero) and a packet counter. A gap loses the part-filled packet, and the
* counter tells the decoder to conceal it.
*/
class G1MicAssembler {
private val frames = ByteArray(Lc3PacketFramer.LC3_BYTES)
private var have = 0
private var firstCounter = 0
private var nextCounter = -1
fun reset() {
have = 0
nextCounter = -1
}
fun ingest(first: Int, count: Int, data: ByteArray, start: Int, emit: (ByteArray) -> Unit) {
val size = Lc3PacketFramer.FRAME_BYTES
for (i in 0 until count) {
val counter = (first + i) and 0xffff
if (counter != nextCounter) have = 0
if (have == 0) firstCounter = counter
data.copyInto(frames, have * size, start + i * size, start + (i + 1) * size)
have++
nextCounter = (counter + 1) and 0xffff
if (have == Lc3PacketFramer.FRAMES_PER_PACKET) {
val packet = ByteArray(Lc3PacketFramer.PACKET_BYTES)
frames.copyInto(packet)
packet[Lc3PacketFramer.COUNTER_OFFSET] = (firstCounter / Lc3PacketFramer.FRAMES_PER_PACKET).toByte()
emit(packet)
have = 0
}
}
}
}
@@ -0,0 +1,311 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/** The part of CBOR that MCUmgr uses: integers, byte and text strings, arrays, maps, booleans and null. */
object G1Cbor {
fun encode(value: Any?): ByteArray {
val out = Output()
write(out, value)
return out.toByteArray()
}
fun decode(data: ByteArray, start: Int = 0): Any? = Reader(data, start).value()
private class Output {
private var bytes = ByteArray(256)
private var size = 0
private fun reserve(extra: Int) {
if (size + extra > bytes.size) bytes = bytes.copyOf(maxOf(bytes.size * 2, size + extra))
}
fun add(byte: Int) {
reserve(1)
bytes[size++] = byte.toByte()
}
fun add(data: ByteArray) {
reserve(data.size)
data.copyInto(bytes, size)
size += data.size
}
fun toByteArray(): ByteArray = bytes.copyOf(size)
}
private fun head(out: Output, major: Int, n: Long) {
val first = major shl 5
val width = when {
n < 24 -> {
out.add(first or n.toInt())
return
}
n < 0x100L -> 1
n < 0x10000L -> 2
n < 0x100000000L -> 4
else -> 8
}
out.add(first or (24 + width.countTrailingZeroBits()))
for (i in width - 1 downTo 0) out.add(((n ushr (8 * i)) and 0xff).toInt())
}
private fun write(out: Output, value: Any?) {
when (value) {
null -> out.add(0xf6)
is Boolean -> out.add(if (value) 0xf5 else 0xf4)
is Int -> write(out, value.toLong())
is Long -> if (value >= 0) head(out, 0, value) else head(out, 1, -1 - value)
is ByteArray -> {
head(out, 2, value.size.toLong())
out.add(value)
}
is String -> {
val text = value.encodeToByteArray()
head(out, 3, text.size.toLong())
out.add(text)
}
is List<*> -> {
head(out, 4, value.size.toLong())
for (item in value) write(out, item)
}
is Map<*, *> -> {
head(out, 5, value.size.toLong())
for ((key, item) in value) {
write(out, key)
write(out, item)
}
}
else -> throw IllegalArgumentException("not CBOR: $value")
}
}
private class Reader(private val data: ByteArray, private var at: Int) {
private fun u8(): Int {
if (at >= data.size) throw IllegalArgumentException("CBOR cut short")
return data[at++].toInt() and 0xff
}
fun value(): Any? {
val first = u8()
val major = first ushr 5
val info = first and 31
val n: Long? = when {
info < 24 -> info.toLong()
info in 24..27 -> {
var v = 0L
repeat(1 shl (info - 24)) { v = (v shl 8) or u8().toLong() }
v
}
info == 31 -> null
else -> throw IllegalArgumentException("bad CBOR header $first")
}
return when (major) {
0 -> n ?: throw IllegalArgumentException("bad CBOR integer")
1 -> -1 - (n ?: throw IllegalArgumentException("bad CBOR integer"))
2, 3 -> {
val size = (n ?: throw IllegalArgumentException("indefinite CBOR string")).toInt()
if (size < 0 || at + size > data.size) throw IllegalArgumentException("CBOR cut short")
val bytes = data.copyOfRange(at, at + size)
at += size
if (major == 2) bytes else bytes.decodeToString()
}
4 -> {
val items = ArrayList<Any?>()
while (more(n, items.size)) items.add(value())
if (n == null) at++
items
}
5 -> {
val items = LinkedHashMap<Any?, Any?>()
while (more(n, items.size)) {
val key = value()
items[key] = value()
}
if (n == null) at++
items
}
7 -> when (info) {
20 -> false
21 -> true
22 -> null
else -> throw IllegalArgumentException("unsupported CBOR simple value $info")
}
else -> throw IllegalArgumentException("unsupported CBOR major type $major")
}
}
/** An indefinite-length array or map (what Zephyr's image list uses) ends at a 0xFF. */
private fun more(n: Long?, count: Int): Boolean {
if (n != null) return count < n
if (at >= data.size) throw IllegalArgumentException("CBOR cut short")
return (data[at].toInt() and 0xff) != 0xff
}
}
}
/**
* MCUmgr SMP, the update service of both Even's firmware and g1v0: as much of it as installing an image takes, and what
* an MCUboot image says about itself. The same requests as scouter_kit's tools/g1host.py (class Smp), which was proven on
* real arms first.
*/
object G1Smp {
const val SMP_CHAR_UUID = "da2e7828-fbce-4e01-ae9e-261174997c48"
const val NUS_RX_UUID = "6e400002-b5a3-f393-e0a9-e50e24dcca9e"
const val NUS_TX_UUID = "6e400003-b5a3-f393-e0a9-e50e24dcca9e"
const val OP_READ = 0
const val OP_WRITE = 2
const val GROUP_OS = 0
const val GROUP_IMAGE = 1
const val CMD_IMAGE_STATE = 0
const val CMD_IMAGE_UPLOAD = 1
const val CMD_OS_RESET = 5
const val UPLOAD_CHUNK = 2048
private const val HEADER = 8
/** Even's firmware drops a link it hears no heartbeat on. */
fun stockHeartbeat(seq: Int): ByteArray =
byteArrayOf(0x25, 0x06, 0x00, seq.toByte(), 0x04, seq.toByte())
/** Even's firmware's battery query; its reply is `2C 66`, then the percentage. The bytes its real firmware answered in emulation. */
val STOCK_BATTERY_QUERY = byteArrayOf(0x2c, 0x04, 0x00, 0x00)
/** The percentage in a reply to [STOCK_BATTERY_QUERY]; null for anything else, or a value that can't be one. */
fun parseStockBattery(data: ByteArray): Int? {
if (data.size < 3 || data[0].toInt() != 0x2c || (data[1].toInt() and 0xff) != 0x66) return null
return (data[2].toInt() and 0xff).takeIf { it in 1..100 }
}
fun frame(op: Int, group: Int, seq: Int, cmd: Int, payload: Map<String, Any?>): ByteArray {
val body = G1Cbor.encode(payload)
val out = ByteArray(HEADER + body.size)
out[0] = op.toByte()
out[2] = (body.size ushr 8).toByte()
out[3] = body.size.toByte()
out[4] = (group ushr 8).toByte()
out[5] = group.toByte()
out[6] = seq.toByte()
out[7] = cmd.toByte()
body.copyInto(out, HEADER)
return out
}
/** One ATT write at most [mtu] - 3 bytes long; MCUmgr joins them again. */
fun split(frame: ByteArray, mtu: Int): List<ByteArray> {
val size = maxOf(mtu - 3, 20)
return (frame.indices step size).map { frame.copyOfRange(it, minOf(it + size, frame.size)) }
}
/** MCUboot's image numbers on an nRF5340: the application core, and the network core (the Bluetooth controller). */
const val IMAGE_APP = 0
const val IMAGE_NET_CORE = 1
/**
* The first chunk also says what is coming, which lets the arm resume an upload of the same file, and which image it
* is: each has its own second slot.
*/
fun uploadRequest(image: ByteArray, offset: Int, fileSha256: ByteArray, imageIndex: Int = IMAGE_APP): Map<String, Any?> {
val data = image.copyOfRange(offset, minOf(offset + UPLOAD_CHUNK, image.size))
return if (offset == 0) {
mapOf("image" to imageIndex, "len" to image.size, "off" to 0, "sha" to fileSha256, "data" to data, "upgrade" to false)
} else {
mapOf("off" to offset, "data" to data)
}
}
/**
* Marks the image with this hash for MCUboot's next boot. [permanent] false is a test install, which is what Even's app
* and g1v0's own network-core fallback ask for on image 1; the network core has no copy to go back to either way.
*/
fun confirmRequest(imageHash: ByteArray, permanent: Boolean = true): Map<String, Any?> = mapOf("hash" to imageHash, "confirm" to permanent)
/** Joins notifications into replies; [feed] gives each complete one as sequence number and body. */
class Assembler {
private var pending = ByteArray(0)
fun reset() {
pending = ByteArray(0)
}
fun feed(data: ByteArray): List<Pair<Int, Map<*, *>>> {
pending += data
val replies = ArrayList<Pair<Int, Map<*, *>>>()
while (pending.size >= HEADER) {
val size = ((pending[2].toInt() and 0xff) shl 8) or (pending[3].toInt() and 0xff)
if (pending.size < HEADER + size) break
val seq = pending[6].toInt() and 0xff
val body = try {
if (size == 0) emptyMap<Any?, Any?>() else G1Cbor.decode(pending, HEADER) as? Map<*, *>
} catch (e: IllegalArgumentException) {
null
}
if (body == null) {
// A frame cut by a lost link: the request is sent again.
pending = ByteArray(0)
break
}
pending = pending.copyOfRange(HEADER + size, pending.size)
replies.add(seq to body)
}
return replies
}
}
class Slot(
@JvmField val image: Int,
@JvmField val slot: Int,
@JvmField val version: String,
@JvmField val hash: ByteArray,
@JvmField val flags: List<String>,
) {
override fun toString(): String =
"image $image slot $slot: $version ${BinaryEncoding.bytesToHex(hash.copyOf(minOf(8, hash.size)))} ${flags.joinToString(" ")}".trim()
}
fun parseSlots(reply: Map<*, *>): List<Slot> {
val images = reply["images"] as? List<*> ?: return emptyList()
return images.mapNotNull { item ->
val entry = item as? Map<*, *> ?: return@mapNotNull null
val hash = entry["hash"] as? ByteArray ?: return@mapNotNull null
Slot((entry["image"] as? Long ?: 0L).toInt(), (entry["slot"] as? Long ?: -1L).toInt(), entry["version"] as? String ?: "?", hash,
listOf("bootable", "pending", "confirmed", "active", "permanent").filter { entry[it] == true })
}
}
fun replyCode(reply: Map<*, *>): Int = (reply["rc"] as? Long ?: 0L).toInt()
fun replyOffset(reply: Map<*, *>): Int? = (reply["off"] as? Long)?.toInt()
/** The dotted version in an MCUboot header. */
fun imageVersion(image: ByteArray): String {
requireImage(image)
val revision = (image[22].toInt() and 0xff) or ((image[23].toInt() and 0xff) shl 8)
return "${image[20].toInt() and 0xff}.${image[21].toInt() and 0xff}.$revision"
}
/** The SHA-256 TLV: how MCUmgr names an image. */
fun imageHash(image: ByteArray): ByteArray {
requireImage(image)
var at = u16(image, 8) + u32(image, 12)
require(at + 4 <= image.size && u16(image, at) == 0x6907) { "the image has no TLV area" }
val end = at + u16(image, at + 2)
require(end <= image.size) { "the image's TLV area is cut short" }
at += 4
while (at + 4 <= end) {
val type = u16(image, at)
val size = u16(image, at + 2)
if (type == 0x10 && size == 32 && at + 4 + size <= end) return image.copyOfRange(at + 4, at + 4 + size)
at += 4 + size
}
throw IllegalArgumentException("the image has no SHA-256 TLV")
}
private fun requireImage(image: ByteArray) {
require(image.size > 32 && u32(image, 0) == 0x96f3b83d.toInt()) { "not an MCUboot image" }
}
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
private fun u32(data: ByteArray, at: Int): Int = u16(data, at) or (u16(data, at + 2) shl 16)
}
@@ -119,6 +119,19 @@ class BleProtocol {
*/
const val FACECLAW_ALS_REPORT_FIELD: Int = 105
/**
* CFW keyboard bridge (Faceclaw/39): the master temple hosts one BLE HID keyboard. The
* host writes ['K','B',ver,op,request,epoch,...] controls as settings field 130; results,
* pairing prompts and the keyboard's raw input reports come back as commandId=3 pushes
* carrying field 131 (['K','B',ver,kind,...], decoded in app/g2/keyboard/protocol.ts).
*/
const val FACECLAW_KEYBOARD_CONTROL_FIELD: Int = 130
const val FACECLAW_KEYBOARD_EVENT_FIELD: Int = 131
/** Fixed header of a field-131 record; fragments follow it. */
const val FACECLAW_KEYBOARD_EVENT_HEADER: Int = 24
const val FACECLAW_WAKE_OP_ACQUIRE: Int = 1
const val FACECLAW_WAKE_OP_RELEASE: Int = 2
@@ -788,6 +801,42 @@ class BleProtocol {
return body
}
/**
* Wrap an already-encoded ['K','B',ver,op,...] keyboard control as a settings write
* carrying CFW field 130. Magic zero, like the other private controls: the answer is a
* field-131 record naming the control's request id, not the settings ACK.
*/
@JvmStatic
fun buildFaceclawKeyboardControl(record: ByteArray): ByteArray {
return concat(
CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, 0),
encodeBytesField(FACECLAW_KEYBOARD_CONTROL_FIELD, record),
)
)
}
/**
* Extract a CFW keyboard record (field 131) from a settings frame, or null when the frame
* carries none. Only the ['K','B'] prelude and the fixed header's length are checked
* here; the TS side validates the version and decodes the rest.
*/
@JvmStatic
fun parseFaceclawKeyboardEvent(pb: ByteArray?): ByteArray? {
if (pb == null) {
return null
}
val body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_KEYBOARD_EVENT_FIELD)
if (
body == null || body.size < FACECLAW_KEYBOARD_EVENT_HEADER ||
body[0].toInt() != 75 || body[1].toInt() != 66
) {
return null
}
return body
}
/**
* Return the raw CFW ambient-light report from a sid-0x09 frame, or null when this is an
* ordinary settings frame. Only the 65.toByte(),76.toByte(),version prelude is validated
@@ -33,7 +33,7 @@ class CfwMessageWindow private constructor() {
fun acknowledgedHead(messages: Iterable<OutboundMessage>): OutboundMessage? {
for (message in messages) {
if (message.sid != CfwTransport.SID) continue
return if (!message.cfwRetryPending && message.cfwAckLenses == CfwTransport.BOTH)
return if (!message.cfwRetryPending && message.isCfwAcked)
message
else null
}
@@ -49,7 +49,7 @@ class CfwMessageWindow private constructor() {
failed =
failed ||
message.cfwRetryPending ||
(message.cfwAckLenses != CfwTransport.BOTH &&
(!message.isCfwAcked &&
message.ackDeadlineAtMs > 0 &&
message.ackDeadlineAtMs <= nowMs)
replay.add(message)
@@ -28,7 +28,18 @@ class CfwTransport(private val platform: ProtocolPlatform) {
fun encode(message: ByteArray?, streamId: Int, lenses: Int, mtu: Int): List<ByteArray> =
lock.withLock {
validate(message, streamId, lenses, mtu)
val input = message!!
g2Packets(encodeRecord(message!!, lenses), streamId, mtu)
}
/**
* One record, `[flags][wire length u16][decoded CRC u16][body]`, compressed with this
* transport's persistent context. The packet envelope is the caller's: [encode] wraps it
* in G2 TPL packets, [RayNeoIoProtocol.cfwPackets] in RayNeo iO business-0xF0 packets.
*/
fun encodeRecord(message: ByteArray, lenses: Int): ByteArray =
lock.withLock {
require(message.size <= MAX_MESSAGE && lenses in 1..BOTH) { "invalid CFW record parameters" }
val input = message
if (previousLenses != lenses) reset()
val reset = resetPending
val encoder = deflater ?: platform.createDeflater().also { deflater = it }
@@ -44,7 +55,7 @@ class CfwTransport(private val platform: ProtocolPlatform) {
resetPending = false
previousLenses = lenses
}
frameRecord(body, crc(input), streamId, flags, mtu)
record(body, crc(input), flags)
}
companion object {
@@ -71,18 +82,12 @@ class CfwTransport(private val platform: ProtocolPlatform) {
@JvmStatic
fun frame(message: ByteArray?, streamId: Int, lenses: Int, mtu: Int): List<ByteArray> {
validate(message, streamId, lenses, mtu)
return frameRecord(message!!, crc(message), streamId, lenses or RESET_CONTEXT, mtu)
return g2Packets(record(message!!, crc(message), lenses or RESET_CONTEXT), streamId, mtu)
}
private fun frameRecord(
body: ByteArray,
checksum: Int,
streamId: Int,
flags: Int,
mtu: Int,
): List<ByteArray> {
val lenses = flags and BOTH
val capacity = minOf(252, mtu - 14)
/** `[flags][wire length u16][decoded CRC u16][body]`, shared by every packet envelope. */
@JvmStatic
fun record(body: ByteArray, checksum: Int, flags: Int): ByteArray {
val stream = ByteArray(body.size + 5)
stream[0] = flags.toByte()
stream[1] = body.size.toByte()
@@ -90,6 +95,13 @@ class CfwTransport(private val platform: ProtocolPlatform) {
stream[3] = checksum.toByte()
stream[4] = (checksum ushr 8).toByte()
body.copyInto(stream, 5)
return stream
}
/** G2 TPL packets (SID 0xf0) carrying one record: options byte = lenses | RESET | END. */
private fun g2Packets(stream: ByteArray, streamId: Int, mtu: Int): List<ByteArray> {
val lenses = stream[0].toInt() and BOTH
val capacity = minOf(252, mtu - 14)
val packets = mutableListOf<ByteArray>()
for (offset in stream.indices step capacity) {
val count = minOf(capacity, stream.size - offset)
@@ -337,6 +337,24 @@ class MessageBuilder {
)
}
/**
* CFW keyboard control (settings field 130). Always the right arm: only the master temple
* acts on it, and a copy sent down each arm would run the control twice.
*/
fun faceclawKeyboardControl(record: ByteArray, label: String): OutboundMessage {
return OutboundMessage(
"keyboard-control",
"keyboard control $label",
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
0,
BleProtocol.buildFaceclawKeyboardControl(record),
ACK_TIMEOUT_MS,
-1,
false,
)
}
fun faceclawWearQuery(leftArm: Boolean): OutboundMessage {
return OutboundMessage(
"wear-query-control",
@@ -18,6 +18,8 @@ class OutboundMessage(
@JvmField
val cfwChecksum: Int = if (sid == CfwTransport.SID) CfwTransport.crc(this.message) else 0
@JvmField var cfwAckLenses = 0
/** The lenses that must ACK this attempt: both on a G2, whichever the message went to on a G1. */
@JvmField var cfwRequiredLenses = CfwTransport.BOTH
@JvmField var cfwRetries = 0
@JvmField var cfwRetryPending = false
@JvmField var imageUpdateId = 0
@@ -47,9 +49,11 @@ class OutboundMessage(
}
if (message.size != ack.size || cfwChecksum != ack.checksum) return false
cfwAckLenses = cfwAckLenses or ack.lens
return !cfwRetryPending && cfwAckLenses == CfwTransport.BOTH
return !cfwRetryPending && isCfwAcked
}
val isCfwAcked: Boolean get() = cfwAckLenses and cfwRequiredLenses == cfwRequiredLenses
/** Replace an attempt atomically under the communicator lock. */
fun prepareCfwReplay(streamId: Int) {
magic = streamId
@@ -48,6 +48,21 @@ interface StockLink {
fun bondState(address: String): BondState
/** The ATT MTU agreed on this link. */
fun negotiatedMtu(address: String): Int = 23
/** Ask the OS to pair with a connected device; a platform that pairs by itself on demand does nothing. */
fun requestBond(address: String) {}
/** Remove this phone's half of a pairing; false where the platform can't or refused. */
fun removeBond(address: String): Boolean = false
/** After discovery: whether the service table has this characteristic. Enabling notifications on one it lacks may throw. */
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean = true
/** Read the service table again, past whatever the OS cached for this address. */
fun refreshServices(address: String, timeoutMs: Int): Boolean = discoverServices(address, timeoutMs)
fun disconnect(address: String)
fun close()
@@ -4,6 +4,11 @@ package com.faceclaw.app
* queued state and sensor freshness do not. Private commands are ordered with
* frames, so wake starts on the intended PRESENT, never on an older black frame. */
internal fun GlassesSessionCore.enqueueBrightnessLocked(frameVisible: Boolean? = null) {
if (isIo) {
ioNoteFrameVisibleLocked(frameVisible)
// Older iO firmware has no message 30; ioMaintain drives the launcher's (and the screen) instead.
if (!ioFirmwareOwnsBrightness) return
}
if (!customFirmwareDetected || !sessionReady || !fixedLayoutCreated || shutdownRequested) return
val visible = frameVisible ?: brightnessSentVisible ?: return
val target = brightnessPolicy.target
@@ -14,29 +19,38 @@ internal fun GlassesSessionCore.enqueueBrightnessLocked(frameVisible: Boolean? =
else if (brightnessPolicy.automatic) 1200 else 120
val payload = byteArrayOf(CFW_MSG_BRIGHTNESS.toByte(), 1, target.toByte(),
(if (visible) 1 else 0).toByte(), duration.toByte(), (duration shr 8).toByte())
val message = messageBuilder.customMessage("brightness-output", payload,
"brightness target=$target visible=$visible fade=$duration", -1, connectionOptions.sendImagesToLeft)
message.onTimeout = MessageCallback {
val label = "brightness target=$target visible=$visible fade=$duration"
val retry = MessageCallback {
brightnessSentLevel = -1
brightnessSentVisible = null
// Retry with a PRESENT even if the scene did not change during recovery.
lastEnqueuedFingerprint = ""
}
pendingMessages.addLast(message)
if (isG1) {
g1EnqueueBrightnessLocked(payload, label, brightnessSentVisible != visible, retry)
} else {
val message = messageBuilder.customMessage("brightness-output", payload, label, -1, connectionOptions.sendImagesToLeft)
message.onTimeout = retry
pendingMessages.addLast(message)
}
brightnessSentLevel = target
brightnessSentVisible = visible
if (isIo) io.lastBrightnessAtMs = now()
}
internal fun GlassesSessionCore.maintainBrightnessLocked(now: Long) {
if (!customFirmwareDetected || !sessionReady || !fixedLayoutCreated || shutdownRequested) return
// Passive polling owns stock ALS even in manual mode. Slower manual polling
// keeps stock from writing the panel without needlessly streaming samples.
if (now - brightnessAlsStartedAt >= 2000 && !hasPendingOrInflightKindLocked("als-control")) {
val interval = if (brightnessDemoPolling) 250 else if (brightnessPolicy.automatic) 500 else 5000
val interval = if (brightnessDemoPolling) 250 else if (brightnessPolicy.automatic) 500 else 5000
// The G2's stock can close the sensor, so it hears this again; the iO's polling lasts as long as the page.
val alsDue = if (isIo) interval != io.alsIntervalSent else now - brightnessAlsStartedAt >= 2000
if (alsDue && !hasPendingOrInflightKindLocked("als-control")) {
val payload = byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 1, 1,
interval.toByte(), (interval shr 8).toByte(), 0, 0, 0xe8.toByte(), 3)
enqueueAmbientLightControlLocked(payload, "brightness passive ALS", false)
brightnessAlsStartedAt = now
if (isIo) io.alsIntervalSent = interval
}
enqueueBrightnessLocked()
}
@@ -86,13 +86,16 @@ internal fun GlassesSessionCore.enqueueBenchmarkMessageLocked() {
kotlin.random.Random.nextBytes(payload)
payload[0] = CFW_MSG_DIAGNOSTICS.toByte() // CFW diagnostic-control mode...
payload[1] = 0x7f.toByte() // ...with an unused sub-op: acked, no effect
// A G1's arms are two links and every frame goes down both, so the run alternates between them.
val leftArm = if (isG1) benchmarkMessageCounter++ % 2 == 0 else connectionOptions.sendImagesToLeft
val message = messageBuilder.imagePayload(
"bandwidth",
DASHBOARD_TILE,
nextMapSessionId(),
payload,
"bandwidth no-op " + payload.size + "B",
connectionOptions.sendImagesToLeft)
leftArm)
if (isG1) message.cfwRequiredLenses = if (leftArm) G1Timings.LENS_LEFT else G1Timings.LENS_RIGHT
val payloadBytes = payload.size
// Logical custom-message bytes; excludes length tag and packet framing.
val wireBytes = message.message.size
@@ -345,6 +348,8 @@ internal fun GlassesSessionCore.releaseFaceclawFramebufferLease(): Boolean {
/** The address of a configured arm the OS no longer holds a bond for, or null. */
internal fun GlassesSessionCore.firstUnpairedArm(): String? {
// The iO identifies the phone by its protocol id, not an OS bond.
if (isIo) return null
if (!link.isBonded(rightAddress)) return rightAddress
if (!link.isBonded(leftAddress)) return leftAddress
return null
@@ -491,7 +496,7 @@ internal fun GlassesSessionCore.scheduleReconnectLocked(sessionWasReady: Boolean
if (sessionWasReady && now - lastSessionReadyAtMs >= ConnectionOptions.STABLE_SESSION_MS) {
consecutiveReconnects = 0
}
val schedule = ConnectionOptions.RECONNECT_BACKOFF_MS
val schedule = if (isG1) G1Timings.RECONNECT_BACKOFF_MS else ConnectionOptions.RECONNECT_BACKOFF_MS
val delayMs = schedule[minOf(consecutiveReconnects, schedule.size - 1)].toLong()
consecutiveReconnects++
reconnectAfterMs = now + delayMs
@@ -581,6 +586,22 @@ internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String)
}
}
internal fun GlassesSessionCore.emitKeyboardEvent(body: ByteArray) {
if (keyboardEventListeners.isEmpty()) {
return
}
val copy = body.copyOf()
host.postToMain {
for (keyboardListener in keyboardEventListeners) {
try {
keyboardListener.onKeyboardEvent(copy)
} catch (t: Throwable) {
logWarn("keyboard event listener failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitAudioMonitorPacket(data: ByteArray, arm: String, arrivalMs: Long) {
if (audioMonitorListeners.isEmpty()) {
return
@@ -107,6 +107,13 @@ class GlassesSessionCore(
@Volatile internal var listener: FaceclawBleCommunicatorListener? = null
/** G2 unless [useRayNeoIo] was called before [start]. */
@Volatile internal var glassesType = GlassesType.G2
/** RayNeo iO link state (GlassesSessionIo.kt); guarded by [monitor]. */
internal val io = IoSessionState()
/** Even G1 link state (GlassesSessionG1.kt); guarded by [monitor]. */
internal val g1 = G1SessionState()
/**
* Optional tap for relay frames the session does not interpret: notifications on the
* right arm's data characteristic whose first two bytes are 'A','N' (the iOS ANCS
@@ -124,6 +131,7 @@ class GlassesSessionCore(
internal val compassSubscriptions = CopyOnWriteList<CompassSubscription>(platform)
internal val ambientLightListeners = CopyOnWriteList<FaceclawAmbientLightListener>(platform)
internal val micStatusListeners = CopyOnWriteList<FaceclawMicStatusListener>(platform)
internal val keyboardEventListeners = CopyOnWriteList<FaceclawKeyboardEventListener>(platform)
internal val audioMonitorListeners = CopyOnWriteList<FaceclawAudioPacketListener>(platform)
@Volatile internal var running = false
@Volatile internal var userDisconnectRequested = false
@@ -451,6 +459,7 @@ class GlassesSessionCore(
internal var benchmarkLastAckAtMs = 0L
internal var benchmarkEndAtMs = 0L // 0 while running; set when the run drains
internal var benchmarkMessagesSent = 0
internal var benchmarkMessageCounter = 0
internal var benchmarkMessagesAcked = 0
internal var benchmarkTimeouts = 0
internal var benchmarkPayloadBytesAcked = 0L
@@ -538,7 +547,8 @@ class GlassesSessionCore(
/**
* Write one already-framed packet to an arm outside the message scheduler (the ANCS
* relay client builds its own packets). Blocks like any link write; false when the
* arm is unknown, the link rejects the write, or the session is not running.
* arm is unknown, the link rejects the write, or the session is not running. A G1 has a
* characteristic for them, whose ATT errors say why a command was refused.
*/
fun writeRawPacket(arm: String, packet: ByteArray): Boolean {
val address = when (arm.uppercase()) {
@@ -549,8 +559,11 @@ class GlassesSessionCore(
val active = monitor.withLock { running }
if (!active) return false
return try {
val result = link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE,
ConnectionOptions.WRITE_TIMEOUT_MS)
val result = if (isG1) {
link.writeFrames(address, G1Ble.ANCS_CHAR_UUID, listOf(packet), GattWriteMode.WITH_RESPONSE, ConnectionOptions.WRITE_TIMEOUT_MS)
} else {
link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE, ConnectionOptions.WRITE_TIMEOUT_MS)
}
if (result) monitor.withLock { noteWriteCompletedLocked(address, now()) }
result
} catch (t: Throwable) {
@@ -574,6 +587,39 @@ class GlassesSessionCore(
requiredFirmwareRevision = revision
}
/**
* Drive a RayNeo iO instead of a G2: one BLE link (pass its address as both arms; no
* ring), the iO handshake and page lease, and the Faceclaw display lists of the
* faceclaw-rayneo-cfw firmware. [phoneId] is this phone's stable 6-byte protocol identity:
* the glasses bind to the first one that authenticates and refuse others until they are
* put in pairing mode. Call before [start].
*/
fun useRayNeoIo(phoneId: ByteArray, phoneName: String, manufacturer: String, model: String) {
require(phoneId.size == 6) { "RayNeo iO phone id must be 6 bytes" }
require(rightAddress.equals(leftAddress, ignoreCase = true)) { "RayNeo iO uses one address for both arms" }
monitor.withLock {
glassesType = GlassesType.RAYNEO_IO
io.phoneId = phoneId.copyOf()
io.phoneName = phoneName.ifEmpty { "Faceclaw" }
io.manufacturer = manufacturer.ifEmpty { "Android" }
io.model = model.ifEmpty { "Android" }
}
}
/** Drive an Even G1 on g1v0 instead of a G2 (see [G1SessionState]). Before [start]. */
fun useG1() {
monitor.withLock { glassesType = GlassesType.G1 }
}
/** True when this session drives an Even G1. */
fun isEvenG1(): Boolean = isG1
/** Whether the glasses can relay iPhone notifications, as far as the session knows: a G2's depends on its firmware revision, which TS checks. */
fun ancsRelayAvailable(): Boolean = !isG1 || monitor.withLock { g1.ancsRelay }
/** True when this session drives a RayNeo iO. */
fun isRayNeoIo(): Boolean = isIo
/** Start the worker (through the host); false when it was already running. */
fun start(): Boolean {
monitor.withLock {
@@ -606,7 +652,7 @@ class GlassesSessionCore(
* quiescing its producers. Also cover direct/early close callers here;
* a successful cleanup must remain the final BLE message. Older CFWs
* fall back to the standalone framebuffer-lease release. */
if (!cfwCleanupDelivered && !sendCfwCleanup()) {
if (!cfwCleanupDelivered && !sendCfwCleanup() && isG2) {
releaseFaceclawFramebufferLease()
}
monitor.withLock {
@@ -625,6 +671,14 @@ class GlassesSessionCore(
host.joinWorker(5_000)
ringDone?.await(5_000)
ringWorkerDone = null
if (isIo && link.isConnected(rightAddress)) {
// Otherwise the blank page stays up until the firmware's 120 s lease runs out.
try {
ioWriteControl(RayNeoIoProtocol.CTRL_PAGE_CLOSE)
} catch (t: Throwable) {
logLine("RayNeo iO page close failed: " + safeMessage(t))
}
}
monitor.withLock {
resetSessionStateLocked()
clearAllMessagesLocked("disconnect")
@@ -666,6 +720,9 @@ class GlassesSessionCore(
if (listener == null) {
throw IllegalArgumentException("listener is required")
}
// The iO's packets are Opus from its remote audio stream (see ioStartAudioCapture).
if (isIo) return ioStartAudioCapture(listener)
if (isG1) return g1StartAudioCapture(listener)
var magic = 0
monitor.withLock {
if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) {
@@ -683,6 +740,8 @@ class GlassesSessionCore(
}
fun stopG2AudioCapture() {
if (isIo) return ioStopAudioCapture()
if (isG1) return g1StopAudioCapture()
var magic = 0
monitor.withLock {
audioPacketListener = null
@@ -790,6 +849,15 @@ class GlassesSessionCore(
* that track a capture across those events must check this rather than
* assume their earlier enable still holds.
*/
/**
* Whether the glasses' own wake word should reach Faceclaw (its configured wake action).
* RayNeo iO: the launcher's "Hey RayNeo" switch, and the stock session it opens is
* ended in favor of Faceclaw's. G2 glasses use the wake lease instead (no-op here).
*/
fun setVoiceWakeEnabled(enabled: Boolean) {
if (isIo) ioSetVoiceWakeEnabled(enabled)
}
fun isAudioCaptureActive(): Boolean {
monitor.withLock {
return running && sessionReady && !shutdownRequested && audioCaptureActive
@@ -806,6 +874,8 @@ class GlassesSessionCore(
val generation: Int
monitor.withLock {
faceclawWakeLeaseEnabled = enabled
// A G2 firmware lease; the iO's wake word is setVoiceWakeEnabled.
if (!isG2) return true
if (!running || !sessionReady) {
return !enabled
}
@@ -874,6 +944,7 @@ class GlassesSessionCore(
* (ignored on disable).
*/
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) {
if (isG1) return g1SetImuReport(enable)
monitor.withLock {
if (!running || !sessionReady) {
logLine("skip IMU " + (if (enable) "enable" else "disable") + "; session not ready")
@@ -1286,6 +1357,11 @@ class GlassesSessionCore(
/** Request one CFW ambient-light report (image-handler mode 16 op 0). */
fun queryAmbientLight() {
monitor.withLock {
if (isG1) {
g1.lastLightAtMs = 0
interruptibleSleep.interrupt()
return
}
enqueueAmbientLightControlLocked(byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 0.toByte()), "als query", false)
}
interruptibleSleep.interrupt()
@@ -1368,7 +1444,7 @@ class GlassesSessionCore(
* queued (no session, or a malformed record).
*/
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean {
if (record == null || record.size < 4) {
if (!isG2 || record == null || record.size < 4) {
return false
}
monitor.withLock {
@@ -1388,6 +1464,39 @@ class GlassesSessionCore(
return true
}
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
if (listener != null) {
keyboardEventListeners.add(listener)
}
}
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
if (listener != null) {
keyboardEventListeners.remove(listener)
}
}
/**
* Queue a CFW keyboard-bridge control (['K','B',ver,op,request,epoch,...]) as a settings
* field-130 write to the right temple. Fire-and-forget: the firmware answers with field-131
* records, which arrive through addKeyboardEventListener. Returns false when nothing was
* queued (not a G2, no session, or a malformed record).
*/
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean {
if (!isG2 || record == null || record.size < 8) {
return false
}
monitor.withLock {
if (!running || !sessionReady) {
logLine("skip keyboard control ($label); session not ready")
return false
}
pendingMessages.addLast(messageBuilder.faceclawKeyboardControl(record, label))
}
interruptibleSleep.interrupt()
return true
}
/**
* Forward render-characteristic audio packets to the listener WITHOUT
* sending the stock EvenHub audio-control enable. Used for the CFW
@@ -1619,6 +1728,8 @@ class GlassesSessionCore(
// EvenHub page lifecycle
fun sendShutdown(exitMode: Int): Boolean {
if (isIo) return ioSuspend()
if (isG1) return g1Suspend()
return sendShutdownInternal(exitMode, true)
}
@@ -1629,6 +1740,17 @@ class GlassesSessionCore(
* can use the legacy shutdown-and-lease-release path.
*/
fun sendCfwCleanup(): Boolean {
if (isG1) {
// Fire and forget: the firmware keeps no session that could outlive the link.
monitor.withLock {
if (cfwCleanupDelivered) return true
shutdownRequested = true
cfwCleanupDelivered = true
clearPendingMessagesLocked("G1 cleanup")
}
g1HideBeforeDisconnect()
return true
}
val magic: Int
monitor.withLock {
if (cfwCleanupDelivered) {
@@ -1710,6 +1832,8 @@ class GlassesSessionCore(
* reconnecting Bluetooth here would defeat the power-saving mode.
*/
fun suspendEvenHubSession(): Boolean {
if (isIo) return ioSuspend()
if (isG1) return g1Suspend()
// Ending the plugin task releases the fb lease, which frees the
// on-glasses resource cache; forget it phone-side either way (a lost
// ack may still have taken effect).
@@ -1733,6 +1857,8 @@ class GlassesSessionCore(
* the desired frame.
*/
fun resumeEvenHubSession(): Boolean {
if (isIo) return ioResume()
if (isG1) return g1Resume()
var claimGeneration = 0
monitor.withLock {
if (!running || !sessionReady || chargingMode) {
@@ -1813,13 +1939,13 @@ class GlassesSessionCore(
continue
}
logWarn("Attempting to connect")
connectLoopOnce()
if (isIo) ioConnectLoopOnce() else if (isG1) g1ConnectLoopOnce() else connectLoopOnce()
continue
}
val sleepMs = driveSession()
if (sleepMs > 0) {
interruptibleSleep.sleep(sleepMs)
interruptibleSleep.sleep(if (isG1) minOf(sleepMs, g1MaxSleepMs()) else sleepMs)
}
} catch (t: Throwable) {
if (!running) {
@@ -1840,10 +1966,19 @@ class GlassesSessionCore(
return
}
val uuid = characteristicUuid.lowercase()
// The phone's own R1 link, beside either glasses (the iO's ring is always direct).
if (isDirectRingNotification(address, uuid)) {
handleDirectRingNotification(uuid, data)
return
}
if (isIo) {
ioOnNotification(address, uuid, data)
return
}
if (isG1) {
g1OnNotification(address, uuid, data)
return
}
if (BleProtocol.RENDER_NOTIFY_UUID == uuid) {
handleRenderNotification(address, data)
return
@@ -1854,28 +1989,7 @@ class GlassesSessionCore(
if (data.size >= 7 && (data[6].toInt() and 255) == CfwTransport.SID) {
val acks = CfwTransport.parseAcks(data) ?: return
monitor.withLock {
lastIncomingAtMs = now()
val arm = armIndex(address)
if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs
for (ack in acks) {
for (message in inFlightMessages) {
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) {
message.acceptCfwAck(ack)
if (ack.nack) ConnectionCounters.increment("g2.cfw-nack")
logLine("CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId
+ " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2].toInt() and 255)
+ " redundant=" + (ack !== acks[0])
+ " size=" + ack.size + " crc=" + ack.checksum
+ " expected=" + message.message.size + "/" + message.cfwChecksum
+ " ackedLenses=" + message.cfwAckLenses
+ " ageMs=" + (lastIncomingAtMs - message.sentAtMs))
message.ackPayload = data.copyOf()
break
}
}
}
drainCfwAcknowledgementsLocked()
acceptCfwAcksLocked(address, acks, data[2].toInt() and 255, data)
}
interruptibleSleep.interrupt()
return
@@ -1889,6 +2003,18 @@ class GlassesSessionCore(
}
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
val frame = BleProtocol.parseFrame(data)
if (frame.ok && frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW keyboard-bridge records (field 131) come from whichever temple hosts the
// keyboard. They are their own stream: neither an ACK nor glasses input, and the
// once-a-second status record among them must not count as user activity (which
// would hold off battery polling).
val keyboardEvent = BleProtocol.parseFaceclawKeyboardEvent(frame.pb)
if (keyboardEvent != null) {
monitor.withLock { lastIncomingAtMs = now() }
emitKeyboardEvent(keyboardEvent)
return
}
}
val decodedWearState = BleProtocol.parseWearState(frame)
val compassEvent = if (address.equals(rightAddress, ignoreCase = true))
BleProtocol.parseCompassEvent(frame)
@@ -2169,6 +2295,7 @@ class GlassesSessionCore(
lastConnectionOrInputAtMs = arrivalMs
lastDirectRingInputAtMs = arrivalMs
lastDirectRingInputLabel = decoded.label + " " + decoded.detail
if (isIo) ioReclaimPageOnInputLocked()
}
logLine("direct ring " + decoded.label + " " + decoded.detail + " raw=" + hex(data))
val frameId = frameTimings.startFrame("input:ring:" + decoded.label)
@@ -2196,6 +2323,35 @@ class GlassesSessionCore(
}
}
/**
* Apply CFW ACKs/NACKs from [address] (the G2's `aa 12` frames, or the iO's 9-byte bodies)
* to the in-flight messages they name, then complete what they finished.
*/
internal fun acceptCfwAcksLocked(address: String, acks: Array<CfwTransport.Ack>, txSeq: Int, payload: ByteArray) {
lastIncomingAtMs = now()
val arm = armIndex(address)
if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs
for (ack in acks) {
for (message in inFlightMessages) {
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) {
message.acceptCfwAck(ack)
if (ack.nack) ConnectionCounters.increment("g2.cfw-nack")
logLine("CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId
+ " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + txSeq
+ " redundant=" + (ack !== acks[0])
+ " size=" + ack.size + " crc=" + ack.checksum
+ " expected=" + message.message.size + "/" + message.cfwChecksum
+ " ackedLenses=" + message.cfwAckLenses
+ " ageMs=" + (lastIncomingAtMs - message.sentAtMs))
message.ackPayload = payload.copyOf()
break
}
}
}
drainCfwAcknowledgementsLocked()
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (address == null) {
return
@@ -2231,12 +2387,17 @@ class GlassesSessionCore(
monitor.withLock {
if (address.equals(rightAddress, ignoreCase = true)) {
rightConnected = connected
// The iO's one link stands for both arms.
if (isIo) leftConnected = connected
} else if (address.equals(leftAddress, ignoreCase = true)) {
leftConnected = connected
} else {
return
}
if (!connected) {
if (!connected && isIo) io.reassembler.reset()
if (!connected && isG1 && g1CarryOnWithoutArmLocked(address)) {
// The session goes on; the worker brings this arm back.
} else if (!connected) {
// Before the session is ready, this is an attempt failing (or an arm
// dropping mid-setup): the connect sequence notices on its own and
// schedules the retry, so leave the backoff alone.
@@ -0,0 +1,774 @@
package com.faceclaw.app
import com.faceclaw.app.GlassesSessionCore.Companion.hex
import com.faceclaw.app.GlassesSessionCore.Companion.safeMessage
import kotlin.random.Random
/** Which glasses a session drives. Set before [GlassesSessionCore.start]; see [GlassesSessionCore.useRayNeoIo]. */
enum class GlassesType { G2, RAYNEO_IO, G1 }
/**
* RayNeo iO session state ([GlassesSessionCore.io]); guarded by the core's monitor.
*
* The iO is one SoC on ONE BLE link that drives both eyes, so the core runs it with the same
* address in both arm slots: CFW messages keep going to the "left" arm, per-arm bookkeeping
* resolves to index 0, and the firmware ACKs each lens of a record separately, exactly what
* the existing window expects. What differs lives here: the AA55 envelope, the TLV handshake
* and guest authentication, IDENT-based capability detection, and the page lease
* (PAGE_OPEN + KEEPALIVE) that stands in for the G2's EvenHub page and framebuffer lease.
* Wire codecs: [RayNeoIoProtocol]; firmware: faceclaw-rayneo-cfw (docs/FACECLAW.md).
*/
internal class IoSessionState {
var phoneId = ByteArray(6)
var phoneName = "Faceclaw"
var manufacturer = "Android"
var model = "Android"
val reassembler = RayNeoIoProtocol.FrameReassembler()
var messageNumber = Random.nextInt(0, 0x10000)
var launcherSequence = 0
var versionReply: RayNeoIoProtocol.VersionReply? = null
var authReply: ByteArray? = null
var ident: RayNeoIoProtocol.Ident? = null
var osVersion = ""
var maxChunk = RayNeoIoProtocol.DEFAULT_MAX_CHUNK
/** Last PAGE state the glasses reported: 1 open, 0 closed, -1 none yet this link. */
var pageState = -1
var pageReason = 0
var pageReplies = 0
/** Whether the phone wants the page open (false while suspended or after the wearer closed it). */
var pageWanted = true
var pageRequestSentAtMs = 0L
var lastKeepaliveAtMs = 0L
var lastStatusAtMs = 0L
var refusals = 0
/** Whether Faceclaw's latest frame is meant to be seen (false for its blanked frames). */
var displayVisible = false
var brightnessSentLevel = -1
var lastBrightnessAtMs = 0L
/** The light sensor poll interval this page was last asked for ([ioFirmwareOwnsBrightness]), -1 none. */
var alsIntervalSent = -1
/** Last `screen_status` push: 1 on, 0 off, -1 unknown. */
var screenStatus = -1
/** Whether Faceclaw answers "Hey RayNeo" (null until the dashboard says); what this link was last told. */
var voiceWakeWanted: Boolean? = null
var voiceWakeSent = -1
/** The stock assistant opened a session (wake word): end it, it must not take the display. */
var voiceAbortPending = false
/** Ask for the launcher's general settings (logged: the wake-word preferences among them). */
var settingsRequestPending = true
/** When a wake was last reported: one phrase can wake twice (fast and second stage). */
var lastVoiceWakeAtMs = 0L
/** A capture wants the remote mic stream; whether this link was asked to open it, and when. */
var micWanted = false
var micStreamOpen = false
var micStartSentAtMs = 0L
var micPackets = 0L
var lastMicPacketAtMs = 0L
fun resetLink() {
reassembler.reset()
versionReply = null
authReply = null
ident = null
pageState = -1
pageReason = 0
pageRequestSentAtMs = 0
lastKeepaliveAtMs = 0
lastStatusAtMs = 0
brightnessSentLevel = -1
lastBrightnessAtMs = 0
alsIntervalSent = -1
screenStatus = -1
voiceWakeSent = -1
voiceAbortPending = false
settingsRequestPending = true
micStreamOpen = false
micStartSentAtMs = 0
}
}
internal object IoTimings {
const val VERSION_REPLY_MS = 4_000L
const val AUTH_REPLY_MS = 4_000L
const val OS_VERSION_MS = 1_500L
const val IDENT_MS = 3_000L
const val PAGE_REPLY_MS = 4_000L
/** Well inside the firmware's 120 s page lease. */
const val KEEPALIVE_MS = 30_000L
const val STATUS_MS = 60_000L
const val PAGE_RETRY_MS = 3_000L
/** Retry cadence while the glasses refuse this phone id (waiting for pairing mode). */
const val PAIRING_RETRY_MS = 2_000L
const val PAGE_REASON_USER = 4
/** While Faceclaw shows content, re-assert the screen this often (the stock auto-off is short). */
const val SCREEN_REFRESH_MS = 5_000L
/** A wanted mic stream that hasn't delivered a packet this long is asked for again. */
const val MIC_RETRY_MS = 3_000L
/** Wakes this close to a reported one are the same phrase (fast stage, then second stage). */
const val VOICE_WAKE_DEDUP_MS = 3_000L
}
/** The glasses hung up at the version request: bound to another phone id, or not in pairing mode. */
internal class IoPairingRefusedException : IllegalStateException("glasses hung up at the version request")
/** CFW message types the iO firmware implements (fc_glue.c + raster cleanup). */
internal fun ioSupportsCfwType(type: Int): Boolean =
type == CFW_MSG_CLEANUP || type == CFW_MSG_UPLOAD_RESOURCE || type == CFW_MSG_EVICT_RESOURCE ||
type == CFW_MSG_DRAW_CALLS || type == CFW_MSG_SET_ROOT_DISPLAY_LIST ||
type == CFW_MSG_PRESENT || type == CFW_MSG_CREATE_SURFACE
internal val GlassesSessionCore.isIo: Boolean get() = glassesType == GlassesType.RAYNEO_IO
/**
* The firmware runs message 30 and the light sensor (faceclaw-rayneo-cfw bright.c), with the G2's
* wire formats, so the shared brightness code drives it. Older firmware gets the launcher's
* brightness command, which is all it has (see [ioMaintain]).
*/
internal val GlassesSessionCore.ioFirmwareOwnsBrightness: Boolean get() = io.ident?.brightnessControl == true
/** Messages the iO cannot take (stock EvenHub/settings traffic, the G2's other hardware CFW modes). */
internal fun GlassesSessionCore.ioSupportsMessage(message: OutboundMessage): Boolean {
if (message.sid != CfwTransport.SID || message.message.isEmpty()) return false
val type = message.message[0].toInt() and 255
return ioSupportsCfwType(type) ||
(ioFirmwareOwnsBrightness && (type == CFW_MSG_BRIGHTNESS || type == CFW_MSG_AMBIENT_LIGHT))
}
/** A page opened (or reopened): the firmware gave brightness back when the last one closed. */
private fun GlassesSessionCore.ioResetBrightnessLocked() {
brightnessSentVisible = null
brightnessSentLevel = -1
brightnessAlsStartedAt = -2000
brightnessPolicy.resetSamples()
io.alsIntervalSent = -1
}
// ---------------------------------------------------------------------------------------------
// Writes
private fun GlassesSessionCore.ioNextMessageNumberLocked(): Int {
val n = io.messageNumber and 0xffff
io.messageNumber = (io.messageNumber + 1) and 0xffff
return n
}
/** One AA55 frame on the iO write characteristic; worker thread, outside the monitor. */
internal fun GlassesSessionCore.ioWriteFrame(business: Int, payload: ByteArray): Boolean {
val frame = monitor.withLock { RayNeoIoProtocol.encodeFrame(ioNextMessageNumberLocked(), business, payload) }
return link.writeFrames(
rightAddress, RayNeoIoBle.WRITE_CHAR_UUID,
RayNeoIoProtocol.writes(frame, link.negotiatedMtu(rightAddress)),
GattWriteMode.WITHOUT_RESPONSE, ConnectionOptions.WRITE_TIMEOUT_MS,
)
}
internal fun GlassesSessionCore.ioWriteControl(ctrl: Int): Boolean =
ioWriteFrame(RayNeoIoProtocol.BUSINESS_CFW, RayNeoIoProtocol.controlPacket(ctrl))
private fun GlassesSessionCore.ioWriteLauncher(type: Int, json: String): Boolean {
val sequence = monitor.withLock {
io.launcherSequence = if (io.launcherSequence >= 0xffff) 1 else io.launcherSequence + 1
io.launcherSequence
}
val envelope = RayNeoIoProtocol.Envelope(type, json, sequence = sequence, mode = RayNeoIoProtocol.MODE_REQUEST)
return ioWriteFrame(RayNeoIoProtocol.BUSINESS_LAUNCHER, RayNeoIoProtocol.encodeEnvelope(envelope))
}
internal fun GlassesSessionCore.ioRequestStatus(): Boolean =
ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_GENERAL_STATUS, RayNeoIoProtocol.GENERAL_STATUS_REQUEST)
/** From the image path (enqueueBrightnessLocked): a visible frame wakes the screen at once. */
internal fun GlassesSessionCore.ioNoteFrameVisibleLocked(frameVisible: Boolean?) {
if (frameVisible == null) return
if (frameVisible && !io.displayVisible) io.lastBrightnessAtMs = 0
io.displayVisible = frameVisible
}
/**
* The writes for one queued CFW message: the shared record (persistent deflate context, as
* on the G2) in `[opt][stream][chunk]` packets, each its own AA55 frame. The stream byte is
* the message's magic, which the firmware echoes in its per-lens ACKs.
*/
internal fun GlassesSessionCore.ioCfwWrites(message: OutboundMessage, mtu: Int): List<ByteArray> {
val transport = cfwTransports[0]
if (message.cfwRetries > 0) transport.reset()
val record = transport.encodeRecord(message.message, CfwTransport.BOTH)
return monitor.withLock {
RayNeoIoProtocol.cfwPackets(record, message.magic, io.maxChunk).flatMap { packet ->
RayNeoIoProtocol.writes(RayNeoIoProtocol.encodeFrame(ioNextMessageNumberLocked(), RayNeoIoProtocol.BUSINESS_CFW, packet), mtu)
}
}
}
/** A message the iO can't run (see [ioSupportsMessage]): never sent, never waited on. */
internal fun GlassesSessionCore.ioDropUnsupported(message: OutboundMessage) {
monitor.withLock {
inFlightMessages.remove(message)
if (message.magic != 0) magicPool.release(message.sid, message.magic, message.label, "unsupported on these glasses")
}
logDebug("dropped a message these glasses don't take: " + message.label + " sid=" + message.sid)
}
// ---------------------------------------------------------------------------------------------
// Connect (worker thread)
private fun GlassesSessionCore.ioAwait(timeoutMs: Long, what: String, ready: () -> Boolean) {
val deadline = now() + timeoutMs
monitor.withLock {
while (true) {
if (ready()) return
if (!running || userDisconnectRequested) throw IllegalStateException("stopped while waiting for $what")
if (!rightConnected) throw IoLinkLostException(what)
val remaining = deadline - now()
if (remaining <= 0) throw IllegalStateException("$what timed out")
monitor.awaitMs(minOf(remaining, 100L))
}
}
}
private class IoLinkLostException(what: String) : IllegalStateException("link lost while waiting for $what")
internal fun GlassesSessionCore.ioConnectLoopOnce() {
monitor.withLock {
wearState = -1
io.resetLink()
}
setStateDisplay("connecting", "Connecting to the glasses...")
ConnectionCounters.increment("io.connect-attempt")
val address = rightAddress
try {
if (!link.connect(address, ConnectionOptions.ARM_CONNECT_WINDOW_MS)) {
handleTransportFailure("glasses not found", "Can't find the glasses. They may be off, in their case, or out of range.")
return
}
link.requestHighPriority(address)
link.requestMtu(address, RayNeoIoBle.REQUESTED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!link.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) throw IllegalStateException("discoverServices failed: $address")
if (!link.enableNotifications(address, RayNeoIoBle.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw IllegalStateException("enableNotifications failed: $address")
}
link.requestLe2mPhy(address)
monitor.withLock {
rightConnected = true
leftConnected = true
}
// Version exchange. Glasses bound to another phone id (or not in pairing mode)
// hang up ~100 ms after this request instead of answering.
val phone = monitor.withLock { io.phoneId.copyOf() }
logDebug("RayNeo iO: phone id " + hex(phone))
if (!ioWriteFrame(RayNeoIoProtocol.BUSINESS_HANDSHAKE,
RayNeoIoProtocol.versionRequest(phone, io.phoneName, io.manufacturer, io.model))) {
throw IllegalStateException("version request write failed")
}
try {
ioAwait(IoTimings.VERSION_REPLY_MS, "version reply") { io.versionReply != null }
} catch (lost: IoLinkLostException) {
throw IoPairingRefusedException()
}
val reply = monitor.withLock { io.versionReply!! }
if (reply.accountBound) throw IoPairingRefusedException()
logLine("RayNeo iO: glasses " + hex(reply.glassesId) + " serial " + reply.serial + " protocol " + reply.protocolVersion)
val random = Random.nextBytes(4)
if (!ioWriteFrame(RayNeoIoProtocol.BUSINESS_HANDSHAKE, RayNeoIoProtocol.guestAuthRequest(random, phone, reply.glassesId))) {
throw IllegalStateException("guest auth write failed")
}
ioAwait(IoTimings.AUTH_REPLY_MS, "guest auth reply") { io.authReply != null }
val authReply = monitor.withLock { io.authReply!! }
if (!RayNeoIoProtocol.verifyGuestAuthReply(authReply, phone, reply.glassesId)) {
throw IllegalStateException("guest auth proof mismatch")
}
monitor.withLock { io.refusals = 0 }
// Battery and the OS version: answers arrive asynchronously; the version only labels
// the firmware info, so a missing one is not fatal.
ioRequestStatus()
ioWriteFrame(RayNeoIoProtocol.BUSINESS_OTA,
RayNeoIoProtocol.encodeEnvelope(RayNeoIoProtocol.Envelope(RayNeoIoProtocol.OTA_VERSION, "{}")))
try {
ioAwait(IoTimings.OS_VERSION_MS, "OS version") { io.osVersion.isNotEmpty() }
} catch (e: IoLinkLostException) {
throw e
} catch (_: IllegalStateException) {
}
// Capability: the custom firmware answers QUERY with IDENT; stock firmware stays silent.
ioWriteControl(RayNeoIoProtocol.CTRL_QUERY)
val ident = try {
ioAwait(IoTimings.IDENT_MS, "IDENT") { io.ident != null }
monitor.withLock { io.ident }
} catch (e: IoLinkLostException) {
throw e
} catch (_: IllegalStateException) {
null
}
val os = monitor.withLock { io.osVersion.ifEmpty { "RayNeo iO" } }
val info = BleProtocol.FirmwareInfo(os, os,
if (ident != null && ident.faceclaw) "Faceclaw/" + ident.faceclawRevision else "")
logLine("RayNeo iO: " + os + ", " + (if (ident == null) "no IDENT (stock firmware)"
else ident.name + " " + ident.width + "x" + ident.height + " chunk " + ident.maxChunk
+ " flags 0x" + ident.flags.toString(16) + " faceclaw " + ident.faceclawRevision))
emitFirmwareInfo(info)
if (!info.isCompatible(requiredFirmwareRevision)) {
monitor.withLock {
// The worker loop halts on this and explains it (handleIncompatibleFirmware).
incompatibleFirmware = info
}
return
}
monitor.withLock {
io.maxChunk = ident!!.maxChunk.coerceIn(16, 4000)
io.pageWanted = true
io.pageRequestSentAtMs = now()
}
ioWriteControl(RayNeoIoProtocol.CTRL_PAGE_OPEN)
try {
ioAwait(IoTimings.PAGE_REPLY_MS, "page open") { io.pageReplies > 0 }
} catch (e: IoLinkLostException) {
throw e
} catch (_: IllegalStateException) {
// The launcher may be busy; ioMaintain keeps asking.
}
monitor.withLock {
sessionReady = true
shutdownRequested = false
customFirmwareDetected = true
clearAllMessagesLocked("session ready")
fixedLayoutCreated = io.pageState == 1
displayedFingerprint = ""
lastAckAtMs = now()
lastIncomingAtMs = lastAckAtMs
lastConnectionOrInputAtMs = lastAckAtMs
lastSessionReadyAtMs = lastAckAtMs
imageRetryAfterMs = 0
consecutiveAckTimeouts = 0
cfwCleanupDelivered = false
lastCfwCleanupAckMagic = 0
for (transport in cfwTransports) transport.reset()
io.lastKeepaliveAtMs = lastAckAtMs
io.lastStatusAtMs = lastAckAtMs
ioResetBrightnessLocked()
}
setStateDisplay("connected", "Connected.")
ConnectionCounters.increment("io.session-ready")
logLine("RayNeo iO session ready (page " + (if (fixedLayoutCreated) "open" else "pending") + ")")
} catch (refused: IoPairingRefusedException) {
val count = monitor.withLock {
io.refusals++
reconnectAfterMs = now() + IoTimings.PAIRING_RETRY_MS
io.refusals
}
ConnectionCounters.increment("io.pairing-refused")
logLine("RayNeo iO refused this phone (" + count + "); waiting for pairing mode")
link.disconnect(address)
monitor.withLock {
rightConnected = false
leftConnected = false
}
setStateDisplay("retrying",
"The glasses aren't accepting this phone yet. Put them in pairing mode: on the charger, "
+ "hold the button about 5 seconds until the light blinks blue.")
} catch (t: Throwable) {
logLine("RayNeo iO connect failed: " + safeMessage(t))
handleTransportFailure("connect failed")
}
}
// ---------------------------------------------------------------------------------------------
// Session upkeep (worker thread, from driveSession, outside the monitor)
/** Page open/close requests, keepalives, status polls and the screen (launcher brightness). */
internal fun GlassesSessionCore.ioMaintain() {
var ctrl = -1
var status = false
var brightness = -1
var wakeword = -1
var micRc = -1
var abortVoice = false
var settings = false
monitor.withLock {
if (!sessionReady) return
val now = now()
val wake = io.voiceWakeWanted
if (wake != null && io.voiceWakeSent != (if (wake) 1 else 0)) {
wakeword = if (wake) 1 else 0
io.voiceWakeSent = wakeword
// Read back what the switch did.
io.settingsRequestPending = true
}
if (io.settingsRequestPending) {
settings = true
io.settingsRequestPending = false
}
if (io.voiceAbortPending) {
abortVoice = true
io.voiceAbortPending = false
}
if (io.micWanted && (!io.micStreamOpen
|| (io.lastMicPacketAtMs < io.micStartSentAtMs && now - io.micStartSentAtMs >= IoTimings.MIC_RETRY_MS))) {
// Not yet asked on this link, a voice session took the recorder, or the ask was lost.
micRc = RayNeoIoProtocol.VOICE_RC_START
io.micStreamOpen = true
io.micStartSentAtMs = now
} else if (!io.micWanted && io.micStreamOpen) {
micRc = RayNeoIoProtocol.VOICE_RC_STOP
io.micStreamOpen = false
}
if (io.pageWanted && !fixedLayoutCreated && now - io.pageRequestSentAtMs >= IoTimings.PAGE_RETRY_MS) {
ctrl = RayNeoIoProtocol.CTRL_PAGE_OPEN
io.pageRequestSentAtMs = now
} else if (!io.pageWanted && io.pageState == 1 && now - io.pageRequestSentAtMs >= IoTimings.PAGE_RETRY_MS) {
ctrl = RayNeoIoProtocol.CTRL_PAGE_CLOSE
io.pageRequestSentAtMs = now
} else if (io.pageWanted && fixedLayoutCreated && now - io.lastKeepaliveAtMs >= IoTimings.KEEPALIVE_MS) {
ctrl = RayNeoIoProtocol.CTRL_KEEPALIVE
io.lastKeepaliveAtMs = now
}
if (now - io.lastStatusAtMs >= IoTimings.STATUS_MS) {
status = true
io.lastStatusAtMs = now
}
// Faceclaw decides when the display is dark (blanked frames); while it shows
// content, keep the iO's screen on and at Faceclaw's brightness.
val showing = fixedLayoutCreated && io.pageWanted && io.displayVisible && !shutdownRequested
if (ioFirmwareOwnsBrightness) {
// Repeating message 30 starts no fade; visible=1 is what holds the screen on.
if (showing && now - io.lastBrightnessAtMs >= IoTimings.SCREEN_REFRESH_MS) {
io.lastBrightnessAtMs = now
brightnessSentLevel = -1
enqueueBrightnessLocked()
}
} else {
val level = RayNeoIoProtocol.brightnessLevel(brightnessPolicy.target)
if (showing && (level != io.brightnessSentLevel || now - io.lastBrightnessAtMs >= IoTimings.SCREEN_REFRESH_MS)) {
brightness = level
io.brightnessSentLevel = level
io.lastBrightnessAtMs = now
}
}
}
if (ctrl >= 0) {
logDebug("RayNeo iO: control $ctrl")
ioWriteControl(ctrl)
}
if (status) ioRequestStatus()
if (brightness > 0) ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_BRIGHTNESS, RayNeoIoProtocol.launcherBrightness(brightness))
if (wakeword >= 0) {
logLine("RayNeo iO: wake word " + (if (wakeword == 1) "on" else "off"))
ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_AI_VOICE_WAKEUP, RayNeoIoProtocol.launcherSetAiVoiceWakeup(wakeword == 1))
}
if (settings) ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_GENERAL_SETTINGS, RayNeoIoProtocol.GENERAL_SETTINGS_REQUEST)
if (abortVoice) {
// As Glassly does: decline the stock session, then abort it, so its page goes.
ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_RC, RayNeoIoProtocol.VOICE_RC_STOP))
ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_ABORT, RayNeoIoProtocol.VOICE_RC_START))
}
if (micRc > 0) {
logLine("RayNeo iO mic: " + (if (micRc == RayNeoIoProtocol.VOICE_RC_START) "open" else "close") + " the remote audio stream")
ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_STREAM_CONTROL, micRc))
}
}
/**
* The glasses' mic for a voice capture: the remote audio stream (16 kHz mono Opus packets to
* [listener]), which needs IDENT_FLAG_MIC firmware. ioMaintain sends the open (and repeats it
* while no packet comes); a voice session or a call can take the recorder meanwhile.
*/
internal fun GlassesSessionCore.ioStartAudioCapture(listener: FaceclawAudioPacketListener): Boolean {
monitor.withLock {
if (!running || !sessionReady || shutdownRequested) {
logLine("skip RayNeo iO mic: session not ready")
return false
}
if (io.ident?.micStream != true) {
logLine("skip RayNeo iO mic: the firmware has no remote audio stream (IDENT flag 0x20)")
return false
}
audioPacketListener = listener
audioCaptureActive = true
io.micWanted = true
io.micStreamOpen = false
io.micPackets = 0
}
interruptibleSleep.interrupt()
return true
}
internal fun GlassesSessionCore.ioStopAudioCapture() {
monitor.withLock {
audioPacketListener = null
audioCaptureActive = false
io.micWanted = false
}
interruptibleSleep.interrupt()
}
/** Whether "Hey RayNeo" should reach Faceclaw (the launcher's wake-word switch). */
internal fun GlassesSessionCore.ioSetVoiceWakeEnabled(enabled: Boolean) {
monitor.withLock { io.voiceWakeWanted = enabled }
interruptibleSleep.interrupt()
}
/** Stop drawing and close the page, keeping the BLE link (the EvenHub suspend analogue). */
internal fun GlassesSessionCore.ioSuspend(): Boolean {
monitor.withLock {
if (!running || !sessionReady) return false
shutdownRequested = true
io.pageWanted = false
io.pageRequestSentAtMs = 0
clearAllMessagesLocked("RayNeo iO suspend")
displayedFingerprint = ""
}
interruptibleSleep.interrupt()
return true
}
/** Reopen the page; the drive loop resends the desired frame once it is open. */
internal fun GlassesSessionCore.ioResume(): Boolean {
monitor.withLock {
if (!running || !sessionReady) return false
shutdownRequested = false
io.pageWanted = true
io.pageRequestSentAtMs = 0
fixedLayoutCreated = false
clearAllMessagesLocked("RayNeo iO resume")
displayedFingerprint = ""
imageRetryAfterMs = 0
}
interruptibleSleep.interrupt()
return true
}
// ---------------------------------------------------------------------------------------------
// Inbound (BLE callback thread)
internal fun GlassesSessionCore.ioOnNotification(address: String, characteristicUuid: String, data: ByteArray) {
if (RayNeoIoBle.NOTIFY_CHAR_UUID != characteristicUuid || !address.equals(rightAddress, ignoreCase = true)) return
val frames = monitor.withLock { io.reassembler.ingest(data) }
for (frame in frames) {
when (frame.business) {
RayNeoIoProtocol.BUSINESS_HANDSHAKE -> monitor.withLock {
val version = RayNeoIoProtocol.parseVersionReply(frame.payload)
if (version != null) io.versionReply = version
else if (RayNeoIoProtocol.isGuestAuthReply(frame.payload)) io.authReply = frame.payload
monitor.signalAll()
}
RayNeoIoProtocol.BUSINESS_LAUNCHER -> ioHandleLauncher(frame.payload)
RayNeoIoProtocol.BUSINESS_VOICE -> ioHandleVoice(frame.payload)
RayNeoIoProtocol.BUSINESS_OTA -> {
val envelope = RayNeoIoProtocol.decodeEnvelope(frame.payload) ?: continue
if (envelope.type != RayNeoIoProtocol.OTA_VERSION) continue
val os = try { Json.parse(envelope.json).asObject()?.get("OsVersion").asString() } catch (_: Exception) { null }
if (os != null) monitor.withLock {
io.osVersion = os
monitor.signalAll()
}
}
RayNeoIoProtocol.BUSINESS_CFW -> ioHandleCfw(address, frame)
}
}
}
private fun GlassesSessionCore.ioHandleLauncher(payload: ByteArray) {
val envelope = RayNeoIoProtocol.decodeEnvelope(payload) ?: return
val screen = RayNeoIoProtocol.screenStatusFromLauncherJson(envelope.json)
if (screen >= 0) {
val changed = monitor.withLock {
val was = io.screenStatus
io.screenStatus = screen
// Woken by the wearer (or the timeout ran out): re-assert promptly if Faceclaw is showing.
if (screen == 0) io.lastBrightnessAtMs = 0
was != screen
}
if (changed) logLine("RayNeo iO screen " + (if (screen == 1) "on" else "off"))
return
}
val battery = RayNeoIoProtocol.batteryFromLauncherJson(envelope.json)
if (battery < 0) {
// Replies to our commands and settings (the wake word's among them): logged for now.
logLine("RayNeo iO launcher type " + envelope.type + ": " + envelope.json.take(400))
return
}
val charging = if (envelope.json.contains("\"isCharging\":true")) 1 else if (envelope.json.contains("\"isCharging\":false")) 0 else -1
var changed = false
monitor.withLock {
lastIncomingAtMs = now()
if (battery != headsetBattery || (charging >= 0 && charging != headsetCharging)) {
headsetBattery = battery
if (charging >= 0) headsetCharging = charging
changed = true
}
}
if (changed) emitBatteryState(headsetBattery, headsetCharging)
}
/** The voice channel (business 13): mic packets, and the stock assistant's wake word. */
private fun GlassesSessionCore.ioHandleVoice(payload: ByteArray) {
val envelope = RayNeoIoProtocol.decodeEnvelope(payload) ?: return
when (envelope.type) {
RayNeoIoProtocol.VOICE_STREAM_AUDIO, RayNeoIoProtocol.VOICE_AUDIO -> {
val opus = envelope.binary ?: return
val arrivalMs = now()
var first = false
val listener = monitor.withLock {
lastIncomingAtMs = arrivalMs
io.micPackets++
first = io.micPackets == 1L
io.lastMicPacketAtMs = arrivalMs
if (audioCaptureActive) audioPacketListener else null
}
if (first) logLine("RayNeo iO mic: first audio packet (type " + envelope.type + ", " + opus.size + " bytes)")
if (listener == null) return
try {
listener.onAudioPacket(opus, "R", arrivalMs)
} catch (t: Throwable) {
logLine("RayNeo iO mic packet listener failed: " + safeMessage(t))
}
}
RayNeoIoProtocol.VOICE_WAKE -> {
// The stock assistant heard "Hey RayNeo" and opened its session. Faceclaw's own
// assistant answers instead: end the stock one, and report the wake word as the
// G2's (sid 0x07 EVEN_AI_WAKE_UP), which the dashboard routes to its wake action.
var repeat = false
val forward = monitor.withLock {
val now = now()
io.voiceAbortPending = true
lastIncomingAtMs = now
lastConnectionOrInputAtMs = now
repeat = io.lastVoiceWakeAtMs != 0L && now - io.lastVoiceWakeAtMs < IoTimings.VOICE_WAKE_DEDUP_MS
val wanted = io.voiceWakeWanted == true && sessionReady
if (wanted && !repeat) {
io.lastVoiceWakeAtMs = now
ioReclaimPageOnInputLocked()
}
wanted && !repeat
}
interruptibleSleep.interrupt()
logLine("RayNeo iO: voice wake " + envelope.json + when {
forward -> ""
repeat -> " (same phrase as the last wake)"
else -> " (not forwarded: wake word off)"
})
if (!forward) return
val event = G2Event("even-ai", "", BleProtocol.EVEN_AI_STATUS_WAKE_UP, 0, 0)
val frameId = frameTimings.startFrame("input:even-ai wake")
frameTimings.log(frameId, "wake word from the RayNeo iO's voice channel")
emitRingEvent(event, frameId)
}
RayNeoIoProtocol.VOICE_EXITED -> {
// A voice session that held the recorder ended: ask for the stream again.
val reopen = monitor.withLock {
if (io.micWanted) io.micStreamOpen = false
io.micWanted
}
if (reopen) interruptibleSleep.interrupt()
logLine("RayNeo iO: voice session exited " + envelope.json + (if (reopen) "; reopening the mic stream" else ""))
}
RayNeoIoProtocol.VOICE_VAD -> logDebug("RayNeo iO: voice VAD " + envelope.json)
else -> logLine("RayNeo iO: voice message type " + envelope.type + " " + envelope.json.take(160))
}
}
private fun GlassesSessionCore.ioHandleCfw(address: String, frame: RayNeoIoProtocol.Frame) {
val payload = frame.payload
if (payload.isEmpty()) return
when (payload[0].toInt() and 255) {
RayNeoIoProtocol.KIND_ACK, RayNeoIoProtocol.KIND_NACK -> {
val ack = RayNeoIoProtocol.parseCfwAck(payload) ?: return
monitor.withLock {
acceptCfwAcksLocked(address, arrayOf(ack), frame.messageNumber and 255, payload)
}
interruptibleSleep.interrupt()
}
RayNeoIoProtocol.KIND_IDENT -> monitor.withLock {
io.ident = RayNeoIoProtocol.parseIdent(payload)
monitor.signalAll()
}
RayNeoIoProtocol.KIND_ALS -> {
val body = RayNeoIoProtocol.parseAlsReport(payload) ?: return
monitor.withLock {
lastIncomingAtMs = now()
brightnessPolicy.sample(body, lastIncomingAtMs)
}
emitAmbientLight(body)
interruptibleSleep.interrupt()
}
RayNeoIoProtocol.KIND_PAGE -> ioHandlePage(payload)
RayNeoIoProtocol.KIND_GESTURE -> ioHandleGesture(RayNeoIoProtocol.parseGesture(payload) ?: return)
else -> logDebug("RayNeo iO: CFW reply kind " + (payload[0].toInt() and 255) + " " + hex(payload.copyOf(minOf(payload.size, 24))))
}
}
private fun GlassesSessionCore.ioHandlePage(payload: ByteArray) {
val state = RayNeoIoProtocol.parsePageState(payload)
val reason = if (payload.size >= 3) payload[2].toInt() and 255 else 0
var lost = false
monitor.withLock {
io.pageState = state
io.pageReason = reason
io.pageReplies++
lastIncomingAtMs = now()
if (state == 1) {
if (io.pageWanted && !fixedLayoutCreated && sessionReady) {
// (Re)opened: the drive loop resends the desired frame from scratch.
fixedLayoutCreated = true
displayedFingerprint = ""
imageRetryAfterMs = 0
io.lastKeepaliveAtMs = now()
io.lastBrightnessAtMs = 0
ioResetBrightnessLocked()
}
} else if (state == 0 && fixedLayoutCreated) {
// The page closed under us (lease, the wearer's long press): the firmware freed
// this session's resources, so start over once it reopens.
lost = true
fixedLayoutCreated = false
clearAllMessagesLocked("RayNeo iO page closed (reason $reason)")
displayedFingerprint = ""
if (reason == IoTimings.PAGE_REASON_USER) io.pageWanted = false
io.pageRequestSentAtMs = now()
}
monitor.signalAll()
}
if (lost) logLine("RayNeo iO page closed by the glasses (reason $reason)")
interruptibleSleep.interrupt()
}
/**
* The wearer gave input (a temple touch, or the R1's) after closing our page:
* take it back. Caller holds `monitor`.
*/
internal fun GlassesSessionCore.ioReclaimPageOnInputLocked() {
if (io.pageWanted || !sessionReady) return
io.pageWanted = true
shutdownRequested = false
io.pageRequestSentAtMs = 0
}
private fun GlassesSessionCore.ioHandleGesture(gesture: RayNeoIoProtocol.Gesture) {
val event: G2Event? = when (gesture.eventType) {
// Same numbering as the G2's OsEventTypeList; head up wakes like the G2 CFW's forward.
BleProtocol.EVENT_HEAD_UP -> G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, BleProtocol.EVENT_SOURCE_GLASSES_R, 0)
BleProtocol.EVENT_HEAD_DOWN -> null
else -> G2Event("sys-event", "", gesture.eventType, BleProtocol.EVENT_SOURCE_GLASSES_R, 0)
}
monitor.withLock {
lastIncomingAtMs = now()
if (event != null) lastConnectionOrInputAtMs = lastIncomingAtMs
if (event != null && "sys-event" == event.kind) ioReclaimPageOnInputLocked()
}
logLine("RayNeo iO gesture type=" + gesture.eventType + " source=" + gesture.source + (if (event == null) " (ignored)" else ""))
if (event == null) return
val frameId = frameTimings.startFrame("input:" + event.kind + " type=" + event.eventType + " src=" + event.eventSource)
frameTimings.log(frameId, "input event decoded from RayNeo iO notification")
emitRingEvent(event, frameId)
}
@@ -971,6 +971,13 @@ internal fun macWireBytes(address: String): ByteArray? {
internal fun GlassesSessionCore.queueGlassesRingLinkCommandLocked(
action: String, fallbackAddress: String, fallbackName: String,
): Boolean {
if (!isG2) {
// A RayNeo iO or a G1 has no ring link of its own; the phone holds the ring directly.
lastRingLinkCommand = "$action: not applicable ($glassesType)"
lastRingLinkCommandAtMs = now()
logLine("glasses ring $action skipped: $glassesType glasses have no ring link")
return false
}
val reported = glassesRingMac
val mac = reported ?: macWireBytes(fallbackAddress)
if (mac == null) {
@@ -1091,6 +1098,9 @@ internal fun GlassesSessionCore.queueRingConfigCommandLocked(
frame = RingProtocol.buildPairAuth(seq)
}
"targets-glasses" -> {
// The iO never connects to the ring: naming it would make the ring drop
// the phone's glasses-role link (the advStart check, ~20 s later).
if (!isG2) return "not sent: these glasses don't hold the ring"
val right = macWireBytes(rightAddress)
val left = macWireBytes(leftAddress)
if (right == null || left == null) return "not sent: glasses addresses unparseable"
@@ -44,6 +44,8 @@ internal fun GlassesSessionCore.driveSession(): Long {
benchmarkReadyAtMs = now() + 1_000
}
}
if (isIo) ioMaintain()
if (isG1) g1Maintain()
var messageToWrite: OutboundMessage? = null
var messageToPrewrite: OutboundMessage? = null
var heartbeatToWrite: OutboundMessage? = null
@@ -74,7 +76,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
clearPendingMessagesLocked("after CFW cleanup")
return 250
}
if (sessionReady
if (sessionReady && isG2
&& now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS
&& !hasPendingOrInflightKindLocked("framebuffer-lease-control")) {
enqueueFaceclawFramebufferControlLocked(
@@ -83,7 +85,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
)
}
if (faceclawWakeLeaseEnabled
&& sessionReady
&& sessionReady && isG2
&& now - lastFaceclawWakeLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS
&& !hasPendingOrInflightKindLocked("wake-lease-control")) {
enqueueFaceclawWakeControlLocked(
@@ -158,6 +160,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
}
} else {
if (messageToPrewrite == null
&& isG2 // the iO's page lease is ioMaintain's; a G1 has none
&& !shutdownRequested
&& !fixedLayoutCreated
&& pendingMessages.isEmpty()
@@ -168,14 +171,14 @@ internal fun GlassesSessionCore.driveSession(): Long {
// Prewrite outside the lock; the logical message remains pending until
// its final BLE frame is sent after the current protocol ACK.
}
if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
if (messageToPrewrite == null && isG2 && !shutdownRequested && fixedLayoutCreated
&& (if (firmwareDebugFlagsEnabled) 2 else 1) != firmwareDebugFlagsLastSent
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()) {
logLine("enqueueing firmware debug flags " + (if (firmwareDebugFlagsEnabled) "show" else "hide"))
enqueueFirmwareDebugFlagsLocked()
}
if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
if (messageToPrewrite == null && isG2 && !shutdownRequested && fixedLayoutCreated
&& caseInfoReadPending
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()) {
caseInfoReadPending = false
@@ -183,7 +186,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
pendingMessages.addLast(messageBuilder.caseInfoQuery())
}
if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
if (messageToPrewrite == null && isG2 && !shutdownRequested && fixedLayoutCreated
&& (if (compassOwners.isEmpty()) 0 else 1) != compassControlLastSent
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()) {
val wanted = compassOwners.isNotEmpty()
@@ -194,7 +197,9 @@ internal fun GlassesSessionCore.driveSession(): Long {
if (benchmarkActive) {
maintainBenchmarkLocked(now)
}
if (!benchmarkActive) maintainBrightnessLocked(now)
// A G1's brightness follows the policy's target; its light sensor isn't wired up yet.
if (!benchmarkActive && isG1) enqueueBrightnessLocked()
else if (!benchmarkActive && (!isIo || ioFirmwareOwnsBrightness)) maintainBrightnessLocked(now)
// Up to WINDOW_SIZE messages may be in flight at once (full
// pipelining); a slot frees when an ack arrives. An active
@@ -337,6 +342,7 @@ internal enum class HeartbeatStep {
}
internal fun GlassesSessionCore.heartbeatStepLocked(imageWaiting: Boolean): HeartbeatStep {
if (!isG2) return HeartbeatStep.NONE
val now = now()
val heartbeatEligible = !shutdownRequested && fixedLayoutCreated
val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat")
@@ -387,6 +393,21 @@ internal fun GlassesSessionCore.createHeartbeatMessage(): OutboundMessage {
}
internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean {
if (isG1 && !g1SupportsMessage(message)) {
// G2-only traffic a shared feature queued anyway: it would only wait for an ACK that never comes.
ioDropUnsupported(message)
return true
}
if (isG1 && g1ArmIsAway(message)) {
ioDropUnsupported(message)
return true
}
if (isIo && !ioSupportsMessage(message)) {
// G2-only traffic (EvenHub/settings frames, the G2 hardware CFW modes) that a feature
// queued anyway: the iO would NACK it into the replay limit, so it never goes out.
ioDropUnsupported(message)
return true
}
val now = now()
message.writeStartedAtMs = now
message.sentAtMs = now
@@ -416,6 +437,10 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean
frames = listOf(prewrittenFrames[prewrittenFrames.size - 1])
prewrittenMessage = null
prewrittenFrames = emptyList()
} else if (isIo) {
frames = ioCfwWrites(message, link.negotiatedMtu(writeAddress))
} else if (isG1) {
frames = g1CfwWrites(message, link.negotiatedMtu(writeAddress))
} else if (message.sid == CfwTransport.SID) {
if (message.cfwRetries > 0) cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset()
frames = cfwTransports[if (message.isLeftArmMessage) 0 else 1].encode(
@@ -431,9 +456,9 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean
}
val result = link.writeFrames(
writeAddress,
BleProtocol.WRITE_CHAR_UUID,
if (isIo) RayNeoIoBle.WRITE_CHAR_UUID else if (isG1) G1Ble.FC_CHAR_UUID else BleProtocol.WRITE_CHAR_UUID,
frames,
ConnectionOptions.WRITE_MODE,
if (isIo || isG1) GattWriteMode.WITHOUT_RESPONSE else ConnectionOptions.WRITE_MODE,
ConnectionOptions.WRITE_TIMEOUT_MS
)
@@ -836,6 +861,10 @@ internal fun GlassesSessionCore.enqueueDesiredImageLocked() {
frameTimings.spanEnd(frameId, "pack-4bpp")
// Visibility belongs to this immutable composite, not the latest UI request.
enqueueBrightnessLocked(!fingerprint.startsWith("blanked:"))
if (isG1 && packedFrame.size > 0) {
g1EnqueueFrameLocked(packedFrame, width, height, scene, fingerprint, paintMs, frameId)
return
}
if (customFirmwareDetected && packedFrame.size > 0) {
val rendered = try {
scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId,
@@ -981,10 +1010,13 @@ internal fun GlassesSessionCore.enqueueCustomImageLocked(
fingerprint: String,
updateId: Int,
messageNumber: Int,
messageCount: Int
messageCount: Int,
leftArm: Boolean = connectionOptions.sendImagesToLeft,
requiredLenses: Int = CfwTransport.BOTH,
) {
val message = messageBuilder.customMessage("image", payload,
"image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft)
"image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, leftArm)
message.cfwRequiredLenses = requiredLenses
message.setImageUpdatePosition(updateId, messageNumber, messageCount)
message.onAck = MessageCallback {
imageRetryAfterMs = 0
@@ -1037,7 +1069,8 @@ internal fun GlassesSessionCore.createAudioControlMessageLocked(enable: Boolean)
}
internal fun GlassesSessionCore.shouldPollBatteryLocked(now: Long): Boolean {
return !shutdownRequested
return isG2 // the others' status polls are their own maintain's
&& !shutdownRequested
&& sessionReady
&& pendingMessages.isEmpty()
&& inFlightMessages.isEmpty()
@@ -1424,6 +1457,7 @@ internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) {
// drop the pipelined delta base so the next image is a full keyframe.
lastEnqueuedPacked = ByteArray(0)
lastEnqueuedFingerprint = ""
g1ForgetFramesLocked()
}
}
@@ -1431,12 +1465,14 @@ internal fun GlassesSessionCore.clearAllMessagesLocked(reason: String) {
brightnessSentVisible = null
brightnessSentLevel = -1
brightnessAlsStartedAt = -2000
io.alsIntervalSent = -1
clearPendingMessagesLocked(reason)
clearInFlightMessagesLocked(reason)
// The image pipeline is gone: drop the pipelined delta base so the next
// image is a full keyframe rather than a delta onto a stale base.
lastEnqueuedPacked = ByteArray(0)
lastEnqueuedFingerprint = ""
g1ForgetFramesLocked()
// Queued resource commands (if any) were dropped with the rest, and the
// session churn behind a full clear may have freed the on-glasses
// cache; forget it phone-side so glyphs re-upload lazily.
@@ -1452,6 +1488,7 @@ internal fun GlassesSessionCore.clearAllMessagesPreservingWakeLeaseLocked(reason
brightnessSentVisible = null
brightnessSentLevel = -1
brightnessAlsStartedAt = -2000
io.alsIntervalSent = -1
val pendingIterator = pendingMessages.iterator()
while (pendingIterator.hasNext()) {
val message = pendingIterator.next()
@@ -68,6 +68,18 @@ interface SessionLink {
/** Whether the OS still holds a pairing bond for the address (iOS: always true). */
fun isBonded(address: String): Boolean
/** Ask the OS to bond with a connected device, without waiting. A no-op where it pairs on demand only (CoreBluetooth). */
fun requestBond(address: String) {}
/** Remove this phone's half of a pairing; false where the platform can't or refused. */
fun removeBond(address: String): Boolean = false
/** After discovery: whether the service table has this characteristic. Enabling notifications on one it lacks may throw. */
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean = true
/** Read the service table again, past whatever the OS cached for this address. */
fun refreshServices(address: String, timeoutMs: Int): Boolean = discoverServices(address, timeoutMs)
/** Bandwidth-benchmark link tuning; 0: current link, 1: HIGH priority, 2: 2M PHY, 3: both. */
fun prepareBenchmarkLink(address: String, mode: Int)
@@ -0,0 +1,120 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.max
import kotlin.math.pow
import kotlin.math.roundToInt
/**
* Turns a color glyph (an emoji from a color font the text stack fell back to) into the
* lit-on-black coverage the glyph pipeline carries, so it keeps its internal detail instead of
* showing as its alpha silhouette (a face as a plain disc, ✅ as a plain square). Platform
* rasterizers draw the glyph in white and pass unpremultiplied ARGB; this object owns the
* pixel math so Android and iOS agree.
*
* Shading is by OKLab lightness relative to the glyph's body, its median lightness:
* - the body lights fully, so a one-color emoji (❤️, ⭐, 🖤) reads as its shape at full
* brightness whatever its color, the way the alpha did, rather than dim for being dark;
* - darker detail (eyes, mouths, the digits on 📅) falls toward [DARKEST_LIT], never off, so
* the silhouette stays whole;
* - when a fair share of the glyph is clearly lighter than the body (the check on ✅, the
* circle on 🎱, the eyes on 💩) the body dims toward [DIMMEST_BODY] to leave that detail
* room above it. Small highlights (❤️'s glint) don't count.
* Lit values go through the notification icons' tone curve ([NotificationIconArt.GAMMA]),
* then scale by the pixel's alpha, which stays the glyph's coverage.
*/
object ColorGlyphArt {
/** A pixel this opaque that is not white means the glyph was not drawn in the paint's white. */
private const val MIN_COLOR_ALPHA = 32
/** How far below 255 a channel may sit and still count as white (unpremultiply rounding). */
private const val WHITE_TOLERANCE = 24
/** Lit level of the glyph's darkest detail, before the tone curve. */
private const val DARKEST_LIT = 0.3
/** Lit level the body dims to when the glyph has plenty of detail lighter than it. */
private const val DIMMEST_BODY = 0.6
/** Lightness quantile that must be clearly above the body for the body to dim: about the lightest 10%. */
private const val LIGHT_DETAIL_QUANTILE = 0.9
/** OKLab lightness above the body where light detail starts to dim the body, and the span to fully dimmed. */
private const val LIGHT_DETAIL_START = 0.05
private const val LIGHT_DETAIL_SPAN = 0.2
/**
* The smallest lightness spans the ramps above and below the body stretch over, so faint
* shading on a nearly one-color glyph stays faint instead of stretching to the full range.
*/
private const val MIN_LIGHT_SPAN = 0.15
private const val MIN_DARK_SPAN = 0.3
/**
* Whether a glyph drawn in white came out in other colors, i.e. came from a color font.
* Ordinary glyphs are white wherever they have coverage.
*/
@JvmStatic
fun isColor(argb: IntArray): Boolean {
val threshold = 255 - WHITE_TOLERANCE
for (p in argb) {
if (alpha(p) < MIN_COLOR_ALPHA) continue
if (((p shr 16) and 0xff) < threshold || ((p shr 8) and 0xff) < threshold || (p and 0xff) < threshold) {
return true
}
}
return false
}
/**
* Coverage (one byte per pixel) of a glyph drawn in white as unpremultiplied ARGB: its alpha,
* or for a color glyph its [shade].
*/
@JvmStatic
fun coverage(argb: IntArray): ByteArray =
if (isColor(argb)) shade(argb) else ByteArray(argb.size) { alpha(argb[it]).toByte() }
/** Lit-on-black shade (one byte per pixel) of a color glyph as unpremultiplied ARGB. */
@JvmStatic
fun shade(argb: IntArray): ByteArray {
var peak = 0
for (p in argb) peak = max(peak, alpha(p))
val out = ByteArray(argb.size)
if (peak == 0) return out
// Statistics over the solid part, so antialiased edges don't skew the body.
val cutoff = max(1, (peak + 1) / 2)
var solid = 0
for (p in argb) if (alpha(p) >= cutoff) solid += 1
val sorted = DoubleArray(solid)
var next = 0
for (p in argb) if (alpha(p) >= cutoff) sorted[next++] = Oklab.lightness(p)
sorted.sort()
val body = quantile(sorted, 0.5)
val darkest = quantile(sorted, 0.05)
val lightest = quantile(sorted, 0.95)
val lightDetail = quantile(sorted, LIGHT_DETAIL_QUANTILE) - body
val dimming = ((lightDetail - LIGHT_DETAIL_START) / LIGHT_DETAIL_SPAN).coerceIn(0.0, 1.0)
val bodyLit = 1.0 - (1.0 - DIMMEST_BODY) * dimming
val lightSpan = max(lightest - body, MIN_LIGHT_SPAN)
val darkSpan = max(body - darkest, MIN_DARK_SPAN)
for (i in argb.indices) {
val p = argb[i]
val a = alpha(p)
if (a == 0) continue
val l = Oklab.lightness(p)
val lit = if (l >= body) {
bodyLit + (1.0 - bodyLit) * ((l - body) / lightSpan).coerceAtMost(1.0)
} else {
bodyLit - (bodyLit - DARKEST_LIT) * ((body - l) / darkSpan).coerceAtMost(1.0)
}
out[i] = (a * lit.pow(NotificationIconArt.GAMMA)).roundToInt().coerceIn(0, 255).toByte()
}
return out
}
private fun quantile(sorted: DoubleArray, q: Double): Double = sorted[((sorted.size - 1) * q).roundToInt()]
private fun alpha(p: Int): Int = (p ushr 24) and 0xff
}
@@ -2,7 +2,6 @@ package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.abs
import kotlin.math.cbrt
import kotlin.math.max
import kotlin.math.min
import kotlin.math.pow
@@ -72,12 +71,6 @@ object NotificationIconArt {
class Art(val kind: Kind, val gray: ByteArray)
/** sRGB byte value to linear light. */
private val SRGB_TO_LINEAR = DoubleArray(256) { value ->
val c = value / 255.0
if (c <= 0.04045) c / 12.92 else ((c + 0.055) / 1.055).pow(2.4)
}
/**
* Classify and convert an icon rendered at (outputSize * ANALYSIS_SCALE) square, row-major
* ARGB (not premultiplied).
@@ -232,7 +225,7 @@ object NotificationIconArt {
private fun readTile(argb: IntArray, mask: BooleanArray, size: Int, maxAlpha: Int, outputSize: Int): Art {
val lab = DoubleArray(argb.size * 3)
for (i in argb.indices) {
if (mask[i]) toOklab(argb[i], lab, i * 3)
if (mask[i]) Oklab.toOklab(argb[i], lab, i * 3)
}
// The background is the color of the tile's rim, about one output pixel deep.
@@ -347,18 +340,6 @@ object NotificationIconArt {
return gray
}
private fun toOklab(pixel: Int, out: DoubleArray, offset: Int) {
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
val b = SRGB_TO_LINEAR[pixel and 0xff]
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
out[offset] = 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
out[offset + 1] = 1.9779984951 * l - 2.4285922050 * m + 0.4505937099 * s
out[offset + 2] = 0.0259040371 * l + 0.7827717662 * m - 0.8086757660 * s
}
private fun distance(lab: DoubleArray, index: Int, background: DoubleArray): Double {
val dl = lab[index * 3] - background[0]
val da = lab[index * 3 + 1] - background[1]
@@ -0,0 +1,40 @@
package com.faceclaw.app
import kotlin.math.cbrt
import kotlin.math.pow
/**
* OKLab, a perceptual color space: L is lightness (0 = black, 1 = white) and equal distances
* look about equally different. The color analysis of icons and color glyphs works in it.
*/
internal object Oklab {
/** sRGB byte value to linear light. */
private val SRGB_TO_LINEAR = DoubleArray(256) { value ->
val c = value / 255.0
if (c <= 0.04045) c / 12.92 else ((c + 0.055) / 1.055).pow(2.4)
}
/** Writes the (L, a, b) of an ARGB pixel's color (its alpha ignored) at out[offset]. */
fun toOklab(pixel: Int, out: DoubleArray, offset: Int) {
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
val b = SRGB_TO_LINEAR[pixel and 0xff]
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
out[offset] = 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
out[offset + 1] = 1.9779984951 * l - 2.4285922050 * m + 0.4505937099 * s
out[offset + 2] = 0.0259040371 * l + 0.7827717662 * m - 0.8086757660 * s
}
/** The OKLab lightness of an ARGB pixel's color (its alpha ignored). */
fun lightness(pixel: Int): Double {
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
val b = SRGB_TO_LINEAR[pixel and 0xff]
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
return 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
}
}
@@ -64,6 +64,21 @@ class ShellScene(val layers: List<Layer>, val selections: List<RetainedDrawing>
return output
}
/** Whether the lenses differ: stereo depth somewhere in the scene. */
val hasStereoDepth: Boolean get() = screenDepth != 0 || layers.any { it.depth != 0 } || allSelections.isNotEmpty()
/**
* What one lens shows of [screen] (packed 4bpp) with this scene over it and every animation settled, for
* glasses that have no display lists (a G1). [screen] itself when there is no scene.
*/
fun composed(screen: ByteArray, width: Int, height: Int, rightLens: Boolean): ByteArray {
if (layers.isEmpty() && selections.isEmpty() && screenDepth == 0) return screen
val output = ByteArray(screen.size)
DisplayListRenderer(rendererResources(width, height), rightLens = rightLens).render(511,
DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height), SETTLED_MS)
return output
}
/** Layer surfaces, then retained resources, by index, and the root list at 511. */
private fun rendererResources(width: Int, height: Int): HashMap<Int, ByteArray> {
val resources = HashMap<Int, ByteArray>()
@@ -0,0 +1,540 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/**
* RayNeo iO wire codecs (stock link plus the Faceclaw custom-firmware link). Pure functions
* and small value types; no I/O, threads or platform APIs.
*
* The iO is one SoC driving both eyes over ONE BLE connection. Everything travels in AA55
* frames on one write characteristic and one notify characteristic; the frame's "business"
* byte selects the subsystem: 0x10 handshake (TLV), 9 OTA and 13..22 apps (protobuf-style
* envelope + JSON), 0xF0 the custom-firmware link. The CFW link carries Faceclaw's G2 records
* unchanged (see [CfwTransport.encodeRecord]) inside `[opt][stream][chunk]` packets, and the
* firmware answers with the G2's 9-byte ACK body, one ACK per targeted lens.
*
* Protocol knowledge from the MIT-licensed Glassly SDK and CFW (tetramo-labs/glassly,
* tetramo-labs/glassly-rayneo-cfw); the Faceclaw firmware fork documents its additions in
* faceclaw-rayneo-cfw/docs/FACECLAW.md.
*/
object RayNeoIoBle {
const val SERVICE_UUID = "0000b81d-0000-1000-8000-00805f9b34fb"
/** Phone -> glasses, write without response. */
const val WRITE_CHAR_UUID = "ea8b70d5-2bd3-49ab-9c31-9c38b2c3c4f9"
/** Glasses -> phone, notifications. */
const val NOTIFY_CHAR_UUID = "7db3e235-3608-41f3-a03c-955fcbd2ea4b"
const val REQUESTED_MTU = 512
/** Advertised manufacturer data company id; the payload is `[07][MAC x6]...` (not in pairing mode). */
const val MANUFACTURER_ID = 0x5716
@JvmStatic fun matchesName(name: String?): Boolean = name != null && name.contains("rayneo", ignoreCase = true)
/** The glasses' MAC ("00:6A:5E:5F:33:C2") from manufacturer data, or null. */
@JvmStatic
fun macFromManufacturerData(data: ByteArray?): String? {
if (data == null || data.size < 7 || data[0].toInt() != 7) return null
return (1..6).joinToString(":") { i -> (data[i].toInt() and 255).toString(16).padStart(2, '0').uppercase() }
}
}
object RayNeoIoProtocol {
const val BUSINESS_OTA = 9
const val BUSINESS_VOICE = 13
const val BUSINESS_LAUNCHER = 15
const val BUSINESS_HANDSHAKE = 0x10
const val BUSINESS_CFW = 0xf0
const val FRAME_OVERHEAD = 10
const val MIN_FRAME_BYTES = 10
/** Program constant of the stock app's guest authentication (not a secret). */
private val SALT = byteArrayOf(
0x07, 0x12, 0xc8.toByte(), 0xad.toByte(), 0x27, 0x74, 0x41, 0x95.toByte(),
0xde.toByte(), 0x5f, 0xa1.toByte(), 0xea.toByte(), 0x6d, 0x02, 0x5f, 0xb8.toByte(),
)
// ---- CRC-16/XMODEM (poly 0x1021, init 0) -----------------------------------------------
@JvmStatic
fun crc16Xmodem(bytes: ByteArray, from: Int = 0, to: Int = bytes.size): Int {
var crc = 0
for (i in from until to) {
crc = crc xor ((bytes[i].toInt() and 255) shl 8)
repeat(8) { crc = if (crc and 0x8000 != 0) ((crc shl 1) xor 0x1021) and 0xffff else (crc shl 1) and 0xffff }
}
return crc
}
// ---- AA55 frames --------------------------------------------------------------------------
class Frame(
@JvmField val messageNumber: Int,
@JvmField val packetType: Int,
@JvmField val business: Int,
@JvmField val payload: ByteArray,
)
/** `AA 55 [len u16 BE = total-6] [msg u16 BE] [packetType] [business] payload [crc16 BE of bytes 4..end-2]`. */
@JvmStatic
fun encodeFrame(messageNumber: Int, business: Int, payload: ByteArray, packetType: Int = 0): ByteArray {
val frame = ByteArray(FRAME_OVERHEAD + payload.size)
val length = frame.size - 6
frame[0] = 0xaa.toByte()
frame[1] = 0x55
frame[2] = (length ushr 8).toByte()
frame[3] = length.toByte()
frame[4] = (messageNumber ushr 8).toByte()
frame[5] = messageNumber.toByte()
frame[6] = packetType.toByte()
frame[7] = business.toByte()
payload.copyInto(frame, 8)
val crc = crc16Xmodem(frame, 4, frame.size - 2)
frame[frame.size - 2] = (crc ushr 8).toByte()
frame[frame.size - 1] = crc.toByte()
return frame
}
/** Split a frame into ATT writes of at most `mtu - 3` bytes. */
@JvmStatic
fun writes(frame: ByteArray, mtu: Int): List<ByteArray> {
val chunk = (mtu - 3).coerceAtLeast(20)
return (frame.indices step chunk).map { frame.copyOfRange(it, minOf(it + chunk, frame.size)) }
}
@JvmStatic
fun decodeFrame(bytes: ByteArray): Frame? {
if (bytes.size < MIN_FRAME_BYTES || bytes[0] != 0xaa.toByte() || bytes[1] != 0x55.toByte()) return null
val total = (((bytes[2].toInt() and 255) shl 8) or (bytes[3].toInt() and 255)) + 6
if (total != bytes.size) return null
val crc = ((bytes[total - 2].toInt() and 255) shl 8) or (bytes[total - 1].toInt() and 255)
if (crc != crc16Xmodem(bytes, 4, total - 2)) return null
val number = ((bytes[4].toInt() and 255) shl 8) or (bytes[5].toInt() and 255)
return Frame(number, bytes[6].toInt() and 255, bytes[7].toInt() and 255, bytes.copyOfRange(8, total - 2))
}
/**
* Frames from notification chunks: a frame may span several notifications. Bytes before a
* frame head are skipped; a head whose CRC fails is skipped past its magic.
*/
class FrameReassembler {
private var buffer = ByteArray(0)
fun reset() {
buffer = ByteArray(0)
}
fun ingest(chunk: ByteArray): List<Frame> {
buffer += chunk
val out = ArrayList<Frame>()
var offset = 0
while (true) {
while (offset + 1 < buffer.size && !(buffer[offset] == 0xaa.toByte() && buffer[offset + 1] == 0x55.toByte())) offset++
if (buffer.size - offset < 4) break
val total = (((buffer[offset + 2].toInt() and 255) shl 8) or (buffer[offset + 3].toInt() and 255)) + 6
if (total < MIN_FRAME_BYTES) { offset += 2; continue }
if (buffer.size - offset < total) break
val frame = decodeFrame(buffer.copyOfRange(offset, offset + total))
if (frame == null) { offset += 2; continue }
out.add(frame)
offset += total
}
buffer = buffer.copyOfRange(minOf(offset, buffer.size), buffer.size)
return out
}
val pendingBytes: Int get() = buffer.size
}
// ---- TLV (tag u8, length u16 BE) ------------------------------------------------------------
class Tlv(@JvmField val tag: Int, @JvmField val value: ByteArray)
@JvmStatic
fun tlv(tag: Int, value: ByteArray): ByteArray {
val out = ByteArray(3 + value.size)
out[0] = tag.toByte()
out[1] = (value.size ushr 8).toByte()
out[2] = value.size.toByte()
value.copyInto(out, 3)
return out
}
@JvmStatic
fun decodeTlvs(bytes: ByteArray): List<Tlv>? {
val out = ArrayList<Tlv>()
var i = 0
while (i < bytes.size) {
if (bytes.size - i < 3) return null
val len = ((bytes[i + 1].toInt() and 255) shl 8) or (bytes[i + 2].toInt() and 255)
if (bytes.size - i - 3 < len) return null
out.add(Tlv(bytes[i].toInt() and 255, bytes.copyOfRange(i + 3, i + 3 + len)))
i += 3 + len
}
return out
}
private fun List<Tlv>.value(tag: Int): ByteArray? = firstOrNull { it.tag == tag }?.value
// ---- Handshake (business 0x10) ----------------------------------------------------------------
const val PLATFORM_IOS = 1
const val PLATFORM_ANDROID = 2
/** Version request. `phoneId` is a stable 6-byte protocol identity the glasses bind to. */
@JvmStatic
fun versionRequest(phoneId: ByteArray, phoneName: String, manufacturer: String, model: String, platform: Int = PLATFORM_ANDROID): ByteArray {
require(phoneId.size == 6)
val inner = tlv(0x10, byteArrayOf(1)) + tlv(0x11, phoneId) + tlv(0x12, phoneName.encodeToByteArray()) +
tlv(0x13, byteArrayOf(platform.toByte())) + tlv(0x16, manufacturer.encodeToByteArray()) +
tlv(0x17, model.encodeToByteArray())
return tlv(0x11, inner)
}
class VersionReply(
@JvmField val protocolVersion: Int,
/** The glasses' Bluetooth MAC, which the auth proof covers. */
@JvmField val glassesId: ByteArray,
@JvmField val serial: String?,
/** Tags 0x19/0x1a: bound to another phone's account; guest auth will not work. */
@JvmField val accountBound: Boolean,
)
@JvmStatic
fun parseVersionReply(payload: ByteArray): VersionReply? {
val body = decodeTlvs(payload)?.value(0x11) ?: return null
val fields = decodeTlvs(body) ?: return null
val id = fields.value(0x11)?.takeIf { it.size == 6 } ?: return null
return VersionReply(
fields.value(0x10)?.firstOrNull()?.toInt()?.and(255) ?: 0,
id,
fields.value(0x14)?.decodeToString()?.trimEnd('\u0000'),
fields.value(0x19) != null || fields.value(0x1a) != null,
)
}
/** `SHA-256(random4 ‖ phoneId6 ‖ glassesId6 ‖ SALT)`, both directions. */
@JvmStatic
fun authProof(random: ByteArray, phoneId: ByteArray, glassesId: ByteArray): ByteArray {
val digest = Sha256Digest()
digest.update(random)
digest.update(phoneId)
digest.update(glassesId)
digest.update(SALT)
return digest.digest()
}
@JvmStatic
fun guestAuthRequest(random: ByteArray, phoneId: ByteArray, glassesId: ByteArray): ByteArray {
require(random.size == 4 && phoneId.size == 6 && glassesId.size == 6)
return tlv(0x18, tlv(0x10, random) + tlv(0x11, authProof(random, phoneId, glassesId)))
}
/** The glasses' answer proves they know the same phone and glasses ids. */
@JvmStatic
fun verifyGuestAuthReply(payload: ByteArray, phoneId: ByteArray, glassesId: ByteArray): Boolean {
val body = decodeTlvs(payload)?.value(0x19) ?: return false
val fields = decodeTlvs(body) ?: return false
val random = fields.value(0x10)?.takeIf { it.size == 4 } ?: return false
val proof = fields.value(0x11)?.takeIf { it.size == 32 } ?: return false
return proof.contentEquals(authProof(random, phoneId, glassesId))
}
/** True for a guest-auth reply (outer tag 0x19), valid or not. */
@JvmStatic fun isGuestAuthReply(payload: ByteArray): Boolean = decodeTlvs(payload)?.value(0x19) != null
// ---- Business envelope (9, 13..22): 08 version, 10 type, 1A json, 22 bytes, 28 seq, 30 mode ----
const val MODE_REQUEST = 1
const val MODE_REPLY = 2
const val LAUNCHER_GENERAL_STATUS = 1
const val LAUNCHER_BRIGHTNESS = 2
const val LAUNCHER_GENERAL_SETTINGS = 4
const val LAUNCHER_AI_VOICE_WAKEUP = 16
const val OTA_VERSION = 1
const val GENERAL_STATUS_REQUEST = """{"cmd":"request_general_status","payload":{"data":"","mode":0,"value":0}}"""
/** The reply (type 4) carries `generalSettings`, among them the wake-word preferences. */
const val GENERAL_SETTINGS_REQUEST = """{"cmd":"request_general_settings","payload":{"value":0,"mode":0,"data":""}}"""
/**
* The launcher's wake-word switch. Every `set_ai_*` handler takes the new value from
* `mode` and ignores `value` (RnLinkMsgProcessor::handleAssistantSettingCommand, see
* notes/rayneo-io-wakeword-re.md): mode 1 on, 0 off.
*/
@JvmStatic
fun launcherSetAiVoiceWakeup(enabled: Boolean): String =
"""{"cmd":"set_ai_voice_wakeup","payload":{"value":0,"mode":${if (enabled) 1 else 0},"data":""}}"""
// ---- Voice assistant (business 13), as Glassly's RayNeoIOProtocol maps it ----
// The stock assistant: the wake word (or a temple long/double press) opens a session
// and sends VOICE_WAKE; the phone answers VOICE_RC {"rc":1} for Opus as VOICE_AUDIO,
// or {"rc":2} plus VOICE_ABORT to end it; VOICE_EXITED reports it gone. The remote
// stream (VOICE_STREAM_CONTROL {"rc":1|2}, packets as VOICE_STREAM_AUDIO) needs
// IDENT_FLAG_MIC firmware and runs without a session or its page, until a session,
// a call, SCO or a recording takes the recorder. Packets are 16 kHz mono Opus in
// the envelope's bytes field.
const val VOICE_WAKE = 1
const val VOICE_RC = 2
const val VOICE_AUDIO = 3
const val VOICE_VAD = 4
const val VOICE_ABORT = 7
const val VOICE_EXITED = 8
const val VOICE_STREAM_CONTROL = 58
const val VOICE_STREAM_AUDIO = 59
const val VOICE_RC_START = 1
const val VOICE_RC_STOP = 2
const val VOICE_SAMPLE_RATE = 16_000
/** A voice-channel message: `{"rc":n}` as [type] (VOICE_RC, VOICE_STREAM_CONTROL or VOICE_ABORT). */
@JvmStatic
fun voiceRc(type: Int, rc: Int): ByteArray = encodeEnvelope(Envelope(type, """{"rc":$rc}"""))
class Envelope(
@JvmField val type: Int,
@JvmField val json: String,
@JvmField val binary: ByteArray? = null,
@JvmField val sequence: Int? = null,
@JvmField val mode: Int? = null,
)
private fun varint(out: MutableList<Byte>, value: Long) {
var v = value
while (v and 0x7fL.inv() != 0L) { out.add(((v and 0x7f) or 0x80).toByte()); v = v ushr 7 }
out.add(v.toByte())
}
@JvmStatic
fun encodeEnvelope(e: Envelope): ByteArray {
val out = ArrayList<Byte>()
out.add(0x08); varint(out, 1)
out.add(0x10); varint(out, e.type.toLong())
val json = e.json.encodeToByteArray()
out.add(0x1a); varint(out, json.size.toLong()); json.forEach { out.add(it) }
e.binary?.let { b -> out.add(0x22); varint(out, b.size.toLong()); b.forEach { out.add(it) } }
e.sequence?.let { out.add(0x28); varint(out, it.toLong()) }
e.mode?.let { out.add(0x30); varint(out, it.toLong()) }
return out.toByteArray()
}
@JvmStatic
fun decodeEnvelope(bytes: ByteArray): Envelope? {
var pos = 0
fun readVarint(): Long? {
var shift = 0
var result = 0L
while (pos < bytes.size && shift <= 63) {
val b = bytes[pos++].toInt() and 255
result = result or ((b and 0x7f).toLong() shl shift)
if (b and 0x80 == 0) return result
shift += 7
}
return null
}
var type: Int? = null
var json = ""
var binary: ByteArray? = null
var sequence: Int? = null
var mode: Int? = null
while (pos < bytes.size) {
val key = readVarint() ?: return null
when ((key and 7).toInt()) {
0 -> {
val v = readVarint() ?: return null
when ((key ushr 3).toInt()) { 2 -> type = v.toInt(); 5 -> sequence = v.toInt(); 6 -> mode = v.toInt() }
}
2 -> {
val len = readVarint() ?: return null
if (len < 0 || len > bytes.size - pos) return null
val v = bytes.copyOfRange(pos, pos + len.toInt())
pos += len.toInt()
when ((key ushr 3).toInt()) { 3 -> json = v.decodeToString(); 4 -> binary = v }
}
1 -> pos += 8
5 -> pos += 4
else -> return null
}
}
return Envelope(type ?: return null, json, binary, sequence, mode)
}
/**
* Set the launcher's brightness (1..8). The stock handler also turns the screen on
* (BrightnessMonitor::setBrightness -> turnOnScreenForSetting), which is how the phone
* wakes the iO's display and keeps the stock auto-off from blanking a static page.
* Only for firmware without [Ident.brightnessControl]: stock auto-brightness soon
* replaces the level, and each one writes the glasses' preferences.
*/
@JvmStatic
fun launcherBrightness(level: Int): String =
"""{"cmd":"brightness_change","payload":{"value":${level.coerceIn(1, 8)},"mode":0,"data":""}}"""
/** Faceclaw's 0..100 brightness target on the launcher's 1..8 scale. */
@JvmStatic fun brightnessLevel(target: Int): Int = maxOf(1, (target.coerceIn(0, 100) * 8 + 50) / 100)
/** `screen_status` push value (1 on, 0 off), or -1 for anything else. */
@JvmStatic
fun screenStatusFromLauncherJson(json: String): Int {
val root = try { Json.parse(json).asObject() } catch (_: Exception) { null } ?: return -1
if (root["cmd"].asString() != "screen_status") return -1
return root["payload"].asObject()?.get("value").asInt() ?: -1
}
/** `generalStatus.battery` from a launcher reply or push, or -1. */
@JvmStatic
fun batteryFromLauncherJson(json: String): Int {
val root = try { Json.parse(json).asObject() } catch (_: Exception) { null } ?: return -1
val status = root["generalStatus"].asObject() ?: root["payload"].asObject()?.get("generalStatus").asObject()
status?.get("battery").asInt()?.let { return it }
// pushes: {"cmd":"battery_change","payload":{"value":N,...}}
if (root["cmd"].asString() == "battery_change") return root["payload"].asObject()?.get("value").asInt() ?: -1
return -1
}
// ---- Custom-firmware link (business 0xF0) -----------------------------------------------------
const val OPT_LENS = 0x01
const val OPT_CONTROL = 0x10
const val OPT_END = 0x40
const val OPT_RESET = 0x80
const val CTRL_QUERY = 1
const val CTRL_PAGE_OPEN = 2
const val CTRL_PAGE_CLOSE = 3
const val CTRL_KEEPALIVE = 4
/** Records up to this many bytes per packet (IDENT reports the firmware's maxChunk). */
const val DEFAULT_MAX_CHUNK = 240
/** The page closes this long after the last PAGE_OPEN/KEEPALIVE. */
const val PAGE_LEASE_MS = 120_000L
@JvmStatic
fun controlPacket(ctrl: Int, arg: Int? = null): ByteArray =
if (arg == null) byteArrayOf(OPT_CONTROL.toByte(), 0, ctrl.toByte())
else byteArrayOf(OPT_CONTROL.toByte(), 0, ctrl.toByte(), arg.toByte())
/**
* One record (from [CfwTransport.encodeRecord]) as business-0xF0 payloads
* `[opt][stream][chunk]`. `stream` is echoed in the ACKs, so it plays the G2 stream id's role.
*/
@JvmStatic
fun cfwPackets(record: ByteArray, streamId: Int, maxChunk: Int = DEFAULT_MAX_CHUNK): List<ByteArray> {
require(streamId in 0..255 && maxChunk > 0)
val out = ArrayList<ByteArray>()
var offset = 0
do {
val end = minOf(offset + maxChunk, record.size)
var opt = OPT_LENS
if (offset == 0) opt = opt or OPT_RESET
if (end >= record.size) opt = opt or OPT_END
val p = ByteArray(2 + end - offset)
p[0] = opt.toByte()
p[1] = streamId.toByte()
record.copyInto(p, 2, offset, end)
out.add(p)
offset = end
} while (offset < record.size)
return out
}
const val KIND_ACK = 1
const val KIND_NACK = 3
const val KIND_REPLY = 5
const val KIND_GESTURE = 7
const val KIND_IDENT = 0x11
const val KIND_PAGE = 0x13
const val KIND_DIAG = 0x15
const val KIND_ALS = 0x1f
/** `[01|03][stream][ordinal u16][lens][size u16][crc u16]`: the G2 ACK body, one per lens. */
@JvmStatic
fun parseCfwAck(payload: ByteArray): CfwTransport.Ack? {
if (payload.size != 9) return null
val kind = payload[0].toInt() and 255
val lens = payload[4].toInt() and 255
if ((kind != KIND_ACK && kind != KIND_NACK) || (lens != 1 && lens != 2)) return null
fun u16(o: Int) = (payload[o].toInt() and 255) or ((payload[o + 1].toInt() and 255) shl 8)
return CfwTransport.Ack(kind == KIND_NACK, payload[1].toInt() and 255, u16(2), lens, u16(5), u16(7))
}
const val IDENT_FLAG_ZLIB = 0x01
const val IDENT_FLAG_STEREO = 0x08
/** The assistant's remote audio stream (type 58/59 below) is allowed (CFW revision 10). */
const val IDENT_FLAG_MIC = 0x20
const val IDENT_FLAG_FACECLAW = 0x80
/** Message 30: the firmware sets the panel level and runs fades, with stock auto-brightness off. */
const val FEATURE_BRIGHTNESS = "bright30"
/** Message 16: the firmware reads the light sensor and sends [KIND_ALS] reports. */
const val FEATURE_ALS = "als16"
class Ident(
@JvmField val width: Int,
@JvmField val height: Int,
@JvmField val maxChunk: Int,
@JvmField val revision: Int,
@JvmField val flags: Int,
@JvmField val name: String,
/** "Faceclaw/<n>" -> n when the firmware runs the Faceclaw display lists, else -1. */
@JvmField val faceclawRevision: Int,
/** What the firmware adds to the display lists (a third string in IDENT). */
@JvmField val features: List<String> = emptyList(),
) {
val acceptsCompressed: Boolean get() = flags and IDENT_FLAG_ZLIB != 0
val micStream: Boolean get() = flags and IDENT_FLAG_MIC != 0
val faceclaw: Boolean get() = flags and IDENT_FLAG_FACECLAW != 0 && faceclawRevision > 0
/** Faceclaw owns brightness and the light sensor, as on the G2 custom firmware. */
val brightnessControl: Boolean get() = faceclaw && FEATURE_BRIGHTNESS in features && FEATURE_ALS in features
}
/** `[11][ver][w u16][h u16][maxChunk u16][storeKiB u16][rev u16][flags]name\0[lineHeight]["Faceclaw/n"\0["features"\0]]`. */
@JvmStatic
fun parseIdent(payload: ByteArray): Ident? {
if (payload.size < 13 || payload[0].toInt() != KIND_IDENT) return null
fun u16(o: Int) = (payload[o].toInt() and 255) or ((payload[o + 1].toInt() and 255) shl 8)
var end = 13
while (end < payload.size && payload[end].toInt() != 0) end++
val name = payload.copyOfRange(13, end).decodeToString()
var faceclaw = -1
var features = emptyList<String>()
val extra = end + 2 // NUL, line height
if (extra < payload.size) {
var e = extra
while (e < payload.size && payload[e].toInt() != 0) e++
val text = payload.copyOfRange(extra, e).decodeToString()
if (text.startsWith("Faceclaw/")) faceclaw = text.removePrefix("Faceclaw/").toIntOrNull() ?: -1
val more = e + 1
if (more < payload.size) {
var f = more
while (f < payload.size && payload[f].toInt() != 0) f++
features = payload.copyOfRange(more, f).decodeToString().split(' ').filter { it.isNotEmpty() }
}
}
return Ident(u16(2), u16(4), u16(6), u16(10), payload[12].toInt() and 255, name, faceclaw, features)
}
/**
* `[1f]` + the 24-byte `['A','L',1,reason,...]` body the G2 custom firmware sends as settings
* field 105 (value in tenths of a lux), or null. The body goes to the same consumers.
*/
@JvmStatic
fun parseAlsReport(payload: ByteArray): ByteArray? {
if (payload.size < 25 || (payload[0].toInt() and 255) != KIND_ALS) return null
val body = payload.copyOfRange(1, 25)
return if (body[0].toInt() == 65 && body[1].toInt() == 76 && body[2].toInt() == 1) body else null
}
/** PAGE `[13][state][reason]`: state 1 open. */
@JvmStatic fun parsePageState(payload: ByteArray): Int = if (payload.size >= 2 && payload[0].toInt() == KIND_PAGE) payload[1].toInt() and 255 else -1
class Gesture(@JvmField val eventType: Int, @JvmField val source: Int, @JvmField val tick: Long)
/**
* GESTURE `[07][eventType][source][tick u32]`. Event types use the G2's OsEventTypeList numbering:
* 0 single tap, 1 swipe up, 2 swipe down, 3 double tap, 11 tap-then-hold, 12 head up, 15 head down.
*/
@JvmStatic
fun parseGesture(payload: ByteArray): Gesture? {
if (payload.size < 7 || payload[0].toInt() != KIND_GESTURE) return null
val tick = (payload[3].toLong() and 255) or ((payload[4].toLong() and 255) shl 8) or
((payload[5].toLong() and 255) shl 16) or ((payload[6].toLong() and 255) shl 24)
return Gesture(payload[1].toInt() and 255, payload[2].toInt() and 255, tick)
}
}
@@ -0,0 +1,37 @@
package com.faceclaw.app
import platform.Foundation.NSData
/**
* Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's
* captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record:
* 16 kHz mono PCM16, as its recognizer hands it to TypeScript.
*/
class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) {
/** The recording stopped early: the file could not be written. */
val failed: Boolean get() = writer.failed
/** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */
fun append(pcm16le: NSData) {
val bytes = pcm16le.byteArray()
val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) }
writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs()))
}
/** Completes the file. Returns the recording's length in ms. */
fun finish(): Long = writer.finish()
}
object IosMicRecordings {
/**
* A recorder writing a new file at [path], or null when the file cannot be created.
* [configJson] describes the capture and is stored in the file.
*/
fun open(path: String, configJson: String?): IosMicRecorder? =
try {
IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform))
} catch (error: Throwable) {
PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}")
null
}
}
@@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped {
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
actual fun openFileOutput(path: String): SeekableOutput {
val file = fopen(path, "wb+") ?: error("Cannot write $path")
return object : SeekableOutput {
override var size = 0L
private set
private fun put(bytes: ByteArray, offset: Int, length: Int) {
if (length == 0) return
bytes.usePinned {
check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert<platform.posix.size_t>()) { "Short write to $path" }
}
}
override fun write(bytes: ByteArray, offset: Int, length: Int) {
put(bytes, offset, length)
size += length
// Each write is a complete element; hand it to the OS so a crash keeps it.
platform.posix.fflush(file)
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0)
put(bytes, 0, bytes.size)
check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0)
platform.posix.fflush(file)
}
override fun close() {
fclose(file)
}
}
}
actual fun openFileInput(path: String): SeekableInput {
val file = fopen(path, "rb") ?: error("Cannot read $path")
val fileSize = fileLength(path)
return object : SeekableInput {
override val length = fileSize
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0
return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() }
}
override fun close() {
fclose(file)
}
}
}
@@ -461,6 +461,20 @@ class IosBleCentral(private val platform: ProtocolPlatform = IosProtocolPlatform
override fun isBonded(address: String): Boolean = true
/** CoreBluetooth pairs by itself when an attribute needs it. */
override fun requestBond(address: String) {}
/** Only iOS Settings can remove a pairing. */
override fun removeBond(address: String): Boolean = false
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean {
val peer = connectedPeer(address) ?: return false
return cond.withLock { peer.characteristics.containsKey(characteristicUuid.lowercase()) }
}
/** CoreBluetooth follows Service Changed itself; discovering again is all there is. */
override fun refreshServices(address: String, timeoutMs: Int): Boolean = discoverServices(address, timeoutMs)
override fun prepareBenchmarkLink(address: String, mode: Int) {}
override fun recordDisplayFrameSent() {
@@ -34,6 +34,11 @@ interface IosMicStatusListener {
fun onNsDataMicStatus(body: NSData, arm: String)
}
/** CFW keyboard-bridge records (the raw field-131 record) as NSData. Main queue. */
interface IosKeyboardEventListener {
fun onNsDataKeyboardEvent(body: NSData)
}
/**
* ObjC-facing facade over the shared [GlassesSessionCore] for the NativeScript app: the iOS
* counterpart of Android's FaceclawBleCommunicator with the same method names. GATT goes
@@ -86,6 +91,9 @@ class IosGlassesSession internal constructor(
fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision)
/** Drive an Even G1 on g1v0 (see GlassesSessionG1.kt). Before [start]. */
fun useG1() = core.useG1()
fun start(): Boolean = core.start()
fun disconnect() = core.disconnect()
@@ -105,6 +113,8 @@ class IosGlassesSession internal constructor(
fun isAncsAuthorized(): Boolean = central?.isAncsAuthorized(rightIdentifier) ?: false
fun ancsRelayAvailable(): Boolean = core.ancsRelayAvailable()
fun setAncsListener(listener: IosAncsListener?) {
if (listener == null) {
core.setRawFrameTap(null)
@@ -225,6 +235,20 @@ class IosGlassesSession internal constructor(
return adapter
}
fun addKeyboardEventListenerNsData(listener: IosKeyboardEventListener): FaceclawKeyboardEventListener {
val adapter = object : FaceclawKeyboardEventListener {
override fun onKeyboardEvent(body: ByteArray?) =
listener.onNsDataKeyboardEvent((body ?: ByteArray(0)).data())
}
core.addKeyboardEventListener(adapter)
return adapter
}
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
fun sendFaceclawKeyboardControl(record: NSData?, label: String): Boolean =
core.sendFaceclawKeyboardControl(record?.byteArray(), label)
/** Every audio-characteristic packet, whether or not a capture is active (passive; sends nothing). */
fun addAudioMonitorListenerNsData(listener: IosAudioPacketListener): FaceclawAudioPacketListener {
val adapter = object : FaceclawAudioPacketListener {
@@ -171,3 +171,57 @@ class IosFirmwareFlasher(rightAddress: String, leftAddress: String, firmwarePath
lock.withLock { listener }?.let(action)
}
}
/** iOS host for the shared [G1ArmFlashFlow]: one G1 arm's install, its restore to Even's firmware, or a question about what it runs. */
class IosG1Flasher(address: String, side: String, firmwarePath: String, netCorePath: String, mode: String, connectWindowMs: Int) {
private val link = IosBleCentral()
private val lock = IosProtocolPlatform.createLock()
private var started = false
private var listener: FaceclawFirmwareFlasherListener? = null
private val flow = G1ArmFlashFlow(
link,
address,
side,
firmwarePath,
netCorePath,
G1ArmFlashFlow.mode(mode),
connectWindowMs,
object : FaceclawFirmwareFlasherListener {
override fun onLog(line: String?) {
PlatformLog.i(G1ArmFlashFlow.TAG, line ?: "")
post { it.onLog(line) }
}
override fun onProgress(lens: String?, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int, bytesSent: Long, bytesTotal: Long) =
post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
},
)
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
lock.withLock { this.listener = listener }
}
fun start() {
lock.withLock {
if (started) return
started = true
}
runOnThread("faceclaw-g1-flash") { flow.run() }
}
fun cancel() = flow.cancel()
fun close() {
cancel()
runCatching { link.close() }
}
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) = main {
lock.withLock { listener }?.let(action)
}
}
@@ -0,0 +1,28 @@
@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class)
package com.faceclaw.app
import platform.Foundation.NSData
/** ColorGlyphArt for FaceclawFontRenderer.m, which draws color glyphs with CoreGraphics. */
object IosColorGlyph {
/**
* The shade (one byte per pixel) of a color glyph drawn as premultiplied RGBA bytes, as a
* kCGImageAlphaPremultipliedLast context lays them out. Empty data for a malformed buffer
* rather than an exception, which would terminate the app on its way into Objective-C.
*/
fun shade(rgba: NSData): NSData {
val bytes = rgba.byteArray()
if (bytes.isEmpty() || bytes.size % 4 != 0) return NSData()
val argb = IntArray(bytes.size / 4) { i ->
val a = bytes[i * 4 + 3].toInt() and 0xff
fun channel(offset: Int): Int {
if (a == 0) return 0
val premultiplied = bytes[i * 4 + offset].toInt() and 0xff
return ((premultiplied * 255 + a / 2) / a).coerceAtMost(255)
}
(a shl 24) or (channel(0) shl 16) or (channel(1) shl 8) or channel(2)
}
return ColorGlyphArt.shade(argb).data()
}
}