forked from Blendman974/faceclaw
Merge branch 'main' into glanceboard-tilt-angle-and-glance-down
This commit is contained in:
commit
37d81d0308
416 files changed
+34629
-3885
No files matched your search
@@ -11,9 +11,10 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
|
||||
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
|
||||
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
|
||||
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
|
||||
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
|
||||
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
|
||||
GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
|
||||
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
|
||||
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
|
||||
`alarms/` (AlarmSchedule, AlarmRingingPolicy), `settings/SettingsChangeHub`,
|
||||
the stock-firmware flows over a `StockLink` GATT port (`g2protocol/StockLinkSession`,
|
||||
|
||||
@@ -16,7 +16,7 @@ kotlin {
|
||||
}
|
||||
android {
|
||||
namespace = "com.faceclaw.shared"
|
||||
compileSdk = 35
|
||||
compileSdk = 36
|
||||
minSdk = 24
|
||||
compilerOptions { jvmTarget.set(JvmTarget.JVM_17) }
|
||||
withHostTestBuilder {}.configure {}
|
||||
|
||||
+20
-10
@@ -40,6 +40,10 @@ constructor(
|
||||
private var file: RandomAccessFile?
|
||||
private var dataBytes = 0L
|
||||
|
||||
// The PCM bytes the header on disk accounts for. It is rewritten about once a second of
|
||||
// audio, so a process death mid-recording still leaves a playable file.
|
||||
private var headerDataBytes = 0L
|
||||
|
||||
init {
|
||||
val parent = File(path).parentFile
|
||||
if (parent != null && !parent.exists() && !parent.mkdirs())
|
||||
@@ -58,11 +62,26 @@ constructor(
|
||||
try {
|
||||
current.write(pcm16le)
|
||||
dataBytes += pcm16le.size
|
||||
if (dataBytes - headerDataBytes >= sampleRate * channels * 2L) writeHeader(current)
|
||||
} catch (e: IOException) {
|
||||
Log.w("FaceclawWavRecorder", "append failed", e)
|
||||
}
|
||||
}
|
||||
|
||||
/** Rewrite the header to cover the PCM written so far, leaving the position at the end. */
|
||||
private fun writeHeader(target: RandomAccessFile) {
|
||||
val header =
|
||||
BinaryEncoding.wavHeader(
|
||||
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
|
||||
sampleRate,
|
||||
channels,
|
||||
)
|
||||
target.seek(0)
|
||||
target.write(header)
|
||||
target.seek(header.size + dataBytes)
|
||||
headerDataBytes = dataBytes
|
||||
}
|
||||
|
||||
@Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L)
|
||||
|
||||
fun getPath(): String = path
|
||||
@@ -72,16 +91,7 @@ constructor(
|
||||
val current = file ?: return dataBytes
|
||||
file = null
|
||||
try {
|
||||
current.use {
|
||||
it.seek(0)
|
||||
it.write(
|
||||
BinaryEncoding.wavHeader(
|
||||
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
|
||||
sampleRate,
|
||||
channels,
|
||||
)
|
||||
)
|
||||
}
|
||||
current.use { writeHeader(it) }
|
||||
} catch (e: IOException) {
|
||||
Log.w("FaceclawWavRecorder", "finish failed", e)
|
||||
}
|
||||
|
||||
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
|
||||
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
|
||||
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
file.write(bytes, offset, length)
|
||||
size += length
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
file.seek(position)
|
||||
file.write(bytes)
|
||||
file.seek(size)
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
|
||||
private val file = java.io.RandomAccessFile(path, "r")
|
||||
override val length = file.length()
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
file.seek(position)
|
||||
var total = 0
|
||||
while (total < length) {
|
||||
val read = file.read(buffer, offset + total, length - total)
|
||||
if (read <= 0) break
|
||||
total += read
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
|
||||
expect fun deleteFile(path: String): Boolean
|
||||
|
||||
expect fun renameFile(from: String, to: String): Boolean
|
||||
|
||||
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
|
||||
interface SeekableOutput {
|
||||
/** Bytes appended so far. */
|
||||
val size: Long
|
||||
|
||||
/** Appends [length] bytes of [bytes] from [offset]. */
|
||||
fun write(bytes: ByteArray, offset: Int, length: Int)
|
||||
|
||||
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
|
||||
fun writeAt(position: Long, bytes: ByteArray)
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** A file read at arbitrary positions without loading it whole. */
|
||||
interface SeekableInput {
|
||||
val length: Long
|
||||
|
||||
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
|
||||
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
|
||||
expect fun openFileOutput(path: String): SeekableOutput
|
||||
|
||||
/** Opens [path] for reading. Throws when it cannot. */
|
||||
expect fun openFileInput(path: String): SeekableInput
|
||||
@@ -0,0 +1,764 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
/**
|
||||
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
|
||||
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
|
||||
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
|
||||
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
|
||||
*
|
||||
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
|
||||
* about a second each, cues and the segment size written when the recording finishes. A
|
||||
* recording cut short by a crash is still a valid live-style stream of everything up to its last
|
||||
* complete cluster, and the reader takes it as such.
|
||||
*/
|
||||
internal object MatroskaIds {
|
||||
const val EBML = 0x1A45DFA3
|
||||
const val EBML_VERSION = 0x4286
|
||||
const val EBML_READ_VERSION = 0x42F7
|
||||
const val EBML_MAX_ID_LENGTH = 0x42F2
|
||||
const val EBML_MAX_SIZE_LENGTH = 0x42F3
|
||||
const val DOC_TYPE = 0x4282
|
||||
const val DOC_TYPE_VERSION = 0x4287
|
||||
const val DOC_TYPE_READ_VERSION = 0x4285
|
||||
const val VOID = 0xEC
|
||||
const val SEGMENT = 0x18538067
|
||||
const val SEEK_HEAD = 0x114D9B74
|
||||
const val SEEK = 0x4DBB
|
||||
const val SEEK_ID = 0x53AB
|
||||
const val SEEK_POSITION = 0x53AC
|
||||
const val INFO = 0x1549A966
|
||||
const val TIMESTAMP_SCALE = 0x2AD7B1
|
||||
const val DURATION = 0x4489
|
||||
const val DATE_UTC = 0x4461
|
||||
const val TITLE = 0x7BA9
|
||||
const val MUXING_APP = 0x4D80
|
||||
const val WRITING_APP = 0x5741
|
||||
const val TRACKS = 0x1654AE6B
|
||||
const val TRACK_ENTRY = 0xAE
|
||||
const val TRACK_NUMBER = 0xD7
|
||||
const val TRACK_UID = 0x73C5
|
||||
const val TRACK_TYPE = 0x83
|
||||
const val FLAG_DEFAULT = 0x88
|
||||
const val FLAG_LACING = 0x9C
|
||||
const val NAME = 0x536E
|
||||
const val LANGUAGE = 0x22B59C
|
||||
const val CODEC_ID = 0x86
|
||||
const val AUDIO = 0xE1
|
||||
const val SAMPLING_FREQUENCY = 0xB5
|
||||
const val CHANNELS = 0x9F
|
||||
const val BIT_DEPTH = 0x6264
|
||||
const val ATTACHMENTS = 0x1941A469
|
||||
const val ATTACHED_FILE = 0x61A7
|
||||
const val FILE_NAME = 0x466E
|
||||
const val FILE_MIME_TYPE = 0x4660
|
||||
const val FILE_DATA = 0x465C
|
||||
const val FILE_UID = 0x46AE
|
||||
const val CLUSTER = 0x1F43B675
|
||||
const val TIMESTAMP = 0xE7
|
||||
const val SIMPLE_BLOCK = 0xA3
|
||||
const val BLOCK_GROUP = 0xA0
|
||||
const val BLOCK = 0xA1
|
||||
const val BLOCK_DURATION = 0x9B
|
||||
const val CUES = 0x1C53BB6B
|
||||
const val CUE_POINT = 0xBB
|
||||
const val CUE_TIME = 0xB3
|
||||
const val CUE_TRACK_POSITIONS = 0xB7
|
||||
const val CUE_TRACK = 0xF7
|
||||
const val CUE_CLUSTER_POSITION = 0xF1
|
||||
}
|
||||
|
||||
/** Builds EBML elements into a byte buffer. */
|
||||
internal class EbmlWriter(capacity: Int = 256) {
|
||||
private val sink = ByteSink(capacity)
|
||||
|
||||
fun size(): Int = sink.size()
|
||||
|
||||
fun bytes(): ByteArray = sink.toByteArray()
|
||||
|
||||
/** An element ID, written as given: its length marker is part of the value. */
|
||||
fun id(id: Int) {
|
||||
if (id > 0xFFFFFF) sink.write(id ushr 24)
|
||||
if (id > 0xFFFF) sink.write(id ushr 16)
|
||||
if (id > 0xFF) sink.write(id ushr 8)
|
||||
sink.write(id)
|
||||
}
|
||||
|
||||
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
|
||||
fun dataSize(size: Long, width: Int = 0) {
|
||||
var bytes = width
|
||||
if (bytes == 0) {
|
||||
bytes = 1
|
||||
// The all-ones value of each width means "unknown size".
|
||||
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
|
||||
}
|
||||
val marked = size or (1L shl (7 * bytes))
|
||||
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun uint(id: Int, value: Long) {
|
||||
var bytes = 1
|
||||
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
|
||||
id(id)
|
||||
dataSize(bytes.toLong())
|
||||
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
/** A fixed eight-byte integer (dates). */
|
||||
fun int64(id: Int, value: Long) {
|
||||
id(id)
|
||||
dataSize(8)
|
||||
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
|
||||
|
||||
fun string(id: Int, text: String) {
|
||||
val bytes = text.encodeToByteArray()
|
||||
binary(id, bytes, 0, bytes.size)
|
||||
}
|
||||
|
||||
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
|
||||
id(id)
|
||||
dataSize(length.toLong())
|
||||
sink.write(bytes, offset, length)
|
||||
}
|
||||
|
||||
fun raw(bytes: ByteArray) = sink.write(bytes)
|
||||
|
||||
fun raw(value: Int) = sink.write(value)
|
||||
|
||||
fun master(id: Int, build: EbmlWriter.() -> Unit) {
|
||||
val child = EbmlWriter()
|
||||
child.build()
|
||||
id(id)
|
||||
dataSize(child.size().toLong())
|
||||
sink.write(child.bytes())
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
|
||||
fun padded(element: ByteArray, reserved: Int): ByteArray? {
|
||||
val filler = reserved - element.size
|
||||
// A Void element is at least two bytes.
|
||||
if (filler < 0 || filler == 1) return null
|
||||
val out = element.copyOf(reserved)
|
||||
if (filler == 0) return out
|
||||
val at = element.size
|
||||
out[at] = MatroskaIds.VOID.toByte()
|
||||
if (filler < 9) {
|
||||
out[at + 1] = (0x80 or (filler - 2)).toByte()
|
||||
} else {
|
||||
val size = (filler - 9).toLong()
|
||||
out[at + 1] = 0x01
|
||||
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
|
||||
|
||||
/**
|
||||
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
|
||||
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
|
||||
* milliseconds from the start of the file. Not thread-safe.
|
||||
*/
|
||||
class MkaWriter(
|
||||
private val output: SeekableOutput,
|
||||
title: String?,
|
||||
/** When the recording began (Unix epoch ms), for the file's date. */
|
||||
dateUtcMs: Long,
|
||||
attachments: List<MkaAttachment>,
|
||||
) {
|
||||
companion object {
|
||||
const val CODEC_PCM = "A_PCM/INT/LIT"
|
||||
const val CODEC_TEXT = "S_TEXT/UTF8"
|
||||
const val TRACK_AUDIO = 2
|
||||
const val TRACK_SUBTITLE = 0x11
|
||||
const val APP_NAME = "Faceclaw"
|
||||
private const val SEEK_RESERVED = 128
|
||||
private const val TRACKS_RESERVED = 4096
|
||||
private const val CLUSTER_SPAN_MS = 1000L
|
||||
private const val CLUSTER_MAX_BYTES = 1 shl 20
|
||||
// Block timestamps are signed 16-bit offsets from their cluster's.
|
||||
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
|
||||
private const val DURATION_PATCH_CLUSTERS = 5
|
||||
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
|
||||
private const val DATE_EPOCH_MS = 978_307_200_000L
|
||||
}
|
||||
|
||||
private class Track(
|
||||
val number: Int,
|
||||
val type: Int,
|
||||
val name: String,
|
||||
val sampleRate: Int,
|
||||
val channels: Int,
|
||||
val preferred: Boolean,
|
||||
)
|
||||
|
||||
private class Cue(val clusterPosition: Long, val timeMs: Long)
|
||||
|
||||
private val tracks = ArrayList<Track>()
|
||||
private val cues = ArrayList<Cue>()
|
||||
private val segmentSizePosition: Long
|
||||
private val segmentStart: Long
|
||||
private val seekPosition: Long
|
||||
private val infoPosition: Long
|
||||
private val durationPosition: Long
|
||||
private val tracksPosition: Long
|
||||
private val attachmentsPosition: Long
|
||||
private var cluster: EbmlWriter? = null
|
||||
private var clusterBaseMs = 0L
|
||||
private var clustersSincePatch = 0
|
||||
private var finished = false
|
||||
private val hasAttachments = attachments.isNotEmpty()
|
||||
|
||||
/** The end of the latest audio or text written, in ms from the start of the file. */
|
||||
var durationMs = 0L
|
||||
private set
|
||||
|
||||
init {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.EBML) {
|
||||
uint(MatroskaIds.EBML_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_READ_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
|
||||
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
|
||||
string(MatroskaIds.DOC_TYPE, "matroska")
|
||||
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
|
||||
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
|
||||
}
|
||||
head.id(MatroskaIds.SEGMENT)
|
||||
segmentSizePosition = head.size().toLong()
|
||||
// Unknown size (a live stream) until finish() knows it.
|
||||
head.raw(0x01)
|
||||
repeat(7) { head.raw(0xFF) }
|
||||
segmentStart = head.size().toLong()
|
||||
append(head.bytes())
|
||||
|
||||
seekPosition = output.size
|
||||
infoPosition = seekPosition + SEEK_RESERVED
|
||||
val info = EbmlWriter()
|
||||
info.master(MatroskaIds.INFO) {
|
||||
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
|
||||
string(MatroskaIds.MUXING_APP, APP_NAME)
|
||||
string(MatroskaIds.WRITING_APP, APP_NAME)
|
||||
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
|
||||
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
|
||||
// Last, so its eight bytes are the element's final ones.
|
||||
float64(MatroskaIds.DURATION, 0.0)
|
||||
}
|
||||
durationPosition = infoPosition + info.size() - 8
|
||||
tracksPosition = infoPosition + info.size()
|
||||
attachmentsPosition = tracksPosition + TRACKS_RESERVED
|
||||
|
||||
append(seekHead(-1))
|
||||
append(info.bytes())
|
||||
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
|
||||
if (attachments.isNotEmpty()) {
|
||||
val files = EbmlWriter()
|
||||
files.master(MatroskaIds.ATTACHMENTS) {
|
||||
attachments.forEachIndexed { index, file ->
|
||||
master(MatroskaIds.ATTACHED_FILE) {
|
||||
string(MatroskaIds.FILE_NAME, file.name)
|
||||
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
|
||||
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
|
||||
uint(MatroskaIds.FILE_UID, index + 1L)
|
||||
}
|
||||
}
|
||||
}
|
||||
append(files.bytes())
|
||||
}
|
||||
}
|
||||
|
||||
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
|
||||
|
||||
private fun seekHead(cuesPosition: Long): ByteArray {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.SEEK_HEAD) {
|
||||
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
|
||||
val idBytes = EbmlWriter()
|
||||
idBytes.id(id)
|
||||
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
|
||||
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
|
||||
}
|
||||
entry(MatroskaIds.INFO, infoPosition)
|
||||
entry(MatroskaIds.TRACKS, tracksPosition)
|
||||
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
|
||||
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
|
||||
}
|
||||
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
|
||||
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
|
||||
* the one players pick by default.
|
||||
*/
|
||||
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
|
||||
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
|
||||
|
||||
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
|
||||
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
|
||||
|
||||
private fun addTrack(track: Track): Int {
|
||||
check(!finished) { "recording already finished" }
|
||||
// A block's track number is written in one byte.
|
||||
if (track.number > 126) return -1
|
||||
tracks.add(track)
|
||||
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
|
||||
if (region == null) {
|
||||
tracks.remove(track)
|
||||
return -1
|
||||
}
|
||||
output.writeAt(tracksPosition, region)
|
||||
return track.number
|
||||
}
|
||||
|
||||
private fun listedTracks(): List<Track> =
|
||||
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
|
||||
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
|
||||
tracks.filter { it.type != TRACK_AUDIO }
|
||||
|
||||
private fun tracksElement(): ByteArray {
|
||||
val listed = listedTracks()
|
||||
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
|
||||
val out = EbmlWriter(1024)
|
||||
out.master(MatroskaIds.TRACKS) {
|
||||
for (track in listed) {
|
||||
master(MatroskaIds.TRACK_ENTRY) {
|
||||
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_UID, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
|
||||
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
|
||||
uint(MatroskaIds.FLAG_LACING, 0)
|
||||
string(MatroskaIds.NAME, track.name)
|
||||
string(MatroskaIds.LANGUAGE, "und")
|
||||
if (track.type == TRACK_AUDIO) {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_PCM)
|
||||
master(MatroskaIds.AUDIO) {
|
||||
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
|
||||
uint(MatroskaIds.CHANNELS, track.channels.toLong())
|
||||
uint(MatroskaIds.BIT_DEPTH, 16)
|
||||
}
|
||||
} else {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.bytes()
|
||||
}
|
||||
|
||||
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
|
||||
|
||||
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
|
||||
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
|
||||
val track = track(trackNumber)
|
||||
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
|
||||
if (length <= 0) return
|
||||
val block = openCluster(timeMs)
|
||||
block.id(MatroskaIds.SIMPLE_BLOCK)
|
||||
block.dataSize(4L + length)
|
||||
blockHeader(block, trackNumber, timeMs, 0x80)
|
||||
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
|
||||
val frames = length / (2 * track.channels)
|
||||
noteEnd(timeMs + frames * 1000L / track.sampleRate)
|
||||
}
|
||||
|
||||
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
|
||||
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
|
||||
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
|
||||
val bytes = text.encodeToByteArray()
|
||||
val block = openCluster(timeMs)
|
||||
block.master(MatroskaIds.BLOCK_GROUP) {
|
||||
id(MatroskaIds.BLOCK)
|
||||
dataSize(4L + bytes.size)
|
||||
blockHeader(this, trackNumber, timeMs, 0)
|
||||
raw(bytes)
|
||||
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
|
||||
}
|
||||
noteEnd(timeMs + maxOf(1L, durationMs))
|
||||
}
|
||||
|
||||
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
|
||||
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
|
||||
block.raw(0x80 or trackNumber)
|
||||
block.raw(relative shr 8)
|
||||
block.raw(relative)
|
||||
block.raw(flags)
|
||||
}
|
||||
|
||||
private fun noteEnd(endMs: Long) {
|
||||
if (endMs > durationMs) durationMs = endMs
|
||||
}
|
||||
|
||||
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
|
||||
private fun openCluster(timeMs: Long): EbmlWriter {
|
||||
check(!finished) { "recording already finished" }
|
||||
val time = maxOf(0L, timeMs)
|
||||
val open = cluster
|
||||
if (open != null) {
|
||||
val relative = time - clusterBaseMs
|
||||
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
|
||||
flushCluster()
|
||||
}
|
||||
clusterBaseMs = time
|
||||
return EbmlWriter(64 * 1024).also { cluster = it }
|
||||
}
|
||||
|
||||
private fun flushCluster() {
|
||||
val body = cluster ?: return
|
||||
cluster = null
|
||||
if (body.size() == 0) return
|
||||
val timestamp = EbmlWriter(16)
|
||||
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
|
||||
val header = EbmlWriter(32)
|
||||
header.id(MatroskaIds.CLUSTER)
|
||||
header.dataSize(timestamp.size().toLong() + body.size())
|
||||
header.raw(timestamp.bytes())
|
||||
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
|
||||
append(header.bytes())
|
||||
append(body.bytes())
|
||||
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
|
||||
}
|
||||
|
||||
private fun patchDuration() {
|
||||
clustersSincePatch = 0
|
||||
val value = durationMs.toDouble().toRawBits()
|
||||
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
|
||||
}
|
||||
|
||||
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
|
||||
fun finish() {
|
||||
if (finished) return
|
||||
try {
|
||||
flushCluster()
|
||||
finished = true
|
||||
val cuesPosition = output.size
|
||||
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
|
||||
if (cues.isNotEmpty()) {
|
||||
val index = EbmlWriter(cues.size * 16 + 16)
|
||||
index.master(MatroskaIds.CUES) {
|
||||
for (cue in cues) {
|
||||
master(MatroskaIds.CUE_POINT) {
|
||||
uint(MatroskaIds.CUE_TIME, cue.timeMs)
|
||||
master(MatroskaIds.CUE_TRACK_POSITIONS) {
|
||||
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
|
||||
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
append(index.bytes())
|
||||
output.writeAt(seekPosition, seekHead(cuesPosition))
|
||||
}
|
||||
patchDuration()
|
||||
val size = EbmlWriter(8)
|
||||
size.dataSize(output.size - segmentStart, 8)
|
||||
output.writeAt(segmentSizePosition, size.bytes())
|
||||
} finally {
|
||||
finished = true
|
||||
output.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaTrackInfo(
|
||||
@JvmField val number: Int,
|
||||
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
|
||||
@JvmField val type: Int,
|
||||
@JvmField val codecId: String,
|
||||
@JvmField val name: String,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val bitDepth: Int,
|
||||
) {
|
||||
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
|
||||
}
|
||||
|
||||
class MkaBlock(
|
||||
@JvmField val track: Int,
|
||||
@JvmField val timeMs: Long,
|
||||
@JvmField val data: ByteArray,
|
||||
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
|
||||
@JvmField val durationMs: Long,
|
||||
)
|
||||
|
||||
/**
|
||||
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
|
||||
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
|
||||
* IllegalArgumentException for anything that is not such a file.
|
||||
*/
|
||||
class MkaReader(private val input: SeekableInput) {
|
||||
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
|
||||
|
||||
private val clusters = ArrayList<ClusterRef>()
|
||||
private val trackList = ArrayList<MkaTrackInfo>()
|
||||
private val attachmentList = ArrayList<MkaAttachment>()
|
||||
private var timestampScale = 1_000_000L
|
||||
private var infoDurationMs = 0L
|
||||
private var nextCluster = 0
|
||||
private var blocks: List<MkaBlock> = emptyList()
|
||||
private var blockIndex = 0
|
||||
private var skipBeforeMs = 0L
|
||||
|
||||
val tracks: List<MkaTrackInfo> get() = trackList
|
||||
val attachments: List<MkaAttachment> get() = attachmentList
|
||||
|
||||
/** The end of the last audio or text in the file, in ms. */
|
||||
val durationMs: Long
|
||||
|
||||
init {
|
||||
scan()
|
||||
var end = infoDurationMs
|
||||
if (clusters.isNotEmpty()) {
|
||||
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
|
||||
}
|
||||
durationMs = end
|
||||
}
|
||||
|
||||
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
|
||||
|
||||
/** Continue from the first block at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
|
||||
nextCluster = maxOf(0, index)
|
||||
blocks = emptyList()
|
||||
blockIndex = 0
|
||||
skipBeforeMs = timeMs
|
||||
}
|
||||
|
||||
/** The next block, or null at the end of the file. */
|
||||
fun next(): MkaBlock? {
|
||||
while (true) {
|
||||
while (blockIndex >= blocks.size) {
|
||||
if (nextCluster >= clusters.size) return null
|
||||
blocks = readCluster(clusters[nextCluster++])
|
||||
blockIndex = 0
|
||||
}
|
||||
val block = blocks[blockIndex++]
|
||||
if (block.timeMs >= skipBeforeMs) return block
|
||||
}
|
||||
}
|
||||
|
||||
fun close() = input.close()
|
||||
|
||||
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
|
||||
|
||||
private fun blockSpanMs(block: MkaBlock): Long {
|
||||
if (block.durationMs >= 0) return block.durationMs
|
||||
val track = track(block.track) ?: return 0
|
||||
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
|
||||
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
|
||||
}
|
||||
|
||||
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
|
||||
private class Header(val id: Int, val size: Long, val length: Int)
|
||||
|
||||
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
|
||||
if (at >= end) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val idLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (idLength > 4 || at + idLength >= end) return null
|
||||
var id = 0
|
||||
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
|
||||
val sizeAt = at + idLength
|
||||
val sizeFirst = bytes[sizeAt].toInt() and 0xff
|
||||
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
|
||||
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
|
||||
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
|
||||
var allOnes = size == (0xff shr sizeLength).toLong()
|
||||
for (index in 1 until sizeLength) {
|
||||
val byte = bytes[sizeAt + index].toInt() and 0xff
|
||||
if (byte != 0xff) allOnes = false
|
||||
size = (size shl 8) or byte.toLong()
|
||||
}
|
||||
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
|
||||
}
|
||||
|
||||
private fun readAt(position: Long, length: Int): ByteArray {
|
||||
val buffer = ByteArray(length)
|
||||
val read = input.readAt(position, buffer, 0, length)
|
||||
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
|
||||
}
|
||||
|
||||
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
|
||||
var value = 0L
|
||||
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
|
||||
return value
|
||||
}
|
||||
|
||||
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
|
||||
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
|
||||
8 -> Double.fromBits(uint(bytes, at, 8))
|
||||
else -> 0.0
|
||||
}
|
||||
|
||||
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
|
||||
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
|
||||
var at = start
|
||||
while (true) {
|
||||
val child = header(bytes, at, end) ?: return
|
||||
val payload = at + child.length
|
||||
if (child.size < 0 || child.size > end - payload) return
|
||||
visit(child.id, payload, child.size.toInt())
|
||||
at = payload + child.size.toInt()
|
||||
}
|
||||
}
|
||||
|
||||
private fun scan() {
|
||||
val length = input.length
|
||||
val start = readAt(0, 64)
|
||||
val ebml = header(start, 0, start.size)
|
||||
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
|
||||
var position = ebml.length + ebml.size
|
||||
val segment = readAt(position, 12).let { header(it, 0, it.size) }
|
||||
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
|
||||
position += segment.length
|
||||
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
|
||||
while (position < end) {
|
||||
val head = readAt(position, 12)
|
||||
val element = header(head, 0, head.size) ?: break
|
||||
val payload = position + element.length
|
||||
if (element.id == MatroskaIds.CLUSTER) {
|
||||
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
|
||||
// A recording cut short ends in a partial cluster; its complete blocks still count.
|
||||
val available = minOf(element.size, end - payload)
|
||||
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
|
||||
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
|
||||
} else if (element.size < 0) {
|
||||
break
|
||||
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
|
||||
when (element.id) {
|
||||
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
|
||||
}
|
||||
}
|
||||
if (element.size < 0) break
|
||||
position = payload + element.size
|
||||
}
|
||||
}
|
||||
|
||||
private fun clusterTime(position: Long, size: Int): Long {
|
||||
val head = readAt(position, minOf(size, 16))
|
||||
val first = header(head, 0, head.size)
|
||||
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
|
||||
return uint(head, first.length, first.size.toInt())
|
||||
}
|
||||
val whole = readAt(position, size)
|
||||
var time = 0L
|
||||
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
|
||||
return time
|
||||
}
|
||||
|
||||
private fun parseInfo(bytes: ByteArray) {
|
||||
var duration = 0.0
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
|
||||
MatroskaIds.DURATION -> duration = float(bytes, at, size)
|
||||
}
|
||||
}
|
||||
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
|
||||
}
|
||||
|
||||
private fun parseTracks(bytes: ByteArray) {
|
||||
trackList.clear()
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.TRACK_ENTRY) {
|
||||
var number = 0
|
||||
var type = 0
|
||||
var codec = ""
|
||||
var name = ""
|
||||
var rate = 8000.0
|
||||
var channels = 1
|
||||
var depth = 0
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
|
||||
when (audio) {
|
||||
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
|
||||
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
|
||||
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val audio = type == MkaWriter.TRACK_AUDIO
|
||||
trackList.add(
|
||||
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseAttachments(bytes: ByteArray) {
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.ATTACHED_FILE) {
|
||||
var name = ""
|
||||
var mime = ""
|
||||
var data = ByteArray(0)
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
|
||||
}
|
||||
}
|
||||
attachmentList.add(MkaAttachment(name, mime, data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
|
||||
val bytes = readAt(cluster.position, cluster.size)
|
||||
val out = ArrayList<MkaBlock>()
|
||||
var clusterRaw = cluster.rawTime
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
|
||||
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
|
||||
MatroskaIds.BLOCK_GROUP -> {
|
||||
var blockAt = -1
|
||||
var blockSize = 0
|
||||
var duration = -1L
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.BLOCK -> {
|
||||
blockAt = fieldAt
|
||||
blockSize = fieldSize
|
||||
}
|
||||
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
|
||||
}
|
||||
}
|
||||
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
|
||||
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
|
||||
if (size < 4) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val trackLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (trackLength > 2 || size < trackLength + 3) return null
|
||||
var track = first and (0xff shr trackLength)
|
||||
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
|
||||
val head = at + trackLength
|
||||
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
|
||||
val flags = bytes[head + 2].toInt() and 0xff
|
||||
if (flags and 0x06 != 0) return null
|
||||
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
|
||||
/**
|
||||
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
|
||||
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
|
||||
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
|
||||
* contract is the header of g2flash patches/mic_control.c).
|
||||
*/
|
||||
object MicArrayProtocol {
|
||||
const val VERSION = 1
|
||||
const val HEADER_BYTES = 21
|
||||
const val OP_CONFIGURE = 1
|
||||
const val OP_STOP = 3
|
||||
const val OP_RENEW = 4
|
||||
/** How long an armed temple streams without a CONFIGURE or RENEW. */
|
||||
const val LEASE_MS = 90_000L
|
||||
const val FLAG_TRUNCATED = 0x80
|
||||
const val CODEC_RAW = 1
|
||||
const val FORMAT_PCM16 = 0
|
||||
|
||||
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
|
||||
class Frame(
|
||||
/** Low 16 bits of the temple's frame counter. */
|
||||
@JvmField val sequence: Int,
|
||||
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
|
||||
@JvmField val deviceTickMs: Long,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val format: Int,
|
||||
@JvmField val codec: Int,
|
||||
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
|
||||
@JvmField val angleDegrees: Int,
|
||||
@JvmField val ssr: Int,
|
||||
/** The firmware cut the payload short of the samples it captured. */
|
||||
@JvmField val truncated: Boolean,
|
||||
@JvmField val payloadOffset: Int,
|
||||
@JvmField val payloadLength: Int,
|
||||
)
|
||||
|
||||
/** A control record with no payload: ['M','C', version, op]. */
|
||||
@JvmStatic
|
||||
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
|
||||
|
||||
@JvmStatic
|
||||
fun isConfigure(record: ByteArray?): Boolean =
|
||||
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
|
||||
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
|
||||
|
||||
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
|
||||
@JvmStatic
|
||||
fun decodeFrame(data: ByteArray): Frame? {
|
||||
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
|
||||
return null
|
||||
}
|
||||
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
|
||||
return Frame(
|
||||
sequence = u16(data, 4),
|
||||
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
|
||||
channels = data[10].toInt() and 0xff,
|
||||
sampleRate = u16(data, 11) * 100,
|
||||
format = data[13].toInt() and 0xff,
|
||||
codec = data[14].toInt() and 0xff,
|
||||
angleDegrees = u16(data, 15).toShort().toInt(),
|
||||
ssr = u16(data, 17).toShort().toInt(),
|
||||
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
|
||||
payloadOffset = HEADER_BYTES,
|
||||
payloadLength = length,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
|
||||
* payload isn't raw 16-bit PCM or holds none.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
|
||||
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
|
||||
val frames = frame.payloadLength / (2 * frame.channels)
|
||||
if (frames <= 0) return null
|
||||
val samples = ShortArray(frames * frame.channels)
|
||||
var at = frame.payloadOffset
|
||||
for (index in samples.indices) {
|
||||
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
|
||||
at += 2
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
|
||||
}
|
||||
@@ -0,0 +1,484 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.concurrent.Volatile
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
import kotlin.math.abs
|
||||
|
||||
/** Which microphones a chunk of capture audio came from. */
|
||||
object MicStream {
|
||||
/** One mixed channel: the glasses' stock stream, or the phone's own microphone. */
|
||||
const val MONO = 0
|
||||
/** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */
|
||||
const val LEFT = 1
|
||||
const val RIGHT = 2
|
||||
/** The mono feed derived from the temples for speech recognition: recorded, never replayed. */
|
||||
const val MIX = 3
|
||||
const val COUNT = 4
|
||||
|
||||
/** The stream's name in frame metadata. */
|
||||
@JvmStatic
|
||||
fun key(stream: Int): String = when (stream) {
|
||||
MONO -> "mic"
|
||||
LEFT -> "left"
|
||||
RIGHT -> "right"
|
||||
else -> "mix"
|
||||
}
|
||||
|
||||
/** The name of the stream's audio tracks in a recording. */
|
||||
@JvmStatic
|
||||
fun trackName(stream: Int): String = when (stream) {
|
||||
MONO -> "mic"
|
||||
LEFT -> "left temple"
|
||||
RIGHT -> "right temple"
|
||||
else -> "mix"
|
||||
}
|
||||
|
||||
/** The stream a recording's track or metadata names, or -1. */
|
||||
@JvmStatic
|
||||
fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1
|
||||
}
|
||||
|
||||
/**
|
||||
* One chunk of microphone audio as a capture source delivered it (about 50 ms), before any
|
||||
* processing. Live captures, recordings and replays all trade in these.
|
||||
*/
|
||||
class MicChunk(
|
||||
@JvmField val stream: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
/** Interleaved PCM16; the first [frames] sample frames are the chunk. */
|
||||
@JvmField val pcm: ShortArray,
|
||||
@JvmField val frames: Int,
|
||||
/** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */
|
||||
@JvmField val timeMs: Long,
|
||||
) {
|
||||
/** Whether the source reported a direction of arrival with the chunk. */
|
||||
@JvmField var hasDirection = false
|
||||
@JvmField var angleDegrees = 0
|
||||
@JvmField var ssr = 0
|
||||
/** The source's packet counter, or -1 when it has none. */
|
||||
@JvmField var sequence = -1
|
||||
/** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */
|
||||
@JvmField var deviceTickMs = -1L
|
||||
/** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */
|
||||
@JvmField var clockStream = stream
|
||||
/** The source cut the chunk short of what it captured. */
|
||||
@JvmField var truncated = false
|
||||
/** Sample frames known to be missing between the previous chunk of this stream and this one. */
|
||||
@JvmField var lostFramesBefore = 0
|
||||
|
||||
val durationMs: Long get() = frames * 1000L / sampleRate
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for
|
||||
* array processing (beamforming, noise cancellation): replays feed it the recorded chunks with
|
||||
* the times they first arrived at, so an implementation that decides from the chunks alone
|
||||
* behaves the same on a replay as it did live.
|
||||
*/
|
||||
interface MicArrayMixer {
|
||||
/** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */
|
||||
fun mix(chunk: MicChunk): MicChunk?
|
||||
|
||||
fun reset()
|
||||
}
|
||||
|
||||
/**
|
||||
* The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix).
|
||||
* The temples sample on independent clocks, so their audio is never summed. The left temple
|
||||
* feeds whenever it is streaming; the right one stands in while it is not.
|
||||
*/
|
||||
class SingleTempleMixer : MicArrayMixer {
|
||||
companion object {
|
||||
/** A temple that has delivered nothing for this long no longer counts as streaming. */
|
||||
const val SILENT_MS = 300L
|
||||
}
|
||||
|
||||
private var leftAtMs = Long.MIN_VALUE
|
||||
|
||||
override fun reset() {
|
||||
leftAtMs = Long.MIN_VALUE
|
||||
}
|
||||
|
||||
override fun mix(chunk: MicChunk): MicChunk? {
|
||||
if (chunk.stream == MicStream.LEFT) {
|
||||
leftAtMs = chunk.timeMs
|
||||
} else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) {
|
||||
return null
|
||||
}
|
||||
val channels = chunk.channels
|
||||
val mono = ShortArray(chunk.frames)
|
||||
for (frame in 0 until chunk.frames) {
|
||||
var sum = 0
|
||||
for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel]
|
||||
mono[frame] = (sum / channels).toShort()
|
||||
}
|
||||
val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs)
|
||||
out.hasDirection = chunk.hasDirection
|
||||
out.angleDegrees = chunk.angleDegrees
|
||||
out.ssr = chunk.ssr
|
||||
out.deviceTickMs = chunk.deviceTickMs
|
||||
out.clockStream = chunk.stream
|
||||
out.truncated = chunk.truncated
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and
|
||||
* layout as each first appears, plus a "frames" text track with one JSON object per chunk that
|
||||
* has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md.
|
||||
*
|
||||
* Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick,
|
||||
* or the count of samples from a source without one) says how far apart its chunks were
|
||||
* captured; the least-delayed recent arrival places that clock on the file's timeline. So a
|
||||
* chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap.
|
||||
*
|
||||
* Safe to share between the capture thread and whoever finishes the recording.
|
||||
*/
|
||||
class MicRecordingWriter(
|
||||
output: SeekableOutput,
|
||||
/** A JSON document describing the capture, stored as the file's "capture.json" attachment. */
|
||||
configJson: String?,
|
||||
/** When the recording began (Unix epoch ms). */
|
||||
startedAtMs: Long,
|
||||
platform: ProtocolPlatform = protocolPlatform(),
|
||||
private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) },
|
||||
) {
|
||||
companion object {
|
||||
const val CONFIG_ATTACHMENT = "capture.json"
|
||||
const val META_TRACK = "frames"
|
||||
/** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */
|
||||
const val DISCONTINUITY_MS = 5000L
|
||||
private const val WINDOW_BUCKETS = 8
|
||||
private const val BUCKET_MS = 1000L
|
||||
}
|
||||
|
||||
/**
|
||||
* Places one track's chunks on the file's timeline. `source` time is the stream's own clock
|
||||
* since this clock was anchored; the offset from it to arrival time is taken as the least
|
||||
* seen over the last few seconds, which is the least-delayed delivery.
|
||||
*/
|
||||
private class TrackClock(val number: Int, val sampleRate: Int) {
|
||||
private var anchored = false
|
||||
private var clockStream = -1
|
||||
private var tickBase = -1L
|
||||
private var lastTick = -1L
|
||||
private var frames = 0L
|
||||
private val minima = LongArray(WINDOW_BUCKETS)
|
||||
private var bucket = 0
|
||||
private var bucketStartMs = 0L
|
||||
var endMs = Long.MIN_VALUE
|
||||
private set
|
||||
|
||||
fun reset() {
|
||||
anchored = false
|
||||
}
|
||||
|
||||
private fun offset(): Long = minima.min()
|
||||
|
||||
fun stamp(chunk: MicChunk): Long {
|
||||
val arrival = chunk.timeMs
|
||||
val ticked = chunk.deviceTickMs >= 0
|
||||
if (anchored) {
|
||||
// Another clock (the feed moved to the other temple), or this one restarted.
|
||||
if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false
|
||||
else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false
|
||||
}
|
||||
var anchorNow = !anchored
|
||||
var source = 0L
|
||||
if (anchored) {
|
||||
frames += chunk.lostFramesBefore
|
||||
source = sourceTime(chunk)
|
||||
if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true
|
||||
}
|
||||
if (anchorNow) {
|
||||
anchored = true
|
||||
clockStream = chunk.clockStream
|
||||
tickBase = if (ticked) chunk.deviceTickMs else -1L
|
||||
frames = 0
|
||||
minima.fill(Long.MAX_VALUE)
|
||||
bucket = 0
|
||||
bucketStartMs = arrival
|
||||
source = 0
|
||||
}
|
||||
if (arrival - bucketStartMs >= BUCKET_MS) {
|
||||
bucket = (bucket + 1) % WINDOW_BUCKETS
|
||||
minima[bucket] = Long.MAX_VALUE
|
||||
bucketStartMs = arrival
|
||||
}
|
||||
if (arrival - source < minima[bucket]) minima[bucket] = arrival - source
|
||||
// Never before the end of the track's previous block.
|
||||
val time = maxOf(source + offset(), endMs)
|
||||
frames += chunk.frames
|
||||
lastTick = chunk.deviceTickMs
|
||||
endMs = time + chunk.durationMs
|
||||
return time
|
||||
}
|
||||
|
||||
private fun sourceTime(chunk: MicChunk): Long =
|
||||
if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate
|
||||
}
|
||||
|
||||
private val lock = platform.createLock()
|
||||
private val writer: MkaWriter
|
||||
private val clocks = HashMap<Long, TrackClock>()
|
||||
private var metaTrack = 0
|
||||
private var originMs = Long.MIN_VALUE
|
||||
private var lastTimeMs = 0L
|
||||
private var closed = false
|
||||
|
||||
/** The recording stopped early: the file could not be written. */
|
||||
@Volatile
|
||||
var failed = false
|
||||
private set
|
||||
|
||||
init {
|
||||
val attachments = if (configJson.isNullOrEmpty()) emptyList()
|
||||
else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray()))
|
||||
writer = MkaWriter(output, null, startedAtMs, attachments)
|
||||
}
|
||||
|
||||
/** Length of the recording so far, in ms. */
|
||||
val durationMs: Long get() = lock.withLock { writer.durationMs }
|
||||
|
||||
/** Appends [chunk] to its stream's track, adding the track on its first chunk. */
|
||||
fun write(chunk: MicChunk) {
|
||||
if (chunk.frames <= 0) return
|
||||
lock.withLock {
|
||||
if (closed) return
|
||||
try {
|
||||
val clock = clockFor(chunk) ?: return
|
||||
if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs
|
||||
val time = clock.stamp(chunk) - originMs
|
||||
val meta = metadata(chunk)
|
||||
if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta)
|
||||
val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels)
|
||||
writer.writeAudio(clock.number, time, bytes, 0, bytes.size)
|
||||
lastTimeMs = maxOf(lastTimeMs, time)
|
||||
} catch (error: Throwable) {
|
||||
fail(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a JSON object as an event at the recording's current position (a capture starting
|
||||
* over with another configuration, say), and has every stream's clock placed afresh by its
|
||||
* next chunk: what follows an event is not continuous with what came before.
|
||||
*/
|
||||
fun event(json: String) {
|
||||
lock.withLock {
|
||||
if (closed) return
|
||||
try {
|
||||
for (clock in clocks.values) clock.reset()
|
||||
if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json)
|
||||
} catch (error: Throwable) {
|
||||
fail(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Completes the file. Returns the recording's length in ms. */
|
||||
fun finish(): Long {
|
||||
lock.withLock {
|
||||
if (!closed) {
|
||||
closed = true
|
||||
try {
|
||||
writer.finish()
|
||||
} catch (error: Throwable) {
|
||||
failed = true
|
||||
logWarn("finishing the recording failed: ${error.message}")
|
||||
}
|
||||
}
|
||||
return writer.durationMs
|
||||
}
|
||||
}
|
||||
|
||||
private fun fail(error: Throwable) {
|
||||
failed = true
|
||||
closed = true
|
||||
logWarn("recording stopped: ${error.message}")
|
||||
try {
|
||||
writer.finish()
|
||||
} catch (ignored: Throwable) {
|
||||
// The file is whatever made it to disk.
|
||||
}
|
||||
}
|
||||
|
||||
private fun ensureMetaTrack(): Boolean {
|
||||
if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK)
|
||||
return metaTrack > 0
|
||||
}
|
||||
|
||||
private fun clockFor(chunk: MicChunk): TrackClock? {
|
||||
val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong()
|
||||
clocks[key]?.let { return it }
|
||||
// The frames track goes in with the first audio track, so it is there for that track's first chunk.
|
||||
ensureMetaTrack()
|
||||
// The mixed streams are what a player should pick; a temple's raw pair is for analysis.
|
||||
val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX
|
||||
val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred)
|
||||
if (number <= 0) {
|
||||
logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio")
|
||||
return null
|
||||
}
|
||||
return TrackClock(number, chunk.sampleRate).also { clocks[key] = it }
|
||||
}
|
||||
|
||||
/** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */
|
||||
private fun metadata(chunk: MicChunk): String? {
|
||||
if (chunk.stream == MicStream.MIX) return null
|
||||
if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null
|
||||
val text = StringBuilder(96)
|
||||
text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs)
|
||||
if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence)
|
||||
if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs)
|
||||
if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr)
|
||||
if (chunk.truncated) text.append(",\"trunc\":true")
|
||||
if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore)
|
||||
return text.append('}').toString()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO]
|
||||
* chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the
|
||||
* metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A
|
||||
* Matroska file from elsewhere plays its first PCM16 track as the mono microphone.
|
||||
*
|
||||
* Throws IllegalArgumentException when the file has no audio a replay can use.
|
||||
*/
|
||||
class MicRecordingReader(input: SeekableInput) {
|
||||
companion object {
|
||||
const val SAMPLE_RATE = 16000
|
||||
/** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */
|
||||
const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20
|
||||
}
|
||||
|
||||
private class Meta(val timeMs: Long, val fields: Map<String, Any?>)
|
||||
|
||||
private val reader = MkaReader(input)
|
||||
private val streams = HashMap<Int, Int>()
|
||||
private var metaTrack = -1
|
||||
private val pendingMeta = arrayOfNulls<Meta>(MicStream.COUNT)
|
||||
private val queued = ArrayDeque<MicChunk>()
|
||||
|
||||
val durationMs: Long get() = reader.durationMs
|
||||
|
||||
/** The capture's "capture.json" attachment, or null. */
|
||||
val configJson: String? =
|
||||
reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString()
|
||||
|
||||
/** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */
|
||||
val description: String
|
||||
|
||||
init {
|
||||
try {
|
||||
val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 }
|
||||
for (track in audio) {
|
||||
val stream = MicStream.named(track.name)
|
||||
if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream
|
||||
}
|
||||
if (streams.isEmpty()) {
|
||||
// Not one of ours: its first usable track stands in for the microphone.
|
||||
val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull()
|
||||
require(first != null) { "no 16-bit PCM audio track" }
|
||||
streams[first.number] = MicStream.MONO
|
||||
}
|
||||
for (number in streams.keys) {
|
||||
val rate = reader.track(number)!!.sampleRate
|
||||
require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" }
|
||||
}
|
||||
metaTrack = reader.tracks.firstOrNull {
|
||||
it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK
|
||||
}?.number ?: -1
|
||||
description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) ->
|
||||
val channels = reader.track(number)!!.channels
|
||||
MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else ""
|
||||
}
|
||||
} catch (error: Throwable) {
|
||||
reader.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Continue from the first chunk at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
queued.clear()
|
||||
pendingMeta.fill(null)
|
||||
reader.seekTo(timeMs)
|
||||
}
|
||||
|
||||
/** The next chunk in file order, or null at the end. */
|
||||
fun next(): MicChunk? {
|
||||
while (queued.isEmpty()) {
|
||||
val block = reader.next() ?: return null
|
||||
if (block.track == metaTrack) {
|
||||
noteMeta(block)
|
||||
continue
|
||||
}
|
||||
val stream = streams[block.track] ?: continue
|
||||
val track = reader.track(block.track) ?: continue
|
||||
queueChunks(block, stream, track)
|
||||
}
|
||||
return queued.removeFirst()
|
||||
}
|
||||
|
||||
fun close() = reader.close()
|
||||
|
||||
private fun noteMeta(block: MkaBlock) {
|
||||
try {
|
||||
val fields = Json.parseObject(block.data.decodeToString())
|
||||
val stream = MicStream.named(fields["s"].asString() ?: return)
|
||||
if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields)
|
||||
} catch (ignored: Throwable) {
|
||||
// An event or a frame we can't read: the audio plays without it.
|
||||
}
|
||||
}
|
||||
|
||||
private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) {
|
||||
val sourceChannels = track.channels
|
||||
val totalFrames = block.data.size / (2 * sourceChannels)
|
||||
if (totalFrames <= 0) return
|
||||
// The mixed microphone is one channel however a foreign file stored it.
|
||||
val channels = if (stream == MicStream.MONO) 1 else sourceChannels
|
||||
val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs }
|
||||
pendingMeta[stream] = null
|
||||
var start = 0
|
||||
while (start < totalFrames) {
|
||||
val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start)
|
||||
val pcm = ShortArray(frames * channels)
|
||||
for (frame in 0 until frames) {
|
||||
val at = (start + frame) * sourceChannels * 2
|
||||
if (channels == sourceChannels) {
|
||||
for (channel in 0 until channels) {
|
||||
pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
|
||||
}
|
||||
} else {
|
||||
var sum = 0
|
||||
for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
|
||||
pcm[frame] = (sum / sourceChannels).toShort()
|
||||
}
|
||||
}
|
||||
val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate)
|
||||
if (meta != null && start == 0) applyMeta(chunk, meta.fields)
|
||||
queued.addLast(chunk)
|
||||
start += frames
|
||||
}
|
||||
}
|
||||
|
||||
private fun applyMeta(chunk: MicChunk, fields: Map<String, Any?>) {
|
||||
chunk.sequence = fields["seq"].asInt() ?: -1
|
||||
chunk.deviceTickMs = fields["tick"].asLong() ?: -1L
|
||||
val angle = fields["angle"].asInt()
|
||||
if (angle != null) {
|
||||
chunk.hasDirection = true
|
||||
chunk.angleDegrees = angle
|
||||
chunk.ssr = fields["ssr"].asInt() ?: 0
|
||||
}
|
||||
chunk.truncated = fields["trunc"].asBoolean() ?: false
|
||||
chunk.lostFramesBefore = fields["lost"].asInt() ?: 0
|
||||
}
|
||||
}
|
||||
+393
-79
@@ -36,17 +36,26 @@ interface VoiceCaptureHost {
|
||||
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
|
||||
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
|
||||
|
||||
/** Where a saved recording goes, or null when recordings cannot be stored. */
|
||||
fun recordingSink(): RecordingSink?
|
||||
/** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */
|
||||
fun micArrayLink(): MicArrayLink? = null
|
||||
|
||||
/** Opens a recording to replay; throws when it cannot be read. */
|
||||
fun openRecording(path: String): SeekableInput = openFileInput(path)
|
||||
}
|
||||
|
||||
/**
|
||||
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
|
||||
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker
|
||||
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, speaker
|
||||
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
|
||||
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
|
||||
* worker thread (started through the shared [startThread]); listener callbacks go through the
|
||||
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
|
||||
*
|
||||
* Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the
|
||||
* phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording
|
||||
* played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go
|
||||
* to the recorder ([setRecorder]) as they came, before any processing, which is what lets a
|
||||
* replay of the recording run the same processing again.
|
||||
*/
|
||||
class VoiceCaptureSession(
|
||||
private val host: VoiceCaptureHost,
|
||||
@@ -98,6 +107,19 @@ class VoiceCaptureSession(
|
||||
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
|
||||
/** Samples per G2 packet; the decode buffer the worker reuses. */
|
||||
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
|
||||
// Mic array upkeep: the temples stop streaming unless their lease is renewed, and a
|
||||
// temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again.
|
||||
const val ARRAY_CHECK_MS = 1000L
|
||||
const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3
|
||||
const val ARRAY_TEMPLE_SILENT_MS = 5000L
|
||||
const val ARRAY_WAITING_MS = 2000L
|
||||
const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..."
|
||||
// A replay skips stretches of the recording with no audio longer than this
|
||||
// (a recording made across a disconnect) instead of sitting through them.
|
||||
const val REPLAY_MAX_GAP_MS = 2000L
|
||||
const val REPLAY_POSITION_INTERVAL_MS = 1000L
|
||||
const val REPLAY_WAIT_SLICE_MS = 20L
|
||||
const val REPLAY_FINISHED_STATUS = "Replay finished."
|
||||
|
||||
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
|
||||
@JvmStatic
|
||||
@@ -123,6 +145,8 @@ class VoiceCaptureSession(
|
||||
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
|
||||
private enum class Source { G2, PHONE, ARRAY, REPLAY }
|
||||
|
||||
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
|
||||
val arm: String = arm ?: "?"
|
||||
}
|
||||
@@ -140,11 +164,33 @@ class VoiceCaptureSession(
|
||||
@Volatile
|
||||
private var workerThreadName: String? = null
|
||||
// Capture from the phone's own microphone instead of the G2 over BLE
|
||||
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
|
||||
// (preview-only mode). Latched into activeSource at start() (under `lock`)
|
||||
// so a mid-session setter call can't switch pipelines underneath the worker.
|
||||
@Volatile
|
||||
private var usePhoneMic = false
|
||||
private var activePhoneMic = false
|
||||
// What the next capture reads (see start()); latched into activeSource and its
|
||||
// companions under `lock`, like usePhoneMic.
|
||||
@Volatile
|
||||
private var micArrayConfigure: ByteArray? = null
|
||||
@Volatile
|
||||
private var replayPath: String? = null
|
||||
@Volatile
|
||||
private var replayStartMs = 0L
|
||||
private var activeSource = Source.G2
|
||||
private var activeConfigure: ByteArray? = null
|
||||
private var activeReplayPath: String? = null
|
||||
private var activeReplayStartMs = 0L
|
||||
// Replay controls, picked up by the worker between chunks.
|
||||
@Volatile
|
||||
private var replaySeekMs = -1L
|
||||
@Volatile
|
||||
private var replayPaused = false
|
||||
@Volatile
|
||||
private var recorder: MicRecordingWriter? = null
|
||||
@Volatile
|
||||
private var mixer: MicArrayMixer = SingleTempleMixer()
|
||||
private var lastMissingPackets = 0L
|
||||
private var arrayRateWarned = false
|
||||
private var mode = Mode.CLOUD
|
||||
@Volatile
|
||||
private var onboardModelKind = VoiceModelKind.MOONSHINE
|
||||
@@ -158,11 +204,8 @@ class VoiceCaptureSession(
|
||||
private var lastTranscriptDecodeAtMs: Long = 0
|
||||
private var lastTranscript = ""
|
||||
@Volatile
|
||||
private var saveRecordings = false
|
||||
@Volatile
|
||||
private var endpointing = false
|
||||
private val endpointDetector = VoiceEndpointDetector()
|
||||
private var recordingPcm: ByteSink? = null
|
||||
@Volatile
|
||||
private var verifySpeakerModelPath: String? = null
|
||||
@Volatile
|
||||
@@ -199,8 +242,45 @@ class VoiceCaptureSession(
|
||||
this.usePhoneMic = usePhoneMic
|
||||
}
|
||||
|
||||
fun setSaveRecordings(saveRecordings: Boolean) {
|
||||
this.saveRecordings = saveRecordings
|
||||
/**
|
||||
* Capture the next glasses session from the temples' own microphones instead of the stock
|
||||
* mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples
|
||||
* (it decides how many microphones each one streams). Null goes back to the mixed stream.
|
||||
*/
|
||||
fun setMicArray(configureRecord: ByteArray?) {
|
||||
this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null
|
||||
}
|
||||
|
||||
/** How the temples' chunks become the mono feed that is transcribed. */
|
||||
fun setMicArrayMixer(mixer: MicArrayMixer) {
|
||||
this.mixer = mixer
|
||||
}
|
||||
|
||||
/**
|
||||
* Have the next capture play the recording at [path] from [startMs] in real time, in place
|
||||
* of any microphone; null goes back to the microphones. A replay needs no glasses.
|
||||
*/
|
||||
fun setReplay(path: String?, startMs: Long) {
|
||||
this.replayPath = path
|
||||
this.replayStartMs = maxOf(0L, startMs)
|
||||
}
|
||||
|
||||
/** Continue the running replay from [positionMs] into its recording. */
|
||||
fun seekReplay(positionMs: Long) {
|
||||
this.replaySeekMs = maxOf(0L, positionMs)
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */
|
||||
fun setReplayPaused(paused: Boolean) {
|
||||
this.replayPaused = paused
|
||||
}
|
||||
|
||||
/**
|
||||
* Where live captures' chunks are recorded as they arrive, or null. Takes effect at once,
|
||||
* also mid-capture; the caller finishes the recording. A replay is never recorded.
|
||||
*/
|
||||
fun setRecorder(recorder: MicRecordingWriter?) {
|
||||
this.recorder = recorder
|
||||
}
|
||||
|
||||
fun setOnboardModelKind(kind: String?) {
|
||||
@@ -268,13 +348,26 @@ class VoiceCaptureSession(
|
||||
emitStopped(captureId)
|
||||
return
|
||||
}
|
||||
if (!usePhoneMic && !host.isG2SessionReady()) {
|
||||
val path = replayPath
|
||||
val configure = micArrayConfigure
|
||||
val source = when {
|
||||
path != null -> Source.REPLAY
|
||||
usePhoneMic -> Source.PHONE
|
||||
configure != null -> Source.ARRAY
|
||||
else -> Source.G2
|
||||
}
|
||||
if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) {
|
||||
emitStatus("Voice control needs an active G2 connection.")
|
||||
emitStopped(captureId)
|
||||
return
|
||||
}
|
||||
mode = parseMode(requestedMode)
|
||||
activePhoneMic = usePhoneMic
|
||||
activeSource = source
|
||||
activeConfigure = configure
|
||||
activeReplayPath = path
|
||||
activeReplayStartMs = replayStartMs
|
||||
replaySeekMs = -1L
|
||||
replayPaused = false
|
||||
started = true
|
||||
audioStarted = false
|
||||
val finished = Latch(1, platform)
|
||||
@@ -291,21 +384,22 @@ class VoiceCaptureSession(
|
||||
*/
|
||||
fun isCapturing(): Boolean {
|
||||
var audioUp = false
|
||||
var phoneMic = false
|
||||
var source = Source.G2
|
||||
lock.withLock {
|
||||
if (!started) {
|
||||
return false
|
||||
}
|
||||
audioUp = audioStarted
|
||||
phoneMic = activePhoneMic
|
||||
source = activeSource
|
||||
}
|
||||
if (!audioUp) {
|
||||
// The worker is still bringing the mic up; report it as running so
|
||||
// a concurrent request shares it instead of restarting it.
|
||||
return true
|
||||
}
|
||||
if (phoneMic) {
|
||||
// The phone mic has no session to lose the enable to; it runs until stop().
|
||||
if (source == Source.PHONE || source == Source.REPLAY) {
|
||||
// Neither has a glasses session to lose the enable to: the phone mic runs
|
||||
// until stop(), a replay until its recording ends.
|
||||
return true
|
||||
}
|
||||
return host.isG2AudioCaptureActive()
|
||||
@@ -327,7 +421,7 @@ class VoiceCaptureSession(
|
||||
finished = workerFinished
|
||||
workerName = workerThreadName
|
||||
}
|
||||
stopG2Audio()
|
||||
stopSource()
|
||||
audioQueue.wake()
|
||||
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
|
||||
finished.await(1500)
|
||||
@@ -363,32 +457,65 @@ class VoiceCaptureSession(
|
||||
}
|
||||
endpointDetector.reset()
|
||||
suppressor?.reset()
|
||||
recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null
|
||||
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
|
||||
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
|
||||
verifyCount = 0
|
||||
if (activePhoneMic) {
|
||||
val record = host.openPhoneMic()
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.")
|
||||
return
|
||||
mixer.reset()
|
||||
lastMissingPackets = 0
|
||||
val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..."
|
||||
when (activeSource) {
|
||||
Source.PHONE -> {
|
||||
val record = host.openPhoneMic()
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
try {
|
||||
processPhoneAudio(record)
|
||||
} finally {
|
||||
record.close()
|
||||
}
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
|
||||
try {
|
||||
processPhoneAudio(record)
|
||||
} finally {
|
||||
record.close()
|
||||
Source.REPLAY -> {
|
||||
val reader = try {
|
||||
MicRecordingReader(host.openRecording(activeReplayPath!!))
|
||||
} catch (error: Throwable) {
|
||||
emitStatus("Could not open the recording: " + error.message)
|
||||
emitReplay("error", 0, 0)
|
||||
return
|
||||
}
|
||||
try {
|
||||
lock.withLock { audioStarted = true }
|
||||
logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s")
|
||||
emitStatus(listening)
|
||||
processReplay(reader)
|
||||
} finally {
|
||||
reader.close()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
packetDecoder = host.createPacketDecoder()
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.")
|
||||
return
|
||||
Source.ARRAY -> {
|
||||
val link = host.micArrayLink()
|
||||
val configure = activeConfigure
|
||||
if (link == null || configure == null || !startArray(link, configure)) {
|
||||
emitStatus("Could not start the glasses microphone array.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
processArray(link, configure, listening)
|
||||
}
|
||||
Source.G2 -> {
|
||||
packetDecoder = host.createPacketDecoder()
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
processG2Audio()
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
|
||||
processG2Audio()
|
||||
}
|
||||
// Verification result must precede the final transcript so the
|
||||
// TS bridge can suppress a non-wearer command before it is acted
|
||||
@@ -403,8 +530,7 @@ class VoiceCaptureSession(
|
||||
logError("Voice control failed", error)
|
||||
emitStatus("Voice control failed: " + error.message)
|
||||
} finally {
|
||||
stopG2Audio()
|
||||
writeRecordingIfAny()
|
||||
stopSource()
|
||||
releaseRecognizer()
|
||||
releaseDecoder()
|
||||
lock.withLock {
|
||||
@@ -418,25 +544,6 @@ class VoiceCaptureSession(
|
||||
}
|
||||
}
|
||||
|
||||
private fun appendRecording(pcm: ShortArray, count: Int) {
|
||||
val out = recordingPcm ?: return
|
||||
out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count))
|
||||
}
|
||||
|
||||
private fun writeRecordingIfAny() {
|
||||
val out = recordingPcm
|
||||
recordingPcm = null
|
||||
if (out == null || out.size() == 0) {
|
||||
return
|
||||
}
|
||||
val sink = host.recordingSink() ?: return
|
||||
try {
|
||||
sink.save(out.toByteArray(), SAMPLE_RATE)
|
||||
} catch (t: Throwable) {
|
||||
logWarn("failed to save voice recording (${t.message})")
|
||||
}
|
||||
}
|
||||
|
||||
private fun startG2Audio(): Boolean {
|
||||
val source = host.g2AudioSource() ?: return false
|
||||
resetAudioStats()
|
||||
@@ -449,9 +556,10 @@ class VoiceCaptureSession(
|
||||
}
|
||||
|
||||
private fun processG2Audio() {
|
||||
val pcm = ShortArray(PACKET_SAMPLES)
|
||||
var pcm = ShortArray(PACKET_SAMPLES)
|
||||
while (started) {
|
||||
val currentDecoder = packetDecoder ?: return
|
||||
if (pcm.size < currentDecoder.maxSamplesPerPacket) pcm = ShortArray(currentDecoder.maxSamplesPerPacket)
|
||||
val packet = takeAudioPacket()
|
||||
if (packet == null) {
|
||||
continue
|
||||
@@ -462,23 +570,214 @@ class VoiceCaptureSession(
|
||||
continue
|
||||
}
|
||||
decodedSamples += count.toLong()
|
||||
// The beam filter drops packets whose firmware direction-of-arrival
|
||||
// falls outside the listening wedge (isolating the aimed talker for
|
||||
// every consumer, recognition included); the spectral noise
|
||||
// suppressor then cleans what remains before it reaches the
|
||||
// recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
val angleDegrees = currentDecoder.lastAngleDegrees
|
||||
val ssr = currentDecoder.lastSsr
|
||||
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
|
||||
emitFrameMeta(angleDegrees, ssr)
|
||||
maybeEmitAudioStats(false)
|
||||
continue
|
||||
}
|
||||
processPcmChunk(pcm, count, angleDegrees, ssr, true)
|
||||
val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs)
|
||||
chunk.hasDirection = currentDecoder.hasFrameMeta
|
||||
chunk.angleDegrees = currentDecoder.lastAngleDegrees
|
||||
chunk.ssr = currentDecoder.lastSsr
|
||||
// Packets the decoder found missing ahead of this one: a gap in a recording.
|
||||
val missing = currentDecoder.missingPackets
|
||||
chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt()
|
||||
lastMissingPackets = missing
|
||||
handleMonoChunk(chunk)
|
||||
maybeEmitAudioStats(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A mixed-microphone chunk from any source: recorded as it came, then on to processing
|
||||
* unless the beam filter drops it. The filter drops chunks whose firmware
|
||||
* direction-of-arrival falls outside the listening wedge (isolating the aimed talker for
|
||||
* every consumer, recognition included); the spectral noise suppressor then cleans what
|
||||
* remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
*/
|
||||
private fun handleMonoChunk(chunk: MicChunk) {
|
||||
recordChunk(chunk)
|
||||
if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) {
|
||||
emitFrameMeta(chunk.angleDegrees, chunk.ssr)
|
||||
return
|
||||
}
|
||||
processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection)
|
||||
}
|
||||
|
||||
/**
|
||||
* A chunk of one temple's own microphones: recorded as it came, then mixed into the mono
|
||||
* feed, which is recorded too (as what was transcribed) and processed like a mixed chunk.
|
||||
* The beam filter does not apply: a temple's bearing is along its own pair, not the
|
||||
* head-frame direction the filter is aimed in.
|
||||
*/
|
||||
private fun handleArrayChunk(chunk: MicChunk) {
|
||||
recordChunk(chunk)
|
||||
val feed = mixer.mix(chunk) ?: return
|
||||
recordChunk(feed)
|
||||
if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) {
|
||||
if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz")
|
||||
arrayRateWarned = true
|
||||
return
|
||||
}
|
||||
processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection)
|
||||
}
|
||||
|
||||
private fun recordChunk(chunk: MicChunk) {
|
||||
if (activeSource != Source.REPLAY) recorder?.write(chunk)
|
||||
}
|
||||
|
||||
private fun handleChunk(chunk: MicChunk) {
|
||||
if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk)
|
||||
}
|
||||
|
||||
private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean {
|
||||
resetAudioStats()
|
||||
audioQueue.clear()
|
||||
if (!startArrayForwarding(link)) return false
|
||||
return link.sendControl(configure, "configure", true, true)
|
||||
}
|
||||
|
||||
private fun startArrayForwarding(link: MicArrayLink): Boolean =
|
||||
link.startForwarding(object : FaceclawAudioPacketListener {
|
||||
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
|
||||
queueAudioPacket(data, arm, arrivalMs)
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream.
|
||||
* Between frames it keeps the streams alive: renews the temples' lease, configures a temple
|
||||
* again when it has gone quiet, and takes forwarding back when the glasses session dropped it.
|
||||
*/
|
||||
private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) {
|
||||
val startedAt = platform.elapsedRealtimeMs()
|
||||
var checkedAt = startedAt
|
||||
var renewedAt = startedAt
|
||||
// Left, right.
|
||||
val armedAt = longArrayOf(startedAt, startedAt)
|
||||
val frameAt = longArrayOf(startedAt, startedAt)
|
||||
var waiting = false
|
||||
while (started) {
|
||||
val packet = takeAudioPacket()
|
||||
val now = platform.elapsedRealtimeMs()
|
||||
if (packet != null) {
|
||||
val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1
|
||||
val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null
|
||||
val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null
|
||||
// Anything else is a stock-format packet from before arming took effect.
|
||||
if (frame != null && pcm != null) {
|
||||
frameAt[side] = now
|
||||
val frames = pcm.size / frame.channels
|
||||
decodedSamples += frames.toLong()
|
||||
val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT
|
||||
val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs)
|
||||
chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0
|
||||
chunk.angleDegrees = frame.angleDegrees
|
||||
chunk.ssr = frame.ssr
|
||||
chunk.sequence = frame.sequence
|
||||
chunk.deviceTickMs = frame.deviceTickMs
|
||||
chunk.truncated = frame.truncated
|
||||
handleArrayChunk(chunk)
|
||||
}
|
||||
}
|
||||
if (now - checkedAt >= ARRAY_CHECK_MS) {
|
||||
checkedAt = now
|
||||
if (!link.isForwarding()) {
|
||||
if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) {
|
||||
logInfo("mic array forwarding re-established")
|
||||
armedAt.fill(now)
|
||||
renewedAt = now
|
||||
}
|
||||
} else {
|
||||
if (now - renewedAt >= ARRAY_RENEW_MS) {
|
||||
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true)
|
||||
renewedAt = now
|
||||
}
|
||||
for (side in 0..1) {
|
||||
if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue
|
||||
logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again")
|
||||
link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0)
|
||||
armedAt[side] = now
|
||||
}
|
||||
}
|
||||
val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS
|
||||
if (silent != waiting) {
|
||||
waiting = silent
|
||||
emitStatus(if (silent) ARRAY_WAITING_STATUS else listening)
|
||||
}
|
||||
}
|
||||
maybeEmitAudioStats(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay loop: hands the recording's chunks to the pipeline at the pace they were captured,
|
||||
* so recognition (its partials, a cloud provider's stream) sees what a live capture would.
|
||||
* Jumps and pauses asked for from outside take effect between chunks.
|
||||
*/
|
||||
private fun processReplay(reader: MicRecordingReader) {
|
||||
val duration = reader.durationMs
|
||||
var position = minOf(activeReplayStartMs, duration)
|
||||
reader.seekTo(position)
|
||||
emitReplay("start", position, duration)
|
||||
var reportedAt = position
|
||||
// Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap.
|
||||
var anchorWall = -1L
|
||||
var anchorFile = 0L
|
||||
var pending: MicChunk? = null
|
||||
while (started) {
|
||||
val seek = replaySeekMs
|
||||
if (seek >= 0) {
|
||||
replaySeekMs = -1L
|
||||
// What played so far is a finished utterance; the rest starts clean.
|
||||
finishReplayedUtterance()
|
||||
position = minOf(seek, duration)
|
||||
reader.seekTo(position)
|
||||
pending = null
|
||||
anchorWall = -1L
|
||||
reportedAt = position
|
||||
emitReplay("seek", position, duration)
|
||||
}
|
||||
if (replayPaused) {
|
||||
anchorWall = -1L
|
||||
sleepMs(REPLAY_WAIT_SLICE_MS)
|
||||
continue
|
||||
}
|
||||
val chunk = pending ?: reader.next()
|
||||
if (chunk == null) {
|
||||
emitReplay("end", duration, duration)
|
||||
emitStatus(REPLAY_FINISHED_STATUS)
|
||||
return
|
||||
}
|
||||
pending = chunk
|
||||
val now = platform.elapsedRealtimeMs()
|
||||
if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) {
|
||||
anchorWall = now
|
||||
anchorFile = chunk.timeMs
|
||||
}
|
||||
val wait = anchorWall + (chunk.timeMs - anchorFile) - now
|
||||
if (wait > 0) {
|
||||
// Short slices, so a stop, jump or pause is noticed promptly.
|
||||
sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS))
|
||||
continue
|
||||
}
|
||||
pending = null
|
||||
position = chunk.timeMs
|
||||
decodedSamples += chunk.frames.toLong()
|
||||
handleChunk(chunk)
|
||||
if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) {
|
||||
reportedAt = position
|
||||
emitReplay("position", position, duration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */
|
||||
private fun finishReplayedUtterance() {
|
||||
mixer.reset()
|
||||
endpointDetector.reset()
|
||||
suppressor?.reset()
|
||||
if (mode != Mode.ONBOARD) return
|
||||
decodeTranscript(true)
|
||||
resetTranscriptState()
|
||||
lastTranscript = ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
|
||||
* decode and the beam filter. hasFrameMeta is false for the phone mic,
|
||||
@@ -488,9 +787,6 @@ class VoiceCaptureSession(
|
||||
if (suppressionEnabled) {
|
||||
applySuppression(pcm, count)
|
||||
}
|
||||
if (recordingPcm != null) {
|
||||
appendRecording(pcm, count)
|
||||
}
|
||||
val verifyBuffer = this.verifyBuffer
|
||||
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
|
||||
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
|
||||
@@ -528,7 +824,7 @@ class VoiceCaptureSession(
|
||||
continue
|
||||
}
|
||||
decodedSamples += read.toLong()
|
||||
processPcmChunk(pcm, read, 0, 0, false)
|
||||
handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -714,6 +1010,11 @@ class VoiceCaptureSession(
|
||||
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
|
||||
}
|
||||
|
||||
private fun emitReplay(event: String, positionMs: Long, durationMs: Long) {
|
||||
val currentListener = listener ?: return
|
||||
host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) }
|
||||
}
|
||||
|
||||
private fun emitSpeechEnd() {
|
||||
val currentListener = listener ?: return
|
||||
host.dispatcher.post { currentListener.onSpeechEnd() }
|
||||
@@ -724,8 +1025,21 @@ class VoiceCaptureSession(
|
||||
host.dispatcher.post { currentListener.onStopped(captureId) }
|
||||
}
|
||||
|
||||
private fun stopG2Audio() {
|
||||
host.g2AudioSource()?.stop()
|
||||
/**
|
||||
* Release what the running capture's source holds on the glasses. Runs on stop() and again
|
||||
* as the worker winds down (it may have started the source after stop() looked), so
|
||||
* everything here is safe to repeat.
|
||||
*/
|
||||
private fun stopSource() {
|
||||
val source = lock.withLock { activeSource }
|
||||
when (source) {
|
||||
Source.G2 -> host.g2AudioSource()?.stop()
|
||||
Source.ARRAY -> host.micArrayLink()?.let { link ->
|
||||
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true)
|
||||
link.stopForwarding()
|
||||
}
|
||||
Source.PHONE, Source.REPLAY -> {}
|
||||
}
|
||||
maybeEmitAudioStats(true)
|
||||
}
|
||||
|
||||
@@ -744,7 +1058,7 @@ class VoiceCaptureSession(
|
||||
if (!started || data == null) {
|
||||
return
|
||||
}
|
||||
if ("L" != arm) {
|
||||
if ("L" != arm && activeSource != Source.ARRAY) {
|
||||
wrongArmPackets++
|
||||
}
|
||||
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
|
||||
|
||||
@@ -34,15 +34,19 @@ fun interface CallbackDispatcher {
|
||||
fun post(action: () -> Unit)
|
||||
}
|
||||
|
||||
/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */
|
||||
fun interface RecordingSink {
|
||||
fun save(pcm16le: ByteArray, sampleRate: Int)
|
||||
}
|
||||
|
||||
/** A G2 microphone packet decoder (the shared [Lc3PacketFramer] over a platform LC3 codec). */
|
||||
/**
|
||||
* A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared
|
||||
* [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus.
|
||||
*/
|
||||
interface AudioPacketDecoder {
|
||||
fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int
|
||||
|
||||
/** The most samples one packet decodes to: the size [decodePacket]'s pcmOut needs. */
|
||||
val maxSamplesPerPacket: Int get() = Lc3PacketFramer.SAMPLES_PER_PACKET
|
||||
|
||||
/** Whether packets carry the G2 firmware's DSP metadata ([lastAngleDegrees], [lastSsr]). */
|
||||
val hasFrameMeta: Boolean get() = true
|
||||
|
||||
val lastAngleDegrees: Int
|
||||
val lastSsr: Int
|
||||
val realPackets: Long
|
||||
@@ -63,6 +67,24 @@ interface PacketAudioSource {
|
||||
fun isActive(): Boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* The custom firmware's mic_control channel on a glasses session: each temple streams its own
|
||||
* microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed
|
||||
* stream uses, while it holds a configuration that the phone keeps renewing.
|
||||
*/
|
||||
interface MicArrayLink {
|
||||
/** Deliver the audio characteristic's packets to [listener]; false when no session is up. */
|
||||
fun startForwarding(listener: FaceclawAudioPacketListener): Boolean
|
||||
|
||||
fun stopForwarding()
|
||||
|
||||
/** Whether forwarding still holds: it lives in the glasses session, which can drop it. */
|
||||
fun isForwarding(): Boolean
|
||||
|
||||
/** Queue a control record for the chosen temples; false when nothing was queued. */
|
||||
fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean
|
||||
}
|
||||
|
||||
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
|
||||
interface PcmAudioSource {
|
||||
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
interface FaceclawKeyboardEventListener {
|
||||
/**
|
||||
* One CFW keyboard-bridge record (settings field 131) from the glasses: the raw
|
||||
* ['K','B',ver,kind,...] bytes, possibly one fragment of a longer record. Decoded and
|
||||
* reassembled on the TS side (app/g2/keyboard/protocol.ts). Delivered in arrival order.
|
||||
*/
|
||||
fun onKeyboardEvent(body: ByteArray?): Unit
|
||||
}
|
||||
+9
@@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener {
|
||||
* speaker model could not be loaded.
|
||||
*/
|
||||
fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit
|
||||
|
||||
/**
|
||||
* Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay):
|
||||
* "start" once the file is open, "position" about once per second of it played, "seek" after
|
||||
* a jump took effect (the transcript of what played before it has been emitted as final),
|
||||
* "end" when the file ran out, and "error" when it could not be opened (the status says
|
||||
* why). Positions are milliseconds into the file.
|
||||
*/
|
||||
fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit
|
||||
}
|
||||
@@ -0,0 +1,794 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.concurrent.Volatile
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
class G1FlashTimings(
|
||||
@JvmField val connectTimeoutMs: Int = ConnectionOptions.CONNECT_TIMEOUT_MS,
|
||||
@JvmField val servicesTimeoutMs: Int = ConnectionOptions.SERVICES_TIMEOUT_MS,
|
||||
@JvmField val descriptorTimeoutMs: Int = ConnectionOptions.DESCRIPTOR_TIMEOUT_MS,
|
||||
@JvmField val writeTimeoutMs: Int = ConnectionOptions.WRITE_TIMEOUT_MS,
|
||||
/** The OS may be showing its pairing dialog. */
|
||||
@JvmField val pairingMs: Int = 60_000,
|
||||
@JvmField val retryDelayMs: Int = 2_000,
|
||||
@JvmField val controlReplyMs: Int = 1_500,
|
||||
@JvmField val controlTries: Int = 4,
|
||||
/** The first chunk of an upload waits for the arm to erase its slot. */
|
||||
@JvmField val uploadReplyMs: Int = 30_000,
|
||||
@JvmField val smpReplyMs: Int = 20_000,
|
||||
/** The arm may go down before its reply leaves. */
|
||||
@JvmField val resetReplyMs: Int = 5_000,
|
||||
@JvmField val heartbeatMs: Int = 8_000,
|
||||
@JvmField val pollMs: Int = 200,
|
||||
@JvmField val uploadRetries: Int = 40,
|
||||
/** How often a store job's state is asked for while its notice hasn't come. */
|
||||
@JvmField val storePollMs: Int = 2_000,
|
||||
/** Erasing the store over Even's old fonts and pictures takes minutes. */
|
||||
@JvmField val wipeMs: Int = 1_800_000,
|
||||
@JvmField val crcMs: Int = 30_000,
|
||||
)
|
||||
|
||||
/**
|
||||
* Installs an MCUboot image on one G1 arm through MCUmgr SMP, or asks an arm what it runs; a port of scouter_kit's
|
||||
* g1_firmware/flash.py. The caller handles one arm at a time and looks for the arm again afterwards: it restarts, and
|
||||
* coming off firmware older than g1v0 0.5.66 it has a new address.
|
||||
*
|
||||
* Nothing changes on the arm until the confirm, which comes only after the arm's image list shows that its second slot
|
||||
* holds exactly this image. MCUboot then overwrites the running firmware with no way back. [run] blocks; the worker thread
|
||||
* is the host's.
|
||||
*
|
||||
* [Mode.RESTORE] puts Even's firmware back on an arm running g1v0: both of a release's images, the application core's
|
||||
* ([firmwarePath]) and the network core's ([netCorePath]), as Even's own update does, so the two always match.
|
||||
*/
|
||||
class G1ArmFlashFlow(
|
||||
private val link: StockLink,
|
||||
private val address: String,
|
||||
private val side: String,
|
||||
private val firmwarePath: String,
|
||||
private val netCorePath: String,
|
||||
private val mode: Mode,
|
||||
/** How long to keep trying to reach the arm and pair with it. */
|
||||
private val connectWindowMs: Int,
|
||||
private val listener: FaceclawFirmwareFlasherListener,
|
||||
private val timings: G1FlashTimings = G1FlashTimings(),
|
||||
platform: ProtocolPlatform = protocolPlatform(),
|
||||
private val readImage: (String) -> ByteArray = ::readFileData,
|
||||
) : StockLinkListener {
|
||||
enum class Mode {
|
||||
/** The arm runs Even's firmware. */
|
||||
STOCK,
|
||||
/** The arm runs g1v0: it answers what it runs, and an arm that already has this version is left alone. */
|
||||
UPDATE,
|
||||
/** Only say what the arm runs, and pair with it. */
|
||||
PROBE,
|
||||
/** The arm runs g1v0 and gets Even's firmware back. */
|
||||
RESTORE,
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val TAG = "FaceclawG1Flash"
|
||||
private const val MIN_BATTERY_PERCENT = 30
|
||||
private const val ANY_STORE_JOB = -1
|
||||
private const val STORE_JOB_INTERRUPTED = -1
|
||||
|
||||
fun mode(name: String): Mode = when (name) {
|
||||
"stock" -> Mode.STOCK
|
||||
"update" -> Mode.UPDATE
|
||||
"probe" -> Mode.PROBE
|
||||
"restore" -> Mode.RESTORE
|
||||
else -> throw IllegalArgumentException("unknown mode $name")
|
||||
}
|
||||
}
|
||||
|
||||
/** One image of an install: MCUboot's number for it, its bytes and the hash its slot is checked against. */
|
||||
private class Part(val imageIndex: Int, val bytes: ByteArray, val label: String) {
|
||||
val hash: ByteArray = G1Smp.imageHash(bytes)
|
||||
}
|
||||
|
||||
private class LinkLostException(message: String) : Exception(message)
|
||||
|
||||
/** The arm has no g1v0 service. */
|
||||
private class NotG1v0Exception : Exception("this arm doesn't run g1v0")
|
||||
|
||||
private val platform = platform
|
||||
private val lock = platform.createLock()
|
||||
private val sleeper = InterruptibleSleep(platform)
|
||||
private val assembler = G1Smp.Assembler()
|
||||
|
||||
private var smpSeq = -1
|
||||
private var smpReply: Map<*, *>? = null
|
||||
private var smpLatch: Latch? = null
|
||||
private var nextSmpSeq = 0
|
||||
|
||||
private var controlTag = -1
|
||||
private var controlReply: G1Protocol.Reply? = null
|
||||
private var controlLatch: Latch? = null
|
||||
private var nextControlTag = 0
|
||||
|
||||
private var stockBattery: Int? = null
|
||||
private var stockBatteryLatch: Latch? = null
|
||||
|
||||
/** `NOTICE` 8s since the store job now waited for was asked for. */
|
||||
private val storeNotices = ArrayList<G1Protocol.StoreNotice>()
|
||||
|
||||
/** The OS's half of a stale pairing goes once per run: a second failure isn't that. */
|
||||
private var bondRemoved = false
|
||||
|
||||
private var linkLost = false
|
||||
private var heartbeats = 0
|
||||
private var nextHeartbeatAtMs = 0L
|
||||
|
||||
@Volatile
|
||||
private var cancelled = false
|
||||
|
||||
fun cancel() {
|
||||
cancelled = true
|
||||
lock.withLock {
|
||||
smpLatch?.countDown()
|
||||
controlLatch?.countDown()
|
||||
}
|
||||
sleeper.interrupt()
|
||||
}
|
||||
|
||||
fun run() {
|
||||
link.setListener(this)
|
||||
try {
|
||||
when (mode) {
|
||||
Mode.PROBE -> probe()
|
||||
Mode.RESTORE -> restore()
|
||||
else -> install()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
val message = if (cancelled) "Cancelled." else StockLinkSession.messageOf(e)
|
||||
log("failed: $message")
|
||||
listener.onState("error", message)
|
||||
listener.onComplete(false, message)
|
||||
} finally {
|
||||
quietly { link.disconnect(address) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun probe() {
|
||||
listener.onState("connecting", side)
|
||||
try {
|
||||
bringUp()
|
||||
} catch (e: NotG1v0Exception) {
|
||||
listener.onComplete(true, """{"firmware":"stock"}""")
|
||||
return
|
||||
}
|
||||
listener.onState("checking", side)
|
||||
val hello = hello()
|
||||
listener.onComplete(true, """{"firmware":"g1v0","version":"${hello.version}","safeMode":${hello.safeMode},"inCase":${hello.inCase},"battery":${batteryPercent()}}""")
|
||||
}
|
||||
|
||||
private fun readFirmware(path: String): ByteArray = try {
|
||||
readImage(path)
|
||||
} catch (e: Exception) {
|
||||
throw IllegalStateException("could not read the firmware: " + StockLinkSession.messageOf(e), e)
|
||||
}
|
||||
|
||||
private fun install() {
|
||||
listener.onState("validating", side)
|
||||
val image = readFirmware(firmwarePath)
|
||||
val version = G1Smp.imageVersion(image)
|
||||
val hash = G1Smp.imageHash(image)
|
||||
log("image $version, ${image.size} bytes, hash ${BinaryEncoding.bytesToHex(hash.copyOf(8))}")
|
||||
|
||||
listener.onState("connecting", side)
|
||||
bringUp()
|
||||
if (mode == Mode.UPDATE) {
|
||||
listener.onState("checking", side)
|
||||
val hello = hello()
|
||||
log("runs g1v0 ${hello.version}, safe mode ${hello.safeMode}, in the case ${hello.inCase}, encrypted ${hello.encrypted}")
|
||||
if (hello.version == version && !hello.safeMode) {
|
||||
listener.onComplete(true, """{"flashed":false,"version":"$version"}""")
|
||||
return
|
||||
}
|
||||
requireBattery(batteryPercent())
|
||||
} else {
|
||||
val battery = stockBatteryPercent()
|
||||
log(if (battery == null) "battery unknown; going on, as the arm is in the case" else "battery $battery%")
|
||||
requireBattery(battery)
|
||||
}
|
||||
|
||||
val slots = retrying { images() }
|
||||
for (slot in slots) log(slot.toString())
|
||||
listener.onState("flashing", side)
|
||||
if (holdsImage(slots, G1Smp.IMAGE_APP, hash)) {
|
||||
log("the second slot already holds this image")
|
||||
listener.onProgress(side, 1, 1, image.size, image.size, image.size.toLong(), image.size.toLong())
|
||||
} else {
|
||||
val startedAt = now()
|
||||
upload(image, G1Smp.IMAGE_APP)
|
||||
log("uploaded in ${(now() - startedAt) / 1000} s")
|
||||
}
|
||||
if (!holdsImage(retrying { images() }, G1Smp.IMAGE_APP, hash)) {
|
||||
throw IllegalStateException("The $side arm's second slot doesn't hold the image after the upload. Nothing was installed.")
|
||||
}
|
||||
throwIfCancelled()
|
||||
|
||||
listener.onState("committing", side)
|
||||
val confirm = smp(G1Smp.OP_WRITE, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_STATE, G1Smp.confirmRequest(hash), timings.smpReplyMs)
|
||||
val code = G1Smp.replyCode(confirm)
|
||||
if (code != 0) throw IllegalStateException("The $side arm refused the image (error $code). Nothing was installed.")
|
||||
log("confirmed: MCUboot installs it at the next boot")
|
||||
reset()
|
||||
listener.onState("rebooting", side)
|
||||
listener.onComplete(true, """{"flashed":true,"version":"$version"}""")
|
||||
}
|
||||
|
||||
/**
|
||||
* Even's firmware back on an arm running g1v0: the store cleared if g1v0 ever wrote to it, then both images uploaded
|
||||
* and checked, and only then marked for MCUboot, the network core first. MCUboot installs both at the reset, the
|
||||
* network core through its own path to that core, as at every update from Even.
|
||||
*/
|
||||
private fun restore() {
|
||||
listener.onState("validating", side)
|
||||
val parts = listOf(
|
||||
Part(G1Smp.IMAGE_NET_CORE, readFirmware(netCorePath), "network core"),
|
||||
Part(G1Smp.IMAGE_APP, readFirmware(firmwarePath), "application"),
|
||||
)
|
||||
for (part in parts) log("${part.label} image: ${part.bytes.size} bytes, hash ${BinaryEncoding.bytesToHex(part.hash.copyOf(8))}")
|
||||
|
||||
listener.onState("connecting", side)
|
||||
bringUp()
|
||||
listener.onState("checking", side)
|
||||
val hello = hello()
|
||||
log("runs g1v0 ${hello.version}, safe mode ${hello.safeMode}, in the case ${hello.inCase}, encrypted ${hello.encrypted}")
|
||||
requireBattery(batteryPercent())
|
||||
val wiped = clearStore(hello)
|
||||
|
||||
val slots = retrying { images() }
|
||||
for (slot in slots) log(slot.toString())
|
||||
listener.onState("flashing", side)
|
||||
val total = parts.sumOf { it.bytes.size }
|
||||
var before = 0
|
||||
for ((index, part) in parts.withIndex()) {
|
||||
if (holdsImage(slots, part.imageIndex, part.hash)) {
|
||||
log("the ${part.label}'s second slot already holds Even's image")
|
||||
listener.onProgress(side, index + 1, parts.size, part.bytes.size, part.bytes.size, (before + part.bytes.size).toLong(), total.toLong())
|
||||
} else {
|
||||
val startedAt = now()
|
||||
upload(part.bytes, part.imageIndex, index + 1, parts.size, before, total)
|
||||
log("${part.label} uploaded in ${(now() - startedAt) / 1000} s")
|
||||
}
|
||||
before += part.bytes.size
|
||||
}
|
||||
val uploaded = retrying { images() }
|
||||
for (part in parts) {
|
||||
if (!holdsImage(uploaded, part.imageIndex, part.hash)) {
|
||||
throw IllegalStateException("The $side arm's slot for its ${part.label} doesn't hold Even's image after the upload. Nothing was installed.")
|
||||
}
|
||||
}
|
||||
throwIfCancelled()
|
||||
|
||||
listener.onState("committing", side)
|
||||
// The network core first. If the second request fails, what's left pending is Even's network core under g1v0,
|
||||
// which g1v0 works with (its fallback installs this very image); Even's application over the older network core
|
||||
// is the combination that has never been tried.
|
||||
for (part in parts) {
|
||||
val permanent = part.imageIndex == G1Smp.IMAGE_APP
|
||||
val reply = smp(G1Smp.OP_WRITE, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_STATE, G1Smp.confirmRequest(part.hash, permanent), timings.smpReplyMs)
|
||||
val code = G1Smp.replyCode(reply)
|
||||
if (code != 0) {
|
||||
throw IllegalStateException("The $side arm refused Even's ${part.label} image (error $code). The arm wasn't restarted; retry to try again.")
|
||||
}
|
||||
}
|
||||
val marked = retrying { images() }
|
||||
for (slot in marked) log(slot.toString())
|
||||
for (part in parts) {
|
||||
if (!marked.any { it.image == part.imageIndex && it.slot == 1 && it.hash.contentEquals(part.hash) && "pending" in it.flags }) {
|
||||
throw IllegalStateException("The $side arm didn't mark Even's ${part.label} image for installing. The arm wasn't restarted; retry to try again.")
|
||||
}
|
||||
}
|
||||
log("both images marked: MCUboot installs them at the next boot")
|
||||
reset()
|
||||
listener.onState("rebooting", side)
|
||||
listener.onComplete(true, """{"flashed":true,"firmware":"stock","wiped":$wiped}""")
|
||||
}
|
||||
|
||||
private fun reset() {
|
||||
try {
|
||||
smp(G1Smp.OP_WRITE, G1Smp.GROUP_OS, G1Smp.CMD_OS_RESET, emptyMap(), timings.resetReplyMs)
|
||||
} catch (e: StockTimeoutException) {
|
||||
log("no answer to the reset")
|
||||
} catch (e: LinkLostException) {
|
||||
log("the link went down with the reset")
|
||||
}
|
||||
}
|
||||
|
||||
// ---- the asset store ---------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Before Even's firmware goes back: g1v0's asset store took the flash where Even's fonts and pictures were, so if it
|
||||
* ever wrote there, Even's firmware must meet erased flash (flash.py's `wipe_store`). A store that was never written
|
||||
* is left alone, and Even's data with it. True when it was erased.
|
||||
*/
|
||||
private fun clearStore(hello: G1Protocol.Hello): Boolean {
|
||||
repeat(3) {
|
||||
val info = assetInfo()
|
||||
if (info == null) {
|
||||
log("no asset store: this firmware never wrote where Even's data is")
|
||||
return false
|
||||
}
|
||||
if (!info.available) {
|
||||
log("the asset store is unavailable, so nothing was written to it")
|
||||
return false
|
||||
}
|
||||
if (info.running) {
|
||||
// A retried request can find its own job running, as can a retry after a link lost during a wipe.
|
||||
log("a storage job is running: waiting for it to end")
|
||||
clearStoreNotices()
|
||||
waitStoreJob(ANY_STORE_JOB, timings.wipeMs.toLong())
|
||||
return@repeat
|
||||
}
|
||||
if (!storeWritten(info, hello)) {
|
||||
log("the asset store was never written: Even's fonts and pictures stay as they are")
|
||||
return false
|
||||
}
|
||||
if (hello.safeMode) throw IllegalStateException(safeModeMessage())
|
||||
listener.onState("wiping", side)
|
||||
log("erasing the asset store (${info.assets} assets, commit ${info.seq})")
|
||||
val startedAt = now()
|
||||
clearStoreNotices()
|
||||
when (val status = control(G1Protocol.OP_ASSET_FORMAT, G1Protocol.assetFormatRequest(wipe = true)).status) {
|
||||
0 -> {
|
||||
val result = waitStoreJob(G1Protocol.STORE_JOB_FORMAT, timings.wipeMs.toLong())
|
||||
if (result == STORE_JOB_INTERRUPTED) {
|
||||
throw IllegalStateException("The $side arm restarted while erasing Faceclaw's storage. Nothing was installed; retry to finish it.")
|
||||
}
|
||||
if (result != 0) {
|
||||
throw IllegalStateException("The $side arm couldn't erase Faceclaw's storage (error $result). Nothing was installed.")
|
||||
}
|
||||
log("asset store erased in ${(now() - startedAt) / 1000} s")
|
||||
return true
|
||||
}
|
||||
G1Protocol.STATUS_SAFE_MODE -> throw IllegalStateException(safeModeMessage())
|
||||
// A job runs: the next round waits for it.
|
||||
G1Protocol.STATUS_ASSET -> {}
|
||||
else -> throw IllegalStateException("The $side arm refused to erase Faceclaw's storage (error $status). Nothing was installed.")
|
||||
}
|
||||
}
|
||||
throw IllegalStateException("The $side arm's storage couldn't be erased. Nothing was installed.")
|
||||
}
|
||||
|
||||
private fun safeModeMessage(): String =
|
||||
"The $side arm is in safe mode (its firmware restarted repeatedly), where it can't erase Faceclaw's storage, and that has to " +
|
||||
"happen before Even's firmware goes back. Install Faceclaw's firmware again, then uninstall it. Nothing was installed."
|
||||
|
||||
/**
|
||||
* Whether g1v0 ever wrote to the store. It only reads it at boot, so with no assets and no commit nothing was written,
|
||||
* unless a format erased the directory's two sectors (a format resets the commit count), which their fingerprint shows.
|
||||
*/
|
||||
private fun storeWritten(info: G1Protocol.AssetInfo, hello: G1Protocol.Hello): Boolean {
|
||||
if (info.assets > 0 || info.seq != 0L) return true
|
||||
if (hello.safeMode) {
|
||||
log("safe mode: the directory can't be checked")
|
||||
return true
|
||||
}
|
||||
val nonErased = flashNonErased(G1Protocol.STORE_ADDRESS, G1Protocol.STORE_DIRECTORY_BYTES)
|
||||
log("the store's directory sectors: ${nonErased ?: "unknown"} bytes not erased")
|
||||
return nonErased == null || nonErased == 0L
|
||||
}
|
||||
|
||||
/** How many bytes of flash [from]…[from] + [n] - 1 aren't 0xFF, or null when the arm didn't say. */
|
||||
private fun flashNonErased(from: Int, n: Int): Long? {
|
||||
clearStoreNotices()
|
||||
val status = try {
|
||||
control(G1Protocol.OP_FLASH_CRC, G1Protocol.flashCrcRequest(from, n)).status
|
||||
} catch (e: StockTimeoutException) {
|
||||
return null
|
||||
}
|
||||
// 13: a crc job runs, which a retried request finds when the first one's reply was lost; its notice says the same.
|
||||
if (status != 0 && status != G1Protocol.STATUS_ASSET) return null
|
||||
val deadline = now() + timings.crcMs
|
||||
while (now() < deadline) {
|
||||
throwIfCancelled()
|
||||
val notice = lock.withLock {
|
||||
storeNotices.firstOrNull { it.job == G1Protocol.STORE_JOB_CRC && it.address == from && it.n == n }
|
||||
}
|
||||
if (notice != null) return if (notice.result == 0) notice.nonErased else null
|
||||
if (!link.isConnected(address)) return null
|
||||
sleeper.sleep(timings.pollMs.toLong())
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun assetInfo(): G1Protocol.AssetInfo? {
|
||||
val reply = control(G1Protocol.OP_ASSET_INFO)
|
||||
return if (reply.status == 0) G1Protocol.parseAssetInfo(reply.body) else null
|
||||
}
|
||||
|
||||
private fun clearStoreNotices() = lock.withLock { storeNotices.clear() }
|
||||
|
||||
/**
|
||||
* The result of the store job running now: its `NOTICE` 8, or `ASSET_INFO`'s state once it has ended (a notice is lost
|
||||
* with the link, and the job goes on without it). [STORE_JOB_INTERRUPTED] when the arm restarted meanwhile, which ends
|
||||
* a job unfinished.
|
||||
*/
|
||||
private fun waitStoreJob(job: Int, timeoutMs: Long): Int {
|
||||
val deadline = now() + timeoutMs
|
||||
var askAt = now() + timings.storePollMs
|
||||
while (true) {
|
||||
throwIfCancelled()
|
||||
val notice = lock.withLock { storeNotices.firstOrNull { job == ANY_STORE_JOB || it.job == job } }
|
||||
if (notice != null) return notice.result
|
||||
if (now() >= deadline) throw StockTimeoutException("the $side arm's storage job didn't end within ${timeoutMs / 60_000} minutes")
|
||||
if (now() >= askAt) {
|
||||
askAt = now() + timings.storePollMs
|
||||
val info = try {
|
||||
assetInfo()
|
||||
} catch (e: Exception) {
|
||||
if (e !is StockTimeoutException && e !is LinkLostException) throw e
|
||||
log("no answer while the storage job runs (${e.message})")
|
||||
reconnectIfNeeded()
|
||||
null
|
||||
}
|
||||
when (info?.state) {
|
||||
G1Protocol.STORE_DONE -> return info.lastError
|
||||
G1Protocol.STORE_FAILED -> return if (info.lastError != 0) info.lastError else G1Protocol.STATUS_ASSET
|
||||
// A job leaves the state at done or failed until the next boot.
|
||||
G1Protocol.STORE_IDLE -> return STORE_JOB_INTERRUPTED
|
||||
}
|
||||
}
|
||||
sleeper.sleep(timings.pollMs.toLong())
|
||||
}
|
||||
}
|
||||
|
||||
private fun requireBattery(percent: Int?) {
|
||||
if (percent != null && percent < MIN_BATTERY_PERCENT) {
|
||||
throw IllegalStateException("The $side arm's battery is at $percent%, too low to install safely. " +
|
||||
"Charge both arms to at least $MIN_BATTERY_PERCENT% in the case, then try again.")
|
||||
}
|
||||
}
|
||||
|
||||
private fun holdsImage(slots: List<G1Smp.Slot>, imageIndex: Int, hash: ByteArray): Boolean =
|
||||
slots.any { it.image == imageIndex && it.slot == 1 && it.hash.contentEquals(hash) }
|
||||
|
||||
private fun images(): List<G1Smp.Slot> =
|
||||
G1Smp.parseSlots(smp(G1Smp.OP_READ, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_STATE, emptyMap(), timings.smpReplyMs))
|
||||
|
||||
/**
|
||||
* MCUmgr answers each chunk with the offset it wants next, so an upload resumes after a lost link. Progress counts
|
||||
* [before] bytes of earlier parts out of [total].
|
||||
*/
|
||||
private fun upload(image: ByteArray, imageIndex: Int, part: Int = 1, parts: Int = 1, before: Int = 0, total: Int = image.size) {
|
||||
val fileSha = sha256(image)
|
||||
var offset = 0
|
||||
var failures = 0
|
||||
while (offset < image.size) {
|
||||
throwIfCancelled()
|
||||
val reply = try {
|
||||
smp(G1Smp.OP_WRITE, G1Smp.GROUP_IMAGE, G1Smp.CMD_IMAGE_UPLOAD, G1Smp.uploadRequest(image, offset, fileSha, imageIndex), timings.uploadReplyMs)
|
||||
} catch (e: Exception) {
|
||||
if (e !is StockTimeoutException && e !is LinkLostException) throw e
|
||||
failures++
|
||||
if (failures > timings.uploadRetries) throw e
|
||||
log("chunk at $offset failed (${e.message}), trying again")
|
||||
reconnectIfNeeded()
|
||||
continue
|
||||
}
|
||||
val code = G1Smp.replyCode(reply)
|
||||
val next = G1Smp.replyOffset(reply)
|
||||
if (code != 0 || next == null || next < 0 || next > image.size) {
|
||||
throw IllegalStateException("The $side arm refused the upload at $offset (error $code). Nothing was installed.")
|
||||
}
|
||||
offset = next
|
||||
listener.onProgress(side, part, parts, offset, image.size, (before + offset).toLong(), total.toLong())
|
||||
}
|
||||
}
|
||||
|
||||
private fun <T> retrying(request: () -> T): T {
|
||||
var attempt = 0
|
||||
while (true) {
|
||||
try {
|
||||
return request()
|
||||
} catch (e: Exception) {
|
||||
if (e !is StockTimeoutException && e !is LinkLostException) throw e
|
||||
attempt++
|
||||
if (attempt >= 10) throw e
|
||||
log("request failed (${e.message}), trying again")
|
||||
reconnectIfNeeded()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun reconnectIfNeeded() {
|
||||
throwIfCancelled()
|
||||
lock.withLock { assembler.reset() }
|
||||
if (!link.isConnected(address)) bringUp()
|
||||
}
|
||||
|
||||
// ---- the link ------------------------------------------------------------
|
||||
|
||||
private fun bringUp() {
|
||||
val deadline = now() + connectWindowMs
|
||||
var attempt = 0
|
||||
while (true) {
|
||||
throwIfCancelled()
|
||||
attempt++
|
||||
val failure = try {
|
||||
connectOnce()
|
||||
log("connected (attempt $attempt)")
|
||||
return
|
||||
} catch (e: NotG1v0Exception) {
|
||||
throw e
|
||||
} catch (e: Exception) {
|
||||
StockLinkSession.messageOf(e)
|
||||
}
|
||||
log("connect attempt $attempt failed: $failure")
|
||||
quietly { link.disconnect(address) }
|
||||
if (now() >= deadline) {
|
||||
throw IllegalStateException("Couldn't reach the $side arm: $failure. Keep the glasses in the case with the lid open; a fully " +
|
||||
"charged arm can take up to 5 minutes to notice it's in the case. If this phone was paired with these glasses before and " +
|
||||
"this keeps happening, remove them from its Bluetooth settings and try again.")
|
||||
}
|
||||
sleeper.sleep(timings.retryDelayMs.toLong())
|
||||
}
|
||||
}
|
||||
|
||||
private fun connectOnce() {
|
||||
if (!link.connect(address, timings.connectTimeoutMs)) throw IllegalStateException("connect failed")
|
||||
lock.withLock {
|
||||
linkLost = false
|
||||
assembler.reset()
|
||||
}
|
||||
link.prepareLink(address, G1Ble.REQUESTED_MTU, timings.connectTimeoutMs)
|
||||
if (!link.discoverServices(address, timings.servicesTimeoutMs)) throw IllegalStateException("service discovery failed")
|
||||
if (mode == Mode.STOCK) {
|
||||
// Even's firmware serves its own service only on an encrypted link, and the heartbeat goes there.
|
||||
settleBond()
|
||||
if (!haveCharacteristic(G1Smp.NUS_TX_UUID)) throw IllegalStateException("no link to Even's service")
|
||||
if (!subscribe(G1Smp.NUS_TX_UUID)) {
|
||||
// Bonded and refused: Even's firmware keeps one bond and has lost this one. This run started in the case.
|
||||
removeStaleBond()
|
||||
throw IllegalStateException("no link to Even's service")
|
||||
}
|
||||
heartbeats = 0
|
||||
nextHeartbeatAtMs = 0
|
||||
} else {
|
||||
if (!haveCharacteristic(G1Ble.EVT_CHAR_UUID)) {
|
||||
if (!link.isConnected(address)) throw LinkLostException("the link went down")
|
||||
throw NotG1v0Exception()
|
||||
}
|
||||
if (!subscribe(G1Ble.EVT_CHAR_UUID)) throw IllegalStateException("no events from the arm")
|
||||
// Out of the case the arm obeys only an encrypted link.
|
||||
settleBond()
|
||||
val paired = try {
|
||||
link.writeFrames(address, G1Ble.PAIR_CHAR_UUID, listOf(byteArrayOf(1)), GattWriteMode.WITH_RESPONSE, timings.pairingMs)
|
||||
} catch (e: IllegalStateException) {
|
||||
false
|
||||
}
|
||||
if (!paired) {
|
||||
// The OS says bonded and the arm disagrees. Giving that up is safe only where a new bond can be made.
|
||||
if (link.isConnected(address) && inCaseWithoutEncryption()) removeStaleBond()
|
||||
throw IllegalStateException("no encrypted link: an arm pairs only in its case")
|
||||
}
|
||||
}
|
||||
if (mode != Mode.PROBE && (!haveCharacteristic(G1Smp.SMP_CHAR_UUID) || !subscribe(G1Smp.SMP_CHAR_UUID))) {
|
||||
throw IllegalStateException("no update service")
|
||||
}
|
||||
}
|
||||
|
||||
/** An arm keeps its address across an install, and the OS may remember the services it had before: look again before believing that. */
|
||||
private fun haveCharacteristic(uuid: String): Boolean {
|
||||
if (link.hasCharacteristic(address, uuid)) return true
|
||||
log("no $uuid in the service table: reading it again")
|
||||
link.refreshServices(address, timings.servicesTimeoutMs)
|
||||
return link.hasCharacteristic(address, uuid)
|
||||
}
|
||||
|
||||
private fun subscribe(uuid: String): Boolean = try {
|
||||
link.enableNotifications(address, uuid, timings.descriptorTimeoutMs)
|
||||
} catch (e: IllegalStateException) {
|
||||
false
|
||||
}
|
||||
|
||||
private fun removeStaleBond() {
|
||||
if (bondRemoved || link.bondState(address) != BondState.BONDED) return
|
||||
if (link.removeBond(address)) {
|
||||
bondRemoved = true
|
||||
log("the arm no longer has this phone's pairing: removed it, pairing again")
|
||||
}
|
||||
}
|
||||
|
||||
/** An unencrypted link gets an answer only from an arm in the case. */
|
||||
private fun inCaseWithoutEncryption(): Boolean = try {
|
||||
val hello = G1Protocol.parseHello(control(G1Protocol.OP_HELLO, tries = 1).body)
|
||||
hello != null && hello.inCase && !hello.encrypted
|
||||
} catch (e: Exception) {
|
||||
false
|
||||
}
|
||||
|
||||
/** Null when no answer came, which firmware that doesn't have the query also gives. */
|
||||
private fun stockBatteryPercent(): Int? {
|
||||
repeat(timings.controlTries) {
|
||||
val latch = Latch(1, platform)
|
||||
lock.withLock {
|
||||
stockBattery = null
|
||||
stockBatteryLatch = latch
|
||||
}
|
||||
try {
|
||||
link.writeFrames(address, G1Smp.NUS_RX_UUID, listOf(G1Smp.STOCK_BATTERY_QUERY), GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
|
||||
} catch (e: IllegalStateException) {
|
||||
return null
|
||||
}
|
||||
latch.await(timings.controlReplyMs.toLong())
|
||||
throwIfCancelled()
|
||||
val answer = lock.withLock {
|
||||
stockBatteryLatch = null
|
||||
stockBattery
|
||||
}
|
||||
if (answer != null) return answer
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Asks the OS to pair if it hasn't, and waits for that to end; a platform that doesn't say pairs by itself. */
|
||||
private fun settleBond() {
|
||||
if (link.bondState(address) == BondState.NONE) {
|
||||
listener.onState("pairing", side)
|
||||
link.requestBond(address)
|
||||
}
|
||||
val deadline = now() + timings.pairingMs
|
||||
var sawBonding = false
|
||||
while (true) {
|
||||
throwIfCancelled()
|
||||
when (link.bondState(address)) {
|
||||
BondState.BONDED, BondState.UNKNOWN -> return
|
||||
BondState.BONDING -> sawBonding = true
|
||||
BondState.NONE -> if (sawBonding) throw IllegalStateException("pairing was refused")
|
||||
}
|
||||
if (!link.isConnected(address)) throw LinkLostException("the link went down while pairing")
|
||||
if (now() >= deadline) throw IllegalStateException("pairing took too long")
|
||||
sleeper.sleep(timings.pollMs.toLong())
|
||||
}
|
||||
}
|
||||
|
||||
private fun heartbeatIfDue() {
|
||||
if (mode != Mode.STOCK || now() < nextHeartbeatAtMs) return
|
||||
val seq = heartbeats % 0xff
|
||||
heartbeats++
|
||||
nextHeartbeatAtMs = now() + timings.heartbeatMs
|
||||
try {
|
||||
link.writeFrames(address, G1Smp.NUS_RX_UUID, listOf(G1Smp.stockHeartbeat(seq)), GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
|
||||
} catch (e: IllegalStateException) {
|
||||
// The request waiting now finds out.
|
||||
}
|
||||
}
|
||||
|
||||
private fun smp(op: Int, group: Int, cmd: Int, payload: Map<String, Any?>, timeoutMs: Int): Map<*, *> {
|
||||
val seq = nextSmpSeq
|
||||
nextSmpSeq = (nextSmpSeq + 1) and 0xff
|
||||
val latch = Latch(1, platform)
|
||||
lock.withLock {
|
||||
smpSeq = seq
|
||||
smpReply = null
|
||||
smpLatch = latch
|
||||
}
|
||||
try {
|
||||
heartbeatIfDue()
|
||||
val frames = G1Smp.split(G1Smp.frame(op, group, seq, cmd, payload), link.negotiatedMtu(address))
|
||||
val written = try {
|
||||
link.writeFrames(address, G1Smp.SMP_CHAR_UUID, frames, GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
|
||||
} catch (e: IllegalStateException) {
|
||||
false
|
||||
}
|
||||
if (!written) throw LinkLostException("write failed")
|
||||
val deadline = now() + timeoutMs
|
||||
while (true) {
|
||||
throwIfCancelled()
|
||||
heartbeatIfDue()
|
||||
val remaining = deadline - now()
|
||||
if (remaining <= 0) throw StockTimeoutException("no answer from the $side arm within $timeoutMs ms")
|
||||
latch.await(minOf(remaining, timings.pollMs.toLong()))
|
||||
val (reply, lost) = lock.withLock { smpReply to linkLost }
|
||||
if (reply != null) return reply
|
||||
if (lost) throw LinkLostException("the link went down")
|
||||
}
|
||||
} finally {
|
||||
lock.withLock {
|
||||
if (smpLatch === latch) smpLatch = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun control(op: Int, body: ByteArray = ByteArray(0), tries: Int = timings.controlTries): G1Protocol.Reply {
|
||||
repeat(tries) {
|
||||
val tag = nextControlTag % 255 + 1
|
||||
nextControlTag = tag
|
||||
val latch = Latch(1, platform)
|
||||
lock.withLock {
|
||||
controlTag = tag
|
||||
controlReply = null
|
||||
controlLatch = latch
|
||||
}
|
||||
val written = try {
|
||||
link.writeFrames(address, G1Ble.CTRL_CHAR_UUID, listOf(G1Protocol.control(op, tag, body)), GattWriteMode.WITHOUT_RESPONSE, timings.writeTimeoutMs)
|
||||
} catch (e: IllegalStateException) {
|
||||
false
|
||||
}
|
||||
if (!written) throw LinkLostException("write failed")
|
||||
latch.await(timings.controlReplyMs.toLong())
|
||||
throwIfCancelled()
|
||||
val (reply, lost) = lock.withLock {
|
||||
controlLatch = null
|
||||
controlReply to linkLost
|
||||
}
|
||||
if (reply != null) return reply
|
||||
if (lost) throw LinkLostException("the link went down")
|
||||
}
|
||||
throw StockTimeoutException("no answer to request $op from the $side arm")
|
||||
}
|
||||
|
||||
private fun hello(): G1Protocol.Hello {
|
||||
val reply = control(G1Protocol.OP_HELLO)
|
||||
return G1Protocol.parseHello(reply.body) ?: throw IllegalStateException("The $side arm's firmware isn't one this app knows.")
|
||||
}
|
||||
|
||||
/** Null when the arm doesn't know yet, or a firmware doesn't say. */
|
||||
private fun batteryPercent(): Int? {
|
||||
val reply = control(G1Protocol.OP_PROPS, byteArrayOf(G1Protocol.PROP_BATT_PCT.toByte()))
|
||||
if (reply.status != 0) return null
|
||||
val value = G1Protocol.parseProps(reply.body, 1)?.get(0) ?: return null
|
||||
return if (value in 0..100) value else null
|
||||
}
|
||||
|
||||
private fun throwIfCancelled() {
|
||||
if (cancelled) throw IllegalStateException("Cancelled.")
|
||||
}
|
||||
|
||||
private fun now(): Long = platform.elapsedRealtimeMs()
|
||||
|
||||
private fun log(line: String) = listener.onLog("$side arm: $line")
|
||||
|
||||
private inline fun quietly(action: () -> Unit) {
|
||||
try {
|
||||
action()
|
||||
} catch (ignored: Exception) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- inbound -------------------------------------------------------------
|
||||
|
||||
override fun onNotification(address: String, characteristicUuid: String, data: ByteArray) {
|
||||
if (characteristicUuid.equals(G1Smp.SMP_CHAR_UUID, ignoreCase = true)) {
|
||||
lock.withLock {
|
||||
for ((seq, body) in assembler.feed(data)) {
|
||||
if (seq == smpSeq) {
|
||||
smpReply = body
|
||||
smpLatch?.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (characteristicUuid.equals(G1Smp.NUS_TX_UUID, ignoreCase = true)) {
|
||||
val percent = G1Smp.parseStockBattery(data) ?: return
|
||||
lock.withLock {
|
||||
stockBattery = percent
|
||||
stockBatteryLatch?.countDown()
|
||||
}
|
||||
} else if (characteristicUuid.equals(G1Ble.EVT_CHAR_UUID, ignoreCase = true)) {
|
||||
val notice = G1Protocol.parseStoreNotice(data)
|
||||
if (notice != null) {
|
||||
lock.withLock { storeNotices.add(notice) }
|
||||
return
|
||||
}
|
||||
val reply = G1Protocol.parseReply(data) ?: return
|
||||
lock.withLock {
|
||||
if (reply.tag == controlTag) {
|
||||
controlReply = reply
|
||||
controlLatch?.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onConnectionStateChange(address: String, connected: Boolean) {
|
||||
if (connected) return
|
||||
lock.withLock {
|
||||
linkLost = true
|
||||
smpLatch?.countDown()
|
||||
controlLatch?.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/**
|
||||
* One lens's update on an Even G1: the rectangles that differ from what the arm's frame holds, as RLE bbox calls
|
||||
* in draw messages. The transport's deflate runs over these messages, as on the G2; there is no resource cache,
|
||||
* so everything visible arrives as pixels.
|
||||
*/
|
||||
object G1FramePlanner {
|
||||
/** The firmware's limit on a decoded message is 16384 bytes (scouter_kit fc.c MAX_MESSAGE). */
|
||||
const val MAX_MESSAGE = 16000
|
||||
|
||||
/** A big rectangle goes out in bands, so that one noisy area doesn't need a long binary split. */
|
||||
private const val BAND_ROWS = 32
|
||||
private const val MAX_RECTS = ConnectionOptions.MULTI_RECT_MAX_RECTS
|
||||
|
||||
/** [previous] null or of another size: a full frame. Empty when nothing changed. */
|
||||
fun plan(previous: ByteArray?, next: ByteArray, width: Int, height: Int): List<ByteArray> {
|
||||
val stride = (width + 1) / 2
|
||||
require(next.size == stride * height)
|
||||
val rects = changedRects(previous, next, width, height)
|
||||
if (rects.isEmpty()) return emptyList()
|
||||
val calls = ArrayList<ByteArray>()
|
||||
for (r in rects) {
|
||||
var top = r[1]
|
||||
val bottom = r[1] + r[3]
|
||||
while (top < bottom) {
|
||||
val rows = minOf(BAND_ROWS, bottom - top)
|
||||
addRect(calls, next, stride, r[0], top, r[2], rows)
|
||||
top += rows
|
||||
}
|
||||
}
|
||||
return DrawProtocol.messages(calls, MAX_MESSAGE)
|
||||
}
|
||||
|
||||
private fun changedRects(previous: ByteArray?, next: ByteArray, width: Int, height: Int): List<IntArray> {
|
||||
if (previous == null || previous.size != next.size) return listOf(intArrayOf(0, 0, width, height))
|
||||
BleImageOptimizer.computeChangedRects(previous, next, width, height, MAX_RECTS)?.let { return it }
|
||||
return listOfNotNull(BleImageOptimizer.computeChangedBox(previous, next, width, height))
|
||||
}
|
||||
|
||||
/** Splits until the call fits a message; a single pixel always does. */
|
||||
private fun addRect(calls: MutableList<ByteArray>, packed: ByteArray, stride: Int, x: Int, y: Int, w: Int, h: Int) {
|
||||
val call = DrawProtocol.bbox(packed, stride, x, y, w, h)
|
||||
if (call.size + MESSAGE_OVERHEAD <= MAX_MESSAGE) {
|
||||
calls.add(call)
|
||||
} else if (h > 1) {
|
||||
addRect(calls, packed, stride, x, y, w, h / 2)
|
||||
addRect(calls, packed, stride, x, y + h / 2, w, h - h / 2)
|
||||
} else {
|
||||
addRect(calls, packed, stride, x, y, w / 2, h)
|
||||
addRect(calls, packed, stride, x + w / 2, y, w - w / 2, h)
|
||||
}
|
||||
}
|
||||
|
||||
/** The message type, the call count and the call's length (DrawProtocol.sequence). */
|
||||
private const val MESSAGE_OVERHEAD = 5
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
/** The GATT service of g1v0, the firmware of an Even G1 (scouter_kit g1_firmware/docs/VM.md §6.1). */
|
||||
object G1Ble {
|
||||
const val CTRL_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4402"
|
||||
const val EVT_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4403"
|
||||
/** Readable and writable only on an encrypted link: a write with response makes the OS pair, then proves the link is encrypted. */
|
||||
const val PAIR_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4404"
|
||||
/** Faceclaw's records (VM.md §6.11). */
|
||||
const val FC_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4405"
|
||||
/** The iPhone notification relay (VM.md §6.12): the G2's `AN` commands and frames, with this characteristic to itself. */
|
||||
const val ANCS_CHAR_UUID = "f3da4d59-277a-4fc9-a1ed-41c9c22d4406"
|
||||
const val REQUESTED_MTU = 247
|
||||
/** The oldest firmware this app works with: `PROP_SET` came in here. */
|
||||
const val REQUIRED_FIRMWARE = "0.5.63"
|
||||
|
||||
/** Dotted versions, component by component. */
|
||||
fun firmwareAtLeast(version: String, required: String = REQUIRED_FIRMWARE): Boolean {
|
||||
val have = version.split(".").map { it.toIntOrNull() ?: 0 }
|
||||
val want = required.split(".").map { it.toIntOrNull() ?: 0 }
|
||||
for (i in 0 until maxOf(have.size, want.size)) {
|
||||
val delta = have.getOrElse(i) { 0 } - want.getOrElse(i) { 0 }
|
||||
if (delta != 0) return delta > 0
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The wire formats of an Even G1 on g1v0: control requests and events (VM.md §6.3, §6.4), which Faceclaw uses as they
|
||||
* are, and the display path's packets and answers (§6.11). No VM instruction is sent; the records inside the packets
|
||||
* are [CfwTransport]'s, the same as on the G2.
|
||||
*/
|
||||
object G1Protocol {
|
||||
const val OP_HELLO = 0x01
|
||||
const val OP_HEALTH_OK = 0x07
|
||||
const val OP_PROPS = 0x0A
|
||||
const val OP_MIC = 0x13
|
||||
const val OP_EVENTS = 0x14
|
||||
const val OP_ASSET_INFO = 0x17
|
||||
const val OP_ASSET_FORMAT = 0x1B
|
||||
const val OP_FLASH_CRC = 0x1C
|
||||
const val OP_FC_INFO = 0x1D
|
||||
const val OP_PROP_SET = 0x1E
|
||||
|
||||
/** Control statuses (VM.md §1). */
|
||||
const val STATUS_SAFE_MODE = 7
|
||||
/** A store request: a job runs, no space, no store. */
|
||||
const val STATUS_ASSET = 13
|
||||
|
||||
const val EV_NOTICE = 0x05
|
||||
const val EV_MIC_DATA = 0x06
|
||||
const val EV_INPUT = 0x07
|
||||
const val EV_FC_ACK = 0x08
|
||||
private const val EV_REPLY = 0x80
|
||||
private const val EVENT_HEADER = 9
|
||||
private const val OPT_START = 0x80
|
||||
private const val OPT_END = 0x40
|
||||
private const val MAX_PACKET = 244
|
||||
private const val PACKET_HEADER = 2
|
||||
|
||||
/** The light sensor converts only while something uses it, as this bit of `EVENTS` does; the left arm has none. */
|
||||
const val TRIGGER_LIGHT = 4
|
||||
const val TRIGGER_BATTERY = 5
|
||||
const val TRIGGER_CASE = 6
|
||||
const val TRIGGER_WEAR = 7
|
||||
const val TRIGGER_TOUCH = 8
|
||||
const val EVENTS_MASK = (1 shl TRIGGER_LIGHT) or (1 shl TRIGGER_BATTERY) or (1 shl TRIGGER_CASE) or
|
||||
(1 shl TRIGGER_WEAR) or (1 shl TRIGGER_TOUCH)
|
||||
|
||||
const val NOTICE_MIC_STOPPED = 5
|
||||
const val MIC_STOPPED_LEASE = 3
|
||||
const val NOTICE_GUARD = 7
|
||||
/** A store job ended (VM.md §6.10). */
|
||||
const val NOTICE_STORE_JOB = 8
|
||||
|
||||
const val STORE_JOB_FORMAT = 3
|
||||
const val STORE_JOB_CRC = 4
|
||||
/** `ASSET_INFO`'s job states. */
|
||||
const val STORE_IDLE = 0
|
||||
const val STORE_DONE = 4
|
||||
const val STORE_FAILED = 5
|
||||
/** The asset store's partition in the arm's QSPI flash, where Even's fonts and pictures were; its first two sectors hold its directory. */
|
||||
const val STORE_ADDRESS = 0x140000
|
||||
const val STORE_DIRECTORY_BYTES = 2 * 4096
|
||||
|
||||
const val PROP_BATT_PCT = 0x11
|
||||
const val PROP_CHARGE = 0x12
|
||||
const val PROP_IN_CASE = 0x13
|
||||
const val PROP_WORN = 0x14
|
||||
const val PROP_ACC_X = 0x18
|
||||
const val PROP_ACC_Y = 0x19
|
||||
const val PROP_ACC_Z = 0x1A
|
||||
/** 0.01 lux, on the right arm, while the panel is on. */
|
||||
const val PROP_LIGHT = 0x1C
|
||||
const val PROP_IMU_RATE = 0x28
|
||||
const val PROP_BRIGHTNESS_MAX = 0x29
|
||||
const val IMU_RATE_IDLE = 13
|
||||
const val IMU_RATE_STREAM = 26
|
||||
const val BRIGHTNESS_LEVEL_MAX = 42
|
||||
/** A property without a value yet (VM_UNKNOWN). */
|
||||
const val UNKNOWN = Int.MIN_VALUE
|
||||
|
||||
const val CHARGE_CHARGING = 1
|
||||
|
||||
const val GESTURE_TAP = 1
|
||||
const val GESTURE_DOUBLE_TAP = 2
|
||||
const val GESTURE_TRIPLE_TAP = 3
|
||||
const val GESTURE_LONG_START = 4
|
||||
const val GESTURE_LONG_END = 5
|
||||
|
||||
/** The packets of one record: `[0x80 first | 0x40 last][stream][the next bytes]`. */
|
||||
fun packets(record: ByteArray, stream: Int, mtu: Int): List<ByteArray> {
|
||||
val chunk = minOf(mtu - 3, MAX_PACKET) - PACKET_HEADER
|
||||
require(record.isNotEmpty() && chunk >= 5 && stream in 0..255)
|
||||
val packets = ArrayList<ByteArray>(record.size / chunk + 1)
|
||||
var offset = 0
|
||||
while (offset < record.size) {
|
||||
val count = minOf(chunk, record.size - offset)
|
||||
val packet = ByteArray(PACKET_HEADER + count)
|
||||
packet[0] = ((if (offset == 0) OPT_START else 0) or (if (offset + count == record.size) OPT_END else 0)).toByte()
|
||||
packet[1] = stream.toByte()
|
||||
record.copyInto(packet, PACKET_HEADER, offset, offset + count)
|
||||
packets.add(packet)
|
||||
offset += count
|
||||
}
|
||||
return packets
|
||||
}
|
||||
|
||||
fun control(op: Int, tag: Int, body: ByteArray = ByteArray(0)): ByteArray =
|
||||
byteArrayOf(op.toByte(), tag.toByte()) + body
|
||||
|
||||
fun eventsRequest(mask: Int): ByteArray = byteArrayOf(mask.toByte(), (mask ushr 8).toByte())
|
||||
|
||||
/** `PROP_SET` (0.5.63+): what SYS_WRITE does, for a host that runs no instructions. */
|
||||
fun propSet(id: Int, value: Int): ByteArray =
|
||||
byteArrayOf(id.toByte(), value.toByte(), (value ushr 8).toByte(), (value ushr 16).toByte(), (value ushr 24).toByte())
|
||||
|
||||
/** On, or a renewal of the lease; 32 kbps, the rate [Lc3PacketFramer] decodes. */
|
||||
fun micRequest(on: Boolean): ByteArray = byteArrayOf(if (on) 1 else 0, 0)
|
||||
|
||||
/** An event's type, or -1 for bytes too short to be one. */
|
||||
fun eventType(data: ByteArray): Int = if (data.size >= EVENT_HEADER) data[0].toInt() and 255 else -1
|
||||
|
||||
class Reply(@JvmField val op: Int, @JvmField val tag: Int, @JvmField val status: Int, @JvmField val body: ByteArray)
|
||||
|
||||
fun parseReply(data: ByteArray): Reply? {
|
||||
val type = eventType(data)
|
||||
if (type < EV_REPLY || data.size < EVENT_HEADER + 2) return null
|
||||
return Reply(type and 0x7f, data[EVENT_HEADER].toInt() and 255, data[EVENT_HEADER + 1].toInt() and 255,
|
||||
data.copyOfRange(EVENT_HEADER + 2, data.size))
|
||||
}
|
||||
|
||||
/** Same shape as [CfwTransport.parseAcks]'s, so the shared window takes it. */
|
||||
fun parseFcAck(data: ByteArray): CfwTransport.Ack? {
|
||||
if (eventType(data) != EV_FC_ACK || data.size < EVENT_HEADER + 9) return null
|
||||
val kind = data[EVENT_HEADER].toInt() and 255
|
||||
if (kind != 1 && kind != 3) return null
|
||||
return CfwTransport.Ack(kind == 3, data[EVENT_HEADER + 1].toInt() and 255, u16(data, EVENT_HEADER + 2),
|
||||
data[EVENT_HEADER + 4].toInt() and 255, u16(data, EVENT_HEADER + 5), u16(data, EVENT_HEADER + 7))
|
||||
}
|
||||
|
||||
class Input(@JvmField val trigger: Int, @JvmField val value: Int)
|
||||
|
||||
fun parseInput(data: ByteArray): Input? {
|
||||
if (eventType(data) != EV_INPUT || data.size < EVENT_HEADER + 7) return null
|
||||
return Input(data[EVENT_HEADER + 2].toInt() and 255, u32(data, EVENT_HEADER + 3))
|
||||
}
|
||||
|
||||
class Notice(@JvmField val kind: Int, @JvmField val body: ByteArray)
|
||||
|
||||
fun parseNotice(data: ByteArray): Notice? {
|
||||
if (eventType(data) != EV_NOTICE || data.size < EVENT_HEADER + 1) return null
|
||||
return Notice(data[EVENT_HEADER].toInt() and 255, data.copyOfRange(EVENT_HEADER + 1, data.size))
|
||||
}
|
||||
|
||||
/** `ASSET_FORMAT` frees the whole store; with [wipe] it also erases every sector that isn't erased already. */
|
||||
fun assetFormatRequest(wipe: Boolean): ByteArray =
|
||||
le32(0x534B4153) + if (wipe) le32(0x50570000) else ByteArray(0)
|
||||
|
||||
/** `FLASH_CRC`: a read-only fingerprint of flash bytes address…address + n - 1, given in `NOTICE` 8. */
|
||||
fun flashCrcRequest(address: Int, n: Int): ByteArray = le32(address) + le32(n)
|
||||
|
||||
class AssetInfo(
|
||||
@JvmField val assets: Int,
|
||||
@JvmField val state: Int,
|
||||
@JvmField val lastError: Int,
|
||||
/** Without a usable flash there is no store, and nothing was ever written to it. */
|
||||
@JvmField val available: Boolean,
|
||||
/** The directory's commit count: 0 for a store that is empty and never was written, or was formatted since. */
|
||||
@JvmField val seq: Long,
|
||||
) {
|
||||
val running: Boolean get() = state in 1..3
|
||||
}
|
||||
|
||||
fun parseAssetInfo(body: ByteArray): AssetInfo? {
|
||||
if (body.size < 20) return null
|
||||
return AssetInfo(body[12].toInt() and 255, body[13].toInt() and 255, body[14].toInt() and 255, body[15].toInt() and 1 != 0,
|
||||
u32(body, 16).toLong() and 0xffffffffL)
|
||||
}
|
||||
|
||||
class StoreNotice(
|
||||
@JvmField val job: Int,
|
||||
@JvmField val result: Int,
|
||||
/** A crc job's range, crc32 and how many of its bytes are not 0xFF; 0 for the other jobs. */
|
||||
@JvmField val address: Int,
|
||||
@JvmField val n: Int,
|
||||
@JvmField val nonErased: Long,
|
||||
)
|
||||
|
||||
fun parseStoreNotice(data: ByteArray): StoreNotice? {
|
||||
val notice = parseNotice(data) ?: return null
|
||||
val body = notice.body
|
||||
if (notice.kind != NOTICE_STORE_JOB || body.size < 18) return null
|
||||
return StoreNotice(body[0].toInt() and 255, body[1].toInt() and 255, u32(body, 2), u32(body, 6), u32(body, 14).toLong() and 0xffffffffL)
|
||||
}
|
||||
|
||||
private fun le32(value: Int): ByteArray =
|
||||
byteArrayOf(value.toByte(), (value ushr 8).toByte(), (value ushr 16).toByte(), (value ushr 24).toByte())
|
||||
|
||||
/**
|
||||
* What the arm's light reading looks like to [BrightnessPolicy] and the Developer app: a live passive poll in the
|
||||
* G2 custom firmware's report (settings field 105), whose value counts tenths of a lux.
|
||||
*/
|
||||
fun alsReport(centilux: Int, brightness: Int, tickMs: Long): ByteArray {
|
||||
val report = ByteArray(24)
|
||||
report[0] = 0x41
|
||||
report[1] = 0x4c
|
||||
report[2] = 1
|
||||
report[3] = 2 // a poll
|
||||
report[4] = 0x0b // opened, passive, read ok
|
||||
report[5] = 3 // polling
|
||||
val tenths = centilux / 10
|
||||
for (i in 0..3) {
|
||||
report[6 + i] = (tenths ushr (8 * i)).toByte()
|
||||
report[10 + i] = (tenths ushr (8 * i)).toByte()
|
||||
report[20 + i] = (tickMs ushr (8 * i)).toByte()
|
||||
}
|
||||
report[14] = brightness.toByte()
|
||||
report[16] = brightness.toByte()
|
||||
report[18] = 0
|
||||
report[19] = 4 // scale 1.0 in Q10
|
||||
return report
|
||||
}
|
||||
|
||||
class Hello(
|
||||
@JvmField val version: String,
|
||||
@JvmField val leftArm: Boolean,
|
||||
@JvmField val safeMode: Boolean,
|
||||
@JvmField val panelOk: Boolean,
|
||||
@JvmField val encrypted: Boolean,
|
||||
@JvmField val inCase: Boolean,
|
||||
/** `MIC` captures here: the right arm. */
|
||||
@JvmField val mic: Boolean,
|
||||
/** The iPhone notification relay answers here (0.5.65+). */
|
||||
@JvmField val ancs: Boolean,
|
||||
)
|
||||
|
||||
fun parseHello(body: ByteArray): Hello? {
|
||||
if (body.size < 7 || body[0].toInt() != 5) return null
|
||||
val flags = body[5].toInt() and 255
|
||||
return Hello("${body[1].toInt() and 255}.${body[2].toInt() and 255}.${body[3].toInt() and 255}",
|
||||
body[4].toInt() == 1, flags and 1 != 0, flags and 2 != 0, flags and 64 != 0, flags and 128 != 0, body[6].toInt() and 4 != 0,
|
||||
body[6].toInt() and 8 != 0)
|
||||
}
|
||||
|
||||
/** The display path's version, or -1 for a body that isn't FC_INFO's. */
|
||||
fun parseFcVersion(body: ByteArray): Int = if (body.size >= 14) body[0].toInt() and 255 else -1
|
||||
|
||||
/** One value per requested property, or null if the reply is short. */
|
||||
fun parseProps(body: ByteArray, count: Int): IntArray? =
|
||||
if (body.size < 4 * count) null else IntArray(count) { u32(body, 4 * it) }
|
||||
|
||||
/**
|
||||
* What a TouchBar gesture means to Faceclaw, which expects a G2's temple: a tap, a double tap and two slides. A
|
||||
* G1's TouchBar cannot slide, but it has one per arm, so a tap moves (left up, right down), a double tap selects
|
||||
* and a triple tap or long press goes back. Null: no event. With the display [dark] nothing can scroll: a tap is
|
||||
* the G2's tap, which shows the Glanceboard, and the session reports a double tap as a wake.
|
||||
*/
|
||||
fun gestureEventType(gesture: Int, leftArm: Boolean, dark: Boolean = false): Int? = when (gesture) {
|
||||
GESTURE_TAP -> if (dark) BleProtocol.EVENT_CLICK else if (leftArm) BleProtocol.EVENT_SCROLL_TOP else BleProtocol.EVENT_SCROLL_BOTTOM
|
||||
GESTURE_DOUBLE_TAP -> if (dark) BleProtocol.EVENT_DOUBLE_CLICK else BleProtocol.EVENT_CLICK
|
||||
GESTURE_TRIPLE_TAP, GESTURE_LONG_START -> BleProtocol.EVENT_DOUBLE_CLICK
|
||||
else -> null
|
||||
}
|
||||
|
||||
/** `MIC_DATA`'s frames: the counter of the first one, how many, and where they start in [data]. */
|
||||
fun micFrames(data: ByteArray): Triple<Int, Int, Int>? {
|
||||
if (eventType(data) != EV_MIC_DATA || data.size < EVENT_HEADER + 3) return null
|
||||
val count = data[EVENT_HEADER + 2].toInt() and 255
|
||||
val start = EVENT_HEADER + 3
|
||||
return if (data.size - start < count * Lc3PacketFramer.FRAME_BYTES) null else Triple(u16(data, EVENT_HEADER), count, start)
|
||||
}
|
||||
|
||||
private fun u16(b: ByteArray, p: Int): Int = (b[p].toInt() and 255) or ((b[p + 1].toInt() and 255) shl 8)
|
||||
|
||||
private fun u32(b: ByteArray, p: Int): Int = u16(b, p) or (u16(b, p + 2) shl 16)
|
||||
}
|
||||
|
||||
/**
|
||||
* A G1's microphone frames as the packets [Lc3PacketFramer] takes from a G2: five 10 ms LC3 frames, then the two values
|
||||
* of the G2's stereo capture (a G1 has one mic: zero) and a packet counter. A gap loses the part-filled packet, and the
|
||||
* counter tells the decoder to conceal it.
|
||||
*/
|
||||
class G1MicAssembler {
|
||||
private val frames = ByteArray(Lc3PacketFramer.LC3_BYTES)
|
||||
private var have = 0
|
||||
private var firstCounter = 0
|
||||
private var nextCounter = -1
|
||||
|
||||
fun reset() {
|
||||
have = 0
|
||||
nextCounter = -1
|
||||
}
|
||||
|
||||
fun ingest(first: Int, count: Int, data: ByteArray, start: Int, emit: (ByteArray) -> Unit) {
|
||||
val size = Lc3PacketFramer.FRAME_BYTES
|
||||
for (i in 0 until count) {
|
||||
val counter = (first + i) and 0xffff
|
||||
if (counter != nextCounter) have = 0
|
||||
if (have == 0) firstCounter = counter
|
||||
data.copyInto(frames, have * size, start + i * size, start + (i + 1) * size)
|
||||
have++
|
||||
nextCounter = (counter + 1) and 0xffff
|
||||
if (have == Lc3PacketFramer.FRAMES_PER_PACKET) {
|
||||
val packet = ByteArray(Lc3PacketFramer.PACKET_BYTES)
|
||||
frames.copyInto(packet)
|
||||
packet[Lc3PacketFramer.COUNTER_OFFSET] = (firstCounter / Lc3PacketFramer.FRAMES_PER_PACKET).toByte()
|
||||
emit(packet)
|
||||
have = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
/** The part of CBOR that MCUmgr uses: integers, byte and text strings, arrays, maps, booleans and null. */
|
||||
object G1Cbor {
|
||||
fun encode(value: Any?): ByteArray {
|
||||
val out = Output()
|
||||
write(out, value)
|
||||
return out.toByteArray()
|
||||
}
|
||||
|
||||
fun decode(data: ByteArray, start: Int = 0): Any? = Reader(data, start).value()
|
||||
|
||||
private class Output {
|
||||
private var bytes = ByteArray(256)
|
||||
private var size = 0
|
||||
|
||||
private fun reserve(extra: Int) {
|
||||
if (size + extra > bytes.size) bytes = bytes.copyOf(maxOf(bytes.size * 2, size + extra))
|
||||
}
|
||||
|
||||
fun add(byte: Int) {
|
||||
reserve(1)
|
||||
bytes[size++] = byte.toByte()
|
||||
}
|
||||
|
||||
fun add(data: ByteArray) {
|
||||
reserve(data.size)
|
||||
data.copyInto(bytes, size)
|
||||
size += data.size
|
||||
}
|
||||
|
||||
fun toByteArray(): ByteArray = bytes.copyOf(size)
|
||||
}
|
||||
|
||||
private fun head(out: Output, major: Int, n: Long) {
|
||||
val first = major shl 5
|
||||
val width = when {
|
||||
n < 24 -> {
|
||||
out.add(first or n.toInt())
|
||||
return
|
||||
}
|
||||
n < 0x100L -> 1
|
||||
n < 0x10000L -> 2
|
||||
n < 0x100000000L -> 4
|
||||
else -> 8
|
||||
}
|
||||
out.add(first or (24 + width.countTrailingZeroBits()))
|
||||
for (i in width - 1 downTo 0) out.add(((n ushr (8 * i)) and 0xff).toInt())
|
||||
}
|
||||
|
||||
private fun write(out: Output, value: Any?) {
|
||||
when (value) {
|
||||
null -> out.add(0xf6)
|
||||
is Boolean -> out.add(if (value) 0xf5 else 0xf4)
|
||||
is Int -> write(out, value.toLong())
|
||||
is Long -> if (value >= 0) head(out, 0, value) else head(out, 1, -1 - value)
|
||||
is ByteArray -> {
|
||||
head(out, 2, value.size.toLong())
|
||||
out.add(value)
|
||||
}
|
||||
is String -> {
|
||||
val text = value.encodeToByteArray()
|
||||
head(out, 3, text.size.toLong())
|
||||
out.add(text)
|
||||
}
|
||||
is List<*> -> {
|
||||
head(out, 4, value.size.toLong())
|
||||
for (item in value) write(out, item)
|
||||
}
|
||||
is Map<*, *> -> {
|
||||
head(out, 5, value.size.toLong())
|
||||
for ((key, item) in value) {
|
||||
write(out, key)
|
||||
write(out, item)
|
||||
}
|
||||
}
|
||||
else -> throw IllegalArgumentException("not CBOR: $value")
|
||||
}
|
||||
}
|
||||
|
||||
private class Reader(private val data: ByteArray, private var at: Int) {
|
||||
private fun u8(): Int {
|
||||
if (at >= data.size) throw IllegalArgumentException("CBOR cut short")
|
||||
return data[at++].toInt() and 0xff
|
||||
}
|
||||
|
||||
fun value(): Any? {
|
||||
val first = u8()
|
||||
val major = first ushr 5
|
||||
val info = first and 31
|
||||
val n: Long? = when {
|
||||
info < 24 -> info.toLong()
|
||||
info in 24..27 -> {
|
||||
var v = 0L
|
||||
repeat(1 shl (info - 24)) { v = (v shl 8) or u8().toLong() }
|
||||
v
|
||||
}
|
||||
info == 31 -> null
|
||||
else -> throw IllegalArgumentException("bad CBOR header $first")
|
||||
}
|
||||
return when (major) {
|
||||
0 -> n ?: throw IllegalArgumentException("bad CBOR integer")
|
||||
1 -> -1 - (n ?: throw IllegalArgumentException("bad CBOR integer"))
|
||||
2, 3 -> {
|
||||
val size = (n ?: throw IllegalArgumentException("indefinite CBOR string")).toInt()
|
||||
if (size < 0 || at + size > data.size) throw IllegalArgumentException("CBOR cut short")
|
||||
val bytes = data.copyOfRange(at, at + size)
|
||||
at += size
|
||||
if (major == 2) bytes else bytes.decodeToString()
|
||||
}
|
||||
4 -> {
|
||||
val items = ArrayList<Any?>()
|
||||
while (more(n, items.size)) items.add(value())
|
||||
if (n == null) at++
|
||||
items
|
||||
}
|
||||
5 -> {
|
||||
val items = LinkedHashMap<Any?, Any?>()
|
||||
while (more(n, items.size)) {
|
||||
val key = value()
|
||||
items[key] = value()
|
||||
}
|
||||
if (n == null) at++
|
||||
items
|
||||
}
|
||||
7 -> when (info) {
|
||||
20 -> false
|
||||
21 -> true
|
||||
22 -> null
|
||||
else -> throw IllegalArgumentException("unsupported CBOR simple value $info")
|
||||
}
|
||||
else -> throw IllegalArgumentException("unsupported CBOR major type $major")
|
||||
}
|
||||
}
|
||||
|
||||
/** An indefinite-length array or map (what Zephyr's image list uses) ends at a 0xFF. */
|
||||
private fun more(n: Long?, count: Int): Boolean {
|
||||
if (n != null) return count < n
|
||||
if (at >= data.size) throw IllegalArgumentException("CBOR cut short")
|
||||
return (data[at].toInt() and 0xff) != 0xff
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* MCUmgr SMP, the update service of both Even's firmware and g1v0: as much of it as installing an image takes, and what
|
||||
* an MCUboot image says about itself. The same requests as scouter_kit's tools/g1host.py (class Smp), which was proven on
|
||||
* real arms first.
|
||||
*/
|
||||
object G1Smp {
|
||||
const val SMP_CHAR_UUID = "da2e7828-fbce-4e01-ae9e-261174997c48"
|
||||
const val NUS_RX_UUID = "6e400002-b5a3-f393-e0a9-e50e24dcca9e"
|
||||
const val NUS_TX_UUID = "6e400003-b5a3-f393-e0a9-e50e24dcca9e"
|
||||
|
||||
const val OP_READ = 0
|
||||
const val OP_WRITE = 2
|
||||
const val GROUP_OS = 0
|
||||
const val GROUP_IMAGE = 1
|
||||
const val CMD_IMAGE_STATE = 0
|
||||
const val CMD_IMAGE_UPLOAD = 1
|
||||
const val CMD_OS_RESET = 5
|
||||
const val UPLOAD_CHUNK = 2048
|
||||
private const val HEADER = 8
|
||||
|
||||
/** Even's firmware drops a link it hears no heartbeat on. */
|
||||
fun stockHeartbeat(seq: Int): ByteArray =
|
||||
byteArrayOf(0x25, 0x06, 0x00, seq.toByte(), 0x04, seq.toByte())
|
||||
|
||||
/** Even's firmware's battery query; its reply is `2C 66`, then the percentage. The bytes its real firmware answered in emulation. */
|
||||
val STOCK_BATTERY_QUERY = byteArrayOf(0x2c, 0x04, 0x00, 0x00)
|
||||
|
||||
/** The percentage in a reply to [STOCK_BATTERY_QUERY]; null for anything else, or a value that can't be one. */
|
||||
fun parseStockBattery(data: ByteArray): Int? {
|
||||
if (data.size < 3 || data[0].toInt() != 0x2c || (data[1].toInt() and 0xff) != 0x66) return null
|
||||
return (data[2].toInt() and 0xff).takeIf { it in 1..100 }
|
||||
}
|
||||
|
||||
fun frame(op: Int, group: Int, seq: Int, cmd: Int, payload: Map<String, Any?>): ByteArray {
|
||||
val body = G1Cbor.encode(payload)
|
||||
val out = ByteArray(HEADER + body.size)
|
||||
out[0] = op.toByte()
|
||||
out[2] = (body.size ushr 8).toByte()
|
||||
out[3] = body.size.toByte()
|
||||
out[4] = (group ushr 8).toByte()
|
||||
out[5] = group.toByte()
|
||||
out[6] = seq.toByte()
|
||||
out[7] = cmd.toByte()
|
||||
body.copyInto(out, HEADER)
|
||||
return out
|
||||
}
|
||||
|
||||
/** One ATT write at most [mtu] - 3 bytes long; MCUmgr joins them again. */
|
||||
fun split(frame: ByteArray, mtu: Int): List<ByteArray> {
|
||||
val size = maxOf(mtu - 3, 20)
|
||||
return (frame.indices step size).map { frame.copyOfRange(it, minOf(it + size, frame.size)) }
|
||||
}
|
||||
|
||||
/** MCUboot's image numbers on an nRF5340: the application core, and the network core (the Bluetooth controller). */
|
||||
const val IMAGE_APP = 0
|
||||
const val IMAGE_NET_CORE = 1
|
||||
|
||||
/**
|
||||
* The first chunk also says what is coming, which lets the arm resume an upload of the same file, and which image it
|
||||
* is: each has its own second slot.
|
||||
*/
|
||||
fun uploadRequest(image: ByteArray, offset: Int, fileSha256: ByteArray, imageIndex: Int = IMAGE_APP): Map<String, Any?> {
|
||||
val data = image.copyOfRange(offset, minOf(offset + UPLOAD_CHUNK, image.size))
|
||||
return if (offset == 0) {
|
||||
mapOf("image" to imageIndex, "len" to image.size, "off" to 0, "sha" to fileSha256, "data" to data, "upgrade" to false)
|
||||
} else {
|
||||
mapOf("off" to offset, "data" to data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks the image with this hash for MCUboot's next boot. [permanent] false is a test install, which is what Even's app
|
||||
* and g1v0's own network-core fallback ask for on image 1; the network core has no copy to go back to either way.
|
||||
*/
|
||||
fun confirmRequest(imageHash: ByteArray, permanent: Boolean = true): Map<String, Any?> = mapOf("hash" to imageHash, "confirm" to permanent)
|
||||
|
||||
/** Joins notifications into replies; [feed] gives each complete one as sequence number and body. */
|
||||
class Assembler {
|
||||
private var pending = ByteArray(0)
|
||||
|
||||
fun reset() {
|
||||
pending = ByteArray(0)
|
||||
}
|
||||
|
||||
fun feed(data: ByteArray): List<Pair<Int, Map<*, *>>> {
|
||||
pending += data
|
||||
val replies = ArrayList<Pair<Int, Map<*, *>>>()
|
||||
while (pending.size >= HEADER) {
|
||||
val size = ((pending[2].toInt() and 0xff) shl 8) or (pending[3].toInt() and 0xff)
|
||||
if (pending.size < HEADER + size) break
|
||||
val seq = pending[6].toInt() and 0xff
|
||||
val body = try {
|
||||
if (size == 0) emptyMap<Any?, Any?>() else G1Cbor.decode(pending, HEADER) as? Map<*, *>
|
||||
} catch (e: IllegalArgumentException) {
|
||||
null
|
||||
}
|
||||
if (body == null) {
|
||||
// A frame cut by a lost link: the request is sent again.
|
||||
pending = ByteArray(0)
|
||||
break
|
||||
}
|
||||
pending = pending.copyOfRange(HEADER + size, pending.size)
|
||||
replies.add(seq to body)
|
||||
}
|
||||
return replies
|
||||
}
|
||||
}
|
||||
|
||||
class Slot(
|
||||
@JvmField val image: Int,
|
||||
@JvmField val slot: Int,
|
||||
@JvmField val version: String,
|
||||
@JvmField val hash: ByteArray,
|
||||
@JvmField val flags: List<String>,
|
||||
) {
|
||||
override fun toString(): String =
|
||||
"image $image slot $slot: $version ${BinaryEncoding.bytesToHex(hash.copyOf(minOf(8, hash.size)))} ${flags.joinToString(" ")}".trim()
|
||||
}
|
||||
|
||||
fun parseSlots(reply: Map<*, *>): List<Slot> {
|
||||
val images = reply["images"] as? List<*> ?: return emptyList()
|
||||
return images.mapNotNull { item ->
|
||||
val entry = item as? Map<*, *> ?: return@mapNotNull null
|
||||
val hash = entry["hash"] as? ByteArray ?: return@mapNotNull null
|
||||
Slot((entry["image"] as? Long ?: 0L).toInt(), (entry["slot"] as? Long ?: -1L).toInt(), entry["version"] as? String ?: "?", hash,
|
||||
listOf("bootable", "pending", "confirmed", "active", "permanent").filter { entry[it] == true })
|
||||
}
|
||||
}
|
||||
|
||||
fun replyCode(reply: Map<*, *>): Int = (reply["rc"] as? Long ?: 0L).toInt()
|
||||
|
||||
fun replyOffset(reply: Map<*, *>): Int? = (reply["off"] as? Long)?.toInt()
|
||||
|
||||
/** The dotted version in an MCUboot header. */
|
||||
fun imageVersion(image: ByteArray): String {
|
||||
requireImage(image)
|
||||
val revision = (image[22].toInt() and 0xff) or ((image[23].toInt() and 0xff) shl 8)
|
||||
return "${image[20].toInt() and 0xff}.${image[21].toInt() and 0xff}.$revision"
|
||||
}
|
||||
|
||||
/** The SHA-256 TLV: how MCUmgr names an image. */
|
||||
fun imageHash(image: ByteArray): ByteArray {
|
||||
requireImage(image)
|
||||
var at = u16(image, 8) + u32(image, 12)
|
||||
require(at + 4 <= image.size && u16(image, at) == 0x6907) { "the image has no TLV area" }
|
||||
val end = at + u16(image, at + 2)
|
||||
require(end <= image.size) { "the image's TLV area is cut short" }
|
||||
at += 4
|
||||
while (at + 4 <= end) {
|
||||
val type = u16(image, at)
|
||||
val size = u16(image, at + 2)
|
||||
if (type == 0x10 && size == 32 && at + 4 + size <= end) return image.copyOfRange(at + 4, at + 4 + size)
|
||||
at += 4 + size
|
||||
}
|
||||
throw IllegalArgumentException("the image has no SHA-256 TLV")
|
||||
}
|
||||
|
||||
private fun requireImage(image: ByteArray) {
|
||||
require(image.size > 32 && u32(image, 0) == 0x96f3b83d.toInt()) { "not an MCUboot image" }
|
||||
}
|
||||
|
||||
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
|
||||
|
||||
private fun u32(data: ByteArray, at: Int): Int = u16(data, at) or (u16(data, at + 2) shl 16)
|
||||
}
|
||||
@@ -119,6 +119,19 @@ class BleProtocol {
|
||||
*/
|
||||
const val FACECLAW_ALS_REPORT_FIELD: Int = 105
|
||||
|
||||
/**
|
||||
* CFW keyboard bridge (Faceclaw/39): the master temple hosts one BLE HID keyboard. The
|
||||
* host writes ['K','B',ver,op,request,epoch,...] controls as settings field 130; results,
|
||||
* pairing prompts and the keyboard's raw input reports come back as commandId=3 pushes
|
||||
* carrying field 131 (['K','B',ver,kind,...], decoded in app/g2/keyboard/protocol.ts).
|
||||
*/
|
||||
const val FACECLAW_KEYBOARD_CONTROL_FIELD: Int = 130
|
||||
|
||||
const val FACECLAW_KEYBOARD_EVENT_FIELD: Int = 131
|
||||
|
||||
/** Fixed header of a field-131 record; fragments follow it. */
|
||||
const val FACECLAW_KEYBOARD_EVENT_HEADER: Int = 24
|
||||
|
||||
const val FACECLAW_WAKE_OP_ACQUIRE: Int = 1
|
||||
|
||||
const val FACECLAW_WAKE_OP_RELEASE: Int = 2
|
||||
@@ -788,6 +801,42 @@ class BleProtocol {
|
||||
return body
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an already-encoded ['K','B',ver,op,...] keyboard control as a settings write
|
||||
* carrying CFW field 130. Magic zero, like the other private controls: the answer is a
|
||||
* field-131 record naming the control's request id, not the settings ACK.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun buildFaceclawKeyboardControl(record: ByteArray): ByteArray {
|
||||
return concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1),
|
||||
encodeVarintField(2, 0),
|
||||
encodeBytesField(FACECLAW_KEYBOARD_CONTROL_FIELD, record),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a CFW keyboard record (field 131) from a settings frame, or null when the frame
|
||||
* carries none. Only the ['K','B'] prelude and the fixed header's length are checked
|
||||
* here; the TS side validates the version and decodes the rest.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun parseFaceclawKeyboardEvent(pb: ByteArray?): ByteArray? {
|
||||
if (pb == null) {
|
||||
return null
|
||||
}
|
||||
val body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_KEYBOARD_EVENT_FIELD)
|
||||
if (
|
||||
body == null || body.size < FACECLAW_KEYBOARD_EVENT_HEADER ||
|
||||
body[0].toInt() != 75 || body[1].toInt() != 66
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the raw CFW ambient-light report from a sid-0x09 frame, or null when this is an
|
||||
* ordinary settings frame. Only the 65.toByte(),76.toByte(),version prelude is validated
|
||||
|
||||
+2
-2
@@ -33,7 +33,7 @@ class CfwMessageWindow private constructor() {
|
||||
fun acknowledgedHead(messages: Iterable<OutboundMessage>): OutboundMessage? {
|
||||
for (message in messages) {
|
||||
if (message.sid != CfwTransport.SID) continue
|
||||
return if (!message.cfwRetryPending && message.cfwAckLenses == CfwTransport.BOTH)
|
||||
return if (!message.cfwRetryPending && message.isCfwAcked)
|
||||
message
|
||||
else null
|
||||
}
|
||||
@@ -49,7 +49,7 @@ class CfwMessageWindow private constructor() {
|
||||
failed =
|
||||
failed ||
|
||||
message.cfwRetryPending ||
|
||||
(message.cfwAckLenses != CfwTransport.BOTH &&
|
||||
(!message.isCfwAcked &&
|
||||
message.ackDeadlineAtMs > 0 &&
|
||||
message.ackDeadlineAtMs <= nowMs)
|
||||
replay.add(message)
|
||||
|
||||
+24
-12
@@ -28,7 +28,18 @@ class CfwTransport(private val platform: ProtocolPlatform) {
|
||||
fun encode(message: ByteArray?, streamId: Int, lenses: Int, mtu: Int): List<ByteArray> =
|
||||
lock.withLock {
|
||||
validate(message, streamId, lenses, mtu)
|
||||
val input = message!!
|
||||
g2Packets(encodeRecord(message!!, lenses), streamId, mtu)
|
||||
}
|
||||
|
||||
/**
|
||||
* One record, `[flags][wire length u16][decoded CRC u16][body]`, compressed with this
|
||||
* transport's persistent context. The packet envelope is the caller's: [encode] wraps it
|
||||
* in G2 TPL packets, [RayNeoIoProtocol.cfwPackets] in RayNeo iO business-0xF0 packets.
|
||||
*/
|
||||
fun encodeRecord(message: ByteArray, lenses: Int): ByteArray =
|
||||
lock.withLock {
|
||||
require(message.size <= MAX_MESSAGE && lenses in 1..BOTH) { "invalid CFW record parameters" }
|
||||
val input = message
|
||||
if (previousLenses != lenses) reset()
|
||||
val reset = resetPending
|
||||
val encoder = deflater ?: platform.createDeflater().also { deflater = it }
|
||||
@@ -44,7 +55,7 @@ class CfwTransport(private val platform: ProtocolPlatform) {
|
||||
resetPending = false
|
||||
previousLenses = lenses
|
||||
}
|
||||
frameRecord(body, crc(input), streamId, flags, mtu)
|
||||
record(body, crc(input), flags)
|
||||
}
|
||||
|
||||
companion object {
|
||||
@@ -71,18 +82,12 @@ class CfwTransport(private val platform: ProtocolPlatform) {
|
||||
@JvmStatic
|
||||
fun frame(message: ByteArray?, streamId: Int, lenses: Int, mtu: Int): List<ByteArray> {
|
||||
validate(message, streamId, lenses, mtu)
|
||||
return frameRecord(message!!, crc(message), streamId, lenses or RESET_CONTEXT, mtu)
|
||||
return g2Packets(record(message!!, crc(message), lenses or RESET_CONTEXT), streamId, mtu)
|
||||
}
|
||||
|
||||
private fun frameRecord(
|
||||
body: ByteArray,
|
||||
checksum: Int,
|
||||
streamId: Int,
|
||||
flags: Int,
|
||||
mtu: Int,
|
||||
): List<ByteArray> {
|
||||
val lenses = flags and BOTH
|
||||
val capacity = minOf(252, mtu - 14)
|
||||
/** `[flags][wire length u16][decoded CRC u16][body]`, shared by every packet envelope. */
|
||||
@JvmStatic
|
||||
fun record(body: ByteArray, checksum: Int, flags: Int): ByteArray {
|
||||
val stream = ByteArray(body.size + 5)
|
||||
stream[0] = flags.toByte()
|
||||
stream[1] = body.size.toByte()
|
||||
@@ -90,6 +95,13 @@ class CfwTransport(private val platform: ProtocolPlatform) {
|
||||
stream[3] = checksum.toByte()
|
||||
stream[4] = (checksum ushr 8).toByte()
|
||||
body.copyInto(stream, 5)
|
||||
return stream
|
||||
}
|
||||
|
||||
/** G2 TPL packets (SID 0xf0) carrying one record: options byte = lenses | RESET | END. */
|
||||
private fun g2Packets(stream: ByteArray, streamId: Int, mtu: Int): List<ByteArray> {
|
||||
val lenses = stream[0].toInt() and BOTH
|
||||
val capacity = minOf(252, mtu - 14)
|
||||
val packets = mutableListOf<ByteArray>()
|
||||
for (offset in stream.indices step capacity) {
|
||||
val count = minOf(capacity, stream.size - offset)
|
||||
|
||||
+18
@@ -337,6 +337,24 @@ class MessageBuilder {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* CFW keyboard control (settings field 130). Always the right arm: only the master temple
|
||||
* acts on it, and a copy sent down each arm would run the control twice.
|
||||
*/
|
||||
fun faceclawKeyboardControl(record: ByteArray, label: String): OutboundMessage {
|
||||
return OutboundMessage(
|
||||
"keyboard-control",
|
||||
"keyboard control $label",
|
||||
BleProtocol.SID_UI_SETTING,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
0,
|
||||
BleProtocol.buildFaceclawKeyboardControl(record),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
fun faceclawWearQuery(leftArm: Boolean): OutboundMessage {
|
||||
return OutboundMessage(
|
||||
"wear-query-control",
|
||||
|
||||
+5
-1
@@ -18,6 +18,8 @@ class OutboundMessage(
|
||||
@JvmField
|
||||
val cfwChecksum: Int = if (sid == CfwTransport.SID) CfwTransport.crc(this.message) else 0
|
||||
@JvmField var cfwAckLenses = 0
|
||||
/** The lenses that must ACK this attempt: both on a G2, whichever the message went to on a G1. */
|
||||
@JvmField var cfwRequiredLenses = CfwTransport.BOTH
|
||||
@JvmField var cfwRetries = 0
|
||||
@JvmField var cfwRetryPending = false
|
||||
@JvmField var imageUpdateId = 0
|
||||
@@ -47,9 +49,11 @@ class OutboundMessage(
|
||||
}
|
||||
if (message.size != ack.size || cfwChecksum != ack.checksum) return false
|
||||
cfwAckLenses = cfwAckLenses or ack.lens
|
||||
return !cfwRetryPending && cfwAckLenses == CfwTransport.BOTH
|
||||
return !cfwRetryPending && isCfwAcked
|
||||
}
|
||||
|
||||
val isCfwAcked: Boolean get() = cfwAckLenses and cfwRequiredLenses == cfwRequiredLenses
|
||||
|
||||
/** Replace an attempt atomically under the communicator lock. */
|
||||
fun prepareCfwReplay(streamId: Int) {
|
||||
magic = streamId
|
||||
|
||||
@@ -48,6 +48,21 @@ interface StockLink {
|
||||
|
||||
fun bondState(address: String): BondState
|
||||
|
||||
/** The ATT MTU agreed on this link. */
|
||||
fun negotiatedMtu(address: String): Int = 23
|
||||
|
||||
/** Ask the OS to pair with a connected device; a platform that pairs by itself on demand does nothing. */
|
||||
fun requestBond(address: String) {}
|
||||
|
||||
/** Remove this phone's half of a pairing; false where the platform can't or refused. */
|
||||
fun removeBond(address: String): Boolean = false
|
||||
|
||||
/** After discovery: whether the service table has this characteristic. Enabling notifications on one it lacks may throw. */
|
||||
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean = true
|
||||
|
||||
/** Read the service table again, past whatever the OS cached for this address. */
|
||||
fun refreshServices(address: String, timeoutMs: Int): Boolean = discoverServices(address, timeoutMs)
|
||||
|
||||
fun disconnect(address: String)
|
||||
|
||||
fun close()
|
||||
|
||||
+20
-6
@@ -4,6 +4,11 @@ package com.faceclaw.app
|
||||
* queued state and sensor freshness do not. Private commands are ordered with
|
||||
* frames, so wake starts on the intended PRESENT, never on an older black frame. */
|
||||
internal fun GlassesSessionCore.enqueueBrightnessLocked(frameVisible: Boolean? = null) {
|
||||
if (isIo) {
|
||||
ioNoteFrameVisibleLocked(frameVisible)
|
||||
// Older iO firmware has no message 30; ioMaintain drives the launcher's (and the screen) instead.
|
||||
if (!ioFirmwareOwnsBrightness) return
|
||||
}
|
||||
if (!customFirmwareDetected || !sessionReady || !fixedLayoutCreated || shutdownRequested) return
|
||||
val visible = frameVisible ?: brightnessSentVisible ?: return
|
||||
val target = brightnessPolicy.target
|
||||
@@ -14,29 +19,38 @@ internal fun GlassesSessionCore.enqueueBrightnessLocked(frameVisible: Boolean? =
|
||||
else if (brightnessPolicy.automatic) 1200 else 120
|
||||
val payload = byteArrayOf(CFW_MSG_BRIGHTNESS.toByte(), 1, target.toByte(),
|
||||
(if (visible) 1 else 0).toByte(), duration.toByte(), (duration shr 8).toByte())
|
||||
val message = messageBuilder.customMessage("brightness-output", payload,
|
||||
"brightness target=$target visible=$visible fade=$duration", -1, connectionOptions.sendImagesToLeft)
|
||||
message.onTimeout = MessageCallback {
|
||||
val label = "brightness target=$target visible=$visible fade=$duration"
|
||||
val retry = MessageCallback {
|
||||
brightnessSentLevel = -1
|
||||
brightnessSentVisible = null
|
||||
// Retry with a PRESENT even if the scene did not change during recovery.
|
||||
lastEnqueuedFingerprint = ""
|
||||
}
|
||||
pendingMessages.addLast(message)
|
||||
if (isG1) {
|
||||
g1EnqueueBrightnessLocked(payload, label, brightnessSentVisible != visible, retry)
|
||||
} else {
|
||||
val message = messageBuilder.customMessage("brightness-output", payload, label, -1, connectionOptions.sendImagesToLeft)
|
||||
message.onTimeout = retry
|
||||
pendingMessages.addLast(message)
|
||||
}
|
||||
brightnessSentLevel = target
|
||||
brightnessSentVisible = visible
|
||||
if (isIo) io.lastBrightnessAtMs = now()
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.maintainBrightnessLocked(now: Long) {
|
||||
if (!customFirmwareDetected || !sessionReady || !fixedLayoutCreated || shutdownRequested) return
|
||||
// Passive polling owns stock ALS even in manual mode. Slower manual polling
|
||||
// keeps stock from writing the panel without needlessly streaming samples.
|
||||
if (now - brightnessAlsStartedAt >= 2000 && !hasPendingOrInflightKindLocked("als-control")) {
|
||||
val interval = if (brightnessDemoPolling) 250 else if (brightnessPolicy.automatic) 500 else 5000
|
||||
val interval = if (brightnessDemoPolling) 250 else if (brightnessPolicy.automatic) 500 else 5000
|
||||
// The G2's stock can close the sensor, so it hears this again; the iO's polling lasts as long as the page.
|
||||
val alsDue = if (isIo) interval != io.alsIntervalSent else now - brightnessAlsStartedAt >= 2000
|
||||
if (alsDue && !hasPendingOrInflightKindLocked("als-control")) {
|
||||
val payload = byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 1, 1,
|
||||
interval.toByte(), (interval shr 8).toByte(), 0, 0, 0xe8.toByte(), 3)
|
||||
enqueueAmbientLightControlLocked(payload, "brightness passive ALS", false)
|
||||
brightnessAlsStartedAt = now
|
||||
if (isIo) io.alsIntervalSent = interval
|
||||
}
|
||||
enqueueBrightnessLocked()
|
||||
}
|
||||
+23
-2
@@ -86,13 +86,16 @@ internal fun GlassesSessionCore.enqueueBenchmarkMessageLocked() {
|
||||
kotlin.random.Random.nextBytes(payload)
|
||||
payload[0] = CFW_MSG_DIAGNOSTICS.toByte() // CFW diagnostic-control mode...
|
||||
payload[1] = 0x7f.toByte() // ...with an unused sub-op: acked, no effect
|
||||
// A G1's arms are two links and every frame goes down both, so the run alternates between them.
|
||||
val leftArm = if (isG1) benchmarkMessageCounter++ % 2 == 0 else connectionOptions.sendImagesToLeft
|
||||
val message = messageBuilder.imagePayload(
|
||||
"bandwidth",
|
||||
DASHBOARD_TILE,
|
||||
nextMapSessionId(),
|
||||
payload,
|
||||
"bandwidth no-op " + payload.size + "B",
|
||||
connectionOptions.sendImagesToLeft)
|
||||
leftArm)
|
||||
if (isG1) message.cfwRequiredLenses = if (leftArm) G1Timings.LENS_LEFT else G1Timings.LENS_RIGHT
|
||||
val payloadBytes = payload.size
|
||||
// Logical custom-message bytes; excludes length tag and packet framing.
|
||||
val wireBytes = message.message.size
|
||||
@@ -345,6 +348,8 @@ internal fun GlassesSessionCore.releaseFaceclawFramebufferLease(): Boolean {
|
||||
|
||||
/** The address of a configured arm the OS no longer holds a bond for, or null. */
|
||||
internal fun GlassesSessionCore.firstUnpairedArm(): String? {
|
||||
// The iO identifies the phone by its protocol id, not an OS bond.
|
||||
if (isIo) return null
|
||||
if (!link.isBonded(rightAddress)) return rightAddress
|
||||
if (!link.isBonded(leftAddress)) return leftAddress
|
||||
return null
|
||||
@@ -491,7 +496,7 @@ internal fun GlassesSessionCore.scheduleReconnectLocked(sessionWasReady: Boolean
|
||||
if (sessionWasReady && now - lastSessionReadyAtMs >= ConnectionOptions.STABLE_SESSION_MS) {
|
||||
consecutiveReconnects = 0
|
||||
}
|
||||
val schedule = ConnectionOptions.RECONNECT_BACKOFF_MS
|
||||
val schedule = if (isG1) G1Timings.RECONNECT_BACKOFF_MS else ConnectionOptions.RECONNECT_BACKOFF_MS
|
||||
val delayMs = schedule[minOf(consecutiveReconnects, schedule.size - 1)].toLong()
|
||||
consecutiveReconnects++
|
||||
reconnectAfterMs = now + delayMs
|
||||
@@ -581,6 +586,22 @@ internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.emitKeyboardEvent(body: ByteArray) {
|
||||
if (keyboardEventListeners.isEmpty()) {
|
||||
return
|
||||
}
|
||||
val copy = body.copyOf()
|
||||
host.postToMain {
|
||||
for (keyboardListener in keyboardEventListeners) {
|
||||
try {
|
||||
keyboardListener.onKeyboardEvent(copy)
|
||||
} catch (t: Throwable) {
|
||||
logWarn("keyboard event listener failed", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.emitAudioMonitorPacket(data: ByteArray, arm: String, arrivalMs: Long) {
|
||||
if (audioMonitorListeners.isEmpty()) {
|
||||
return
|
||||
|
||||
+191
-30
@@ -107,6 +107,13 @@ class GlassesSessionCore(
|
||||
|
||||
@Volatile internal var listener: FaceclawBleCommunicatorListener? = null
|
||||
|
||||
/** G2 unless [useRayNeoIo] was called before [start]. */
|
||||
@Volatile internal var glassesType = GlassesType.G2
|
||||
/** RayNeo iO link state (GlassesSessionIo.kt); guarded by [monitor]. */
|
||||
internal val io = IoSessionState()
|
||||
/** Even G1 link state (GlassesSessionG1.kt); guarded by [monitor]. */
|
||||
internal val g1 = G1SessionState()
|
||||
|
||||
/**
|
||||
* Optional tap for relay frames the session does not interpret: notifications on the
|
||||
* right arm's data characteristic whose first two bytes are 'A','N' (the iOS ANCS
|
||||
@@ -124,6 +131,7 @@ class GlassesSessionCore(
|
||||
internal val compassSubscriptions = CopyOnWriteList<CompassSubscription>(platform)
|
||||
internal val ambientLightListeners = CopyOnWriteList<FaceclawAmbientLightListener>(platform)
|
||||
internal val micStatusListeners = CopyOnWriteList<FaceclawMicStatusListener>(platform)
|
||||
internal val keyboardEventListeners = CopyOnWriteList<FaceclawKeyboardEventListener>(platform)
|
||||
internal val audioMonitorListeners = CopyOnWriteList<FaceclawAudioPacketListener>(platform)
|
||||
@Volatile internal var running = false
|
||||
@Volatile internal var userDisconnectRequested = false
|
||||
@@ -451,6 +459,7 @@ class GlassesSessionCore(
|
||||
internal var benchmarkLastAckAtMs = 0L
|
||||
internal var benchmarkEndAtMs = 0L // 0 while running; set when the run drains
|
||||
internal var benchmarkMessagesSent = 0
|
||||
internal var benchmarkMessageCounter = 0
|
||||
internal var benchmarkMessagesAcked = 0
|
||||
internal var benchmarkTimeouts = 0
|
||||
internal var benchmarkPayloadBytesAcked = 0L
|
||||
@@ -538,7 +547,8 @@ class GlassesSessionCore(
|
||||
/**
|
||||
* Write one already-framed packet to an arm outside the message scheduler (the ANCS
|
||||
* relay client builds its own packets). Blocks like any link write; false when the
|
||||
* arm is unknown, the link rejects the write, or the session is not running.
|
||||
* arm is unknown, the link rejects the write, or the session is not running. A G1 has a
|
||||
* characteristic for them, whose ATT errors say why a command was refused.
|
||||
*/
|
||||
fun writeRawPacket(arm: String, packet: ByteArray): Boolean {
|
||||
val address = when (arm.uppercase()) {
|
||||
@@ -549,8 +559,11 @@ class GlassesSessionCore(
|
||||
val active = monitor.withLock { running }
|
||||
if (!active) return false
|
||||
return try {
|
||||
val result = link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE,
|
||||
ConnectionOptions.WRITE_TIMEOUT_MS)
|
||||
val result = if (isG1) {
|
||||
link.writeFrames(address, G1Ble.ANCS_CHAR_UUID, listOf(packet), GattWriteMode.WITH_RESPONSE, ConnectionOptions.WRITE_TIMEOUT_MS)
|
||||
} else {
|
||||
link.writeFrames(address, BleProtocol.WRITE_CHAR_UUID, listOf(packet), ConnectionOptions.WRITE_MODE, ConnectionOptions.WRITE_TIMEOUT_MS)
|
||||
}
|
||||
if (result) monitor.withLock { noteWriteCompletedLocked(address, now()) }
|
||||
result
|
||||
} catch (t: Throwable) {
|
||||
@@ -574,6 +587,39 @@ class GlassesSessionCore(
|
||||
requiredFirmwareRevision = revision
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive a RayNeo iO instead of a G2: one BLE link (pass its address as both arms; no
|
||||
* ring), the iO handshake and page lease, and the Faceclaw display lists of the
|
||||
* faceclaw-rayneo-cfw firmware. [phoneId] is this phone's stable 6-byte protocol identity:
|
||||
* the glasses bind to the first one that authenticates and refuse others until they are
|
||||
* put in pairing mode. Call before [start].
|
||||
*/
|
||||
fun useRayNeoIo(phoneId: ByteArray, phoneName: String, manufacturer: String, model: String) {
|
||||
require(phoneId.size == 6) { "RayNeo iO phone id must be 6 bytes" }
|
||||
require(rightAddress.equals(leftAddress, ignoreCase = true)) { "RayNeo iO uses one address for both arms" }
|
||||
monitor.withLock {
|
||||
glassesType = GlassesType.RAYNEO_IO
|
||||
io.phoneId = phoneId.copyOf()
|
||||
io.phoneName = phoneName.ifEmpty { "Faceclaw" }
|
||||
io.manufacturer = manufacturer.ifEmpty { "Android" }
|
||||
io.model = model.ifEmpty { "Android" }
|
||||
}
|
||||
}
|
||||
|
||||
/** Drive an Even G1 on g1v0 instead of a G2 (see [G1SessionState]). Before [start]. */
|
||||
fun useG1() {
|
||||
monitor.withLock { glassesType = GlassesType.G1 }
|
||||
}
|
||||
|
||||
/** True when this session drives an Even G1. */
|
||||
fun isEvenG1(): Boolean = isG1
|
||||
|
||||
/** Whether the glasses can relay iPhone notifications, as far as the session knows: a G2's depends on its firmware revision, which TS checks. */
|
||||
fun ancsRelayAvailable(): Boolean = !isG1 || monitor.withLock { g1.ancsRelay }
|
||||
|
||||
/** True when this session drives a RayNeo iO. */
|
||||
fun isRayNeoIo(): Boolean = isIo
|
||||
|
||||
/** Start the worker (through the host); false when it was already running. */
|
||||
fun start(): Boolean {
|
||||
monitor.withLock {
|
||||
@@ -606,7 +652,7 @@ class GlassesSessionCore(
|
||||
* quiescing its producers. Also cover direct/early close callers here;
|
||||
* a successful cleanup must remain the final BLE message. Older CFWs
|
||||
* fall back to the standalone framebuffer-lease release. */
|
||||
if (!cfwCleanupDelivered && !sendCfwCleanup()) {
|
||||
if (!cfwCleanupDelivered && !sendCfwCleanup() && isG2) {
|
||||
releaseFaceclawFramebufferLease()
|
||||
}
|
||||
monitor.withLock {
|
||||
@@ -625,6 +671,14 @@ class GlassesSessionCore(
|
||||
host.joinWorker(5_000)
|
||||
ringDone?.await(5_000)
|
||||
ringWorkerDone = null
|
||||
if (isIo && link.isConnected(rightAddress)) {
|
||||
// Otherwise the blank page stays up until the firmware's 120 s lease runs out.
|
||||
try {
|
||||
ioWriteControl(RayNeoIoProtocol.CTRL_PAGE_CLOSE)
|
||||
} catch (t: Throwable) {
|
||||
logLine("RayNeo iO page close failed: " + safeMessage(t))
|
||||
}
|
||||
}
|
||||
monitor.withLock {
|
||||
resetSessionStateLocked()
|
||||
clearAllMessagesLocked("disconnect")
|
||||
@@ -666,6 +720,9 @@ class GlassesSessionCore(
|
||||
if (listener == null) {
|
||||
throw IllegalArgumentException("listener is required")
|
||||
}
|
||||
// The iO's packets are Opus from its remote audio stream (see ioStartAudioCapture).
|
||||
if (isIo) return ioStartAudioCapture(listener)
|
||||
if (isG1) return g1StartAudioCapture(listener)
|
||||
var magic = 0
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady || shutdownRequested || !fixedLayoutCreated) {
|
||||
@@ -683,6 +740,8 @@ class GlassesSessionCore(
|
||||
}
|
||||
|
||||
fun stopG2AudioCapture() {
|
||||
if (isIo) return ioStopAudioCapture()
|
||||
if (isG1) return g1StopAudioCapture()
|
||||
var magic = 0
|
||||
monitor.withLock {
|
||||
audioPacketListener = null
|
||||
@@ -790,6 +849,15 @@ class GlassesSessionCore(
|
||||
* that track a capture across those events must check this rather than
|
||||
* assume their earlier enable still holds.
|
||||
*/
|
||||
/**
|
||||
* Whether the glasses' own wake word should reach Faceclaw (its configured wake action).
|
||||
* RayNeo iO: the launcher's "Hey RayNeo" switch, and the stock session it opens is
|
||||
* ended in favor of Faceclaw's. G2 glasses use the wake lease instead (no-op here).
|
||||
*/
|
||||
fun setVoiceWakeEnabled(enabled: Boolean) {
|
||||
if (isIo) ioSetVoiceWakeEnabled(enabled)
|
||||
}
|
||||
|
||||
fun isAudioCaptureActive(): Boolean {
|
||||
monitor.withLock {
|
||||
return running && sessionReady && !shutdownRequested && audioCaptureActive
|
||||
@@ -806,6 +874,8 @@ class GlassesSessionCore(
|
||||
val generation: Int
|
||||
monitor.withLock {
|
||||
faceclawWakeLeaseEnabled = enabled
|
||||
// A G2 firmware lease; the iO's wake word is setVoiceWakeEnabled.
|
||||
if (!isG2) return true
|
||||
if (!running || !sessionReady) {
|
||||
return !enabled
|
||||
}
|
||||
@@ -874,6 +944,7 @@ class GlassesSessionCore(
|
||||
* (ignored on disable).
|
||||
*/
|
||||
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) {
|
||||
if (isG1) return g1SetImuReport(enable)
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip IMU " + (if (enable) "enable" else "disable") + "; session not ready")
|
||||
@@ -1286,6 +1357,11 @@ class GlassesSessionCore(
|
||||
/** Request one CFW ambient-light report (image-handler mode 16 op 0). */
|
||||
fun queryAmbientLight() {
|
||||
monitor.withLock {
|
||||
if (isG1) {
|
||||
g1.lastLightAtMs = 0
|
||||
interruptibleSleep.interrupt()
|
||||
return
|
||||
}
|
||||
enqueueAmbientLightControlLocked(byteArrayOf(CFW_MSG_AMBIENT_LIGHT.toByte(), 0.toByte()), "als query", false)
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
@@ -1368,7 +1444,7 @@ class GlassesSessionCore(
|
||||
* queued (no session, or a malformed record).
|
||||
*/
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean {
|
||||
if (record == null || record.size < 4) {
|
||||
if (!isG2 || record == null || record.size < 4) {
|
||||
return false
|
||||
}
|
||||
monitor.withLock {
|
||||
@@ -1388,6 +1464,39 @@ class GlassesSessionCore(
|
||||
return true
|
||||
}
|
||||
|
||||
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
|
||||
if (listener != null) {
|
||||
keyboardEventListeners.add(listener)
|
||||
}
|
||||
}
|
||||
|
||||
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
|
||||
if (listener != null) {
|
||||
keyboardEventListeners.remove(listener)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a CFW keyboard-bridge control (['K','B',ver,op,request,epoch,...]) as a settings
|
||||
* field-130 write to the right temple. Fire-and-forget: the firmware answers with field-131
|
||||
* records, which arrive through addKeyboardEventListener. Returns false when nothing was
|
||||
* queued (not a G2, no session, or a malformed record).
|
||||
*/
|
||||
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean {
|
||||
if (!isG2 || record == null || record.size < 8) {
|
||||
return false
|
||||
}
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip keyboard control ($label); session not ready")
|
||||
return false
|
||||
}
|
||||
pendingMessages.addLast(messageBuilder.faceclawKeyboardControl(record, label))
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward render-characteristic audio packets to the listener WITHOUT
|
||||
* sending the stock EvenHub audio-control enable. Used for the CFW
|
||||
@@ -1619,6 +1728,8 @@ class GlassesSessionCore(
|
||||
// EvenHub page lifecycle
|
||||
|
||||
fun sendShutdown(exitMode: Int): Boolean {
|
||||
if (isIo) return ioSuspend()
|
||||
if (isG1) return g1Suspend()
|
||||
return sendShutdownInternal(exitMode, true)
|
||||
}
|
||||
|
||||
@@ -1629,6 +1740,17 @@ class GlassesSessionCore(
|
||||
* can use the legacy shutdown-and-lease-release path.
|
||||
*/
|
||||
fun sendCfwCleanup(): Boolean {
|
||||
if (isG1) {
|
||||
// Fire and forget: the firmware keeps no session that could outlive the link.
|
||||
monitor.withLock {
|
||||
if (cfwCleanupDelivered) return true
|
||||
shutdownRequested = true
|
||||
cfwCleanupDelivered = true
|
||||
clearPendingMessagesLocked("G1 cleanup")
|
||||
}
|
||||
g1HideBeforeDisconnect()
|
||||
return true
|
||||
}
|
||||
val magic: Int
|
||||
monitor.withLock {
|
||||
if (cfwCleanupDelivered) {
|
||||
@@ -1710,6 +1832,8 @@ class GlassesSessionCore(
|
||||
* reconnecting Bluetooth here would defeat the power-saving mode.
|
||||
*/
|
||||
fun suspendEvenHubSession(): Boolean {
|
||||
if (isIo) return ioSuspend()
|
||||
if (isG1) return g1Suspend()
|
||||
// Ending the plugin task releases the fb lease, which frees the
|
||||
// on-glasses resource cache; forget it phone-side either way (a lost
|
||||
// ack may still have taken effect).
|
||||
@@ -1733,6 +1857,8 @@ class GlassesSessionCore(
|
||||
* the desired frame.
|
||||
*/
|
||||
fun resumeEvenHubSession(): Boolean {
|
||||
if (isIo) return ioResume()
|
||||
if (isG1) return g1Resume()
|
||||
var claimGeneration = 0
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady || chargingMode) {
|
||||
@@ -1813,13 +1939,13 @@ class GlassesSessionCore(
|
||||
continue
|
||||
}
|
||||
logWarn("Attempting to connect")
|
||||
connectLoopOnce()
|
||||
if (isIo) ioConnectLoopOnce() else if (isG1) g1ConnectLoopOnce() else connectLoopOnce()
|
||||
continue
|
||||
}
|
||||
|
||||
val sleepMs = driveSession()
|
||||
if (sleepMs > 0) {
|
||||
interruptibleSleep.sleep(sleepMs)
|
||||
interruptibleSleep.sleep(if (isG1) minOf(sleepMs, g1MaxSleepMs()) else sleepMs)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
if (!running) {
|
||||
@@ -1840,10 +1966,19 @@ class GlassesSessionCore(
|
||||
return
|
||||
}
|
||||
val uuid = characteristicUuid.lowercase()
|
||||
// The phone's own R1 link, beside either glasses (the iO's ring is always direct).
|
||||
if (isDirectRingNotification(address, uuid)) {
|
||||
handleDirectRingNotification(uuid, data)
|
||||
return
|
||||
}
|
||||
if (isIo) {
|
||||
ioOnNotification(address, uuid, data)
|
||||
return
|
||||
}
|
||||
if (isG1) {
|
||||
g1OnNotification(address, uuid, data)
|
||||
return
|
||||
}
|
||||
if (BleProtocol.RENDER_NOTIFY_UUID == uuid) {
|
||||
handleRenderNotification(address, data)
|
||||
return
|
||||
@@ -1854,28 +1989,7 @@ class GlassesSessionCore(
|
||||
if (data.size >= 7 && (data[6].toInt() and 255) == CfwTransport.SID) {
|
||||
val acks = CfwTransport.parseAcks(data) ?: return
|
||||
monitor.withLock {
|
||||
lastIncomingAtMs = now()
|
||||
val arm = armIndex(address)
|
||||
if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs
|
||||
for (ack in acks) {
|
||||
for (message in inFlightMessages) {
|
||||
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
|
||||
if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) {
|
||||
message.acceptCfwAck(ack)
|
||||
if (ack.nack) ConnectionCounters.increment("g2.cfw-nack")
|
||||
logLine("CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId
|
||||
+ " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2].toInt() and 255)
|
||||
+ " redundant=" + (ack !== acks[0])
|
||||
+ " size=" + ack.size + " crc=" + ack.checksum
|
||||
+ " expected=" + message.message.size + "/" + message.cfwChecksum
|
||||
+ " ackedLenses=" + message.cfwAckLenses
|
||||
+ " ageMs=" + (lastIncomingAtMs - message.sentAtMs))
|
||||
message.ackPayload = data.copyOf()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
drainCfwAcknowledgementsLocked()
|
||||
acceptCfwAcksLocked(address, acks, data[2].toInt() and 255, data)
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
return
|
||||
@@ -1889,6 +2003,18 @@ class GlassesSessionCore(
|
||||
}
|
||||
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
|
||||
val frame = BleProtocol.parseFrame(data)
|
||||
if (frame.ok && frame.sid == BleProtocol.SID_UI_SETTING) {
|
||||
// CFW keyboard-bridge records (field 131) come from whichever temple hosts the
|
||||
// keyboard. They are their own stream: neither an ACK nor glasses input, and the
|
||||
// once-a-second status record among them must not count as user activity (which
|
||||
// would hold off battery polling).
|
||||
val keyboardEvent = BleProtocol.parseFaceclawKeyboardEvent(frame.pb)
|
||||
if (keyboardEvent != null) {
|
||||
monitor.withLock { lastIncomingAtMs = now() }
|
||||
emitKeyboardEvent(keyboardEvent)
|
||||
return
|
||||
}
|
||||
}
|
||||
val decodedWearState = BleProtocol.parseWearState(frame)
|
||||
val compassEvent = if (address.equals(rightAddress, ignoreCase = true))
|
||||
BleProtocol.parseCompassEvent(frame)
|
||||
@@ -2169,6 +2295,7 @@ class GlassesSessionCore(
|
||||
lastConnectionOrInputAtMs = arrivalMs
|
||||
lastDirectRingInputAtMs = arrivalMs
|
||||
lastDirectRingInputLabel = decoded.label + " " + decoded.detail
|
||||
if (isIo) ioReclaimPageOnInputLocked()
|
||||
}
|
||||
logLine("direct ring " + decoded.label + " " + decoded.detail + " raw=" + hex(data))
|
||||
val frameId = frameTimings.startFrame("input:ring:" + decoded.label)
|
||||
@@ -2196,6 +2323,35 @@ class GlassesSessionCore(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply CFW ACKs/NACKs from [address] (the G2's `aa 12` frames, or the iO's 9-byte bodies)
|
||||
* to the in-flight messages they name, then complete what they finished.
|
||||
*/
|
||||
internal fun acceptCfwAcksLocked(address: String, acks: Array<CfwTransport.Ack>, txSeq: Int, payload: ByteArray) {
|
||||
lastIncomingAtMs = now()
|
||||
val arm = armIndex(address)
|
||||
if (arm >= 0) lastArmCfwAckAtMs[arm] = lastIncomingAtMs
|
||||
for (ack in acks) {
|
||||
for (message in inFlightMessages) {
|
||||
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
|
||||
if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) {
|
||||
message.acceptCfwAck(ack)
|
||||
if (ack.nack) ConnectionCounters.increment("g2.cfw-nack")
|
||||
logLine("CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId
|
||||
+ " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + txSeq
|
||||
+ " redundant=" + (ack !== acks[0])
|
||||
+ " size=" + ack.size + " crc=" + ack.checksum
|
||||
+ " expected=" + message.message.size + "/" + message.cfwChecksum
|
||||
+ " ackedLenses=" + message.cfwAckLenses
|
||||
+ " ageMs=" + (lastIncomingAtMs - message.sentAtMs))
|
||||
message.ackPayload = payload.copyOf()
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
drainCfwAcknowledgementsLocked()
|
||||
}
|
||||
|
||||
override fun onConnectionStateChange(address: String?, connected: Boolean) {
|
||||
if (address == null) {
|
||||
return
|
||||
@@ -2231,12 +2387,17 @@ class GlassesSessionCore(
|
||||
monitor.withLock {
|
||||
if (address.equals(rightAddress, ignoreCase = true)) {
|
||||
rightConnected = connected
|
||||
// The iO's one link stands for both arms.
|
||||
if (isIo) leftConnected = connected
|
||||
} else if (address.equals(leftAddress, ignoreCase = true)) {
|
||||
leftConnected = connected
|
||||
} else {
|
||||
return
|
||||
}
|
||||
if (!connected) {
|
||||
if (!connected && isIo) io.reassembler.reset()
|
||||
if (!connected && isG1 && g1CarryOnWithoutArmLocked(address)) {
|
||||
// The session goes on; the worker brings this arm back.
|
||||
} else if (!connected) {
|
||||
// Before the session is ready, this is an attempt failing (or an arm
|
||||
// dropping mid-setup): the connect sequence notices on its own and
|
||||
// schedules the retry, so leave the backoff alone.
|
||||
|
||||
+1033
File diff suppressed because it is too large.
Load diff
+774
@@ -0,0 +1,774 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import com.faceclaw.app.GlassesSessionCore.Companion.hex
|
||||
import com.faceclaw.app.GlassesSessionCore.Companion.safeMessage
|
||||
import kotlin.random.Random
|
||||
|
||||
/** Which glasses a session drives. Set before [GlassesSessionCore.start]; see [GlassesSessionCore.useRayNeoIo]. */
|
||||
enum class GlassesType { G2, RAYNEO_IO, G1 }
|
||||
|
||||
/**
|
||||
* RayNeo iO session state ([GlassesSessionCore.io]); guarded by the core's monitor.
|
||||
*
|
||||
* The iO is one SoC on ONE BLE link that drives both eyes, so the core runs it with the same
|
||||
* address in both arm slots: CFW messages keep going to the "left" arm, per-arm bookkeeping
|
||||
* resolves to index 0, and the firmware ACKs each lens of a record separately, exactly what
|
||||
* the existing window expects. What differs lives here: the AA55 envelope, the TLV handshake
|
||||
* and guest authentication, IDENT-based capability detection, and the page lease
|
||||
* (PAGE_OPEN + KEEPALIVE) that stands in for the G2's EvenHub page and framebuffer lease.
|
||||
* Wire codecs: [RayNeoIoProtocol]; firmware: faceclaw-rayneo-cfw (docs/FACECLAW.md).
|
||||
*/
|
||||
internal class IoSessionState {
|
||||
var phoneId = ByteArray(6)
|
||||
var phoneName = "Faceclaw"
|
||||
var manufacturer = "Android"
|
||||
var model = "Android"
|
||||
|
||||
val reassembler = RayNeoIoProtocol.FrameReassembler()
|
||||
var messageNumber = Random.nextInt(0, 0x10000)
|
||||
var launcherSequence = 0
|
||||
|
||||
var versionReply: RayNeoIoProtocol.VersionReply? = null
|
||||
var authReply: ByteArray? = null
|
||||
var ident: RayNeoIoProtocol.Ident? = null
|
||||
var osVersion = ""
|
||||
var maxChunk = RayNeoIoProtocol.DEFAULT_MAX_CHUNK
|
||||
|
||||
/** Last PAGE state the glasses reported: 1 open, 0 closed, -1 none yet this link. */
|
||||
var pageState = -1
|
||||
var pageReason = 0
|
||||
var pageReplies = 0
|
||||
/** Whether the phone wants the page open (false while suspended or after the wearer closed it). */
|
||||
var pageWanted = true
|
||||
var pageRequestSentAtMs = 0L
|
||||
var lastKeepaliveAtMs = 0L
|
||||
var lastStatusAtMs = 0L
|
||||
var refusals = 0
|
||||
|
||||
/** Whether Faceclaw's latest frame is meant to be seen (false for its blanked frames). */
|
||||
var displayVisible = false
|
||||
var brightnessSentLevel = -1
|
||||
var lastBrightnessAtMs = 0L
|
||||
/** The light sensor poll interval this page was last asked for ([ioFirmwareOwnsBrightness]), -1 none. */
|
||||
var alsIntervalSent = -1
|
||||
/** Last `screen_status` push: 1 on, 0 off, -1 unknown. */
|
||||
var screenStatus = -1
|
||||
|
||||
/** Whether Faceclaw answers "Hey RayNeo" (null until the dashboard says); what this link was last told. */
|
||||
var voiceWakeWanted: Boolean? = null
|
||||
var voiceWakeSent = -1
|
||||
/** The stock assistant opened a session (wake word): end it, it must not take the display. */
|
||||
var voiceAbortPending = false
|
||||
/** Ask for the launcher's general settings (logged: the wake-word preferences among them). */
|
||||
var settingsRequestPending = true
|
||||
/** When a wake was last reported: one phrase can wake twice (fast and second stage). */
|
||||
var lastVoiceWakeAtMs = 0L
|
||||
|
||||
/** A capture wants the remote mic stream; whether this link was asked to open it, and when. */
|
||||
var micWanted = false
|
||||
var micStreamOpen = false
|
||||
var micStartSentAtMs = 0L
|
||||
var micPackets = 0L
|
||||
var lastMicPacketAtMs = 0L
|
||||
|
||||
fun resetLink() {
|
||||
reassembler.reset()
|
||||
versionReply = null
|
||||
authReply = null
|
||||
ident = null
|
||||
pageState = -1
|
||||
pageReason = 0
|
||||
pageRequestSentAtMs = 0
|
||||
lastKeepaliveAtMs = 0
|
||||
lastStatusAtMs = 0
|
||||
brightnessSentLevel = -1
|
||||
lastBrightnessAtMs = 0
|
||||
alsIntervalSent = -1
|
||||
screenStatus = -1
|
||||
voiceWakeSent = -1
|
||||
voiceAbortPending = false
|
||||
settingsRequestPending = true
|
||||
micStreamOpen = false
|
||||
micStartSentAtMs = 0
|
||||
}
|
||||
}
|
||||
|
||||
internal object IoTimings {
|
||||
const val VERSION_REPLY_MS = 4_000L
|
||||
const val AUTH_REPLY_MS = 4_000L
|
||||
const val OS_VERSION_MS = 1_500L
|
||||
const val IDENT_MS = 3_000L
|
||||
const val PAGE_REPLY_MS = 4_000L
|
||||
/** Well inside the firmware's 120 s page lease. */
|
||||
const val KEEPALIVE_MS = 30_000L
|
||||
const val STATUS_MS = 60_000L
|
||||
const val PAGE_RETRY_MS = 3_000L
|
||||
/** Retry cadence while the glasses refuse this phone id (waiting for pairing mode). */
|
||||
const val PAIRING_RETRY_MS = 2_000L
|
||||
const val PAGE_REASON_USER = 4
|
||||
/** While Faceclaw shows content, re-assert the screen this often (the stock auto-off is short). */
|
||||
const val SCREEN_REFRESH_MS = 5_000L
|
||||
/** A wanted mic stream that hasn't delivered a packet this long is asked for again. */
|
||||
const val MIC_RETRY_MS = 3_000L
|
||||
/** Wakes this close to a reported one are the same phrase (fast stage, then second stage). */
|
||||
const val VOICE_WAKE_DEDUP_MS = 3_000L
|
||||
}
|
||||
|
||||
/** The glasses hung up at the version request: bound to another phone id, or not in pairing mode. */
|
||||
internal class IoPairingRefusedException : IllegalStateException("glasses hung up at the version request")
|
||||
|
||||
/** CFW message types the iO firmware implements (fc_glue.c + raster cleanup). */
|
||||
internal fun ioSupportsCfwType(type: Int): Boolean =
|
||||
type == CFW_MSG_CLEANUP || type == CFW_MSG_UPLOAD_RESOURCE || type == CFW_MSG_EVICT_RESOURCE ||
|
||||
type == CFW_MSG_DRAW_CALLS || type == CFW_MSG_SET_ROOT_DISPLAY_LIST ||
|
||||
type == CFW_MSG_PRESENT || type == CFW_MSG_CREATE_SURFACE
|
||||
|
||||
internal val GlassesSessionCore.isIo: Boolean get() = glassesType == GlassesType.RAYNEO_IO
|
||||
|
||||
/**
|
||||
* The firmware runs message 30 and the light sensor (faceclaw-rayneo-cfw bright.c), with the G2's
|
||||
* wire formats, so the shared brightness code drives it. Older firmware gets the launcher's
|
||||
* brightness command, which is all it has (see [ioMaintain]).
|
||||
*/
|
||||
internal val GlassesSessionCore.ioFirmwareOwnsBrightness: Boolean get() = io.ident?.brightnessControl == true
|
||||
|
||||
/** Messages the iO cannot take (stock EvenHub/settings traffic, the G2's other hardware CFW modes). */
|
||||
internal fun GlassesSessionCore.ioSupportsMessage(message: OutboundMessage): Boolean {
|
||||
if (message.sid != CfwTransport.SID || message.message.isEmpty()) return false
|
||||
val type = message.message[0].toInt() and 255
|
||||
return ioSupportsCfwType(type) ||
|
||||
(ioFirmwareOwnsBrightness && (type == CFW_MSG_BRIGHTNESS || type == CFW_MSG_AMBIENT_LIGHT))
|
||||
}
|
||||
|
||||
/** A page opened (or reopened): the firmware gave brightness back when the last one closed. */
|
||||
private fun GlassesSessionCore.ioResetBrightnessLocked() {
|
||||
brightnessSentVisible = null
|
||||
brightnessSentLevel = -1
|
||||
brightnessAlsStartedAt = -2000
|
||||
brightnessPolicy.resetSamples()
|
||||
io.alsIntervalSent = -1
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Writes
|
||||
|
||||
private fun GlassesSessionCore.ioNextMessageNumberLocked(): Int {
|
||||
val n = io.messageNumber and 0xffff
|
||||
io.messageNumber = (io.messageNumber + 1) and 0xffff
|
||||
return n
|
||||
}
|
||||
|
||||
/** One AA55 frame on the iO write characteristic; worker thread, outside the monitor. */
|
||||
internal fun GlassesSessionCore.ioWriteFrame(business: Int, payload: ByteArray): Boolean {
|
||||
val frame = monitor.withLock { RayNeoIoProtocol.encodeFrame(ioNextMessageNumberLocked(), business, payload) }
|
||||
return link.writeFrames(
|
||||
rightAddress, RayNeoIoBle.WRITE_CHAR_UUID,
|
||||
RayNeoIoProtocol.writes(frame, link.negotiatedMtu(rightAddress)),
|
||||
GattWriteMode.WITHOUT_RESPONSE, ConnectionOptions.WRITE_TIMEOUT_MS,
|
||||
)
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.ioWriteControl(ctrl: Int): Boolean =
|
||||
ioWriteFrame(RayNeoIoProtocol.BUSINESS_CFW, RayNeoIoProtocol.controlPacket(ctrl))
|
||||
|
||||
private fun GlassesSessionCore.ioWriteLauncher(type: Int, json: String): Boolean {
|
||||
val sequence = monitor.withLock {
|
||||
io.launcherSequence = if (io.launcherSequence >= 0xffff) 1 else io.launcherSequence + 1
|
||||
io.launcherSequence
|
||||
}
|
||||
val envelope = RayNeoIoProtocol.Envelope(type, json, sequence = sequence, mode = RayNeoIoProtocol.MODE_REQUEST)
|
||||
return ioWriteFrame(RayNeoIoProtocol.BUSINESS_LAUNCHER, RayNeoIoProtocol.encodeEnvelope(envelope))
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.ioRequestStatus(): Boolean =
|
||||
ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_GENERAL_STATUS, RayNeoIoProtocol.GENERAL_STATUS_REQUEST)
|
||||
|
||||
/** From the image path (enqueueBrightnessLocked): a visible frame wakes the screen at once. */
|
||||
internal fun GlassesSessionCore.ioNoteFrameVisibleLocked(frameVisible: Boolean?) {
|
||||
if (frameVisible == null) return
|
||||
if (frameVisible && !io.displayVisible) io.lastBrightnessAtMs = 0
|
||||
io.displayVisible = frameVisible
|
||||
}
|
||||
|
||||
/**
|
||||
* The writes for one queued CFW message: the shared record (persistent deflate context, as
|
||||
* on the G2) in `[opt][stream][chunk]` packets, each its own AA55 frame. The stream byte is
|
||||
* the message's magic, which the firmware echoes in its per-lens ACKs.
|
||||
*/
|
||||
internal fun GlassesSessionCore.ioCfwWrites(message: OutboundMessage, mtu: Int): List<ByteArray> {
|
||||
val transport = cfwTransports[0]
|
||||
if (message.cfwRetries > 0) transport.reset()
|
||||
val record = transport.encodeRecord(message.message, CfwTransport.BOTH)
|
||||
return monitor.withLock {
|
||||
RayNeoIoProtocol.cfwPackets(record, message.magic, io.maxChunk).flatMap { packet ->
|
||||
RayNeoIoProtocol.writes(RayNeoIoProtocol.encodeFrame(ioNextMessageNumberLocked(), RayNeoIoProtocol.BUSINESS_CFW, packet), mtu)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A message the iO can't run (see [ioSupportsMessage]): never sent, never waited on. */
|
||||
internal fun GlassesSessionCore.ioDropUnsupported(message: OutboundMessage) {
|
||||
monitor.withLock {
|
||||
inFlightMessages.remove(message)
|
||||
if (message.magic != 0) magicPool.release(message.sid, message.magic, message.label, "unsupported on these glasses")
|
||||
}
|
||||
logDebug("dropped a message these glasses don't take: " + message.label + " sid=" + message.sid)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Connect (worker thread)
|
||||
|
||||
private fun GlassesSessionCore.ioAwait(timeoutMs: Long, what: String, ready: () -> Boolean) {
|
||||
val deadline = now() + timeoutMs
|
||||
monitor.withLock {
|
||||
while (true) {
|
||||
if (ready()) return
|
||||
if (!running || userDisconnectRequested) throw IllegalStateException("stopped while waiting for $what")
|
||||
if (!rightConnected) throw IoLinkLostException(what)
|
||||
val remaining = deadline - now()
|
||||
if (remaining <= 0) throw IllegalStateException("$what timed out")
|
||||
monitor.awaitMs(minOf(remaining, 100L))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class IoLinkLostException(what: String) : IllegalStateException("link lost while waiting for $what")
|
||||
|
||||
internal fun GlassesSessionCore.ioConnectLoopOnce() {
|
||||
monitor.withLock {
|
||||
wearState = -1
|
||||
io.resetLink()
|
||||
}
|
||||
setStateDisplay("connecting", "Connecting to the glasses...")
|
||||
ConnectionCounters.increment("io.connect-attempt")
|
||||
val address = rightAddress
|
||||
try {
|
||||
if (!link.connect(address, ConnectionOptions.ARM_CONNECT_WINDOW_MS)) {
|
||||
handleTransportFailure("glasses not found", "Can't find the glasses. They may be off, in their case, or out of range.")
|
||||
return
|
||||
}
|
||||
link.requestHighPriority(address)
|
||||
link.requestMtu(address, RayNeoIoBle.REQUESTED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
|
||||
if (!link.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) throw IllegalStateException("discoverServices failed: $address")
|
||||
if (!link.enableNotifications(address, RayNeoIoBle.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
|
||||
throw IllegalStateException("enableNotifications failed: $address")
|
||||
}
|
||||
link.requestLe2mPhy(address)
|
||||
monitor.withLock {
|
||||
rightConnected = true
|
||||
leftConnected = true
|
||||
}
|
||||
|
||||
// Version exchange. Glasses bound to another phone id (or not in pairing mode)
|
||||
// hang up ~100 ms after this request instead of answering.
|
||||
val phone = monitor.withLock { io.phoneId.copyOf() }
|
||||
logDebug("RayNeo iO: phone id " + hex(phone))
|
||||
if (!ioWriteFrame(RayNeoIoProtocol.BUSINESS_HANDSHAKE,
|
||||
RayNeoIoProtocol.versionRequest(phone, io.phoneName, io.manufacturer, io.model))) {
|
||||
throw IllegalStateException("version request write failed")
|
||||
}
|
||||
try {
|
||||
ioAwait(IoTimings.VERSION_REPLY_MS, "version reply") { io.versionReply != null }
|
||||
} catch (lost: IoLinkLostException) {
|
||||
throw IoPairingRefusedException()
|
||||
}
|
||||
val reply = monitor.withLock { io.versionReply!! }
|
||||
if (reply.accountBound) throw IoPairingRefusedException()
|
||||
logLine("RayNeo iO: glasses " + hex(reply.glassesId) + " serial " + reply.serial + " protocol " + reply.protocolVersion)
|
||||
|
||||
val random = Random.nextBytes(4)
|
||||
if (!ioWriteFrame(RayNeoIoProtocol.BUSINESS_HANDSHAKE, RayNeoIoProtocol.guestAuthRequest(random, phone, reply.glassesId))) {
|
||||
throw IllegalStateException("guest auth write failed")
|
||||
}
|
||||
ioAwait(IoTimings.AUTH_REPLY_MS, "guest auth reply") { io.authReply != null }
|
||||
val authReply = monitor.withLock { io.authReply!! }
|
||||
if (!RayNeoIoProtocol.verifyGuestAuthReply(authReply, phone, reply.glassesId)) {
|
||||
throw IllegalStateException("guest auth proof mismatch")
|
||||
}
|
||||
monitor.withLock { io.refusals = 0 }
|
||||
|
||||
// Battery and the OS version: answers arrive asynchronously; the version only labels
|
||||
// the firmware info, so a missing one is not fatal.
|
||||
ioRequestStatus()
|
||||
ioWriteFrame(RayNeoIoProtocol.BUSINESS_OTA,
|
||||
RayNeoIoProtocol.encodeEnvelope(RayNeoIoProtocol.Envelope(RayNeoIoProtocol.OTA_VERSION, "{}")))
|
||||
try {
|
||||
ioAwait(IoTimings.OS_VERSION_MS, "OS version") { io.osVersion.isNotEmpty() }
|
||||
} catch (e: IoLinkLostException) {
|
||||
throw e
|
||||
} catch (_: IllegalStateException) {
|
||||
}
|
||||
|
||||
// Capability: the custom firmware answers QUERY with IDENT; stock firmware stays silent.
|
||||
ioWriteControl(RayNeoIoProtocol.CTRL_QUERY)
|
||||
val ident = try {
|
||||
ioAwait(IoTimings.IDENT_MS, "IDENT") { io.ident != null }
|
||||
monitor.withLock { io.ident }
|
||||
} catch (e: IoLinkLostException) {
|
||||
throw e
|
||||
} catch (_: IllegalStateException) {
|
||||
null
|
||||
}
|
||||
val os = monitor.withLock { io.osVersion.ifEmpty { "RayNeo iO" } }
|
||||
val info = BleProtocol.FirmwareInfo(os, os,
|
||||
if (ident != null && ident.faceclaw) "Faceclaw/" + ident.faceclawRevision else "")
|
||||
logLine("RayNeo iO: " + os + ", " + (if (ident == null) "no IDENT (stock firmware)"
|
||||
else ident.name + " " + ident.width + "x" + ident.height + " chunk " + ident.maxChunk
|
||||
+ " flags 0x" + ident.flags.toString(16) + " faceclaw " + ident.faceclawRevision))
|
||||
emitFirmwareInfo(info)
|
||||
if (!info.isCompatible(requiredFirmwareRevision)) {
|
||||
monitor.withLock {
|
||||
// The worker loop halts on this and explains it (handleIncompatibleFirmware).
|
||||
incompatibleFirmware = info
|
||||
}
|
||||
return
|
||||
}
|
||||
monitor.withLock {
|
||||
io.maxChunk = ident!!.maxChunk.coerceIn(16, 4000)
|
||||
io.pageWanted = true
|
||||
io.pageRequestSentAtMs = now()
|
||||
}
|
||||
|
||||
ioWriteControl(RayNeoIoProtocol.CTRL_PAGE_OPEN)
|
||||
try {
|
||||
ioAwait(IoTimings.PAGE_REPLY_MS, "page open") { io.pageReplies > 0 }
|
||||
} catch (e: IoLinkLostException) {
|
||||
throw e
|
||||
} catch (_: IllegalStateException) {
|
||||
// The launcher may be busy; ioMaintain keeps asking.
|
||||
}
|
||||
|
||||
monitor.withLock {
|
||||
sessionReady = true
|
||||
shutdownRequested = false
|
||||
customFirmwareDetected = true
|
||||
clearAllMessagesLocked("session ready")
|
||||
fixedLayoutCreated = io.pageState == 1
|
||||
displayedFingerprint = ""
|
||||
lastAckAtMs = now()
|
||||
lastIncomingAtMs = lastAckAtMs
|
||||
lastConnectionOrInputAtMs = lastAckAtMs
|
||||
lastSessionReadyAtMs = lastAckAtMs
|
||||
imageRetryAfterMs = 0
|
||||
consecutiveAckTimeouts = 0
|
||||
cfwCleanupDelivered = false
|
||||
lastCfwCleanupAckMagic = 0
|
||||
for (transport in cfwTransports) transport.reset()
|
||||
io.lastKeepaliveAtMs = lastAckAtMs
|
||||
io.lastStatusAtMs = lastAckAtMs
|
||||
ioResetBrightnessLocked()
|
||||
}
|
||||
setStateDisplay("connected", "Connected.")
|
||||
ConnectionCounters.increment("io.session-ready")
|
||||
logLine("RayNeo iO session ready (page " + (if (fixedLayoutCreated) "open" else "pending") + ")")
|
||||
} catch (refused: IoPairingRefusedException) {
|
||||
val count = monitor.withLock {
|
||||
io.refusals++
|
||||
reconnectAfterMs = now() + IoTimings.PAIRING_RETRY_MS
|
||||
io.refusals
|
||||
}
|
||||
ConnectionCounters.increment("io.pairing-refused")
|
||||
logLine("RayNeo iO refused this phone (" + count + "); waiting for pairing mode")
|
||||
link.disconnect(address)
|
||||
monitor.withLock {
|
||||
rightConnected = false
|
||||
leftConnected = false
|
||||
}
|
||||
setStateDisplay("retrying",
|
||||
"The glasses aren't accepting this phone yet. Put them in pairing mode: on the charger, "
|
||||
+ "hold the button about 5 seconds until the light blinks blue.")
|
||||
} catch (t: Throwable) {
|
||||
logLine("RayNeo iO connect failed: " + safeMessage(t))
|
||||
handleTransportFailure("connect failed")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Session upkeep (worker thread, from driveSession, outside the monitor)
|
||||
|
||||
/** Page open/close requests, keepalives, status polls and the screen (launcher brightness). */
|
||||
internal fun GlassesSessionCore.ioMaintain() {
|
||||
var ctrl = -1
|
||||
var status = false
|
||||
var brightness = -1
|
||||
var wakeword = -1
|
||||
var micRc = -1
|
||||
var abortVoice = false
|
||||
var settings = false
|
||||
monitor.withLock {
|
||||
if (!sessionReady) return
|
||||
val now = now()
|
||||
val wake = io.voiceWakeWanted
|
||||
if (wake != null && io.voiceWakeSent != (if (wake) 1 else 0)) {
|
||||
wakeword = if (wake) 1 else 0
|
||||
io.voiceWakeSent = wakeword
|
||||
// Read back what the switch did.
|
||||
io.settingsRequestPending = true
|
||||
}
|
||||
if (io.settingsRequestPending) {
|
||||
settings = true
|
||||
io.settingsRequestPending = false
|
||||
}
|
||||
if (io.voiceAbortPending) {
|
||||
abortVoice = true
|
||||
io.voiceAbortPending = false
|
||||
}
|
||||
if (io.micWanted && (!io.micStreamOpen
|
||||
|| (io.lastMicPacketAtMs < io.micStartSentAtMs && now - io.micStartSentAtMs >= IoTimings.MIC_RETRY_MS))) {
|
||||
// Not yet asked on this link, a voice session took the recorder, or the ask was lost.
|
||||
micRc = RayNeoIoProtocol.VOICE_RC_START
|
||||
io.micStreamOpen = true
|
||||
io.micStartSentAtMs = now
|
||||
} else if (!io.micWanted && io.micStreamOpen) {
|
||||
micRc = RayNeoIoProtocol.VOICE_RC_STOP
|
||||
io.micStreamOpen = false
|
||||
}
|
||||
if (io.pageWanted && !fixedLayoutCreated && now - io.pageRequestSentAtMs >= IoTimings.PAGE_RETRY_MS) {
|
||||
ctrl = RayNeoIoProtocol.CTRL_PAGE_OPEN
|
||||
io.pageRequestSentAtMs = now
|
||||
} else if (!io.pageWanted && io.pageState == 1 && now - io.pageRequestSentAtMs >= IoTimings.PAGE_RETRY_MS) {
|
||||
ctrl = RayNeoIoProtocol.CTRL_PAGE_CLOSE
|
||||
io.pageRequestSentAtMs = now
|
||||
} else if (io.pageWanted && fixedLayoutCreated && now - io.lastKeepaliveAtMs >= IoTimings.KEEPALIVE_MS) {
|
||||
ctrl = RayNeoIoProtocol.CTRL_KEEPALIVE
|
||||
io.lastKeepaliveAtMs = now
|
||||
}
|
||||
if (now - io.lastStatusAtMs >= IoTimings.STATUS_MS) {
|
||||
status = true
|
||||
io.lastStatusAtMs = now
|
||||
}
|
||||
// Faceclaw decides when the display is dark (blanked frames); while it shows
|
||||
// content, keep the iO's screen on and at Faceclaw's brightness.
|
||||
val showing = fixedLayoutCreated && io.pageWanted && io.displayVisible && !shutdownRequested
|
||||
if (ioFirmwareOwnsBrightness) {
|
||||
// Repeating message 30 starts no fade; visible=1 is what holds the screen on.
|
||||
if (showing && now - io.lastBrightnessAtMs >= IoTimings.SCREEN_REFRESH_MS) {
|
||||
io.lastBrightnessAtMs = now
|
||||
brightnessSentLevel = -1
|
||||
enqueueBrightnessLocked()
|
||||
}
|
||||
} else {
|
||||
val level = RayNeoIoProtocol.brightnessLevel(brightnessPolicy.target)
|
||||
if (showing && (level != io.brightnessSentLevel || now - io.lastBrightnessAtMs >= IoTimings.SCREEN_REFRESH_MS)) {
|
||||
brightness = level
|
||||
io.brightnessSentLevel = level
|
||||
io.lastBrightnessAtMs = now
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ctrl >= 0) {
|
||||
logDebug("RayNeo iO: control $ctrl")
|
||||
ioWriteControl(ctrl)
|
||||
}
|
||||
if (status) ioRequestStatus()
|
||||
if (brightness > 0) ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_BRIGHTNESS, RayNeoIoProtocol.launcherBrightness(brightness))
|
||||
if (wakeword >= 0) {
|
||||
logLine("RayNeo iO: wake word " + (if (wakeword == 1) "on" else "off"))
|
||||
ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_AI_VOICE_WAKEUP, RayNeoIoProtocol.launcherSetAiVoiceWakeup(wakeword == 1))
|
||||
}
|
||||
if (settings) ioWriteLauncher(RayNeoIoProtocol.LAUNCHER_GENERAL_SETTINGS, RayNeoIoProtocol.GENERAL_SETTINGS_REQUEST)
|
||||
if (abortVoice) {
|
||||
// As Glassly does: decline the stock session, then abort it, so its page goes.
|
||||
ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_RC, RayNeoIoProtocol.VOICE_RC_STOP))
|
||||
ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_ABORT, RayNeoIoProtocol.VOICE_RC_START))
|
||||
}
|
||||
if (micRc > 0) {
|
||||
logLine("RayNeo iO mic: " + (if (micRc == RayNeoIoProtocol.VOICE_RC_START) "open" else "close") + " the remote audio stream")
|
||||
ioWriteFrame(RayNeoIoProtocol.BUSINESS_VOICE, RayNeoIoProtocol.voiceRc(RayNeoIoProtocol.VOICE_STREAM_CONTROL, micRc))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The glasses' mic for a voice capture: the remote audio stream (16 kHz mono Opus packets to
|
||||
* [listener]), which needs IDENT_FLAG_MIC firmware. ioMaintain sends the open (and repeats it
|
||||
* while no packet comes); a voice session or a call can take the recorder meanwhile.
|
||||
*/
|
||||
internal fun GlassesSessionCore.ioStartAudioCapture(listener: FaceclawAudioPacketListener): Boolean {
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady || shutdownRequested) {
|
||||
logLine("skip RayNeo iO mic: session not ready")
|
||||
return false
|
||||
}
|
||||
if (io.ident?.micStream != true) {
|
||||
logLine("skip RayNeo iO mic: the firmware has no remote audio stream (IDENT flag 0x20)")
|
||||
return false
|
||||
}
|
||||
audioPacketListener = listener
|
||||
audioCaptureActive = true
|
||||
io.micWanted = true
|
||||
io.micStreamOpen = false
|
||||
io.micPackets = 0
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
return true
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.ioStopAudioCapture() {
|
||||
monitor.withLock {
|
||||
audioPacketListener = null
|
||||
audioCaptureActive = false
|
||||
io.micWanted = false
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/** Whether "Hey RayNeo" should reach Faceclaw (the launcher's wake-word switch). */
|
||||
internal fun GlassesSessionCore.ioSetVoiceWakeEnabled(enabled: Boolean) {
|
||||
monitor.withLock { io.voiceWakeWanted = enabled }
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/** Stop drawing and close the page, keeping the BLE link (the EvenHub suspend analogue). */
|
||||
internal fun GlassesSessionCore.ioSuspend(): Boolean {
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady) return false
|
||||
shutdownRequested = true
|
||||
io.pageWanted = false
|
||||
io.pageRequestSentAtMs = 0
|
||||
clearAllMessagesLocked("RayNeo iO suspend")
|
||||
displayedFingerprint = ""
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
return true
|
||||
}
|
||||
|
||||
/** Reopen the page; the drive loop resends the desired frame once it is open. */
|
||||
internal fun GlassesSessionCore.ioResume(): Boolean {
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady) return false
|
||||
shutdownRequested = false
|
||||
io.pageWanted = true
|
||||
io.pageRequestSentAtMs = 0
|
||||
fixedLayoutCreated = false
|
||||
clearAllMessagesLocked("RayNeo iO resume")
|
||||
displayedFingerprint = ""
|
||||
imageRetryAfterMs = 0
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
return true
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Inbound (BLE callback thread)
|
||||
|
||||
internal fun GlassesSessionCore.ioOnNotification(address: String, characteristicUuid: String, data: ByteArray) {
|
||||
if (RayNeoIoBle.NOTIFY_CHAR_UUID != characteristicUuid || !address.equals(rightAddress, ignoreCase = true)) return
|
||||
val frames = monitor.withLock { io.reassembler.ingest(data) }
|
||||
for (frame in frames) {
|
||||
when (frame.business) {
|
||||
RayNeoIoProtocol.BUSINESS_HANDSHAKE -> monitor.withLock {
|
||||
val version = RayNeoIoProtocol.parseVersionReply(frame.payload)
|
||||
if (version != null) io.versionReply = version
|
||||
else if (RayNeoIoProtocol.isGuestAuthReply(frame.payload)) io.authReply = frame.payload
|
||||
monitor.signalAll()
|
||||
}
|
||||
RayNeoIoProtocol.BUSINESS_LAUNCHER -> ioHandleLauncher(frame.payload)
|
||||
RayNeoIoProtocol.BUSINESS_VOICE -> ioHandleVoice(frame.payload)
|
||||
RayNeoIoProtocol.BUSINESS_OTA -> {
|
||||
val envelope = RayNeoIoProtocol.decodeEnvelope(frame.payload) ?: continue
|
||||
if (envelope.type != RayNeoIoProtocol.OTA_VERSION) continue
|
||||
val os = try { Json.parse(envelope.json).asObject()?.get("OsVersion").asString() } catch (_: Exception) { null }
|
||||
if (os != null) monitor.withLock {
|
||||
io.osVersion = os
|
||||
monitor.signalAll()
|
||||
}
|
||||
}
|
||||
RayNeoIoProtocol.BUSINESS_CFW -> ioHandleCfw(address, frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun GlassesSessionCore.ioHandleLauncher(payload: ByteArray) {
|
||||
val envelope = RayNeoIoProtocol.decodeEnvelope(payload) ?: return
|
||||
val screen = RayNeoIoProtocol.screenStatusFromLauncherJson(envelope.json)
|
||||
if (screen >= 0) {
|
||||
val changed = monitor.withLock {
|
||||
val was = io.screenStatus
|
||||
io.screenStatus = screen
|
||||
// Woken by the wearer (or the timeout ran out): re-assert promptly if Faceclaw is showing.
|
||||
if (screen == 0) io.lastBrightnessAtMs = 0
|
||||
was != screen
|
||||
}
|
||||
if (changed) logLine("RayNeo iO screen " + (if (screen == 1) "on" else "off"))
|
||||
return
|
||||
}
|
||||
val battery = RayNeoIoProtocol.batteryFromLauncherJson(envelope.json)
|
||||
if (battery < 0) {
|
||||
// Replies to our commands and settings (the wake word's among them): logged for now.
|
||||
logLine("RayNeo iO launcher type " + envelope.type + ": " + envelope.json.take(400))
|
||||
return
|
||||
}
|
||||
val charging = if (envelope.json.contains("\"isCharging\":true")) 1 else if (envelope.json.contains("\"isCharging\":false")) 0 else -1
|
||||
var changed = false
|
||||
monitor.withLock {
|
||||
lastIncomingAtMs = now()
|
||||
if (battery != headsetBattery || (charging >= 0 && charging != headsetCharging)) {
|
||||
headsetBattery = battery
|
||||
if (charging >= 0) headsetCharging = charging
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if (changed) emitBatteryState(headsetBattery, headsetCharging)
|
||||
}
|
||||
|
||||
/** The voice channel (business 13): mic packets, and the stock assistant's wake word. */
|
||||
private fun GlassesSessionCore.ioHandleVoice(payload: ByteArray) {
|
||||
val envelope = RayNeoIoProtocol.decodeEnvelope(payload) ?: return
|
||||
when (envelope.type) {
|
||||
RayNeoIoProtocol.VOICE_STREAM_AUDIO, RayNeoIoProtocol.VOICE_AUDIO -> {
|
||||
val opus = envelope.binary ?: return
|
||||
val arrivalMs = now()
|
||||
var first = false
|
||||
val listener = monitor.withLock {
|
||||
lastIncomingAtMs = arrivalMs
|
||||
io.micPackets++
|
||||
first = io.micPackets == 1L
|
||||
io.lastMicPacketAtMs = arrivalMs
|
||||
if (audioCaptureActive) audioPacketListener else null
|
||||
}
|
||||
if (first) logLine("RayNeo iO mic: first audio packet (type " + envelope.type + ", " + opus.size + " bytes)")
|
||||
if (listener == null) return
|
||||
try {
|
||||
listener.onAudioPacket(opus, "R", arrivalMs)
|
||||
} catch (t: Throwable) {
|
||||
logLine("RayNeo iO mic packet listener failed: " + safeMessage(t))
|
||||
}
|
||||
}
|
||||
RayNeoIoProtocol.VOICE_WAKE -> {
|
||||
// The stock assistant heard "Hey RayNeo" and opened its session. Faceclaw's own
|
||||
// assistant answers instead: end the stock one, and report the wake word as the
|
||||
// G2's (sid 0x07 EVEN_AI_WAKE_UP), which the dashboard routes to its wake action.
|
||||
var repeat = false
|
||||
val forward = monitor.withLock {
|
||||
val now = now()
|
||||
io.voiceAbortPending = true
|
||||
lastIncomingAtMs = now
|
||||
lastConnectionOrInputAtMs = now
|
||||
repeat = io.lastVoiceWakeAtMs != 0L && now - io.lastVoiceWakeAtMs < IoTimings.VOICE_WAKE_DEDUP_MS
|
||||
val wanted = io.voiceWakeWanted == true && sessionReady
|
||||
if (wanted && !repeat) {
|
||||
io.lastVoiceWakeAtMs = now
|
||||
ioReclaimPageOnInputLocked()
|
||||
}
|
||||
wanted && !repeat
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
logLine("RayNeo iO: voice wake " + envelope.json + when {
|
||||
forward -> ""
|
||||
repeat -> " (same phrase as the last wake)"
|
||||
else -> " (not forwarded: wake word off)"
|
||||
})
|
||||
if (!forward) return
|
||||
val event = G2Event("even-ai", "", BleProtocol.EVEN_AI_STATUS_WAKE_UP, 0, 0)
|
||||
val frameId = frameTimings.startFrame("input:even-ai wake")
|
||||
frameTimings.log(frameId, "wake word from the RayNeo iO's voice channel")
|
||||
emitRingEvent(event, frameId)
|
||||
}
|
||||
RayNeoIoProtocol.VOICE_EXITED -> {
|
||||
// A voice session that held the recorder ended: ask for the stream again.
|
||||
val reopen = monitor.withLock {
|
||||
if (io.micWanted) io.micStreamOpen = false
|
||||
io.micWanted
|
||||
}
|
||||
if (reopen) interruptibleSleep.interrupt()
|
||||
logLine("RayNeo iO: voice session exited " + envelope.json + (if (reopen) "; reopening the mic stream" else ""))
|
||||
}
|
||||
RayNeoIoProtocol.VOICE_VAD -> logDebug("RayNeo iO: voice VAD " + envelope.json)
|
||||
else -> logLine("RayNeo iO: voice message type " + envelope.type + " " + envelope.json.take(160))
|
||||
}
|
||||
}
|
||||
|
||||
private fun GlassesSessionCore.ioHandleCfw(address: String, frame: RayNeoIoProtocol.Frame) {
|
||||
val payload = frame.payload
|
||||
if (payload.isEmpty()) return
|
||||
when (payload[0].toInt() and 255) {
|
||||
RayNeoIoProtocol.KIND_ACK, RayNeoIoProtocol.KIND_NACK -> {
|
||||
val ack = RayNeoIoProtocol.parseCfwAck(payload) ?: return
|
||||
monitor.withLock {
|
||||
acceptCfwAcksLocked(address, arrayOf(ack), frame.messageNumber and 255, payload)
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
RayNeoIoProtocol.KIND_IDENT -> monitor.withLock {
|
||||
io.ident = RayNeoIoProtocol.parseIdent(payload)
|
||||
monitor.signalAll()
|
||||
}
|
||||
RayNeoIoProtocol.KIND_ALS -> {
|
||||
val body = RayNeoIoProtocol.parseAlsReport(payload) ?: return
|
||||
monitor.withLock {
|
||||
lastIncomingAtMs = now()
|
||||
brightnessPolicy.sample(body, lastIncomingAtMs)
|
||||
}
|
||||
emitAmbientLight(body)
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
RayNeoIoProtocol.KIND_PAGE -> ioHandlePage(payload)
|
||||
RayNeoIoProtocol.KIND_GESTURE -> ioHandleGesture(RayNeoIoProtocol.parseGesture(payload) ?: return)
|
||||
else -> logDebug("RayNeo iO: CFW reply kind " + (payload[0].toInt() and 255) + " " + hex(payload.copyOf(minOf(payload.size, 24))))
|
||||
}
|
||||
}
|
||||
|
||||
private fun GlassesSessionCore.ioHandlePage(payload: ByteArray) {
|
||||
val state = RayNeoIoProtocol.parsePageState(payload)
|
||||
val reason = if (payload.size >= 3) payload[2].toInt() and 255 else 0
|
||||
var lost = false
|
||||
monitor.withLock {
|
||||
io.pageState = state
|
||||
io.pageReason = reason
|
||||
io.pageReplies++
|
||||
lastIncomingAtMs = now()
|
||||
if (state == 1) {
|
||||
if (io.pageWanted && !fixedLayoutCreated && sessionReady) {
|
||||
// (Re)opened: the drive loop resends the desired frame from scratch.
|
||||
fixedLayoutCreated = true
|
||||
displayedFingerprint = ""
|
||||
imageRetryAfterMs = 0
|
||||
io.lastKeepaliveAtMs = now()
|
||||
io.lastBrightnessAtMs = 0
|
||||
ioResetBrightnessLocked()
|
||||
}
|
||||
} else if (state == 0 && fixedLayoutCreated) {
|
||||
// The page closed under us (lease, the wearer's long press): the firmware freed
|
||||
// this session's resources, so start over once it reopens.
|
||||
lost = true
|
||||
fixedLayoutCreated = false
|
||||
clearAllMessagesLocked("RayNeo iO page closed (reason $reason)")
|
||||
displayedFingerprint = ""
|
||||
if (reason == IoTimings.PAGE_REASON_USER) io.pageWanted = false
|
||||
io.pageRequestSentAtMs = now()
|
||||
}
|
||||
monitor.signalAll()
|
||||
}
|
||||
if (lost) logLine("RayNeo iO page closed by the glasses (reason $reason)")
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/**
|
||||
* The wearer gave input (a temple touch, or the R1's) after closing our page:
|
||||
* take it back. Caller holds `monitor`.
|
||||
*/
|
||||
internal fun GlassesSessionCore.ioReclaimPageOnInputLocked() {
|
||||
if (io.pageWanted || !sessionReady) return
|
||||
io.pageWanted = true
|
||||
shutdownRequested = false
|
||||
io.pageRequestSentAtMs = 0
|
||||
}
|
||||
|
||||
private fun GlassesSessionCore.ioHandleGesture(gesture: RayNeoIoProtocol.Gesture) {
|
||||
val event: G2Event? = when (gesture.eventType) {
|
||||
// Same numbering as the G2's OsEventTypeList; head up wakes like the G2 CFW's forward.
|
||||
BleProtocol.EVENT_HEAD_UP -> G2Event("display-wake", "", BleProtocol.EVENT_HEAD_UP, BleProtocol.EVENT_SOURCE_GLASSES_R, 0)
|
||||
BleProtocol.EVENT_HEAD_DOWN -> null
|
||||
else -> G2Event("sys-event", "", gesture.eventType, BleProtocol.EVENT_SOURCE_GLASSES_R, 0)
|
||||
}
|
||||
monitor.withLock {
|
||||
lastIncomingAtMs = now()
|
||||
if (event != null) lastConnectionOrInputAtMs = lastIncomingAtMs
|
||||
if (event != null && "sys-event" == event.kind) ioReclaimPageOnInputLocked()
|
||||
}
|
||||
logLine("RayNeo iO gesture type=" + gesture.eventType + " source=" + gesture.source + (if (event == null) " (ignored)" else ""))
|
||||
if (event == null) return
|
||||
val frameId = frameTimings.startFrame("input:" + event.kind + " type=" + event.eventType + " src=" + event.eventSource)
|
||||
frameTimings.log(frameId, "input event decoded from RayNeo iO notification")
|
||||
emitRingEvent(event, frameId)
|
||||
}
|
||||
+10
@@ -971,6 +971,13 @@ internal fun macWireBytes(address: String): ByteArray? {
|
||||
internal fun GlassesSessionCore.queueGlassesRingLinkCommandLocked(
|
||||
action: String, fallbackAddress: String, fallbackName: String,
|
||||
): Boolean {
|
||||
if (!isG2) {
|
||||
// A RayNeo iO or a G1 has no ring link of its own; the phone holds the ring directly.
|
||||
lastRingLinkCommand = "$action: not applicable ($glassesType)"
|
||||
lastRingLinkCommandAtMs = now()
|
||||
logLine("glasses ring $action skipped: $glassesType glasses have no ring link")
|
||||
return false
|
||||
}
|
||||
val reported = glassesRingMac
|
||||
val mac = reported ?: macWireBytes(fallbackAddress)
|
||||
if (mac == null) {
|
||||
@@ -1091,6 +1098,9 @@ internal fun GlassesSessionCore.queueRingConfigCommandLocked(
|
||||
frame = RingProtocol.buildPairAuth(seq)
|
||||
}
|
||||
"targets-glasses" -> {
|
||||
// The iO never connects to the ring: naming it would make the ring drop
|
||||
// the phone's glasses-role link (the advStart check, ~20 s later).
|
||||
if (!isG2) return "not sent: these glasses don't hold the ring"
|
||||
val right = macWireBytes(rightAddress)
|
||||
val left = macWireBytes(leftAddress)
|
||||
if (right == null || left == null) return "not sent: glasses addresses unparseable"
|
||||
|
||||
+48
-11
@@ -44,6 +44,8 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
benchmarkReadyAtMs = now() + 1_000
|
||||
}
|
||||
}
|
||||
if (isIo) ioMaintain()
|
||||
if (isG1) g1Maintain()
|
||||
var messageToWrite: OutboundMessage? = null
|
||||
var messageToPrewrite: OutboundMessage? = null
|
||||
var heartbeatToWrite: OutboundMessage? = null
|
||||
@@ -74,7 +76,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
clearPendingMessagesLocked("after CFW cleanup")
|
||||
return 250
|
||||
}
|
||||
if (sessionReady
|
||||
if (sessionReady && isG2
|
||||
&& now - lastFaceclawFramebufferLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS
|
||||
&& !hasPendingOrInflightKindLocked("framebuffer-lease-control")) {
|
||||
enqueueFaceclawFramebufferControlLocked(
|
||||
@@ -83,7 +85,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
)
|
||||
}
|
||||
if (faceclawWakeLeaseEnabled
|
||||
&& sessionReady
|
||||
&& sessionReady && isG2
|
||||
&& now - lastFaceclawWakeLeaseQueuedAtMs >= FACECLAW_WAKE_LEASE_RENEW_MS
|
||||
&& !hasPendingOrInflightKindLocked("wake-lease-control")) {
|
||||
enqueueFaceclawWakeControlLocked(
|
||||
@@ -158,6 +160,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
}
|
||||
} else {
|
||||
if (messageToPrewrite == null
|
||||
&& isG2 // the iO's page lease is ioMaintain's; a G1 has none
|
||||
&& !shutdownRequested
|
||||
&& !fixedLayoutCreated
|
||||
&& pendingMessages.isEmpty()
|
||||
@@ -168,14 +171,14 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
// Prewrite outside the lock; the logical message remains pending until
|
||||
// its final BLE frame is sent after the current protocol ACK.
|
||||
}
|
||||
if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
|
||||
if (messageToPrewrite == null && isG2 && !shutdownRequested && fixedLayoutCreated
|
||||
&& (if (firmwareDebugFlagsEnabled) 2 else 1) != firmwareDebugFlagsLastSent
|
||||
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()) {
|
||||
logLine("enqueueing firmware debug flags " + (if (firmwareDebugFlagsEnabled) "show" else "hide"))
|
||||
enqueueFirmwareDebugFlagsLocked()
|
||||
}
|
||||
|
||||
if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
|
||||
if (messageToPrewrite == null && isG2 && !shutdownRequested && fixedLayoutCreated
|
||||
&& caseInfoReadPending
|
||||
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()) {
|
||||
caseInfoReadPending = false
|
||||
@@ -183,7 +186,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
pendingMessages.addLast(messageBuilder.caseInfoQuery())
|
||||
}
|
||||
|
||||
if (messageToPrewrite == null && !shutdownRequested && fixedLayoutCreated
|
||||
if (messageToPrewrite == null && isG2 && !shutdownRequested && fixedLayoutCreated
|
||||
&& (if (compassOwners.isEmpty()) 0 else 1) != compassControlLastSent
|
||||
&& pendingMessages.isEmpty() && inFlightMessages.isEmpty()) {
|
||||
val wanted = compassOwners.isNotEmpty()
|
||||
@@ -194,7 +197,9 @@ internal fun GlassesSessionCore.driveSession(): Long {
|
||||
if (benchmarkActive) {
|
||||
maintainBenchmarkLocked(now)
|
||||
}
|
||||
if (!benchmarkActive) maintainBrightnessLocked(now)
|
||||
// A G1's brightness follows the policy's target; its light sensor isn't wired up yet.
|
||||
if (!benchmarkActive && isG1) enqueueBrightnessLocked()
|
||||
else if (!benchmarkActive && (!isIo || ioFirmwareOwnsBrightness)) maintainBrightnessLocked(now)
|
||||
|
||||
// Up to WINDOW_SIZE messages may be in flight at once (full
|
||||
// pipelining); a slot frees when an ack arrives. An active
|
||||
@@ -337,6 +342,7 @@ internal enum class HeartbeatStep {
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.heartbeatStepLocked(imageWaiting: Boolean): HeartbeatStep {
|
||||
if (!isG2) return HeartbeatStep.NONE
|
||||
val now = now()
|
||||
val heartbeatEligible = !shutdownRequested && fixedLayoutCreated
|
||||
val heartbeatPending = heartbeatEligible && hasPendingOrInflightKindLocked("heartbeat")
|
||||
@@ -387,6 +393,21 @@ internal fun GlassesSessionCore.createHeartbeatMessage(): OutboundMessage {
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean {
|
||||
if (isG1 && !g1SupportsMessage(message)) {
|
||||
// G2-only traffic a shared feature queued anyway: it would only wait for an ACK that never comes.
|
||||
ioDropUnsupported(message)
|
||||
return true
|
||||
}
|
||||
if (isG1 && g1ArmIsAway(message)) {
|
||||
ioDropUnsupported(message)
|
||||
return true
|
||||
}
|
||||
if (isIo && !ioSupportsMessage(message)) {
|
||||
// G2-only traffic (EvenHub/settings frames, the G2 hardware CFW modes) that a feature
|
||||
// queued anyway: the iO would NACK it into the replay limit, so it never goes out.
|
||||
ioDropUnsupported(message)
|
||||
return true
|
||||
}
|
||||
val now = now()
|
||||
message.writeStartedAtMs = now
|
||||
message.sentAtMs = now
|
||||
@@ -416,6 +437,10 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean
|
||||
frames = listOf(prewrittenFrames[prewrittenFrames.size - 1])
|
||||
prewrittenMessage = null
|
||||
prewrittenFrames = emptyList()
|
||||
} else if (isIo) {
|
||||
frames = ioCfwWrites(message, link.negotiatedMtu(writeAddress))
|
||||
} else if (isG1) {
|
||||
frames = g1CfwWrites(message, link.negotiatedMtu(writeAddress))
|
||||
} else if (message.sid == CfwTransport.SID) {
|
||||
if (message.cfwRetries > 0) cfwTransports[if (message.isLeftArmMessage) 0 else 1].reset()
|
||||
frames = cfwTransports[if (message.isLeftArmMessage) 0 else 1].encode(
|
||||
@@ -431,9 +456,9 @@ internal fun GlassesSessionCore.writeMessage(message: OutboundMessage): Boolean
|
||||
}
|
||||
val result = link.writeFrames(
|
||||
writeAddress,
|
||||
BleProtocol.WRITE_CHAR_UUID,
|
||||
if (isIo) RayNeoIoBle.WRITE_CHAR_UUID else if (isG1) G1Ble.FC_CHAR_UUID else BleProtocol.WRITE_CHAR_UUID,
|
||||
frames,
|
||||
ConnectionOptions.WRITE_MODE,
|
||||
if (isIo || isG1) GattWriteMode.WITHOUT_RESPONSE else ConnectionOptions.WRITE_MODE,
|
||||
ConnectionOptions.WRITE_TIMEOUT_MS
|
||||
)
|
||||
|
||||
@@ -836,6 +861,10 @@ internal fun GlassesSessionCore.enqueueDesiredImageLocked() {
|
||||
frameTimings.spanEnd(frameId, "pack-4bpp")
|
||||
// Visibility belongs to this immutable composite, not the latest UI request.
|
||||
enqueueBrightnessLocked(!fingerprint.startsWith("blanked:"))
|
||||
if (isG1 && packedFrame.size > 0) {
|
||||
g1EnqueueFrameLocked(packedFrame, width, height, scene, fingerprint, paintMs, frameId)
|
||||
return
|
||||
}
|
||||
if (customFirmwareDetected && packedFrame.size > 0) {
|
||||
val rendered = try {
|
||||
scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId,
|
||||
@@ -981,10 +1010,13 @@ internal fun GlassesSessionCore.enqueueCustomImageLocked(
|
||||
fingerprint: String,
|
||||
updateId: Int,
|
||||
messageNumber: Int,
|
||||
messageCount: Int
|
||||
messageCount: Int,
|
||||
leftArm: Boolean = connectionOptions.sendImagesToLeft,
|
||||
requiredLenses: Int = CfwTransport.BOTH,
|
||||
) {
|
||||
val message = messageBuilder.customMessage("image", payload,
|
||||
"image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, connectionOptions.sendImagesToLeft)
|
||||
"image " + plan.tile.name + "#" + messageNumber, plan.tileIndex, leftArm)
|
||||
message.cfwRequiredLenses = requiredLenses
|
||||
message.setImageUpdatePosition(updateId, messageNumber, messageCount)
|
||||
message.onAck = MessageCallback {
|
||||
imageRetryAfterMs = 0
|
||||
@@ -1037,7 +1069,8 @@ internal fun GlassesSessionCore.createAudioControlMessageLocked(enable: Boolean)
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.shouldPollBatteryLocked(now: Long): Boolean {
|
||||
return !shutdownRequested
|
||||
return isG2 // the others' status polls are their own maintain's
|
||||
&& !shutdownRequested
|
||||
&& sessionReady
|
||||
&& pendingMessages.isEmpty()
|
||||
&& inFlightMessages.isEmpty()
|
||||
@@ -1424,6 +1457,7 @@ internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) {
|
||||
// drop the pipelined delta base so the next image is a full keyframe.
|
||||
lastEnqueuedPacked = ByteArray(0)
|
||||
lastEnqueuedFingerprint = ""
|
||||
g1ForgetFramesLocked()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1431,12 +1465,14 @@ internal fun GlassesSessionCore.clearAllMessagesLocked(reason: String) {
|
||||
brightnessSentVisible = null
|
||||
brightnessSentLevel = -1
|
||||
brightnessAlsStartedAt = -2000
|
||||
io.alsIntervalSent = -1
|
||||
clearPendingMessagesLocked(reason)
|
||||
clearInFlightMessagesLocked(reason)
|
||||
// The image pipeline is gone: drop the pipelined delta base so the next
|
||||
// image is a full keyframe rather than a delta onto a stale base.
|
||||
lastEnqueuedPacked = ByteArray(0)
|
||||
lastEnqueuedFingerprint = ""
|
||||
g1ForgetFramesLocked()
|
||||
// Queued resource commands (if any) were dropped with the rest, and the
|
||||
// session churn behind a full clear may have freed the on-glasses
|
||||
// cache; forget it phone-side so glyphs re-upload lazily.
|
||||
@@ -1452,6 +1488,7 @@ internal fun GlassesSessionCore.clearAllMessagesPreservingWakeLeaseLocked(reason
|
||||
brightnessSentVisible = null
|
||||
brightnessSentLevel = -1
|
||||
brightnessAlsStartedAt = -2000
|
||||
io.alsIntervalSent = -1
|
||||
val pendingIterator = pendingMessages.iterator()
|
||||
while (pendingIterator.hasNext()) {
|
||||
val message = pendingIterator.next()
|
||||
|
||||
+12
@@ -68,6 +68,18 @@ interface SessionLink {
|
||||
/** Whether the OS still holds a pairing bond for the address (iOS: always true). */
|
||||
fun isBonded(address: String): Boolean
|
||||
|
||||
/** Ask the OS to bond with a connected device, without waiting. A no-op where it pairs on demand only (CoreBluetooth). */
|
||||
fun requestBond(address: String) {}
|
||||
|
||||
/** Remove this phone's half of a pairing; false where the platform can't or refused. */
|
||||
fun removeBond(address: String): Boolean = false
|
||||
|
||||
/** After discovery: whether the service table has this characteristic. Enabling notifications on one it lacks may throw. */
|
||||
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean = true
|
||||
|
||||
/** Read the service table again, past whatever the OS cached for this address. */
|
||||
fun refreshServices(address: String, timeoutMs: Int): Boolean = discoverServices(address, timeoutMs)
|
||||
|
||||
/** Bandwidth-benchmark link tuning; 0: current link, 1: HIGH priority, 2: 2M PHY, 3: both. */
|
||||
fun prepareBenchmarkLink(address: String, mode: Int)
|
||||
|
||||
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmStatic
|
||||
import kotlin.math.max
|
||||
import kotlin.math.pow
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
* Turns a color glyph (an emoji from a color font the text stack fell back to) into the
|
||||
* lit-on-black coverage the glyph pipeline carries, so it keeps its internal detail instead of
|
||||
* showing as its alpha silhouette (a face as a plain disc, ✅ as a plain square). Platform
|
||||
* rasterizers draw the glyph in white and pass unpremultiplied ARGB; this object owns the
|
||||
* pixel math so Android and iOS agree.
|
||||
*
|
||||
* Shading is by OKLab lightness relative to the glyph's body, its median lightness:
|
||||
* - the body lights fully, so a one-color emoji (❤️, ⭐, 🖤) reads as its shape at full
|
||||
* brightness whatever its color, the way the alpha did, rather than dim for being dark;
|
||||
* - darker detail (eyes, mouths, the digits on 📅) falls toward [DARKEST_LIT], never off, so
|
||||
* the silhouette stays whole;
|
||||
* - when a fair share of the glyph is clearly lighter than the body (the check on ✅, the
|
||||
* circle on 🎱, the eyes on 💩) the body dims toward [DIMMEST_BODY] to leave that detail
|
||||
* room above it. Small highlights (❤️'s glint) don't count.
|
||||
* Lit values go through the notification icons' tone curve ([NotificationIconArt.GAMMA]),
|
||||
* then scale by the pixel's alpha, which stays the glyph's coverage.
|
||||
*/
|
||||
object ColorGlyphArt {
|
||||
/** A pixel this opaque that is not white means the glyph was not drawn in the paint's white. */
|
||||
private const val MIN_COLOR_ALPHA = 32
|
||||
|
||||
/** How far below 255 a channel may sit and still count as white (unpremultiply rounding). */
|
||||
private const val WHITE_TOLERANCE = 24
|
||||
|
||||
/** Lit level of the glyph's darkest detail, before the tone curve. */
|
||||
private const val DARKEST_LIT = 0.3
|
||||
|
||||
/** Lit level the body dims to when the glyph has plenty of detail lighter than it. */
|
||||
private const val DIMMEST_BODY = 0.6
|
||||
|
||||
/** Lightness quantile that must be clearly above the body for the body to dim: about the lightest 10%. */
|
||||
private const val LIGHT_DETAIL_QUANTILE = 0.9
|
||||
|
||||
/** OKLab lightness above the body where light detail starts to dim the body, and the span to fully dimmed. */
|
||||
private const val LIGHT_DETAIL_START = 0.05
|
||||
private const val LIGHT_DETAIL_SPAN = 0.2
|
||||
|
||||
/**
|
||||
* The smallest lightness spans the ramps above and below the body stretch over, so faint
|
||||
* shading on a nearly one-color glyph stays faint instead of stretching to the full range.
|
||||
*/
|
||||
private const val MIN_LIGHT_SPAN = 0.15
|
||||
private const val MIN_DARK_SPAN = 0.3
|
||||
|
||||
/**
|
||||
* Whether a glyph drawn in white came out in other colors, i.e. came from a color font.
|
||||
* Ordinary glyphs are white wherever they have coverage.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun isColor(argb: IntArray): Boolean {
|
||||
val threshold = 255 - WHITE_TOLERANCE
|
||||
for (p in argb) {
|
||||
if (alpha(p) < MIN_COLOR_ALPHA) continue
|
||||
if (((p shr 16) and 0xff) < threshold || ((p shr 8) and 0xff) < threshold || (p and 0xff) < threshold) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage (one byte per pixel) of a glyph drawn in white as unpremultiplied ARGB: its alpha,
|
||||
* or for a color glyph its [shade].
|
||||
*/
|
||||
@JvmStatic
|
||||
fun coverage(argb: IntArray): ByteArray =
|
||||
if (isColor(argb)) shade(argb) else ByteArray(argb.size) { alpha(argb[it]).toByte() }
|
||||
|
||||
/** Lit-on-black shade (one byte per pixel) of a color glyph as unpremultiplied ARGB. */
|
||||
@JvmStatic
|
||||
fun shade(argb: IntArray): ByteArray {
|
||||
var peak = 0
|
||||
for (p in argb) peak = max(peak, alpha(p))
|
||||
val out = ByteArray(argb.size)
|
||||
if (peak == 0) return out
|
||||
|
||||
// Statistics over the solid part, so antialiased edges don't skew the body.
|
||||
val cutoff = max(1, (peak + 1) / 2)
|
||||
var solid = 0
|
||||
for (p in argb) if (alpha(p) >= cutoff) solid += 1
|
||||
val sorted = DoubleArray(solid)
|
||||
var next = 0
|
||||
for (p in argb) if (alpha(p) >= cutoff) sorted[next++] = Oklab.lightness(p)
|
||||
sorted.sort()
|
||||
val body = quantile(sorted, 0.5)
|
||||
val darkest = quantile(sorted, 0.05)
|
||||
val lightest = quantile(sorted, 0.95)
|
||||
val lightDetail = quantile(sorted, LIGHT_DETAIL_QUANTILE) - body
|
||||
val dimming = ((lightDetail - LIGHT_DETAIL_START) / LIGHT_DETAIL_SPAN).coerceIn(0.0, 1.0)
|
||||
val bodyLit = 1.0 - (1.0 - DIMMEST_BODY) * dimming
|
||||
val lightSpan = max(lightest - body, MIN_LIGHT_SPAN)
|
||||
val darkSpan = max(body - darkest, MIN_DARK_SPAN)
|
||||
|
||||
for (i in argb.indices) {
|
||||
val p = argb[i]
|
||||
val a = alpha(p)
|
||||
if (a == 0) continue
|
||||
val l = Oklab.lightness(p)
|
||||
val lit = if (l >= body) {
|
||||
bodyLit + (1.0 - bodyLit) * ((l - body) / lightSpan).coerceAtMost(1.0)
|
||||
} else {
|
||||
bodyLit - (bodyLit - DARKEST_LIT) * ((body - l) / darkSpan).coerceAtMost(1.0)
|
||||
}
|
||||
out[i] = (a * lit.pow(NotificationIconArt.GAMMA)).roundToInt().coerceIn(0, 255).toByte()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private fun quantile(sorted: DoubleArray, q: Double): Double = sorted[((sorted.size - 1) * q).roundToInt()]
|
||||
|
||||
private fun alpha(p: Int): Int = (p ushr 24) and 0xff
|
||||
}
|
||||
+1
-20
@@ -2,7 +2,6 @@ package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmStatic
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.cbrt
|
||||
import kotlin.math.max
|
||||
import kotlin.math.min
|
||||
import kotlin.math.pow
|
||||
@@ -72,12 +71,6 @@ object NotificationIconArt {
|
||||
|
||||
class Art(val kind: Kind, val gray: ByteArray)
|
||||
|
||||
/** sRGB byte value to linear light. */
|
||||
private val SRGB_TO_LINEAR = DoubleArray(256) { value ->
|
||||
val c = value / 255.0
|
||||
if (c <= 0.04045) c / 12.92 else ((c + 0.055) / 1.055).pow(2.4)
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify and convert an icon rendered at (outputSize * ANALYSIS_SCALE) square, row-major
|
||||
* ARGB (not premultiplied).
|
||||
@@ -232,7 +225,7 @@ object NotificationIconArt {
|
||||
private fun readTile(argb: IntArray, mask: BooleanArray, size: Int, maxAlpha: Int, outputSize: Int): Art {
|
||||
val lab = DoubleArray(argb.size * 3)
|
||||
for (i in argb.indices) {
|
||||
if (mask[i]) toOklab(argb[i], lab, i * 3)
|
||||
if (mask[i]) Oklab.toOklab(argb[i], lab, i * 3)
|
||||
}
|
||||
|
||||
// The background is the color of the tile's rim, about one output pixel deep.
|
||||
@@ -347,18 +340,6 @@ object NotificationIconArt {
|
||||
return gray
|
||||
}
|
||||
|
||||
private fun toOklab(pixel: Int, out: DoubleArray, offset: Int) {
|
||||
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
|
||||
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
|
||||
val b = SRGB_TO_LINEAR[pixel and 0xff]
|
||||
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
|
||||
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
|
||||
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
|
||||
out[offset] = 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
|
||||
out[offset + 1] = 1.9779984951 * l - 2.4285922050 * m + 0.4505937099 * s
|
||||
out[offset + 2] = 0.0259040371 * l + 0.7827717662 * m - 0.8086757660 * s
|
||||
}
|
||||
|
||||
private fun distance(lab: DoubleArray, index: Int, background: DoubleArray): Double {
|
||||
val dl = lab[index * 3] - background[0]
|
||||
val da = lab[index * 3 + 1] - background[1]
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.math.cbrt
|
||||
import kotlin.math.pow
|
||||
|
||||
/**
|
||||
* OKLab, a perceptual color space: L is lightness (0 = black, 1 = white) and equal distances
|
||||
* look about equally different. The color analysis of icons and color glyphs works in it.
|
||||
*/
|
||||
internal object Oklab {
|
||||
/** sRGB byte value to linear light. */
|
||||
private val SRGB_TO_LINEAR = DoubleArray(256) { value ->
|
||||
val c = value / 255.0
|
||||
if (c <= 0.04045) c / 12.92 else ((c + 0.055) / 1.055).pow(2.4)
|
||||
}
|
||||
|
||||
/** Writes the (L, a, b) of an ARGB pixel's color (its alpha ignored) at out[offset]. */
|
||||
fun toOklab(pixel: Int, out: DoubleArray, offset: Int) {
|
||||
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
|
||||
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
|
||||
val b = SRGB_TO_LINEAR[pixel and 0xff]
|
||||
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
|
||||
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
|
||||
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
|
||||
out[offset] = 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
|
||||
out[offset + 1] = 1.9779984951 * l - 2.4285922050 * m + 0.4505937099 * s
|
||||
out[offset + 2] = 0.0259040371 * l + 0.7827717662 * m - 0.8086757660 * s
|
||||
}
|
||||
|
||||
/** The OKLab lightness of an ARGB pixel's color (its alpha ignored). */
|
||||
fun lightness(pixel: Int): Double {
|
||||
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
|
||||
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
|
||||
val b = SRGB_TO_LINEAR[pixel and 0xff]
|
||||
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
|
||||
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
|
||||
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
|
||||
return 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,21 @@ class ShellScene(val layers: List<Layer>, val selections: List<RetainedDrawing>
|
||||
return output
|
||||
}
|
||||
|
||||
/** Whether the lenses differ: stereo depth somewhere in the scene. */
|
||||
val hasStereoDepth: Boolean get() = screenDepth != 0 || layers.any { it.depth != 0 } || allSelections.isNotEmpty()
|
||||
|
||||
/**
|
||||
* What one lens shows of [screen] (packed 4bpp) with this scene over it and every animation settled, for
|
||||
* glasses that have no display lists (a G1). [screen] itself when there is no scene.
|
||||
*/
|
||||
fun composed(screen: ByteArray, width: Int, height: Int, rightLens: Boolean): ByteArray {
|
||||
if (layers.isEmpty() && selections.isEmpty() && screenDepth == 0) return screen
|
||||
val output = ByteArray(screen.size)
|
||||
DisplayListRenderer(rendererResources(width, height), rightLens = rightLens).render(511,
|
||||
DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height), SETTLED_MS)
|
||||
return output
|
||||
}
|
||||
|
||||
/** Layer surfaces, then retained resources, by index, and the root list at 511. */
|
||||
private fun rendererResources(width: Int, height: Int): HashMap<Int, ByteArray> {
|
||||
val resources = HashMap<Int, ByteArray>()
|
||||
|
||||
+540
@@ -0,0 +1,540 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
|
||||
/**
|
||||
* RayNeo iO wire codecs (stock link plus the Faceclaw custom-firmware link). Pure functions
|
||||
* and small value types; no I/O, threads or platform APIs.
|
||||
*
|
||||
* The iO is one SoC driving both eyes over ONE BLE connection. Everything travels in AA55
|
||||
* frames on one write characteristic and one notify characteristic; the frame's "business"
|
||||
* byte selects the subsystem: 0x10 handshake (TLV), 9 OTA and 13..22 apps (protobuf-style
|
||||
* envelope + JSON), 0xF0 the custom-firmware link. The CFW link carries Faceclaw's G2 records
|
||||
* unchanged (see [CfwTransport.encodeRecord]) inside `[opt][stream][chunk]` packets, and the
|
||||
* firmware answers with the G2's 9-byte ACK body, one ACK per targeted lens.
|
||||
*
|
||||
* Protocol knowledge from the MIT-licensed Glassly SDK and CFW (tetramo-labs/glassly,
|
||||
* tetramo-labs/glassly-rayneo-cfw); the Faceclaw firmware fork documents its additions in
|
||||
* faceclaw-rayneo-cfw/docs/FACECLAW.md.
|
||||
*/
|
||||
object RayNeoIoBle {
|
||||
const val SERVICE_UUID = "0000b81d-0000-1000-8000-00805f9b34fb"
|
||||
/** Phone -> glasses, write without response. */
|
||||
const val WRITE_CHAR_UUID = "ea8b70d5-2bd3-49ab-9c31-9c38b2c3c4f9"
|
||||
/** Glasses -> phone, notifications. */
|
||||
const val NOTIFY_CHAR_UUID = "7db3e235-3608-41f3-a03c-955fcbd2ea4b"
|
||||
const val REQUESTED_MTU = 512
|
||||
/** Advertised manufacturer data company id; the payload is `[07][MAC x6]...` (not in pairing mode). */
|
||||
const val MANUFACTURER_ID = 0x5716
|
||||
|
||||
@JvmStatic fun matchesName(name: String?): Boolean = name != null && name.contains("rayneo", ignoreCase = true)
|
||||
|
||||
/** The glasses' MAC ("00:6A:5E:5F:33:C2") from manufacturer data, or null. */
|
||||
@JvmStatic
|
||||
fun macFromManufacturerData(data: ByteArray?): String? {
|
||||
if (data == null || data.size < 7 || data[0].toInt() != 7) return null
|
||||
return (1..6).joinToString(":") { i -> (data[i].toInt() and 255).toString(16).padStart(2, '0').uppercase() }
|
||||
}
|
||||
}
|
||||
|
||||
object RayNeoIoProtocol {
|
||||
const val BUSINESS_OTA = 9
|
||||
const val BUSINESS_VOICE = 13
|
||||
const val BUSINESS_LAUNCHER = 15
|
||||
const val BUSINESS_HANDSHAKE = 0x10
|
||||
const val BUSINESS_CFW = 0xf0
|
||||
|
||||
const val FRAME_OVERHEAD = 10
|
||||
const val MIN_FRAME_BYTES = 10
|
||||
|
||||
/** Program constant of the stock app's guest authentication (not a secret). */
|
||||
private val SALT = byteArrayOf(
|
||||
0x07, 0x12, 0xc8.toByte(), 0xad.toByte(), 0x27, 0x74, 0x41, 0x95.toByte(),
|
||||
0xde.toByte(), 0x5f, 0xa1.toByte(), 0xea.toByte(), 0x6d, 0x02, 0x5f, 0xb8.toByte(),
|
||||
)
|
||||
|
||||
// ---- CRC-16/XMODEM (poly 0x1021, init 0) -----------------------------------------------
|
||||
|
||||
@JvmStatic
|
||||
fun crc16Xmodem(bytes: ByteArray, from: Int = 0, to: Int = bytes.size): Int {
|
||||
var crc = 0
|
||||
for (i in from until to) {
|
||||
crc = crc xor ((bytes[i].toInt() and 255) shl 8)
|
||||
repeat(8) { crc = if (crc and 0x8000 != 0) ((crc shl 1) xor 0x1021) and 0xffff else (crc shl 1) and 0xffff }
|
||||
}
|
||||
return crc
|
||||
}
|
||||
|
||||
// ---- AA55 frames --------------------------------------------------------------------------
|
||||
|
||||
class Frame(
|
||||
@JvmField val messageNumber: Int,
|
||||
@JvmField val packetType: Int,
|
||||
@JvmField val business: Int,
|
||||
@JvmField val payload: ByteArray,
|
||||
)
|
||||
|
||||
/** `AA 55 [len u16 BE = total-6] [msg u16 BE] [packetType] [business] payload [crc16 BE of bytes 4..end-2]`. */
|
||||
@JvmStatic
|
||||
fun encodeFrame(messageNumber: Int, business: Int, payload: ByteArray, packetType: Int = 0): ByteArray {
|
||||
val frame = ByteArray(FRAME_OVERHEAD + payload.size)
|
||||
val length = frame.size - 6
|
||||
frame[0] = 0xaa.toByte()
|
||||
frame[1] = 0x55
|
||||
frame[2] = (length ushr 8).toByte()
|
||||
frame[3] = length.toByte()
|
||||
frame[4] = (messageNumber ushr 8).toByte()
|
||||
frame[5] = messageNumber.toByte()
|
||||
frame[6] = packetType.toByte()
|
||||
frame[7] = business.toByte()
|
||||
payload.copyInto(frame, 8)
|
||||
val crc = crc16Xmodem(frame, 4, frame.size - 2)
|
||||
frame[frame.size - 2] = (crc ushr 8).toByte()
|
||||
frame[frame.size - 1] = crc.toByte()
|
||||
return frame
|
||||
}
|
||||
|
||||
/** Split a frame into ATT writes of at most `mtu - 3` bytes. */
|
||||
@JvmStatic
|
||||
fun writes(frame: ByteArray, mtu: Int): List<ByteArray> {
|
||||
val chunk = (mtu - 3).coerceAtLeast(20)
|
||||
return (frame.indices step chunk).map { frame.copyOfRange(it, minOf(it + chunk, frame.size)) }
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun decodeFrame(bytes: ByteArray): Frame? {
|
||||
if (bytes.size < MIN_FRAME_BYTES || bytes[0] != 0xaa.toByte() || bytes[1] != 0x55.toByte()) return null
|
||||
val total = (((bytes[2].toInt() and 255) shl 8) or (bytes[3].toInt() and 255)) + 6
|
||||
if (total != bytes.size) return null
|
||||
val crc = ((bytes[total - 2].toInt() and 255) shl 8) or (bytes[total - 1].toInt() and 255)
|
||||
if (crc != crc16Xmodem(bytes, 4, total - 2)) return null
|
||||
val number = ((bytes[4].toInt() and 255) shl 8) or (bytes[5].toInt() and 255)
|
||||
return Frame(number, bytes[6].toInt() and 255, bytes[7].toInt() and 255, bytes.copyOfRange(8, total - 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Frames from notification chunks: a frame may span several notifications. Bytes before a
|
||||
* frame head are skipped; a head whose CRC fails is skipped past its magic.
|
||||
*/
|
||||
class FrameReassembler {
|
||||
private var buffer = ByteArray(0)
|
||||
|
||||
fun reset() {
|
||||
buffer = ByteArray(0)
|
||||
}
|
||||
|
||||
fun ingest(chunk: ByteArray): List<Frame> {
|
||||
buffer += chunk
|
||||
val out = ArrayList<Frame>()
|
||||
var offset = 0
|
||||
while (true) {
|
||||
while (offset + 1 < buffer.size && !(buffer[offset] == 0xaa.toByte() && buffer[offset + 1] == 0x55.toByte())) offset++
|
||||
if (buffer.size - offset < 4) break
|
||||
val total = (((buffer[offset + 2].toInt() and 255) shl 8) or (buffer[offset + 3].toInt() and 255)) + 6
|
||||
if (total < MIN_FRAME_BYTES) { offset += 2; continue }
|
||||
if (buffer.size - offset < total) break
|
||||
val frame = decodeFrame(buffer.copyOfRange(offset, offset + total))
|
||||
if (frame == null) { offset += 2; continue }
|
||||
out.add(frame)
|
||||
offset += total
|
||||
}
|
||||
buffer = buffer.copyOfRange(minOf(offset, buffer.size), buffer.size)
|
||||
return out
|
||||
}
|
||||
|
||||
val pendingBytes: Int get() = buffer.size
|
||||
}
|
||||
|
||||
// ---- TLV (tag u8, length u16 BE) ------------------------------------------------------------
|
||||
|
||||
class Tlv(@JvmField val tag: Int, @JvmField val value: ByteArray)
|
||||
|
||||
@JvmStatic
|
||||
fun tlv(tag: Int, value: ByteArray): ByteArray {
|
||||
val out = ByteArray(3 + value.size)
|
||||
out[0] = tag.toByte()
|
||||
out[1] = (value.size ushr 8).toByte()
|
||||
out[2] = value.size.toByte()
|
||||
value.copyInto(out, 3)
|
||||
return out
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun decodeTlvs(bytes: ByteArray): List<Tlv>? {
|
||||
val out = ArrayList<Tlv>()
|
||||
var i = 0
|
||||
while (i < bytes.size) {
|
||||
if (bytes.size - i < 3) return null
|
||||
val len = ((bytes[i + 1].toInt() and 255) shl 8) or (bytes[i + 2].toInt() and 255)
|
||||
if (bytes.size - i - 3 < len) return null
|
||||
out.add(Tlv(bytes[i].toInt() and 255, bytes.copyOfRange(i + 3, i + 3 + len)))
|
||||
i += 3 + len
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private fun List<Tlv>.value(tag: Int): ByteArray? = firstOrNull { it.tag == tag }?.value
|
||||
|
||||
// ---- Handshake (business 0x10) ----------------------------------------------------------------
|
||||
|
||||
const val PLATFORM_IOS = 1
|
||||
const val PLATFORM_ANDROID = 2
|
||||
|
||||
/** Version request. `phoneId` is a stable 6-byte protocol identity the glasses bind to. */
|
||||
@JvmStatic
|
||||
fun versionRequest(phoneId: ByteArray, phoneName: String, manufacturer: String, model: String, platform: Int = PLATFORM_ANDROID): ByteArray {
|
||||
require(phoneId.size == 6)
|
||||
val inner = tlv(0x10, byteArrayOf(1)) + tlv(0x11, phoneId) + tlv(0x12, phoneName.encodeToByteArray()) +
|
||||
tlv(0x13, byteArrayOf(platform.toByte())) + tlv(0x16, manufacturer.encodeToByteArray()) +
|
||||
tlv(0x17, model.encodeToByteArray())
|
||||
return tlv(0x11, inner)
|
||||
}
|
||||
|
||||
class VersionReply(
|
||||
@JvmField val protocolVersion: Int,
|
||||
/** The glasses' Bluetooth MAC, which the auth proof covers. */
|
||||
@JvmField val glassesId: ByteArray,
|
||||
@JvmField val serial: String?,
|
||||
/** Tags 0x19/0x1a: bound to another phone's account; guest auth will not work. */
|
||||
@JvmField val accountBound: Boolean,
|
||||
)
|
||||
|
||||
@JvmStatic
|
||||
fun parseVersionReply(payload: ByteArray): VersionReply? {
|
||||
val body = decodeTlvs(payload)?.value(0x11) ?: return null
|
||||
val fields = decodeTlvs(body) ?: return null
|
||||
val id = fields.value(0x11)?.takeIf { it.size == 6 } ?: return null
|
||||
return VersionReply(
|
||||
fields.value(0x10)?.firstOrNull()?.toInt()?.and(255) ?: 0,
|
||||
id,
|
||||
fields.value(0x14)?.decodeToString()?.trimEnd('\u0000'),
|
||||
fields.value(0x19) != null || fields.value(0x1a) != null,
|
||||
)
|
||||
}
|
||||
|
||||
/** `SHA-256(random4 ‖ phoneId6 ‖ glassesId6 ‖ SALT)`, both directions. */
|
||||
@JvmStatic
|
||||
fun authProof(random: ByteArray, phoneId: ByteArray, glassesId: ByteArray): ByteArray {
|
||||
val digest = Sha256Digest()
|
||||
digest.update(random)
|
||||
digest.update(phoneId)
|
||||
digest.update(glassesId)
|
||||
digest.update(SALT)
|
||||
return digest.digest()
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun guestAuthRequest(random: ByteArray, phoneId: ByteArray, glassesId: ByteArray): ByteArray {
|
||||
require(random.size == 4 && phoneId.size == 6 && glassesId.size == 6)
|
||||
return tlv(0x18, tlv(0x10, random) + tlv(0x11, authProof(random, phoneId, glassesId)))
|
||||
}
|
||||
|
||||
/** The glasses' answer proves they know the same phone and glasses ids. */
|
||||
@JvmStatic
|
||||
fun verifyGuestAuthReply(payload: ByteArray, phoneId: ByteArray, glassesId: ByteArray): Boolean {
|
||||
val body = decodeTlvs(payload)?.value(0x19) ?: return false
|
||||
val fields = decodeTlvs(body) ?: return false
|
||||
val random = fields.value(0x10)?.takeIf { it.size == 4 } ?: return false
|
||||
val proof = fields.value(0x11)?.takeIf { it.size == 32 } ?: return false
|
||||
return proof.contentEquals(authProof(random, phoneId, glassesId))
|
||||
}
|
||||
|
||||
/** True for a guest-auth reply (outer tag 0x19), valid or not. */
|
||||
@JvmStatic fun isGuestAuthReply(payload: ByteArray): Boolean = decodeTlvs(payload)?.value(0x19) != null
|
||||
|
||||
// ---- Business envelope (9, 13..22): 08 version, 10 type, 1A json, 22 bytes, 28 seq, 30 mode ----
|
||||
|
||||
const val MODE_REQUEST = 1
|
||||
const val MODE_REPLY = 2
|
||||
const val LAUNCHER_GENERAL_STATUS = 1
|
||||
const val LAUNCHER_BRIGHTNESS = 2
|
||||
const val LAUNCHER_GENERAL_SETTINGS = 4
|
||||
const val LAUNCHER_AI_VOICE_WAKEUP = 16
|
||||
const val OTA_VERSION = 1
|
||||
const val GENERAL_STATUS_REQUEST = """{"cmd":"request_general_status","payload":{"data":"","mode":0,"value":0}}"""
|
||||
/** The reply (type 4) carries `generalSettings`, among them the wake-word preferences. */
|
||||
const val GENERAL_SETTINGS_REQUEST = """{"cmd":"request_general_settings","payload":{"value":0,"mode":0,"data":""}}"""
|
||||
|
||||
/**
|
||||
* The launcher's wake-word switch. Every `set_ai_*` handler takes the new value from
|
||||
* `mode` and ignores `value` (RnLinkMsgProcessor::handleAssistantSettingCommand, see
|
||||
* notes/rayneo-io-wakeword-re.md): mode 1 on, 0 off.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun launcherSetAiVoiceWakeup(enabled: Boolean): String =
|
||||
"""{"cmd":"set_ai_voice_wakeup","payload":{"value":0,"mode":${if (enabled) 1 else 0},"data":""}}"""
|
||||
|
||||
// ---- Voice assistant (business 13), as Glassly's RayNeoIOProtocol maps it ----
|
||||
// The stock assistant: the wake word (or a temple long/double press) opens a session
|
||||
// and sends VOICE_WAKE; the phone answers VOICE_RC {"rc":1} for Opus as VOICE_AUDIO,
|
||||
// or {"rc":2} plus VOICE_ABORT to end it; VOICE_EXITED reports it gone. The remote
|
||||
// stream (VOICE_STREAM_CONTROL {"rc":1|2}, packets as VOICE_STREAM_AUDIO) needs
|
||||
// IDENT_FLAG_MIC firmware and runs without a session or its page, until a session,
|
||||
// a call, SCO or a recording takes the recorder. Packets are 16 kHz mono Opus in
|
||||
// the envelope's bytes field.
|
||||
|
||||
const val VOICE_WAKE = 1
|
||||
const val VOICE_RC = 2
|
||||
const val VOICE_AUDIO = 3
|
||||
const val VOICE_VAD = 4
|
||||
const val VOICE_ABORT = 7
|
||||
const val VOICE_EXITED = 8
|
||||
const val VOICE_STREAM_CONTROL = 58
|
||||
const val VOICE_STREAM_AUDIO = 59
|
||||
const val VOICE_RC_START = 1
|
||||
const val VOICE_RC_STOP = 2
|
||||
const val VOICE_SAMPLE_RATE = 16_000
|
||||
|
||||
/** A voice-channel message: `{"rc":n}` as [type] (VOICE_RC, VOICE_STREAM_CONTROL or VOICE_ABORT). */
|
||||
@JvmStatic
|
||||
fun voiceRc(type: Int, rc: Int): ByteArray = encodeEnvelope(Envelope(type, """{"rc":$rc}"""))
|
||||
|
||||
class Envelope(
|
||||
@JvmField val type: Int,
|
||||
@JvmField val json: String,
|
||||
@JvmField val binary: ByteArray? = null,
|
||||
@JvmField val sequence: Int? = null,
|
||||
@JvmField val mode: Int? = null,
|
||||
)
|
||||
|
||||
private fun varint(out: MutableList<Byte>, value: Long) {
|
||||
var v = value
|
||||
while (v and 0x7fL.inv() != 0L) { out.add(((v and 0x7f) or 0x80).toByte()); v = v ushr 7 }
|
||||
out.add(v.toByte())
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun encodeEnvelope(e: Envelope): ByteArray {
|
||||
val out = ArrayList<Byte>()
|
||||
out.add(0x08); varint(out, 1)
|
||||
out.add(0x10); varint(out, e.type.toLong())
|
||||
val json = e.json.encodeToByteArray()
|
||||
out.add(0x1a); varint(out, json.size.toLong()); json.forEach { out.add(it) }
|
||||
e.binary?.let { b -> out.add(0x22); varint(out, b.size.toLong()); b.forEach { out.add(it) } }
|
||||
e.sequence?.let { out.add(0x28); varint(out, it.toLong()) }
|
||||
e.mode?.let { out.add(0x30); varint(out, it.toLong()) }
|
||||
return out.toByteArray()
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun decodeEnvelope(bytes: ByteArray): Envelope? {
|
||||
var pos = 0
|
||||
fun readVarint(): Long? {
|
||||
var shift = 0
|
||||
var result = 0L
|
||||
while (pos < bytes.size && shift <= 63) {
|
||||
val b = bytes[pos++].toInt() and 255
|
||||
result = result or ((b and 0x7f).toLong() shl shift)
|
||||
if (b and 0x80 == 0) return result
|
||||
shift += 7
|
||||
}
|
||||
return null
|
||||
}
|
||||
var type: Int? = null
|
||||
var json = ""
|
||||
var binary: ByteArray? = null
|
||||
var sequence: Int? = null
|
||||
var mode: Int? = null
|
||||
while (pos < bytes.size) {
|
||||
val key = readVarint() ?: return null
|
||||
when ((key and 7).toInt()) {
|
||||
0 -> {
|
||||
val v = readVarint() ?: return null
|
||||
when ((key ushr 3).toInt()) { 2 -> type = v.toInt(); 5 -> sequence = v.toInt(); 6 -> mode = v.toInt() }
|
||||
}
|
||||
2 -> {
|
||||
val len = readVarint() ?: return null
|
||||
if (len < 0 || len > bytes.size - pos) return null
|
||||
val v = bytes.copyOfRange(pos, pos + len.toInt())
|
||||
pos += len.toInt()
|
||||
when ((key ushr 3).toInt()) { 3 -> json = v.decodeToString(); 4 -> binary = v }
|
||||
}
|
||||
1 -> pos += 8
|
||||
5 -> pos += 4
|
||||
else -> return null
|
||||
}
|
||||
}
|
||||
return Envelope(type ?: return null, json, binary, sequence, mode)
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the launcher's brightness (1..8). The stock handler also turns the screen on
|
||||
* (BrightnessMonitor::setBrightness -> turnOnScreenForSetting), which is how the phone
|
||||
* wakes the iO's display and keeps the stock auto-off from blanking a static page.
|
||||
* Only for firmware without [Ident.brightnessControl]: stock auto-brightness soon
|
||||
* replaces the level, and each one writes the glasses' preferences.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun launcherBrightness(level: Int): String =
|
||||
"""{"cmd":"brightness_change","payload":{"value":${level.coerceIn(1, 8)},"mode":0,"data":""}}"""
|
||||
|
||||
/** Faceclaw's 0..100 brightness target on the launcher's 1..8 scale. */
|
||||
@JvmStatic fun brightnessLevel(target: Int): Int = maxOf(1, (target.coerceIn(0, 100) * 8 + 50) / 100)
|
||||
|
||||
/** `screen_status` push value (1 on, 0 off), or -1 for anything else. */
|
||||
@JvmStatic
|
||||
fun screenStatusFromLauncherJson(json: String): Int {
|
||||
val root = try { Json.parse(json).asObject() } catch (_: Exception) { null } ?: return -1
|
||||
if (root["cmd"].asString() != "screen_status") return -1
|
||||
return root["payload"].asObject()?.get("value").asInt() ?: -1
|
||||
}
|
||||
|
||||
/** `generalStatus.battery` from a launcher reply or push, or -1. */
|
||||
@JvmStatic
|
||||
fun batteryFromLauncherJson(json: String): Int {
|
||||
val root = try { Json.parse(json).asObject() } catch (_: Exception) { null } ?: return -1
|
||||
val status = root["generalStatus"].asObject() ?: root["payload"].asObject()?.get("generalStatus").asObject()
|
||||
status?.get("battery").asInt()?.let { return it }
|
||||
// pushes: {"cmd":"battery_change","payload":{"value":N,...}}
|
||||
if (root["cmd"].asString() == "battery_change") return root["payload"].asObject()?.get("value").asInt() ?: -1
|
||||
return -1
|
||||
}
|
||||
|
||||
// ---- Custom-firmware link (business 0xF0) -----------------------------------------------------
|
||||
|
||||
const val OPT_LENS = 0x01
|
||||
const val OPT_CONTROL = 0x10
|
||||
const val OPT_END = 0x40
|
||||
const val OPT_RESET = 0x80
|
||||
const val CTRL_QUERY = 1
|
||||
const val CTRL_PAGE_OPEN = 2
|
||||
const val CTRL_PAGE_CLOSE = 3
|
||||
const val CTRL_KEEPALIVE = 4
|
||||
/** Records up to this many bytes per packet (IDENT reports the firmware's maxChunk). */
|
||||
const val DEFAULT_MAX_CHUNK = 240
|
||||
/** The page closes this long after the last PAGE_OPEN/KEEPALIVE. */
|
||||
const val PAGE_LEASE_MS = 120_000L
|
||||
|
||||
@JvmStatic
|
||||
fun controlPacket(ctrl: Int, arg: Int? = null): ByteArray =
|
||||
if (arg == null) byteArrayOf(OPT_CONTROL.toByte(), 0, ctrl.toByte())
|
||||
else byteArrayOf(OPT_CONTROL.toByte(), 0, ctrl.toByte(), arg.toByte())
|
||||
|
||||
/**
|
||||
* One record (from [CfwTransport.encodeRecord]) as business-0xF0 payloads
|
||||
* `[opt][stream][chunk]`. `stream` is echoed in the ACKs, so it plays the G2 stream id's role.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun cfwPackets(record: ByteArray, streamId: Int, maxChunk: Int = DEFAULT_MAX_CHUNK): List<ByteArray> {
|
||||
require(streamId in 0..255 && maxChunk > 0)
|
||||
val out = ArrayList<ByteArray>()
|
||||
var offset = 0
|
||||
do {
|
||||
val end = minOf(offset + maxChunk, record.size)
|
||||
var opt = OPT_LENS
|
||||
if (offset == 0) opt = opt or OPT_RESET
|
||||
if (end >= record.size) opt = opt or OPT_END
|
||||
val p = ByteArray(2 + end - offset)
|
||||
p[0] = opt.toByte()
|
||||
p[1] = streamId.toByte()
|
||||
record.copyInto(p, 2, offset, end)
|
||||
out.add(p)
|
||||
offset = end
|
||||
} while (offset < record.size)
|
||||
return out
|
||||
}
|
||||
|
||||
const val KIND_ACK = 1
|
||||
const val KIND_NACK = 3
|
||||
const val KIND_REPLY = 5
|
||||
const val KIND_GESTURE = 7
|
||||
const val KIND_IDENT = 0x11
|
||||
const val KIND_PAGE = 0x13
|
||||
const val KIND_DIAG = 0x15
|
||||
const val KIND_ALS = 0x1f
|
||||
|
||||
/** `[01|03][stream][ordinal u16][lens][size u16][crc u16]`: the G2 ACK body, one per lens. */
|
||||
@JvmStatic
|
||||
fun parseCfwAck(payload: ByteArray): CfwTransport.Ack? {
|
||||
if (payload.size != 9) return null
|
||||
val kind = payload[0].toInt() and 255
|
||||
val lens = payload[4].toInt() and 255
|
||||
if ((kind != KIND_ACK && kind != KIND_NACK) || (lens != 1 && lens != 2)) return null
|
||||
fun u16(o: Int) = (payload[o].toInt() and 255) or ((payload[o + 1].toInt() and 255) shl 8)
|
||||
return CfwTransport.Ack(kind == KIND_NACK, payload[1].toInt() and 255, u16(2), lens, u16(5), u16(7))
|
||||
}
|
||||
|
||||
const val IDENT_FLAG_ZLIB = 0x01
|
||||
const val IDENT_FLAG_STEREO = 0x08
|
||||
/** The assistant's remote audio stream (type 58/59 below) is allowed (CFW revision 10). */
|
||||
const val IDENT_FLAG_MIC = 0x20
|
||||
const val IDENT_FLAG_FACECLAW = 0x80
|
||||
/** Message 30: the firmware sets the panel level and runs fades, with stock auto-brightness off. */
|
||||
const val FEATURE_BRIGHTNESS = "bright30"
|
||||
/** Message 16: the firmware reads the light sensor and sends [KIND_ALS] reports. */
|
||||
const val FEATURE_ALS = "als16"
|
||||
|
||||
class Ident(
|
||||
@JvmField val width: Int,
|
||||
@JvmField val height: Int,
|
||||
@JvmField val maxChunk: Int,
|
||||
@JvmField val revision: Int,
|
||||
@JvmField val flags: Int,
|
||||
@JvmField val name: String,
|
||||
/** "Faceclaw/<n>" -> n when the firmware runs the Faceclaw display lists, else -1. */
|
||||
@JvmField val faceclawRevision: Int,
|
||||
/** What the firmware adds to the display lists (a third string in IDENT). */
|
||||
@JvmField val features: List<String> = emptyList(),
|
||||
) {
|
||||
val acceptsCompressed: Boolean get() = flags and IDENT_FLAG_ZLIB != 0
|
||||
val micStream: Boolean get() = flags and IDENT_FLAG_MIC != 0
|
||||
val faceclaw: Boolean get() = flags and IDENT_FLAG_FACECLAW != 0 && faceclawRevision > 0
|
||||
/** Faceclaw owns brightness and the light sensor, as on the G2 custom firmware. */
|
||||
val brightnessControl: Boolean get() = faceclaw && FEATURE_BRIGHTNESS in features && FEATURE_ALS in features
|
||||
}
|
||||
|
||||
/** `[11][ver][w u16][h u16][maxChunk u16][storeKiB u16][rev u16][flags]name\0[lineHeight]["Faceclaw/n"\0["features"\0]]`. */
|
||||
@JvmStatic
|
||||
fun parseIdent(payload: ByteArray): Ident? {
|
||||
if (payload.size < 13 || payload[0].toInt() != KIND_IDENT) return null
|
||||
fun u16(o: Int) = (payload[o].toInt() and 255) or ((payload[o + 1].toInt() and 255) shl 8)
|
||||
var end = 13
|
||||
while (end < payload.size && payload[end].toInt() != 0) end++
|
||||
val name = payload.copyOfRange(13, end).decodeToString()
|
||||
var faceclaw = -1
|
||||
var features = emptyList<String>()
|
||||
val extra = end + 2 // NUL, line height
|
||||
if (extra < payload.size) {
|
||||
var e = extra
|
||||
while (e < payload.size && payload[e].toInt() != 0) e++
|
||||
val text = payload.copyOfRange(extra, e).decodeToString()
|
||||
if (text.startsWith("Faceclaw/")) faceclaw = text.removePrefix("Faceclaw/").toIntOrNull() ?: -1
|
||||
val more = e + 1
|
||||
if (more < payload.size) {
|
||||
var f = more
|
||||
while (f < payload.size && payload[f].toInt() != 0) f++
|
||||
features = payload.copyOfRange(more, f).decodeToString().split(' ').filter { it.isNotEmpty() }
|
||||
}
|
||||
}
|
||||
return Ident(u16(2), u16(4), u16(6), u16(10), payload[12].toInt() and 255, name, faceclaw, features)
|
||||
}
|
||||
|
||||
/**
|
||||
* `[1f]` + the 24-byte `['A','L',1,reason,...]` body the G2 custom firmware sends as settings
|
||||
* field 105 (value in tenths of a lux), or null. The body goes to the same consumers.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun parseAlsReport(payload: ByteArray): ByteArray? {
|
||||
if (payload.size < 25 || (payload[0].toInt() and 255) != KIND_ALS) return null
|
||||
val body = payload.copyOfRange(1, 25)
|
||||
return if (body[0].toInt() == 65 && body[1].toInt() == 76 && body[2].toInt() == 1) body else null
|
||||
}
|
||||
|
||||
/** PAGE `[13][state][reason]`: state 1 open. */
|
||||
@JvmStatic fun parsePageState(payload: ByteArray): Int = if (payload.size >= 2 && payload[0].toInt() == KIND_PAGE) payload[1].toInt() and 255 else -1
|
||||
|
||||
class Gesture(@JvmField val eventType: Int, @JvmField val source: Int, @JvmField val tick: Long)
|
||||
|
||||
/**
|
||||
* GESTURE `[07][eventType][source][tick u32]`. Event types use the G2's OsEventTypeList numbering:
|
||||
* 0 single tap, 1 swipe up, 2 swipe down, 3 double tap, 11 tap-then-hold, 12 head up, 15 head down.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun parseGesture(payload: ByteArray): Gesture? {
|
||||
if (payload.size < 7 || payload[0].toInt() != KIND_GESTURE) return null
|
||||
val tick = (payload[3].toLong() and 255) or ((payload[4].toLong() and 255) shl 8) or
|
||||
((payload[5].toLong() and 255) shl 16) or ((payload[6].toLong() and 255) shl 24)
|
||||
return Gesture(payload[1].toInt() and 255, payload[2].toInt() and 255, tick)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import platform.Foundation.NSData
|
||||
|
||||
/**
|
||||
* Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's
|
||||
* captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record:
|
||||
* 16 kHz mono PCM16, as its recognizer hands it to TypeScript.
|
||||
*/
|
||||
class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) {
|
||||
/** The recording stopped early: the file could not be written. */
|
||||
val failed: Boolean get() = writer.failed
|
||||
|
||||
/** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */
|
||||
fun append(pcm16le: NSData) {
|
||||
val bytes = pcm16le.byteArray()
|
||||
val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) }
|
||||
writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs()))
|
||||
}
|
||||
|
||||
/** Completes the file. Returns the recording's length in ms. */
|
||||
fun finish(): Long = writer.finish()
|
||||
}
|
||||
|
||||
object IosMicRecordings {
|
||||
/**
|
||||
* A recorder writing a new file at [path], or null when the file cannot be created.
|
||||
* [configJson] describes the capture and is stored in the file.
|
||||
*/
|
||||
fun open(path: String, configJson: String?): IosMicRecorder? =
|
||||
try {
|
||||
IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform))
|
||||
} catch (error: Throwable) {
|
||||
PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}")
|
||||
null
|
||||
}
|
||||
}
|
||||
@@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped {
|
||||
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput {
|
||||
val file = fopen(path, "wb+") ?: error("Cannot write $path")
|
||||
return object : SeekableOutput {
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
private fun put(bytes: ByteArray, offset: Int, length: Int) {
|
||||
if (length == 0) return
|
||||
bytes.usePinned {
|
||||
check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert<platform.posix.size_t>()) { "Short write to $path" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
put(bytes, offset, length)
|
||||
size += length
|
||||
// Each write is a complete element; hand it to the OS so a crash keeps it.
|
||||
platform.posix.fflush(file)
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0)
|
||||
put(bytes, 0, bytes.size)
|
||||
check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0)
|
||||
platform.posix.fflush(file)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput {
|
||||
val file = fopen(path, "rb") ?: error("Cannot read $path")
|
||||
val fileSize = fileLength(path)
|
||||
return object : SeekableInput {
|
||||
override val length = fileSize
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0
|
||||
return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() }
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -461,6 +461,20 @@ class IosBleCentral(private val platform: ProtocolPlatform = IosProtocolPlatform
|
||||
|
||||
override fun isBonded(address: String): Boolean = true
|
||||
|
||||
/** CoreBluetooth pairs by itself when an attribute needs it. */
|
||||
override fun requestBond(address: String) {}
|
||||
|
||||
/** Only iOS Settings can remove a pairing. */
|
||||
override fun removeBond(address: String): Boolean = false
|
||||
|
||||
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean {
|
||||
val peer = connectedPeer(address) ?: return false
|
||||
return cond.withLock { peer.characteristics.containsKey(characteristicUuid.lowercase()) }
|
||||
}
|
||||
|
||||
/** CoreBluetooth follows Service Changed itself; discovering again is all there is. */
|
||||
override fun refreshServices(address: String, timeoutMs: Int): Boolean = discoverServices(address, timeoutMs)
|
||||
|
||||
override fun prepareBenchmarkLink(address: String, mode: Int) {}
|
||||
|
||||
override fun recordDisplayFrameSent() {
|
||||
|
||||
@@ -34,6 +34,11 @@ interface IosMicStatusListener {
|
||||
fun onNsDataMicStatus(body: NSData, arm: String)
|
||||
}
|
||||
|
||||
/** CFW keyboard-bridge records (the raw field-131 record) as NSData. Main queue. */
|
||||
interface IosKeyboardEventListener {
|
||||
fun onNsDataKeyboardEvent(body: NSData)
|
||||
}
|
||||
|
||||
/**
|
||||
* ObjC-facing facade over the shared [GlassesSessionCore] for the NativeScript app: the iOS
|
||||
* counterpart of Android's FaceclawBleCommunicator with the same method names. GATT goes
|
||||
@@ -86,6 +91,9 @@ class IosGlassesSession internal constructor(
|
||||
|
||||
fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision)
|
||||
|
||||
/** Drive an Even G1 on g1v0 (see GlassesSessionG1.kt). Before [start]. */
|
||||
fun useG1() = core.useG1()
|
||||
|
||||
fun start(): Boolean = core.start()
|
||||
|
||||
fun disconnect() = core.disconnect()
|
||||
@@ -105,6 +113,8 @@ class IosGlassesSession internal constructor(
|
||||
|
||||
fun isAncsAuthorized(): Boolean = central?.isAncsAuthorized(rightIdentifier) ?: false
|
||||
|
||||
fun ancsRelayAvailable(): Boolean = core.ancsRelayAvailable()
|
||||
|
||||
fun setAncsListener(listener: IosAncsListener?) {
|
||||
if (listener == null) {
|
||||
core.setRawFrameTap(null)
|
||||
@@ -225,6 +235,20 @@ class IosGlassesSession internal constructor(
|
||||
return adapter
|
||||
}
|
||||
|
||||
fun addKeyboardEventListenerNsData(listener: IosKeyboardEventListener): FaceclawKeyboardEventListener {
|
||||
val adapter = object : FaceclawKeyboardEventListener {
|
||||
override fun onKeyboardEvent(body: ByteArray?) =
|
||||
listener.onNsDataKeyboardEvent((body ?: ByteArray(0)).data())
|
||||
}
|
||||
core.addKeyboardEventListener(adapter)
|
||||
return adapter
|
||||
}
|
||||
|
||||
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
|
||||
|
||||
fun sendFaceclawKeyboardControl(record: NSData?, label: String): Boolean =
|
||||
core.sendFaceclawKeyboardControl(record?.byteArray(), label)
|
||||
|
||||
/** Every audio-characteristic packet, whether or not a capture is active (passive; sends nothing). */
|
||||
fun addAudioMonitorListenerNsData(listener: IosAudioPacketListener): FaceclawAudioPacketListener {
|
||||
val adapter = object : FaceclawAudioPacketListener {
|
||||
|
||||
@@ -171,3 +171,57 @@ class IosFirmwareFlasher(rightAddress: String, leftAddress: String, firmwarePath
|
||||
lock.withLock { listener }?.let(action)
|
||||
}
|
||||
}
|
||||
|
||||
/** iOS host for the shared [G1ArmFlashFlow]: one G1 arm's install, its restore to Even's firmware, or a question about what it runs. */
|
||||
class IosG1Flasher(address: String, side: String, firmwarePath: String, netCorePath: String, mode: String, connectWindowMs: Int) {
|
||||
private val link = IosBleCentral()
|
||||
private val lock = IosProtocolPlatform.createLock()
|
||||
private var started = false
|
||||
private var listener: FaceclawFirmwareFlasherListener? = null
|
||||
|
||||
private val flow = G1ArmFlashFlow(
|
||||
link,
|
||||
address,
|
||||
side,
|
||||
firmwarePath,
|
||||
netCorePath,
|
||||
G1ArmFlashFlow.mode(mode),
|
||||
connectWindowMs,
|
||||
object : FaceclawFirmwareFlasherListener {
|
||||
override fun onLog(line: String?) {
|
||||
PlatformLog.i(G1ArmFlashFlow.TAG, line ?: "")
|
||||
post { it.onLog(line) }
|
||||
}
|
||||
|
||||
override fun onProgress(lens: String?, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int, bytesSent: Long, bytesTotal: Long) =
|
||||
post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
|
||||
|
||||
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
|
||||
|
||||
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
|
||||
},
|
||||
)
|
||||
|
||||
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
|
||||
lock.withLock { this.listener = listener }
|
||||
}
|
||||
|
||||
fun start() {
|
||||
lock.withLock {
|
||||
if (started) return
|
||||
started = true
|
||||
}
|
||||
runOnThread("faceclaw-g1-flash") { flow.run() }
|
||||
}
|
||||
|
||||
fun cancel() = flow.cancel()
|
||||
|
||||
fun close() {
|
||||
cancel()
|
||||
runCatching { link.close() }
|
||||
}
|
||||
|
||||
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) = main {
|
||||
lock.withLock { listener }?.let(action)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class)
|
||||
|
||||
package com.faceclaw.app
|
||||
|
||||
import platform.Foundation.NSData
|
||||
|
||||
/** ColorGlyphArt for FaceclawFontRenderer.m, which draws color glyphs with CoreGraphics. */
|
||||
object IosColorGlyph {
|
||||
/**
|
||||
* The shade (one byte per pixel) of a color glyph drawn as premultiplied RGBA bytes, as a
|
||||
* kCGImageAlphaPremultipliedLast context lays them out. Empty data for a malformed buffer
|
||||
* rather than an exception, which would terminate the app on its way into Objective-C.
|
||||
*/
|
||||
fun shade(rgba: NSData): NSData {
|
||||
val bytes = rgba.byteArray()
|
||||
if (bytes.isEmpty() || bytes.size % 4 != 0) return NSData()
|
||||
val argb = IntArray(bytes.size / 4) { i ->
|
||||
val a = bytes[i * 4 + 3].toInt() and 0xff
|
||||
fun channel(offset: Int): Int {
|
||||
if (a == 0) return 0
|
||||
val premultiplied = bytes[i * 4 + offset].toInt() and 0xff
|
||||
return ((premultiplied * 255 + a / 2) / a).coerceAtMost(255)
|
||||
}
|
||||
(a shl 24) or (channel(0) shl 16) or (channel(1) shl 8) or channel(2)
|
||||
}
|
||||
return ColorGlyphArt.shade(argb).data()
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user