[verified] merge upstream 25cb255b into all-features

This commit is contained in:
MasterControl committed 2026-11-19 22:21:26 +00:00
commit ffcdae660d
169 files changed
+16201 -597

No files matched your search

+2 -1
View File
@@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
override var size = 0L
private set
override fun write(bytes: ByteArray, offset: Int, length: Int) {
file.write(bytes, offset, length)
size += length
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
file.seek(position)
file.write(bytes)
file.seek(size)
}
override fun close() = file.close()
}
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
private val file = java.io.RandomAccessFile(path, "r")
override val length = file.length()
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
file.seek(position)
var total = 0
while (total < length) {
val read = file.read(buffer, offset + total, length - total)
if (read <= 0) break
total += read
}
return total
}
override fun close() = file.close()
}
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
expect fun deleteFile(path: String): Boolean
expect fun renameFile(from: String, to: String): Boolean
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
interface SeekableOutput {
/** Bytes appended so far. */
val size: Long
/** Appends [length] bytes of [bytes] from [offset]. */
fun write(bytes: ByteArray, offset: Int, length: Int)
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
fun writeAt(position: Long, bytes: ByteArray)
fun close()
}
/** A file read at arbitrary positions without loading it whole. */
interface SeekableInput {
val length: Long
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
fun close()
}
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
expect fun openFileOutput(path: String): SeekableOutput
/** Opens [path] for reading. Throws when it cannot. */
expect fun openFileInput(path: String): SeekableInput
@@ -0,0 +1,764 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/**
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
*
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
* about a second each, cues and the segment size written when the recording finishes. A
* recording cut short by a crash is still a valid live-style stream of everything up to its last
* complete cluster, and the reader takes it as such.
*/
internal object MatroskaIds {
const val EBML = 0x1A45DFA3
const val EBML_VERSION = 0x4286
const val EBML_READ_VERSION = 0x42F7
const val EBML_MAX_ID_LENGTH = 0x42F2
const val EBML_MAX_SIZE_LENGTH = 0x42F3
const val DOC_TYPE = 0x4282
const val DOC_TYPE_VERSION = 0x4287
const val DOC_TYPE_READ_VERSION = 0x4285
const val VOID = 0xEC
const val SEGMENT = 0x18538067
const val SEEK_HEAD = 0x114D9B74
const val SEEK = 0x4DBB
const val SEEK_ID = 0x53AB
const val SEEK_POSITION = 0x53AC
const val INFO = 0x1549A966
const val TIMESTAMP_SCALE = 0x2AD7B1
const val DURATION = 0x4489
const val DATE_UTC = 0x4461
const val TITLE = 0x7BA9
const val MUXING_APP = 0x4D80
const val WRITING_APP = 0x5741
const val TRACKS = 0x1654AE6B
const val TRACK_ENTRY = 0xAE
const val TRACK_NUMBER = 0xD7
const val TRACK_UID = 0x73C5
const val TRACK_TYPE = 0x83
const val FLAG_DEFAULT = 0x88
const val FLAG_LACING = 0x9C
const val NAME = 0x536E
const val LANGUAGE = 0x22B59C
const val CODEC_ID = 0x86
const val AUDIO = 0xE1
const val SAMPLING_FREQUENCY = 0xB5
const val CHANNELS = 0x9F
const val BIT_DEPTH = 0x6264
const val ATTACHMENTS = 0x1941A469
const val ATTACHED_FILE = 0x61A7
const val FILE_NAME = 0x466E
const val FILE_MIME_TYPE = 0x4660
const val FILE_DATA = 0x465C
const val FILE_UID = 0x46AE
const val CLUSTER = 0x1F43B675
const val TIMESTAMP = 0xE7
const val SIMPLE_BLOCK = 0xA3
const val BLOCK_GROUP = 0xA0
const val BLOCK = 0xA1
const val BLOCK_DURATION = 0x9B
const val CUES = 0x1C53BB6B
const val CUE_POINT = 0xBB
const val CUE_TIME = 0xB3
const val CUE_TRACK_POSITIONS = 0xB7
const val CUE_TRACK = 0xF7
const val CUE_CLUSTER_POSITION = 0xF1
}
/** Builds EBML elements into a byte buffer. */
internal class EbmlWriter(capacity: Int = 256) {
private val sink = ByteSink(capacity)
fun size(): Int = sink.size()
fun bytes(): ByteArray = sink.toByteArray()
/** An element ID, written as given: its length marker is part of the value. */
fun id(id: Int) {
if (id > 0xFFFFFF) sink.write(id ushr 24)
if (id > 0xFFFF) sink.write(id ushr 16)
if (id > 0xFF) sink.write(id ushr 8)
sink.write(id)
}
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
fun dataSize(size: Long, width: Int = 0) {
var bytes = width
if (bytes == 0) {
bytes = 1
// The all-ones value of each width means "unknown size".
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
}
val marked = size or (1L shl (7 * bytes))
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
}
fun uint(id: Int, value: Long) {
var bytes = 1
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
id(id)
dataSize(bytes.toLong())
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
/** A fixed eight-byte integer (dates). */
fun int64(id: Int, value: Long) {
id(id)
dataSize(8)
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
fun string(id: Int, text: String) {
val bytes = text.encodeToByteArray()
binary(id, bytes, 0, bytes.size)
}
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
id(id)
dataSize(length.toLong())
sink.write(bytes, offset, length)
}
fun raw(bytes: ByteArray) = sink.write(bytes)
fun raw(value: Int) = sink.write(value)
fun master(id: Int, build: EbmlWriter.() -> Unit) {
val child = EbmlWriter()
child.build()
id(id)
dataSize(child.size().toLong())
sink.write(child.bytes())
}
companion object {
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
fun padded(element: ByteArray, reserved: Int): ByteArray? {
val filler = reserved - element.size
// A Void element is at least two bytes.
if (filler < 0 || filler == 1) return null
val out = element.copyOf(reserved)
if (filler == 0) return out
val at = element.size
out[at] = MatroskaIds.VOID.toByte()
if (filler < 9) {
out[at + 1] = (0x80 or (filler - 2)).toByte()
} else {
val size = (filler - 9).toLong()
out[at + 1] = 0x01
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
}
return out
}
}
}
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
/**
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
* milliseconds from the start of the file. Not thread-safe.
*/
class MkaWriter(
private val output: SeekableOutput,
title: String?,
/** When the recording began (Unix epoch ms), for the file's date. */
dateUtcMs: Long,
attachments: List<MkaAttachment>,
) {
companion object {
const val CODEC_PCM = "A_PCM/INT/LIT"
const val CODEC_TEXT = "S_TEXT/UTF8"
const val TRACK_AUDIO = 2
const val TRACK_SUBTITLE = 0x11
const val APP_NAME = "Faceclaw"
private const val SEEK_RESERVED = 128
private const val TRACKS_RESERVED = 4096
private const val CLUSTER_SPAN_MS = 1000L
private const val CLUSTER_MAX_BYTES = 1 shl 20
// Block timestamps are signed 16-bit offsets from their cluster's.
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
private const val DURATION_PATCH_CLUSTERS = 5
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
private const val DATE_EPOCH_MS = 978_307_200_000L
}
private class Track(
val number: Int,
val type: Int,
val name: String,
val sampleRate: Int,
val channels: Int,
val preferred: Boolean,
)
private class Cue(val clusterPosition: Long, val timeMs: Long)
private val tracks = ArrayList<Track>()
private val cues = ArrayList<Cue>()
private val segmentSizePosition: Long
private val segmentStart: Long
private val seekPosition: Long
private val infoPosition: Long
private val durationPosition: Long
private val tracksPosition: Long
private val attachmentsPosition: Long
private var cluster: EbmlWriter? = null
private var clusterBaseMs = 0L
private var clustersSincePatch = 0
private var finished = false
private val hasAttachments = attachments.isNotEmpty()
/** The end of the latest audio or text written, in ms from the start of the file. */
var durationMs = 0L
private set
init {
val head = EbmlWriter()
head.master(MatroskaIds.EBML) {
uint(MatroskaIds.EBML_VERSION, 1)
uint(MatroskaIds.EBML_READ_VERSION, 1)
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
string(MatroskaIds.DOC_TYPE, "matroska")
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
}
head.id(MatroskaIds.SEGMENT)
segmentSizePosition = head.size().toLong()
// Unknown size (a live stream) until finish() knows it.
head.raw(0x01)
repeat(7) { head.raw(0xFF) }
segmentStart = head.size().toLong()
append(head.bytes())
seekPosition = output.size
infoPosition = seekPosition + SEEK_RESERVED
val info = EbmlWriter()
info.master(MatroskaIds.INFO) {
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
string(MatroskaIds.MUXING_APP, APP_NAME)
string(MatroskaIds.WRITING_APP, APP_NAME)
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
// Last, so its eight bytes are the element's final ones.
float64(MatroskaIds.DURATION, 0.0)
}
durationPosition = infoPosition + info.size() - 8
tracksPosition = infoPosition + info.size()
attachmentsPosition = tracksPosition + TRACKS_RESERVED
append(seekHead(-1))
append(info.bytes())
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
if (attachments.isNotEmpty()) {
val files = EbmlWriter()
files.master(MatroskaIds.ATTACHMENTS) {
attachments.forEachIndexed { index, file ->
master(MatroskaIds.ATTACHED_FILE) {
string(MatroskaIds.FILE_NAME, file.name)
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
uint(MatroskaIds.FILE_UID, index + 1L)
}
}
}
append(files.bytes())
}
}
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
private fun seekHead(cuesPosition: Long): ByteArray {
val head = EbmlWriter()
head.master(MatroskaIds.SEEK_HEAD) {
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
val idBytes = EbmlWriter()
idBytes.id(id)
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
}
entry(MatroskaIds.INFO, infoPosition)
entry(MatroskaIds.TRACKS, tracksPosition)
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
}
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
}
/**
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
* the one players pick by default.
*/
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
private fun addTrack(track: Track): Int {
check(!finished) { "recording already finished" }
// A block's track number is written in one byte.
if (track.number > 126) return -1
tracks.add(track)
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
if (region == null) {
tracks.remove(track)
return -1
}
output.writeAt(tracksPosition, region)
return track.number
}
private fun listedTracks(): List<Track> =
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
tracks.filter { it.type != TRACK_AUDIO }
private fun tracksElement(): ByteArray {
val listed = listedTracks()
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
val out = EbmlWriter(1024)
out.master(MatroskaIds.TRACKS) {
for (track in listed) {
master(MatroskaIds.TRACK_ENTRY) {
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
uint(MatroskaIds.TRACK_UID, track.number.toLong())
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
uint(MatroskaIds.FLAG_LACING, 0)
string(MatroskaIds.NAME, track.name)
string(MatroskaIds.LANGUAGE, "und")
if (track.type == TRACK_AUDIO) {
string(MatroskaIds.CODEC_ID, CODEC_PCM)
master(MatroskaIds.AUDIO) {
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
uint(MatroskaIds.CHANNELS, track.channels.toLong())
uint(MatroskaIds.BIT_DEPTH, 16)
}
} else {
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
}
}
}
}
return out.bytes()
}
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
val track = track(trackNumber)
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
if (length <= 0) return
val block = openCluster(timeMs)
block.id(MatroskaIds.SIMPLE_BLOCK)
block.dataSize(4L + length)
blockHeader(block, trackNumber, timeMs, 0x80)
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
val frames = length / (2 * track.channels)
noteEnd(timeMs + frames * 1000L / track.sampleRate)
}
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
val bytes = text.encodeToByteArray()
val block = openCluster(timeMs)
block.master(MatroskaIds.BLOCK_GROUP) {
id(MatroskaIds.BLOCK)
dataSize(4L + bytes.size)
blockHeader(this, trackNumber, timeMs, 0)
raw(bytes)
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
}
noteEnd(timeMs + maxOf(1L, durationMs))
}
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
block.raw(0x80 or trackNumber)
block.raw(relative shr 8)
block.raw(relative)
block.raw(flags)
}
private fun noteEnd(endMs: Long) {
if (endMs > durationMs) durationMs = endMs
}
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
private fun openCluster(timeMs: Long): EbmlWriter {
check(!finished) { "recording already finished" }
val time = maxOf(0L, timeMs)
val open = cluster
if (open != null) {
val relative = time - clusterBaseMs
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
flushCluster()
}
clusterBaseMs = time
return EbmlWriter(64 * 1024).also { cluster = it }
}
private fun flushCluster() {
val body = cluster ?: return
cluster = null
if (body.size() == 0) return
val timestamp = EbmlWriter(16)
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
val header = EbmlWriter(32)
header.id(MatroskaIds.CLUSTER)
header.dataSize(timestamp.size().toLong() + body.size())
header.raw(timestamp.bytes())
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
append(header.bytes())
append(body.bytes())
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
}
private fun patchDuration() {
clustersSincePatch = 0
val value = durationMs.toDouble().toRawBits()
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
}
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
fun finish() {
if (finished) return
try {
flushCluster()
finished = true
val cuesPosition = output.size
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
if (cues.isNotEmpty()) {
val index = EbmlWriter(cues.size * 16 + 16)
index.master(MatroskaIds.CUES) {
for (cue in cues) {
master(MatroskaIds.CUE_POINT) {
uint(MatroskaIds.CUE_TIME, cue.timeMs)
master(MatroskaIds.CUE_TRACK_POSITIONS) {
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
}
}
}
}
append(index.bytes())
output.writeAt(seekPosition, seekHead(cuesPosition))
}
patchDuration()
val size = EbmlWriter(8)
size.dataSize(output.size - segmentStart, 8)
output.writeAt(segmentSizePosition, size.bytes())
} finally {
finished = true
output.close()
}
}
}
class MkaTrackInfo(
@JvmField val number: Int,
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
@JvmField val type: Int,
@JvmField val codecId: String,
@JvmField val name: String,
@JvmField val sampleRate: Int,
@JvmField val channels: Int,
@JvmField val bitDepth: Int,
) {
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
}
class MkaBlock(
@JvmField val track: Int,
@JvmField val timeMs: Long,
@JvmField val data: ByteArray,
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
@JvmField val durationMs: Long,
)
/**
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
* IllegalArgumentException for anything that is not such a file.
*/
class MkaReader(private val input: SeekableInput) {
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
private val clusters = ArrayList<ClusterRef>()
private val trackList = ArrayList<MkaTrackInfo>()
private val attachmentList = ArrayList<MkaAttachment>()
private var timestampScale = 1_000_000L
private var infoDurationMs = 0L
private var nextCluster = 0
private var blocks: List<MkaBlock> = emptyList()
private var blockIndex = 0
private var skipBeforeMs = 0L
val tracks: List<MkaTrackInfo> get() = trackList
val attachments: List<MkaAttachment> get() = attachmentList
/** The end of the last audio or text in the file, in ms. */
val durationMs: Long
init {
scan()
var end = infoDurationMs
if (clusters.isNotEmpty()) {
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
}
durationMs = end
}
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
/** Continue from the first block at or after [timeMs]. */
fun seekTo(timeMs: Long) {
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
nextCluster = maxOf(0, index)
blocks = emptyList()
blockIndex = 0
skipBeforeMs = timeMs
}
/** The next block, or null at the end of the file. */
fun next(): MkaBlock? {
while (true) {
while (blockIndex >= blocks.size) {
if (nextCluster >= clusters.size) return null
blocks = readCluster(clusters[nextCluster++])
blockIndex = 0
}
val block = blocks[blockIndex++]
if (block.timeMs >= skipBeforeMs) return block
}
}
fun close() = input.close()
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
private fun blockSpanMs(block: MkaBlock): Long {
if (block.durationMs >= 0) return block.durationMs
val track = track(block.track) ?: return 0
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
}
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
private class Header(val id: Int, val size: Long, val length: Int)
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
if (at >= end) return null
val first = bytes[at].toInt() and 0xff
val idLength = first.countLeadingZeroBits() - 24 + 1
if (idLength > 4 || at + idLength >= end) return null
var id = 0
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
val sizeAt = at + idLength
val sizeFirst = bytes[sizeAt].toInt() and 0xff
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
var allOnes = size == (0xff shr sizeLength).toLong()
for (index in 1 until sizeLength) {
val byte = bytes[sizeAt + index].toInt() and 0xff
if (byte != 0xff) allOnes = false
size = (size shl 8) or byte.toLong()
}
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
}
private fun readAt(position: Long, length: Int): ByteArray {
val buffer = ByteArray(length)
val read = input.readAt(position, buffer, 0, length)
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
}
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
var value = 0L
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
return value
}
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
8 -> Double.fromBits(uint(bytes, at, 8))
else -> 0.0
}
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
var at = start
while (true) {
val child = header(bytes, at, end) ?: return
val payload = at + child.length
if (child.size < 0 || child.size > end - payload) return
visit(child.id, payload, child.size.toInt())
at = payload + child.size.toInt()
}
}
private fun scan() {
val length = input.length
val start = readAt(0, 64)
val ebml = header(start, 0, start.size)
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
var position = ebml.length + ebml.size
val segment = readAt(position, 12).let { header(it, 0, it.size) }
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
position += segment.length
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
while (position < end) {
val head = readAt(position, 12)
val element = header(head, 0, head.size) ?: break
val payload = position + element.length
if (element.id == MatroskaIds.CLUSTER) {
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
// A recording cut short ends in a partial cluster; its complete blocks still count.
val available = minOf(element.size, end - payload)
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
} else if (element.size < 0) {
break
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
when (element.id) {
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
}
}
if (element.size < 0) break
position = payload + element.size
}
}
private fun clusterTime(position: Long, size: Int): Long {
val head = readAt(position, minOf(size, 16))
val first = header(head, 0, head.size)
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
return uint(head, first.length, first.size.toInt())
}
val whole = readAt(position, size)
var time = 0L
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
return time
}
private fun parseInfo(bytes: ByteArray) {
var duration = 0.0
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
MatroskaIds.DURATION -> duration = float(bytes, at, size)
}
}
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
}
private fun parseTracks(bytes: ByteArray) {
trackList.clear()
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.TRACK_ENTRY) {
var number = 0
var type = 0
var codec = ""
var name = ""
var rate = 8000.0
var channels = 1
var depth = 0
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
when (audio) {
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
}
}
}
}
val audio = type == MkaWriter.TRACK_AUDIO
trackList.add(
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
)
}
}
}
private fun parseAttachments(bytes: ByteArray) {
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.ATTACHED_FILE) {
var name = ""
var mime = ""
var data = ByteArray(0)
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
}
}
attachmentList.add(MkaAttachment(name, mime, data))
}
}
}
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
val bytes = readAt(cluster.position, cluster.size)
val out = ArrayList<MkaBlock>()
var clusterRaw = cluster.rawTime
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
MatroskaIds.BLOCK_GROUP -> {
var blockAt = -1
var blockSize = 0
var duration = -1L
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.BLOCK -> {
blockAt = fieldAt
blockSize = fieldSize
}
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
}
}
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
}
}
}
return out
}
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
if (size < 4) return null
val first = bytes[at].toInt() and 0xff
val trackLength = first.countLeadingZeroBits() - 24 + 1
if (trackLength > 2 || size < trackLength + 3) return null
var track = first and (0xff shr trackLength)
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
val head = at + trackLength
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
val flags = bytes[head + 2].toInt() and 0xff
if (flags and 0x06 != 0) return null
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
}
}
@@ -0,0 +1,93 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/**
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
* contract is the header of g2flash patches/mic_control.c).
*/
object MicArrayProtocol {
const val VERSION = 1
const val HEADER_BYTES = 21
const val OP_CONFIGURE = 1
const val OP_STOP = 3
const val OP_RENEW = 4
/** How long an armed temple streams without a CONFIGURE or RENEW. */
const val LEASE_MS = 90_000L
const val FLAG_TRUNCATED = 0x80
const val CODEC_RAW = 1
const val FORMAT_PCM16 = 0
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
class Frame(
/** Low 16 bits of the temple's frame counter. */
@JvmField val sequence: Int,
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
@JvmField val deviceTickMs: Long,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
@JvmField val format: Int,
@JvmField val codec: Int,
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
@JvmField val angleDegrees: Int,
@JvmField val ssr: Int,
/** The firmware cut the payload short of the samples it captured. */
@JvmField val truncated: Boolean,
@JvmField val payloadOffset: Int,
@JvmField val payloadLength: Int,
)
/** A control record with no payload: ['M','C', version, op]. */
@JvmStatic
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
@JvmStatic
fun isConfigure(record: ByteArray?): Boolean =
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
@JvmStatic
fun decodeFrame(data: ByteArray): Frame? {
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
return null
}
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
return Frame(
sequence = u16(data, 4),
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
channels = data[10].toInt() and 0xff,
sampleRate = u16(data, 11) * 100,
format = data[13].toInt() and 0xff,
codec = data[14].toInt() and 0xff,
angleDegrees = u16(data, 15).toShort().toInt(),
ssr = u16(data, 17).toShort().toInt(),
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
payloadOffset = HEADER_BYTES,
payloadLength = length,
)
}
/**
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
* payload isn't raw 16-bit PCM or holds none.
*/
@JvmStatic
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
val frames = frame.payloadLength / (2 * frame.channels)
if (frames <= 0) return null
val samples = ShortArray(frames * frame.channels)
var at = frame.payloadOffset
for (index in samples.indices) {
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
at += 2
}
return samples
}
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
}
@@ -0,0 +1,484 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
import kotlin.math.abs
/** Which microphones a chunk of capture audio came from. */
object MicStream {
/** One mixed channel: the glasses' stock stream, or the phone's own microphone. */
const val MONO = 0
/** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */
const val LEFT = 1
const val RIGHT = 2
/** The mono feed derived from the temples for speech recognition: recorded, never replayed. */
const val MIX = 3
const val COUNT = 4
/** The stream's name in frame metadata. */
@JvmStatic
fun key(stream: Int): String = when (stream) {
MONO -> "mic"
LEFT -> "left"
RIGHT -> "right"
else -> "mix"
}
/** The name of the stream's audio tracks in a recording. */
@JvmStatic
fun trackName(stream: Int): String = when (stream) {
MONO -> "mic"
LEFT -> "left temple"
RIGHT -> "right temple"
else -> "mix"
}
/** The stream a recording's track or metadata names, or -1. */
@JvmStatic
fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1
}
/**
* One chunk of microphone audio as a capture source delivered it (about 50 ms), before any
* processing. Live captures, recordings and replays all trade in these.
*/
class MicChunk(
@JvmField val stream: Int,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
/** Interleaved PCM16; the first [frames] sample frames are the chunk. */
@JvmField val pcm: ShortArray,
@JvmField val frames: Int,
/** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */
@JvmField val timeMs: Long,
) {
/** Whether the source reported a direction of arrival with the chunk. */
@JvmField var hasDirection = false
@JvmField var angleDegrees = 0
@JvmField var ssr = 0
/** The source's packet counter, or -1 when it has none. */
@JvmField var sequence = -1
/** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */
@JvmField var deviceTickMs = -1L
/** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */
@JvmField var clockStream = stream
/** The source cut the chunk short of what it captured. */
@JvmField var truncated = false
/** Sample frames known to be missing between the previous chunk of this stream and this one. */
@JvmField var lostFramesBefore = 0
val durationMs: Long get() = frames * 1000L / sampleRate
}
/**
* Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for
* array processing (beamforming, noise cancellation): replays feed it the recorded chunks with
* the times they first arrived at, so an implementation that decides from the chunks alone
* behaves the same on a replay as it did live.
*/
interface MicArrayMixer {
/** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */
fun mix(chunk: MicChunk): MicChunk?
fun reset()
}
/**
* The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix).
* The temples sample on independent clocks, so their audio is never summed. The left temple
* feeds whenever it is streaming; the right one stands in while it is not.
*/
class SingleTempleMixer : MicArrayMixer {
companion object {
/** A temple that has delivered nothing for this long no longer counts as streaming. */
const val SILENT_MS = 300L
}
private var leftAtMs = Long.MIN_VALUE
override fun reset() {
leftAtMs = Long.MIN_VALUE
}
override fun mix(chunk: MicChunk): MicChunk? {
if (chunk.stream == MicStream.LEFT) {
leftAtMs = chunk.timeMs
} else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) {
return null
}
val channels = chunk.channels
val mono = ShortArray(chunk.frames)
for (frame in 0 until chunk.frames) {
var sum = 0
for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel]
mono[frame] = (sum / channels).toShort()
}
val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs)
out.hasDirection = chunk.hasDirection
out.angleDegrees = chunk.angleDegrees
out.ssr = chunk.ssr
out.deviceTickMs = chunk.deviceTickMs
out.clockStream = chunk.stream
out.truncated = chunk.truncated
return out
}
}
/**
* Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and
* layout as each first appears, plus a "frames" text track with one JSON object per chunk that
* has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md.
*
* Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick,
* or the count of samples from a source without one) says how far apart its chunks were
* captured; the least-delayed recent arrival places that clock on the file's timeline. So a
* chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap.
*
* Safe to share between the capture thread and whoever finishes the recording.
*/
class MicRecordingWriter(
output: SeekableOutput,
/** A JSON document describing the capture, stored as the file's "capture.json" attachment. */
configJson: String?,
/** When the recording began (Unix epoch ms). */
startedAtMs: Long,
platform: ProtocolPlatform = protocolPlatform(),
private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) },
) {
companion object {
const val CONFIG_ATTACHMENT = "capture.json"
const val META_TRACK = "frames"
/** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */
const val DISCONTINUITY_MS = 5000L
private const val WINDOW_BUCKETS = 8
private const val BUCKET_MS = 1000L
}
/**
* Places one track's chunks on the file's timeline. `source` time is the stream's own clock
* since this clock was anchored; the offset from it to arrival time is taken as the least
* seen over the last few seconds, which is the least-delayed delivery.
*/
private class TrackClock(val number: Int, val sampleRate: Int) {
private var anchored = false
private var clockStream = -1
private var tickBase = -1L
private var lastTick = -1L
private var frames = 0L
private val minima = LongArray(WINDOW_BUCKETS)
private var bucket = 0
private var bucketStartMs = 0L
var endMs = Long.MIN_VALUE
private set
fun reset() {
anchored = false
}
private fun offset(): Long = minima.min()
fun stamp(chunk: MicChunk): Long {
val arrival = chunk.timeMs
val ticked = chunk.deviceTickMs >= 0
if (anchored) {
// Another clock (the feed moved to the other temple), or this one restarted.
if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false
else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false
}
var anchorNow = !anchored
var source = 0L
if (anchored) {
frames += chunk.lostFramesBefore
source = sourceTime(chunk)
if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true
}
if (anchorNow) {
anchored = true
clockStream = chunk.clockStream
tickBase = if (ticked) chunk.deviceTickMs else -1L
frames = 0
minima.fill(Long.MAX_VALUE)
bucket = 0
bucketStartMs = arrival
source = 0
}
if (arrival - bucketStartMs >= BUCKET_MS) {
bucket = (bucket + 1) % WINDOW_BUCKETS
minima[bucket] = Long.MAX_VALUE
bucketStartMs = arrival
}
if (arrival - source < minima[bucket]) minima[bucket] = arrival - source
// Never before the end of the track's previous block.
val time = maxOf(source + offset(), endMs)
frames += chunk.frames
lastTick = chunk.deviceTickMs
endMs = time + chunk.durationMs
return time
}
private fun sourceTime(chunk: MicChunk): Long =
if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate
}
private val lock = platform.createLock()
private val writer: MkaWriter
private val clocks = HashMap<Long, TrackClock>()
private var metaTrack = 0
private var originMs = Long.MIN_VALUE
private var lastTimeMs = 0L
private var closed = false
/** The recording stopped early: the file could not be written. */
@Volatile
var failed = false
private set
init {
val attachments = if (configJson.isNullOrEmpty()) emptyList()
else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray()))
writer = MkaWriter(output, null, startedAtMs, attachments)
}
/** Length of the recording so far, in ms. */
val durationMs: Long get() = lock.withLock { writer.durationMs }
/** Appends [chunk] to its stream's track, adding the track on its first chunk. */
fun write(chunk: MicChunk) {
if (chunk.frames <= 0) return
lock.withLock {
if (closed) return
try {
val clock = clockFor(chunk) ?: return
if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs
val time = clock.stamp(chunk) - originMs
val meta = metadata(chunk)
if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta)
val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels)
writer.writeAudio(clock.number, time, bytes, 0, bytes.size)
lastTimeMs = maxOf(lastTimeMs, time)
} catch (error: Throwable) {
fail(error)
}
}
}
/**
* Records a JSON object as an event at the recording's current position (a capture starting
* over with another configuration, say), and has every stream's clock placed afresh by its
* next chunk: what follows an event is not continuous with what came before.
*/
fun event(json: String) {
lock.withLock {
if (closed) return
try {
for (clock in clocks.values) clock.reset()
if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json)
} catch (error: Throwable) {
fail(error)
}
}
}
/** Completes the file. Returns the recording's length in ms. */
fun finish(): Long {
lock.withLock {
if (!closed) {
closed = true
try {
writer.finish()
} catch (error: Throwable) {
failed = true
logWarn("finishing the recording failed: ${error.message}")
}
}
return writer.durationMs
}
}
private fun fail(error: Throwable) {
failed = true
closed = true
logWarn("recording stopped: ${error.message}")
try {
writer.finish()
} catch (ignored: Throwable) {
// The file is whatever made it to disk.
}
}
private fun ensureMetaTrack(): Boolean {
if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK)
return metaTrack > 0
}
private fun clockFor(chunk: MicChunk): TrackClock? {
val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong()
clocks[key]?.let { return it }
// The frames track goes in with the first audio track, so it is there for that track's first chunk.
ensureMetaTrack()
// The mixed streams are what a player should pick; a temple's raw pair is for analysis.
val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX
val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred)
if (number <= 0) {
logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio")
return null
}
return TrackClock(number, chunk.sampleRate).also { clocks[key] = it }
}
/** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */
private fun metadata(chunk: MicChunk): String? {
if (chunk.stream == MicStream.MIX) return null
if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null
val text = StringBuilder(96)
text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs)
if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence)
if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs)
if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr)
if (chunk.truncated) text.append(",\"trunc\":true")
if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore)
return text.append('}').toString()
}
}
/**
* Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO]
* chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the
* metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A
* Matroska file from elsewhere plays its first PCM16 track as the mono microphone.
*
* Throws IllegalArgumentException when the file has no audio a replay can use.
*/
class MicRecordingReader(input: SeekableInput) {
companion object {
const val SAMPLE_RATE = 16000
/** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */
const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20
}
private class Meta(val timeMs: Long, val fields: Map<String, Any?>)
private val reader = MkaReader(input)
private val streams = HashMap<Int, Int>()
private var metaTrack = -1
private val pendingMeta = arrayOfNulls<Meta>(MicStream.COUNT)
private val queued = ArrayDeque<MicChunk>()
val durationMs: Long get() = reader.durationMs
/** The capture's "capture.json" attachment, or null. */
val configJson: String? =
reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString()
/** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */
val description: String
init {
try {
val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 }
for (track in audio) {
val stream = MicStream.named(track.name)
if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream
}
if (streams.isEmpty()) {
// Not one of ours: its first usable track stands in for the microphone.
val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull()
require(first != null) { "no 16-bit PCM audio track" }
streams[first.number] = MicStream.MONO
}
for (number in streams.keys) {
val rate = reader.track(number)!!.sampleRate
require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" }
}
metaTrack = reader.tracks.firstOrNull {
it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK
}?.number ?: -1
description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) ->
val channels = reader.track(number)!!.channels
MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else ""
}
} catch (error: Throwable) {
reader.close()
throw error
}
}
/** Continue from the first chunk at or after [timeMs]. */
fun seekTo(timeMs: Long) {
queued.clear()
pendingMeta.fill(null)
reader.seekTo(timeMs)
}
/** The next chunk in file order, or null at the end. */
fun next(): MicChunk? {
while (queued.isEmpty()) {
val block = reader.next() ?: return null
if (block.track == metaTrack) {
noteMeta(block)
continue
}
val stream = streams[block.track] ?: continue
val track = reader.track(block.track) ?: continue
queueChunks(block, stream, track)
}
return queued.removeFirst()
}
fun close() = reader.close()
private fun noteMeta(block: MkaBlock) {
try {
val fields = Json.parseObject(block.data.decodeToString())
val stream = MicStream.named(fields["s"].asString() ?: return)
if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields)
} catch (ignored: Throwable) {
// An event or a frame we can't read: the audio plays without it.
}
}
private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) {
val sourceChannels = track.channels
val totalFrames = block.data.size / (2 * sourceChannels)
if (totalFrames <= 0) return
// The mixed microphone is one channel however a foreign file stored it.
val channels = if (stream == MicStream.MONO) 1 else sourceChannels
val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs }
pendingMeta[stream] = null
var start = 0
while (start < totalFrames) {
val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start)
val pcm = ShortArray(frames * channels)
for (frame in 0 until frames) {
val at = (start + frame) * sourceChannels * 2
if (channels == sourceChannels) {
for (channel in 0 until channels) {
pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
}
} else {
var sum = 0
for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
pcm[frame] = (sum / sourceChannels).toShort()
}
}
val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate)
if (meta != null && start == 0) applyMeta(chunk, meta.fields)
queued.addLast(chunk)
start += frames
}
}
private fun applyMeta(chunk: MicChunk, fields: Map<String, Any?>) {
chunk.sequence = fields["seq"].asInt() ?: -1
chunk.deviceTickMs = fields["tick"].asLong() ?: -1L
val angle = fields["angle"].asInt()
if (angle != null) {
chunk.hasDirection = true
chunk.angleDegrees = angle
chunk.ssr = fields["ssr"].asInt() ?: 0
}
chunk.truncated = fields["trunc"].asBoolean() ?: false
chunk.lostFramesBefore = fields["lost"].asInt() ?: 0
}
}
@@ -35,6 +35,12 @@ interface VoiceCaptureHost {
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
/** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */
fun micArrayLink(): MicArrayLink? = null
/** Opens a recording to replay; throws when it cannot be read. */
fun openRecording(path: String): SeekableInput = openFileInput(path)
}
/**
@@ -44,6 +50,12 @@ interface VoiceCaptureHost {
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
* worker thread (started through the shared [startThread]); listener callbacks go through the
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
*
* Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the
* phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording
* played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go
* to the recorder ([setRecorder]) as they came, before any processing, which is what lets a
* replay of the recording run the same processing again.
*/
class VoiceCaptureSession(
private val host: VoiceCaptureHost,
@@ -95,6 +107,19 @@ class VoiceCaptureSession(
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
/** Samples per G2 packet; the decode buffer the worker reuses. */
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
// Mic array upkeep: the temples stop streaming unless their lease is renewed, and a
// temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again.
const val ARRAY_CHECK_MS = 1000L
const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3
const val ARRAY_TEMPLE_SILENT_MS = 5000L
const val ARRAY_WAITING_MS = 2000L
const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..."
// A replay skips stretches of the recording with no audio longer than this
// (a recording made across a disconnect) instead of sitting through them.
const val REPLAY_MAX_GAP_MS = 2000L
const val REPLAY_POSITION_INTERVAL_MS = 1000L
const val REPLAY_WAIT_SLICE_MS = 20L
const val REPLAY_FINISHED_STATUS = "Replay finished."
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
@JvmStatic
@@ -120,6 +145,8 @@ class VoiceCaptureSession(
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
}
private enum class Source { G2, PHONE, ARRAY, REPLAY }
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
val arm: String = arm ?: "?"
}
@@ -137,11 +164,33 @@ class VoiceCaptureSession(
@Volatile
private var workerThreadName: String? = null
// Capture from the phone's own microphone instead of the G2 over BLE
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
// (preview-only mode). Latched into activeSource at start() (under `lock`)
// so a mid-session setter call can't switch pipelines underneath the worker.
@Volatile
private var usePhoneMic = false
private var activePhoneMic = false
// What the next capture reads (see start()); latched into activeSource and its
// companions under `lock`, like usePhoneMic.
@Volatile
private var micArrayConfigure: ByteArray? = null
@Volatile
private var replayPath: String? = null
@Volatile
private var replayStartMs = 0L
private var activeSource = Source.G2
private var activeConfigure: ByteArray? = null
private var activeReplayPath: String? = null
private var activeReplayStartMs = 0L
// Replay controls, picked up by the worker between chunks.
@Volatile
private var replaySeekMs = -1L
@Volatile
private var replayPaused = false
@Volatile
private var recorder: MicRecordingWriter? = null
@Volatile
private var mixer: MicArrayMixer = SingleTempleMixer()
private var lastMissingPackets = 0L
private var arrayRateWarned = false
private var mode = Mode.CLOUD
@Volatile
private var onboardModelKind = VoiceModelKind.MOONSHINE
@@ -193,6 +242,47 @@ class VoiceCaptureSession(
this.usePhoneMic = usePhoneMic
}
/**
* Capture the next glasses session from the temples' own microphones instead of the stock
* mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples
* (it decides how many microphones each one streams). Null goes back to the mixed stream.
*/
fun setMicArray(configureRecord: ByteArray?) {
this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null
}
/** How the temples' chunks become the mono feed that is transcribed. */
fun setMicArrayMixer(mixer: MicArrayMixer) {
this.mixer = mixer
}
/**
* Have the next capture play the recording at [path] from [startMs] in real time, in place
* of any microphone; null goes back to the microphones. A replay needs no glasses.
*/
fun setReplay(path: String?, startMs: Long) {
this.replayPath = path
this.replayStartMs = maxOf(0L, startMs)
}
/** Continue the running replay from [positionMs] into its recording. */
fun seekReplay(positionMs: Long) {
this.replaySeekMs = maxOf(0L, positionMs)
}
/** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */
fun setReplayPaused(paused: Boolean) {
this.replayPaused = paused
}
/**
* Where live captures' chunks are recorded as they arrive, or null. Takes effect at once,
* also mid-capture; the caller finishes the recording. A replay is never recorded.
*/
fun setRecorder(recorder: MicRecordingWriter?) {
this.recorder = recorder
}
fun setOnboardModelKind(kind: String?) {
this.onboardModelKind = parseModelKind(kind)
}
@@ -258,13 +348,26 @@ class VoiceCaptureSession(
emitStopped(captureId)
return
}
if (!usePhoneMic && !host.isG2SessionReady()) {
val path = replayPath
val configure = micArrayConfigure
val source = when {
path != null -> Source.REPLAY
usePhoneMic -> Source.PHONE
configure != null -> Source.ARRAY
else -> Source.G2
}
if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) {
emitStatus("Voice control needs an active G2 connection.")
emitStopped(captureId)
return
}
mode = parseMode(requestedMode)
activePhoneMic = usePhoneMic
activeSource = source
activeConfigure = configure
activeReplayPath = path
activeReplayStartMs = replayStartMs
replaySeekMs = -1L
replayPaused = false
started = true
audioStarted = false
val finished = Latch(1, platform)
@@ -281,21 +384,22 @@ class VoiceCaptureSession(
*/
fun isCapturing(): Boolean {
var audioUp = false
var phoneMic = false
var source = Source.G2
lock.withLock {
if (!started) {
return false
}
audioUp = audioStarted
phoneMic = activePhoneMic
source = activeSource
}
if (!audioUp) {
// The worker is still bringing the mic up; report it as running so
// a concurrent request shares it instead of restarting it.
return true
}
if (phoneMic) {
// The phone mic has no session to lose the enable to; it runs until stop().
if (source == Source.PHONE || source == Source.REPLAY) {
// Neither has a glasses session to lose the enable to: the phone mic runs
// until stop(), a replay until its recording ends.
return true
}
return host.isG2AudioCaptureActive()
@@ -317,7 +421,7 @@ class VoiceCaptureSession(
finished = workerFinished
workerName = workerThreadName
}
stopG2Audio()
stopSource()
audioQueue.wake()
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
finished.await(1500)
@@ -356,28 +460,62 @@ class VoiceCaptureSession(
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
verifyCount = 0
if (activePhoneMic) {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
mixer.reset()
lastMissingPackets = 0
val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..."
when (activeSource) {
Source.PHONE -> {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
try {
processPhoneAudio(record)
} finally {
record.close()
}
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
try {
processPhoneAudio(record)
} finally {
record.close()
Source.REPLAY -> {
val reader = try {
MicRecordingReader(host.openRecording(activeReplayPath!!))
} catch (error: Throwable) {
emitStatus("Could not open the recording: " + error.message)
emitReplay("error", 0, 0)
return
}
try {
lock.withLock { audioStarted = true }
logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s")
emitStatus(listening)
processReplay(reader)
} finally {
reader.close()
}
}
} else {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
Source.ARRAY -> {
val link = host.micArrayLink()
val configure = activeConfigure
if (link == null || configure == null || !startArray(link, configure)) {
emitStatus("Could not start the glasses microphone array.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
processArray(link, configure, listening)
}
Source.G2 -> {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
processG2Audio()
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
processG2Audio()
}
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
@@ -392,7 +530,7 @@ class VoiceCaptureSession(
logError("Voice control failed", error)
emitStatus("Voice control failed: " + error.message)
} finally {
stopG2Audio()
stopSource()
releaseRecognizer()
releaseDecoder()
lock.withLock {
@@ -432,23 +570,214 @@ class VoiceCaptureSession(
continue
}
decodedSamples += count.toLong()
// The beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker for
// every consumer, recognition included); the spectral noise
// suppressor then cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
val angleDegrees = currentDecoder.lastAngleDegrees
val ssr = currentDecoder.lastSsr
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr)
maybeEmitAudioStats(false)
continue
}
processPcmChunk(pcm, count, angleDegrees, ssr, currentDecoder.hasFrameMeta)
val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs)
chunk.hasDirection = currentDecoder.hasFrameMeta
chunk.angleDegrees = currentDecoder.lastAngleDegrees
chunk.ssr = currentDecoder.lastSsr
// Packets the decoder found missing ahead of this one: a gap in a recording.
val missing = currentDecoder.missingPackets
chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt()
lastMissingPackets = missing
handleMonoChunk(chunk)
maybeEmitAudioStats(false)
}
}
/**
* A mixed-microphone chunk from any source: recorded as it came, then on to processing
* unless the beam filter drops it. The filter drops chunks whose firmware
* direction-of-arrival falls outside the listening wedge (isolating the aimed talker for
* every consumer, recognition included); the spectral noise suppressor then cleans what
* remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification.
*/
private fun handleMonoChunk(chunk: MicChunk) {
recordChunk(chunk)
if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) {
emitFrameMeta(chunk.angleDegrees, chunk.ssr)
return
}
processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection)
}
/**
* A chunk of one temple's own microphones: recorded as it came, then mixed into the mono
* feed, which is recorded too (as what was transcribed) and processed like a mixed chunk.
* The beam filter does not apply: a temple's bearing is along its own pair, not the
* head-frame direction the filter is aimed in.
*/
private fun handleArrayChunk(chunk: MicChunk) {
recordChunk(chunk)
val feed = mixer.mix(chunk) ?: return
recordChunk(feed)
if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) {
if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz")
arrayRateWarned = true
return
}
processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection)
}
private fun recordChunk(chunk: MicChunk) {
if (activeSource != Source.REPLAY) recorder?.write(chunk)
}
private fun handleChunk(chunk: MicChunk) {
if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk)
}
private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean {
resetAudioStats()
audioQueue.clear()
if (!startArrayForwarding(link)) return false
return link.sendControl(configure, "configure", true, true)
}
private fun startArrayForwarding(link: MicArrayLink): Boolean =
link.startForwarding(object : FaceclawAudioPacketListener {
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
queueAudioPacket(data, arm, arrivalMs)
}
})
/**
* Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream.
* Between frames it keeps the streams alive: renews the temples' lease, configures a temple
* again when it has gone quiet, and takes forwarding back when the glasses session dropped it.
*/
private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) {
val startedAt = platform.elapsedRealtimeMs()
var checkedAt = startedAt
var renewedAt = startedAt
// Left, right.
val armedAt = longArrayOf(startedAt, startedAt)
val frameAt = longArrayOf(startedAt, startedAt)
var waiting = false
while (started) {
val packet = takeAudioPacket()
val now = platform.elapsedRealtimeMs()
if (packet != null) {
val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1
val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null
val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null
// Anything else is a stock-format packet from before arming took effect.
if (frame != null && pcm != null) {
frameAt[side] = now
val frames = pcm.size / frame.channels
decodedSamples += frames.toLong()
val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT
val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs)
chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0
chunk.angleDegrees = frame.angleDegrees
chunk.ssr = frame.ssr
chunk.sequence = frame.sequence
chunk.deviceTickMs = frame.deviceTickMs
chunk.truncated = frame.truncated
handleArrayChunk(chunk)
}
}
if (now - checkedAt >= ARRAY_CHECK_MS) {
checkedAt = now
if (!link.isForwarding()) {
if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) {
logInfo("mic array forwarding re-established")
armedAt.fill(now)
renewedAt = now
}
} else {
if (now - renewedAt >= ARRAY_RENEW_MS) {
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true)
renewedAt = now
}
for (side in 0..1) {
if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue
logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again")
link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0)
armedAt[side] = now
}
}
val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS
if (silent != waiting) {
waiting = silent
emitStatus(if (silent) ARRAY_WAITING_STATUS else listening)
}
}
maybeEmitAudioStats(false)
}
}
/**
* Replay loop: hands the recording's chunks to the pipeline at the pace they were captured,
* so recognition (its partials, a cloud provider's stream) sees what a live capture would.
* Jumps and pauses asked for from outside take effect between chunks.
*/
private fun processReplay(reader: MicRecordingReader) {
val duration = reader.durationMs
var position = minOf(activeReplayStartMs, duration)
reader.seekTo(position)
emitReplay("start", position, duration)
var reportedAt = position
// Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap.
var anchorWall = -1L
var anchorFile = 0L
var pending: MicChunk? = null
while (started) {
val seek = replaySeekMs
if (seek >= 0) {
replaySeekMs = -1L
// What played so far is a finished utterance; the rest starts clean.
finishReplayedUtterance()
position = minOf(seek, duration)
reader.seekTo(position)
pending = null
anchorWall = -1L
reportedAt = position
emitReplay("seek", position, duration)
}
if (replayPaused) {
anchorWall = -1L
sleepMs(REPLAY_WAIT_SLICE_MS)
continue
}
val chunk = pending ?: reader.next()
if (chunk == null) {
emitReplay("end", duration, duration)
emitStatus(REPLAY_FINISHED_STATUS)
return
}
pending = chunk
val now = platform.elapsedRealtimeMs()
if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) {
anchorWall = now
anchorFile = chunk.timeMs
}
val wait = anchorWall + (chunk.timeMs - anchorFile) - now
if (wait > 0) {
// Short slices, so a stop, jump or pause is noticed promptly.
sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS))
continue
}
pending = null
position = chunk.timeMs
decodedSamples += chunk.frames.toLong()
handleChunk(chunk)
if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) {
reportedAt = position
emitReplay("position", position, duration)
}
}
}
/** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */
private fun finishReplayedUtterance() {
mixer.reset()
endpointDetector.reset()
suppressor?.reset()
if (mode != Mode.ONBOARD) return
decodeTranscript(true)
resetTranscriptState()
lastTranscript = ""
}
/**
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
* decode and the beam filter. hasFrameMeta is false for the phone mic,
@@ -495,7 +824,7 @@ class VoiceCaptureSession(
continue
}
decodedSamples += read.toLong()
processPcmChunk(pcm, read, 0, 0, false)
handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs()))
}
}
@@ -681,6 +1010,11 @@ class VoiceCaptureSession(
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
}
private fun emitReplay(event: String, positionMs: Long, durationMs: Long) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) }
}
private fun emitSpeechEnd() {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onSpeechEnd() }
@@ -691,8 +1025,21 @@ class VoiceCaptureSession(
host.dispatcher.post { currentListener.onStopped(captureId) }
}
private fun stopG2Audio() {
host.g2AudioSource()?.stop()
/**
* Release what the running capture's source holds on the glasses. Runs on stop() and again
* as the worker winds down (it may have started the source after stop() looked), so
* everything here is safe to repeat.
*/
private fun stopSource() {
val source = lock.withLock { activeSource }
when (source) {
Source.G2 -> host.g2AudioSource()?.stop()
Source.ARRAY -> host.micArrayLink()?.let { link ->
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true)
link.stopForwarding()
}
Source.PHONE, Source.REPLAY -> {}
}
maybeEmitAudioStats(true)
}
@@ -711,7 +1058,7 @@ class VoiceCaptureSession(
if (!started || data == null) {
return
}
if ("L" != arm) {
if ("L" != arm && activeSource != Source.ARRAY) {
wrongArmPackets++
}
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
@@ -67,6 +67,24 @@ interface PacketAudioSource {
fun isActive(): Boolean
}
/**
* The custom firmware's mic_control channel on a glasses session: each temple streams its own
* microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed
* stream uses, while it holds a configuration that the phone keeps renewing.
*/
interface MicArrayLink {
/** Deliver the audio characteristic's packets to [listener]; false when no session is up. */
fun startForwarding(listener: FaceclawAudioPacketListener): Boolean
fun stopForwarding()
/** Whether forwarding still holds: it lives in the glasses session, which can drop it. */
fun isForwarding(): Boolean
/** Queue a control record for the chosen temples; false when nothing was queued. */
fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean
}
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
interface PcmAudioSource {
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
@@ -3,6 +3,8 @@ package com.faceclaw.app
/**
* Receives IMU (accelerometer) readings decoded from the glasses' sys-event stream. Registered on
* FaceclawBleCommunicator via addImuListener; callbacks are delivered on the Android main thread.
* x, y, z are in g, in the canonical frame of every glasses model (+x forward, +y left, +z up;
* see ImuOrientation).
*/
interface FaceclawImuListener {
fun onImuData(x: Double, y: Double, z: Double, eventSource: Int): Unit
@@ -0,0 +1,10 @@
package com.faceclaw.app
interface FaceclawKeyboardEventListener {
/**
* One CFW keyboard-bridge record (settings field 131) from the glasses: the raw
* ['K','B',ver,kind,...] bytes, possibly one fragment of a longer record. Decoded and
* reassembled on the TS side (app/g2/keyboard/protocol.ts). Delivered in arrival order.
*/
fun onKeyboardEvent(body: ByteArray?): Unit
}
@@ -0,0 +1,9 @@
package com.faceclaw.app
/**
* Events from FaceclawSshChannel, delivered on the thread that opened the
* channel. See native/ssh/faceclawssh.go (Listener) for the kinds and payloads.
*/
interface FaceclawSshListener {
fun onEvent(kind: String?, payload: String?): Unit
}
@@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener {
* speaker model could not be loaded.
*/
fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit
/**
* Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay):
* "start" once the file is open, "position" about once per second of it played, "seek" after
* a jump took effect (the transcript of what played before it has been emitted as final),
* "end" when the file ran out, and "error" when it could not be opened (the status says
* why). Positions are milliseconds into the file.
*/
fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit
}
@@ -12,6 +12,12 @@ import kotlin.jvm.JvmStatic
*/
class BleProtocol {
companion object {
/** The service holding [WRITE_CHAR_UUID] and [NOTIFY_CHAR_UUID]. */
const val PROTOCOL_SERVICE_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e5450"
/** The service holding [RENDER_NOTIFY_UUID]. */
const val RENDER_SERVICE_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e6450"
const val WRITE_CHAR_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e5401"
const val NOTIFY_CHAR_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e5402"
@@ -119,6 +125,19 @@ class BleProtocol {
*/
const val FACECLAW_ALS_REPORT_FIELD: Int = 105
/**
* CFW keyboard bridge (Faceclaw/39): the master temple hosts one BLE HID keyboard. The
* host writes ['K','B',ver,op,request,epoch,...] controls as settings field 130; results,
* pairing prompts and the keyboard's raw input reports come back as commandId=3 pushes
* carrying field 131 (['K','B',ver,kind,...], decoded in app/g2/keyboard/protocol.ts).
*/
const val FACECLAW_KEYBOARD_CONTROL_FIELD: Int = 130
const val FACECLAW_KEYBOARD_EVENT_FIELD: Int = 131
/** Fixed header of a field-131 record; fragments follow it. */
const val FACECLAW_KEYBOARD_EVENT_HEADER: Int = 24
const val FACECLAW_WAKE_OP_ACQUIRE: Int = 1
const val FACECLAW_WAKE_OP_RELEASE: Int = 2
@@ -497,22 +516,79 @@ class BleProtocol {
)
}
/** Stock HeadUpSetting: switch and angle are separate oneof writes (field 4 of device info). */
/** Compatibility helper: preserve the fork's explicit switch write. */
@JvmStatic
fun buildSetHeadUpWake(magic: Int, enabled: Boolean): ByteArray = concat(
CollectionUtils.listOf(
encodeVarintField(1, 1), encodeVarintField(2, magic),
encodeMessageField(3, encodeMessageField(4, encodeVarintField(1, if (enabled) 1 else 0))),
)
)
fun buildSetHeadUpWake(magic: Int, enabled: Boolean): ByteArray =
buildSetHeadUp(magic, HEAD_UP_FIELD_SWITCH, if (enabled) 1 else 0)
/** Compatibility helper: only the fork's angle API clamps to its 0..60 preference range. */
@JvmStatic
fun buildSetHeadUpAngle(magic: Int, angle: Int): ByteArray = concat(
CollectionUtils.listOf(
encodeVarintField(1, 1), encodeVarintField(2, magic),
encodeMessageField(3, encodeMessageField(4, encodeVarintField(2, angle.coerceIn(0, 60)))),
fun buildSetHeadUpAngle(magic: Int, angle: Int): ByteArray =
buildSetHeadUp(magic, HEAD_UP_FIELD_ANGLE, angle.coerceIn(0, 60))
// Field numbers from g2_setting.proto (package g2.g2_setting; see g2-kit-unofficial
// ble/gen/g2_setting_pb.ts), named after the schema fields.
/** G2SettingPackage.commandId */
private const val G2_SETTING_PACKAGE_COMMAND_ID: Int = 1
/** G2SettingPackage.magicRandom */
private const val G2_SETTING_PACKAGE_MAGIC_RANDOM: Int = 2
/** G2SettingPackage.deviceReceiveInfoFromApp */
private const val G2_SETTING_PACKAGE_DEVICE_RECEIVE_INFO_FROM_APP: Int = 3
/** G2SettingPackage.deviceReceiveRequestFromApp (the settings-read reply) */
private const val G2_SETTING_PACKAGE_DEVICE_RECEIVE_REQUEST_FROM_APP: Int = 4
/** g2_settingCommandId.DeviceReceiveInfo */
private const val G2_SETTING_COMMAND_DEVICE_RECEIVE_INFO: Int = 1
/** DeviceReceiveInfoFromAPP.deviceReceiveHeadUpSetting */
private const val DEVICE_RECEIVE_INFO_HEAD_UP_SETTING: Int = 4
/** DeviceReceiveRequestFromAPP.headUpSwitchRestored */
private const val DEVICE_RECEIVE_REQUEST_HEAD_UP_SWITCH_RESTORED: Int = 7
/** DeviceReceiveRequestFromAPP.headUpAngleRestored */
private const val DEVICE_RECEIVE_REQUEST_HEAD_UP_ANGLE_RESTORED: Int = 8
/** DeviceReceiveRequestFromAPP.headUpAngleCalibrationRestored */
private const val DEVICE_RECEIVE_REQUEST_HEAD_UP_CALIBRATION_RESTORED: Int = 9
/**
* One head-up (look-up wake) setting: DeviceReceiveInfoFromAPP.deviceReceiveHeadUpSetting,
* a DeviceReceive_Head_UP_Setting carrying one of headUpSwitch, headUpAngle,
* headUpCalibrationSwitch or headUpCalibration ([field] is one of the HEAD_UP_FIELD_*).
* The firmware treats that submessage as a oneof: setting_handle_head_up_setting
* (2.3.0.24, 0x46c39a) receives a single {field, value} and applies it, so the switch and
* the angle go out as separate packages. headUpCalibration is not stored from the
* phone; it is the pitch the glasses measured on their own calibration screen. The
* effective trigger pitch is angle + calibration, clamped to -90..90 and accepted only in
* 1..90 (SVC_Settings_HeadUpConfig 0x470580, DRV_IMUHUDParameterConfig). The angle is a
* uint32 on the wire but used as a signed int, so a negative offset is sent as its
* 32-bit two's complement.
*/
@JvmStatic
fun buildSetHeadUp(magic: Int, field: Int, value: Int): ByteArray {
var headUp: ByteArray = concat(CollectionUtils.listOf(encodeKey(field, 0), encodeUint32(value)))
return concat(
CollectionUtils.listOf(
encodeVarintField(G2_SETTING_PACKAGE_COMMAND_ID, G2_SETTING_COMMAND_DEVICE_RECEIVE_INFO),
encodeVarintField(G2_SETTING_PACKAGE_MAGIC_RANDOM, magic),
encodeMessageField(
G2_SETTING_PACKAGE_DEVICE_RECEIVE_INFO_FROM_APP,
encodeMessageField(DEVICE_RECEIVE_INFO_HEAD_UP_SETTING, headUp),
),
)
)
)
}
/** DeviceReceive_Head_UP_Setting.headUpSwitch */
const val HEAD_UP_FIELD_SWITCH: Int = 1
/** DeviceReceive_Head_UP_Setting.headUpAngle */
const val HEAD_UP_FIELD_ANGLE: Int = 2
/** Enable the firmware's wear detector (DeviceReceiveInfoFromAPP field 5). */
@JvmStatic
@@ -741,6 +817,42 @@ class BleProtocol {
return body
}
/**
* Wrap an already-encoded ['K','B',ver,op,...] keyboard control as a settings write
* carrying CFW field 130. Magic zero, like the other private controls: the answer is a
* field-131 record naming the control's request id, not the settings ACK.
*/
@JvmStatic
fun buildFaceclawKeyboardControl(record: ByteArray): ByteArray {
return concat(
CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, 0),
encodeBytesField(FACECLAW_KEYBOARD_CONTROL_FIELD, record),
)
)
}
/**
* Extract a CFW keyboard record (field 131) from a settings frame, or null when the frame
* carries none. Only the ['K','B'] prelude and the fixed header's length are checked
* here; the TS side validates the version and decodes the rest.
*/
@JvmStatic
fun parseFaceclawKeyboardEvent(pb: ByteArray?): ByteArray? {
if (pb == null) {
return null
}
val body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_KEYBOARD_EVENT_FIELD)
if (
body == null || body.size < FACECLAW_KEYBOARD_EVENT_HEADER ||
body[0].toInt() != 75 || body[1].toInt() != 66
) {
return null
}
return body
}
/**
* Return the raw CFW ambient-light report from a sid-0x09 frame, or null when this is an
* ordinary settings frame. Only the 65.toByte(),76.toByte(),version prelude is validated
@@ -922,6 +1034,26 @@ class BleProtocol {
return BatterySnapshot(battery, charging, silentMode)
}
/**
* The head-up settings in a settings READ ack: deviceReceiveRequestFromApp (field 4)
* headUpSwitchRestored=7, headUpAngleRestored=8, headUpAngleCalibrationRestored=9.
* Returns null when the ack has no request submessage. Absent fields (proto3 drops
* zeros) read as 0; angle and calibration are signed ints carried as uint32.
*/
@JvmStatic
fun parseSettingsHeadUp(pb: ByteArray): IntArray? {
var root: ByteArray = stripTrailingCrc(pb)
var request: ByteArray? = readFieldBytes(root, G2_SETTING_PACKAGE_DEVICE_RECEIVE_REQUEST_FROM_APP)
if ((request == null)) {
return null
}
return intArrayOf(
readVarintFieldValue(request, DEVICE_RECEIVE_REQUEST_HEAD_UP_SWITCH_RESTORED, 0),
readVarintFieldValue(request, DEVICE_RECEIVE_REQUEST_HEAD_UP_ANGLE_RESTORED, 0),
readVarintFieldValue(request, DEVICE_RECEIVE_REQUEST_HEAD_UP_CALIBRATION_RESTORED, 0),
)
}
/** GlassesCaseDataPackage{commandId=CASE_INFO, magicRandom, caseInfo={}}, as the Even app sends it. */
@JvmStatic
fun buildCaseInfoQuery(magic: Int): ByteArray {
@@ -1106,6 +1238,18 @@ class BleProtocol {
return out.toByteArray()
}
/** A varint of the value's 32-bit two's complement (encodeVarint stops short on negatives). */
private fun encodeUint32(value: Int): ByteArray {
var out: ByteSink = ByteSink()
var v: Long = (value.toLong() and 0xffffffffL)
while ((v >= 0x80L)) {
out.write((((v and 0x7fL).toInt()) or 0x80))
v = (v ushr 7)
}
out.write(v.toInt())
return out.toByteArray()
}
private fun crcBytesLe(data: ByteArray): ByteArray {
var crc: Int = 0xffff
for (b in data) {
@@ -188,14 +188,31 @@ class MessageBuilder {
)
}
fun headUpControl(leftArm: Boolean, enabled: Boolean?, angle: Int): OutboundMessage {
/** Fork preference API: retain per-arm targeting and the bounded preference angle. */
fun headUpControl(leftArm: Boolean, enabled: Boolean?, angle: Int): OutboundMessage =
headUpSetting(
if (enabled == null) BleProtocol.HEAD_UP_FIELD_ANGLE else BleProtocol.HEAD_UP_FIELD_SWITCH,
if (enabled == null) angle.coerceIn(0, 60) else if (enabled) 1 else 0,
leftArm,
if (enabled == null) "head-up angle=$angle" else "head-up wake=$enabled",
)
/** One head-up setting field, including signed calibration-relative angles. */
fun setHeadUp(field: Int, value: Int): OutboundMessage =
headUpSetting(field, value, false, "head-up field $field=$value")
internal fun headUpSetting(field: Int, value: Int, leftArm: Boolean, label: String): OutboundMessage {
val magic = magicPool.allocate()
return OutboundMessage(
"head-up-control", if (enabled == null) "head-up angle=$angle" else "head-up wake=$enabled",
BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, magic,
if (enabled == null) BleProtocol.buildSetHeadUpAngle(magic, angle)
else BleProtocol.buildSetHeadUpWake(magic, enabled),
ACK_TIMEOUT_MS, -1, leftArm,
"head-up-control",
label,
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
magic,
BleProtocol.buildSetHeadUp(magic, field, value),
ACK_TIMEOUT_MS,
-1,
leftArm,
)
}
@@ -332,6 +349,24 @@ class MessageBuilder {
)
}
/**
* CFW keyboard control (settings field 130). Always the right arm: only the master temple
* acts on it, and a copy sent down each arm would run the control twice.
*/
fun faceclawKeyboardControl(record: ByteArray, label: String): OutboundMessage {
return OutboundMessage(
"keyboard-control",
"keyboard control $label",
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
0,
BleProtocol.buildFaceclawKeyboardControl(record),
ACK_TIMEOUT_MS,
-1,
false,
)
}
fun faceclawWearQuery(leftArm: Boolean): OutboundMessage {
return OutboundMessage(
"wear-query-control",
@@ -20,11 +20,18 @@ class CachedResource @JvmOverloads constructor(bytes: ByteArray, val owner: Stri
*/
class ResourceCacheState {
companion object {
/** Mirrors g2flash/patches/resource_cache.h. */
const val CACHE_SIZE = 196608
/**
* Mirrors g2flash/patches/resource_cache.h: the id table, then 320 KiB shared by the
* resources and their block headers. Firmware before Faceclaw/40 had 192 KiB in all,
* so this must not grow past what REQUIRED_FACECLAW_FIRMWARE_VERSION guarantees.
* RayNeo iO sessions use the same budget: that firmware's own copy of the header
* (faceclaw-rayneo-cfw patches/fc/resource_cache.h) needs at least this much before
* it reports the required revision.
*/
const val RESOURCE_COUNT = 512
const val TABLE_BYTES = RESOURCE_COUNT * 4
const val ARENA_BYTES = CACHE_SIZE - TABLE_BYTES
const val ARENA_BYTES = 320 * 1024
const val CACHE_SIZE = TABLE_BYTES + ARENA_BYTES
const val MAX_RESOURCE_SIZE = 65536
const val BLOCK_HEADER_BYTES = 16
@@ -506,6 +506,10 @@ internal fun GlassesSessionCore.scheduleReconnectLocked(sessionWasReady: Boolean
internal fun GlassesSessionCore.resetSessionStateLocked() {
ringBattery = -1
ringCharging = -1
headUpSwitch = -1
headUpSent = null
headUpReadAtMs = 0L
headUpVerifyPending = false
directRingBattery = -1
directRingCharging = -1
sessionReady = false
@@ -583,6 +587,22 @@ internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String)
}
}
internal fun GlassesSessionCore.emitKeyboardEvent(body: ByteArray) {
if (keyboardEventListeners.isEmpty()) {
return
}
val copy = body.copyOf()
host.postToMain {
for (keyboardListener in keyboardEventListeners) {
try {
keyboardListener.onKeyboardEvent(copy)
} catch (t: Throwable) {
logWarn("keyboard event listener failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitAudioMonitorPacket(data: ByteArray, arm: String, arrivalMs: Long) {
if (audioMonitorListeners.isEmpty()) {
return
@@ -634,14 +654,16 @@ internal fun GlassesSessionCore.finishFrame(frameId: Int, outcome: String) {
}
}
/** [x], [y], [z] as the hardware reports them; listeners get them in [ImuOrientation]'s frame. */
internal fun GlassesSessionCore.emitImuData(x: Double, y: Double, z: Double, eventSource: Int) {
if (imuListeners.isEmpty()) {
return
}
val (cx, cy, cz) = ImuOrientation.toCanonical(glassesType, x, y, z)
host.postToMain {
for (imuListener in imuListeners) {
try {
imuListener.onImuData(x, y, z, eventSource)
imuListener.onImuData(cx, cy, cz, eventSource)
} catch (t: Throwable) {
logWarn("listener onImuData failed", t)
}
@@ -131,6 +131,7 @@ class GlassesSessionCore(
internal val compassSubscriptions = CopyOnWriteList<CompassSubscription>(platform)
internal val ambientLightListeners = CopyOnWriteList<FaceclawAmbientLightListener>(platform)
internal val micStatusListeners = CopyOnWriteList<FaceclawMicStatusListener>(platform)
internal val keyboardEventListeners = CopyOnWriteList<FaceclawKeyboardEventListener>(platform)
internal val audioMonitorListeners = CopyOnWriteList<FaceclawAudioPacketListener>(platform)
@Volatile internal var running = false
@Volatile internal var userDisconnectRequested = false
@@ -312,7 +313,8 @@ class GlassesSessionCore(
internal var brightnessAlsStartedAt = -2000L
internal var brightnessDemoPolling = false
internal var headUpPreferences: Pair<Int, Int>? = null
internal var headUpSent: Pair<Int, Int>? = null
// Effective queued policy, not the raw preferences; null angle means omitted.
internal var headUpSent: Pair<Int, Int?>? = null
internal var cfwCleanupDelivered = false
internal var lastCfwCleanupAckMagic = 0
@@ -359,6 +361,26 @@ class GlassesSessionCore(
internal var lastGlassesRingInputAtMs = 0L
internal var lastGlassesRingInputLabel = ""
internal var lastGlassesRingBatteryAtMs = 0L
// Head-up (look-up wake) state from settings reads: -1 switch = not read yet.
internal var headUpSwitch = -1
internal var headUpAngle = 0
internal var headUpCalibration = 0
internal var headUpReadAtMs = 0L
// The trigger pitch TS asked for (degrees, 1..90), or -1 to leave the glasses'
// own setting alone. Reapplied after every read that disagrees with it.
internal var desiredHeadUpTrigger = -1
// The glasses' own switch and angle from before Faceclaw first changed them
// (switch -1 = unknown): captured before the first write, or handed back by TS,
// which persists them across app restarts. Written back while
// headUpRestorePending (TS stopped managing the trigger), then forgotten once a
// read confirms them.
internal var headUpOriginalSwitch = -1
internal var headUpOriginalAngle = 0
internal var headUpRestorePending = false
// Set after queueing a correction so the next settings read comes right away to
// confirm it; cleared by that read, so a firmware that refuses the value is
// retried on the normal poll cadence, not in a loop.
internal var headUpVerifyPending = false
internal var ringConnectedChangedAtMs = 0L
internal var ringConnectCount = 0
internal var lastDirectRingError = ""
@@ -488,7 +510,7 @@ class GlassesSessionCore(
// Preview/screenshot/recording callers render those explicitly on demand.
internal val compositor = SurfaceCompositor(false)
// Phone-side model of the CFW's 192 KiB resource cache.
// Phone-side model of the CFW's resource cache (ResourceCacheState.CACHE_SIZE).
// Reset whenever the image pipeline / EvenHub session is torn down: the
// firmware frees the cache with the fb lease, and after any resync the
// cheap safe assumption is an empty cache (glyphs re-upload lazily).
@@ -996,7 +1018,12 @@ class GlassesSessionCore(
fun configureHeadUpWake(enabled: Int, angle: Int) {
require(enabled in -1..1)
monitor.withLock {
headUpPreferences = Pair(enabled, if (angle < 0) -1 else angle.coerceIn(0, 60))
val preferences = Pair(enabled, if (angle < 0) -1 else angle.coerceIn(0, 60))
if (headUpPreferences != preferences) {
clearMessagesOfKindLocked("head-up-control")
headUpVerifyPending = false
headUpPreferences = preferences
}
}
interruptibleSleep.interrupt()
}
@@ -1026,6 +1053,61 @@ class GlassesSessionCore(
interruptibleSleep.interrupt()
}
/**
* Set the pitch (degrees, 1..90) at which looking up wakes the display, or 0 to stop
* managing it. Kept for the life of this object and reapplied on every connection: the
* next settings read supplies the glasses' calibration, then the switch and angle are
* written (see reconcileHeadUpLocked) and a follow-up read confirms them.
*
* [originalSwitch] (0/1, or -1 when unknown) and [originalAngle] are the glasses' own
* values from before Faceclaw changed them, as TS saved them from getHeadUpStatus; they
* take precedence over what this object captured. With degrees 0 they are written back.
*/
fun setHeadUpTrigger(degrees: Int, originalSwitch: Int, originalAngle: Int) {
monitor.withLock {
if (originalSwitch in 0..1) {
headUpOriginalSwitch = originalSwitch
headUpOriginalAngle = originalAngle
}
// A change of mind drops writes still queued for the previous value.
clearMessagesOfKindLocked("head-up-control")
headUpVerifyPending = false
desiredHeadUpTrigger = if (degrees in 1..90) degrees else -1
headUpRestorePending = desiredHeadUpTrigger < 0 && headUpOriginalSwitch >= 0
logLine(when {
desiredHeadUpTrigger > 0 -> "head-up trigger requested: ${desiredHeadUpTrigger}°"
headUpRestorePending -> "head-up trigger: restore the glasses' own " +
"(switch=$headUpOriginalSwitch angle=$headUpOriginalAngle)"
else -> "head-up trigger: left to the glasses"
})
if (!running || !sessionReady) return
if (desiredHeadUpTrigger <= 0 && !headUpRestorePending) return
if (headUpSwitch >= 0) {
applyHeadUpLocked(verifying = false)
} else {
// Calibration unknown: read the settings first; the ack applies the trigger.
lastBatteryRefreshAtMs = 0L
}
}
interruptibleSleep.interrupt()
}
/** Head-up settings as last read, as JSON (switch -1 = not read on this connection). */
fun getHeadUpStatus(): String {
monitor.withLock {
val status = linkedMapOf<String, Any?>()
status["switch"] = headUpSwitch
status["angle"] = headUpAngle
status["calibration"] = headUpCalibration
status["desired"] = desiredHeadUpTrigger
status["originalSwitch"] = headUpOriginalSwitch
status["originalAngle"] = headUpOriginalAngle
status["restorePending"] = headUpRestorePending
status["readAgeMs"] = if (headUpReadAtMs == 0L) -1L else maxOf(0L, now() - headUpReadAtMs)
return try { Json.write(status) } catch (e: Exception) { "{}" }
}
}
// ---------------------------------------------------------------------
// Bandwidth benchmark
@@ -1399,6 +1481,39 @@ class GlassesSessionCore(
return true
}
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
if (listener != null) {
keyboardEventListeners.add(listener)
}
}
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
if (listener != null) {
keyboardEventListeners.remove(listener)
}
}
/**
* Queue a CFW keyboard-bridge control (['K','B',ver,op,request,epoch,...]) as a settings
* field-130 write to the right temple. Fire-and-forget: the firmware answers with field-131
* records, which arrive through addKeyboardEventListener. Returns false when nothing was
* queued (not a G2, no session, or a malformed record).
*/
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean {
if (!isG2 || record == null || record.size < 8) {
return false
}
monitor.withLock {
if (!running || !sessionReady) {
logLine("skip keyboard control ($label); session not ready")
return false
}
pendingMessages.addLast(messageBuilder.faceclawKeyboardControl(record, label))
}
interruptibleSleep.interrupt()
return true
}
/**
* Forward render-characteristic audio packets to the listener WITHOUT
* sending the stock EvenHub audio-control enable. Used for the CFW
@@ -1903,8 +2018,21 @@ class GlassesSessionCore(
tap(data.copyOf())
return
}
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
val frame = BleProtocol.parseFrame(data)
if (frame.ok && frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW keyboard-bridge records (field 131) come from whichever temple hosts the
// keyboard. They are their own stream: neither an ACK nor glasses input, and the
// once-a-second status record among them must not count as user activity (which
// would hold off battery polling). A touchpad sends a hundred of them a second
// while a finger is on it, so they are not logged one by one either.
val keyboardEvent = BleProtocol.parseFaceclawKeyboardEvent(frame.pb)
if (keyboardEvent != null) {
monitor.withLock { lastIncomingAtMs = now() }
emitKeyboardEvent(keyboardEvent)
return
}
}
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
val decodedWearState = BleProtocol.parseWearState(frame)
val compassEvent = if (address.equals(rightAddress, ignoreCase = true))
BleProtocol.parseCompassEvent(frame)
@@ -2072,11 +2200,10 @@ class GlassesSessionCore(
} else if (decoded != null
&& "sys-event" == decoded.kind
&& decoded.eventType == BleProtocol.EVENT_HEAD_DOWN) {
// Faceclaw/38 also forwards the head coming back down. Nothing
// uses it yet; as an unknown input it would count as activity
// and keep the screen awake, so drop it here.
logLine("head-down forwarded by CFW (ignored)")
decoded = null
// Faceclaw/38 also forwards the head coming back down. TS uses
// it to end a head-tilt Glanceboard and otherwise drops it
// before the shell, so it never counts as activity.
logLine("head-down forwarded by CFW")
}
event = decoded
if (decoded != null) {
@@ -299,7 +299,8 @@ internal fun GlassesSessionCore.g1StopAudioCapture() {
}
/**
* There is no magnetometer on a G1, only the accelerometer and gyroscope: x, y, z in g, head frame, from the right arm.
* There is no magnetometer on a G1, only the accelerometer and gyroscope: x, y, z in g from the right arm, in the
* G1's own axes, which [emitImuData] rotates into the G2's ([ImuOrientation]).
* [enable] raises the sample rate and polls it; [g1MaxSleepMs] follows.
*/
internal fun GlassesSessionCore.g1SetImuReport(enable: Boolean) {
@@ -1,23 +1,7 @@
package com.faceclaw.app
/** Both arms receive separate stock settings messages; no CFW changes required. */
/** The sender and settings polls share one reconciler; never queue a second policy. */
internal fun GlassesSessionCore.enqueueHeadUpPreferencesLocked() {
if (!isG2) return
val desired = headUpPreferences ?: return
if (!sessionReady || shutdownRequested || chargingMode || headUpSent == desired
|| hasPendingOrInflightKindLocked("head-up-control")) return
for (leftArm in listOf(true, false)) {
// Set the angle before enabling wake. Omitted fields preserve device preferences.
if (desired.second >= 0) {
val message = messageBuilder.headUpControl(leftArm, null, desired.second)
message.onTimeout = MessageCallback { headUpSent = null; logLine("head-up angle ack timeout") }
pendingMessages.addLast(message)
}
if (desired.first >= 0) {
val message = messageBuilder.headUpControl(leftArm, desired.first == 1, -1)
message.onTimeout = MessageCallback { headUpSent = null; logLine("head-up wake ack timeout") }
pendingMessages.addLast(message)
}
}
headUpSent = desired
if (headUpPreferences == null) return
applyHeadUpLocked(verifying = false)
}
@@ -1079,6 +1079,103 @@ internal fun GlassesSessionCore.shouldPollBatteryLocked(now: Long): Boolean {
&& (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS)
}
/**
* Bring the glasses' head-up trigger to [GlassesSessionCore.desiredHeadUpTrigger] after a settings
* read: switch on, then angle = trigger - calibration (the firmware adds its stored calibration).
*/
internal fun GlassesSessionCore.reconcileHeadUpLocked(ackPayload: ByteArray?) {
val verifying = headUpVerifyPending
headUpVerifyPending = false
val headUp = if (ackPayload == null) null else BleProtocol.parseSettingsHeadUp(ackPayload)
if (headUp != null) {
headUpSwitch = headUp[0]
headUpAngle = headUp[1]
headUpCalibration = headUp[2]
headUpReadAtMs = now()
val sent = headUpSent
if (!verifying && sent != null && !hasPendingOrInflightKindLocked("head-up-control")
&& ((sent.first >= 0 && sent.first != headUpSwitch)
|| (sent.second != null && sent.second != headUpAngle))) {
headUpSent = null // a regular poll retries a refused/drifted setting
}
}
if (headUpSwitch < 0 || !sessionReady) return
applyHeadUpLocked(verifying)
}
/**
* Compare the last read with what is wanted (the trigger, or the glasses' original values
* while a restore is pending) and queue writes when they differ. After a verifying read
* that still disagrees nothing is queued: the next regular poll retries, so a firmware
* that clamps or refuses the value costs one attempt per poll interval, not a write loop.
*/
internal fun GlassesSessionCore.applyHeadUpLocked(verifying: Boolean) {
if (!isG2 || !sessionReady || shutdownRequested || chargingMode
|| hasPendingOrInflightKindLocked("head-up-control")) return
val preferences = headUpPreferences
val bothArms = preferences != null && (preferences.first >= 0 || preferences.second >= 0)
// Explicit Off is authoritative. Glanceboard is a temporary angle override,
// never a separate writer that can re-enable a forbidden wake.
val trigger = desiredHeadUpTrigger > 0 && preferences?.first != 0
if (trigger && headUpSwitch < 0) return // calibration comes from the settings read
val wantSwitch = when {
trigger -> 1
preferences != null && preferences.first >= 0 -> preferences.first
headUpRestorePending -> headUpOriginalSwitch
else -> -1
}
val wantAngle = when {
trigger -> desiredHeadUpTrigger - headUpCalibration
preferences != null && preferences.second >= 0 -> preferences.second
headUpRestorePending -> headUpOriginalAngle
else -> null // omitted; negative generic angles are valid
}
if (wantSwitch < 0 && wantAngle == null) return
val matches = (wantSwitch < 0 || headUpSwitch == wantSwitch) &&
(wantAngle == null || headUpAngle == wantAngle)
// A right-arm read cannot prove the left arm was configured. The cache is
// the complete queued policy, invalidated on cancellation, timeout and reset.
val effective = wantSwitch to wantAngle
if (matches && (!bothArms || headUpSent == effective)) {
if (!trigger && headUpRestorePending) {
headUpRestorePending = false
headUpOriginalSwitch = -1
headUpOriginalAngle = 0
}
return
}
if (verifying) {
logLine("head-up setting not applied yet; retrying on next poll")
return
}
if (headUpSent == effective) return
if (trigger && headUpOriginalSwitch < 0) {
headUpOriginalSwitch = headUpSwitch
headUpOriginalAngle = headUpAngle
}
queueHeadUpWritesLocked(wantSwitch, wantAngle, bothArms)
headUpSent = effective
}
internal fun GlassesSessionCore.queueHeadUpWritesLocked(switch: Int, angle: Int?, bothArms: Boolean) {
clearMessagesOfKindLocked("head-up-control")
for (leftArm in if (bothArms) listOf(true, false) else listOf(false)) {
val set = if (angle == null) null else if (bothArms)
messageBuilder.headUpSetting(BleProtocol.HEAD_UP_FIELD_ANGLE, angle, leftArm, "head-up angle=$angle")
else messageBuilder.setHeadUp(BleProtocol.HEAD_UP_FIELD_ANGLE, angle)
val sw = if (switch < 0) null else if (bothArms)
messageBuilder.headUpSetting(BleProtocol.HEAD_UP_FIELD_SWITCH, switch, leftArm, "head-up wake=${switch == 1}")
else messageBuilder.setHeadUp(BleProtocol.HEAD_UP_FIELD_SWITCH, switch)
set?.onTimeout = MessageCallback { headUpSent = null; logLine("head-up angle ack timeout") }
sw?.onTimeout = MessageCallback { headUpSent = null; logLine("head-up switch ack timeout") }
// Fork: angle before wake on both arms. Unset: preserve upstream ordering.
val writes = if (!bothArms && switch == 1) listOfNotNull(sw, set) else listOfNotNull(set, sw)
for (message in writes) pendingMessages.addLast(message)
}
headUpVerifyPending = true
lastBatteryRefreshAtMs = 0L
}
internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessage {
val message = messageBuilder.batteryQuery()
message.onAck = MessageCallback {
@@ -1087,6 +1184,7 @@ internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessa
ringBattery = if (ring == null) -1 else ring.battery
ringCharging = if (ring == null) -1 else ring.charging
if (ringBattery >= 0) releaseRingFromGlassesLocked("glasses report the ring connected")
reconcileHeadUpLocked(message.ackPayload)
val snapshot = BleProtocol.parseSettingsBattery(message.ackPayload)
if (snapshot != null) {
headsetBattery = snapshot.battery
@@ -1350,6 +1448,7 @@ internal fun GlassesSessionCore.handleAckTimeoutLocked(message: OutboundMessage)
}
internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) {
if (kind == "head-up-control") headUpSent = null
val pendingIterator = pendingMessages.iterator()
while (pendingIterator.hasNext()) {
val message = pendingIterator.next()
@@ -1362,10 +1461,12 @@ internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) {
magicPool.release(message.sid, message.magic, message.label, "cleared pending $kind")
}
}
// Head-up writes already handed to BLE must finish before a replacement
// policy is queued; dropping their ACK tracking would race the old write.
val inFlightIterator = inFlightMessages.iterator()
while (inFlightIterator.hasNext()) {
val message = inFlightIterator.next()
if (kind == message.kind) {
if (kind == message.kind && kind != "head-up-control") {
inFlightIterator.remove()
if ("image" == kind) {
discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared ($kind)")
@@ -1406,6 +1507,7 @@ internal fun GlassesSessionCore.clearAllMessagesLocked(reason: String) {
* direct prelude write.
*/
internal fun GlassesSessionCore.clearAllMessagesPreservingWakeLeaseLocked(reason: String) {
headUpSent = null
brightnessSentVisible = null
brightnessSentLevel = -1
brightnessAlsStartedAt = -2000
@@ -1448,6 +1550,7 @@ internal fun GlassesSessionCore.hasPendingImageLocked(): Boolean {
internal fun GlassesSessionCore.clearPendingMessagesLocked(reason: String) {
while (!pendingMessages.isEmpty()) {
val message = pendingMessages.removeFirst()
if (message.kind == "head-up-control") headUpSent = null
discardPrewriteIfMatchesLocked(message)
discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared: $reason")
magicPool.release(message.sid, message.magic, message.label, "cleared pending: $reason")
@@ -1457,6 +1560,7 @@ internal fun GlassesSessionCore.clearPendingMessagesLocked(reason: String) {
internal fun GlassesSessionCore.clearInFlightMessagesLocked(reason: String) {
while (!inFlightMessages.isEmpty()) {
val message = inFlightMessages.removeFirst()
if (message.kind == "head-up-control") headUpSent = null
discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared: $reason")
magicPool.release(message.sid, message.magic, message.label, "cleared inflight: $reason")
}
@@ -0,0 +1,29 @@
package com.faceclaw.app
/**
* Faceclaw's accelerometer frame. Every model's samples are rotated into it before they reach a
* FaceclawImuListener ([GlassesSessionCore.emitImuData]), so Faceclaw apps and EvenHub pages see
* the same frame whichever glasses are worn. It is the G2's own frame, in g, glasses worn:
*
* +x forward (the way the wearer faces), +y to the wearer's left, +z up through the top of the head.
*
* At rest a reading is the "up" direction (the reaction to gravity): head level reads (0, 0, 1),
* looking 45° up (0.71, 0, 0.71), and the head tilted 45° to the right (right ear down)
* (0, 0.71, 0.71). Pitch above level is atan2(x, hypot(y, z)).
*/
object ImuOrientation {
/**
* A sample as [type]'s hardware reports it, in the canonical frame. Axes measured on hardware
* with the Developer app's accelerometer page:
* - G2: already canonical.
* - G1 (g1v0): +x right, +y up, +z back. Level reads (0, 1, 0), 45° up (0, 0.71, -0.71),
* 45° to the right (-0.71, 0.71, 0).
* - RayNeo iO: no accelerometer stream yet, so nothing to rotate; measure its axes when one
* is added.
*/
fun toCanonical(type: GlassesType, x: Double, y: Double, z: Double): DoubleArray = when (type) {
GlassesType.G2 -> doubleArrayOf(x, y, z)
GlassesType.G1 -> doubleArrayOf(-z, -x, y)
GlassesType.RAYNEO_IO -> doubleArrayOf(x, y, z)
}
}
@@ -0,0 +1,37 @@
package com.faceclaw.app
import platform.Foundation.NSData
/**
* Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's
* captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record:
* 16 kHz mono PCM16, as its recognizer hands it to TypeScript.
*/
class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) {
/** The recording stopped early: the file could not be written. */
val failed: Boolean get() = writer.failed
/** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */
fun append(pcm16le: NSData) {
val bytes = pcm16le.byteArray()
val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) }
writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs()))
}
/** Completes the file. Returns the recording's length in ms. */
fun finish(): Long = writer.finish()
}
object IosMicRecordings {
/**
* A recorder writing a new file at [path], or null when the file cannot be created.
* [configJson] describes the capture and is stored in the file.
*/
fun open(path: String, configJson: String?): IosMicRecorder? =
try {
IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform))
} catch (error: Throwable) {
PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}")
null
}
}
@@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped {
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
actual fun openFileOutput(path: String): SeekableOutput {
val file = fopen(path, "wb+") ?: error("Cannot write $path")
return object : SeekableOutput {
override var size = 0L
private set
private fun put(bytes: ByteArray, offset: Int, length: Int) {
if (length == 0) return
bytes.usePinned {
check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert<platform.posix.size_t>()) { "Short write to $path" }
}
}
override fun write(bytes: ByteArray, offset: Int, length: Int) {
put(bytes, offset, length)
size += length
// Each write is a complete element; hand it to the OS so a crash keeps it.
platform.posix.fflush(file)
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0)
put(bytes, 0, bytes.size)
check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0)
platform.posix.fflush(file)
}
override fun close() {
fclose(file)
}
}
}
actual fun openFileInput(path: String): SeekableInput {
val file = fopen(path, "rb") ?: error("Cannot read $path")
val fileSize = fileLength(path)
return object : SeekableInput {
override val length = fileSize
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0
return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() }
}
override fun close() {
fclose(file)
}
}
}
@@ -29,6 +29,7 @@ import platform.CoreBluetooth.CBPeripheralDelegateProtocol
import platform.CoreBluetooth.CBPeripheralStateConnected
import platform.CoreBluetooth.CBPeripheralStateDisconnected
import platform.CoreBluetooth.CBService
import platform.CoreBluetooth.CBUUID
import platform.Foundation.NSData
import platform.Foundation.NSError
import platform.Foundation.NSNumber
@@ -47,6 +48,9 @@ interface IosBleScanListener {
fun onAdvertisement(identifier: String, name: String, manufacturerDataHex: String, rssi: Int, connectable: Boolean)
}
/** A peripheral the system holds a connection to; see [IosBleCentral.systemConnectedG2Arms]. */
class IosConnectedPeripheral(val identifier: String, val name: String)
/** ANCS authorization changes for a connected peripheral; delivered on the BLE queue. */
fun interface IosAncsAuthorizationListener {
fun onAncsAuthorization(identifier: String, authorized: Boolean)
@@ -214,6 +218,25 @@ class IosBleCentral(private val platform: ProtocolPlatform = IosProtocolPlatform
manager.scanForPeripheralsWithServices(null, mapOf(CBCentralManagerScanOptionAllowDuplicatesKey to true))
}
/**
* G2 arms the system is connected to, through this app, another app or iOS itself. A
* connected arm stops advertising, and iOS keeps the right arm's ANCS link open after a
* session ends, so a scan alone never hears it. Empty until Bluetooth is powered on.
*/
fun systemConnectedG2Arms(): List<IosConnectedPeripheral> {
val found = ArrayList<IosConnectedPeripheral>()
dispatch_sync(queue) {
val manager = central ?: return@dispatch_sync
if (manager.state != CBManagerStatePoweredOn) return@dispatch_sync
val services = listOf(BleProtocol.PROTOCOL_SERVICE_UUID, BleProtocol.RENDER_SERVICE_UUID).map { CBUUID.UUIDWithString(it) }
for (item in manager.retrieveConnectedPeripheralsWithServices(services)) {
val peripheral = item as? CBPeripheral ?: continue
found.add(IosConnectedPeripheral(peripheral.identifier.UUIDString.uppercase(), peripheral.name ?: ""))
}
}
return found
}
/** Known peripheral identifiers from this process's scans/retrievals (uppercase). */
fun knownIdentifiers(): List<String> = cond.withLock { peers.keys.toList() }
@@ -34,6 +34,11 @@ interface IosMicStatusListener {
fun onNsDataMicStatus(body: NSData, arm: String)
}
/** CFW keyboard-bridge records (the raw field-131 record) as NSData. Main queue. */
interface IosKeyboardEventListener {
fun onNsDataKeyboardEvent(body: NSData)
}
/**
* ObjC-facing facade over the shared [GlassesSessionCore] for the NativeScript app: the iOS
* counterpart of Android's FaceclawBleCommunicator with the same method names. GATT goes
@@ -230,6 +235,20 @@ class IosGlassesSession internal constructor(
return adapter
}
fun addKeyboardEventListenerNsData(listener: IosKeyboardEventListener): FaceclawKeyboardEventListener {
val adapter = object : FaceclawKeyboardEventListener {
override fun onKeyboardEvent(body: ByteArray?) =
listener.onNsDataKeyboardEvent((body ?: ByteArray(0)).data())
}
core.addKeyboardEventListener(adapter)
return adapter
}
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
fun sendFaceclawKeyboardControl(record: NSData?, label: String): Boolean =
core.sendFaceclawKeyboardControl(record?.byteArray(), label)
/** Every audio-characteristic packet, whether or not a capture is active (passive; sends nothing). */
fun addAudioMonitorListenerNsData(listener: IosAudioPacketListener): FaceclawAudioPacketListener {
val adapter = object : FaceclawAudioPacketListener {
+43
View File
@@ -0,0 +1,43 @@
# Faceclaw SSH transport
The Terminal app's SSH client: a small Go package over
[golang.org/x/crypto/ssh](https://pkg.go.dev/golang.org/x/crypto/ssh), bound
into the apps with gomobile. It provides text-message channels over SSH:
- **exec**: run a command on the host, exchanging newline-delimited text
(used for tmux control mode and the tmux session poller; see
`app/apps/terminal/tmux-client.ts`).
- **websocket**: a WebSocket to an address as seen from the host, through a
`direct-tcpip` channel (used to tunnel g2mirror-server's protocol).
Channels to the same host and credentials share one SSH connection, with a
keepalive that fails them promptly when the peer silently disappears. Host
keys are pinned by the app (first connection reports the presented key so
the user can verify its fingerprint). Authentication uses a non-exportable
ECDSA P-256 key held by the platform keystore (Android Keystore/StrongBox,
iOS Secure Enclave; see `FaceclawSsh.kt` / `FaceclawSsh.m`) and, if the
server allows it, a password saved encrypted on the phone.
## Building
`scripts/go-ssh-build.cjs` runs `gomobile bind` (pinned in `go.mod`'s tool
block) from the NativeScript before-prepare hook and stages
`faceclaw-ssh.aar` / `Faceclawssh.xcframework` into the local
`@faceclaw/kotlin` plugin. It needs Go 1.26+ on PATH, plus the Android NDK
(Android) or Xcode (iOS); a content stamp skips rebuilds when the Go sources
haven't changed.
## Testing
`go test -race ./...` runs the transport against an in-process SSH server
(host-key pinning, password and keyboard-interactive auth, key auth through
a fake platform signer, exec framing, the websocket tunnel, connection
sharing and loss).
## Keeping it current
x/crypto/ssh is the security boundary against a malicious host or a
man in the middle; update it promptly when advisories appear
(`go get golang.org/x/crypto@latest && go mod tidy`). A Go panic in any
goroutine terminates the app: the package recovers in the goroutines it
owns, but not inside the library's own.
+254
View File
@@ -0,0 +1,254 @@
package faceclawssh
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/url"
"strings"
"time"
"unicode/utf8"
"github.com/gorilla/websocket"
"golang.org/x/crypto/ssh"
)
type closedPayload struct {
ExitStatus *int `json:"exitStatus,omitempty"`
Signal string `json:"signal,omitempty"`
Code *int `json:"code,omitempty"`
Reason string `json:"reason,omitempty"`
}
// runExec runs command on the host and relays newline-delimited text both
// ways until the command exits, the connection drops, or Close.
func (c *Channel) runExec(conn *pooledConn, command string) {
session, err := conn.client.NewSession()
if err != nil {
c.fail(channelOpenFailure(conn, err))
return
}
defer session.Close()
stdin, err1 := session.StdinPipe()
stdout, err2 := session.StdoutPipe()
stderr, err3 := session.StderrPipe()
if err := errors.Join(err1, err2, err3); err != nil {
c.fail(&failure{Kind: "channel", Message: err.Error()})
return
}
if err := session.Start(command); err != nil {
c.fail(&failure{Kind: "channel", Message: "could not run command: " + err.Error()})
return
}
go func() {
<-c.ctx.Done()
session.Close()
}()
c.markOpen()
go func() {
defer c.guard()
for text := range c.sendQ {
if _, err := io.WriteString(stdin, text+"\n"); err != nil {
return
}
}
// Close was called and the queue has drained: EOF tells the command
// we're done (tmux control mode detaches on it).
stdin.Close()
}()
stderrDone := make(chan struct{})
go func() {
defer close(stderrDone)
defer c.guard()
readLines(stderr, func(line string) { c.emit("stderr", line) })
}()
readErr := readLines(stdout, func(line string) { c.emit("text", line) })
// Deliver the command's last words (e.g. "tmux: command not found")
// before the close event, but don't hang on a stderr that never ends.
select {
case <-stderrDone:
case <-time.After(500 * time.Millisecond):
}
if c.ctx.Err() != nil {
return
}
waitErr := session.Wait()
var exitErr *ssh.ExitError
var missing *ssh.ExitMissingError
switch {
case waitErr == nil:
status := 0
c.finish("closed", closedPayload{ExitStatus: &status})
case errors.As(waitErr, &exitErr):
status := exitErr.ExitStatus()
c.finish("closed", closedPayload{ExitStatus: &status, Signal: exitErr.Signal()})
case errors.As(waitErr, &missing):
// The channel closed without an exit status: usually the connection
// went away under it.
c.fail(lostFailure(conn, readErr))
default:
c.fail(lostFailure(conn, waitErr))
}
}
// readLines calls emit for each line of r (without its newline) and returns
// the read error that ended it (nil at EOF). Lines may be arbitrarily long.
func readLines(r io.Reader, emit func(string)) error {
reader := bufio.NewReaderSize(r, 64*1024)
for {
line, err := reader.ReadBytes('\n')
if len(line) > 0 {
if line[len(line)-1] == '\n' {
line = line[:len(line)-1]
}
emit(escapeLine(line))
}
if err == io.EOF {
return nil
}
if err != nil {
return err
}
}
}
// escapeLine makes a line safe to hand across the gomobile boundary as a
// string (which must be valid UTF-8). Valid UTF-8 passes through. Otherwise
// every byte >= 0x80 becomes "\ooo" (backslash, three octal digits): the same
// escape tmux control mode uses for control characters and backslashes in
// %output, so a %output line split mid-character still decodes back to its
// exact bytes. (Outside %output a literal backslash is ambiguous with this
// escape; such lines are only ever displayed.)
func escapeLine(line []byte) string {
if utf8.Valid(line) {
return string(line)
}
var b strings.Builder
for _, c := range line {
if c >= 0x80 {
fmt.Fprintf(&b, "\\%03o", c)
} else {
b.WriteByte(c)
}
}
return b.String()
}
// runWebSocket opens rawURL (ws://host:port/path) through the SSH
// connection — host:port is resolved on the SSH server's side — and relays
// text frames.
func (c *Channel) runWebSocket(conn *pooledConn, rawURL string) {
parsed, err := url.Parse(rawURL)
if err != nil || parsed.Scheme != "ws" || parsed.Host == "" {
c.fail(&failure{Kind: "config", Message: fmt.Sprintf("bad websocket URL %q", rawURL)})
return
}
target := parsed.Host
if parsed.Port() == "" {
target = net.JoinHostPort(parsed.Hostname(), "80")
}
dialer := websocket.Dialer{
NetDialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
forwarded, err := conn.client.DialContext(ctx, "tcp", target)
if err != nil {
return nil, err
}
return noDeadlineConn{forwarded}, nil
},
}
// The handshake timeout rides the context: SSH channels can't do deadlines.
handshakeCtx, cancelHandshake := context.WithTimeout(c.ctx, defaultConnectTimeout)
ws, _, err := dialer.DialContext(handshakeCtx, rawURL, nil)
cancelHandshake()
if err != nil {
if c.ctx.Err() != nil {
return
}
c.fail(channelOpenFailure(conn, fmt.Errorf("could not reach %s on the host: %w", target, err)))
return
}
defer ws.Close()
go func() {
<-c.ctx.Done()
ws.Close()
}()
c.markOpen()
go func() {
defer c.guard()
for text := range c.sendQ {
if err := ws.WriteMessage(websocket.TextMessage, []byte(text)); err != nil {
return
}
}
ws.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "bye"),
time.Now().Add(time.Second))
}()
for {
kind, data, err := ws.ReadMessage()
if err != nil {
if c.ctx.Err() != nil {
return
}
var closeErr *websocket.CloseError
if errors.As(err, &closeErr) {
code := closeErr.Code
c.finish("closed", closedPayload{Code: &code, Reason: closeErr.Text})
return
}
c.fail(lostFailure(conn, err))
return
}
if kind == websocket.TextMessage {
c.emit("text", string(data))
}
}
}
// noDeadlineConn adapts an SSH direct-tcpip channel for the websocket
// library, which sets deadlines the channel doesn't support. Timeouts come
// from contexts and the SSH keepalive instead.
type noDeadlineConn struct{ net.Conn }
func (noDeadlineConn) SetDeadline(time.Time) error { return nil }
func (noDeadlineConn) SetReadDeadline(time.Time) error { return nil }
func (noDeadlineConn) SetWriteDeadline(time.Time) error { return nil }
// channelOpenFailure classifies a failure to open a channel: if the SSH
// connection itself is gone it's a network failure (reconnect), otherwise
// the host refused this particular channel.
func channelOpenFailure(conn *pooledConn, err error) *failure {
if isConnDead(conn) {
return &failure{Kind: "network", Message: "connection lost: " + err.Error()}
}
return &failure{Kind: "channel", Message: err.Error()}
}
func lostFailure(conn *pooledConn, err error) *failure {
message := "connection lost"
if err != nil && !errors.Is(err, io.EOF) {
message += ": " + err.Error()
}
if !isConnDead(conn) {
// The connection is fine; only this channel ended abruptly.
return &failure{Kind: "channel", Message: "channel closed unexpectedly"}
}
return &failure{Kind: "network", Message: message}
}
func isConnDead(conn *pooledConn) bool {
// client.Wait's goroutine marks the connection dead slightly after the
// channels see their streams end; give it a moment to catch up.
deadline := time.Now().Add(200 * time.Millisecond)
for {
poolMu.Lock()
dead := conn.dead
poolMu.Unlock()
if dead || time.Now().After(deadline) {
return dead
}
time.Sleep(10 * time.Millisecond)
}
}
+304
View File
@@ -0,0 +1,304 @@
// Package faceclawssh is Faceclaw's SSH transport, bound into the Android and
// iOS apps with gomobile. It exposes text-message channels over SSH:
//
// - "exec": run a command on the host and exchange newline-delimited text
// with it (stdin lines in, stdout lines out). Used for tmux control mode
// and the tmux session poller.
// - "websocket": open a direct-tcpip channel to an address as seen from the
// host (normally g2mirror-server on 127.0.0.1:8737) and speak WebSocket
// text frames over it.
//
// Channels to the same host, user and credentials share one SSH connection
// (see pool.go). Host keys are pinned by the caller: a connection whose host
// presents a key that isn't in the spec's hostKeys fails with a
// "hostkey-unknown" or "hostkey-mismatch" failure carrying the presented key,
// so the app can show the fingerprint and retry once the user has trusted it.
// Authentication uses a platform-held, non-exportable ECDSA P-256 key (the
// Signer interface; Android Keystore / iOS Secure Enclave) and, if the server
// allows it, a password.
//
// Everything crossing the gomobile boundary is a string, []byte or one of the
// interfaces below; structured values are JSON strings.
package faceclawssh
import (
"context"
"encoding/json"
"fmt"
"runtime/debug"
"sync"
"time"
)
// Signer is implemented by the platform: an ECDSA P-256 key whose private
// half never leaves the device's keystore.
type Signer interface {
// PublicKey returns the key's public half, either as X.509
// SubjectPublicKeyInfo DER (Android's PublicKey.getEncoded()) or as an
// uncompressed X9.62 point, 0x04 || X || Y (iOS's
// SecKeyCopyExternalRepresentation).
PublicKey() []byte
// Sign returns the ASN.1 DER ECDSA signature of SHA-256(data).
Sign(data []byte) ([]byte, error)
}
// Listener receives a channel's events. Calls arrive on arbitrary goroutines
// (never concurrently for one channel); the platform wrapper hops them onto
// the thread that owns the channel.
//
// Event kinds and payloads:
// - "open": the channel is ready; payload empty. Sends made before this are
// queued and delivered once open.
// - "text": one message (websocket text frame, or one stdout line without
// its newline). Exec lines that aren't valid UTF-8 have every byte >= 0x80
// escaped as a backslash and three octal digits, matching tmux control
// mode's escaping of control characters (see escapeLine).
// - "stderr": one stderr line of an exec channel, escaped the same way.
// - "closed": the channel ended normally; payload is JSON
// {"exitStatus": n, "signal": "...", "code": n, "reason": "..."} with the
// fields that apply (exec exit status/signal, websocket close code/reason).
// Terminal: no events follow.
// - "failure": the channel could not be opened or was lost; payload is a
// JSON Failure. Terminal: no events follow.
type Listener interface {
OnEvent(kind string, payload string)
}
// Failure describes why a channel failed, as the JSON payload of a "failure"
// event.
type failure struct {
// "hostkey-unknown": no host key is pinned yet; HostKey is what the host
// presented. "hostkey-mismatch": the host presented a key that isn't
// pinned. "auth": the host refused every credential we have.
// "network": couldn't reach the host, or the connection dropped.
// "channel": the SSH connection is fine but the channel couldn't be
// opened (exec refused, nothing listening at the forwarded address...).
// "config": the spec was malformed. "internal": a bug.
Kind string `json:"kind"`
Message string `json:"message"`
HostKey *hostKeyInfo `json:"hostKey,omitempty"`
// For "auth": the host accepts password (or keyboard-interactive)
// authentication, whether a password was sent, and whether our key was
// offered.
PasswordAllowed bool `json:"passwordAllowed,omitempty"`
PasswordTried bool `json:"passwordTried,omitempty"`
KeyTried bool `json:"keyTried,omitempty"`
}
func (f *failure) Error() string { return f.Message }
type hostKeyInfo struct {
// SSH key type, e.g. "ssh-ed25519".
Type string `json:"type"`
// OpenSSH-style SHA-256 fingerprint, "SHA256:<base64>".
Fingerprint string `json:"fingerprint"`
// The key in SSH wire format, base64: what goes in a spec's hostKeys.
Key string `json:"key"`
}
type channelSpec struct {
// "exec" or "websocket".
Type string `json:"type"`
// exec: the command line, run by the user's login shell.
Command string `json:"command"`
// websocket: ws:// URL, resolved on the host's side of the connection.
URL string `json:"url"`
}
type openSpec struct {
Host string `json:"host"`
Port int `json:"port"`
User string `json:"user"`
Password string `json:"password"`
// Pinned host keys, SSH wire format, base64. Empty: the first connection
// fails with "hostkey-unknown".
HostKeys []string `json:"hostKeys"`
// Offer the platform key (the Signer passed to Open).
UseKey bool `json:"useKey"`
ConnectTimeoutMs int `json:"connectTimeoutMs"`
KeepaliveIntervalMs int `json:"keepaliveIntervalMs"`
Channel channelSpec `json:"channel"`
}
const (
defaultConnectTimeout = 15 * time.Second
defaultKeepaliveInterval = 20 * time.Second
// Messages queued on a channel before Send starts refusing them.
sendQueueCapacity = 4096
// How long Close waits for queued messages to flush and the far end to
// finish before tearing the channel down.
closeGracePeriod = 2 * time.Second
)
// Channel is one open (or opening) channel. Methods are safe to call from
// any thread.
type Channel struct {
listener Listener
ctx context.Context
cancel context.CancelFunc
mu sync.Mutex
closed bool // Close was called: no more sends, no more events
opened bool
sendQ chan string
finishOnce sync.Once
}
// Open starts opening a channel described by spec (a JSON openSpec) and
// returns immediately; progress arrives as listener events. signer may be
// nil (no key authentication).
func Open(spec string, signer Signer, listener Listener) *Channel {
ctx, cancel := context.WithCancel(context.Background())
c := &Channel{
listener: listener,
ctx: ctx,
cancel: cancel,
sendQ: make(chan string, sendQueueCapacity),
}
go c.run(spec, signer)
return c
}
// Send queues one message: a websocket text frame, or a line for an exec
// channel's stdin (the newline is added). Returns false if the channel is
// closed or its queue is full.
func (c *Channel) Send(text string) bool {
c.mu.Lock()
defer c.mu.Unlock()
if c.closed {
return false
}
select {
case c.sendQ <- text:
return true
default:
return false
}
}
// Close ends the channel. Messages already queued are still sent (so a
// parting "unview" reaches the host), then the channel's write side is
// closed and, after a short grace period, the channel is torn down. No
// events are delivered after Close.
func (c *Channel) Close() {
c.mu.Lock()
if c.closed {
c.mu.Unlock()
return
}
c.closed = true
opened := c.opened
close(c.sendQ)
c.mu.Unlock()
if !opened {
c.cancel()
return
}
time.AfterFunc(closeGracePeriod, c.cancel)
}
func (c *Channel) isClosed() bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.closed
}
func (c *Channel) emit(kind, payload string) {
if c.isClosed() {
return
}
c.listener.OnEvent(kind, payload)
}
func (c *Channel) markOpen() {
c.mu.Lock()
c.opened = true
c.mu.Unlock()
c.emit("open", "")
}
// finish delivers the channel's one terminal event ("closed" or "failure").
func (c *Channel) finish(kind string, payload any) {
c.finishOnce.Do(func() {
encoded, err := json.Marshal(payload)
if err != nil {
encoded = []byte(`{"kind":"internal","message":"unencodable event"}`)
}
c.emit(kind, string(encoded))
c.cancel()
})
}
func (c *Channel) fail(f *failure) { c.finish("failure", f) }
// guard turns a panic in one of our goroutines into a failure event instead
// of crashing the app (an unrecovered Go panic kills the whole process).
func (c *Channel) guard() {
if r := recover(); r != nil {
c.fail(&failure{Kind: "internal", Message: fmt.Sprintf("internal error: %v\n%s", r, debug.Stack())})
}
}
func (c *Channel) run(specJSON string, signer Signer) {
defer c.guard()
var spec openSpec
if err := json.Unmarshal([]byte(specJSON), &spec); err != nil {
c.fail(&failure{Kind: "config", Message: "bad channel spec: " + err.Error()})
return
}
if spec.Host == "" || spec.User == "" {
c.fail(&failure{Kind: "config", Message: "host and user are required"})
return
}
if spec.Port == 0 {
spec.Port = 22
}
if !spec.UseKey {
signer = nil
}
conn, err := acquire(c.ctx, &spec, signer)
if err != nil {
if c.ctx.Err() != nil {
return
}
c.fail(asFailure(err))
return
}
defer conn.release()
switch spec.Channel.Type {
case "exec":
c.runExec(conn, spec.Channel.Command)
case "websocket":
c.runWebSocket(conn, spec.Channel.URL)
default:
c.fail(&failure{Kind: "config", Message: fmt.Sprintf("unknown channel type %q", spec.Channel.Type)})
}
}
func asFailure(err error) *failure {
if f, ok := err.(*failure); ok {
return f
}
return &failure{Kind: "network", Message: err.Error()}
}
// AuthorizedKey formats a platform public key (see Signer.PublicKey) as an
// authorized_keys line: "ecdsa-sha2-nistp256 AAAA... comment".
func AuthorizedKey(publicKey []byte, comment string) (string, error) {
key, err := sshPublicKey(publicKey)
if err != nil {
return "", err
}
return authorizedKeyLine(key, comment), nil
}
// Fingerprint returns the OpenSSH SHA-256 fingerprint ("SHA256:...") of a
// platform public key.
func Fingerprint(publicKey []byte) (string, error) {
key, err := sshPublicKey(publicKey)
if err != nil {
return "", err
}
return fingerprint(key), nil
}
+526
View File
@@ -0,0 +1,526 @@
package faceclawssh
import (
"bufio"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/gorilla/websocket"
"golang.org/x/crypto/ssh"
)
// ---- test SSH server ----
type testServer struct {
addr string
hostKey ssh.Signer
accepts atomic.Int32
listener net.Listener
}
type serverOptions struct {
password string // "" = password auth disabled
keyboardInteractive bool // offer the password via keyboard-interactive instead
authorizedKey ssh.PublicKey
}
func newTestServer(t *testing.T, opts serverOptions) *testServer {
t.Helper()
_, priv, _ := ed25519.GenerateKey(rand.Reader)
hostKey, err := ssh.NewSignerFromKey(priv)
if err != nil {
t.Fatal(err)
}
config := &ssh.ServerConfig{}
if opts.password != "" && !opts.keyboardInteractive {
config.PasswordCallback = func(_ ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
if string(password) == opts.password {
return nil, nil
}
return nil, fmt.Errorf("wrong password")
}
}
if opts.password != "" && opts.keyboardInteractive {
config.KeyboardInteractiveCallback = func(_ ssh.ConnMetadata, challenge ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
answers, err := challenge("", "", []string{"Password: "}, []bool{false})
if err != nil {
return nil, err
}
if len(answers) == 1 && answers[0] == opts.password {
return nil, nil
}
return nil, fmt.Errorf("wrong password")
}
}
if opts.authorizedKey != nil {
config.PublicKeyCallback = func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
if string(key.Marshal()) == string(opts.authorizedKey.Marshal()) {
return nil, nil
}
return nil, fmt.Errorf("unknown key")
}
}
config.AddHostKey(hostKey)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
server := &testServer{addr: listener.Addr().String(), hostKey: hostKey, listener: listener}
t.Cleanup(func() { listener.Close() })
go func() {
for {
tcp, err := listener.Accept()
if err != nil {
return
}
server.accepts.Add(1)
go server.serve(tcp, config)
}
}()
return server
}
func (s *testServer) serve(tcp net.Conn, config *ssh.ServerConfig) {
conn, chans, reqs, err := ssh.NewServerConn(tcp, config)
if err != nil {
tcp.Close()
return
}
defer conn.Close()
go ssh.DiscardRequests(reqs) // replies false to keepalives, which still counts as alive
for newChannel := range chans {
switch newChannel.ChannelType() {
case "session":
go serveSession(newChannel)
case "direct-tcpip":
go serveDirectTCPIP(newChannel)
default:
newChannel.Reject(ssh.UnknownChannelType, "nope")
}
}
}
// serveSession implements a few fake commands:
// - "echo": echo stdin lines back with a "> " prefix until EOF, exit 0
// - "bytes": print an invalid-UTF-8 line and a stderr line, exit 3
func serveSession(newChannel ssh.NewChannel) {
channel, requests, err := newChannel.Accept()
if err != nil {
return
}
defer channel.Close()
for req := range requests {
if req.Type != "exec" {
req.Reply(false, nil)
continue
}
var payload struct{ Command string }
ssh.Unmarshal(req.Payload, &payload)
req.Reply(true, nil)
status := 0
switch payload.Command {
case "echo":
scanner := bufio.NewScanner(channel)
for scanner.Scan() {
fmt.Fprintf(channel, "> %s\n", scanner.Text())
}
case "bytes":
channel.Write([]byte("%output %1 caf\xc3\n%output %1 \xa9 ok\nplain ✓\n"))
channel.Stderr().Write([]byte("oops\n"))
status = 3
default:
channel.Stderr().Write([]byte("unknown command\n"))
status = 127
}
channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{uint32(status)}))
return
}
}
func serveDirectTCPIP(newChannel ssh.NewChannel) {
var target struct {
Host string
Port uint32
OriginHost string
OriginPort uint32
}
if err := ssh.Unmarshal(newChannel.ExtraData(), &target); err != nil {
newChannel.Reject(ssh.ConnectionFailed, "bad payload")
return
}
upstream, err := net.Dial("tcp", net.JoinHostPort(target.Host, fmt.Sprint(target.Port)))
if err != nil {
newChannel.Reject(ssh.ConnectionFailed, err.Error())
return
}
channel, requests, err := newChannel.Accept()
if err != nil {
upstream.Close()
return
}
go ssh.DiscardRequests(requests)
go func() { io.Copy(upstream, channel); upstream.Close() }()
io.Copy(channel, upstream)
channel.Close()
}
// ---- test client plumbing ----
type event struct{ kind, payload string }
type recorder struct {
events chan event
}
func newRecorder() *recorder { return &recorder{events: make(chan event, 256)} }
func (r *recorder) OnEvent(kind, payload string) { r.events <- event{kind, payload} }
func (r *recorder) next(t *testing.T) event {
t.Helper()
select {
case e := <-r.events:
return e
case <-time.After(10 * time.Second):
t.Fatal("timed out waiting for event")
return event{}
}
}
func (r *recorder) expect(t *testing.T, kind string) event {
t.Helper()
e := r.next(t)
if e.kind != kind {
t.Fatalf("got event %s(%s), want %s", e.kind, e.payload, kind)
}
return e
}
func (r *recorder) expectFailure(t *testing.T) failure {
t.Helper()
e := r.expect(t, "failure")
var f failure
if err := json.Unmarshal([]byte(e.payload), &f); err != nil {
t.Fatal(err)
}
return f
}
type softwareSigner struct{ key *ecdsa.PrivateKey }
func newSoftwareSigner(t *testing.T) *softwareSigner {
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
return &softwareSigner{key: key}
}
func (s *softwareSigner) PublicKey() []byte {
der, _ := x509.MarshalPKIXPublicKey(&s.key.PublicKey)
return der
}
func (s *softwareSigner) Sign(data []byte) ([]byte, error) {
digest := sha256.Sum256(data)
return ecdsa.SignASN1(rand.Reader, s.key, digest[:])
}
func (s *softwareSigner) sshKey(t *testing.T) ssh.PublicKey {
key, err := ssh.NewPublicKey(&s.key.PublicKey)
if err != nil {
t.Fatal(err)
}
return key
}
func specFor(server *testServer, mutate func(*openSpec)) string {
host, port, _ := net.SplitHostPort(server.addr)
var portNumber int
fmt.Sscan(port, &portNumber)
spec := openSpec{
Host: host,
Port: portNumber,
User: "jim",
HostKeys: []string{base64.StdEncoding.EncodeToString(server.hostKey.PublicKey().Marshal())},
Channel: channelSpec{Type: "exec", Command: "echo"},
}
if mutate != nil {
mutate(&spec)
}
encoded, _ := json.Marshal(spec)
return string(encoded)
}
func resetPool() {
poolMu.Lock()
defer poolMu.Unlock()
for key, conn := range pool {
conn.client.Close()
delete(pool, key)
}
}
// ---- tests ----
func TestHostKeyPinning(t *testing.T) {
defer resetPool()
server := newTestServer(t, serverOptions{password: "pw"})
wantKey := base64.StdEncoding.EncodeToString(server.hostKey.PublicKey().Marshal())
rec := newRecorder()
Open(specFor(server, func(s *openSpec) { s.HostKeys = nil; s.Password = "pw" }), nil, rec)
f := rec.expectFailure(t)
if f.Kind != "hostkey-unknown" || f.HostKey == nil || f.HostKey.Key != wantKey ||
f.HostKey.Type != ssh.KeyAlgoED25519 || !strings.HasPrefix(f.HostKey.Fingerprint, "SHA256:") {
t.Fatalf("unexpected failure %+v", f)
}
_, otherPriv, _ := ed25519.GenerateKey(rand.Reader)
other, _ := ssh.NewSignerFromKey(otherPriv)
rec = newRecorder()
Open(specFor(server, func(s *openSpec) {
s.HostKeys = []string{base64.StdEncoding.EncodeToString(other.PublicKey().Marshal())}
s.Password = "pw"
}), nil, rec)
f = rec.expectFailure(t)
if f.Kind != "hostkey-mismatch" || f.HostKey.Key != wantKey {
t.Fatalf("unexpected failure %+v", f)
}
rec = newRecorder()
ch := Open(specFor(server, func(s *openSpec) { s.Password = "pw" }), nil, rec)
rec.expect(t, "open")
ch.Close()
}
func TestPasswordAuth(t *testing.T) {
for _, interactive := range []bool{false, true} {
t.Run(fmt.Sprintf("keyboardInteractive=%v", interactive), func(t *testing.T) {
defer resetPool()
server := newTestServer(t, serverOptions{password: "secret", keyboardInteractive: interactive})
rec := newRecorder()
Open(specFor(server, nil), nil, rec)
f := rec.expectFailure(t)
if f.Kind != "auth" || !f.PasswordAllowed || f.PasswordTried {
t.Fatalf("no password: %+v", f)
}
rec = newRecorder()
Open(specFor(server, func(s *openSpec) { s.Password = "wrong" }), nil, rec)
f = rec.expectFailure(t)
if f.Kind != "auth" || !f.PasswordTried {
t.Fatalf("wrong password: %+v", f)
}
rec = newRecorder()
ch := Open(specFor(server, func(s *openSpec) { s.Password = "secret" }), nil, rec)
rec.expect(t, "open")
ch.Send("hi")
if e := rec.expect(t, "text"); e.payload != "> hi" {
t.Fatalf("got %q", e.payload)
}
ch.Close()
})
}
}
func TestKeyAuth(t *testing.T) {
defer resetPool()
signer := newSoftwareSigner(t)
server := newTestServer(t, serverOptions{authorizedKey: signer.sshKey(t)})
rec := newRecorder()
ch := Open(specFor(server, func(s *openSpec) { s.UseKey = true }), signer, rec)
rec.expect(t, "open")
ch.Close()
// A key the server doesn't know, and no password support.
resetPool()
rec = newRecorder()
Open(specFor(server, func(s *openSpec) { s.UseKey = true }), newSoftwareSigner(t), rec)
f := rec.expectFailure(t)
if f.Kind != "auth" || !f.KeyTried || f.PasswordAllowed {
t.Fatalf("unexpected failure %+v", f)
}
line, err := AuthorizedKey(signer.PublicKey(), "Faceclaw on my phone")
if err != nil || !strings.HasPrefix(line, "ecdsa-sha2-nistp256 AAAA") || !strings.HasSuffix(line, " Faceclaw-on-my-phone") {
t.Fatalf("AuthorizedKey = %q, %v", line, err)
}
parsed, _, _, _, err := ssh.ParseAuthorizedKey([]byte(line))
if err != nil || string(parsed.Marshal()) != string(signer.sshKey(t).Marshal()) {
t.Fatalf("authorized key does not round-trip: %v", err)
}
// The uncompressed-point form iOS hands us parses to the same key.
point := elliptic.Marshal(elliptic.P256(), signer.key.X, signer.key.Y) //nolint:staticcheck
fromPoint, err := Fingerprint(point)
fromDER, _ := Fingerprint(signer.PublicKey())
if err != nil || fromPoint != fromDER {
t.Fatalf("fingerprints differ: %s vs %s (%v)", fromPoint, fromDER, err)
}
}
func TestExecFraming(t *testing.T) {
defer resetPool()
server := newTestServer(t, serverOptions{password: "pw"})
rec := newRecorder()
Open(specFor(server, func(s *openSpec) { s.Password = "pw"; s.Channel.Command = "bytes" }), nil, rec)
rec.expect(t, "open")
var texts, stderr []string
var closed closedPayload
for done := false; !done; {
e := rec.next(t)
switch e.kind {
case "text":
texts = append(texts, e.payload)
case "stderr":
stderr = append(stderr, e.payload)
case "closed":
json.Unmarshal([]byte(e.payload), &closed)
done = true
default:
t.Fatalf("unexpected %s(%s)", e.kind, e.payload)
}
}
want := []string{`%output %1 caf\303`, `%output %1 \251 ok`, "plain ✓"}
if strings.Join(texts, "|") != strings.Join(want, "|") {
t.Fatalf("texts = %q, want %q", texts, want)
}
if len(stderr) != 1 || stderr[0] != "oops" {
t.Fatalf("stderr = %q", stderr)
}
if closed.ExitStatus == nil || *closed.ExitStatus != 3 {
t.Fatalf("closed = %+v", closed)
}
}
func TestWebSocketTunnelAndSharing(t *testing.T) {
defer resetPool()
upgrader := websocket.Upgrader{}
var serverConns sync.WaitGroup
wsServer := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
serverConns.Add(1)
defer serverConns.Done()
defer conn.Close()
for {
kind, data, err := conn.ReadMessage()
if err != nil {
return
}
if string(data) == "quit" {
conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(4000, "asked to quit"))
return
}
conn.WriteMessage(kind, append([]byte("echo:"), data...))
}
})}
wsListener, _ := net.Listen("tcp", "127.0.0.1:0")
go wsServer.Serve(wsListener)
defer wsServer.Close()
server := newTestServer(t, serverOptions{password: "pw"})
wsSpec := specFor(server, func(s *openSpec) {
s.Password = "pw"
s.Channel = channelSpec{Type: "websocket", URL: "ws://" + wsListener.Addr().String() + "/"}
})
recA, recB := newRecorder(), newRecorder()
a := Open(wsSpec, nil, recA)
b := Open(wsSpec, nil, recB)
// Sent before open: queued until the channel is up.
a.Send(`{"type":"init"}`)
recA.expect(t, "open")
recB.expect(t, "open")
if e := recA.expect(t, "text"); e.payload != `echo:{"type":"init"}` {
t.Fatalf("got %q", e.payload)
}
b.Send("quit")
e := recB.expect(t, "closed")
var closed closedPayload
json.Unmarshal([]byte(e.payload), &closed)
if closed.Code == nil || *closed.Code != 4000 || closed.Reason != "asked to quit" {
t.Fatalf("closed = %+v", closed)
}
if n := server.accepts.Load(); n != 1 {
t.Fatalf("expected both channels on one SSH connection, got %d connections", n)
}
// Close flushes queued messages before tearing down.
a.Send("last words")
a.Close()
time.Sleep(300 * time.Millisecond)
select {
case e := <-recA.events:
t.Fatalf("event after Close: %s(%s)", e.kind, e.payload)
default:
}
// Nothing listening at the forwarded address: a channel failure, not a
// network one (the SSH connection is fine).
recC := newRecorder()
Open(specFor(server, func(s *openSpec) {
s.Password = "pw"
s.Channel = channelSpec{Type: "websocket", URL: "ws://127.0.0.1:1/"}
}), nil, recC)
if f := recC.expectFailure(t); f.Kind != "channel" {
t.Fatalf("unexpected failure %+v", f)
}
}
func TestConnectionLoss(t *testing.T) {
defer resetPool()
server := newTestServer(t, serverOptions{password: "pw"})
rec := newRecorder()
Open(specFor(server, func(s *openSpec) { s.Password = "pw" }), nil, rec)
rec.expect(t, "open")
// Kill the SSH connection under the channel.
poolMu.Lock()
for _, conn := range pool {
conn.client.Close()
}
poolMu.Unlock()
if f := rec.expectFailure(t); f.Kind != "network" {
t.Fatalf("unexpected failure %+v", f)
}
// The next channel dials a fresh connection.
rec = newRecorder()
ch := Open(specFor(server, func(s *openSpec) { s.Password = "pw" }), nil, rec)
rec.expect(t, "open")
ch.Close()
if n := server.accepts.Load(); n != 2 {
t.Fatalf("expected a second connection, got %d", n)
}
}
func TestNetworkFailure(t *testing.T) {
rec := newRecorder()
Open(`{"host":"127.0.0.1","port":1,"user":"jim","channel":{"type":"exec","command":"x"}}`, nil, rec)
if f := rec.expectFailure(t); f.Kind != "network" {
t.Fatalf("unexpected failure %+v", f)
}
rec = newRecorder()
Open(`not json`, nil, rec)
if f := rec.expectFailure(t); f.Kind != "config" {
t.Fatalf("unexpected failure %+v", f)
}
}
+21
View File
@@ -0,0 +1,21 @@
module github.com/jimrandomh/faceclaw/native/ssh
go 1.26.0
require (
github.com/gorilla/websocket v1.5.3
golang.org/x/crypto v0.59.0
)
require (
golang.org/x/mobile v0.0.0-20261011034528-197770444867 // indirect
golang.org/x/mod v0.42.0 // indirect
golang.org/x/sync v0.24.0 // indirect
golang.org/x/sys v0.50.0 // indirect
golang.org/x/tools v0.52.0 // indirect
)
tool (
golang.org/x/mobile/cmd/gobind
golang.org/x/mobile/cmd/gomobile
)
+16
View File
@@ -0,0 +1,16 @@
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c=
golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U=
golang.org/x/mobile v0.0.0-20261011034528-197770444867 h1:rav73cdd7OQUPgt1aIFbZQTiPNnjfueKXvfOiLCY0Ss=
golang.org/x/mobile v0.0.0-20261011034528-197770444867/go.mod h1:HR/iBzuQmVTl/us733QPwrqHp43a8m9ZLgyKXD3ST6U=
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk=
golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4=
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
+58
View File
@@ -0,0 +1,58 @@
package faceclawssh
import (
"encoding/json"
"net"
"os"
"strconv"
"testing"
)
// TestOpenSSHInterop checks the parts of the handshake that need no
// credentials against a real sshd: key exchange, host key reporting and
// pinning, and how an authentication refusal is classified. Opt-in:
//
// FACECLAW_SSH_INTEROP_ADDR=127.0.0.1:22 go test -run Interop -v
//
// Compare the logged fingerprint with `ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub`.
// It makes one failed login attempt (no password, an unknown key) as
// FACECLAW_SSH_INTEROP_USER (default: $USER).
func TestOpenSSHInterop(t *testing.T) {
addr := os.Getenv("FACECLAW_SSH_INTEROP_ADDR")
if addr == "" {
t.Skip("FACECLAW_SSH_INTEROP_ADDR not set")
}
defer resetPool()
user := os.Getenv("FACECLAW_SSH_INTEROP_USER")
if user == "" {
user = os.Getenv("USER")
}
host, portText, err := net.SplitHostPort(addr)
if err != nil {
t.Fatal(err)
}
port, _ := strconv.Atoi(portText)
spec := func(hostKeys []string) string {
encoded, _ := json.Marshal(openSpec{
Host: host, Port: port, User: user, HostKeys: hostKeys, UseKey: true,
Channel: channelSpec{Type: "exec", Command: "true"},
})
return string(encoded)
}
rec := newRecorder()
Open(spec(nil), newSoftwareSigner(t), rec)
f := rec.expectFailure(t)
if f.Kind != "hostkey-unknown" || f.HostKey == nil {
t.Fatalf("first contact: %+v", f)
}
t.Logf("host key: %s %s", f.HostKey.Type, f.HostKey.Fingerprint)
rec = newRecorder()
Open(spec([]string{f.HostKey.Key}), newSoftwareSigner(t), rec)
auth := rec.expectFailure(t)
if auth.Kind != "auth" || !auth.KeyTried {
t.Fatalf("pinned, unknown key: %+v", auth)
}
t.Logf("auth refusal: %+v", auth)
}
+91
View File
@@ -0,0 +1,91 @@
package faceclawssh
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/sha256"
"crypto/x509"
"encoding/asn1"
"encoding/base64"
"errors"
"fmt"
"io"
"math/big"
"strings"
"golang.org/x/crypto/ssh"
)
// platformSigner adapts the platform's keystore-backed Signer to ssh.Signer.
// The platform signs SHA-256(data) with ECDSA P-256 and returns ASN.1 DER;
// SSH wants the signature as two mpints (RFC 5656 §3.1.2).
type platformSigner struct {
signer Signer
pub ssh.PublicKey
}
func newPlatformSigner(signer Signer) (*platformSigner, error) {
pub, err := sshPublicKey(signer.PublicKey())
if err != nil {
return nil, err
}
return &platformSigner{signer: signer, pub: pub}, nil
}
func (p *platformSigner) PublicKey() ssh.PublicKey { return p.pub }
func (p *platformSigner) Sign(_ io.Reader, data []byte) (*ssh.Signature, error) {
der, err := p.signer.Sign(data)
if err != nil {
return nil, fmt.Errorf("platform key signing failed: %w", err)
}
var sig struct{ R, S *big.Int }
rest, err := asn1.Unmarshal(der, &sig)
if err != nil || len(rest) != 0 || sig.R == nil || sig.S == nil || sig.R.Sign() <= 0 || sig.S.Sign() <= 0 {
return nil, errors.New("platform key returned a malformed signature")
}
blob := ssh.Marshal(struct{ R, S *big.Int }{sig.R, sig.S})
return &ssh.Signature{Format: p.pub.Type(), Blob: blob}, nil
}
// sshPublicKey parses a platform public key (SubjectPublicKeyInfo DER or an
// uncompressed P-256 point) into an ecdsa-sha2-nistp256 SSH key.
func sshPublicKey(encoded []byte) (ssh.PublicKey, error) {
var pub *ecdsa.PublicKey
if len(encoded) == 65 && encoded[0] == 4 {
parsed, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), encoded)
if err != nil {
return nil, fmt.Errorf("bad P-256 point: %v", err)
}
pub = parsed
} else {
parsed, err := x509.ParsePKIXPublicKey(encoded)
if err != nil {
return nil, fmt.Errorf("bad public key: %v", err)
}
ecdsaKey, ok := parsed.(*ecdsa.PublicKey)
if !ok {
return nil, fmt.Errorf("unsupported public key type %T", parsed)
}
pub = ecdsaKey
}
if pub.Curve != elliptic.P256() {
return nil, errors.New("only P-256 keys are supported")
}
return ssh.NewPublicKey(pub)
}
func authorizedKeyLine(key ssh.PublicKey, comment string) string {
line := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key)))
comment = strings.Join(strings.Fields(comment), "-")
if comment != "" {
line += " " + comment
}
return line
}
// fingerprint is OpenSSH's SHA256 fingerprint format (unpadded base64).
func fingerprint(key ssh.PublicKey) string {
sum := sha256.Sum256(key.Marshal())
return "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:])
}
+416
View File
@@ -0,0 +1,416 @@
package faceclawssh
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"log"
"net"
"strconv"
"strings"
"sync"
"time"
"golang.org/x/crypto/ssh"
)
// SSH connections are shared between channels with the same host, port,
// user, credentials and pinned keys: the Terminal app's session poller and
// each of its view windows ride one TCP connection. A connection with no
// channels left lingers briefly (a view window closing and reopening shouldn't
// pay a fresh handshake) and is then closed. A keepalive probe closes
// connections whose peer has silently gone away, which fails every channel on
// them so the app's reconnect logic takes over.
const idleLinger = 15 * time.Second
type pooledConn struct {
key string
client *ssh.Client
// Guarded by poolMu.
refs int
dead bool
linger *time.Timer
}
type dialCall struct {
done chan struct{}
err error
}
var (
poolMu sync.Mutex
pool = map[string]*pooledConn{}
dialing = map[string]*dialCall{}
)
func poolKey(spec *openSpec, signer *platformSigner) string {
h := sha256.New()
fmt.Fprintf(h, "%s\x00%d\x00%s\x00%s\x00", spec.Host, spec.Port, spec.User, spec.Password)
for _, key := range spec.HostKeys {
fmt.Fprintf(h, "%s\x00", key)
}
if signer != nil {
h.Write(signer.pub.Marshal())
}
fmt.Fprintf(h, "\x00%d", spec.KeepaliveIntervalMs)
return hex.EncodeToString(h.Sum(nil))
}
// acquire returns a live pooled connection for spec, dialing one if needed
// (concurrent callers for the same key share a single dial). The caller must
// release() it.
func acquire(ctx context.Context, spec *openSpec, signer Signer) (*pooledConn, error) {
var ps *platformSigner
if signer != nil {
var err error
if ps, err = newPlatformSigner(signer); err != nil {
return nil, &failure{Kind: "config", Message: "client key unusable: " + err.Error()}
}
}
key := poolKey(spec, ps)
for {
poolMu.Lock()
if conn := pool[key]; conn != nil && !conn.dead {
conn.refs++
if conn.linger != nil {
conn.linger.Stop()
conn.linger = nil
}
poolMu.Unlock()
return conn, nil
}
if call := dialing[key]; call != nil {
poolMu.Unlock()
select {
case <-call.done:
case <-ctx.Done():
return nil, ctx.Err()
}
if call.err != nil {
return nil, call.err
}
continue
}
call := &dialCall{done: make(chan struct{})}
dialing[key] = call
poolMu.Unlock()
conn, err := dial(spec, ps, key)
poolMu.Lock()
delete(dialing, key)
if err == nil {
pool[key] = conn
}
call.err = err
close(call.done)
poolMu.Unlock()
if err != nil {
return nil, err
}
// Loop around to take a reference like any other caller.
}
}
func (conn *pooledConn) release() {
poolMu.Lock()
defer poolMu.Unlock()
conn.refs--
if conn.refs > 0 || conn.dead {
return
}
conn.linger = time.AfterFunc(idleLinger, func() {
poolMu.Lock()
idle := conn.refs == 0 && !conn.dead
if idle {
conn.dead = true
if pool[conn.key] == conn {
delete(pool, conn.key)
}
}
poolMu.Unlock()
if idle {
conn.client.Close()
}
})
}
// markDead removes a connection from the pool once its transport has ended,
// so the next acquire dials afresh.
func (conn *pooledConn) markDead() {
poolMu.Lock()
defer poolMu.Unlock()
conn.dead = true
if conn.linger != nil {
conn.linger.Stop()
conn.linger = nil
}
if pool[conn.key] == conn {
delete(pool, conn.key)
}
}
// hostKeyChecker pins the host key: only keys listed in the spec are
// accepted, and whatever the host presented is kept for the failure report.
type hostKeyChecker struct {
known [][]byte
presented ssh.PublicKey
rejected bool
}
func newHostKeyChecker(encoded []string) (*hostKeyChecker, error) {
checker := &hostKeyChecker{}
for _, entry := range encoded {
wire, err := base64.StdEncoding.DecodeString(strings.TrimSpace(entry))
if err != nil {
return nil, fmt.Errorf("bad pinned host key: %v", err)
}
if _, err := ssh.ParsePublicKey(wire); err != nil {
return nil, fmt.Errorf("bad pinned host key: %v", err)
}
checker.known = append(checker.known, wire)
}
return checker, nil
}
func (h *hostKeyChecker) check(_ string, _ net.Addr, key ssh.PublicKey) error {
h.presented = key
wire := key.Marshal()
for _, known := range h.known {
if bytes.Equal(known, wire) {
return nil
}
}
h.rejected = true
return errors.New("host key not trusted")
}
// algorithms lists host key algorithms with the pinned keys' types first, so
// a host with several host keys presents the one we pinned rather than
// whichever the default preference order favors.
func (h *hostKeyChecker) algorithms() []string {
var preferred []string
seen := map[string]bool{}
add := func(algorithm string) {
if !seen[algorithm] {
seen[algorithm] = true
preferred = append(preferred, algorithm)
}
}
for _, wire := range h.known {
key, err := ssh.ParsePublicKey(wire)
if err != nil {
continue
}
if key.Type() == ssh.KeyAlgoRSA {
add(ssh.KeyAlgoRSASHA512)
add(ssh.KeyAlgoRSASHA256)
continue
}
add(key.Type())
}
if len(preferred) == 0 {
return nil // the library default
}
for _, algorithm := range ssh.SupportedAlgorithms().HostKeys {
add(algorithm)
}
return preferred
}
func (h *hostKeyChecker) failure() *failure {
if h.presented == nil {
return nil
}
info := &hostKeyInfo{
Type: h.presented.Type(),
Fingerprint: fingerprint(h.presented),
Key: base64.StdEncoding.EncodeToString(h.presented.Marshal()),
}
if len(h.known) == 0 {
return &failure{Kind: "hostkey-unknown", Message: "Unknown host key " + info.Fingerprint, HostKey: info}
}
return &failure{Kind: "hostkey-mismatch", Message: "Host key changed: now " + info.Fingerprint, HostKey: info}
}
// errNoPassword aborts authentication when the host would accept a password
// but we don't have one; the "auth" failure then asks the app to prompt.
var errNoPassword = errors.New("password required")
type authState struct {
mu sync.Mutex
passwordAllowed bool
passwordTried bool
keyTried bool
}
func (a *authState) set(update func(*authState)) {
a.mu.Lock()
update(a)
a.mu.Unlock()
}
func authMethods(spec *openSpec, signer *platformSigner, state *authState) []ssh.AuthMethod {
var methods []ssh.AuthMethod
if signer != nil {
methods = append(methods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
state.set(func(a *authState) { a.keyTried = true })
return []ssh.Signer{signer}, nil
}))
}
password := spec.Password
methods = append(methods,
ssh.PasswordCallback(func() (string, error) {
state.set(func(a *authState) { a.passwordAllowed = true })
if password == "" {
return "", errNoPassword
}
state.set(func(a *authState) { a.passwordTried = true })
return password, nil
}),
// Many hosts (PAM-backed sshd, e.g. macOS) take passwords through
// keyboard-interactive. Answer hidden prompts with the password and
// visible ones with nothing.
ssh.KeyboardInteractive(func(_, _ string, questions []string, echos []bool) ([]string, error) {
if len(questions) == 0 {
return nil, nil
}
state.set(func(a *authState) { a.passwordAllowed = true })
if password == "" {
return nil, errNoPassword
}
state.set(func(a *authState) { a.passwordTried = true })
answers := make([]string, len(questions))
for i := range questions {
if !echos[i] {
answers[i] = password
}
}
return answers, nil
}),
)
return methods
}
func dial(spec *openSpec, signer *platformSigner, key string) (*pooledConn, error) {
timeout := defaultConnectTimeout
if spec.ConnectTimeoutMs > 0 {
timeout = time.Duration(spec.ConnectTimeoutMs) * time.Millisecond
}
checker, err := newHostKeyChecker(spec.HostKeys)
if err != nil {
return nil, &failure{Kind: "config", Message: err.Error()}
}
addr := net.JoinHostPort(spec.Host, strconv.Itoa(spec.Port))
dialer := net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second}
tcp, err := dialer.Dial("tcp", addr)
if err != nil {
return nil, &failure{Kind: "network", Message: describeNetError(err)}
}
// Bound the whole handshake, not just the TCP connect.
tcp.SetDeadline(time.Now().Add(timeout))
state := &authState{}
config := &ssh.ClientConfig{
User: spec.User,
Auth: authMethods(spec, signer, state),
HostKeyCallback: checker.check,
HostKeyAlgorithms: checker.algorithms(),
ClientVersion: "SSH-2.0-Faceclaw",
Timeout: timeout,
}
sshConn, chans, reqs, err := ssh.NewClientConn(tcp, addr, config)
if err != nil {
tcp.Close()
if checker.rejected {
return nil, checker.failure()
}
state.mu.Lock()
defer state.mu.Unlock()
if errors.Is(err, errNoPassword) || strings.Contains(err.Error(), "unable to authenticate") {
message := "The host refused this device's key."
switch {
case state.passwordTried:
message = "The host refused the password."
case state.passwordAllowed:
message = "The host wants a password."
case signer == nil:
message = "The host accepts neither passwords nor keys we have."
}
return nil, &failure{
Kind: "auth",
Message: message,
PasswordAllowed: state.passwordAllowed,
PasswordTried: state.passwordTried,
KeyTried: state.keyTried,
}
}
return nil, &failure{Kind: "network", Message: "SSH handshake failed: " + describeNetError(err)}
}
tcp.SetDeadline(time.Time{})
client := ssh.NewClient(sshConn, chans, reqs)
conn := &pooledConn{key: key, client: client}
log.Printf("faceclawssh: connected to %s@%s", spec.User, addr)
go func() {
err := client.Wait()
log.Printf("faceclawssh: connection to %s ended: %v", addr, err)
conn.markDead()
}()
interval := defaultKeepaliveInterval
if spec.KeepaliveIntervalMs > 0 {
interval = time.Duration(spec.KeepaliveIntervalMs) * time.Millisecond
}
go keepalive(conn, interval)
return conn, nil
}
// keepalive probes the host every interval and closes the connection if a
// probe goes unanswered for a whole interval, so a dead peer (phone switched
// networks, laptop asleep) fails the channels promptly instead of hanging.
func keepalive(conn *pooledConn, interval time.Duration) {
defer func() {
if r := recover(); r != nil {
log.Printf("faceclawssh: keepalive panic: %v", r)
conn.client.Close()
}
}()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for range ticker.C {
poolMu.Lock()
dead := conn.dead
poolMu.Unlock()
if dead {
return
}
reply := make(chan error, 1)
go func() {
_, _, err := conn.client.SendRequest("keepalive@openssh.com", true, nil)
reply <- err
}()
select {
case err := <-reply:
if err != nil {
conn.client.Close()
return
}
case <-time.After(interval):
log.Printf("faceclawssh: keepalive timed out; closing")
conn.client.Close()
return
}
}
}
func describeNetError(err error) string {
message := err.Error()
// "dial tcp 1.2.3.4:22: connect: connection refused" → keep the tail.
if index := strings.LastIndex(message, ": "); index >= 0 && strings.HasPrefix(message, "dial ") {
return message[index+2:]
}
return message
}