forked from Blendman974/faceclaw
[verified] merge upstream 25cb255b into all-features
This commit is contained in:
commit
ffcdae660d
169 files changed
+16201
-597
No files matched your search
@@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
|
||||
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
|
||||
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
|
||||
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
|
||||
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
|
||||
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
|
||||
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
|
||||
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
|
||||
|
||||
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
|
||||
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
|
||||
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
file.write(bytes, offset, length)
|
||||
size += length
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
file.seek(position)
|
||||
file.write(bytes)
|
||||
file.seek(size)
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
|
||||
private val file = java.io.RandomAccessFile(path, "r")
|
||||
override val length = file.length()
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
file.seek(position)
|
||||
var total = 0
|
||||
while (total < length) {
|
||||
val read = file.read(buffer, offset + total, length - total)
|
||||
if (read <= 0) break
|
||||
total += read
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
|
||||
expect fun deleteFile(path: String): Boolean
|
||||
|
||||
expect fun renameFile(from: String, to: String): Boolean
|
||||
|
||||
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
|
||||
interface SeekableOutput {
|
||||
/** Bytes appended so far. */
|
||||
val size: Long
|
||||
|
||||
/** Appends [length] bytes of [bytes] from [offset]. */
|
||||
fun write(bytes: ByteArray, offset: Int, length: Int)
|
||||
|
||||
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
|
||||
fun writeAt(position: Long, bytes: ByteArray)
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** A file read at arbitrary positions without loading it whole. */
|
||||
interface SeekableInput {
|
||||
val length: Long
|
||||
|
||||
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
|
||||
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
|
||||
expect fun openFileOutput(path: String): SeekableOutput
|
||||
|
||||
/** Opens [path] for reading. Throws when it cannot. */
|
||||
expect fun openFileInput(path: String): SeekableInput
|
||||
@@ -0,0 +1,764 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
/**
|
||||
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
|
||||
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
|
||||
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
|
||||
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
|
||||
*
|
||||
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
|
||||
* about a second each, cues and the segment size written when the recording finishes. A
|
||||
* recording cut short by a crash is still a valid live-style stream of everything up to its last
|
||||
* complete cluster, and the reader takes it as such.
|
||||
*/
|
||||
internal object MatroskaIds {
|
||||
const val EBML = 0x1A45DFA3
|
||||
const val EBML_VERSION = 0x4286
|
||||
const val EBML_READ_VERSION = 0x42F7
|
||||
const val EBML_MAX_ID_LENGTH = 0x42F2
|
||||
const val EBML_MAX_SIZE_LENGTH = 0x42F3
|
||||
const val DOC_TYPE = 0x4282
|
||||
const val DOC_TYPE_VERSION = 0x4287
|
||||
const val DOC_TYPE_READ_VERSION = 0x4285
|
||||
const val VOID = 0xEC
|
||||
const val SEGMENT = 0x18538067
|
||||
const val SEEK_HEAD = 0x114D9B74
|
||||
const val SEEK = 0x4DBB
|
||||
const val SEEK_ID = 0x53AB
|
||||
const val SEEK_POSITION = 0x53AC
|
||||
const val INFO = 0x1549A966
|
||||
const val TIMESTAMP_SCALE = 0x2AD7B1
|
||||
const val DURATION = 0x4489
|
||||
const val DATE_UTC = 0x4461
|
||||
const val TITLE = 0x7BA9
|
||||
const val MUXING_APP = 0x4D80
|
||||
const val WRITING_APP = 0x5741
|
||||
const val TRACKS = 0x1654AE6B
|
||||
const val TRACK_ENTRY = 0xAE
|
||||
const val TRACK_NUMBER = 0xD7
|
||||
const val TRACK_UID = 0x73C5
|
||||
const val TRACK_TYPE = 0x83
|
||||
const val FLAG_DEFAULT = 0x88
|
||||
const val FLAG_LACING = 0x9C
|
||||
const val NAME = 0x536E
|
||||
const val LANGUAGE = 0x22B59C
|
||||
const val CODEC_ID = 0x86
|
||||
const val AUDIO = 0xE1
|
||||
const val SAMPLING_FREQUENCY = 0xB5
|
||||
const val CHANNELS = 0x9F
|
||||
const val BIT_DEPTH = 0x6264
|
||||
const val ATTACHMENTS = 0x1941A469
|
||||
const val ATTACHED_FILE = 0x61A7
|
||||
const val FILE_NAME = 0x466E
|
||||
const val FILE_MIME_TYPE = 0x4660
|
||||
const val FILE_DATA = 0x465C
|
||||
const val FILE_UID = 0x46AE
|
||||
const val CLUSTER = 0x1F43B675
|
||||
const val TIMESTAMP = 0xE7
|
||||
const val SIMPLE_BLOCK = 0xA3
|
||||
const val BLOCK_GROUP = 0xA0
|
||||
const val BLOCK = 0xA1
|
||||
const val BLOCK_DURATION = 0x9B
|
||||
const val CUES = 0x1C53BB6B
|
||||
const val CUE_POINT = 0xBB
|
||||
const val CUE_TIME = 0xB3
|
||||
const val CUE_TRACK_POSITIONS = 0xB7
|
||||
const val CUE_TRACK = 0xF7
|
||||
const val CUE_CLUSTER_POSITION = 0xF1
|
||||
}
|
||||
|
||||
/** Builds EBML elements into a byte buffer. */
|
||||
internal class EbmlWriter(capacity: Int = 256) {
|
||||
private val sink = ByteSink(capacity)
|
||||
|
||||
fun size(): Int = sink.size()
|
||||
|
||||
fun bytes(): ByteArray = sink.toByteArray()
|
||||
|
||||
/** An element ID, written as given: its length marker is part of the value. */
|
||||
fun id(id: Int) {
|
||||
if (id > 0xFFFFFF) sink.write(id ushr 24)
|
||||
if (id > 0xFFFF) sink.write(id ushr 16)
|
||||
if (id > 0xFF) sink.write(id ushr 8)
|
||||
sink.write(id)
|
||||
}
|
||||
|
||||
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
|
||||
fun dataSize(size: Long, width: Int = 0) {
|
||||
var bytes = width
|
||||
if (bytes == 0) {
|
||||
bytes = 1
|
||||
// The all-ones value of each width means "unknown size".
|
||||
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
|
||||
}
|
||||
val marked = size or (1L shl (7 * bytes))
|
||||
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun uint(id: Int, value: Long) {
|
||||
var bytes = 1
|
||||
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
|
||||
id(id)
|
||||
dataSize(bytes.toLong())
|
||||
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
/** A fixed eight-byte integer (dates). */
|
||||
fun int64(id: Int, value: Long) {
|
||||
id(id)
|
||||
dataSize(8)
|
||||
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
|
||||
|
||||
fun string(id: Int, text: String) {
|
||||
val bytes = text.encodeToByteArray()
|
||||
binary(id, bytes, 0, bytes.size)
|
||||
}
|
||||
|
||||
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
|
||||
id(id)
|
||||
dataSize(length.toLong())
|
||||
sink.write(bytes, offset, length)
|
||||
}
|
||||
|
||||
fun raw(bytes: ByteArray) = sink.write(bytes)
|
||||
|
||||
fun raw(value: Int) = sink.write(value)
|
||||
|
||||
fun master(id: Int, build: EbmlWriter.() -> Unit) {
|
||||
val child = EbmlWriter()
|
||||
child.build()
|
||||
id(id)
|
||||
dataSize(child.size().toLong())
|
||||
sink.write(child.bytes())
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
|
||||
fun padded(element: ByteArray, reserved: Int): ByteArray? {
|
||||
val filler = reserved - element.size
|
||||
// A Void element is at least two bytes.
|
||||
if (filler < 0 || filler == 1) return null
|
||||
val out = element.copyOf(reserved)
|
||||
if (filler == 0) return out
|
||||
val at = element.size
|
||||
out[at] = MatroskaIds.VOID.toByte()
|
||||
if (filler < 9) {
|
||||
out[at + 1] = (0x80 or (filler - 2)).toByte()
|
||||
} else {
|
||||
val size = (filler - 9).toLong()
|
||||
out[at + 1] = 0x01
|
||||
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
|
||||
|
||||
/**
|
||||
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
|
||||
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
|
||||
* milliseconds from the start of the file. Not thread-safe.
|
||||
*/
|
||||
class MkaWriter(
|
||||
private val output: SeekableOutput,
|
||||
title: String?,
|
||||
/** When the recording began (Unix epoch ms), for the file's date. */
|
||||
dateUtcMs: Long,
|
||||
attachments: List<MkaAttachment>,
|
||||
) {
|
||||
companion object {
|
||||
const val CODEC_PCM = "A_PCM/INT/LIT"
|
||||
const val CODEC_TEXT = "S_TEXT/UTF8"
|
||||
const val TRACK_AUDIO = 2
|
||||
const val TRACK_SUBTITLE = 0x11
|
||||
const val APP_NAME = "Faceclaw"
|
||||
private const val SEEK_RESERVED = 128
|
||||
private const val TRACKS_RESERVED = 4096
|
||||
private const val CLUSTER_SPAN_MS = 1000L
|
||||
private const val CLUSTER_MAX_BYTES = 1 shl 20
|
||||
// Block timestamps are signed 16-bit offsets from their cluster's.
|
||||
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
|
||||
private const val DURATION_PATCH_CLUSTERS = 5
|
||||
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
|
||||
private const val DATE_EPOCH_MS = 978_307_200_000L
|
||||
}
|
||||
|
||||
private class Track(
|
||||
val number: Int,
|
||||
val type: Int,
|
||||
val name: String,
|
||||
val sampleRate: Int,
|
||||
val channels: Int,
|
||||
val preferred: Boolean,
|
||||
)
|
||||
|
||||
private class Cue(val clusterPosition: Long, val timeMs: Long)
|
||||
|
||||
private val tracks = ArrayList<Track>()
|
||||
private val cues = ArrayList<Cue>()
|
||||
private val segmentSizePosition: Long
|
||||
private val segmentStart: Long
|
||||
private val seekPosition: Long
|
||||
private val infoPosition: Long
|
||||
private val durationPosition: Long
|
||||
private val tracksPosition: Long
|
||||
private val attachmentsPosition: Long
|
||||
private var cluster: EbmlWriter? = null
|
||||
private var clusterBaseMs = 0L
|
||||
private var clustersSincePatch = 0
|
||||
private var finished = false
|
||||
private val hasAttachments = attachments.isNotEmpty()
|
||||
|
||||
/** The end of the latest audio or text written, in ms from the start of the file. */
|
||||
var durationMs = 0L
|
||||
private set
|
||||
|
||||
init {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.EBML) {
|
||||
uint(MatroskaIds.EBML_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_READ_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
|
||||
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
|
||||
string(MatroskaIds.DOC_TYPE, "matroska")
|
||||
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
|
||||
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
|
||||
}
|
||||
head.id(MatroskaIds.SEGMENT)
|
||||
segmentSizePosition = head.size().toLong()
|
||||
// Unknown size (a live stream) until finish() knows it.
|
||||
head.raw(0x01)
|
||||
repeat(7) { head.raw(0xFF) }
|
||||
segmentStart = head.size().toLong()
|
||||
append(head.bytes())
|
||||
|
||||
seekPosition = output.size
|
||||
infoPosition = seekPosition + SEEK_RESERVED
|
||||
val info = EbmlWriter()
|
||||
info.master(MatroskaIds.INFO) {
|
||||
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
|
||||
string(MatroskaIds.MUXING_APP, APP_NAME)
|
||||
string(MatroskaIds.WRITING_APP, APP_NAME)
|
||||
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
|
||||
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
|
||||
// Last, so its eight bytes are the element's final ones.
|
||||
float64(MatroskaIds.DURATION, 0.0)
|
||||
}
|
||||
durationPosition = infoPosition + info.size() - 8
|
||||
tracksPosition = infoPosition + info.size()
|
||||
attachmentsPosition = tracksPosition + TRACKS_RESERVED
|
||||
|
||||
append(seekHead(-1))
|
||||
append(info.bytes())
|
||||
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
|
||||
if (attachments.isNotEmpty()) {
|
||||
val files = EbmlWriter()
|
||||
files.master(MatroskaIds.ATTACHMENTS) {
|
||||
attachments.forEachIndexed { index, file ->
|
||||
master(MatroskaIds.ATTACHED_FILE) {
|
||||
string(MatroskaIds.FILE_NAME, file.name)
|
||||
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
|
||||
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
|
||||
uint(MatroskaIds.FILE_UID, index + 1L)
|
||||
}
|
||||
}
|
||||
}
|
||||
append(files.bytes())
|
||||
}
|
||||
}
|
||||
|
||||
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
|
||||
|
||||
private fun seekHead(cuesPosition: Long): ByteArray {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.SEEK_HEAD) {
|
||||
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
|
||||
val idBytes = EbmlWriter()
|
||||
idBytes.id(id)
|
||||
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
|
||||
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
|
||||
}
|
||||
entry(MatroskaIds.INFO, infoPosition)
|
||||
entry(MatroskaIds.TRACKS, tracksPosition)
|
||||
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
|
||||
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
|
||||
}
|
||||
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
|
||||
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
|
||||
* the one players pick by default.
|
||||
*/
|
||||
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
|
||||
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
|
||||
|
||||
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
|
||||
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
|
||||
|
||||
private fun addTrack(track: Track): Int {
|
||||
check(!finished) { "recording already finished" }
|
||||
// A block's track number is written in one byte.
|
||||
if (track.number > 126) return -1
|
||||
tracks.add(track)
|
||||
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
|
||||
if (region == null) {
|
||||
tracks.remove(track)
|
||||
return -1
|
||||
}
|
||||
output.writeAt(tracksPosition, region)
|
||||
return track.number
|
||||
}
|
||||
|
||||
private fun listedTracks(): List<Track> =
|
||||
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
|
||||
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
|
||||
tracks.filter { it.type != TRACK_AUDIO }
|
||||
|
||||
private fun tracksElement(): ByteArray {
|
||||
val listed = listedTracks()
|
||||
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
|
||||
val out = EbmlWriter(1024)
|
||||
out.master(MatroskaIds.TRACKS) {
|
||||
for (track in listed) {
|
||||
master(MatroskaIds.TRACK_ENTRY) {
|
||||
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_UID, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
|
||||
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
|
||||
uint(MatroskaIds.FLAG_LACING, 0)
|
||||
string(MatroskaIds.NAME, track.name)
|
||||
string(MatroskaIds.LANGUAGE, "und")
|
||||
if (track.type == TRACK_AUDIO) {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_PCM)
|
||||
master(MatroskaIds.AUDIO) {
|
||||
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
|
||||
uint(MatroskaIds.CHANNELS, track.channels.toLong())
|
||||
uint(MatroskaIds.BIT_DEPTH, 16)
|
||||
}
|
||||
} else {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.bytes()
|
||||
}
|
||||
|
||||
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
|
||||
|
||||
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
|
||||
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
|
||||
val track = track(trackNumber)
|
||||
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
|
||||
if (length <= 0) return
|
||||
val block = openCluster(timeMs)
|
||||
block.id(MatroskaIds.SIMPLE_BLOCK)
|
||||
block.dataSize(4L + length)
|
||||
blockHeader(block, trackNumber, timeMs, 0x80)
|
||||
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
|
||||
val frames = length / (2 * track.channels)
|
||||
noteEnd(timeMs + frames * 1000L / track.sampleRate)
|
||||
}
|
||||
|
||||
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
|
||||
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
|
||||
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
|
||||
val bytes = text.encodeToByteArray()
|
||||
val block = openCluster(timeMs)
|
||||
block.master(MatroskaIds.BLOCK_GROUP) {
|
||||
id(MatroskaIds.BLOCK)
|
||||
dataSize(4L + bytes.size)
|
||||
blockHeader(this, trackNumber, timeMs, 0)
|
||||
raw(bytes)
|
||||
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
|
||||
}
|
||||
noteEnd(timeMs + maxOf(1L, durationMs))
|
||||
}
|
||||
|
||||
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
|
||||
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
|
||||
block.raw(0x80 or trackNumber)
|
||||
block.raw(relative shr 8)
|
||||
block.raw(relative)
|
||||
block.raw(flags)
|
||||
}
|
||||
|
||||
private fun noteEnd(endMs: Long) {
|
||||
if (endMs > durationMs) durationMs = endMs
|
||||
}
|
||||
|
||||
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
|
||||
private fun openCluster(timeMs: Long): EbmlWriter {
|
||||
check(!finished) { "recording already finished" }
|
||||
val time = maxOf(0L, timeMs)
|
||||
val open = cluster
|
||||
if (open != null) {
|
||||
val relative = time - clusterBaseMs
|
||||
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
|
||||
flushCluster()
|
||||
}
|
||||
clusterBaseMs = time
|
||||
return EbmlWriter(64 * 1024).also { cluster = it }
|
||||
}
|
||||
|
||||
private fun flushCluster() {
|
||||
val body = cluster ?: return
|
||||
cluster = null
|
||||
if (body.size() == 0) return
|
||||
val timestamp = EbmlWriter(16)
|
||||
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
|
||||
val header = EbmlWriter(32)
|
||||
header.id(MatroskaIds.CLUSTER)
|
||||
header.dataSize(timestamp.size().toLong() + body.size())
|
||||
header.raw(timestamp.bytes())
|
||||
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
|
||||
append(header.bytes())
|
||||
append(body.bytes())
|
||||
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
|
||||
}
|
||||
|
||||
private fun patchDuration() {
|
||||
clustersSincePatch = 0
|
||||
val value = durationMs.toDouble().toRawBits()
|
||||
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
|
||||
}
|
||||
|
||||
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
|
||||
fun finish() {
|
||||
if (finished) return
|
||||
try {
|
||||
flushCluster()
|
||||
finished = true
|
||||
val cuesPosition = output.size
|
||||
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
|
||||
if (cues.isNotEmpty()) {
|
||||
val index = EbmlWriter(cues.size * 16 + 16)
|
||||
index.master(MatroskaIds.CUES) {
|
||||
for (cue in cues) {
|
||||
master(MatroskaIds.CUE_POINT) {
|
||||
uint(MatroskaIds.CUE_TIME, cue.timeMs)
|
||||
master(MatroskaIds.CUE_TRACK_POSITIONS) {
|
||||
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
|
||||
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
append(index.bytes())
|
||||
output.writeAt(seekPosition, seekHead(cuesPosition))
|
||||
}
|
||||
patchDuration()
|
||||
val size = EbmlWriter(8)
|
||||
size.dataSize(output.size - segmentStart, 8)
|
||||
output.writeAt(segmentSizePosition, size.bytes())
|
||||
} finally {
|
||||
finished = true
|
||||
output.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaTrackInfo(
|
||||
@JvmField val number: Int,
|
||||
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
|
||||
@JvmField val type: Int,
|
||||
@JvmField val codecId: String,
|
||||
@JvmField val name: String,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val bitDepth: Int,
|
||||
) {
|
||||
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
|
||||
}
|
||||
|
||||
class MkaBlock(
|
||||
@JvmField val track: Int,
|
||||
@JvmField val timeMs: Long,
|
||||
@JvmField val data: ByteArray,
|
||||
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
|
||||
@JvmField val durationMs: Long,
|
||||
)
|
||||
|
||||
/**
|
||||
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
|
||||
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
|
||||
* IllegalArgumentException for anything that is not such a file.
|
||||
*/
|
||||
class MkaReader(private val input: SeekableInput) {
|
||||
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
|
||||
|
||||
private val clusters = ArrayList<ClusterRef>()
|
||||
private val trackList = ArrayList<MkaTrackInfo>()
|
||||
private val attachmentList = ArrayList<MkaAttachment>()
|
||||
private var timestampScale = 1_000_000L
|
||||
private var infoDurationMs = 0L
|
||||
private var nextCluster = 0
|
||||
private var blocks: List<MkaBlock> = emptyList()
|
||||
private var blockIndex = 0
|
||||
private var skipBeforeMs = 0L
|
||||
|
||||
val tracks: List<MkaTrackInfo> get() = trackList
|
||||
val attachments: List<MkaAttachment> get() = attachmentList
|
||||
|
||||
/** The end of the last audio or text in the file, in ms. */
|
||||
val durationMs: Long
|
||||
|
||||
init {
|
||||
scan()
|
||||
var end = infoDurationMs
|
||||
if (clusters.isNotEmpty()) {
|
||||
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
|
||||
}
|
||||
durationMs = end
|
||||
}
|
||||
|
||||
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
|
||||
|
||||
/** Continue from the first block at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
|
||||
nextCluster = maxOf(0, index)
|
||||
blocks = emptyList()
|
||||
blockIndex = 0
|
||||
skipBeforeMs = timeMs
|
||||
}
|
||||
|
||||
/** The next block, or null at the end of the file. */
|
||||
fun next(): MkaBlock? {
|
||||
while (true) {
|
||||
while (blockIndex >= blocks.size) {
|
||||
if (nextCluster >= clusters.size) return null
|
||||
blocks = readCluster(clusters[nextCluster++])
|
||||
blockIndex = 0
|
||||
}
|
||||
val block = blocks[blockIndex++]
|
||||
if (block.timeMs >= skipBeforeMs) return block
|
||||
}
|
||||
}
|
||||
|
||||
fun close() = input.close()
|
||||
|
||||
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
|
||||
|
||||
private fun blockSpanMs(block: MkaBlock): Long {
|
||||
if (block.durationMs >= 0) return block.durationMs
|
||||
val track = track(block.track) ?: return 0
|
||||
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
|
||||
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
|
||||
}
|
||||
|
||||
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
|
||||
private class Header(val id: Int, val size: Long, val length: Int)
|
||||
|
||||
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
|
||||
if (at >= end) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val idLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (idLength > 4 || at + idLength >= end) return null
|
||||
var id = 0
|
||||
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
|
||||
val sizeAt = at + idLength
|
||||
val sizeFirst = bytes[sizeAt].toInt() and 0xff
|
||||
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
|
||||
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
|
||||
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
|
||||
var allOnes = size == (0xff shr sizeLength).toLong()
|
||||
for (index in 1 until sizeLength) {
|
||||
val byte = bytes[sizeAt + index].toInt() and 0xff
|
||||
if (byte != 0xff) allOnes = false
|
||||
size = (size shl 8) or byte.toLong()
|
||||
}
|
||||
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
|
||||
}
|
||||
|
||||
private fun readAt(position: Long, length: Int): ByteArray {
|
||||
val buffer = ByteArray(length)
|
||||
val read = input.readAt(position, buffer, 0, length)
|
||||
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
|
||||
}
|
||||
|
||||
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
|
||||
var value = 0L
|
||||
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
|
||||
return value
|
||||
}
|
||||
|
||||
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
|
||||
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
|
||||
8 -> Double.fromBits(uint(bytes, at, 8))
|
||||
else -> 0.0
|
||||
}
|
||||
|
||||
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
|
||||
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
|
||||
var at = start
|
||||
while (true) {
|
||||
val child = header(bytes, at, end) ?: return
|
||||
val payload = at + child.length
|
||||
if (child.size < 0 || child.size > end - payload) return
|
||||
visit(child.id, payload, child.size.toInt())
|
||||
at = payload + child.size.toInt()
|
||||
}
|
||||
}
|
||||
|
||||
private fun scan() {
|
||||
val length = input.length
|
||||
val start = readAt(0, 64)
|
||||
val ebml = header(start, 0, start.size)
|
||||
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
|
||||
var position = ebml.length + ebml.size
|
||||
val segment = readAt(position, 12).let { header(it, 0, it.size) }
|
||||
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
|
||||
position += segment.length
|
||||
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
|
||||
while (position < end) {
|
||||
val head = readAt(position, 12)
|
||||
val element = header(head, 0, head.size) ?: break
|
||||
val payload = position + element.length
|
||||
if (element.id == MatroskaIds.CLUSTER) {
|
||||
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
|
||||
// A recording cut short ends in a partial cluster; its complete blocks still count.
|
||||
val available = minOf(element.size, end - payload)
|
||||
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
|
||||
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
|
||||
} else if (element.size < 0) {
|
||||
break
|
||||
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
|
||||
when (element.id) {
|
||||
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
|
||||
}
|
||||
}
|
||||
if (element.size < 0) break
|
||||
position = payload + element.size
|
||||
}
|
||||
}
|
||||
|
||||
private fun clusterTime(position: Long, size: Int): Long {
|
||||
val head = readAt(position, minOf(size, 16))
|
||||
val first = header(head, 0, head.size)
|
||||
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
|
||||
return uint(head, first.length, first.size.toInt())
|
||||
}
|
||||
val whole = readAt(position, size)
|
||||
var time = 0L
|
||||
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
|
||||
return time
|
||||
}
|
||||
|
||||
private fun parseInfo(bytes: ByteArray) {
|
||||
var duration = 0.0
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
|
||||
MatroskaIds.DURATION -> duration = float(bytes, at, size)
|
||||
}
|
||||
}
|
||||
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
|
||||
}
|
||||
|
||||
private fun parseTracks(bytes: ByteArray) {
|
||||
trackList.clear()
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.TRACK_ENTRY) {
|
||||
var number = 0
|
||||
var type = 0
|
||||
var codec = ""
|
||||
var name = ""
|
||||
var rate = 8000.0
|
||||
var channels = 1
|
||||
var depth = 0
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
|
||||
when (audio) {
|
||||
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
|
||||
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
|
||||
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val audio = type == MkaWriter.TRACK_AUDIO
|
||||
trackList.add(
|
||||
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseAttachments(bytes: ByteArray) {
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.ATTACHED_FILE) {
|
||||
var name = ""
|
||||
var mime = ""
|
||||
var data = ByteArray(0)
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
|
||||
}
|
||||
}
|
||||
attachmentList.add(MkaAttachment(name, mime, data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
|
||||
val bytes = readAt(cluster.position, cluster.size)
|
||||
val out = ArrayList<MkaBlock>()
|
||||
var clusterRaw = cluster.rawTime
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
|
||||
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
|
||||
MatroskaIds.BLOCK_GROUP -> {
|
||||
var blockAt = -1
|
||||
var blockSize = 0
|
||||
var duration = -1L
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.BLOCK -> {
|
||||
blockAt = fieldAt
|
||||
blockSize = fieldSize
|
||||
}
|
||||
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
|
||||
}
|
||||
}
|
||||
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
|
||||
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
|
||||
if (size < 4) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val trackLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (trackLength > 2 || size < trackLength + 3) return null
|
||||
var track = first and (0xff shr trackLength)
|
||||
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
|
||||
val head = at + trackLength
|
||||
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
|
||||
val flags = bytes[head + 2].toInt() and 0xff
|
||||
if (flags and 0x06 != 0) return null
|
||||
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
|
||||
/**
|
||||
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
|
||||
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
|
||||
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
|
||||
* contract is the header of g2flash patches/mic_control.c).
|
||||
*/
|
||||
object MicArrayProtocol {
|
||||
const val VERSION = 1
|
||||
const val HEADER_BYTES = 21
|
||||
const val OP_CONFIGURE = 1
|
||||
const val OP_STOP = 3
|
||||
const val OP_RENEW = 4
|
||||
/** How long an armed temple streams without a CONFIGURE or RENEW. */
|
||||
const val LEASE_MS = 90_000L
|
||||
const val FLAG_TRUNCATED = 0x80
|
||||
const val CODEC_RAW = 1
|
||||
const val FORMAT_PCM16 = 0
|
||||
|
||||
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
|
||||
class Frame(
|
||||
/** Low 16 bits of the temple's frame counter. */
|
||||
@JvmField val sequence: Int,
|
||||
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
|
||||
@JvmField val deviceTickMs: Long,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val format: Int,
|
||||
@JvmField val codec: Int,
|
||||
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
|
||||
@JvmField val angleDegrees: Int,
|
||||
@JvmField val ssr: Int,
|
||||
/** The firmware cut the payload short of the samples it captured. */
|
||||
@JvmField val truncated: Boolean,
|
||||
@JvmField val payloadOffset: Int,
|
||||
@JvmField val payloadLength: Int,
|
||||
)
|
||||
|
||||
/** A control record with no payload: ['M','C', version, op]. */
|
||||
@JvmStatic
|
||||
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
|
||||
|
||||
@JvmStatic
|
||||
fun isConfigure(record: ByteArray?): Boolean =
|
||||
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
|
||||
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
|
||||
|
||||
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
|
||||
@JvmStatic
|
||||
fun decodeFrame(data: ByteArray): Frame? {
|
||||
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
|
||||
return null
|
||||
}
|
||||
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
|
||||
return Frame(
|
||||
sequence = u16(data, 4),
|
||||
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
|
||||
channels = data[10].toInt() and 0xff,
|
||||
sampleRate = u16(data, 11) * 100,
|
||||
format = data[13].toInt() and 0xff,
|
||||
codec = data[14].toInt() and 0xff,
|
||||
angleDegrees = u16(data, 15).toShort().toInt(),
|
||||
ssr = u16(data, 17).toShort().toInt(),
|
||||
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
|
||||
payloadOffset = HEADER_BYTES,
|
||||
payloadLength = length,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
|
||||
* payload isn't raw 16-bit PCM or holds none.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
|
||||
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
|
||||
val frames = frame.payloadLength / (2 * frame.channels)
|
||||
if (frames <= 0) return null
|
||||
val samples = ShortArray(frames * frame.channels)
|
||||
var at = frame.payloadOffset
|
||||
for (index in samples.indices) {
|
||||
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
|
||||
at += 2
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
|
||||
}
|
||||
@@ -0,0 +1,484 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.concurrent.Volatile
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
import kotlin.math.abs
|
||||
|
||||
/** Which microphones a chunk of capture audio came from. */
|
||||
object MicStream {
|
||||
/** One mixed channel: the glasses' stock stream, or the phone's own microphone. */
|
||||
const val MONO = 0
|
||||
/** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */
|
||||
const val LEFT = 1
|
||||
const val RIGHT = 2
|
||||
/** The mono feed derived from the temples for speech recognition: recorded, never replayed. */
|
||||
const val MIX = 3
|
||||
const val COUNT = 4
|
||||
|
||||
/** The stream's name in frame metadata. */
|
||||
@JvmStatic
|
||||
fun key(stream: Int): String = when (stream) {
|
||||
MONO -> "mic"
|
||||
LEFT -> "left"
|
||||
RIGHT -> "right"
|
||||
else -> "mix"
|
||||
}
|
||||
|
||||
/** The name of the stream's audio tracks in a recording. */
|
||||
@JvmStatic
|
||||
fun trackName(stream: Int): String = when (stream) {
|
||||
MONO -> "mic"
|
||||
LEFT -> "left temple"
|
||||
RIGHT -> "right temple"
|
||||
else -> "mix"
|
||||
}
|
||||
|
||||
/** The stream a recording's track or metadata names, or -1. */
|
||||
@JvmStatic
|
||||
fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1
|
||||
}
|
||||
|
||||
/**
|
||||
* One chunk of microphone audio as a capture source delivered it (about 50 ms), before any
|
||||
* processing. Live captures, recordings and replays all trade in these.
|
||||
*/
|
||||
class MicChunk(
|
||||
@JvmField val stream: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
/** Interleaved PCM16; the first [frames] sample frames are the chunk. */
|
||||
@JvmField val pcm: ShortArray,
|
||||
@JvmField val frames: Int,
|
||||
/** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */
|
||||
@JvmField val timeMs: Long,
|
||||
) {
|
||||
/** Whether the source reported a direction of arrival with the chunk. */
|
||||
@JvmField var hasDirection = false
|
||||
@JvmField var angleDegrees = 0
|
||||
@JvmField var ssr = 0
|
||||
/** The source's packet counter, or -1 when it has none. */
|
||||
@JvmField var sequence = -1
|
||||
/** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */
|
||||
@JvmField var deviceTickMs = -1L
|
||||
/** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */
|
||||
@JvmField var clockStream = stream
|
||||
/** The source cut the chunk short of what it captured. */
|
||||
@JvmField var truncated = false
|
||||
/** Sample frames known to be missing between the previous chunk of this stream and this one. */
|
||||
@JvmField var lostFramesBefore = 0
|
||||
|
||||
val durationMs: Long get() = frames * 1000L / sampleRate
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for
|
||||
* array processing (beamforming, noise cancellation): replays feed it the recorded chunks with
|
||||
* the times they first arrived at, so an implementation that decides from the chunks alone
|
||||
* behaves the same on a replay as it did live.
|
||||
*/
|
||||
interface MicArrayMixer {
|
||||
/** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */
|
||||
fun mix(chunk: MicChunk): MicChunk?
|
||||
|
||||
fun reset()
|
||||
}
|
||||
|
||||
/**
|
||||
* The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix).
|
||||
* The temples sample on independent clocks, so their audio is never summed. The left temple
|
||||
* feeds whenever it is streaming; the right one stands in while it is not.
|
||||
*/
|
||||
class SingleTempleMixer : MicArrayMixer {
|
||||
companion object {
|
||||
/** A temple that has delivered nothing for this long no longer counts as streaming. */
|
||||
const val SILENT_MS = 300L
|
||||
}
|
||||
|
||||
private var leftAtMs = Long.MIN_VALUE
|
||||
|
||||
override fun reset() {
|
||||
leftAtMs = Long.MIN_VALUE
|
||||
}
|
||||
|
||||
override fun mix(chunk: MicChunk): MicChunk? {
|
||||
if (chunk.stream == MicStream.LEFT) {
|
||||
leftAtMs = chunk.timeMs
|
||||
} else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) {
|
||||
return null
|
||||
}
|
||||
val channels = chunk.channels
|
||||
val mono = ShortArray(chunk.frames)
|
||||
for (frame in 0 until chunk.frames) {
|
||||
var sum = 0
|
||||
for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel]
|
||||
mono[frame] = (sum / channels).toShort()
|
||||
}
|
||||
val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs)
|
||||
out.hasDirection = chunk.hasDirection
|
||||
out.angleDegrees = chunk.angleDegrees
|
||||
out.ssr = chunk.ssr
|
||||
out.deviceTickMs = chunk.deviceTickMs
|
||||
out.clockStream = chunk.stream
|
||||
out.truncated = chunk.truncated
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and
|
||||
* layout as each first appears, plus a "frames" text track with one JSON object per chunk that
|
||||
* has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md.
|
||||
*
|
||||
* Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick,
|
||||
* or the count of samples from a source without one) says how far apart its chunks were
|
||||
* captured; the least-delayed recent arrival places that clock on the file's timeline. So a
|
||||
* chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap.
|
||||
*
|
||||
* Safe to share between the capture thread and whoever finishes the recording.
|
||||
*/
|
||||
class MicRecordingWriter(
|
||||
output: SeekableOutput,
|
||||
/** A JSON document describing the capture, stored as the file's "capture.json" attachment. */
|
||||
configJson: String?,
|
||||
/** When the recording began (Unix epoch ms). */
|
||||
startedAtMs: Long,
|
||||
platform: ProtocolPlatform = protocolPlatform(),
|
||||
private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) },
|
||||
) {
|
||||
companion object {
|
||||
const val CONFIG_ATTACHMENT = "capture.json"
|
||||
const val META_TRACK = "frames"
|
||||
/** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */
|
||||
const val DISCONTINUITY_MS = 5000L
|
||||
private const val WINDOW_BUCKETS = 8
|
||||
private const val BUCKET_MS = 1000L
|
||||
}
|
||||
|
||||
/**
|
||||
* Places one track's chunks on the file's timeline. `source` time is the stream's own clock
|
||||
* since this clock was anchored; the offset from it to arrival time is taken as the least
|
||||
* seen over the last few seconds, which is the least-delayed delivery.
|
||||
*/
|
||||
private class TrackClock(val number: Int, val sampleRate: Int) {
|
||||
private var anchored = false
|
||||
private var clockStream = -1
|
||||
private var tickBase = -1L
|
||||
private var lastTick = -1L
|
||||
private var frames = 0L
|
||||
private val minima = LongArray(WINDOW_BUCKETS)
|
||||
private var bucket = 0
|
||||
private var bucketStartMs = 0L
|
||||
var endMs = Long.MIN_VALUE
|
||||
private set
|
||||
|
||||
fun reset() {
|
||||
anchored = false
|
||||
}
|
||||
|
||||
private fun offset(): Long = minima.min()
|
||||
|
||||
fun stamp(chunk: MicChunk): Long {
|
||||
val arrival = chunk.timeMs
|
||||
val ticked = chunk.deviceTickMs >= 0
|
||||
if (anchored) {
|
||||
// Another clock (the feed moved to the other temple), or this one restarted.
|
||||
if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false
|
||||
else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false
|
||||
}
|
||||
var anchorNow = !anchored
|
||||
var source = 0L
|
||||
if (anchored) {
|
||||
frames += chunk.lostFramesBefore
|
||||
source = sourceTime(chunk)
|
||||
if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true
|
||||
}
|
||||
if (anchorNow) {
|
||||
anchored = true
|
||||
clockStream = chunk.clockStream
|
||||
tickBase = if (ticked) chunk.deviceTickMs else -1L
|
||||
frames = 0
|
||||
minima.fill(Long.MAX_VALUE)
|
||||
bucket = 0
|
||||
bucketStartMs = arrival
|
||||
source = 0
|
||||
}
|
||||
if (arrival - bucketStartMs >= BUCKET_MS) {
|
||||
bucket = (bucket + 1) % WINDOW_BUCKETS
|
||||
minima[bucket] = Long.MAX_VALUE
|
||||
bucketStartMs = arrival
|
||||
}
|
||||
if (arrival - source < minima[bucket]) minima[bucket] = arrival - source
|
||||
// Never before the end of the track's previous block.
|
||||
val time = maxOf(source + offset(), endMs)
|
||||
frames += chunk.frames
|
||||
lastTick = chunk.deviceTickMs
|
||||
endMs = time + chunk.durationMs
|
||||
return time
|
||||
}
|
||||
|
||||
private fun sourceTime(chunk: MicChunk): Long =
|
||||
if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate
|
||||
}
|
||||
|
||||
private val lock = platform.createLock()
|
||||
private val writer: MkaWriter
|
||||
private val clocks = HashMap<Long, TrackClock>()
|
||||
private var metaTrack = 0
|
||||
private var originMs = Long.MIN_VALUE
|
||||
private var lastTimeMs = 0L
|
||||
private var closed = false
|
||||
|
||||
/** The recording stopped early: the file could not be written. */
|
||||
@Volatile
|
||||
var failed = false
|
||||
private set
|
||||
|
||||
init {
|
||||
val attachments = if (configJson.isNullOrEmpty()) emptyList()
|
||||
else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray()))
|
||||
writer = MkaWriter(output, null, startedAtMs, attachments)
|
||||
}
|
||||
|
||||
/** Length of the recording so far, in ms. */
|
||||
val durationMs: Long get() = lock.withLock { writer.durationMs }
|
||||
|
||||
/** Appends [chunk] to its stream's track, adding the track on its first chunk. */
|
||||
fun write(chunk: MicChunk) {
|
||||
if (chunk.frames <= 0) return
|
||||
lock.withLock {
|
||||
if (closed) return
|
||||
try {
|
||||
val clock = clockFor(chunk) ?: return
|
||||
if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs
|
||||
val time = clock.stamp(chunk) - originMs
|
||||
val meta = metadata(chunk)
|
||||
if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta)
|
||||
val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels)
|
||||
writer.writeAudio(clock.number, time, bytes, 0, bytes.size)
|
||||
lastTimeMs = maxOf(lastTimeMs, time)
|
||||
} catch (error: Throwable) {
|
||||
fail(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Records a JSON object as an event at the recording's current position (a capture starting
|
||||
* over with another configuration, say), and has every stream's clock placed afresh by its
|
||||
* next chunk: what follows an event is not continuous with what came before.
|
||||
*/
|
||||
fun event(json: String) {
|
||||
lock.withLock {
|
||||
if (closed) return
|
||||
try {
|
||||
for (clock in clocks.values) clock.reset()
|
||||
if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json)
|
||||
} catch (error: Throwable) {
|
||||
fail(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Completes the file. Returns the recording's length in ms. */
|
||||
fun finish(): Long {
|
||||
lock.withLock {
|
||||
if (!closed) {
|
||||
closed = true
|
||||
try {
|
||||
writer.finish()
|
||||
} catch (error: Throwable) {
|
||||
failed = true
|
||||
logWarn("finishing the recording failed: ${error.message}")
|
||||
}
|
||||
}
|
||||
return writer.durationMs
|
||||
}
|
||||
}
|
||||
|
||||
private fun fail(error: Throwable) {
|
||||
failed = true
|
||||
closed = true
|
||||
logWarn("recording stopped: ${error.message}")
|
||||
try {
|
||||
writer.finish()
|
||||
} catch (ignored: Throwable) {
|
||||
// The file is whatever made it to disk.
|
||||
}
|
||||
}
|
||||
|
||||
private fun ensureMetaTrack(): Boolean {
|
||||
if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK)
|
||||
return metaTrack > 0
|
||||
}
|
||||
|
||||
private fun clockFor(chunk: MicChunk): TrackClock? {
|
||||
val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong()
|
||||
clocks[key]?.let { return it }
|
||||
// The frames track goes in with the first audio track, so it is there for that track's first chunk.
|
||||
ensureMetaTrack()
|
||||
// The mixed streams are what a player should pick; a temple's raw pair is for analysis.
|
||||
val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX
|
||||
val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred)
|
||||
if (number <= 0) {
|
||||
logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio")
|
||||
return null
|
||||
}
|
||||
return TrackClock(number, chunk.sampleRate).also { clocks[key] = it }
|
||||
}
|
||||
|
||||
/** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */
|
||||
private fun metadata(chunk: MicChunk): String? {
|
||||
if (chunk.stream == MicStream.MIX) return null
|
||||
if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null
|
||||
val text = StringBuilder(96)
|
||||
text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs)
|
||||
if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence)
|
||||
if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs)
|
||||
if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr)
|
||||
if (chunk.truncated) text.append(",\"trunc\":true")
|
||||
if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore)
|
||||
return text.append('}').toString()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO]
|
||||
* chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the
|
||||
* metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A
|
||||
* Matroska file from elsewhere plays its first PCM16 track as the mono microphone.
|
||||
*
|
||||
* Throws IllegalArgumentException when the file has no audio a replay can use.
|
||||
*/
|
||||
class MicRecordingReader(input: SeekableInput) {
|
||||
companion object {
|
||||
const val SAMPLE_RATE = 16000
|
||||
/** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */
|
||||
const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20
|
||||
}
|
||||
|
||||
private class Meta(val timeMs: Long, val fields: Map<String, Any?>)
|
||||
|
||||
private val reader = MkaReader(input)
|
||||
private val streams = HashMap<Int, Int>()
|
||||
private var metaTrack = -1
|
||||
private val pendingMeta = arrayOfNulls<Meta>(MicStream.COUNT)
|
||||
private val queued = ArrayDeque<MicChunk>()
|
||||
|
||||
val durationMs: Long get() = reader.durationMs
|
||||
|
||||
/** The capture's "capture.json" attachment, or null. */
|
||||
val configJson: String? =
|
||||
reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString()
|
||||
|
||||
/** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */
|
||||
val description: String
|
||||
|
||||
init {
|
||||
try {
|
||||
val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 }
|
||||
for (track in audio) {
|
||||
val stream = MicStream.named(track.name)
|
||||
if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream
|
||||
}
|
||||
if (streams.isEmpty()) {
|
||||
// Not one of ours: its first usable track stands in for the microphone.
|
||||
val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull()
|
||||
require(first != null) { "no 16-bit PCM audio track" }
|
||||
streams[first.number] = MicStream.MONO
|
||||
}
|
||||
for (number in streams.keys) {
|
||||
val rate = reader.track(number)!!.sampleRate
|
||||
require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" }
|
||||
}
|
||||
metaTrack = reader.tracks.firstOrNull {
|
||||
it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK
|
||||
}?.number ?: -1
|
||||
description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) ->
|
||||
val channels = reader.track(number)!!.channels
|
||||
MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else ""
|
||||
}
|
||||
} catch (error: Throwable) {
|
||||
reader.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Continue from the first chunk at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
queued.clear()
|
||||
pendingMeta.fill(null)
|
||||
reader.seekTo(timeMs)
|
||||
}
|
||||
|
||||
/** The next chunk in file order, or null at the end. */
|
||||
fun next(): MicChunk? {
|
||||
while (queued.isEmpty()) {
|
||||
val block = reader.next() ?: return null
|
||||
if (block.track == metaTrack) {
|
||||
noteMeta(block)
|
||||
continue
|
||||
}
|
||||
val stream = streams[block.track] ?: continue
|
||||
val track = reader.track(block.track) ?: continue
|
||||
queueChunks(block, stream, track)
|
||||
}
|
||||
return queued.removeFirst()
|
||||
}
|
||||
|
||||
fun close() = reader.close()
|
||||
|
||||
private fun noteMeta(block: MkaBlock) {
|
||||
try {
|
||||
val fields = Json.parseObject(block.data.decodeToString())
|
||||
val stream = MicStream.named(fields["s"].asString() ?: return)
|
||||
if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields)
|
||||
} catch (ignored: Throwable) {
|
||||
// An event or a frame we can't read: the audio plays without it.
|
||||
}
|
||||
}
|
||||
|
||||
private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) {
|
||||
val sourceChannels = track.channels
|
||||
val totalFrames = block.data.size / (2 * sourceChannels)
|
||||
if (totalFrames <= 0) return
|
||||
// The mixed microphone is one channel however a foreign file stored it.
|
||||
val channels = if (stream == MicStream.MONO) 1 else sourceChannels
|
||||
val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs }
|
||||
pendingMeta[stream] = null
|
||||
var start = 0
|
||||
while (start < totalFrames) {
|
||||
val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start)
|
||||
val pcm = ShortArray(frames * channels)
|
||||
for (frame in 0 until frames) {
|
||||
val at = (start + frame) * sourceChannels * 2
|
||||
if (channels == sourceChannels) {
|
||||
for (channel in 0 until channels) {
|
||||
pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
|
||||
}
|
||||
} else {
|
||||
var sum = 0
|
||||
for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
|
||||
pcm[frame] = (sum / sourceChannels).toShort()
|
||||
}
|
||||
}
|
||||
val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate)
|
||||
if (meta != null && start == 0) applyMeta(chunk, meta.fields)
|
||||
queued.addLast(chunk)
|
||||
start += frames
|
||||
}
|
||||
}
|
||||
|
||||
private fun applyMeta(chunk: MicChunk, fields: Map<String, Any?>) {
|
||||
chunk.sequence = fields["seq"].asInt() ?: -1
|
||||
chunk.deviceTickMs = fields["tick"].asLong() ?: -1L
|
||||
val angle = fields["angle"].asInt()
|
||||
if (angle != null) {
|
||||
chunk.hasDirection = true
|
||||
chunk.angleDegrees = angle
|
||||
chunk.ssr = fields["ssr"].asInt() ?: 0
|
||||
}
|
||||
chunk.truncated = fields["trunc"].asBoolean() ?: false
|
||||
chunk.lostFramesBefore = fields["lost"].asInt() ?: 0
|
||||
}
|
||||
}
|
||||
+393
-46
@@ -35,6 +35,12 @@ interface VoiceCaptureHost {
|
||||
|
||||
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
|
||||
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
|
||||
|
||||
/** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */
|
||||
fun micArrayLink(): MicArrayLink? = null
|
||||
|
||||
/** Opens a recording to replay; throws when it cannot be read. */
|
||||
fun openRecording(path: String): SeekableInput = openFileInput(path)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -44,6 +50,12 @@ interface VoiceCaptureHost {
|
||||
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
|
||||
* worker thread (started through the shared [startThread]); listener callbacks go through the
|
||||
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
|
||||
*
|
||||
* Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the
|
||||
* phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording
|
||||
* played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go
|
||||
* to the recorder ([setRecorder]) as they came, before any processing, which is what lets a
|
||||
* replay of the recording run the same processing again.
|
||||
*/
|
||||
class VoiceCaptureSession(
|
||||
private val host: VoiceCaptureHost,
|
||||
@@ -95,6 +107,19 @@ class VoiceCaptureSession(
|
||||
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
|
||||
/** Samples per G2 packet; the decode buffer the worker reuses. */
|
||||
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
|
||||
// Mic array upkeep: the temples stop streaming unless their lease is renewed, and a
|
||||
// temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again.
|
||||
const val ARRAY_CHECK_MS = 1000L
|
||||
const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3
|
||||
const val ARRAY_TEMPLE_SILENT_MS = 5000L
|
||||
const val ARRAY_WAITING_MS = 2000L
|
||||
const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..."
|
||||
// A replay skips stretches of the recording with no audio longer than this
|
||||
// (a recording made across a disconnect) instead of sitting through them.
|
||||
const val REPLAY_MAX_GAP_MS = 2000L
|
||||
const val REPLAY_POSITION_INTERVAL_MS = 1000L
|
||||
const val REPLAY_WAIT_SLICE_MS = 20L
|
||||
const val REPLAY_FINISHED_STATUS = "Replay finished."
|
||||
|
||||
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
|
||||
@JvmStatic
|
||||
@@ -120,6 +145,8 @@ class VoiceCaptureSession(
|
||||
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
|
||||
private enum class Source { G2, PHONE, ARRAY, REPLAY }
|
||||
|
||||
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
|
||||
val arm: String = arm ?: "?"
|
||||
}
|
||||
@@ -137,11 +164,33 @@ class VoiceCaptureSession(
|
||||
@Volatile
|
||||
private var workerThreadName: String? = null
|
||||
// Capture from the phone's own microphone instead of the G2 over BLE
|
||||
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
|
||||
// (preview-only mode). Latched into activeSource at start() (under `lock`)
|
||||
// so a mid-session setter call can't switch pipelines underneath the worker.
|
||||
@Volatile
|
||||
private var usePhoneMic = false
|
||||
private var activePhoneMic = false
|
||||
// What the next capture reads (see start()); latched into activeSource and its
|
||||
// companions under `lock`, like usePhoneMic.
|
||||
@Volatile
|
||||
private var micArrayConfigure: ByteArray? = null
|
||||
@Volatile
|
||||
private var replayPath: String? = null
|
||||
@Volatile
|
||||
private var replayStartMs = 0L
|
||||
private var activeSource = Source.G2
|
||||
private var activeConfigure: ByteArray? = null
|
||||
private var activeReplayPath: String? = null
|
||||
private var activeReplayStartMs = 0L
|
||||
// Replay controls, picked up by the worker between chunks.
|
||||
@Volatile
|
||||
private var replaySeekMs = -1L
|
||||
@Volatile
|
||||
private var replayPaused = false
|
||||
@Volatile
|
||||
private var recorder: MicRecordingWriter? = null
|
||||
@Volatile
|
||||
private var mixer: MicArrayMixer = SingleTempleMixer()
|
||||
private var lastMissingPackets = 0L
|
||||
private var arrayRateWarned = false
|
||||
private var mode = Mode.CLOUD
|
||||
@Volatile
|
||||
private var onboardModelKind = VoiceModelKind.MOONSHINE
|
||||
@@ -193,6 +242,47 @@ class VoiceCaptureSession(
|
||||
this.usePhoneMic = usePhoneMic
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture the next glasses session from the temples' own microphones instead of the stock
|
||||
* mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples
|
||||
* (it decides how many microphones each one streams). Null goes back to the mixed stream.
|
||||
*/
|
||||
fun setMicArray(configureRecord: ByteArray?) {
|
||||
this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null
|
||||
}
|
||||
|
||||
/** How the temples' chunks become the mono feed that is transcribed. */
|
||||
fun setMicArrayMixer(mixer: MicArrayMixer) {
|
||||
this.mixer = mixer
|
||||
}
|
||||
|
||||
/**
|
||||
* Have the next capture play the recording at [path] from [startMs] in real time, in place
|
||||
* of any microphone; null goes back to the microphones. A replay needs no glasses.
|
||||
*/
|
||||
fun setReplay(path: String?, startMs: Long) {
|
||||
this.replayPath = path
|
||||
this.replayStartMs = maxOf(0L, startMs)
|
||||
}
|
||||
|
||||
/** Continue the running replay from [positionMs] into its recording. */
|
||||
fun seekReplay(positionMs: Long) {
|
||||
this.replaySeekMs = maxOf(0L, positionMs)
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */
|
||||
fun setReplayPaused(paused: Boolean) {
|
||||
this.replayPaused = paused
|
||||
}
|
||||
|
||||
/**
|
||||
* Where live captures' chunks are recorded as they arrive, or null. Takes effect at once,
|
||||
* also mid-capture; the caller finishes the recording. A replay is never recorded.
|
||||
*/
|
||||
fun setRecorder(recorder: MicRecordingWriter?) {
|
||||
this.recorder = recorder
|
||||
}
|
||||
|
||||
fun setOnboardModelKind(kind: String?) {
|
||||
this.onboardModelKind = parseModelKind(kind)
|
||||
}
|
||||
@@ -258,13 +348,26 @@ class VoiceCaptureSession(
|
||||
emitStopped(captureId)
|
||||
return
|
||||
}
|
||||
if (!usePhoneMic && !host.isG2SessionReady()) {
|
||||
val path = replayPath
|
||||
val configure = micArrayConfigure
|
||||
val source = when {
|
||||
path != null -> Source.REPLAY
|
||||
usePhoneMic -> Source.PHONE
|
||||
configure != null -> Source.ARRAY
|
||||
else -> Source.G2
|
||||
}
|
||||
if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) {
|
||||
emitStatus("Voice control needs an active G2 connection.")
|
||||
emitStopped(captureId)
|
||||
return
|
||||
}
|
||||
mode = parseMode(requestedMode)
|
||||
activePhoneMic = usePhoneMic
|
||||
activeSource = source
|
||||
activeConfigure = configure
|
||||
activeReplayPath = path
|
||||
activeReplayStartMs = replayStartMs
|
||||
replaySeekMs = -1L
|
||||
replayPaused = false
|
||||
started = true
|
||||
audioStarted = false
|
||||
val finished = Latch(1, platform)
|
||||
@@ -281,21 +384,22 @@ class VoiceCaptureSession(
|
||||
*/
|
||||
fun isCapturing(): Boolean {
|
||||
var audioUp = false
|
||||
var phoneMic = false
|
||||
var source = Source.G2
|
||||
lock.withLock {
|
||||
if (!started) {
|
||||
return false
|
||||
}
|
||||
audioUp = audioStarted
|
||||
phoneMic = activePhoneMic
|
||||
source = activeSource
|
||||
}
|
||||
if (!audioUp) {
|
||||
// The worker is still bringing the mic up; report it as running so
|
||||
// a concurrent request shares it instead of restarting it.
|
||||
return true
|
||||
}
|
||||
if (phoneMic) {
|
||||
// The phone mic has no session to lose the enable to; it runs until stop().
|
||||
if (source == Source.PHONE || source == Source.REPLAY) {
|
||||
// Neither has a glasses session to lose the enable to: the phone mic runs
|
||||
// until stop(), a replay until its recording ends.
|
||||
return true
|
||||
}
|
||||
return host.isG2AudioCaptureActive()
|
||||
@@ -317,7 +421,7 @@ class VoiceCaptureSession(
|
||||
finished = workerFinished
|
||||
workerName = workerThreadName
|
||||
}
|
||||
stopG2Audio()
|
||||
stopSource()
|
||||
audioQueue.wake()
|
||||
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
|
||||
finished.await(1500)
|
||||
@@ -356,28 +460,62 @@ class VoiceCaptureSession(
|
||||
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
|
||||
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
|
||||
verifyCount = 0
|
||||
if (activePhoneMic) {
|
||||
val record = host.openPhoneMic()
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.")
|
||||
return
|
||||
mixer.reset()
|
||||
lastMissingPackets = 0
|
||||
val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..."
|
||||
when (activeSource) {
|
||||
Source.PHONE -> {
|
||||
val record = host.openPhoneMic()
|
||||
if (record == null) {
|
||||
emitStatus("Could not start the phone microphone.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
try {
|
||||
processPhoneAudio(record)
|
||||
} finally {
|
||||
record.close()
|
||||
}
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
|
||||
try {
|
||||
processPhoneAudio(record)
|
||||
} finally {
|
||||
record.close()
|
||||
Source.REPLAY -> {
|
||||
val reader = try {
|
||||
MicRecordingReader(host.openRecording(activeReplayPath!!))
|
||||
} catch (error: Throwable) {
|
||||
emitStatus("Could not open the recording: " + error.message)
|
||||
emitReplay("error", 0, 0)
|
||||
return
|
||||
}
|
||||
try {
|
||||
lock.withLock { audioStarted = true }
|
||||
logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s")
|
||||
emitStatus(listening)
|
||||
processReplay(reader)
|
||||
} finally {
|
||||
reader.close()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
packetDecoder = host.createPacketDecoder()
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.")
|
||||
return
|
||||
Source.ARRAY -> {
|
||||
val link = host.micArrayLink()
|
||||
val configure = activeConfigure
|
||||
if (link == null || configure == null || !startArray(link, configure)) {
|
||||
emitStatus("Could not start the glasses microphone array.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
processArray(link, configure, listening)
|
||||
}
|
||||
Source.G2 -> {
|
||||
packetDecoder = host.createPacketDecoder()
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.")
|
||||
return
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(listening)
|
||||
processG2Audio()
|
||||
}
|
||||
lock.withLock { audioStarted = true }
|
||||
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
|
||||
processG2Audio()
|
||||
}
|
||||
// Verification result must precede the final transcript so the
|
||||
// TS bridge can suppress a non-wearer command before it is acted
|
||||
@@ -392,7 +530,7 @@ class VoiceCaptureSession(
|
||||
logError("Voice control failed", error)
|
||||
emitStatus("Voice control failed: " + error.message)
|
||||
} finally {
|
||||
stopG2Audio()
|
||||
stopSource()
|
||||
releaseRecognizer()
|
||||
releaseDecoder()
|
||||
lock.withLock {
|
||||
@@ -432,23 +570,214 @@ class VoiceCaptureSession(
|
||||
continue
|
||||
}
|
||||
decodedSamples += count.toLong()
|
||||
// The beam filter drops packets whose firmware direction-of-arrival
|
||||
// falls outside the listening wedge (isolating the aimed talker for
|
||||
// every consumer, recognition included); the spectral noise
|
||||
// suppressor then cleans what remains before it reaches the
|
||||
// recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
val angleDegrees = currentDecoder.lastAngleDegrees
|
||||
val ssr = currentDecoder.lastSsr
|
||||
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
|
||||
emitFrameMeta(angleDegrees, ssr)
|
||||
maybeEmitAudioStats(false)
|
||||
continue
|
||||
}
|
||||
processPcmChunk(pcm, count, angleDegrees, ssr, currentDecoder.hasFrameMeta)
|
||||
val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs)
|
||||
chunk.hasDirection = currentDecoder.hasFrameMeta
|
||||
chunk.angleDegrees = currentDecoder.lastAngleDegrees
|
||||
chunk.ssr = currentDecoder.lastSsr
|
||||
// Packets the decoder found missing ahead of this one: a gap in a recording.
|
||||
val missing = currentDecoder.missingPackets
|
||||
chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt()
|
||||
lastMissingPackets = missing
|
||||
handleMonoChunk(chunk)
|
||||
maybeEmitAudioStats(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A mixed-microphone chunk from any source: recorded as it came, then on to processing
|
||||
* unless the beam filter drops it. The filter drops chunks whose firmware
|
||||
* direction-of-arrival falls outside the listening wedge (isolating the aimed talker for
|
||||
* every consumer, recognition included); the spectral noise suppressor then cleans what
|
||||
* remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
*/
|
||||
private fun handleMonoChunk(chunk: MicChunk) {
|
||||
recordChunk(chunk)
|
||||
if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) {
|
||||
emitFrameMeta(chunk.angleDegrees, chunk.ssr)
|
||||
return
|
||||
}
|
||||
processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection)
|
||||
}
|
||||
|
||||
/**
|
||||
* A chunk of one temple's own microphones: recorded as it came, then mixed into the mono
|
||||
* feed, which is recorded too (as what was transcribed) and processed like a mixed chunk.
|
||||
* The beam filter does not apply: a temple's bearing is along its own pair, not the
|
||||
* head-frame direction the filter is aimed in.
|
||||
*/
|
||||
private fun handleArrayChunk(chunk: MicChunk) {
|
||||
recordChunk(chunk)
|
||||
val feed = mixer.mix(chunk) ?: return
|
||||
recordChunk(feed)
|
||||
if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) {
|
||||
if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz")
|
||||
arrayRateWarned = true
|
||||
return
|
||||
}
|
||||
processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection)
|
||||
}
|
||||
|
||||
private fun recordChunk(chunk: MicChunk) {
|
||||
if (activeSource != Source.REPLAY) recorder?.write(chunk)
|
||||
}
|
||||
|
||||
private fun handleChunk(chunk: MicChunk) {
|
||||
if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk)
|
||||
}
|
||||
|
||||
private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean {
|
||||
resetAudioStats()
|
||||
audioQueue.clear()
|
||||
if (!startArrayForwarding(link)) return false
|
||||
return link.sendControl(configure, "configure", true, true)
|
||||
}
|
||||
|
||||
private fun startArrayForwarding(link: MicArrayLink): Boolean =
|
||||
link.startForwarding(object : FaceclawAudioPacketListener {
|
||||
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
|
||||
queueAudioPacket(data, arm, arrivalMs)
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream.
|
||||
* Between frames it keeps the streams alive: renews the temples' lease, configures a temple
|
||||
* again when it has gone quiet, and takes forwarding back when the glasses session dropped it.
|
||||
*/
|
||||
private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) {
|
||||
val startedAt = platform.elapsedRealtimeMs()
|
||||
var checkedAt = startedAt
|
||||
var renewedAt = startedAt
|
||||
// Left, right.
|
||||
val armedAt = longArrayOf(startedAt, startedAt)
|
||||
val frameAt = longArrayOf(startedAt, startedAt)
|
||||
var waiting = false
|
||||
while (started) {
|
||||
val packet = takeAudioPacket()
|
||||
val now = platform.elapsedRealtimeMs()
|
||||
if (packet != null) {
|
||||
val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1
|
||||
val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null
|
||||
val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null
|
||||
// Anything else is a stock-format packet from before arming took effect.
|
||||
if (frame != null && pcm != null) {
|
||||
frameAt[side] = now
|
||||
val frames = pcm.size / frame.channels
|
||||
decodedSamples += frames.toLong()
|
||||
val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT
|
||||
val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs)
|
||||
chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0
|
||||
chunk.angleDegrees = frame.angleDegrees
|
||||
chunk.ssr = frame.ssr
|
||||
chunk.sequence = frame.sequence
|
||||
chunk.deviceTickMs = frame.deviceTickMs
|
||||
chunk.truncated = frame.truncated
|
||||
handleArrayChunk(chunk)
|
||||
}
|
||||
}
|
||||
if (now - checkedAt >= ARRAY_CHECK_MS) {
|
||||
checkedAt = now
|
||||
if (!link.isForwarding()) {
|
||||
if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) {
|
||||
logInfo("mic array forwarding re-established")
|
||||
armedAt.fill(now)
|
||||
renewedAt = now
|
||||
}
|
||||
} else {
|
||||
if (now - renewedAt >= ARRAY_RENEW_MS) {
|
||||
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true)
|
||||
renewedAt = now
|
||||
}
|
||||
for (side in 0..1) {
|
||||
if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue
|
||||
logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again")
|
||||
link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0)
|
||||
armedAt[side] = now
|
||||
}
|
||||
}
|
||||
val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS
|
||||
if (silent != waiting) {
|
||||
waiting = silent
|
||||
emitStatus(if (silent) ARRAY_WAITING_STATUS else listening)
|
||||
}
|
||||
}
|
||||
maybeEmitAudioStats(false)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay loop: hands the recording's chunks to the pipeline at the pace they were captured,
|
||||
* so recognition (its partials, a cloud provider's stream) sees what a live capture would.
|
||||
* Jumps and pauses asked for from outside take effect between chunks.
|
||||
*/
|
||||
private fun processReplay(reader: MicRecordingReader) {
|
||||
val duration = reader.durationMs
|
||||
var position = minOf(activeReplayStartMs, duration)
|
||||
reader.seekTo(position)
|
||||
emitReplay("start", position, duration)
|
||||
var reportedAt = position
|
||||
// Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap.
|
||||
var anchorWall = -1L
|
||||
var anchorFile = 0L
|
||||
var pending: MicChunk? = null
|
||||
while (started) {
|
||||
val seek = replaySeekMs
|
||||
if (seek >= 0) {
|
||||
replaySeekMs = -1L
|
||||
// What played so far is a finished utterance; the rest starts clean.
|
||||
finishReplayedUtterance()
|
||||
position = minOf(seek, duration)
|
||||
reader.seekTo(position)
|
||||
pending = null
|
||||
anchorWall = -1L
|
||||
reportedAt = position
|
||||
emitReplay("seek", position, duration)
|
||||
}
|
||||
if (replayPaused) {
|
||||
anchorWall = -1L
|
||||
sleepMs(REPLAY_WAIT_SLICE_MS)
|
||||
continue
|
||||
}
|
||||
val chunk = pending ?: reader.next()
|
||||
if (chunk == null) {
|
||||
emitReplay("end", duration, duration)
|
||||
emitStatus(REPLAY_FINISHED_STATUS)
|
||||
return
|
||||
}
|
||||
pending = chunk
|
||||
val now = platform.elapsedRealtimeMs()
|
||||
if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) {
|
||||
anchorWall = now
|
||||
anchorFile = chunk.timeMs
|
||||
}
|
||||
val wait = anchorWall + (chunk.timeMs - anchorFile) - now
|
||||
if (wait > 0) {
|
||||
// Short slices, so a stop, jump or pause is noticed promptly.
|
||||
sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS))
|
||||
continue
|
||||
}
|
||||
pending = null
|
||||
position = chunk.timeMs
|
||||
decodedSamples += chunk.frames.toLong()
|
||||
handleChunk(chunk)
|
||||
if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) {
|
||||
reportedAt = position
|
||||
emitReplay("position", position, duration)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */
|
||||
private fun finishReplayedUtterance() {
|
||||
mixer.reset()
|
||||
endpointDetector.reset()
|
||||
suppressor?.reset()
|
||||
if (mode != Mode.ONBOARD) return
|
||||
decodeTranscript(true)
|
||||
resetTranscriptState()
|
||||
lastTranscript = ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
|
||||
* decode and the beam filter. hasFrameMeta is false for the phone mic,
|
||||
@@ -495,7 +824,7 @@ class VoiceCaptureSession(
|
||||
continue
|
||||
}
|
||||
decodedSamples += read.toLong()
|
||||
processPcmChunk(pcm, read, 0, 0, false)
|
||||
handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -681,6 +1010,11 @@ class VoiceCaptureSession(
|
||||
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
|
||||
}
|
||||
|
||||
private fun emitReplay(event: String, positionMs: Long, durationMs: Long) {
|
||||
val currentListener = listener ?: return
|
||||
host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) }
|
||||
}
|
||||
|
||||
private fun emitSpeechEnd() {
|
||||
val currentListener = listener ?: return
|
||||
host.dispatcher.post { currentListener.onSpeechEnd() }
|
||||
@@ -691,8 +1025,21 @@ class VoiceCaptureSession(
|
||||
host.dispatcher.post { currentListener.onStopped(captureId) }
|
||||
}
|
||||
|
||||
private fun stopG2Audio() {
|
||||
host.g2AudioSource()?.stop()
|
||||
/**
|
||||
* Release what the running capture's source holds on the glasses. Runs on stop() and again
|
||||
* as the worker winds down (it may have started the source after stop() looked), so
|
||||
* everything here is safe to repeat.
|
||||
*/
|
||||
private fun stopSource() {
|
||||
val source = lock.withLock { activeSource }
|
||||
when (source) {
|
||||
Source.G2 -> host.g2AudioSource()?.stop()
|
||||
Source.ARRAY -> host.micArrayLink()?.let { link ->
|
||||
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true)
|
||||
link.stopForwarding()
|
||||
}
|
||||
Source.PHONE, Source.REPLAY -> {}
|
||||
}
|
||||
maybeEmitAudioStats(true)
|
||||
}
|
||||
|
||||
@@ -711,7 +1058,7 @@ class VoiceCaptureSession(
|
||||
if (!started || data == null) {
|
||||
return
|
||||
}
|
||||
if ("L" != arm) {
|
||||
if ("L" != arm && activeSource != Source.ARRAY) {
|
||||
wrongArmPackets++
|
||||
}
|
||||
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
|
||||
|
||||
@@ -67,6 +67,24 @@ interface PacketAudioSource {
|
||||
fun isActive(): Boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* The custom firmware's mic_control channel on a glasses session: each temple streams its own
|
||||
* microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed
|
||||
* stream uses, while it holds a configuration that the phone keeps renewing.
|
||||
*/
|
||||
interface MicArrayLink {
|
||||
/** Deliver the audio characteristic's packets to [listener]; false when no session is up. */
|
||||
fun startForwarding(listener: FaceclawAudioPacketListener): Boolean
|
||||
|
||||
fun stopForwarding()
|
||||
|
||||
/** Whether forwarding still holds: it lives in the glasses session, which can drop it. */
|
||||
fun isForwarding(): Boolean
|
||||
|
||||
/** Queue a control record for the chosen temples; false when nothing was queued. */
|
||||
fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean
|
||||
}
|
||||
|
||||
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
|
||||
interface PcmAudioSource {
|
||||
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
|
||||
|
||||
+2
@@ -3,6 +3,8 @@ package com.faceclaw.app
|
||||
/**
|
||||
* Receives IMU (accelerometer) readings decoded from the glasses' sys-event stream. Registered on
|
||||
* FaceclawBleCommunicator via addImuListener; callbacks are delivered on the Android main thread.
|
||||
* x, y, z are in g, in the canonical frame of every glasses model (+x forward, +y left, +z up;
|
||||
* see ImuOrientation).
|
||||
*/
|
||||
interface FaceclawImuListener {
|
||||
fun onImuData(x: Double, y: Double, z: Double, eventSource: Int): Unit
|
||||
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
interface FaceclawKeyboardEventListener {
|
||||
/**
|
||||
* One CFW keyboard-bridge record (settings field 131) from the glasses: the raw
|
||||
* ['K','B',ver,kind,...] bytes, possibly one fragment of a longer record. Decoded and
|
||||
* reassembled on the TS side (app/g2/keyboard/protocol.ts). Delivered in arrival order.
|
||||
*/
|
||||
fun onKeyboardEvent(body: ByteArray?): Unit
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/**
|
||||
* Events from FaceclawSshChannel, delivered on the thread that opened the
|
||||
* channel. See native/ssh/faceclawssh.go (Listener) for the kinds and payloads.
|
||||
*/
|
||||
interface FaceclawSshListener {
|
||||
fun onEvent(kind: String?, payload: String?): Unit
|
||||
}
|
||||
+9
@@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener {
|
||||
* speaker model could not be loaded.
|
||||
*/
|
||||
fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit
|
||||
|
||||
/**
|
||||
* Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay):
|
||||
* "start" once the file is open, "position" about once per second of it played, "seek" after
|
||||
* a jump took effect (the transcript of what played before it has been emitted as final),
|
||||
* "end" when the file ran out, and "error" when it could not be opened (the status says
|
||||
* why). Positions are milliseconds into the file.
|
||||
*/
|
||||
fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit
|
||||
}
|
||||
+156
-12
@@ -12,6 +12,12 @@ import kotlin.jvm.JvmStatic
|
||||
*/
|
||||
class BleProtocol {
|
||||
companion object {
|
||||
/** The service holding [WRITE_CHAR_UUID] and [NOTIFY_CHAR_UUID]. */
|
||||
const val PROTOCOL_SERVICE_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e5450"
|
||||
|
||||
/** The service holding [RENDER_NOTIFY_UUID]. */
|
||||
const val RENDER_SERVICE_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e6450"
|
||||
|
||||
const val WRITE_CHAR_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e5401"
|
||||
|
||||
const val NOTIFY_CHAR_UUID: String = "00002760-08c2-11e1-9073-0e8ac72e5402"
|
||||
@@ -119,6 +125,19 @@ class BleProtocol {
|
||||
*/
|
||||
const val FACECLAW_ALS_REPORT_FIELD: Int = 105
|
||||
|
||||
/**
|
||||
* CFW keyboard bridge (Faceclaw/39): the master temple hosts one BLE HID keyboard. The
|
||||
* host writes ['K','B',ver,op,request,epoch,...] controls as settings field 130; results,
|
||||
* pairing prompts and the keyboard's raw input reports come back as commandId=3 pushes
|
||||
* carrying field 131 (['K','B',ver,kind,...], decoded in app/g2/keyboard/protocol.ts).
|
||||
*/
|
||||
const val FACECLAW_KEYBOARD_CONTROL_FIELD: Int = 130
|
||||
|
||||
const val FACECLAW_KEYBOARD_EVENT_FIELD: Int = 131
|
||||
|
||||
/** Fixed header of a field-131 record; fragments follow it. */
|
||||
const val FACECLAW_KEYBOARD_EVENT_HEADER: Int = 24
|
||||
|
||||
const val FACECLAW_WAKE_OP_ACQUIRE: Int = 1
|
||||
|
||||
const val FACECLAW_WAKE_OP_RELEASE: Int = 2
|
||||
@@ -497,22 +516,79 @@ class BleProtocol {
|
||||
)
|
||||
}
|
||||
|
||||
/** Stock HeadUpSetting: switch and angle are separate oneof writes (field 4 of device info). */
|
||||
/** Compatibility helper: preserve the fork's explicit switch write. */
|
||||
@JvmStatic
|
||||
fun buildSetHeadUpWake(magic: Int, enabled: Boolean): ByteArray = concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1), encodeVarintField(2, magic),
|
||||
encodeMessageField(3, encodeMessageField(4, encodeVarintField(1, if (enabled) 1 else 0))),
|
||||
)
|
||||
)
|
||||
fun buildSetHeadUpWake(magic: Int, enabled: Boolean): ByteArray =
|
||||
buildSetHeadUp(magic, HEAD_UP_FIELD_SWITCH, if (enabled) 1 else 0)
|
||||
|
||||
/** Compatibility helper: only the fork's angle API clamps to its 0..60 preference range. */
|
||||
@JvmStatic
|
||||
fun buildSetHeadUpAngle(magic: Int, angle: Int): ByteArray = concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1), encodeVarintField(2, magic),
|
||||
encodeMessageField(3, encodeMessageField(4, encodeVarintField(2, angle.coerceIn(0, 60)))),
|
||||
fun buildSetHeadUpAngle(magic: Int, angle: Int): ByteArray =
|
||||
buildSetHeadUp(magic, HEAD_UP_FIELD_ANGLE, angle.coerceIn(0, 60))
|
||||
|
||||
// Field numbers from g2_setting.proto (package g2.g2_setting; see g2-kit-unofficial
|
||||
// ble/gen/g2_setting_pb.ts), named after the schema fields.
|
||||
|
||||
/** G2SettingPackage.commandId */
|
||||
private const val G2_SETTING_PACKAGE_COMMAND_ID: Int = 1
|
||||
|
||||
/** G2SettingPackage.magicRandom */
|
||||
private const val G2_SETTING_PACKAGE_MAGIC_RANDOM: Int = 2
|
||||
|
||||
/** G2SettingPackage.deviceReceiveInfoFromApp */
|
||||
private const val G2_SETTING_PACKAGE_DEVICE_RECEIVE_INFO_FROM_APP: Int = 3
|
||||
|
||||
/** G2SettingPackage.deviceReceiveRequestFromApp (the settings-read reply) */
|
||||
private const val G2_SETTING_PACKAGE_DEVICE_RECEIVE_REQUEST_FROM_APP: Int = 4
|
||||
|
||||
/** g2_settingCommandId.DeviceReceiveInfo */
|
||||
private const val G2_SETTING_COMMAND_DEVICE_RECEIVE_INFO: Int = 1
|
||||
|
||||
/** DeviceReceiveInfoFromAPP.deviceReceiveHeadUpSetting */
|
||||
private const val DEVICE_RECEIVE_INFO_HEAD_UP_SETTING: Int = 4
|
||||
|
||||
/** DeviceReceiveRequestFromAPP.headUpSwitchRestored */
|
||||
private const val DEVICE_RECEIVE_REQUEST_HEAD_UP_SWITCH_RESTORED: Int = 7
|
||||
|
||||
/** DeviceReceiveRequestFromAPP.headUpAngleRestored */
|
||||
private const val DEVICE_RECEIVE_REQUEST_HEAD_UP_ANGLE_RESTORED: Int = 8
|
||||
|
||||
/** DeviceReceiveRequestFromAPP.headUpAngleCalibrationRestored */
|
||||
private const val DEVICE_RECEIVE_REQUEST_HEAD_UP_CALIBRATION_RESTORED: Int = 9
|
||||
|
||||
/**
|
||||
* One head-up (look-up wake) setting: DeviceReceiveInfoFromAPP.deviceReceiveHeadUpSetting,
|
||||
* a DeviceReceive_Head_UP_Setting carrying one of headUpSwitch, headUpAngle,
|
||||
* headUpCalibrationSwitch or headUpCalibration ([field] is one of the HEAD_UP_FIELD_*).
|
||||
* The firmware treats that submessage as a oneof: setting_handle_head_up_setting
|
||||
* (2.3.0.24, 0x46c39a) receives a single {field, value} and applies it, so the switch and
|
||||
* the angle go out as separate packages. headUpCalibration is not stored from the
|
||||
* phone; it is the pitch the glasses measured on their own calibration screen. The
|
||||
* effective trigger pitch is angle + calibration, clamped to -90..90 and accepted only in
|
||||
* 1..90 (SVC_Settings_HeadUpConfig 0x470580, DRV_IMUHUDParameterConfig). The angle is a
|
||||
* uint32 on the wire but used as a signed int, so a negative offset is sent as its
|
||||
* 32-bit two's complement.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun buildSetHeadUp(magic: Int, field: Int, value: Int): ByteArray {
|
||||
var headUp: ByteArray = concat(CollectionUtils.listOf(encodeKey(field, 0), encodeUint32(value)))
|
||||
return concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(G2_SETTING_PACKAGE_COMMAND_ID, G2_SETTING_COMMAND_DEVICE_RECEIVE_INFO),
|
||||
encodeVarintField(G2_SETTING_PACKAGE_MAGIC_RANDOM, magic),
|
||||
encodeMessageField(
|
||||
G2_SETTING_PACKAGE_DEVICE_RECEIVE_INFO_FROM_APP,
|
||||
encodeMessageField(DEVICE_RECEIVE_INFO_HEAD_UP_SETTING, headUp),
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/** DeviceReceive_Head_UP_Setting.headUpSwitch */
|
||||
const val HEAD_UP_FIELD_SWITCH: Int = 1
|
||||
|
||||
/** DeviceReceive_Head_UP_Setting.headUpAngle */
|
||||
const val HEAD_UP_FIELD_ANGLE: Int = 2
|
||||
|
||||
/** Enable the firmware's wear detector (DeviceReceiveInfoFromAPP field 5). */
|
||||
@JvmStatic
|
||||
@@ -741,6 +817,42 @@ class BleProtocol {
|
||||
return body
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an already-encoded ['K','B',ver,op,...] keyboard control as a settings write
|
||||
* carrying CFW field 130. Magic zero, like the other private controls: the answer is a
|
||||
* field-131 record naming the control's request id, not the settings ACK.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun buildFaceclawKeyboardControl(record: ByteArray): ByteArray {
|
||||
return concat(
|
||||
CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1),
|
||||
encodeVarintField(2, 0),
|
||||
encodeBytesField(FACECLAW_KEYBOARD_CONTROL_FIELD, record),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a CFW keyboard record (field 131) from a settings frame, or null when the frame
|
||||
* carries none. Only the ['K','B'] prelude and the fixed header's length are checked
|
||||
* here; the TS side validates the version and decodes the rest.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun parseFaceclawKeyboardEvent(pb: ByteArray?): ByteArray? {
|
||||
if (pb == null) {
|
||||
return null
|
||||
}
|
||||
val body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_KEYBOARD_EVENT_FIELD)
|
||||
if (
|
||||
body == null || body.size < FACECLAW_KEYBOARD_EVENT_HEADER ||
|
||||
body[0].toInt() != 75 || body[1].toInt() != 66
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the raw CFW ambient-light report from a sid-0x09 frame, or null when this is an
|
||||
* ordinary settings frame. Only the 65.toByte(),76.toByte(),version prelude is validated
|
||||
@@ -922,6 +1034,26 @@ class BleProtocol {
|
||||
return BatterySnapshot(battery, charging, silentMode)
|
||||
}
|
||||
|
||||
/**
|
||||
* The head-up settings in a settings READ ack: deviceReceiveRequestFromApp (field 4)
|
||||
* headUpSwitchRestored=7, headUpAngleRestored=8, headUpAngleCalibrationRestored=9.
|
||||
* Returns null when the ack has no request submessage. Absent fields (proto3 drops
|
||||
* zeros) read as 0; angle and calibration are signed ints carried as uint32.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun parseSettingsHeadUp(pb: ByteArray): IntArray? {
|
||||
var root: ByteArray = stripTrailingCrc(pb)
|
||||
var request: ByteArray? = readFieldBytes(root, G2_SETTING_PACKAGE_DEVICE_RECEIVE_REQUEST_FROM_APP)
|
||||
if ((request == null)) {
|
||||
return null
|
||||
}
|
||||
return intArrayOf(
|
||||
readVarintFieldValue(request, DEVICE_RECEIVE_REQUEST_HEAD_UP_SWITCH_RESTORED, 0),
|
||||
readVarintFieldValue(request, DEVICE_RECEIVE_REQUEST_HEAD_UP_ANGLE_RESTORED, 0),
|
||||
readVarintFieldValue(request, DEVICE_RECEIVE_REQUEST_HEAD_UP_CALIBRATION_RESTORED, 0),
|
||||
)
|
||||
}
|
||||
|
||||
/** GlassesCaseDataPackage{commandId=CASE_INFO, magicRandom, caseInfo={}}, as the Even app sends it. */
|
||||
@JvmStatic
|
||||
fun buildCaseInfoQuery(magic: Int): ByteArray {
|
||||
@@ -1106,6 +1238,18 @@ class BleProtocol {
|
||||
return out.toByteArray()
|
||||
}
|
||||
|
||||
/** A varint of the value's 32-bit two's complement (encodeVarint stops short on negatives). */
|
||||
private fun encodeUint32(value: Int): ByteArray {
|
||||
var out: ByteSink = ByteSink()
|
||||
var v: Long = (value.toLong() and 0xffffffffL)
|
||||
while ((v >= 0x80L)) {
|
||||
out.write((((v and 0x7fL).toInt()) or 0x80))
|
||||
v = (v ushr 7)
|
||||
}
|
||||
out.write(v.toInt())
|
||||
return out.toByteArray()
|
||||
}
|
||||
|
||||
private fun crcBytesLe(data: ByteArray): ByteArray {
|
||||
var crc: Int = 0xffff
|
||||
for (b in data) {
|
||||
|
||||
+41
-6
@@ -188,14 +188,31 @@ class MessageBuilder {
|
||||
)
|
||||
}
|
||||
|
||||
fun headUpControl(leftArm: Boolean, enabled: Boolean?, angle: Int): OutboundMessage {
|
||||
/** Fork preference API: retain per-arm targeting and the bounded preference angle. */
|
||||
fun headUpControl(leftArm: Boolean, enabled: Boolean?, angle: Int): OutboundMessage =
|
||||
headUpSetting(
|
||||
if (enabled == null) BleProtocol.HEAD_UP_FIELD_ANGLE else BleProtocol.HEAD_UP_FIELD_SWITCH,
|
||||
if (enabled == null) angle.coerceIn(0, 60) else if (enabled) 1 else 0,
|
||||
leftArm,
|
||||
if (enabled == null) "head-up angle=$angle" else "head-up wake=$enabled",
|
||||
)
|
||||
|
||||
/** One head-up setting field, including signed calibration-relative angles. */
|
||||
fun setHeadUp(field: Int, value: Int): OutboundMessage =
|
||||
headUpSetting(field, value, false, "head-up field $field=$value")
|
||||
|
||||
internal fun headUpSetting(field: Int, value: Int, leftArm: Boolean, label: String): OutboundMessage {
|
||||
val magic = magicPool.allocate()
|
||||
return OutboundMessage(
|
||||
"head-up-control", if (enabled == null) "head-up angle=$angle" else "head-up wake=$enabled",
|
||||
BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, magic,
|
||||
if (enabled == null) BleProtocol.buildSetHeadUpAngle(magic, angle)
|
||||
else BleProtocol.buildSetHeadUpWake(magic, enabled),
|
||||
ACK_TIMEOUT_MS, -1, leftArm,
|
||||
"head-up-control",
|
||||
label,
|
||||
BleProtocol.SID_UI_SETTING,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
magic,
|
||||
BleProtocol.buildSetHeadUp(magic, field, value),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
leftArm,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -332,6 +349,24 @@ class MessageBuilder {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* CFW keyboard control (settings field 130). Always the right arm: only the master temple
|
||||
* acts on it, and a copy sent down each arm would run the control twice.
|
||||
*/
|
||||
fun faceclawKeyboardControl(record: ByteArray, label: String): OutboundMessage {
|
||||
return OutboundMessage(
|
||||
"keyboard-control",
|
||||
"keyboard control $label",
|
||||
BleProtocol.SID_UI_SETTING,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
0,
|
||||
BleProtocol.buildFaceclawKeyboardControl(record),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
fun faceclawWearQuery(leftArm: Boolean): OutboundMessage {
|
||||
return OutboundMessage(
|
||||
"wear-query-control",
|
||||
|
||||
+10
-3
@@ -20,11 +20,18 @@ class CachedResource @JvmOverloads constructor(bytes: ByteArray, val owner: Stri
|
||||
*/
|
||||
class ResourceCacheState {
|
||||
companion object {
|
||||
/** Mirrors g2flash/patches/resource_cache.h. */
|
||||
const val CACHE_SIZE = 196608
|
||||
/**
|
||||
* Mirrors g2flash/patches/resource_cache.h: the id table, then 320 KiB shared by the
|
||||
* resources and their block headers. Firmware before Faceclaw/40 had 192 KiB in all,
|
||||
* so this must not grow past what REQUIRED_FACECLAW_FIRMWARE_VERSION guarantees.
|
||||
* RayNeo iO sessions use the same budget: that firmware's own copy of the header
|
||||
* (faceclaw-rayneo-cfw patches/fc/resource_cache.h) needs at least this much before
|
||||
* it reports the required revision.
|
||||
*/
|
||||
const val RESOURCE_COUNT = 512
|
||||
const val TABLE_BYTES = RESOURCE_COUNT * 4
|
||||
const val ARENA_BYTES = CACHE_SIZE - TABLE_BYTES
|
||||
const val ARENA_BYTES = 320 * 1024
|
||||
const val CACHE_SIZE = TABLE_BYTES + ARENA_BYTES
|
||||
const val MAX_RESOURCE_SIZE = 65536
|
||||
const val BLOCK_HEADER_BYTES = 16
|
||||
|
||||
|
||||
+23
-1
@@ -506,6 +506,10 @@ internal fun GlassesSessionCore.scheduleReconnectLocked(sessionWasReady: Boolean
|
||||
internal fun GlassesSessionCore.resetSessionStateLocked() {
|
||||
ringBattery = -1
|
||||
ringCharging = -1
|
||||
headUpSwitch = -1
|
||||
headUpSent = null
|
||||
headUpReadAtMs = 0L
|
||||
headUpVerifyPending = false
|
||||
directRingBattery = -1
|
||||
directRingCharging = -1
|
||||
sessionReady = false
|
||||
@@ -583,6 +587,22 @@ internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.emitKeyboardEvent(body: ByteArray) {
|
||||
if (keyboardEventListeners.isEmpty()) {
|
||||
return
|
||||
}
|
||||
val copy = body.copyOf()
|
||||
host.postToMain {
|
||||
for (keyboardListener in keyboardEventListeners) {
|
||||
try {
|
||||
keyboardListener.onKeyboardEvent(copy)
|
||||
} catch (t: Throwable) {
|
||||
logWarn("keyboard event listener failed", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.emitAudioMonitorPacket(data: ByteArray, arm: String, arrivalMs: Long) {
|
||||
if (audioMonitorListeners.isEmpty()) {
|
||||
return
|
||||
@@ -634,14 +654,16 @@ internal fun GlassesSessionCore.finishFrame(frameId: Int, outcome: String) {
|
||||
}
|
||||
}
|
||||
|
||||
/** [x], [y], [z] as the hardware reports them; listeners get them in [ImuOrientation]'s frame. */
|
||||
internal fun GlassesSessionCore.emitImuData(x: Double, y: Double, z: Double, eventSource: Int) {
|
||||
if (imuListeners.isEmpty()) {
|
||||
return
|
||||
}
|
||||
val (cx, cy, cz) = ImuOrientation.toCanonical(glassesType, x, y, z)
|
||||
host.postToMain {
|
||||
for (imuListener in imuListeners) {
|
||||
try {
|
||||
imuListener.onImuData(x, y, z, eventSource)
|
||||
imuListener.onImuData(cx, cy, cz, eventSource)
|
||||
} catch (t: Throwable) {
|
||||
logWarn("listener onImuData failed", t)
|
||||
}
|
||||
|
||||
+136
-9
@@ -131,6 +131,7 @@ class GlassesSessionCore(
|
||||
internal val compassSubscriptions = CopyOnWriteList<CompassSubscription>(platform)
|
||||
internal val ambientLightListeners = CopyOnWriteList<FaceclawAmbientLightListener>(platform)
|
||||
internal val micStatusListeners = CopyOnWriteList<FaceclawMicStatusListener>(platform)
|
||||
internal val keyboardEventListeners = CopyOnWriteList<FaceclawKeyboardEventListener>(platform)
|
||||
internal val audioMonitorListeners = CopyOnWriteList<FaceclawAudioPacketListener>(platform)
|
||||
@Volatile internal var running = false
|
||||
@Volatile internal var userDisconnectRequested = false
|
||||
@@ -312,7 +313,8 @@ class GlassesSessionCore(
|
||||
internal var brightnessAlsStartedAt = -2000L
|
||||
internal var brightnessDemoPolling = false
|
||||
internal var headUpPreferences: Pair<Int, Int>? = null
|
||||
internal var headUpSent: Pair<Int, Int>? = null
|
||||
// Effective queued policy, not the raw preferences; null angle means omitted.
|
||||
internal var headUpSent: Pair<Int, Int?>? = null
|
||||
internal var cfwCleanupDelivered = false
|
||||
internal var lastCfwCleanupAckMagic = 0
|
||||
|
||||
@@ -359,6 +361,26 @@ class GlassesSessionCore(
|
||||
internal var lastGlassesRingInputAtMs = 0L
|
||||
internal var lastGlassesRingInputLabel = ""
|
||||
internal var lastGlassesRingBatteryAtMs = 0L
|
||||
// Head-up (look-up wake) state from settings reads: -1 switch = not read yet.
|
||||
internal var headUpSwitch = -1
|
||||
internal var headUpAngle = 0
|
||||
internal var headUpCalibration = 0
|
||||
internal var headUpReadAtMs = 0L
|
||||
// The trigger pitch TS asked for (degrees, 1..90), or -1 to leave the glasses'
|
||||
// own setting alone. Reapplied after every read that disagrees with it.
|
||||
internal var desiredHeadUpTrigger = -1
|
||||
// The glasses' own switch and angle from before Faceclaw first changed them
|
||||
// (switch -1 = unknown): captured before the first write, or handed back by TS,
|
||||
// which persists them across app restarts. Written back while
|
||||
// headUpRestorePending (TS stopped managing the trigger), then forgotten once a
|
||||
// read confirms them.
|
||||
internal var headUpOriginalSwitch = -1
|
||||
internal var headUpOriginalAngle = 0
|
||||
internal var headUpRestorePending = false
|
||||
// Set after queueing a correction so the next settings read comes right away to
|
||||
// confirm it; cleared by that read, so a firmware that refuses the value is
|
||||
// retried on the normal poll cadence, not in a loop.
|
||||
internal var headUpVerifyPending = false
|
||||
internal var ringConnectedChangedAtMs = 0L
|
||||
internal var ringConnectCount = 0
|
||||
internal var lastDirectRingError = ""
|
||||
@@ -488,7 +510,7 @@ class GlassesSessionCore(
|
||||
// Preview/screenshot/recording callers render those explicitly on demand.
|
||||
internal val compositor = SurfaceCompositor(false)
|
||||
|
||||
// Phone-side model of the CFW's 192 KiB resource cache.
|
||||
// Phone-side model of the CFW's resource cache (ResourceCacheState.CACHE_SIZE).
|
||||
// Reset whenever the image pipeline / EvenHub session is torn down: the
|
||||
// firmware frees the cache with the fb lease, and after any resync the
|
||||
// cheap safe assumption is an empty cache (glyphs re-upload lazily).
|
||||
@@ -996,7 +1018,12 @@ class GlassesSessionCore(
|
||||
fun configureHeadUpWake(enabled: Int, angle: Int) {
|
||||
require(enabled in -1..1)
|
||||
monitor.withLock {
|
||||
headUpPreferences = Pair(enabled, if (angle < 0) -1 else angle.coerceIn(0, 60))
|
||||
val preferences = Pair(enabled, if (angle < 0) -1 else angle.coerceIn(0, 60))
|
||||
if (headUpPreferences != preferences) {
|
||||
clearMessagesOfKindLocked("head-up-control")
|
||||
headUpVerifyPending = false
|
||||
headUpPreferences = preferences
|
||||
}
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
@@ -1026,6 +1053,61 @@ class GlassesSessionCore(
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the pitch (degrees, 1..90) at which looking up wakes the display, or 0 to stop
|
||||
* managing it. Kept for the life of this object and reapplied on every connection: the
|
||||
* next settings read supplies the glasses' calibration, then the switch and angle are
|
||||
* written (see reconcileHeadUpLocked) and a follow-up read confirms them.
|
||||
*
|
||||
* [originalSwitch] (0/1, or -1 when unknown) and [originalAngle] are the glasses' own
|
||||
* values from before Faceclaw changed them, as TS saved them from getHeadUpStatus; they
|
||||
* take precedence over what this object captured. With degrees 0 they are written back.
|
||||
*/
|
||||
fun setHeadUpTrigger(degrees: Int, originalSwitch: Int, originalAngle: Int) {
|
||||
monitor.withLock {
|
||||
if (originalSwitch in 0..1) {
|
||||
headUpOriginalSwitch = originalSwitch
|
||||
headUpOriginalAngle = originalAngle
|
||||
}
|
||||
// A change of mind drops writes still queued for the previous value.
|
||||
clearMessagesOfKindLocked("head-up-control")
|
||||
headUpVerifyPending = false
|
||||
desiredHeadUpTrigger = if (degrees in 1..90) degrees else -1
|
||||
headUpRestorePending = desiredHeadUpTrigger < 0 && headUpOriginalSwitch >= 0
|
||||
logLine(when {
|
||||
desiredHeadUpTrigger > 0 -> "head-up trigger requested: ${desiredHeadUpTrigger}°"
|
||||
headUpRestorePending -> "head-up trigger: restore the glasses' own " +
|
||||
"(switch=$headUpOriginalSwitch angle=$headUpOriginalAngle)"
|
||||
else -> "head-up trigger: left to the glasses"
|
||||
})
|
||||
if (!running || !sessionReady) return
|
||||
if (desiredHeadUpTrigger <= 0 && !headUpRestorePending) return
|
||||
if (headUpSwitch >= 0) {
|
||||
applyHeadUpLocked(verifying = false)
|
||||
} else {
|
||||
// Calibration unknown: read the settings first; the ack applies the trigger.
|
||||
lastBatteryRefreshAtMs = 0L
|
||||
}
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
}
|
||||
|
||||
/** Head-up settings as last read, as JSON (switch -1 = not read on this connection). */
|
||||
fun getHeadUpStatus(): String {
|
||||
monitor.withLock {
|
||||
val status = linkedMapOf<String, Any?>()
|
||||
status["switch"] = headUpSwitch
|
||||
status["angle"] = headUpAngle
|
||||
status["calibration"] = headUpCalibration
|
||||
status["desired"] = desiredHeadUpTrigger
|
||||
status["originalSwitch"] = headUpOriginalSwitch
|
||||
status["originalAngle"] = headUpOriginalAngle
|
||||
status["restorePending"] = headUpRestorePending
|
||||
status["readAgeMs"] = if (headUpReadAtMs == 0L) -1L else maxOf(0L, now() - headUpReadAtMs)
|
||||
return try { Json.write(status) } catch (e: Exception) { "{}" }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Bandwidth benchmark
|
||||
|
||||
@@ -1399,6 +1481,39 @@ class GlassesSessionCore(
|
||||
return true
|
||||
}
|
||||
|
||||
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
|
||||
if (listener != null) {
|
||||
keyboardEventListeners.add(listener)
|
||||
}
|
||||
}
|
||||
|
||||
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) {
|
||||
if (listener != null) {
|
||||
keyboardEventListeners.remove(listener)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a CFW keyboard-bridge control (['K','B',ver,op,request,epoch,...]) as a settings
|
||||
* field-130 write to the right temple. Fire-and-forget: the firmware answers with field-131
|
||||
* records, which arrive through addKeyboardEventListener. Returns false when nothing was
|
||||
* queued (not a G2, no session, or a malformed record).
|
||||
*/
|
||||
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean {
|
||||
if (!isG2 || record == null || record.size < 8) {
|
||||
return false
|
||||
}
|
||||
monitor.withLock {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip keyboard control ($label); session not ready")
|
||||
return false
|
||||
}
|
||||
pendingMessages.addLast(messageBuilder.faceclawKeyboardControl(record, label))
|
||||
}
|
||||
interruptibleSleep.interrupt()
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward render-characteristic audio packets to the listener WITHOUT
|
||||
* sending the stock EvenHub audio-control enable. Used for the CFW
|
||||
@@ -1903,8 +2018,21 @@ class GlassesSessionCore(
|
||||
tap(data.copyOf())
|
||||
return
|
||||
}
|
||||
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
|
||||
val frame = BleProtocol.parseFrame(data)
|
||||
if (frame.ok && frame.sid == BleProtocol.SID_UI_SETTING) {
|
||||
// CFW keyboard-bridge records (field 131) come from whichever temple hosts the
|
||||
// keyboard. They are their own stream: neither an ACK nor glasses input, and the
|
||||
// once-a-second status record among them must not count as user activity (which
|
||||
// would hold off battery polling). A touchpad sends a hundred of them a second
|
||||
// while a finger is on it, so they are not logged one by one either.
|
||||
val keyboardEvent = BleProtocol.parseFaceclawKeyboardEvent(frame.pb)
|
||||
if (keyboardEvent != null) {
|
||||
monitor.withLock { lastIncomingAtMs = now() }
|
||||
emitKeyboardEvent(keyboardEvent)
|
||||
return
|
||||
}
|
||||
}
|
||||
logDebug("onNotification: address=" + address + " characteristicUuid=" + characteristicUuid + " data.length=" + data.size)
|
||||
val decodedWearState = BleProtocol.parseWearState(frame)
|
||||
val compassEvent = if (address.equals(rightAddress, ignoreCase = true))
|
||||
BleProtocol.parseCompassEvent(frame)
|
||||
@@ -2072,11 +2200,10 @@ class GlassesSessionCore(
|
||||
} else if (decoded != null
|
||||
&& "sys-event" == decoded.kind
|
||||
&& decoded.eventType == BleProtocol.EVENT_HEAD_DOWN) {
|
||||
// Faceclaw/38 also forwards the head coming back down. Nothing
|
||||
// uses it yet; as an unknown input it would count as activity
|
||||
// and keep the screen awake, so drop it here.
|
||||
logLine("head-down forwarded by CFW (ignored)")
|
||||
decoded = null
|
||||
// Faceclaw/38 also forwards the head coming back down. TS uses
|
||||
// it to end a head-tilt Glanceboard and otherwise drops it
|
||||
// before the shell, so it never counts as activity.
|
||||
logLine("head-down forwarded by CFW")
|
||||
}
|
||||
event = decoded
|
||||
if (decoded != null) {
|
||||
|
||||
+2
-1
@@ -299,7 +299,8 @@ internal fun GlassesSessionCore.g1StopAudioCapture() {
|
||||
}
|
||||
|
||||
/**
|
||||
* There is no magnetometer on a G1, only the accelerometer and gyroscope: x, y, z in g, head frame, from the right arm.
|
||||
* There is no magnetometer on a G1, only the accelerometer and gyroscope: x, y, z in g from the right arm, in the
|
||||
* G1's own axes, which [emitImuData] rotates into the G2's ([ImuOrientation]).
|
||||
* [enable] raises the sample rate and polls it; [g1MaxSleepMs] follows.
|
||||
*/
|
||||
internal fun GlassesSessionCore.g1SetImuReport(enable: Boolean) {
|
||||
|
||||
+3
-19
@@ -1,23 +1,7 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/** Both arms receive separate stock settings messages; no CFW changes required. */
|
||||
/** The sender and settings polls share one reconciler; never queue a second policy. */
|
||||
internal fun GlassesSessionCore.enqueueHeadUpPreferencesLocked() {
|
||||
if (!isG2) return
|
||||
val desired = headUpPreferences ?: return
|
||||
if (!sessionReady || shutdownRequested || chargingMode || headUpSent == desired
|
||||
|| hasPendingOrInflightKindLocked("head-up-control")) return
|
||||
for (leftArm in listOf(true, false)) {
|
||||
// Set the angle before enabling wake. Omitted fields preserve device preferences.
|
||||
if (desired.second >= 0) {
|
||||
val message = messageBuilder.headUpControl(leftArm, null, desired.second)
|
||||
message.onTimeout = MessageCallback { headUpSent = null; logLine("head-up angle ack timeout") }
|
||||
pendingMessages.addLast(message)
|
||||
}
|
||||
if (desired.first >= 0) {
|
||||
val message = messageBuilder.headUpControl(leftArm, desired.first == 1, -1)
|
||||
message.onTimeout = MessageCallback { headUpSent = null; logLine("head-up wake ack timeout") }
|
||||
pendingMessages.addLast(message)
|
||||
}
|
||||
}
|
||||
headUpSent = desired
|
||||
if (headUpPreferences == null) return
|
||||
applyHeadUpLocked(verifying = false)
|
||||
}
|
||||
+105
-1
@@ -1079,6 +1079,103 @@ internal fun GlassesSessionCore.shouldPollBatteryLocked(now: Long): Boolean {
|
||||
&& (lastBatteryRefreshAtMs == 0L || now - lastBatteryRefreshAtMs >= ConnectionOptions.BATTERY_REFRESH_INTERVAL_MS)
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring the glasses' head-up trigger to [GlassesSessionCore.desiredHeadUpTrigger] after a settings
|
||||
* read: switch on, then angle = trigger - calibration (the firmware adds its stored calibration).
|
||||
*/
|
||||
internal fun GlassesSessionCore.reconcileHeadUpLocked(ackPayload: ByteArray?) {
|
||||
val verifying = headUpVerifyPending
|
||||
headUpVerifyPending = false
|
||||
val headUp = if (ackPayload == null) null else BleProtocol.parseSettingsHeadUp(ackPayload)
|
||||
if (headUp != null) {
|
||||
headUpSwitch = headUp[0]
|
||||
headUpAngle = headUp[1]
|
||||
headUpCalibration = headUp[2]
|
||||
headUpReadAtMs = now()
|
||||
val sent = headUpSent
|
||||
if (!verifying && sent != null && !hasPendingOrInflightKindLocked("head-up-control")
|
||||
&& ((sent.first >= 0 && sent.first != headUpSwitch)
|
||||
|| (sent.second != null && sent.second != headUpAngle))) {
|
||||
headUpSent = null // a regular poll retries a refused/drifted setting
|
||||
}
|
||||
}
|
||||
if (headUpSwitch < 0 || !sessionReady) return
|
||||
applyHeadUpLocked(verifying)
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare the last read with what is wanted (the trigger, or the glasses' original values
|
||||
* while a restore is pending) and queue writes when they differ. After a verifying read
|
||||
* that still disagrees nothing is queued: the next regular poll retries, so a firmware
|
||||
* that clamps or refuses the value costs one attempt per poll interval, not a write loop.
|
||||
*/
|
||||
internal fun GlassesSessionCore.applyHeadUpLocked(verifying: Boolean) {
|
||||
if (!isG2 || !sessionReady || shutdownRequested || chargingMode
|
||||
|| hasPendingOrInflightKindLocked("head-up-control")) return
|
||||
val preferences = headUpPreferences
|
||||
val bothArms = preferences != null && (preferences.first >= 0 || preferences.second >= 0)
|
||||
// Explicit Off is authoritative. Glanceboard is a temporary angle override,
|
||||
// never a separate writer that can re-enable a forbidden wake.
|
||||
val trigger = desiredHeadUpTrigger > 0 && preferences?.first != 0
|
||||
if (trigger && headUpSwitch < 0) return // calibration comes from the settings read
|
||||
val wantSwitch = when {
|
||||
trigger -> 1
|
||||
preferences != null && preferences.first >= 0 -> preferences.first
|
||||
headUpRestorePending -> headUpOriginalSwitch
|
||||
else -> -1
|
||||
}
|
||||
val wantAngle = when {
|
||||
trigger -> desiredHeadUpTrigger - headUpCalibration
|
||||
preferences != null && preferences.second >= 0 -> preferences.second
|
||||
headUpRestorePending -> headUpOriginalAngle
|
||||
else -> null // omitted; negative generic angles are valid
|
||||
}
|
||||
if (wantSwitch < 0 && wantAngle == null) return
|
||||
val matches = (wantSwitch < 0 || headUpSwitch == wantSwitch) &&
|
||||
(wantAngle == null || headUpAngle == wantAngle)
|
||||
// A right-arm read cannot prove the left arm was configured. The cache is
|
||||
// the complete queued policy, invalidated on cancellation, timeout and reset.
|
||||
val effective = wantSwitch to wantAngle
|
||||
if (matches && (!bothArms || headUpSent == effective)) {
|
||||
if (!trigger && headUpRestorePending) {
|
||||
headUpRestorePending = false
|
||||
headUpOriginalSwitch = -1
|
||||
headUpOriginalAngle = 0
|
||||
}
|
||||
return
|
||||
}
|
||||
if (verifying) {
|
||||
logLine("head-up setting not applied yet; retrying on next poll")
|
||||
return
|
||||
}
|
||||
if (headUpSent == effective) return
|
||||
if (trigger && headUpOriginalSwitch < 0) {
|
||||
headUpOriginalSwitch = headUpSwitch
|
||||
headUpOriginalAngle = headUpAngle
|
||||
}
|
||||
queueHeadUpWritesLocked(wantSwitch, wantAngle, bothArms)
|
||||
headUpSent = effective
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.queueHeadUpWritesLocked(switch: Int, angle: Int?, bothArms: Boolean) {
|
||||
clearMessagesOfKindLocked("head-up-control")
|
||||
for (leftArm in if (bothArms) listOf(true, false) else listOf(false)) {
|
||||
val set = if (angle == null) null else if (bothArms)
|
||||
messageBuilder.headUpSetting(BleProtocol.HEAD_UP_FIELD_ANGLE, angle, leftArm, "head-up angle=$angle")
|
||||
else messageBuilder.setHeadUp(BleProtocol.HEAD_UP_FIELD_ANGLE, angle)
|
||||
val sw = if (switch < 0) null else if (bothArms)
|
||||
messageBuilder.headUpSetting(BleProtocol.HEAD_UP_FIELD_SWITCH, switch, leftArm, "head-up wake=${switch == 1}")
|
||||
else messageBuilder.setHeadUp(BleProtocol.HEAD_UP_FIELD_SWITCH, switch)
|
||||
set?.onTimeout = MessageCallback { headUpSent = null; logLine("head-up angle ack timeout") }
|
||||
sw?.onTimeout = MessageCallback { headUpSent = null; logLine("head-up switch ack timeout") }
|
||||
// Fork: angle before wake on both arms. Unset: preserve upstream ordering.
|
||||
val writes = if (!bothArms && switch == 1) listOfNotNull(sw, set) else listOfNotNull(set, sw)
|
||||
for (message in writes) pendingMessages.addLast(message)
|
||||
}
|
||||
headUpVerifyPending = true
|
||||
lastBatteryRefreshAtMs = 0L
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessage {
|
||||
val message = messageBuilder.batteryQuery()
|
||||
message.onAck = MessageCallback {
|
||||
@@ -1087,6 +1184,7 @@ internal fun GlassesSessionCore.createBatteryQueryMessageLocked(): OutboundMessa
|
||||
ringBattery = if (ring == null) -1 else ring.battery
|
||||
ringCharging = if (ring == null) -1 else ring.charging
|
||||
if (ringBattery >= 0) releaseRingFromGlassesLocked("glasses report the ring connected")
|
||||
reconcileHeadUpLocked(message.ackPayload)
|
||||
val snapshot = BleProtocol.parseSettingsBattery(message.ackPayload)
|
||||
if (snapshot != null) {
|
||||
headsetBattery = snapshot.battery
|
||||
@@ -1350,6 +1448,7 @@ internal fun GlassesSessionCore.handleAckTimeoutLocked(message: OutboundMessage)
|
||||
}
|
||||
|
||||
internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) {
|
||||
if (kind == "head-up-control") headUpSent = null
|
||||
val pendingIterator = pendingMessages.iterator()
|
||||
while (pendingIterator.hasNext()) {
|
||||
val message = pendingIterator.next()
|
||||
@@ -1362,10 +1461,12 @@ internal fun GlassesSessionCore.clearMessagesOfKindLocked(kind: String) {
|
||||
magicPool.release(message.sid, message.magic, message.label, "cleared pending $kind")
|
||||
}
|
||||
}
|
||||
// Head-up writes already handed to BLE must finish before a replacement
|
||||
// policy is queued; dropping their ACK tracking would race the old write.
|
||||
val inFlightIterator = inFlightMessages.iterator()
|
||||
while (inFlightIterator.hasNext()) {
|
||||
val message = inFlightIterator.next()
|
||||
if (kind == message.kind) {
|
||||
if (kind == message.kind && kind != "head-up-control") {
|
||||
inFlightIterator.remove()
|
||||
if ("image" == kind) {
|
||||
discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared ($kind)")
|
||||
@@ -1406,6 +1507,7 @@ internal fun GlassesSessionCore.clearAllMessagesLocked(reason: String) {
|
||||
* direct prelude write.
|
||||
*/
|
||||
internal fun GlassesSessionCore.clearAllMessagesPreservingWakeLeaseLocked(reason: String) {
|
||||
headUpSent = null
|
||||
brightnessSentVisible = null
|
||||
brightnessSentLevel = -1
|
||||
brightnessAlsStartedAt = -2000
|
||||
@@ -1448,6 +1550,7 @@ internal fun GlassesSessionCore.hasPendingImageLocked(): Boolean {
|
||||
internal fun GlassesSessionCore.clearPendingMessagesLocked(reason: String) {
|
||||
while (!pendingMessages.isEmpty()) {
|
||||
val message = pendingMessages.removeFirst()
|
||||
if (message.kind == "head-up-control") headUpSent = null
|
||||
discardPrewriteIfMatchesLocked(message)
|
||||
discardImageUpdateStatsLocked(message.imageUpdateId, "pending messages cleared: $reason")
|
||||
magicPool.release(message.sid, message.magic, message.label, "cleared pending: $reason")
|
||||
@@ -1457,6 +1560,7 @@ internal fun GlassesSessionCore.clearPendingMessagesLocked(reason: String) {
|
||||
internal fun GlassesSessionCore.clearInFlightMessagesLocked(reason: String) {
|
||||
while (!inFlightMessages.isEmpty()) {
|
||||
val message = inFlightMessages.removeFirst()
|
||||
if (message.kind == "head-up-control") headUpSent = null
|
||||
discardImageUpdateStatsLocked(message.imageUpdateId, "inflight messages cleared: $reason")
|
||||
magicPool.release(message.sid, message.magic, message.label, "cleared inflight: $reason")
|
||||
}
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/**
|
||||
* Faceclaw's accelerometer frame. Every model's samples are rotated into it before they reach a
|
||||
* FaceclawImuListener ([GlassesSessionCore.emitImuData]), so Faceclaw apps and EvenHub pages see
|
||||
* the same frame whichever glasses are worn. It is the G2's own frame, in g, glasses worn:
|
||||
*
|
||||
* +x forward (the way the wearer faces), +y to the wearer's left, +z up through the top of the head.
|
||||
*
|
||||
* At rest a reading is the "up" direction (the reaction to gravity): head level reads (0, 0, 1),
|
||||
* looking 45° up (0.71, 0, 0.71), and the head tilted 45° to the right (right ear down)
|
||||
* (0, 0.71, 0.71). Pitch above level is atan2(x, hypot(y, z)).
|
||||
*/
|
||||
object ImuOrientation {
|
||||
/**
|
||||
* A sample as [type]'s hardware reports it, in the canonical frame. Axes measured on hardware
|
||||
* with the Developer app's accelerometer page:
|
||||
* - G2: already canonical.
|
||||
* - G1 (g1v0): +x right, +y up, +z back. Level reads (0, 1, 0), 45° up (0, 0.71, -0.71),
|
||||
* 45° to the right (-0.71, 0.71, 0).
|
||||
* - RayNeo iO: no accelerometer stream yet, so nothing to rotate; measure its axes when one
|
||||
* is added.
|
||||
*/
|
||||
fun toCanonical(type: GlassesType, x: Double, y: Double, z: Double): DoubleArray = when (type) {
|
||||
GlassesType.G2 -> doubleArrayOf(x, y, z)
|
||||
GlassesType.G1 -> doubleArrayOf(-z, -x, y)
|
||||
GlassesType.RAYNEO_IO -> doubleArrayOf(x, y, z)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import platform.Foundation.NSData
|
||||
|
||||
/**
|
||||
* Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's
|
||||
* captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record:
|
||||
* 16 kHz mono PCM16, as its recognizer hands it to TypeScript.
|
||||
*/
|
||||
class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) {
|
||||
/** The recording stopped early: the file could not be written. */
|
||||
val failed: Boolean get() = writer.failed
|
||||
|
||||
/** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */
|
||||
fun append(pcm16le: NSData) {
|
||||
val bytes = pcm16le.byteArray()
|
||||
val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) }
|
||||
writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs()))
|
||||
}
|
||||
|
||||
/** Completes the file. Returns the recording's length in ms. */
|
||||
fun finish(): Long = writer.finish()
|
||||
}
|
||||
|
||||
object IosMicRecordings {
|
||||
/**
|
||||
* A recorder writing a new file at [path], or null when the file cannot be created.
|
||||
* [configJson] describes the capture and is stored in the file.
|
||||
*/
|
||||
fun open(path: String, configJson: String?): IosMicRecorder? =
|
||||
try {
|
||||
IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform))
|
||||
} catch (error: Throwable) {
|
||||
PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}")
|
||||
null
|
||||
}
|
||||
}
|
||||
@@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped {
|
||||
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput {
|
||||
val file = fopen(path, "wb+") ?: error("Cannot write $path")
|
||||
return object : SeekableOutput {
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
private fun put(bytes: ByteArray, offset: Int, length: Int) {
|
||||
if (length == 0) return
|
||||
bytes.usePinned {
|
||||
check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert<platform.posix.size_t>()) { "Short write to $path" }
|
||||
}
|
||||
}
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
put(bytes, offset, length)
|
||||
size += length
|
||||
// Each write is a complete element; hand it to the OS so a crash keeps it.
|
||||
platform.posix.fflush(file)
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0)
|
||||
put(bytes, 0, bytes.size)
|
||||
check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0)
|
||||
platform.posix.fflush(file)
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput {
|
||||
val file = fopen(path, "rb") ?: error("Cannot read $path")
|
||||
val fileSize = fileLength(path)
|
||||
return object : SeekableInput {
|
||||
override val length = fileSize
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0
|
||||
return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() }
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
fclose(file)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import platform.CoreBluetooth.CBPeripheralDelegateProtocol
|
||||
import platform.CoreBluetooth.CBPeripheralStateConnected
|
||||
import platform.CoreBluetooth.CBPeripheralStateDisconnected
|
||||
import platform.CoreBluetooth.CBService
|
||||
import platform.CoreBluetooth.CBUUID
|
||||
import platform.Foundation.NSData
|
||||
import platform.Foundation.NSError
|
||||
import platform.Foundation.NSNumber
|
||||
@@ -47,6 +48,9 @@ interface IosBleScanListener {
|
||||
fun onAdvertisement(identifier: String, name: String, manufacturerDataHex: String, rssi: Int, connectable: Boolean)
|
||||
}
|
||||
|
||||
/** A peripheral the system holds a connection to; see [IosBleCentral.systemConnectedG2Arms]. */
|
||||
class IosConnectedPeripheral(val identifier: String, val name: String)
|
||||
|
||||
/** ANCS authorization changes for a connected peripheral; delivered on the BLE queue. */
|
||||
fun interface IosAncsAuthorizationListener {
|
||||
fun onAncsAuthorization(identifier: String, authorized: Boolean)
|
||||
@@ -214,6 +218,25 @@ class IosBleCentral(private val platform: ProtocolPlatform = IosProtocolPlatform
|
||||
manager.scanForPeripheralsWithServices(null, mapOf(CBCentralManagerScanOptionAllowDuplicatesKey to true))
|
||||
}
|
||||
|
||||
/**
|
||||
* G2 arms the system is connected to, through this app, another app or iOS itself. A
|
||||
* connected arm stops advertising, and iOS keeps the right arm's ANCS link open after a
|
||||
* session ends, so a scan alone never hears it. Empty until Bluetooth is powered on.
|
||||
*/
|
||||
fun systemConnectedG2Arms(): List<IosConnectedPeripheral> {
|
||||
val found = ArrayList<IosConnectedPeripheral>()
|
||||
dispatch_sync(queue) {
|
||||
val manager = central ?: return@dispatch_sync
|
||||
if (manager.state != CBManagerStatePoweredOn) return@dispatch_sync
|
||||
val services = listOf(BleProtocol.PROTOCOL_SERVICE_UUID, BleProtocol.RENDER_SERVICE_UUID).map { CBUUID.UUIDWithString(it) }
|
||||
for (item in manager.retrieveConnectedPeripheralsWithServices(services)) {
|
||||
val peripheral = item as? CBPeripheral ?: continue
|
||||
found.add(IosConnectedPeripheral(peripheral.identifier.UUIDString.uppercase(), peripheral.name ?: ""))
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
/** Known peripheral identifiers from this process's scans/retrievals (uppercase). */
|
||||
fun knownIdentifiers(): List<String> = cond.withLock { peers.keys.toList() }
|
||||
|
||||
|
||||
@@ -34,6 +34,11 @@ interface IosMicStatusListener {
|
||||
fun onNsDataMicStatus(body: NSData, arm: String)
|
||||
}
|
||||
|
||||
/** CFW keyboard-bridge records (the raw field-131 record) as NSData. Main queue. */
|
||||
interface IosKeyboardEventListener {
|
||||
fun onNsDataKeyboardEvent(body: NSData)
|
||||
}
|
||||
|
||||
/**
|
||||
* ObjC-facing facade over the shared [GlassesSessionCore] for the NativeScript app: the iOS
|
||||
* counterpart of Android's FaceclawBleCommunicator with the same method names. GATT goes
|
||||
@@ -230,6 +235,20 @@ class IosGlassesSession internal constructor(
|
||||
return adapter
|
||||
}
|
||||
|
||||
fun addKeyboardEventListenerNsData(listener: IosKeyboardEventListener): FaceclawKeyboardEventListener {
|
||||
val adapter = object : FaceclawKeyboardEventListener {
|
||||
override fun onKeyboardEvent(body: ByteArray?) =
|
||||
listener.onNsDataKeyboardEvent((body ?: ByteArray(0)).data())
|
||||
}
|
||||
core.addKeyboardEventListener(adapter)
|
||||
return adapter
|
||||
}
|
||||
|
||||
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
|
||||
|
||||
fun sendFaceclawKeyboardControl(record: NSData?, label: String): Boolean =
|
||||
core.sendFaceclawKeyboardControl(record?.byteArray(), label)
|
||||
|
||||
/** Every audio-characteristic packet, whether or not a capture is active (passive; sends nothing). */
|
||||
fun addAudioMonitorListenerNsData(listener: IosAudioPacketListener): FaceclawAudioPacketListener {
|
||||
val adapter = object : FaceclawAudioPacketListener {
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# Faceclaw SSH transport
|
||||
|
||||
The Terminal app's SSH client: a small Go package over
|
||||
[golang.org/x/crypto/ssh](https://pkg.go.dev/golang.org/x/crypto/ssh), bound
|
||||
into the apps with gomobile. It provides text-message channels over SSH:
|
||||
|
||||
- **exec**: run a command on the host, exchanging newline-delimited text
|
||||
(used for tmux control mode and the tmux session poller; see
|
||||
`app/apps/terminal/tmux-client.ts`).
|
||||
- **websocket**: a WebSocket to an address as seen from the host, through a
|
||||
`direct-tcpip` channel (used to tunnel g2mirror-server's protocol).
|
||||
|
||||
Channels to the same host and credentials share one SSH connection, with a
|
||||
keepalive that fails them promptly when the peer silently disappears. Host
|
||||
keys are pinned by the app (first connection reports the presented key so
|
||||
the user can verify its fingerprint). Authentication uses a non-exportable
|
||||
ECDSA P-256 key held by the platform keystore (Android Keystore/StrongBox,
|
||||
iOS Secure Enclave; see `FaceclawSsh.kt` / `FaceclawSsh.m`) and, if the
|
||||
server allows it, a password saved encrypted on the phone.
|
||||
|
||||
## Building
|
||||
|
||||
`scripts/go-ssh-build.cjs` runs `gomobile bind` (pinned in `go.mod`'s tool
|
||||
block) from the NativeScript before-prepare hook and stages
|
||||
`faceclaw-ssh.aar` / `Faceclawssh.xcframework` into the local
|
||||
`@faceclaw/kotlin` plugin. It needs Go 1.26+ on PATH, plus the Android NDK
|
||||
(Android) or Xcode (iOS); a content stamp skips rebuilds when the Go sources
|
||||
haven't changed.
|
||||
|
||||
## Testing
|
||||
|
||||
`go test -race ./...` runs the transport against an in-process SSH server
|
||||
(host-key pinning, password and keyboard-interactive auth, key auth through
|
||||
a fake platform signer, exec framing, the websocket tunnel, connection
|
||||
sharing and loss).
|
||||
|
||||
## Keeping it current
|
||||
|
||||
x/crypto/ssh is the security boundary against a malicious host or a
|
||||
man in the middle; update it promptly when advisories appear
|
||||
(`go get golang.org/x/crypto@latest && go mod tidy`). A Go panic in any
|
||||
goroutine terminates the app: the package recovers in the goroutines it
|
||||
owns, but not inside the library's own.
|
||||
@@ -0,0 +1,254 @@
|
||||
package faceclawssh
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type closedPayload struct {
|
||||
ExitStatus *int `json:"exitStatus,omitempty"`
|
||||
Signal string `json:"signal,omitempty"`
|
||||
Code *int `json:"code,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// runExec runs command on the host and relays newline-delimited text both
|
||||
// ways until the command exits, the connection drops, or Close.
|
||||
func (c *Channel) runExec(conn *pooledConn, command string) {
|
||||
session, err := conn.client.NewSession()
|
||||
if err != nil {
|
||||
c.fail(channelOpenFailure(conn, err))
|
||||
return
|
||||
}
|
||||
defer session.Close()
|
||||
stdin, err1 := session.StdinPipe()
|
||||
stdout, err2 := session.StdoutPipe()
|
||||
stderr, err3 := session.StderrPipe()
|
||||
if err := errors.Join(err1, err2, err3); err != nil {
|
||||
c.fail(&failure{Kind: "channel", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
if err := session.Start(command); err != nil {
|
||||
c.fail(&failure{Kind: "channel", Message: "could not run command: " + err.Error()})
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
<-c.ctx.Done()
|
||||
session.Close()
|
||||
}()
|
||||
c.markOpen()
|
||||
go func() {
|
||||
defer c.guard()
|
||||
for text := range c.sendQ {
|
||||
if _, err := io.WriteString(stdin, text+"\n"); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
// Close was called and the queue has drained: EOF tells the command
|
||||
// we're done (tmux control mode detaches on it).
|
||||
stdin.Close()
|
||||
}()
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
defer c.guard()
|
||||
readLines(stderr, func(line string) { c.emit("stderr", line) })
|
||||
}()
|
||||
readErr := readLines(stdout, func(line string) { c.emit("text", line) })
|
||||
// Deliver the command's last words (e.g. "tmux: command not found")
|
||||
// before the close event, but don't hang on a stderr that never ends.
|
||||
select {
|
||||
case <-stderrDone:
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
if c.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
waitErr := session.Wait()
|
||||
var exitErr *ssh.ExitError
|
||||
var missing *ssh.ExitMissingError
|
||||
switch {
|
||||
case waitErr == nil:
|
||||
status := 0
|
||||
c.finish("closed", closedPayload{ExitStatus: &status})
|
||||
case errors.As(waitErr, &exitErr):
|
||||
status := exitErr.ExitStatus()
|
||||
c.finish("closed", closedPayload{ExitStatus: &status, Signal: exitErr.Signal()})
|
||||
case errors.As(waitErr, &missing):
|
||||
// The channel closed without an exit status: usually the connection
|
||||
// went away under it.
|
||||
c.fail(lostFailure(conn, readErr))
|
||||
default:
|
||||
c.fail(lostFailure(conn, waitErr))
|
||||
}
|
||||
}
|
||||
|
||||
// readLines calls emit for each line of r (without its newline) and returns
|
||||
// the read error that ended it (nil at EOF). Lines may be arbitrarily long.
|
||||
func readLines(r io.Reader, emit func(string)) error {
|
||||
reader := bufio.NewReaderSize(r, 64*1024)
|
||||
for {
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if len(line) > 0 {
|
||||
if line[len(line)-1] == '\n' {
|
||||
line = line[:len(line)-1]
|
||||
}
|
||||
emit(escapeLine(line))
|
||||
}
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// escapeLine makes a line safe to hand across the gomobile boundary as a
|
||||
// string (which must be valid UTF-8). Valid UTF-8 passes through. Otherwise
|
||||
// every byte >= 0x80 becomes "\ooo" (backslash, three octal digits): the same
|
||||
// escape tmux control mode uses for control characters and backslashes in
|
||||
// %output, so a %output line split mid-character still decodes back to its
|
||||
// exact bytes. (Outside %output a literal backslash is ambiguous with this
|
||||
// escape; such lines are only ever displayed.)
|
||||
func escapeLine(line []byte) string {
|
||||
if utf8.Valid(line) {
|
||||
return string(line)
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, c := range line {
|
||||
if c >= 0x80 {
|
||||
fmt.Fprintf(&b, "\\%03o", c)
|
||||
} else {
|
||||
b.WriteByte(c)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// runWebSocket opens rawURL (ws://host:port/path) through the SSH
|
||||
// connection — host:port is resolved on the SSH server's side — and relays
|
||||
// text frames.
|
||||
func (c *Channel) runWebSocket(conn *pooledConn, rawURL string) {
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err != nil || parsed.Scheme != "ws" || parsed.Host == "" {
|
||||
c.fail(&failure{Kind: "config", Message: fmt.Sprintf("bad websocket URL %q", rawURL)})
|
||||
return
|
||||
}
|
||||
target := parsed.Host
|
||||
if parsed.Port() == "" {
|
||||
target = net.JoinHostPort(parsed.Hostname(), "80")
|
||||
}
|
||||
dialer := websocket.Dialer{
|
||||
NetDialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
forwarded, err := conn.client.DialContext(ctx, "tcp", target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return noDeadlineConn{forwarded}, nil
|
||||
},
|
||||
}
|
||||
// The handshake timeout rides the context: SSH channels can't do deadlines.
|
||||
handshakeCtx, cancelHandshake := context.WithTimeout(c.ctx, defaultConnectTimeout)
|
||||
ws, _, err := dialer.DialContext(handshakeCtx, rawURL, nil)
|
||||
cancelHandshake()
|
||||
if err != nil {
|
||||
if c.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
c.fail(channelOpenFailure(conn, fmt.Errorf("could not reach %s on the host: %w", target, err)))
|
||||
return
|
||||
}
|
||||
defer ws.Close()
|
||||
go func() {
|
||||
<-c.ctx.Done()
|
||||
ws.Close()
|
||||
}()
|
||||
c.markOpen()
|
||||
go func() {
|
||||
defer c.guard()
|
||||
for text := range c.sendQ {
|
||||
if err := ws.WriteMessage(websocket.TextMessage, []byte(text)); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
ws.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "bye"),
|
||||
time.Now().Add(time.Second))
|
||||
}()
|
||||
for {
|
||||
kind, data, err := ws.ReadMessage()
|
||||
if err != nil {
|
||||
if c.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var closeErr *websocket.CloseError
|
||||
if errors.As(err, &closeErr) {
|
||||
code := closeErr.Code
|
||||
c.finish("closed", closedPayload{Code: &code, Reason: closeErr.Text})
|
||||
return
|
||||
}
|
||||
c.fail(lostFailure(conn, err))
|
||||
return
|
||||
}
|
||||
if kind == websocket.TextMessage {
|
||||
c.emit("text", string(data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// noDeadlineConn adapts an SSH direct-tcpip channel for the websocket
|
||||
// library, which sets deadlines the channel doesn't support. Timeouts come
|
||||
// from contexts and the SSH keepalive instead.
|
||||
type noDeadlineConn struct{ net.Conn }
|
||||
|
||||
func (noDeadlineConn) SetDeadline(time.Time) error { return nil }
|
||||
func (noDeadlineConn) SetReadDeadline(time.Time) error { return nil }
|
||||
func (noDeadlineConn) SetWriteDeadline(time.Time) error { return nil }
|
||||
|
||||
// channelOpenFailure classifies a failure to open a channel: if the SSH
|
||||
// connection itself is gone it's a network failure (reconnect), otherwise
|
||||
// the host refused this particular channel.
|
||||
func channelOpenFailure(conn *pooledConn, err error) *failure {
|
||||
if isConnDead(conn) {
|
||||
return &failure{Kind: "network", Message: "connection lost: " + err.Error()}
|
||||
}
|
||||
return &failure{Kind: "channel", Message: err.Error()}
|
||||
}
|
||||
|
||||
func lostFailure(conn *pooledConn, err error) *failure {
|
||||
message := "connection lost"
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
message += ": " + err.Error()
|
||||
}
|
||||
if !isConnDead(conn) {
|
||||
// The connection is fine; only this channel ended abruptly.
|
||||
return &failure{Kind: "channel", Message: "channel closed unexpectedly"}
|
||||
}
|
||||
return &failure{Kind: "network", Message: message}
|
||||
}
|
||||
|
||||
func isConnDead(conn *pooledConn) bool {
|
||||
// client.Wait's goroutine marks the connection dead slightly after the
|
||||
// channels see their streams end; give it a moment to catch up.
|
||||
deadline := time.Now().Add(200 * time.Millisecond)
|
||||
for {
|
||||
poolMu.Lock()
|
||||
dead := conn.dead
|
||||
poolMu.Unlock()
|
||||
if dead || time.Now().After(deadline) {
|
||||
return dead
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
// Package faceclawssh is Faceclaw's SSH transport, bound into the Android and
|
||||
// iOS apps with gomobile. It exposes text-message channels over SSH:
|
||||
//
|
||||
// - "exec": run a command on the host and exchange newline-delimited text
|
||||
// with it (stdin lines in, stdout lines out). Used for tmux control mode
|
||||
// and the tmux session poller.
|
||||
// - "websocket": open a direct-tcpip channel to an address as seen from the
|
||||
// host (normally g2mirror-server on 127.0.0.1:8737) and speak WebSocket
|
||||
// text frames over it.
|
||||
//
|
||||
// Channels to the same host, user and credentials share one SSH connection
|
||||
// (see pool.go). Host keys are pinned by the caller: a connection whose host
|
||||
// presents a key that isn't in the spec's hostKeys fails with a
|
||||
// "hostkey-unknown" or "hostkey-mismatch" failure carrying the presented key,
|
||||
// so the app can show the fingerprint and retry once the user has trusted it.
|
||||
// Authentication uses a platform-held, non-exportable ECDSA P-256 key (the
|
||||
// Signer interface; Android Keystore / iOS Secure Enclave) and, if the server
|
||||
// allows it, a password.
|
||||
//
|
||||
// Everything crossing the gomobile boundary is a string, []byte or one of the
|
||||
// interfaces below; structured values are JSON strings.
|
||||
package faceclawssh
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Signer is implemented by the platform: an ECDSA P-256 key whose private
|
||||
// half never leaves the device's keystore.
|
||||
type Signer interface {
|
||||
// PublicKey returns the key's public half, either as X.509
|
||||
// SubjectPublicKeyInfo DER (Android's PublicKey.getEncoded()) or as an
|
||||
// uncompressed X9.62 point, 0x04 || X || Y (iOS's
|
||||
// SecKeyCopyExternalRepresentation).
|
||||
PublicKey() []byte
|
||||
// Sign returns the ASN.1 DER ECDSA signature of SHA-256(data).
|
||||
Sign(data []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
// Listener receives a channel's events. Calls arrive on arbitrary goroutines
|
||||
// (never concurrently for one channel); the platform wrapper hops them onto
|
||||
// the thread that owns the channel.
|
||||
//
|
||||
// Event kinds and payloads:
|
||||
// - "open": the channel is ready; payload empty. Sends made before this are
|
||||
// queued and delivered once open.
|
||||
// - "text": one message (websocket text frame, or one stdout line without
|
||||
// its newline). Exec lines that aren't valid UTF-8 have every byte >= 0x80
|
||||
// escaped as a backslash and three octal digits, matching tmux control
|
||||
// mode's escaping of control characters (see escapeLine).
|
||||
// - "stderr": one stderr line of an exec channel, escaped the same way.
|
||||
// - "closed": the channel ended normally; payload is JSON
|
||||
// {"exitStatus": n, "signal": "...", "code": n, "reason": "..."} with the
|
||||
// fields that apply (exec exit status/signal, websocket close code/reason).
|
||||
// Terminal: no events follow.
|
||||
// - "failure": the channel could not be opened or was lost; payload is a
|
||||
// JSON Failure. Terminal: no events follow.
|
||||
type Listener interface {
|
||||
OnEvent(kind string, payload string)
|
||||
}
|
||||
|
||||
// Failure describes why a channel failed, as the JSON payload of a "failure"
|
||||
// event.
|
||||
type failure struct {
|
||||
// "hostkey-unknown": no host key is pinned yet; HostKey is what the host
|
||||
// presented. "hostkey-mismatch": the host presented a key that isn't
|
||||
// pinned. "auth": the host refused every credential we have.
|
||||
// "network": couldn't reach the host, or the connection dropped.
|
||||
// "channel": the SSH connection is fine but the channel couldn't be
|
||||
// opened (exec refused, nothing listening at the forwarded address...).
|
||||
// "config": the spec was malformed. "internal": a bug.
|
||||
Kind string `json:"kind"`
|
||||
Message string `json:"message"`
|
||||
HostKey *hostKeyInfo `json:"hostKey,omitempty"`
|
||||
// For "auth": the host accepts password (or keyboard-interactive)
|
||||
// authentication, whether a password was sent, and whether our key was
|
||||
// offered.
|
||||
PasswordAllowed bool `json:"passwordAllowed,omitempty"`
|
||||
PasswordTried bool `json:"passwordTried,omitempty"`
|
||||
KeyTried bool `json:"keyTried,omitempty"`
|
||||
}
|
||||
|
||||
func (f *failure) Error() string { return f.Message }
|
||||
|
||||
type hostKeyInfo struct {
|
||||
// SSH key type, e.g. "ssh-ed25519".
|
||||
Type string `json:"type"`
|
||||
// OpenSSH-style SHA-256 fingerprint, "SHA256:<base64>".
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
// The key in SSH wire format, base64: what goes in a spec's hostKeys.
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
type channelSpec struct {
|
||||
// "exec" or "websocket".
|
||||
Type string `json:"type"`
|
||||
// exec: the command line, run by the user's login shell.
|
||||
Command string `json:"command"`
|
||||
// websocket: ws:// URL, resolved on the host's side of the connection.
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type openSpec struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
// Pinned host keys, SSH wire format, base64. Empty: the first connection
|
||||
// fails with "hostkey-unknown".
|
||||
HostKeys []string `json:"hostKeys"`
|
||||
// Offer the platform key (the Signer passed to Open).
|
||||
UseKey bool `json:"useKey"`
|
||||
ConnectTimeoutMs int `json:"connectTimeoutMs"`
|
||||
KeepaliveIntervalMs int `json:"keepaliveIntervalMs"`
|
||||
Channel channelSpec `json:"channel"`
|
||||
}
|
||||
|
||||
const (
|
||||
defaultConnectTimeout = 15 * time.Second
|
||||
defaultKeepaliveInterval = 20 * time.Second
|
||||
// Messages queued on a channel before Send starts refusing them.
|
||||
sendQueueCapacity = 4096
|
||||
// How long Close waits for queued messages to flush and the far end to
|
||||
// finish before tearing the channel down.
|
||||
closeGracePeriod = 2 * time.Second
|
||||
)
|
||||
|
||||
// Channel is one open (or opening) channel. Methods are safe to call from
|
||||
// any thread.
|
||||
type Channel struct {
|
||||
listener Listener
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
|
||||
mu sync.Mutex
|
||||
closed bool // Close was called: no more sends, no more events
|
||||
opened bool
|
||||
sendQ chan string
|
||||
|
||||
finishOnce sync.Once
|
||||
}
|
||||
|
||||
// Open starts opening a channel described by spec (a JSON openSpec) and
|
||||
// returns immediately; progress arrives as listener events. signer may be
|
||||
// nil (no key authentication).
|
||||
func Open(spec string, signer Signer, listener Listener) *Channel {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &Channel{
|
||||
listener: listener,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
sendQ: make(chan string, sendQueueCapacity),
|
||||
}
|
||||
go c.run(spec, signer)
|
||||
return c
|
||||
}
|
||||
|
||||
// Send queues one message: a websocket text frame, or a line for an exec
|
||||
// channel's stdin (the newline is added). Returns false if the channel is
|
||||
// closed or its queue is full.
|
||||
func (c *Channel) Send(text string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.closed {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case c.sendQ <- text:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Close ends the channel. Messages already queued are still sent (so a
|
||||
// parting "unview" reaches the host), then the channel's write side is
|
||||
// closed and, after a short grace period, the channel is torn down. No
|
||||
// events are delivered after Close.
|
||||
func (c *Channel) Close() {
|
||||
c.mu.Lock()
|
||||
if c.closed {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
c.closed = true
|
||||
opened := c.opened
|
||||
close(c.sendQ)
|
||||
c.mu.Unlock()
|
||||
if !opened {
|
||||
c.cancel()
|
||||
return
|
||||
}
|
||||
time.AfterFunc(closeGracePeriod, c.cancel)
|
||||
}
|
||||
|
||||
func (c *Channel) isClosed() bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.closed
|
||||
}
|
||||
|
||||
func (c *Channel) emit(kind, payload string) {
|
||||
if c.isClosed() {
|
||||
return
|
||||
}
|
||||
c.listener.OnEvent(kind, payload)
|
||||
}
|
||||
|
||||
func (c *Channel) markOpen() {
|
||||
c.mu.Lock()
|
||||
c.opened = true
|
||||
c.mu.Unlock()
|
||||
c.emit("open", "")
|
||||
}
|
||||
|
||||
// finish delivers the channel's one terminal event ("closed" or "failure").
|
||||
func (c *Channel) finish(kind string, payload any) {
|
||||
c.finishOnce.Do(func() {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
encoded = []byte(`{"kind":"internal","message":"unencodable event"}`)
|
||||
}
|
||||
c.emit(kind, string(encoded))
|
||||
c.cancel()
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Channel) fail(f *failure) { c.finish("failure", f) }
|
||||
|
||||
// guard turns a panic in one of our goroutines into a failure event instead
|
||||
// of crashing the app (an unrecovered Go panic kills the whole process).
|
||||
func (c *Channel) guard() {
|
||||
if r := recover(); r != nil {
|
||||
c.fail(&failure{Kind: "internal", Message: fmt.Sprintf("internal error: %v\n%s", r, debug.Stack())})
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Channel) run(specJSON string, signer Signer) {
|
||||
defer c.guard()
|
||||
var spec openSpec
|
||||
if err := json.Unmarshal([]byte(specJSON), &spec); err != nil {
|
||||
c.fail(&failure{Kind: "config", Message: "bad channel spec: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if spec.Host == "" || spec.User == "" {
|
||||
c.fail(&failure{Kind: "config", Message: "host and user are required"})
|
||||
return
|
||||
}
|
||||
if spec.Port == 0 {
|
||||
spec.Port = 22
|
||||
}
|
||||
if !spec.UseKey {
|
||||
signer = nil
|
||||
}
|
||||
conn, err := acquire(c.ctx, &spec, signer)
|
||||
if err != nil {
|
||||
if c.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
c.fail(asFailure(err))
|
||||
return
|
||||
}
|
||||
defer conn.release()
|
||||
switch spec.Channel.Type {
|
||||
case "exec":
|
||||
c.runExec(conn, spec.Channel.Command)
|
||||
case "websocket":
|
||||
c.runWebSocket(conn, spec.Channel.URL)
|
||||
default:
|
||||
c.fail(&failure{Kind: "config", Message: fmt.Sprintf("unknown channel type %q", spec.Channel.Type)})
|
||||
}
|
||||
}
|
||||
|
||||
func asFailure(err error) *failure {
|
||||
if f, ok := err.(*failure); ok {
|
||||
return f
|
||||
}
|
||||
return &failure{Kind: "network", Message: err.Error()}
|
||||
}
|
||||
|
||||
// AuthorizedKey formats a platform public key (see Signer.PublicKey) as an
|
||||
// authorized_keys line: "ecdsa-sha2-nistp256 AAAA... comment".
|
||||
func AuthorizedKey(publicKey []byte, comment string) (string, error) {
|
||||
key, err := sshPublicKey(publicKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return authorizedKeyLine(key, comment), nil
|
||||
}
|
||||
|
||||
// Fingerprint returns the OpenSSH SHA-256 fingerprint ("SHA256:...") of a
|
||||
// platform public key.
|
||||
func Fingerprint(publicKey []byte) (string, error) {
|
||||
key, err := sshPublicKey(publicKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fingerprint(key), nil
|
||||
}
|
||||
@@ -0,0 +1,526 @@
|
||||
package faceclawssh
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/ecdsa"
|
||||
"crypto/ed25519"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// ---- test SSH server ----
|
||||
|
||||
type testServer struct {
|
||||
addr string
|
||||
hostKey ssh.Signer
|
||||
accepts atomic.Int32
|
||||
listener net.Listener
|
||||
}
|
||||
|
||||
type serverOptions struct {
|
||||
password string // "" = password auth disabled
|
||||
keyboardInteractive bool // offer the password via keyboard-interactive instead
|
||||
authorizedKey ssh.PublicKey
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, opts serverOptions) *testServer {
|
||||
t.Helper()
|
||||
_, priv, _ := ed25519.GenerateKey(rand.Reader)
|
||||
hostKey, err := ssh.NewSignerFromKey(priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := &ssh.ServerConfig{}
|
||||
if opts.password != "" && !opts.keyboardInteractive {
|
||||
config.PasswordCallback = func(_ ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) {
|
||||
if string(password) == opts.password {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("wrong password")
|
||||
}
|
||||
}
|
||||
if opts.password != "" && opts.keyboardInteractive {
|
||||
config.KeyboardInteractiveCallback = func(_ ssh.ConnMetadata, challenge ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
|
||||
answers, err := challenge("", "", []string{"Password: "}, []bool{false})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(answers) == 1 && answers[0] == opts.password {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("wrong password")
|
||||
}
|
||||
}
|
||||
if opts.authorizedKey != nil {
|
||||
config.PublicKeyCallback = func(_ ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
|
||||
if string(key.Marshal()) == string(opts.authorizedKey.Marshal()) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown key")
|
||||
}
|
||||
}
|
||||
config.AddHostKey(hostKey)
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := &testServer{addr: listener.Addr().String(), hostKey: hostKey, listener: listener}
|
||||
t.Cleanup(func() { listener.Close() })
|
||||
go func() {
|
||||
for {
|
||||
tcp, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
server.accepts.Add(1)
|
||||
go server.serve(tcp, config)
|
||||
}
|
||||
}()
|
||||
return server
|
||||
}
|
||||
|
||||
func (s *testServer) serve(tcp net.Conn, config *ssh.ServerConfig) {
|
||||
conn, chans, reqs, err := ssh.NewServerConn(tcp, config)
|
||||
if err != nil {
|
||||
tcp.Close()
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
go ssh.DiscardRequests(reqs) // replies false to keepalives, which still counts as alive
|
||||
for newChannel := range chans {
|
||||
switch newChannel.ChannelType() {
|
||||
case "session":
|
||||
go serveSession(newChannel)
|
||||
case "direct-tcpip":
|
||||
go serveDirectTCPIP(newChannel)
|
||||
default:
|
||||
newChannel.Reject(ssh.UnknownChannelType, "nope")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// serveSession implements a few fake commands:
|
||||
// - "echo": echo stdin lines back with a "> " prefix until EOF, exit 0
|
||||
// - "bytes": print an invalid-UTF-8 line and a stderr line, exit 3
|
||||
func serveSession(newChannel ssh.NewChannel) {
|
||||
channel, requests, err := newChannel.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer channel.Close()
|
||||
for req := range requests {
|
||||
if req.Type != "exec" {
|
||||
req.Reply(false, nil)
|
||||
continue
|
||||
}
|
||||
var payload struct{ Command string }
|
||||
ssh.Unmarshal(req.Payload, &payload)
|
||||
req.Reply(true, nil)
|
||||
status := 0
|
||||
switch payload.Command {
|
||||
case "echo":
|
||||
scanner := bufio.NewScanner(channel)
|
||||
for scanner.Scan() {
|
||||
fmt.Fprintf(channel, "> %s\n", scanner.Text())
|
||||
}
|
||||
case "bytes":
|
||||
channel.Write([]byte("%output %1 caf\xc3\n%output %1 \xa9 ok\nplain ✓\n"))
|
||||
channel.Stderr().Write([]byte("oops\n"))
|
||||
status = 3
|
||||
default:
|
||||
channel.Stderr().Write([]byte("unknown command\n"))
|
||||
status = 127
|
||||
}
|
||||
channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{uint32(status)}))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func serveDirectTCPIP(newChannel ssh.NewChannel) {
|
||||
var target struct {
|
||||
Host string
|
||||
Port uint32
|
||||
OriginHost string
|
||||
OriginPort uint32
|
||||
}
|
||||
if err := ssh.Unmarshal(newChannel.ExtraData(), &target); err != nil {
|
||||
newChannel.Reject(ssh.ConnectionFailed, "bad payload")
|
||||
return
|
||||
}
|
||||
upstream, err := net.Dial("tcp", net.JoinHostPort(target.Host, fmt.Sprint(target.Port)))
|
||||
if err != nil {
|
||||
newChannel.Reject(ssh.ConnectionFailed, err.Error())
|
||||
return
|
||||
}
|
||||
channel, requests, err := newChannel.Accept()
|
||||
if err != nil {
|
||||
upstream.Close()
|
||||
return
|
||||
}
|
||||
go ssh.DiscardRequests(requests)
|
||||
go func() { io.Copy(upstream, channel); upstream.Close() }()
|
||||
io.Copy(channel, upstream)
|
||||
channel.Close()
|
||||
}
|
||||
|
||||
// ---- test client plumbing ----
|
||||
|
||||
type event struct{ kind, payload string }
|
||||
|
||||
type recorder struct {
|
||||
events chan event
|
||||
}
|
||||
|
||||
func newRecorder() *recorder { return &recorder{events: make(chan event, 256)} }
|
||||
|
||||
func (r *recorder) OnEvent(kind, payload string) { r.events <- event{kind, payload} }
|
||||
|
||||
func (r *recorder) next(t *testing.T) event {
|
||||
t.Helper()
|
||||
select {
|
||||
case e := <-r.events:
|
||||
return e
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("timed out waiting for event")
|
||||
return event{}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *recorder) expect(t *testing.T, kind string) event {
|
||||
t.Helper()
|
||||
e := r.next(t)
|
||||
if e.kind != kind {
|
||||
t.Fatalf("got event %s(%s), want %s", e.kind, e.payload, kind)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func (r *recorder) expectFailure(t *testing.T) failure {
|
||||
t.Helper()
|
||||
e := r.expect(t, "failure")
|
||||
var f failure
|
||||
if err := json.Unmarshal([]byte(e.payload), &f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
type softwareSigner struct{ key *ecdsa.PrivateKey }
|
||||
|
||||
func newSoftwareSigner(t *testing.T) *softwareSigner {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &softwareSigner{key: key}
|
||||
}
|
||||
|
||||
func (s *softwareSigner) PublicKey() []byte {
|
||||
der, _ := x509.MarshalPKIXPublicKey(&s.key.PublicKey)
|
||||
return der
|
||||
}
|
||||
|
||||
func (s *softwareSigner) Sign(data []byte) ([]byte, error) {
|
||||
digest := sha256.Sum256(data)
|
||||
return ecdsa.SignASN1(rand.Reader, s.key, digest[:])
|
||||
}
|
||||
|
||||
func (s *softwareSigner) sshKey(t *testing.T) ssh.PublicKey {
|
||||
key, err := ssh.NewPublicKey(&s.key.PublicKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
func specFor(server *testServer, mutate func(*openSpec)) string {
|
||||
host, port, _ := net.SplitHostPort(server.addr)
|
||||
var portNumber int
|
||||
fmt.Sscan(port, &portNumber)
|
||||
spec := openSpec{
|
||||
Host: host,
|
||||
Port: portNumber,
|
||||
User: "jim",
|
||||
HostKeys: []string{base64.StdEncoding.EncodeToString(server.hostKey.PublicKey().Marshal())},
|
||||
Channel: channelSpec{Type: "exec", Command: "echo"},
|
||||
}
|
||||
if mutate != nil {
|
||||
mutate(&spec)
|
||||
}
|
||||
encoded, _ := json.Marshal(spec)
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
func resetPool() {
|
||||
poolMu.Lock()
|
||||
defer poolMu.Unlock()
|
||||
for key, conn := range pool {
|
||||
conn.client.Close()
|
||||
delete(pool, key)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- tests ----
|
||||
|
||||
func TestHostKeyPinning(t *testing.T) {
|
||||
defer resetPool()
|
||||
server := newTestServer(t, serverOptions{password: "pw"})
|
||||
wantKey := base64.StdEncoding.EncodeToString(server.hostKey.PublicKey().Marshal())
|
||||
|
||||
rec := newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) { s.HostKeys = nil; s.Password = "pw" }), nil, rec)
|
||||
f := rec.expectFailure(t)
|
||||
if f.Kind != "hostkey-unknown" || f.HostKey == nil || f.HostKey.Key != wantKey ||
|
||||
f.HostKey.Type != ssh.KeyAlgoED25519 || !strings.HasPrefix(f.HostKey.Fingerprint, "SHA256:") {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
|
||||
_, otherPriv, _ := ed25519.GenerateKey(rand.Reader)
|
||||
other, _ := ssh.NewSignerFromKey(otherPriv)
|
||||
rec = newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) {
|
||||
s.HostKeys = []string{base64.StdEncoding.EncodeToString(other.PublicKey().Marshal())}
|
||||
s.Password = "pw"
|
||||
}), nil, rec)
|
||||
f = rec.expectFailure(t)
|
||||
if f.Kind != "hostkey-mismatch" || f.HostKey.Key != wantKey {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
|
||||
rec = newRecorder()
|
||||
ch := Open(specFor(server, func(s *openSpec) { s.Password = "pw" }), nil, rec)
|
||||
rec.expect(t, "open")
|
||||
ch.Close()
|
||||
}
|
||||
|
||||
func TestPasswordAuth(t *testing.T) {
|
||||
for _, interactive := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("keyboardInteractive=%v", interactive), func(t *testing.T) {
|
||||
defer resetPool()
|
||||
server := newTestServer(t, serverOptions{password: "secret", keyboardInteractive: interactive})
|
||||
|
||||
rec := newRecorder()
|
||||
Open(specFor(server, nil), nil, rec)
|
||||
f := rec.expectFailure(t)
|
||||
if f.Kind != "auth" || !f.PasswordAllowed || f.PasswordTried {
|
||||
t.Fatalf("no password: %+v", f)
|
||||
}
|
||||
|
||||
rec = newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) { s.Password = "wrong" }), nil, rec)
|
||||
f = rec.expectFailure(t)
|
||||
if f.Kind != "auth" || !f.PasswordTried {
|
||||
t.Fatalf("wrong password: %+v", f)
|
||||
}
|
||||
|
||||
rec = newRecorder()
|
||||
ch := Open(specFor(server, func(s *openSpec) { s.Password = "secret" }), nil, rec)
|
||||
rec.expect(t, "open")
|
||||
ch.Send("hi")
|
||||
if e := rec.expect(t, "text"); e.payload != "> hi" {
|
||||
t.Fatalf("got %q", e.payload)
|
||||
}
|
||||
ch.Close()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyAuth(t *testing.T) {
|
||||
defer resetPool()
|
||||
signer := newSoftwareSigner(t)
|
||||
server := newTestServer(t, serverOptions{authorizedKey: signer.sshKey(t)})
|
||||
|
||||
rec := newRecorder()
|
||||
ch := Open(specFor(server, func(s *openSpec) { s.UseKey = true }), signer, rec)
|
||||
rec.expect(t, "open")
|
||||
ch.Close()
|
||||
|
||||
// A key the server doesn't know, and no password support.
|
||||
resetPool()
|
||||
rec = newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) { s.UseKey = true }), newSoftwareSigner(t), rec)
|
||||
f := rec.expectFailure(t)
|
||||
if f.Kind != "auth" || !f.KeyTried || f.PasswordAllowed {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
|
||||
line, err := AuthorizedKey(signer.PublicKey(), "Faceclaw on my phone")
|
||||
if err != nil || !strings.HasPrefix(line, "ecdsa-sha2-nistp256 AAAA") || !strings.HasSuffix(line, " Faceclaw-on-my-phone") {
|
||||
t.Fatalf("AuthorizedKey = %q, %v", line, err)
|
||||
}
|
||||
parsed, _, _, _, err := ssh.ParseAuthorizedKey([]byte(line))
|
||||
if err != nil || string(parsed.Marshal()) != string(signer.sshKey(t).Marshal()) {
|
||||
t.Fatalf("authorized key does not round-trip: %v", err)
|
||||
}
|
||||
// The uncompressed-point form iOS hands us parses to the same key.
|
||||
point := elliptic.Marshal(elliptic.P256(), signer.key.X, signer.key.Y) //nolint:staticcheck
|
||||
fromPoint, err := Fingerprint(point)
|
||||
fromDER, _ := Fingerprint(signer.PublicKey())
|
||||
if err != nil || fromPoint != fromDER {
|
||||
t.Fatalf("fingerprints differ: %s vs %s (%v)", fromPoint, fromDER, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecFraming(t *testing.T) {
|
||||
defer resetPool()
|
||||
server := newTestServer(t, serverOptions{password: "pw"})
|
||||
rec := newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) { s.Password = "pw"; s.Channel.Command = "bytes" }), nil, rec)
|
||||
rec.expect(t, "open")
|
||||
var texts, stderr []string
|
||||
var closed closedPayload
|
||||
for done := false; !done; {
|
||||
e := rec.next(t)
|
||||
switch e.kind {
|
||||
case "text":
|
||||
texts = append(texts, e.payload)
|
||||
case "stderr":
|
||||
stderr = append(stderr, e.payload)
|
||||
case "closed":
|
||||
json.Unmarshal([]byte(e.payload), &closed)
|
||||
done = true
|
||||
default:
|
||||
t.Fatalf("unexpected %s(%s)", e.kind, e.payload)
|
||||
}
|
||||
}
|
||||
want := []string{`%output %1 caf\303`, `%output %1 \251 ok`, "plain ✓"}
|
||||
if strings.Join(texts, "|") != strings.Join(want, "|") {
|
||||
t.Fatalf("texts = %q, want %q", texts, want)
|
||||
}
|
||||
if len(stderr) != 1 || stderr[0] != "oops" {
|
||||
t.Fatalf("stderr = %q", stderr)
|
||||
}
|
||||
if closed.ExitStatus == nil || *closed.ExitStatus != 3 {
|
||||
t.Fatalf("closed = %+v", closed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebSocketTunnelAndSharing(t *testing.T) {
|
||||
defer resetPool()
|
||||
upgrader := websocket.Upgrader{}
|
||||
var serverConns sync.WaitGroup
|
||||
wsServer := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
serverConns.Add(1)
|
||||
defer serverConns.Done()
|
||||
defer conn.Close()
|
||||
for {
|
||||
kind, data, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if string(data) == "quit" {
|
||||
conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(4000, "asked to quit"))
|
||||
return
|
||||
}
|
||||
conn.WriteMessage(kind, append([]byte("echo:"), data...))
|
||||
}
|
||||
})}
|
||||
wsListener, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
go wsServer.Serve(wsListener)
|
||||
defer wsServer.Close()
|
||||
|
||||
server := newTestServer(t, serverOptions{password: "pw"})
|
||||
wsSpec := specFor(server, func(s *openSpec) {
|
||||
s.Password = "pw"
|
||||
s.Channel = channelSpec{Type: "websocket", URL: "ws://" + wsListener.Addr().String() + "/"}
|
||||
})
|
||||
recA, recB := newRecorder(), newRecorder()
|
||||
a := Open(wsSpec, nil, recA)
|
||||
b := Open(wsSpec, nil, recB)
|
||||
// Sent before open: queued until the channel is up.
|
||||
a.Send(`{"type":"init"}`)
|
||||
recA.expect(t, "open")
|
||||
recB.expect(t, "open")
|
||||
if e := recA.expect(t, "text"); e.payload != `echo:{"type":"init"}` {
|
||||
t.Fatalf("got %q", e.payload)
|
||||
}
|
||||
b.Send("quit")
|
||||
e := recB.expect(t, "closed")
|
||||
var closed closedPayload
|
||||
json.Unmarshal([]byte(e.payload), &closed)
|
||||
if closed.Code == nil || *closed.Code != 4000 || closed.Reason != "asked to quit" {
|
||||
t.Fatalf("closed = %+v", closed)
|
||||
}
|
||||
if n := server.accepts.Load(); n != 1 {
|
||||
t.Fatalf("expected both channels on one SSH connection, got %d connections", n)
|
||||
}
|
||||
|
||||
// Close flushes queued messages before tearing down.
|
||||
a.Send("last words")
|
||||
a.Close()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
select {
|
||||
case e := <-recA.events:
|
||||
t.Fatalf("event after Close: %s(%s)", e.kind, e.payload)
|
||||
default:
|
||||
}
|
||||
|
||||
// Nothing listening at the forwarded address: a channel failure, not a
|
||||
// network one (the SSH connection is fine).
|
||||
recC := newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) {
|
||||
s.Password = "pw"
|
||||
s.Channel = channelSpec{Type: "websocket", URL: "ws://127.0.0.1:1/"}
|
||||
}), nil, recC)
|
||||
if f := recC.expectFailure(t); f.Kind != "channel" {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionLoss(t *testing.T) {
|
||||
defer resetPool()
|
||||
server := newTestServer(t, serverOptions{password: "pw"})
|
||||
rec := newRecorder()
|
||||
Open(specFor(server, func(s *openSpec) { s.Password = "pw" }), nil, rec)
|
||||
rec.expect(t, "open")
|
||||
// Kill the SSH connection under the channel.
|
||||
poolMu.Lock()
|
||||
for _, conn := range pool {
|
||||
conn.client.Close()
|
||||
}
|
||||
poolMu.Unlock()
|
||||
if f := rec.expectFailure(t); f.Kind != "network" {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
// The next channel dials a fresh connection.
|
||||
rec = newRecorder()
|
||||
ch := Open(specFor(server, func(s *openSpec) { s.Password = "pw" }), nil, rec)
|
||||
rec.expect(t, "open")
|
||||
ch.Close()
|
||||
if n := server.accepts.Load(); n != 2 {
|
||||
t.Fatalf("expected a second connection, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkFailure(t *testing.T) {
|
||||
rec := newRecorder()
|
||||
Open(`{"host":"127.0.0.1","port":1,"user":"jim","channel":{"type":"exec","command":"x"}}`, nil, rec)
|
||||
if f := rec.expectFailure(t); f.Kind != "network" {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
rec = newRecorder()
|
||||
Open(`not json`, nil, rec)
|
||||
if f := rec.expectFailure(t); f.Kind != "config" {
|
||||
t.Fatalf("unexpected failure %+v", f)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
module github.com/jimrandomh/faceclaw/native/ssh
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
golang.org/x/crypto v0.59.0
|
||||
)
|
||||
|
||||
require (
|
||||
golang.org/x/mobile v0.0.0-20261011034528-197770444867 // indirect
|
||||
golang.org/x/mod v0.42.0 // indirect
|
||||
golang.org/x/sync v0.24.0 // indirect
|
||||
golang.org/x/sys v0.50.0 // indirect
|
||||
golang.org/x/tools v0.52.0 // indirect
|
||||
)
|
||||
|
||||
tool (
|
||||
golang.org/x/mobile/cmd/gobind
|
||||
golang.org/x/mobile/cmd/gomobile
|
||||
)
|
||||
@@ -0,0 +1,16 @@
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c=
|
||||
golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U=
|
||||
golang.org/x/mobile v0.0.0-20261011034528-197770444867 h1:rav73cdd7OQUPgt1aIFbZQTiPNnjfueKXvfOiLCY0Ss=
|
||||
golang.org/x/mobile v0.0.0-20261011034528-197770444867/go.mod h1:HR/iBzuQmVTl/us733QPwrqHp43a8m9ZLgyKXD3ST6U=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.48.0 h1:G6UU1eiSB8nl3DzeUGt7Mg+2SctI8tPS6Z5/mBt6cOk=
|
||||
golang.org/x/term v0.48.0/go.mod h1:BRDLHCZr2SUjIeBHHpRyQeC9xzSAVbzY2rvKgafnHf4=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||
@@ -0,0 +1,58 @@
|
||||
package faceclawssh
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOpenSSHInterop checks the parts of the handshake that need no
|
||||
// credentials against a real sshd: key exchange, host key reporting and
|
||||
// pinning, and how an authentication refusal is classified. Opt-in:
|
||||
//
|
||||
// FACECLAW_SSH_INTEROP_ADDR=127.0.0.1:22 go test -run Interop -v
|
||||
//
|
||||
// Compare the logged fingerprint with `ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub`.
|
||||
// It makes one failed login attempt (no password, an unknown key) as
|
||||
// FACECLAW_SSH_INTEROP_USER (default: $USER).
|
||||
func TestOpenSSHInterop(t *testing.T) {
|
||||
addr := os.Getenv("FACECLAW_SSH_INTEROP_ADDR")
|
||||
if addr == "" {
|
||||
t.Skip("FACECLAW_SSH_INTEROP_ADDR not set")
|
||||
}
|
||||
defer resetPool()
|
||||
user := os.Getenv("FACECLAW_SSH_INTEROP_USER")
|
||||
if user == "" {
|
||||
user = os.Getenv("USER")
|
||||
}
|
||||
host, portText, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port, _ := strconv.Atoi(portText)
|
||||
spec := func(hostKeys []string) string {
|
||||
encoded, _ := json.Marshal(openSpec{
|
||||
Host: host, Port: port, User: user, HostKeys: hostKeys, UseKey: true,
|
||||
Channel: channelSpec{Type: "exec", Command: "true"},
|
||||
})
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
rec := newRecorder()
|
||||
Open(spec(nil), newSoftwareSigner(t), rec)
|
||||
f := rec.expectFailure(t)
|
||||
if f.Kind != "hostkey-unknown" || f.HostKey == nil {
|
||||
t.Fatalf("first contact: %+v", f)
|
||||
}
|
||||
t.Logf("host key: %s %s", f.HostKey.Type, f.HostKey.Fingerprint)
|
||||
|
||||
rec = newRecorder()
|
||||
Open(spec([]string{f.HostKey.Key}), newSoftwareSigner(t), rec)
|
||||
auth := rec.expectFailure(t)
|
||||
if auth.Kind != "auth" || !auth.KeyTried {
|
||||
t.Fatalf("pinned, unknown key: %+v", auth)
|
||||
}
|
||||
t.Logf("auth refusal: %+v", auth)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package faceclawssh
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/asn1"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// platformSigner adapts the platform's keystore-backed Signer to ssh.Signer.
|
||||
// The platform signs SHA-256(data) with ECDSA P-256 and returns ASN.1 DER;
|
||||
// SSH wants the signature as two mpints (RFC 5656 §3.1.2).
|
||||
type platformSigner struct {
|
||||
signer Signer
|
||||
pub ssh.PublicKey
|
||||
}
|
||||
|
||||
func newPlatformSigner(signer Signer) (*platformSigner, error) {
|
||||
pub, err := sshPublicKey(signer.PublicKey())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &platformSigner{signer: signer, pub: pub}, nil
|
||||
}
|
||||
|
||||
func (p *platformSigner) PublicKey() ssh.PublicKey { return p.pub }
|
||||
|
||||
func (p *platformSigner) Sign(_ io.Reader, data []byte) (*ssh.Signature, error) {
|
||||
der, err := p.signer.Sign(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("platform key signing failed: %w", err)
|
||||
}
|
||||
var sig struct{ R, S *big.Int }
|
||||
rest, err := asn1.Unmarshal(der, &sig)
|
||||
if err != nil || len(rest) != 0 || sig.R == nil || sig.S == nil || sig.R.Sign() <= 0 || sig.S.Sign() <= 0 {
|
||||
return nil, errors.New("platform key returned a malformed signature")
|
||||
}
|
||||
blob := ssh.Marshal(struct{ R, S *big.Int }{sig.R, sig.S})
|
||||
return &ssh.Signature{Format: p.pub.Type(), Blob: blob}, nil
|
||||
}
|
||||
|
||||
// sshPublicKey parses a platform public key (SubjectPublicKeyInfo DER or an
|
||||
// uncompressed P-256 point) into an ecdsa-sha2-nistp256 SSH key.
|
||||
func sshPublicKey(encoded []byte) (ssh.PublicKey, error) {
|
||||
var pub *ecdsa.PublicKey
|
||||
if len(encoded) == 65 && encoded[0] == 4 {
|
||||
parsed, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), encoded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad P-256 point: %v", err)
|
||||
}
|
||||
pub = parsed
|
||||
} else {
|
||||
parsed, err := x509.ParsePKIXPublicKey(encoded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad public key: %v", err)
|
||||
}
|
||||
ecdsaKey, ok := parsed.(*ecdsa.PublicKey)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported public key type %T", parsed)
|
||||
}
|
||||
pub = ecdsaKey
|
||||
}
|
||||
if pub.Curve != elliptic.P256() {
|
||||
return nil, errors.New("only P-256 keys are supported")
|
||||
}
|
||||
return ssh.NewPublicKey(pub)
|
||||
}
|
||||
|
||||
func authorizedKeyLine(key ssh.PublicKey, comment string) string {
|
||||
line := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(key)))
|
||||
comment = strings.Join(strings.Fields(comment), "-")
|
||||
if comment != "" {
|
||||
line += " " + comment
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// fingerprint is OpenSSH's SHA256 fingerprint format (unpadded base64).
|
||||
func fingerprint(key ssh.PublicKey) string {
|
||||
sum := sha256.Sum256(key.Marshal())
|
||||
return "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
package faceclawssh
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// SSH connections are shared between channels with the same host, port,
|
||||
// user, credentials and pinned keys: the Terminal app's session poller and
|
||||
// each of its view windows ride one TCP connection. A connection with no
|
||||
// channels left lingers briefly (a view window closing and reopening shouldn't
|
||||
// pay a fresh handshake) and is then closed. A keepalive probe closes
|
||||
// connections whose peer has silently gone away, which fails every channel on
|
||||
// them so the app's reconnect logic takes over.
|
||||
|
||||
const idleLinger = 15 * time.Second
|
||||
|
||||
type pooledConn struct {
|
||||
key string
|
||||
client *ssh.Client
|
||||
|
||||
// Guarded by poolMu.
|
||||
refs int
|
||||
dead bool
|
||||
linger *time.Timer
|
||||
}
|
||||
|
||||
type dialCall struct {
|
||||
done chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
var (
|
||||
poolMu sync.Mutex
|
||||
pool = map[string]*pooledConn{}
|
||||
dialing = map[string]*dialCall{}
|
||||
)
|
||||
|
||||
func poolKey(spec *openSpec, signer *platformSigner) string {
|
||||
h := sha256.New()
|
||||
fmt.Fprintf(h, "%s\x00%d\x00%s\x00%s\x00", spec.Host, spec.Port, spec.User, spec.Password)
|
||||
for _, key := range spec.HostKeys {
|
||||
fmt.Fprintf(h, "%s\x00", key)
|
||||
}
|
||||
if signer != nil {
|
||||
h.Write(signer.pub.Marshal())
|
||||
}
|
||||
fmt.Fprintf(h, "\x00%d", spec.KeepaliveIntervalMs)
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// acquire returns a live pooled connection for spec, dialing one if needed
|
||||
// (concurrent callers for the same key share a single dial). The caller must
|
||||
// release() it.
|
||||
func acquire(ctx context.Context, spec *openSpec, signer Signer) (*pooledConn, error) {
|
||||
var ps *platformSigner
|
||||
if signer != nil {
|
||||
var err error
|
||||
if ps, err = newPlatformSigner(signer); err != nil {
|
||||
return nil, &failure{Kind: "config", Message: "client key unusable: " + err.Error()}
|
||||
}
|
||||
}
|
||||
key := poolKey(spec, ps)
|
||||
for {
|
||||
poolMu.Lock()
|
||||
if conn := pool[key]; conn != nil && !conn.dead {
|
||||
conn.refs++
|
||||
if conn.linger != nil {
|
||||
conn.linger.Stop()
|
||||
conn.linger = nil
|
||||
}
|
||||
poolMu.Unlock()
|
||||
return conn, nil
|
||||
}
|
||||
if call := dialing[key]; call != nil {
|
||||
poolMu.Unlock()
|
||||
select {
|
||||
case <-call.done:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if call.err != nil {
|
||||
return nil, call.err
|
||||
}
|
||||
continue
|
||||
}
|
||||
call := &dialCall{done: make(chan struct{})}
|
||||
dialing[key] = call
|
||||
poolMu.Unlock()
|
||||
|
||||
conn, err := dial(spec, ps, key)
|
||||
poolMu.Lock()
|
||||
delete(dialing, key)
|
||||
if err == nil {
|
||||
pool[key] = conn
|
||||
}
|
||||
call.err = err
|
||||
close(call.done)
|
||||
poolMu.Unlock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Loop around to take a reference like any other caller.
|
||||
}
|
||||
}
|
||||
|
||||
func (conn *pooledConn) release() {
|
||||
poolMu.Lock()
|
||||
defer poolMu.Unlock()
|
||||
conn.refs--
|
||||
if conn.refs > 0 || conn.dead {
|
||||
return
|
||||
}
|
||||
conn.linger = time.AfterFunc(idleLinger, func() {
|
||||
poolMu.Lock()
|
||||
idle := conn.refs == 0 && !conn.dead
|
||||
if idle {
|
||||
conn.dead = true
|
||||
if pool[conn.key] == conn {
|
||||
delete(pool, conn.key)
|
||||
}
|
||||
}
|
||||
poolMu.Unlock()
|
||||
if idle {
|
||||
conn.client.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// markDead removes a connection from the pool once its transport has ended,
|
||||
// so the next acquire dials afresh.
|
||||
func (conn *pooledConn) markDead() {
|
||||
poolMu.Lock()
|
||||
defer poolMu.Unlock()
|
||||
conn.dead = true
|
||||
if conn.linger != nil {
|
||||
conn.linger.Stop()
|
||||
conn.linger = nil
|
||||
}
|
||||
if pool[conn.key] == conn {
|
||||
delete(pool, conn.key)
|
||||
}
|
||||
}
|
||||
|
||||
// hostKeyChecker pins the host key: only keys listed in the spec are
|
||||
// accepted, and whatever the host presented is kept for the failure report.
|
||||
type hostKeyChecker struct {
|
||||
known [][]byte
|
||||
presented ssh.PublicKey
|
||||
rejected bool
|
||||
}
|
||||
|
||||
func newHostKeyChecker(encoded []string) (*hostKeyChecker, error) {
|
||||
checker := &hostKeyChecker{}
|
||||
for _, entry := range encoded {
|
||||
wire, err := base64.StdEncoding.DecodeString(strings.TrimSpace(entry))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad pinned host key: %v", err)
|
||||
}
|
||||
if _, err := ssh.ParsePublicKey(wire); err != nil {
|
||||
return nil, fmt.Errorf("bad pinned host key: %v", err)
|
||||
}
|
||||
checker.known = append(checker.known, wire)
|
||||
}
|
||||
return checker, nil
|
||||
}
|
||||
|
||||
func (h *hostKeyChecker) check(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||
h.presented = key
|
||||
wire := key.Marshal()
|
||||
for _, known := range h.known {
|
||||
if bytes.Equal(known, wire) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
h.rejected = true
|
||||
return errors.New("host key not trusted")
|
||||
}
|
||||
|
||||
// algorithms lists host key algorithms with the pinned keys' types first, so
|
||||
// a host with several host keys presents the one we pinned rather than
|
||||
// whichever the default preference order favors.
|
||||
func (h *hostKeyChecker) algorithms() []string {
|
||||
var preferred []string
|
||||
seen := map[string]bool{}
|
||||
add := func(algorithm string) {
|
||||
if !seen[algorithm] {
|
||||
seen[algorithm] = true
|
||||
preferred = append(preferred, algorithm)
|
||||
}
|
||||
}
|
||||
for _, wire := range h.known {
|
||||
key, err := ssh.ParsePublicKey(wire)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if key.Type() == ssh.KeyAlgoRSA {
|
||||
add(ssh.KeyAlgoRSASHA512)
|
||||
add(ssh.KeyAlgoRSASHA256)
|
||||
continue
|
||||
}
|
||||
add(key.Type())
|
||||
}
|
||||
if len(preferred) == 0 {
|
||||
return nil // the library default
|
||||
}
|
||||
for _, algorithm := range ssh.SupportedAlgorithms().HostKeys {
|
||||
add(algorithm)
|
||||
}
|
||||
return preferred
|
||||
}
|
||||
|
||||
func (h *hostKeyChecker) failure() *failure {
|
||||
if h.presented == nil {
|
||||
return nil
|
||||
}
|
||||
info := &hostKeyInfo{
|
||||
Type: h.presented.Type(),
|
||||
Fingerprint: fingerprint(h.presented),
|
||||
Key: base64.StdEncoding.EncodeToString(h.presented.Marshal()),
|
||||
}
|
||||
if len(h.known) == 0 {
|
||||
return &failure{Kind: "hostkey-unknown", Message: "Unknown host key " + info.Fingerprint, HostKey: info}
|
||||
}
|
||||
return &failure{Kind: "hostkey-mismatch", Message: "Host key changed: now " + info.Fingerprint, HostKey: info}
|
||||
}
|
||||
|
||||
// errNoPassword aborts authentication when the host would accept a password
|
||||
// but we don't have one; the "auth" failure then asks the app to prompt.
|
||||
var errNoPassword = errors.New("password required")
|
||||
|
||||
type authState struct {
|
||||
mu sync.Mutex
|
||||
passwordAllowed bool
|
||||
passwordTried bool
|
||||
keyTried bool
|
||||
}
|
||||
|
||||
func (a *authState) set(update func(*authState)) {
|
||||
a.mu.Lock()
|
||||
update(a)
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func authMethods(spec *openSpec, signer *platformSigner, state *authState) []ssh.AuthMethod {
|
||||
var methods []ssh.AuthMethod
|
||||
if signer != nil {
|
||||
methods = append(methods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
|
||||
state.set(func(a *authState) { a.keyTried = true })
|
||||
return []ssh.Signer{signer}, nil
|
||||
}))
|
||||
}
|
||||
password := spec.Password
|
||||
methods = append(methods,
|
||||
ssh.PasswordCallback(func() (string, error) {
|
||||
state.set(func(a *authState) { a.passwordAllowed = true })
|
||||
if password == "" {
|
||||
return "", errNoPassword
|
||||
}
|
||||
state.set(func(a *authState) { a.passwordTried = true })
|
||||
return password, nil
|
||||
}),
|
||||
// Many hosts (PAM-backed sshd, e.g. macOS) take passwords through
|
||||
// keyboard-interactive. Answer hidden prompts with the password and
|
||||
// visible ones with nothing.
|
||||
ssh.KeyboardInteractive(func(_, _ string, questions []string, echos []bool) ([]string, error) {
|
||||
if len(questions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
state.set(func(a *authState) { a.passwordAllowed = true })
|
||||
if password == "" {
|
||||
return nil, errNoPassword
|
||||
}
|
||||
state.set(func(a *authState) { a.passwordTried = true })
|
||||
answers := make([]string, len(questions))
|
||||
for i := range questions {
|
||||
if !echos[i] {
|
||||
answers[i] = password
|
||||
}
|
||||
}
|
||||
return answers, nil
|
||||
}),
|
||||
)
|
||||
return methods
|
||||
}
|
||||
|
||||
func dial(spec *openSpec, signer *platformSigner, key string) (*pooledConn, error) {
|
||||
timeout := defaultConnectTimeout
|
||||
if spec.ConnectTimeoutMs > 0 {
|
||||
timeout = time.Duration(spec.ConnectTimeoutMs) * time.Millisecond
|
||||
}
|
||||
checker, err := newHostKeyChecker(spec.HostKeys)
|
||||
if err != nil {
|
||||
return nil, &failure{Kind: "config", Message: err.Error()}
|
||||
}
|
||||
addr := net.JoinHostPort(spec.Host, strconv.Itoa(spec.Port))
|
||||
dialer := net.Dialer{Timeout: timeout, KeepAlive: 30 * time.Second}
|
||||
tcp, err := dialer.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, &failure{Kind: "network", Message: describeNetError(err)}
|
||||
}
|
||||
// Bound the whole handshake, not just the TCP connect.
|
||||
tcp.SetDeadline(time.Now().Add(timeout))
|
||||
state := &authState{}
|
||||
config := &ssh.ClientConfig{
|
||||
User: spec.User,
|
||||
Auth: authMethods(spec, signer, state),
|
||||
HostKeyCallback: checker.check,
|
||||
HostKeyAlgorithms: checker.algorithms(),
|
||||
ClientVersion: "SSH-2.0-Faceclaw",
|
||||
Timeout: timeout,
|
||||
}
|
||||
sshConn, chans, reqs, err := ssh.NewClientConn(tcp, addr, config)
|
||||
if err != nil {
|
||||
tcp.Close()
|
||||
if checker.rejected {
|
||||
return nil, checker.failure()
|
||||
}
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
if errors.Is(err, errNoPassword) || strings.Contains(err.Error(), "unable to authenticate") {
|
||||
message := "The host refused this device's key."
|
||||
switch {
|
||||
case state.passwordTried:
|
||||
message = "The host refused the password."
|
||||
case state.passwordAllowed:
|
||||
message = "The host wants a password."
|
||||
case signer == nil:
|
||||
message = "The host accepts neither passwords nor keys we have."
|
||||
}
|
||||
return nil, &failure{
|
||||
Kind: "auth",
|
||||
Message: message,
|
||||
PasswordAllowed: state.passwordAllowed,
|
||||
PasswordTried: state.passwordTried,
|
||||
KeyTried: state.keyTried,
|
||||
}
|
||||
}
|
||||
return nil, &failure{Kind: "network", Message: "SSH handshake failed: " + describeNetError(err)}
|
||||
}
|
||||
tcp.SetDeadline(time.Time{})
|
||||
client := ssh.NewClient(sshConn, chans, reqs)
|
||||
conn := &pooledConn{key: key, client: client}
|
||||
log.Printf("faceclawssh: connected to %s@%s", spec.User, addr)
|
||||
go func() {
|
||||
err := client.Wait()
|
||||
log.Printf("faceclawssh: connection to %s ended: %v", addr, err)
|
||||
conn.markDead()
|
||||
}()
|
||||
interval := defaultKeepaliveInterval
|
||||
if spec.KeepaliveIntervalMs > 0 {
|
||||
interval = time.Duration(spec.KeepaliveIntervalMs) * time.Millisecond
|
||||
}
|
||||
go keepalive(conn, interval)
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// keepalive probes the host every interval and closes the connection if a
|
||||
// probe goes unanswered for a whole interval, so a dead peer (phone switched
|
||||
// networks, laptop asleep) fails the channels promptly instead of hanging.
|
||||
func keepalive(conn *pooledConn, interval time.Duration) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("faceclawssh: keepalive panic: %v", r)
|
||||
conn.client.Close()
|
||||
}
|
||||
}()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
poolMu.Lock()
|
||||
dead := conn.dead
|
||||
poolMu.Unlock()
|
||||
if dead {
|
||||
return
|
||||
}
|
||||
reply := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := conn.client.SendRequest("keepalive@openssh.com", true, nil)
|
||||
reply <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-reply:
|
||||
if err != nil {
|
||||
conn.client.Close()
|
||||
return
|
||||
}
|
||||
case <-time.After(interval):
|
||||
log.Printf("faceclawssh: keepalive timed out; closing")
|
||||
conn.client.Close()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func describeNetError(err error) string {
|
||||
message := err.Error()
|
||||
// "dial tcp 1.2.3.4:22: connect: connection refused" → keep the tail.
|
||||
if index := strings.LastIndex(message, ": "); index >= 0 && strings.HasPrefix(message, "dial ") {
|
||||
return message[index+2:]
|
||||
}
|
||||
return message
|
||||
}
|
||||
Reference in new issue
Block a user