Multi-mic support and replay mode for Transcribe app

This commit is contained in:
James Babcock committed 2026-11-12 21:54:57 -08:00
1 parent c162c8e633
commit 9372b2742c
45 files changed
+3926 -288

No files matched your search

+2 -1
View File
@@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
override var size = 0L
private set
override fun write(bytes: ByteArray, offset: Int, length: Int) {
file.write(bytes, offset, length)
size += length
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
file.seek(position)
file.write(bytes)
file.seek(size)
}
override fun close() = file.close()
}
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
private val file = java.io.RandomAccessFile(path, "r")
override val length = file.length()
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
file.seek(position)
var total = 0
while (total < length) {
val read = file.read(buffer, offset + total, length - total)
if (read <= 0) break
total += read
}
return total
}
override fun close() = file.close()
}
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
expect fun deleteFile(path: String): Boolean
expect fun renameFile(from: String, to: String): Boolean
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
interface SeekableOutput {
/** Bytes appended so far. */
val size: Long
/** Appends [length] bytes of [bytes] from [offset]. */
fun write(bytes: ByteArray, offset: Int, length: Int)
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
fun writeAt(position: Long, bytes: ByteArray)
fun close()
}
/** A file read at arbitrary positions without loading it whole. */
interface SeekableInput {
val length: Long
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
fun close()
}
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
expect fun openFileOutput(path: String): SeekableOutput
/** Opens [path] for reading. Throws when it cannot. */
expect fun openFileInput(path: String): SeekableInput
@@ -0,0 +1,764 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/**
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
*
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
* about a second each, cues and the segment size written when the recording finishes. A
* recording cut short by a crash is still a valid live-style stream of everything up to its last
* complete cluster, and the reader takes it as such.
*/
internal object MatroskaIds {
const val EBML = 0x1A45DFA3
const val EBML_VERSION = 0x4286
const val EBML_READ_VERSION = 0x42F7
const val EBML_MAX_ID_LENGTH = 0x42F2
const val EBML_MAX_SIZE_LENGTH = 0x42F3
const val DOC_TYPE = 0x4282
const val DOC_TYPE_VERSION = 0x4287
const val DOC_TYPE_READ_VERSION = 0x4285
const val VOID = 0xEC
const val SEGMENT = 0x18538067
const val SEEK_HEAD = 0x114D9B74
const val SEEK = 0x4DBB
const val SEEK_ID = 0x53AB
const val SEEK_POSITION = 0x53AC
const val INFO = 0x1549A966
const val TIMESTAMP_SCALE = 0x2AD7B1
const val DURATION = 0x4489
const val DATE_UTC = 0x4461
const val TITLE = 0x7BA9
const val MUXING_APP = 0x4D80
const val WRITING_APP = 0x5741
const val TRACKS = 0x1654AE6B
const val TRACK_ENTRY = 0xAE
const val TRACK_NUMBER = 0xD7
const val TRACK_UID = 0x73C5
const val TRACK_TYPE = 0x83
const val FLAG_DEFAULT = 0x88
const val FLAG_LACING = 0x9C
const val NAME = 0x536E
const val LANGUAGE = 0x22B59C
const val CODEC_ID = 0x86
const val AUDIO = 0xE1
const val SAMPLING_FREQUENCY = 0xB5
const val CHANNELS = 0x9F
const val BIT_DEPTH = 0x6264
const val ATTACHMENTS = 0x1941A469
const val ATTACHED_FILE = 0x61A7
const val FILE_NAME = 0x466E
const val FILE_MIME_TYPE = 0x4660
const val FILE_DATA = 0x465C
const val FILE_UID = 0x46AE
const val CLUSTER = 0x1F43B675
const val TIMESTAMP = 0xE7
const val SIMPLE_BLOCK = 0xA3
const val BLOCK_GROUP = 0xA0
const val BLOCK = 0xA1
const val BLOCK_DURATION = 0x9B
const val CUES = 0x1C53BB6B
const val CUE_POINT = 0xBB
const val CUE_TIME = 0xB3
const val CUE_TRACK_POSITIONS = 0xB7
const val CUE_TRACK = 0xF7
const val CUE_CLUSTER_POSITION = 0xF1
}
/** Builds EBML elements into a byte buffer. */
internal class EbmlWriter(capacity: Int = 256) {
private val sink = ByteSink(capacity)
fun size(): Int = sink.size()
fun bytes(): ByteArray = sink.toByteArray()
/** An element ID, written as given: its length marker is part of the value. */
fun id(id: Int) {
if (id > 0xFFFFFF) sink.write(id ushr 24)
if (id > 0xFFFF) sink.write(id ushr 16)
if (id > 0xFF) sink.write(id ushr 8)
sink.write(id)
}
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
fun dataSize(size: Long, width: Int = 0) {
var bytes = width
if (bytes == 0) {
bytes = 1
// The all-ones value of each width means "unknown size".
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
}
val marked = size or (1L shl (7 * bytes))
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
}
fun uint(id: Int, value: Long) {
var bytes = 1
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
id(id)
dataSize(bytes.toLong())
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
/** A fixed eight-byte integer (dates). */
fun int64(id: Int, value: Long) {
id(id)
dataSize(8)
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
fun string(id: Int, text: String) {
val bytes = text.encodeToByteArray()
binary(id, bytes, 0, bytes.size)
}
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
id(id)
dataSize(length.toLong())
sink.write(bytes, offset, length)
}
fun raw(bytes: ByteArray) = sink.write(bytes)
fun raw(value: Int) = sink.write(value)
fun master(id: Int, build: EbmlWriter.() -> Unit) {
val child = EbmlWriter()
child.build()
id(id)
dataSize(child.size().toLong())
sink.write(child.bytes())
}
companion object {
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
fun padded(element: ByteArray, reserved: Int): ByteArray? {
val filler = reserved - element.size
// A Void element is at least two bytes.
if (filler < 0 || filler == 1) return null
val out = element.copyOf(reserved)
if (filler == 0) return out
val at = element.size
out[at] = MatroskaIds.VOID.toByte()
if (filler < 9) {
out[at + 1] = (0x80 or (filler - 2)).toByte()
} else {
val size = (filler - 9).toLong()
out[at + 1] = 0x01
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
}
return out
}
}
}
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
/**
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
* milliseconds from the start of the file. Not thread-safe.
*/
class MkaWriter(
private val output: SeekableOutput,
title: String?,
/** When the recording began (Unix epoch ms), for the file's date. */
dateUtcMs: Long,
attachments: List<MkaAttachment>,
) {
companion object {
const val CODEC_PCM = "A_PCM/INT/LIT"
const val CODEC_TEXT = "S_TEXT/UTF8"
const val TRACK_AUDIO = 2
const val TRACK_SUBTITLE = 0x11
const val APP_NAME = "Faceclaw"
private const val SEEK_RESERVED = 128
private const val TRACKS_RESERVED = 4096
private const val CLUSTER_SPAN_MS = 1000L
private const val CLUSTER_MAX_BYTES = 1 shl 20
// Block timestamps are signed 16-bit offsets from their cluster's.
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
private const val DURATION_PATCH_CLUSTERS = 5
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
private const val DATE_EPOCH_MS = 978_307_200_000L
}
private class Track(
val number: Int,
val type: Int,
val name: String,
val sampleRate: Int,
val channels: Int,
val preferred: Boolean,
)
private class Cue(val clusterPosition: Long, val timeMs: Long)
private val tracks = ArrayList<Track>()
private val cues = ArrayList<Cue>()
private val segmentSizePosition: Long
private val segmentStart: Long
private val seekPosition: Long
private val infoPosition: Long
private val durationPosition: Long
private val tracksPosition: Long
private val attachmentsPosition: Long
private var cluster: EbmlWriter? = null
private var clusterBaseMs = 0L
private var clustersSincePatch = 0
private var finished = false
private val hasAttachments = attachments.isNotEmpty()
/** The end of the latest audio or text written, in ms from the start of the file. */
var durationMs = 0L
private set
init {
val head = EbmlWriter()
head.master(MatroskaIds.EBML) {
uint(MatroskaIds.EBML_VERSION, 1)
uint(MatroskaIds.EBML_READ_VERSION, 1)
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
string(MatroskaIds.DOC_TYPE, "matroska")
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
}
head.id(MatroskaIds.SEGMENT)
segmentSizePosition = head.size().toLong()
// Unknown size (a live stream) until finish() knows it.
head.raw(0x01)
repeat(7) { head.raw(0xFF) }
segmentStart = head.size().toLong()
append(head.bytes())
seekPosition = output.size
infoPosition = seekPosition + SEEK_RESERVED
val info = EbmlWriter()
info.master(MatroskaIds.INFO) {
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
string(MatroskaIds.MUXING_APP, APP_NAME)
string(MatroskaIds.WRITING_APP, APP_NAME)
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
// Last, so its eight bytes are the element's final ones.
float64(MatroskaIds.DURATION, 0.0)
}
durationPosition = infoPosition + info.size() - 8
tracksPosition = infoPosition + info.size()
attachmentsPosition = tracksPosition + TRACKS_RESERVED
append(seekHead(-1))
append(info.bytes())
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
if (attachments.isNotEmpty()) {
val files = EbmlWriter()
files.master(MatroskaIds.ATTACHMENTS) {
attachments.forEachIndexed { index, file ->
master(MatroskaIds.ATTACHED_FILE) {
string(MatroskaIds.FILE_NAME, file.name)
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
uint(MatroskaIds.FILE_UID, index + 1L)
}
}
}
append(files.bytes())
}
}
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
private fun seekHead(cuesPosition: Long): ByteArray {
val head = EbmlWriter()
head.master(MatroskaIds.SEEK_HEAD) {
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
val idBytes = EbmlWriter()
idBytes.id(id)
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
}
entry(MatroskaIds.INFO, infoPosition)
entry(MatroskaIds.TRACKS, tracksPosition)
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
}
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
}
/**
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
* the one players pick by default.
*/
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
private fun addTrack(track: Track): Int {
check(!finished) { "recording already finished" }
// A block's track number is written in one byte.
if (track.number > 126) return -1
tracks.add(track)
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
if (region == null) {
tracks.remove(track)
return -1
}
output.writeAt(tracksPosition, region)
return track.number
}
private fun listedTracks(): List<Track> =
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
tracks.filter { it.type != TRACK_AUDIO }
private fun tracksElement(): ByteArray {
val listed = listedTracks()
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
val out = EbmlWriter(1024)
out.master(MatroskaIds.TRACKS) {
for (track in listed) {
master(MatroskaIds.TRACK_ENTRY) {
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
uint(MatroskaIds.TRACK_UID, track.number.toLong())
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
uint(MatroskaIds.FLAG_LACING, 0)
string(MatroskaIds.NAME, track.name)
string(MatroskaIds.LANGUAGE, "und")
if (track.type == TRACK_AUDIO) {
string(MatroskaIds.CODEC_ID, CODEC_PCM)
master(MatroskaIds.AUDIO) {
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
uint(MatroskaIds.CHANNELS, track.channels.toLong())
uint(MatroskaIds.BIT_DEPTH, 16)
}
} else {
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
}
}
}
}
return out.bytes()
}
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
val track = track(trackNumber)
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
if (length <= 0) return
val block = openCluster(timeMs)
block.id(MatroskaIds.SIMPLE_BLOCK)
block.dataSize(4L + length)
blockHeader(block, trackNumber, timeMs, 0x80)
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
val frames = length / (2 * track.channels)
noteEnd(timeMs + frames * 1000L / track.sampleRate)
}
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
val bytes = text.encodeToByteArray()
val block = openCluster(timeMs)
block.master(MatroskaIds.BLOCK_GROUP) {
id(MatroskaIds.BLOCK)
dataSize(4L + bytes.size)
blockHeader(this, trackNumber, timeMs, 0)
raw(bytes)
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
}
noteEnd(timeMs + maxOf(1L, durationMs))
}
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
block.raw(0x80 or trackNumber)
block.raw(relative shr 8)
block.raw(relative)
block.raw(flags)
}
private fun noteEnd(endMs: Long) {
if (endMs > durationMs) durationMs = endMs
}
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
private fun openCluster(timeMs: Long): EbmlWriter {
check(!finished) { "recording already finished" }
val time = maxOf(0L, timeMs)
val open = cluster
if (open != null) {
val relative = time - clusterBaseMs
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
flushCluster()
}
clusterBaseMs = time
return EbmlWriter(64 * 1024).also { cluster = it }
}
private fun flushCluster() {
val body = cluster ?: return
cluster = null
if (body.size() == 0) return
val timestamp = EbmlWriter(16)
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
val header = EbmlWriter(32)
header.id(MatroskaIds.CLUSTER)
header.dataSize(timestamp.size().toLong() + body.size())
header.raw(timestamp.bytes())
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
append(header.bytes())
append(body.bytes())
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
}
private fun patchDuration() {
clustersSincePatch = 0
val value = durationMs.toDouble().toRawBits()
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
}
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
fun finish() {
if (finished) return
try {
flushCluster()
finished = true
val cuesPosition = output.size
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
if (cues.isNotEmpty()) {
val index = EbmlWriter(cues.size * 16 + 16)
index.master(MatroskaIds.CUES) {
for (cue in cues) {
master(MatroskaIds.CUE_POINT) {
uint(MatroskaIds.CUE_TIME, cue.timeMs)
master(MatroskaIds.CUE_TRACK_POSITIONS) {
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
}
}
}
}
append(index.bytes())
output.writeAt(seekPosition, seekHead(cuesPosition))
}
patchDuration()
val size = EbmlWriter(8)
size.dataSize(output.size - segmentStart, 8)
output.writeAt(segmentSizePosition, size.bytes())
} finally {
finished = true
output.close()
}
}
}
class MkaTrackInfo(
@JvmField val number: Int,
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
@JvmField val type: Int,
@JvmField val codecId: String,
@JvmField val name: String,
@JvmField val sampleRate: Int,
@JvmField val channels: Int,
@JvmField val bitDepth: Int,
) {
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
}
class MkaBlock(
@JvmField val track: Int,
@JvmField val timeMs: Long,
@JvmField val data: ByteArray,
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
@JvmField val durationMs: Long,
)
/**
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
* IllegalArgumentException for anything that is not such a file.
*/
class MkaReader(private val input: SeekableInput) {
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
private val clusters = ArrayList<ClusterRef>()
private val trackList = ArrayList<MkaTrackInfo>()
private val attachmentList = ArrayList<MkaAttachment>()
private var timestampScale = 1_000_000L
private var infoDurationMs = 0L
private var nextCluster = 0
private var blocks: List<MkaBlock> = emptyList()
private var blockIndex = 0
private var skipBeforeMs = 0L
val tracks: List<MkaTrackInfo> get() = trackList
val attachments: List<MkaAttachment> get() = attachmentList
/** The end of the last audio or text in the file, in ms. */
val durationMs: Long
init {
scan()
var end = infoDurationMs
if (clusters.isNotEmpty()) {
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
}
durationMs = end
}
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
/** Continue from the first block at or after [timeMs]. */
fun seekTo(timeMs: Long) {
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
nextCluster = maxOf(0, index)
blocks = emptyList()
blockIndex = 0
skipBeforeMs = timeMs
}
/** The next block, or null at the end of the file. */
fun next(): MkaBlock? {
while (true) {
while (blockIndex >= blocks.size) {
if (nextCluster >= clusters.size) return null
blocks = readCluster(clusters[nextCluster++])
blockIndex = 0
}
val block = blocks[blockIndex++]
if (block.timeMs >= skipBeforeMs) return block
}
}
fun close() = input.close()
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
private fun blockSpanMs(block: MkaBlock): Long {
if (block.durationMs >= 0) return block.durationMs
val track = track(block.track) ?: return 0
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
}
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
private class Header(val id: Int, val size: Long, val length: Int)
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
if (at >= end) return null
val first = bytes[at].toInt() and 0xff
val idLength = first.countLeadingZeroBits() - 24 + 1
if (idLength > 4 || at + idLength >= end) return null
var id = 0
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
val sizeAt = at + idLength
val sizeFirst = bytes[sizeAt].toInt() and 0xff
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
var allOnes = size == (0xff shr sizeLength).toLong()
for (index in 1 until sizeLength) {
val byte = bytes[sizeAt + index].toInt() and 0xff
if (byte != 0xff) allOnes = false
size = (size shl 8) or byte.toLong()
}
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
}
private fun readAt(position: Long, length: Int): ByteArray {
val buffer = ByteArray(length)
val read = input.readAt(position, buffer, 0, length)
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
}
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
var value = 0L
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
return value
}
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
8 -> Double.fromBits(uint(bytes, at, 8))
else -> 0.0
}
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
var at = start
while (true) {
val child = header(bytes, at, end) ?: return
val payload = at + child.length
if (child.size < 0 || child.size > end - payload) return
visit(child.id, payload, child.size.toInt())
at = payload + child.size.toInt()
}
}
private fun scan() {
val length = input.length
val start = readAt(0, 64)
val ebml = header(start, 0, start.size)
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
var position = ebml.length + ebml.size
val segment = readAt(position, 12).let { header(it, 0, it.size) }
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
position += segment.length
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
while (position < end) {
val head = readAt(position, 12)
val element = header(head, 0, head.size) ?: break
val payload = position + element.length
if (element.id == MatroskaIds.CLUSTER) {
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
// A recording cut short ends in a partial cluster; its complete blocks still count.
val available = minOf(element.size, end - payload)
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
} else if (element.size < 0) {
break
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
when (element.id) {
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
}
}
if (element.size < 0) break
position = payload + element.size
}
}
private fun clusterTime(position: Long, size: Int): Long {
val head = readAt(position, minOf(size, 16))
val first = header(head, 0, head.size)
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
return uint(head, first.length, first.size.toInt())
}
val whole = readAt(position, size)
var time = 0L
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
return time
}
private fun parseInfo(bytes: ByteArray) {
var duration = 0.0
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
MatroskaIds.DURATION -> duration = float(bytes, at, size)
}
}
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
}
private fun parseTracks(bytes: ByteArray) {
trackList.clear()
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.TRACK_ENTRY) {
var number = 0
var type = 0
var codec = ""
var name = ""
var rate = 8000.0
var channels = 1
var depth = 0
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
when (audio) {
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
}
}
}
}
val audio = type == MkaWriter.TRACK_AUDIO
trackList.add(
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
)
}
}
}
private fun parseAttachments(bytes: ByteArray) {
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.ATTACHED_FILE) {
var name = ""
var mime = ""
var data = ByteArray(0)
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
}
}
attachmentList.add(MkaAttachment(name, mime, data))
}
}
}
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
val bytes = readAt(cluster.position, cluster.size)
val out = ArrayList<MkaBlock>()
var clusterRaw = cluster.rawTime
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
MatroskaIds.BLOCK_GROUP -> {
var blockAt = -1
var blockSize = 0
var duration = -1L
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.BLOCK -> {
blockAt = fieldAt
blockSize = fieldSize
}
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
}
}
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
}
}
}
return out
}
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
if (size < 4) return null
val first = bytes[at].toInt() and 0xff
val trackLength = first.countLeadingZeroBits() - 24 + 1
if (trackLength > 2 || size < trackLength + 3) return null
var track = first and (0xff shr trackLength)
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
val head = at + trackLength
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
val flags = bytes[head + 2].toInt() and 0xff
if (flags and 0x06 != 0) return null
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
}
}
@@ -0,0 +1,93 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/**
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
* contract is the header of g2flash patches/mic_control.c).
*/
object MicArrayProtocol {
const val VERSION = 1
const val HEADER_BYTES = 21
const val OP_CONFIGURE = 1
const val OP_STOP = 3
const val OP_RENEW = 4
/** How long an armed temple streams without a CONFIGURE or RENEW. */
const val LEASE_MS = 90_000L
const val FLAG_TRUNCATED = 0x80
const val CODEC_RAW = 1
const val FORMAT_PCM16 = 0
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
class Frame(
/** Low 16 bits of the temple's frame counter. */
@JvmField val sequence: Int,
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
@JvmField val deviceTickMs: Long,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
@JvmField val format: Int,
@JvmField val codec: Int,
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
@JvmField val angleDegrees: Int,
@JvmField val ssr: Int,
/** The firmware cut the payload short of the samples it captured. */
@JvmField val truncated: Boolean,
@JvmField val payloadOffset: Int,
@JvmField val payloadLength: Int,
)
/** A control record with no payload: ['M','C', version, op]. */
@JvmStatic
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
@JvmStatic
fun isConfigure(record: ByteArray?): Boolean =
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
@JvmStatic
fun decodeFrame(data: ByteArray): Frame? {
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
return null
}
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
return Frame(
sequence = u16(data, 4),
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
channels = data[10].toInt() and 0xff,
sampleRate = u16(data, 11) * 100,
format = data[13].toInt() and 0xff,
codec = data[14].toInt() and 0xff,
angleDegrees = u16(data, 15).toShort().toInt(),
ssr = u16(data, 17).toShort().toInt(),
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
payloadOffset = HEADER_BYTES,
payloadLength = length,
)
}
/**
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
* payload isn't raw 16-bit PCM or holds none.
*/
@JvmStatic
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
val frames = frame.payloadLength / (2 * frame.channels)
if (frames <= 0) return null
val samples = ShortArray(frames * frame.channels)
var at = frame.payloadOffset
for (index in samples.indices) {
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
at += 2
}
return samples
}
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
}
@@ -0,0 +1,484 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
import kotlin.math.abs
/** Which microphones a chunk of capture audio came from. */
object MicStream {
/** One mixed channel: the glasses' stock stream, or the phone's own microphone. */
const val MONO = 0
/** A temple's own microphones, on that temple's clock (channel 0 front, channel 1 rear). */
const val LEFT = 1
const val RIGHT = 2
/** The mono feed derived from the temples for speech recognition: recorded, never replayed. */
const val MIX = 3
const val COUNT = 4
/** The stream's name in frame metadata. */
@JvmStatic
fun key(stream: Int): String = when (stream) {
MONO -> "mic"
LEFT -> "left"
RIGHT -> "right"
else -> "mix"
}
/** The name of the stream's audio tracks in a recording. */
@JvmStatic
fun trackName(stream: Int): String = when (stream) {
MONO -> "mic"
LEFT -> "left temple"
RIGHT -> "right temple"
else -> "mix"
}
/** The stream a recording's track or metadata names, or -1. */
@JvmStatic
fun named(name: String): Int = (0 until COUNT).firstOrNull { trackName(it) == name || key(it) == name } ?: -1
}
/**
* One chunk of microphone audio as a capture source delivered it (about 50 ms), before any
* processing. Live captures, recordings and replays all trade in these.
*/
class MicChunk(
@JvmField val stream: Int,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
/** Interleaved PCM16; the first [frames] sample frames are the chunk. */
@JvmField val pcm: ShortArray,
@JvmField val frames: Int,
/** When the chunk arrived: the phone's monotonic clock for a live capture, the offset into the file for a replay. */
@JvmField val timeMs: Long,
) {
/** Whether the source reported a direction of arrival with the chunk. */
@JvmField var hasDirection = false
@JvmField var angleDegrees = 0
@JvmField var ssr = 0
/** The source's packet counter, or -1 when it has none. */
@JvmField var sequence = -1
/** The source's own clock when it captured the chunk (unsigned 32-bit ms), or -1 when it has none. */
@JvmField var deviceTickMs = -1L
/** Whose clock [deviceTickMs] is on: the stream the samples were captured by. */
@JvmField var clockStream = stream
/** The source cut the chunk short of what it captured. */
@JvmField var truncated = false
/** Sample frames known to be missing between the previous chunk of this stream and this one. */
@JvmField var lostFramesBefore = 0
val durationMs: Long get() = frames * 1000L / sampleRate
}
/**
* Derives the mono feed speech recognition hears from the temples' chunks. This is the seam for
* array processing (beamforming, noise cancellation): replays feed it the recorded chunks with
* the times they first arrived at, so an implementation that decides from the chunks alone
* behaves the same on a replay as it did live.
*/
interface MicArrayMixer {
/** The mono chunk [chunk] contributes to the feed, or null when it contributes nothing. */
fun mix(chunk: MicChunk): MicChunk?
fun reset()
}
/**
* The feed is one temple at a time, its microphones averaged (the firmware's own stock downmix).
* The temples sample on independent clocks, so their audio is never summed. The left temple
* feeds whenever it is streaming; the right one stands in while it is not.
*/
class SingleTempleMixer : MicArrayMixer {
companion object {
/** A temple that has delivered nothing for this long no longer counts as streaming. */
const val SILENT_MS = 300L
}
private var leftAtMs = Long.MIN_VALUE
override fun reset() {
leftAtMs = Long.MIN_VALUE
}
override fun mix(chunk: MicChunk): MicChunk? {
if (chunk.stream == MicStream.LEFT) {
leftAtMs = chunk.timeMs
} else if (chunk.stream != MicStream.RIGHT || (leftAtMs != Long.MIN_VALUE && chunk.timeMs - leftAtMs < SILENT_MS)) {
return null
}
val channels = chunk.channels
val mono = ShortArray(chunk.frames)
for (frame in 0 until chunk.frames) {
var sum = 0
for (channel in 0 until channels) sum += chunk.pcm[frame * channels + channel]
mono[frame] = (sum / channels).toShort()
}
val out = MicChunk(MicStream.MIX, 1, chunk.sampleRate, mono, chunk.frames, chunk.timeMs)
out.hasDirection = chunk.hasDirection
out.angleDegrees = chunk.angleDegrees
out.ssr = chunk.ssr
out.deviceTickMs = chunk.deviceTickMs
out.clockStream = chunk.stream
out.truncated = chunk.truncated
return out
}
}
/**
* Records a capture's chunks to a Matroska audio file (.mka), one PCM16 track per stream and
* layout as each first appears, plus a "frames" text track with one JSON object per chunk that
* has metadata, written just ahead of the chunk's audio. Format: notes/mic-recording-format.md.
*
* Block timestamps are capture times, not arrival times. A stream's own clock (a temple's tick,
* or the count of samples from a source without one) says how far apart its chunks were
* captured; the least-delayed recent arrival places that clock on the file's timeline. So a
* chunk held up over Bluetooth still lands where it was captured, and lost audio leaves a gap.
*
* Safe to share between the capture thread and whoever finishes the recording.
*/
class MicRecordingWriter(
output: SeekableOutput,
/** A JSON document describing the capture, stored as the file's "capture.json" attachment. */
configJson: String?,
/** When the recording began (Unix epoch ms). */
startedAtMs: Long,
platform: ProtocolPlatform = protocolPlatform(),
private val logWarn: (String) -> Unit = { PlatformLog.w("FaceclawMicRecording", it) },
) {
companion object {
const val CONFIG_ATTACHMENT = "capture.json"
const val META_TRACK = "frames"
/** How far a chunk's arrival may sit from where its stream's clock puts it before the clock is re-placed. */
const val DISCONTINUITY_MS = 5000L
private const val WINDOW_BUCKETS = 8
private const val BUCKET_MS = 1000L
}
/**
* Places one track's chunks on the file's timeline. `source` time is the stream's own clock
* since this clock was anchored; the offset from it to arrival time is taken as the least
* seen over the last few seconds, which is the least-delayed delivery.
*/
private class TrackClock(val number: Int, val sampleRate: Int) {
private var anchored = false
private var clockStream = -1
private var tickBase = -1L
private var lastTick = -1L
private var frames = 0L
private val minima = LongArray(WINDOW_BUCKETS)
private var bucket = 0
private var bucketStartMs = 0L
var endMs = Long.MIN_VALUE
private set
fun reset() {
anchored = false
}
private fun offset(): Long = minima.min()
fun stamp(chunk: MicChunk): Long {
val arrival = chunk.timeMs
val ticked = chunk.deviceTickMs >= 0
if (anchored) {
// Another clock (the feed moved to the other temple), or this one restarted.
if (chunk.clockStream != clockStream || ticked != (tickBase >= 0)) anchored = false
else if (ticked && (chunk.deviceTickMs - lastTick).toInt() < 0) anchored = false
}
var anchorNow = !anchored
var source = 0L
if (anchored) {
frames += chunk.lostFramesBefore
source = sourceTime(chunk)
if (abs(arrival - source - offset()) > DISCONTINUITY_MS) anchorNow = true
}
if (anchorNow) {
anchored = true
clockStream = chunk.clockStream
tickBase = if (ticked) chunk.deviceTickMs else -1L
frames = 0
minima.fill(Long.MAX_VALUE)
bucket = 0
bucketStartMs = arrival
source = 0
}
if (arrival - bucketStartMs >= BUCKET_MS) {
bucket = (bucket + 1) % WINDOW_BUCKETS
minima[bucket] = Long.MAX_VALUE
bucketStartMs = arrival
}
if (arrival - source < minima[bucket]) minima[bucket] = arrival - source
// Never before the end of the track's previous block.
val time = maxOf(source + offset(), endMs)
frames += chunk.frames
lastTick = chunk.deviceTickMs
endMs = time + chunk.durationMs
return time
}
private fun sourceTime(chunk: MicChunk): Long =
if (tickBase >= 0) (chunk.deviceTickMs - tickBase) and 0xffffffffL else frames * 1000L / sampleRate
}
private val lock = platform.createLock()
private val writer: MkaWriter
private val clocks = HashMap<Long, TrackClock>()
private var metaTrack = 0
private var originMs = Long.MIN_VALUE
private var lastTimeMs = 0L
private var closed = false
/** The recording stopped early: the file could not be written. */
@Volatile
var failed = false
private set
init {
val attachments = if (configJson.isNullOrEmpty()) emptyList()
else listOf(MkaAttachment(CONFIG_ATTACHMENT, "application/json", configJson.encodeToByteArray()))
writer = MkaWriter(output, null, startedAtMs, attachments)
}
/** Length of the recording so far, in ms. */
val durationMs: Long get() = lock.withLock { writer.durationMs }
/** Appends [chunk] to its stream's track, adding the track on its first chunk. */
fun write(chunk: MicChunk) {
if (chunk.frames <= 0) return
lock.withLock {
if (closed) return
try {
val clock = clockFor(chunk) ?: return
if (originMs == Long.MIN_VALUE) originMs = chunk.timeMs
val time = clock.stamp(chunk) - originMs
val meta = metadata(chunk)
if (meta != null && metaTrack > 0) writer.writeText(metaTrack, time, maxOf(1L, chunk.durationMs), meta)
val bytes = AudioSegmentation.pcm16ToLittleEndian(chunk.pcm, 0, chunk.frames * chunk.channels)
writer.writeAudio(clock.number, time, bytes, 0, bytes.size)
lastTimeMs = maxOf(lastTimeMs, time)
} catch (error: Throwable) {
fail(error)
}
}
}
/**
* Records a JSON object as an event at the recording's current position (a capture starting
* over with another configuration, say), and has every stream's clock placed afresh by its
* next chunk: what follows an event is not continuous with what came before.
*/
fun event(json: String) {
lock.withLock {
if (closed) return
try {
for (clock in clocks.values) clock.reset()
if (ensureMetaTrack()) writer.writeText(metaTrack, maxOf(lastTimeMs, writer.durationMs), 1, json)
} catch (error: Throwable) {
fail(error)
}
}
}
/** Completes the file. Returns the recording's length in ms. */
fun finish(): Long {
lock.withLock {
if (!closed) {
closed = true
try {
writer.finish()
} catch (error: Throwable) {
failed = true
logWarn("finishing the recording failed: ${error.message}")
}
}
return writer.durationMs
}
}
private fun fail(error: Throwable) {
failed = true
closed = true
logWarn("recording stopped: ${error.message}")
try {
writer.finish()
} catch (ignored: Throwable) {
// The file is whatever made it to disk.
}
}
private fun ensureMetaTrack(): Boolean {
if (metaTrack == 0) metaTrack = writer.addTextTrack(META_TRACK)
return metaTrack > 0
}
private fun clockFor(chunk: MicChunk): TrackClock? {
val key = (chunk.stream.toLong() shl 48) or (chunk.channels.toLong() shl 32) or chunk.sampleRate.toLong()
clocks[key]?.let { return it }
// The frames track goes in with the first audio track, so it is there for that track's first chunk.
ensureMetaTrack()
// The mixed streams are what a player should pick; a temple's raw pair is for analysis.
val preferred = chunk.stream == MicStream.MONO || chunk.stream == MicStream.MIX
val number = writer.addAudioTrack(MicStream.trackName(chunk.stream), chunk.sampleRate, chunk.channels, preferred)
if (number <= 0) {
logWarn("recording has no room for another track; dropping ${MicStream.trackName(chunk.stream)} audio")
return null
}
return TrackClock(number, chunk.sampleRate).also { clocks[key] = it }
}
/** The chunk's frame-metadata JSON, or null when it has nothing to say beyond its audio. */
private fun metadata(chunk: MicChunk): String? {
if (chunk.stream == MicStream.MIX) return null
if (!chunk.hasDirection && chunk.sequence < 0 && chunk.deviceTickMs < 0 && !chunk.truncated && chunk.lostFramesBefore == 0) return null
val text = StringBuilder(96)
text.append("{\"s\":\"").append(MicStream.key(chunk.stream)).append("\",\"rx\":").append(chunk.timeMs - originMs)
if (chunk.sequence >= 0) text.append(",\"seq\":").append(chunk.sequence)
if (chunk.deviceTickMs >= 0) text.append(",\"tick\":").append(chunk.deviceTickMs)
if (chunk.hasDirection) text.append(",\"angle\":").append(chunk.angleDegrees).append(",\"ssr\":").append(chunk.ssr)
if (chunk.truncated) text.append(",\"trunc\":true")
if (chunk.lostFramesBefore > 0) text.append(",\"lost\":").append(chunk.lostFramesBefore)
return text.append('}').toString()
}
}
/**
* Reads a recording back as the chunks its capture delivered: the "mic" track as [MicStream.MONO]
* chunks and the temple tracks as [MicStream.LEFT] / [MicStream.RIGHT] ones, each with the
* metadata its frame carried. The "mix" track is skipped: a replay derives the feed again. A
* Matroska file from elsewhere plays its first PCM16 track as the mono microphone.
*
* Throws IllegalArgumentException when the file has no audio a replay can use.
*/
class MicRecordingReader(input: SeekableInput) {
companion object {
const val SAMPLE_RATE = 16000
/** Blocks longer than this many frames are handed out in pieces, like a live capture's chunks. */
const val MAX_CHUNK_FRAMES = SAMPLE_RATE / 20
}
private class Meta(val timeMs: Long, val fields: Map<String, Any?>)
private val reader = MkaReader(input)
private val streams = HashMap<Int, Int>()
private var metaTrack = -1
private val pendingMeta = arrayOfNulls<Meta>(MicStream.COUNT)
private val queued = ArrayDeque<MicChunk>()
val durationMs: Long get() = reader.durationMs
/** The capture's "capture.json" attachment, or null. */
val configJson: String? =
reader.attachments.firstOrNull { it.name == MicRecordingWriter.CONFIG_ATTACHMENT }?.data?.decodeToString()
/** What the replay plays, for a status line: "left temple 2ch + right temple 2ch". */
val description: String
init {
try {
val audio = reader.tracks.filter { it.isPcm16 && it.channels in 1..8 }
for (track in audio) {
val stream = MicStream.named(track.name)
if (stream == MicStream.MONO || stream == MicStream.LEFT || stream == MicStream.RIGHT) streams[track.number] = stream
}
if (streams.isEmpty()) {
// Not one of ours: its first usable track stands in for the microphone.
val first = audio.firstOrNull { MicStream.named(it.name) != MicStream.MIX } ?: audio.firstOrNull()
require(first != null) { "no 16-bit PCM audio track" }
streams[first.number] = MicStream.MONO
}
for (number in streams.keys) {
val rate = reader.track(number)!!.sampleRate
require(rate == SAMPLE_RATE) { "unsupported sample rate ($rate Hz; replay needs ${SAMPLE_RATE / 1000} kHz)" }
}
metaTrack = reader.tracks.firstOrNull {
it.type == MkaWriter.TRACK_SUBTITLE && it.name == MicRecordingWriter.META_TRACK
}?.number ?: -1
description = streams.entries.sortedBy { it.value }.joinToString(" + ") { (number, stream) ->
val channels = reader.track(number)!!.channels
MicStream.trackName(stream) + if (channels > 1 || stream != MicStream.MONO) " ${channels}ch" else ""
}
} catch (error: Throwable) {
reader.close()
throw error
}
}
/** Continue from the first chunk at or after [timeMs]. */
fun seekTo(timeMs: Long) {
queued.clear()
pendingMeta.fill(null)
reader.seekTo(timeMs)
}
/** The next chunk in file order, or null at the end. */
fun next(): MicChunk? {
while (queued.isEmpty()) {
val block = reader.next() ?: return null
if (block.track == metaTrack) {
noteMeta(block)
continue
}
val stream = streams[block.track] ?: continue
val track = reader.track(block.track) ?: continue
queueChunks(block, stream, track)
}
return queued.removeFirst()
}
fun close() = reader.close()
private fun noteMeta(block: MkaBlock) {
try {
val fields = Json.parseObject(block.data.decodeToString())
val stream = MicStream.named(fields["s"].asString() ?: return)
if (stream >= 0) pendingMeta[stream] = Meta(block.timeMs, fields)
} catch (ignored: Throwable) {
// An event or a frame we can't read: the audio plays without it.
}
}
private fun queueChunks(block: MkaBlock, stream: Int, track: MkaTrackInfo) {
val sourceChannels = track.channels
val totalFrames = block.data.size / (2 * sourceChannels)
if (totalFrames <= 0) return
// The mixed microphone is one channel however a foreign file stored it.
val channels = if (stream == MicStream.MONO) 1 else sourceChannels
val meta = pendingMeta[stream]?.takeIf { it.timeMs == block.timeMs }
pendingMeta[stream] = null
var start = 0
while (start < totalFrames) {
val frames = minOf(MAX_CHUNK_FRAMES, totalFrames - start)
val pcm = ShortArray(frames * channels)
for (frame in 0 until frames) {
val at = (start + frame) * sourceChannels * 2
if (channels == sourceChannels) {
for (channel in 0 until channels) {
pcm[frame * channels + channel] = AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
}
} else {
var sum = 0
for (channel in 0 until sourceChannels) sum += AudioSegmentation.pcm16le(block.data[at + channel * 2], block.data[at + channel * 2 + 1])
pcm[frame] = (sum / sourceChannels).toShort()
}
}
val chunk = MicChunk(stream, channels, track.sampleRate, pcm, frames, block.timeMs + start * 1000L / track.sampleRate)
if (meta != null && start == 0) applyMeta(chunk, meta.fields)
queued.addLast(chunk)
start += frames
}
}
private fun applyMeta(chunk: MicChunk, fields: Map<String, Any?>) {
chunk.sequence = fields["seq"].asInt() ?: -1
chunk.deviceTickMs = fields["tick"].asLong() ?: -1L
val angle = fields["angle"].asInt()
if (angle != null) {
chunk.hasDirection = true
chunk.angleDegrees = angle
chunk.ssr = fields["ssr"].asInt() ?: 0
}
chunk.truncated = fields["trunc"].asBoolean() ?: false
chunk.lostFramesBefore = fields["lost"].asInt() ?: 0
}
}
@@ -35,6 +35,12 @@ interface VoiceCaptureHost {
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
/** The glasses' per-temple microphone channel, or null when there are no glasses that have one. */
fun micArrayLink(): MicArrayLink? = null
/** Opens a recording to replay; throws when it cannot be read. */
fun openRecording(path: String): SeekableInput = openFileInput(path)
}
/**
@@ -44,6 +50,12 @@ interface VoiceCaptureHost {
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
* worker thread (started through the shared [startThread]); listener callbacks go through the
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
*
* Audio comes from one of four sources, chosen at start(): the glasses' stock mixed stream, the
* phone's microphone, the glasses' per-temple microphones ([setMicArray]), or a recording
* played back in real time ([setReplay]). Each delivers [MicChunk]s; a live capture's chunks go
* to the recorder ([setRecorder]) as they came, before any processing, which is what lets a
* replay of the recording run the same processing again.
*/
class VoiceCaptureSession(
private val host: VoiceCaptureHost,
@@ -95,6 +107,19 @@ class VoiceCaptureSession(
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
/** Samples per G2 packet; the decode buffer the worker reuses. */
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
// Mic array upkeep: the temples stop streaming unless their lease is renewed, and a
// temple that goes quiet (a lost CONFIGURE, or it restarted) is configured again.
const val ARRAY_CHECK_MS = 1000L
const val ARRAY_RENEW_MS = MicArrayProtocol.LEASE_MS / 3
const val ARRAY_TEMPLE_SILENT_MS = 5000L
const val ARRAY_WAITING_MS = 2000L
const val ARRAY_WAITING_STATUS = "Waiting for the glasses microphones..."
// A replay skips stretches of the recording with no audio longer than this
// (a recording made across a disconnect) instead of sitting through them.
const val REPLAY_MAX_GAP_MS = 2000L
const val REPLAY_POSITION_INTERVAL_MS = 1000L
const val REPLAY_WAIT_SLICE_MS = 20L
const val REPLAY_FINISHED_STATUS = "Replay finished."
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
@JvmStatic
@@ -120,6 +145,8 @@ class VoiceCaptureSession(
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
}
private enum class Source { G2, PHONE, ARRAY, REPLAY }
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
val arm: String = arm ?: "?"
}
@@ -137,11 +164,33 @@ class VoiceCaptureSession(
@Volatile
private var workerThreadName: String? = null
// Capture from the phone's own microphone instead of the G2 over BLE
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
// (preview-only mode). Latched into activeSource at start() (under `lock`)
// so a mid-session setter call can't switch pipelines underneath the worker.
@Volatile
private var usePhoneMic = false
private var activePhoneMic = false
// What the next capture reads (see start()); latched into activeSource and its
// companions under `lock`, like usePhoneMic.
@Volatile
private var micArrayConfigure: ByteArray? = null
@Volatile
private var replayPath: String? = null
@Volatile
private var replayStartMs = 0L
private var activeSource = Source.G2
private var activeConfigure: ByteArray? = null
private var activeReplayPath: String? = null
private var activeReplayStartMs = 0L
// Replay controls, picked up by the worker between chunks.
@Volatile
private var replaySeekMs = -1L
@Volatile
private var replayPaused = false
@Volatile
private var recorder: MicRecordingWriter? = null
@Volatile
private var mixer: MicArrayMixer = SingleTempleMixer()
private var lastMissingPackets = 0L
private var arrayRateWarned = false
private var mode = Mode.CLOUD
@Volatile
private var onboardModelKind = VoiceModelKind.MOONSHINE
@@ -193,6 +242,47 @@ class VoiceCaptureSession(
this.usePhoneMic = usePhoneMic
}
/**
* Capture the next glasses session from the temples' own microphones instead of the stock
* mixed stream: [configureRecord] is the mic_control CONFIGURE record sent to both temples
* (it decides how many microphones each one streams). Null goes back to the mixed stream.
*/
fun setMicArray(configureRecord: ByteArray?) {
this.micArrayConfigure = if (MicArrayProtocol.isConfigure(configureRecord)) configureRecord else null
}
/** How the temples' chunks become the mono feed that is transcribed. */
fun setMicArrayMixer(mixer: MicArrayMixer) {
this.mixer = mixer
}
/**
* Have the next capture play the recording at [path] from [startMs] in real time, in place
* of any microphone; null goes back to the microphones. A replay needs no glasses.
*/
fun setReplay(path: String?, startMs: Long) {
this.replayPath = path
this.replayStartMs = maxOf(0L, startMs)
}
/** Continue the running replay from [positionMs] into its recording. */
fun seekReplay(positionMs: Long) {
this.replaySeekMs = maxOf(0L, positionMs)
}
/** Hold the running replay where it is, or let it go on. A new capture starts unpaused. */
fun setReplayPaused(paused: Boolean) {
this.replayPaused = paused
}
/**
* Where live captures' chunks are recorded as they arrive, or null. Takes effect at once,
* also mid-capture; the caller finishes the recording. A replay is never recorded.
*/
fun setRecorder(recorder: MicRecordingWriter?) {
this.recorder = recorder
}
fun setOnboardModelKind(kind: String?) {
this.onboardModelKind = parseModelKind(kind)
}
@@ -258,13 +348,26 @@ class VoiceCaptureSession(
emitStopped(captureId)
return
}
if (!usePhoneMic && !host.isG2SessionReady()) {
val path = replayPath
val configure = micArrayConfigure
val source = when {
path != null -> Source.REPLAY
usePhoneMic -> Source.PHONE
configure != null -> Source.ARRAY
else -> Source.G2
}
if ((source == Source.G2 || source == Source.ARRAY) && !host.isG2SessionReady()) {
emitStatus("Voice control needs an active G2 connection.")
emitStopped(captureId)
return
}
mode = parseMode(requestedMode)
activePhoneMic = usePhoneMic
activeSource = source
activeConfigure = configure
activeReplayPath = path
activeReplayStartMs = replayStartMs
replaySeekMs = -1L
replayPaused = false
started = true
audioStarted = false
val finished = Latch(1, platform)
@@ -281,21 +384,22 @@ class VoiceCaptureSession(
*/
fun isCapturing(): Boolean {
var audioUp = false
var phoneMic = false
var source = Source.G2
lock.withLock {
if (!started) {
return false
}
audioUp = audioStarted
phoneMic = activePhoneMic
source = activeSource
}
if (!audioUp) {
// The worker is still bringing the mic up; report it as running so
// a concurrent request shares it instead of restarting it.
return true
}
if (phoneMic) {
// The phone mic has no session to lose the enable to; it runs until stop().
if (source == Source.PHONE || source == Source.REPLAY) {
// Neither has a glasses session to lose the enable to: the phone mic runs
// until stop(), a replay until its recording ends.
return true
}
return host.isG2AudioCaptureActive()
@@ -317,7 +421,7 @@ class VoiceCaptureSession(
finished = workerFinished
workerName = workerThreadName
}
stopG2Audio()
stopSource()
audioQueue.wake()
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
finished.await(1500)
@@ -356,28 +460,62 @@ class VoiceCaptureSession(
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
verifyCount = 0
if (activePhoneMic) {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
mixer.reset()
lastMissingPackets = 0
val listening = if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening..."
when (activeSource) {
Source.PHONE -> {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
try {
processPhoneAudio(record)
} finally {
record.close()
}
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
try {
processPhoneAudio(record)
} finally {
record.close()
Source.REPLAY -> {
val reader = try {
MicRecordingReader(host.openRecording(activeReplayPath!!))
} catch (error: Throwable) {
emitStatus("Could not open the recording: " + error.message)
emitReplay("error", 0, 0)
return
}
try {
lock.withLock { audioStarted = true }
logInfo("replaying " + reader.description + ", " + formatFixed(reader.durationMs / 1000.0, 1) + " s")
emitStatus(listening)
processReplay(reader)
} finally {
reader.close()
}
}
} else {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
Source.ARRAY -> {
val link = host.micArrayLink()
val configure = activeConfigure
if (link == null || configure == null || !startArray(link, configure)) {
emitStatus("Could not start the glasses microphone array.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
processArray(link, configure, listening)
}
Source.G2 -> {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
}
lock.withLock { audioStarted = true }
emitStatus(listening)
processG2Audio()
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
processG2Audio()
}
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
@@ -392,7 +530,7 @@ class VoiceCaptureSession(
logError("Voice control failed", error)
emitStatus("Voice control failed: " + error.message)
} finally {
stopG2Audio()
stopSource()
releaseRecognizer()
releaseDecoder()
lock.withLock {
@@ -432,23 +570,214 @@ class VoiceCaptureSession(
continue
}
decodedSamples += count.toLong()
// The beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker for
// every consumer, recognition included); the spectral noise
// suppressor then cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
val angleDegrees = currentDecoder.lastAngleDegrees
val ssr = currentDecoder.lastSsr
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr)
maybeEmitAudioStats(false)
continue
}
processPcmChunk(pcm, count, angleDegrees, ssr, currentDecoder.hasFrameMeta)
val chunk = MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, count, packet.arrivalMs)
chunk.hasDirection = currentDecoder.hasFrameMeta
chunk.angleDegrees = currentDecoder.lastAngleDegrees
chunk.ssr = currentDecoder.lastSsr
// Packets the decoder found missing ahead of this one: a gap in a recording.
val missing = currentDecoder.missingPackets
chunk.lostFramesBefore = ((missing - lastMissingPackets) * currentDecoder.maxSamplesPerPacket).toInt()
lastMissingPackets = missing
handleMonoChunk(chunk)
maybeEmitAudioStats(false)
}
}
/**
* A mixed-microphone chunk from any source: recorded as it came, then on to processing
* unless the beam filter drops it. The filter drops chunks whose firmware
* direction-of-arrival falls outside the listening wedge (isolating the aimed talker for
* every consumer, recognition included); the spectral noise suppressor then cleans what
* remains before it reaches the recognizer, cloud PCM, endpointing, or speaker verification.
*/
private fun handleMonoChunk(chunk: MicChunk) {
recordChunk(chunk)
if (beamFilterEnabled && chunk.ssr > 0 && !withinBeam(chunk.angleDegrees)) {
emitFrameMeta(chunk.angleDegrees, chunk.ssr)
return
}
processPcmChunk(chunk.pcm, chunk.frames, chunk.angleDegrees, chunk.ssr, chunk.hasDirection)
}
/**
* A chunk of one temple's own microphones: recorded as it came, then mixed into the mono
* feed, which is recorded too (as what was transcribed) and processed like a mixed chunk.
* The beam filter does not apply: a temple's bearing is along its own pair, not the
* head-frame direction the filter is aimed in.
*/
private fun handleArrayChunk(chunk: MicChunk) {
recordChunk(chunk)
val feed = mixer.mix(chunk) ?: return
recordChunk(feed)
if (feed.sampleRate != SAMPLE_RATE || feed.channels != 1) {
if (!arrayRateWarned) logWarn("mic array feed is ${feed.channels}ch ${feed.sampleRate} Hz; recognition needs mono $SAMPLE_RATE Hz")
arrayRateWarned = true
return
}
processPcmChunk(feed.pcm, feed.frames, feed.angleDegrees, feed.ssr, feed.hasDirection)
}
private fun recordChunk(chunk: MicChunk) {
if (activeSource != Source.REPLAY) recorder?.write(chunk)
}
private fun handleChunk(chunk: MicChunk) {
if (chunk.stream == MicStream.MONO) handleMonoChunk(chunk) else handleArrayChunk(chunk)
}
private fun startArray(link: MicArrayLink, configure: ByteArray): Boolean {
resetAudioStats()
audioQueue.clear()
if (!startArrayForwarding(link)) return false
return link.sendControl(configure, "configure", true, true)
}
private fun startArrayForwarding(link: MicArrayLink): Boolean =
link.startForwarding(object : FaceclawAudioPacketListener {
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
queueAudioPacket(data, arm, arrivalMs)
}
})
/**
* Mic array capture loop: each temple's 'SM' frames become chunks of that temple's stream.
* Between frames it keeps the streams alive: renews the temples' lease, configures a temple
* again when it has gone quiet, and takes forwarding back when the glasses session dropped it.
*/
private fun processArray(link: MicArrayLink, configure: ByteArray, listening: String) {
val startedAt = platform.elapsedRealtimeMs()
var checkedAt = startedAt
var renewedAt = startedAt
// Left, right.
val armedAt = longArrayOf(startedAt, startedAt)
val frameAt = longArrayOf(startedAt, startedAt)
var waiting = false
while (started) {
val packet = takeAudioPacket()
val now = platform.elapsedRealtimeMs()
if (packet != null) {
val side = if (packet.arm == "L") 0 else if (packet.arm == "R") 1 else -1
val frame = if (side >= 0) MicArrayProtocol.decodeFrame(packet.data) else null
val pcm = if (frame != null) MicArrayProtocol.pcm16(packet.data, frame) else null
// Anything else is a stock-format packet from before arming took effect.
if (frame != null && pcm != null) {
frameAt[side] = now
val frames = pcm.size / frame.channels
decodedSamples += frames.toLong()
val stream = if (side == 0) MicStream.LEFT else MicStream.RIGHT
val chunk = MicChunk(stream, frame.channels, frame.sampleRate, pcm, frames, packet.arrivalMs)
chunk.hasDirection = frame.angleDegrees != 0 || frame.ssr != 0
chunk.angleDegrees = frame.angleDegrees
chunk.ssr = frame.ssr
chunk.sequence = frame.sequence
chunk.deviceTickMs = frame.deviceTickMs
chunk.truncated = frame.truncated
handleArrayChunk(chunk)
}
}
if (now - checkedAt >= ARRAY_CHECK_MS) {
checkedAt = now
if (!link.isForwarding()) {
if (host.isG2SessionReady() && startArrayForwarding(link) && link.sendControl(configure, "configure", true, true)) {
logInfo("mic array forwarding re-established")
armedAt.fill(now)
renewedAt = now
}
} else {
if (now - renewedAt >= ARRAY_RENEW_MS) {
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_RENEW), "renew", true, true)
renewedAt = now
}
for (side in 0..1) {
if (now - maxOf(frameAt[side], armedAt[side]) < ARRAY_TEMPLE_SILENT_MS) continue
logWarn("mic array: " + (if (side == 0) "left" else "right") + " temple silent; configuring it again")
link.sendControl(configure, "configure-" + (if (side == 0) "left" else "right"), side == 1, side == 0)
armedAt[side] = now
}
}
val silent = now - maxOf(frameAt[0], frameAt[1]) >= ARRAY_WAITING_MS
if (silent != waiting) {
waiting = silent
emitStatus(if (silent) ARRAY_WAITING_STATUS else listening)
}
}
maybeEmitAudioStats(false)
}
}
/**
* Replay loop: hands the recording's chunks to the pipeline at the pace they were captured,
* so recognition (its partials, a cloud provider's stream) sees what a live capture would.
* Jumps and pauses asked for from outside take effect between chunks.
*/
private fun processReplay(reader: MicRecordingReader) {
val duration = reader.durationMs
var position = minOf(activeReplayStartMs, duration)
reader.seekTo(position)
emitReplay("start", position, duration)
var reportedAt = position
// Wall-clock time at which file time `anchorFile` plays; unset after a jump, pause or gap.
var anchorWall = -1L
var anchorFile = 0L
var pending: MicChunk? = null
while (started) {
val seek = replaySeekMs
if (seek >= 0) {
replaySeekMs = -1L
// What played so far is a finished utterance; the rest starts clean.
finishReplayedUtterance()
position = minOf(seek, duration)
reader.seekTo(position)
pending = null
anchorWall = -1L
reportedAt = position
emitReplay("seek", position, duration)
}
if (replayPaused) {
anchorWall = -1L
sleepMs(REPLAY_WAIT_SLICE_MS)
continue
}
val chunk = pending ?: reader.next()
if (chunk == null) {
emitReplay("end", duration, duration)
emitStatus(REPLAY_FINISHED_STATUS)
return
}
pending = chunk
val now = platform.elapsedRealtimeMs()
if (anchorWall < 0 || chunk.timeMs - position > REPLAY_MAX_GAP_MS) {
anchorWall = now
anchorFile = chunk.timeMs
}
val wait = anchorWall + (chunk.timeMs - anchorFile) - now
if (wait > 0) {
// Short slices, so a stop, jump or pause is noticed promptly.
sleepMs(minOf(wait, REPLAY_WAIT_SLICE_MS))
continue
}
pending = null
position = chunk.timeMs
decodedSamples += chunk.frames.toLong()
handleChunk(chunk)
if (position - reportedAt >= REPLAY_POSITION_INTERVAL_MS) {
reportedAt = position
emitReplay("position", position, duration)
}
}
}
/** Ends the utterance a replay was in the middle of when it jumped, so the audio on both sides isn't joined. */
private fun finishReplayedUtterance() {
mixer.reset()
endpointDetector.reset()
suppressor?.reset()
if (mode != Mode.ONBOARD) return
decodeTranscript(true)
resetTranscriptState()
lastTranscript = ""
}
/**
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
* decode and the beam filter. hasFrameMeta is false for the phone mic,
@@ -495,7 +824,7 @@ class VoiceCaptureSession(
continue
}
decodedSamples += read.toLong()
processPcmChunk(pcm, read, 0, 0, false)
handleMonoChunk(MicChunk(MicStream.MONO, 1, SAMPLE_RATE, pcm, read, platform.elapsedRealtimeMs()))
}
}
@@ -681,6 +1010,11 @@ class VoiceCaptureSession(
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
}
private fun emitReplay(event: String, positionMs: Long, durationMs: Long) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onReplay(event, positionMs, durationMs) }
}
private fun emitSpeechEnd() {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onSpeechEnd() }
@@ -691,8 +1025,21 @@ class VoiceCaptureSession(
host.dispatcher.post { currentListener.onStopped(captureId) }
}
private fun stopG2Audio() {
host.g2AudioSource()?.stop()
/**
* Release what the running capture's source holds on the glasses. Runs on stop() and again
* as the worker winds down (it may have started the source after stop() looked), so
* everything here is safe to repeat.
*/
private fun stopSource() {
val source = lock.withLock { activeSource }
when (source) {
Source.G2 -> host.g2AudioSource()?.stop()
Source.ARRAY -> host.micArrayLink()?.let { link ->
link.sendControl(MicArrayProtocol.control(MicArrayProtocol.OP_STOP), "stop", true, true)
link.stopForwarding()
}
Source.PHONE, Source.REPLAY -> {}
}
maybeEmitAudioStats(true)
}
@@ -711,7 +1058,7 @@ class VoiceCaptureSession(
if (!started || data == null) {
return
}
if ("L" != arm) {
if ("L" != arm && activeSource != Source.ARRAY) {
wrongArmPackets++
}
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
@@ -67,6 +67,24 @@ interface PacketAudioSource {
fun isActive(): Boolean
}
/**
* The custom firmware's mic_control channel on a glasses session: each temple streams its own
* microphones as 'SM' frames (see [MicArrayProtocol]) on the characteristic the stock mixed
* stream uses, while it holds a configuration that the phone keeps renewing.
*/
interface MicArrayLink {
/** Deliver the audio characteristic's packets to [listener]; false when no session is up. */
fun startForwarding(listener: FaceclawAudioPacketListener): Boolean
fun stopForwarding()
/** Whether forwarding still holds: it lives in the glasses session, which can drop it. */
fun isForwarding(): Boolean
/** Queue a control record for the chosen temples; false when nothing was queued. */
fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean
}
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
interface PcmAudioSource {
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
@@ -41,4 +41,13 @@ interface FaceclawVoiceControllerListener {
* speaker model could not be loaded.
*/
fun onSpeakerVerified(isWearer: Boolean, similarity: Float): Unit
/**
* Progress of a capture that replays a recording (see VoiceCaptureSession.setReplay):
* "start" once the file is open, "position" about once per second of it played, "seek" after
* a jump took effect (the transcript of what played before it has been emitted as final),
* "end" when the file ran out, and "error" when it could not be opened (the status says
* why). Positions are milliseconds into the file.
*/
fun onReplay(event: String?, positionMs: Long, durationMs: Long): Unit
}
@@ -0,0 +1,37 @@
package com.faceclaw.app
import platform.Foundation.NSData
/**
* Records the iOS voice bridge's audio to a Matroska audio file (.mka), the format Android's
* captures are recorded in (see [MicRecordingWriter]). The bridge has one stream to record:
* 16 kHz mono PCM16, as its recognizer hands it to TypeScript.
*/
class IosMicRecorder internal constructor(private val writer: MicRecordingWriter) {
/** The recording stopped early: the file could not be written. */
val failed: Boolean get() = writer.failed
/** Appends a chunk of 16 kHz mono PCM16 (little-endian), as having arrived now. */
fun append(pcm16le: NSData) {
val bytes = pcm16le.byteArray()
val samples = ShortArray(bytes.size / 2) { AudioSegmentation.pcm16le(bytes[it * 2], bytes[it * 2 + 1]) }
writer.write(MicChunk(MicStream.MONO, 1, 16000, samples, samples.size, IosProtocolPlatform.elapsedRealtimeMs()))
}
/** Completes the file. Returns the recording's length in ms. */
fun finish(): Long = writer.finish()
}
object IosMicRecordings {
/**
* A recorder writing a new file at [path], or null when the file cannot be created.
* [configJson] describes the capture and is stored in the file.
*/
fun open(path: String, configJson: String?): IosMicRecorder? =
try {
IosMicRecorder(MicRecordingWriter(openFileOutput(path), configJson, currentTimeMillis(), IosProtocolPlatform))
} catch (error: Throwable) {
PlatformLog.w("FaceclawMicRecording", "could not create $path: ${error.message}")
null
}
}
@@ -114,3 +114,54 @@ actual fun fileLength(path: String): Long = memScoped {
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
actual fun openFileOutput(path: String): SeekableOutput {
val file = fopen(path, "wb+") ?: error("Cannot write $path")
return object : SeekableOutput {
override var size = 0L
private set
private fun put(bytes: ByteArray, offset: Int, length: Int) {
if (length == 0) return
bytes.usePinned {
check(fwrite(it.addressOf(offset), 1u, length.convert(), file) == length.convert<platform.posix.size_t>()) { "Short write to $path" }
}
}
override fun write(bytes: ByteArray, offset: Int, length: Int) {
put(bytes, offset, length)
size += length
// Each write is a complete element; hand it to the OS so a crash keeps it.
platform.posix.fflush(file)
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
check(platform.posix.fseek(file, position, platform.posix.SEEK_SET) == 0)
put(bytes, 0, bytes.size)
check(platform.posix.fseek(file, size, platform.posix.SEEK_SET) == 0)
platform.posix.fflush(file)
}
override fun close() {
fclose(file)
}
}
}
actual fun openFileInput(path: String): SeekableInput {
val file = fopen(path, "rb") ?: error("Cannot read $path")
val fileSize = fileLength(path)
return object : SeekableInput {
override val length = fileSize
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
if (length == 0 || platform.posix.fseek(file, position, platform.posix.SEEK_SET) != 0) return 0
return buffer.usePinned { platform.posix.fread(it.addressOf(offset), 1u, length.convert(), file).toInt() }
}
override fun close() {
fclose(file)
}
}
}