Add microphone support on RayNeo iO

This commit is contained in:
James Babcock committed 2026-11-01 21:08:25 -08:00
1 parent fde847a31a
commit 9ddd4c4d03
13 files changed
+407 -5

No files matched your search

+3
View File
@@ -19,6 +19,9 @@ dependencies {
// app's captions (translation models download on demand, per language).
implementation 'com.google.mlkit:language-id:17.0.6'
implementation 'com.google.mlkit:translate:17.0.3'
// Pure-Java Opus decoder for the RayNeo iO's microphone (16 kHz mono Opus over
// BLE); the same one Glassly uses. See AndroidSpeechEngines.OpusDecoderAdapter.
implementation 'io.github.jaredmdobson:concentus:1.0.1'
}
def faceclawSherpaOnnxVersion = "1.13.0"
@@ -94,6 +94,44 @@ internal object AndroidSpeechEngines {
override fun close() = decoder.close()
}
/**
* The RayNeo iO's microphone: one 16 kHz mono Opus packet per BLE message (its remote
* audio stream, GlassesSessionIo), decoded with Concentus. No G2 DSP metadata; packets
* carry no counter, so "missing" stays 0 and a lost one is just a gap.
*/
class OpusDecoderAdapter : AudioPacketDecoder {
private val decoder = io.github.jaredmdobson.OpusDecoder(RayNeoIoProtocol.VOICE_SAMPLE_RATE, 1)
private var packets = 0L
private var errors = 0L
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
if (packet == null || packet.isEmpty() || pcmOut == null) return 0
return try {
val count = decoder.decode(packet, 0, packet.size, pcmOut, 0, pcmOut.size, false)
packets++
count
} catch (e: io.github.jaredmdobson.OpusException) {
errors++
if (errors == 1L || errors % 100 == 0L) {
android.util.Log.w("FaceclawVoice", "Opus decode error ($errors so far): ${e.message}")
}
0
}
}
/** 120 ms, Opus's longest frame. */
override val maxSamplesPerPacket: Int get() = RayNeoIoProtocol.VOICE_SAMPLE_RATE * 120 / 1000
override val hasFrameMeta: Boolean get() = false
override val lastAngleDegrees: Int get() = 0
override val lastSsr: Int get() = 0
override val realPackets: Long get() = packets
override val duplicatePackets: Long get() = 0
override val missingPackets: Long get() = 0
override val decodeErrors: Long get() = errors
override fun close() = decoder.resetState()
}
/**
* Open the phone's own microphone at the pipeline's native format (16 kHz
* mono PCM16), or null when it cannot start — the permission is missing
@@ -254,6 +254,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled)
/** See [GlassesSessionCore.setVoiceWakeEnabled]. */
fun setVoiceWakeEnabled(enabled: Boolean) = core.setVoiceWakeEnabled(enabled)
fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs)
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq)
@@ -114,7 +114,9 @@ class FaceclawVoiceController(context: Context) {
}
}
override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter()
override fun createPacketDecoder(): AudioPacketDecoder =
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
else AndroidSpeechEngines.Lc3DecoderAdapter()
override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES)