forked from Blendman974/faceclaw
Add microphone support on RayNeo iO
This commit is contained in:
1 parent
fde847a31a
commit
9ddd4c4d03
13 files changed
+407
-5
No files matched your search
@@ -19,6 +19,9 @@ dependencies {
|
||||
// app's captions (translation models download on demand, per language).
|
||||
implementation 'com.google.mlkit:language-id:17.0.6'
|
||||
implementation 'com.google.mlkit:translate:17.0.3'
|
||||
// Pure-Java Opus decoder for the RayNeo iO's microphone (16 kHz mono Opus over
|
||||
// BLE); the same one Glassly uses. See AndroidSpeechEngines.OpusDecoderAdapter.
|
||||
implementation 'io.github.jaredmdobson:concentus:1.0.1'
|
||||
}
|
||||
|
||||
def faceclawSherpaOnnxVersion = "1.13.0"
|
||||
|
||||
@@ -94,6 +94,44 @@ internal object AndroidSpeechEngines {
|
||||
override fun close() = decoder.close()
|
||||
}
|
||||
|
||||
/**
|
||||
* The RayNeo iO's microphone: one 16 kHz mono Opus packet per BLE message (its remote
|
||||
* audio stream, GlassesSessionIo), decoded with Concentus. No G2 DSP metadata; packets
|
||||
* carry no counter, so "missing" stays 0 and a lost one is just a gap.
|
||||
*/
|
||||
class OpusDecoderAdapter : AudioPacketDecoder {
|
||||
private val decoder = io.github.jaredmdobson.OpusDecoder(RayNeoIoProtocol.VOICE_SAMPLE_RATE, 1)
|
||||
private var packets = 0L
|
||||
private var errors = 0L
|
||||
|
||||
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
|
||||
if (packet == null || packet.isEmpty() || pcmOut == null) return 0
|
||||
return try {
|
||||
val count = decoder.decode(packet, 0, packet.size, pcmOut, 0, pcmOut.size, false)
|
||||
packets++
|
||||
count
|
||||
} catch (e: io.github.jaredmdobson.OpusException) {
|
||||
errors++
|
||||
if (errors == 1L || errors % 100 == 0L) {
|
||||
android.util.Log.w("FaceclawVoice", "Opus decode error ($errors so far): ${e.message}")
|
||||
}
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/** 120 ms, Opus's longest frame. */
|
||||
override val maxSamplesPerPacket: Int get() = RayNeoIoProtocol.VOICE_SAMPLE_RATE * 120 / 1000
|
||||
override val hasFrameMeta: Boolean get() = false
|
||||
override val lastAngleDegrees: Int get() = 0
|
||||
override val lastSsr: Int get() = 0
|
||||
override val realPackets: Long get() = packets
|
||||
override val duplicatePackets: Long get() = 0
|
||||
override val missingPackets: Long get() = 0
|
||||
override val decodeErrors: Long get() = errors
|
||||
|
||||
override fun close() = decoder.resetState()
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the phone's own microphone at the pipeline's native format (16 kHz
|
||||
* mono PCM16), or null when it cannot start — the permission is missing
|
||||
|
||||
@@ -254,6 +254,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled)
|
||||
|
||||
/** See [GlassesSessionCore.setVoiceWakeEnabled]. */
|
||||
fun setVoiceWakeEnabled(enabled: Boolean) = core.setVoiceWakeEnabled(enabled)
|
||||
|
||||
fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs)
|
||||
|
||||
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq)
|
||||
|
||||
@@ -114,7 +114,9 @@ class FaceclawVoiceController(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
override fun createPacketDecoder(): AudioPacketDecoder =
|
||||
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
|
||||
else AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
|
||||
override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES)
|
||||
|
||||
|
||||
Reference in new issue
Block a user