forked from Blendman974/faceclaw
Merge upstream main preserving custom feature integrations
This commit is contained in:
commit
6150a1ac57
228 files changed
+16844
-1388
No files matched your search
@@ -19,6 +19,9 @@ dependencies {
|
||||
// app's captions (translation models download on demand, per language).
|
||||
implementation 'com.google.mlkit:language-id:17.0.6'
|
||||
implementation 'com.google.mlkit:translate:17.0.3'
|
||||
// Pure-Java Opus decoder for the RayNeo iO's microphone (16 kHz mono Opus over
|
||||
// BLE); the same one Glassly uses. See AndroidSpeechEngines.OpusDecoderAdapter.
|
||||
implementation 'io.github.jaredmdobson:concentus:1.0.1'
|
||||
}
|
||||
|
||||
def faceclawSherpaOnnxVersion = "1.13.0"
|
||||
|
||||
@@ -46,6 +46,15 @@ class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLi
|
||||
|
||||
override fun isBonded(address: String): Boolean = bleManager.isBonded(address)
|
||||
|
||||
override fun requestBond(address: String) = bleManager.requestBond(address)
|
||||
|
||||
override fun removeBond(address: String): Boolean = bleManager.removeBond(address)
|
||||
|
||||
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean =
|
||||
bleManager.hasCharacteristic(address, characteristicUuid)
|
||||
|
||||
override fun refreshServices(address: String, timeoutMs: Int): Boolean = bleManager.refreshServices(address, timeoutMs)
|
||||
|
||||
override fun prepareBenchmarkLink(address: String, mode: Int) = bleManager.prepareBenchmarkLink(address, mode)
|
||||
|
||||
override fun recordDisplayFrameSent() = FaceclawBleManager.recordDisplayFrameSent()
|
||||
|
||||
@@ -97,6 +97,44 @@ internal object AndroidSpeechEngines {
|
||||
override fun close() = decoder.close()
|
||||
}
|
||||
|
||||
/**
|
||||
* The RayNeo iO's microphone: one 16 kHz mono Opus packet per BLE message (its remote
|
||||
* audio stream, GlassesSessionIo), decoded with Concentus. No G2 DSP metadata; packets
|
||||
* carry no counter, so "missing" stays 0 and a lost one is just a gap.
|
||||
*/
|
||||
class OpusDecoderAdapter : AudioPacketDecoder {
|
||||
private val decoder = io.github.jaredmdobson.OpusDecoder(RayNeoIoProtocol.VOICE_SAMPLE_RATE, 1)
|
||||
private var packets = 0L
|
||||
private var errors = 0L
|
||||
|
||||
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
|
||||
if (packet == null || packet.isEmpty() || pcmOut == null) return 0
|
||||
return try {
|
||||
val count = decoder.decode(packet, 0, packet.size, pcmOut, 0, pcmOut.size, false)
|
||||
packets++
|
||||
count
|
||||
} catch (e: io.github.jaredmdobson.OpusException) {
|
||||
errors++
|
||||
if (errors == 1L || errors % 100 == 0L) {
|
||||
android.util.Log.w("FaceclawVoice", "Opus decode error ($errors so far): ${e.message}")
|
||||
}
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/** 120 ms, Opus's longest frame. */
|
||||
override val maxSamplesPerPacket: Int get() = RayNeoIoProtocol.VOICE_SAMPLE_RATE * 120 / 1000
|
||||
override val hasFrameMeta: Boolean get() = false
|
||||
override val lastAngleDegrees: Int get() = 0
|
||||
override val lastSsr: Int get() = 0
|
||||
override val realPackets: Long get() = packets
|
||||
override val duplicatePackets: Long get() = 0
|
||||
override val missingPackets: Long get() = 0
|
||||
override val decodeErrors: Long get() = errors
|
||||
|
||||
override fun close() = decoder.resetState()
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the phone's own microphone at the pipeline's native format (16 kHz
|
||||
* mono PCM16), or null when it cannot start — the permission is missing
|
||||
|
||||
@@ -48,6 +48,17 @@ class AndroidStockLink(context: Context) : StockLink, FaceclawBleListener {
|
||||
else -> BondState.UNKNOWN
|
||||
}
|
||||
|
||||
override fun negotiatedMtu(address: String): Int = bleManager.getNegotiatedMtu(address)
|
||||
|
||||
override fun requestBond(address: String) = bleManager.requestBond(address)
|
||||
|
||||
override fun removeBond(address: String): Boolean = bleManager.removeBond(address)
|
||||
|
||||
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean =
|
||||
bleManager.hasCharacteristic(address, characteristicUuid)
|
||||
|
||||
override fun refreshServices(address: String, timeoutMs: Int): Boolean = bleManager.refreshServices(address, timeoutMs)
|
||||
|
||||
override fun disconnect(address: String) = bleManager.disconnect(address)
|
||||
|
||||
override fun close() = bleManager.close()
|
||||
|
||||
@@ -189,6 +189,28 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision)
|
||||
|
||||
/**
|
||||
* Drive a RayNeo iO (constructed with its address as both arms and no ring). The phone's
|
||||
* protocol identity is derived from ANDROID_ID so it survives reinstalls: the glasses bind
|
||||
* to the first identity that authenticates and refuse others until put in pairing mode.
|
||||
*/
|
||||
fun useRayNeoIo() {
|
||||
val androidId = try {
|
||||
android.provider.Settings.Secure.getString(appContext.contentResolver, android.provider.Settings.Secure.ANDROID_ID)
|
||||
} catch (_: Exception) {
|
||||
null
|
||||
}
|
||||
val seed = "faceclaw-rayneo-phone-id:" + (androidId ?: android.os.Build.FINGERPRINT)
|
||||
val phoneId = java.security.MessageDigest.getInstance("SHA-256").digest(seed.toByteArray(Charsets.UTF_8)).copyOf(6)
|
||||
val model = android.os.Build.MODEL ?: "Android"
|
||||
core.useRayNeoIo(phoneId, model, android.os.Build.MANUFACTURER ?: "Android", model)
|
||||
}
|
||||
|
||||
fun isRayNeoIo(): Boolean = core.isRayNeoIo()
|
||||
|
||||
/** Drive an Even G1 on g1v0 (two arms, as a G2 pair's addresses). */
|
||||
fun useG1() = core.useG1()
|
||||
|
||||
fun start() {
|
||||
if (core.start()) {
|
||||
activeInstance = this
|
||||
@@ -235,6 +257,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled)
|
||||
|
||||
/** See [GlassesSessionCore.setVoiceWakeEnabled]. */
|
||||
fun setVoiceWakeEnabled(enabled: Boolean) = core.setVoiceWakeEnabled(enabled)
|
||||
|
||||
fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs)
|
||||
|
||||
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq)
|
||||
|
||||
@@ -27,6 +27,11 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class FaceclawBleManager(context: Context) {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawBle"
|
||||
|
||||
/** Not public API, but long stable, and the only way short of sending the user to Settings. */
|
||||
fun removeBond(device: BluetoothDevice): Boolean =
|
||||
device.bondState == BluetoothDevice.BOND_NONE || device.javaClass.getMethod("removeBond").invoke(device) == true
|
||||
|
||||
private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200)
|
||||
|
||||
// Gap after each LE 2M PHY request before checking whether it landed and asking
|
||||
@@ -160,6 +165,36 @@ class FaceclawBleManager(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove this phone's half of a pairing the other side no longer has. False when the platform refused.
|
||||
*/
|
||||
fun removeBond(address: String?): Boolean {
|
||||
if (address == null || address.trim().isEmpty()) {
|
||||
return false
|
||||
}
|
||||
return try {
|
||||
removeBond(bluetoothAdapter.getRemoteDevice(address))
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "removeBond failed: $address", t)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/** Start bonding, without waiting: the OS may ask the user, and [isBonded] tells when it is done. */
|
||||
fun requestBond(address: String?) {
|
||||
if (address == null || address.trim().isEmpty()) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
val device = bluetoothAdapter.getRemoteDevice(address)
|
||||
if (device.bondState == BluetoothDevice.BOND_NONE) {
|
||||
Log.i(TAG, "requestBond " + address + ": " + device.createBond())
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "requestBond failed: " + address, t)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Android's bond state for this address (BluetoothDevice.BOND_NONE /
|
||||
* BOND_BONDING / BOND_BONDED), or -1 when it cannot be determined.
|
||||
@@ -364,6 +399,31 @@ class FaceclawBleManager(context: Context) {
|
||||
return status == BluetoothGatt.GATT_SUCCESS
|
||||
}
|
||||
|
||||
/** After discovery: whether the service table has this characteristic. */
|
||||
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean {
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = links[address]?.gatt ?: return false
|
||||
val target = UUID.fromString(characteristicUuid)
|
||||
return gatt.services.any { it.getCharacteristic(target) != null }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget the service table Android remembers for a bonded device and read it again. A G1 arm keeps its address across an
|
||||
* install that changes its services, and a cached table would make new firmware look like the old one.
|
||||
*/
|
||||
fun refreshServices(address: String, timeoutMs: Int): Boolean {
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = links[address]?.gatt ?: return false
|
||||
try {
|
||||
gatt.javaClass.getMethod("refresh").invoke(gatt)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "refresh failed: $address", t)
|
||||
}
|
||||
}
|
||||
return discoverServices(address, timeoutMs)
|
||||
}
|
||||
|
||||
fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean {
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = requireGatt(address)
|
||||
|
||||
@@ -129,13 +129,21 @@ class FaceclawDeviceDiscovery(context: Context) {
|
||||
&& manufacturerData[0] == 0x45.toByte() && manufacturerData[1] == 0x52.toByte()) {
|
||||
return true
|
||||
}
|
||||
// RayNeo iO: manufacturer company 0x5716 ([07][MAC]...) outside pairing mode;
|
||||
// in pairing mode only the "RayNeo iO-xxxx" name identifies it.
|
||||
if (manufacturerData != null && manufacturerData.size >= 2
|
||||
&& manufacturerData[0] == 0x16.toByte() && manufacturerData[1] == 0x57.toByte()) {
|
||||
return true
|
||||
}
|
||||
if (name == null) {
|
||||
return false
|
||||
}
|
||||
val upper = name.uppercase()
|
||||
// A bare "R1" substring also matches unrelated hardware ("Oppo Enco
|
||||
// R1"), and TS only classifies rings from the stock name prefix.
|
||||
return upper.contains("G2") || upper.startsWith("EVEN R1")
|
||||
// Even G1: its stock name, or g1v0's G1CFW_L_XXXXXX (before 0.5.66: G1v0_L, G1v0S_R).
|
||||
return upper.contains("G2") || upper.startsWith("EVEN R1") || upper.startsWith("RAYNEO") ||
|
||||
upper.startsWith("EVEN G1") || upper.startsWith("G1CFW_") || upper.startsWith("G1V0")
|
||||
}
|
||||
|
||||
private fun buildJson(
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
|
||||
/**
|
||||
* Android host for the shared [G1ArmFlashFlow]: owns the GATT link and the worker thread for one G1 arm's install, its
|
||||
* restore to Even's firmware, or a question about what it runs, and delivers listener callbacks on the main thread. For app/native/g1-flasher.ts.
|
||||
*/
|
||||
class FaceclawG1Flasher(
|
||||
context: Context,
|
||||
address: String?,
|
||||
side: String?,
|
||||
firmwarePath: String?,
|
||||
netCorePath: String?,
|
||||
mode: String?,
|
||||
connectWindowMs: Int,
|
||||
) {
|
||||
private val link = AndroidStockLink(context.applicationContext)
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val lock = Any()
|
||||
|
||||
@Volatile
|
||||
private var listener: FaceclawFirmwareFlasherListener? = null
|
||||
|
||||
@Volatile
|
||||
private var worker: Thread? = null
|
||||
|
||||
private val flow =
|
||||
G1ArmFlashFlow(
|
||||
link,
|
||||
address ?: "",
|
||||
side ?: "",
|
||||
firmwarePath ?: "",
|
||||
netCorePath ?: "",
|
||||
G1ArmFlashFlow.mode(mode ?: ""),
|
||||
connectWindowMs,
|
||||
object : FaceclawFirmwareFlasherListener {
|
||||
override fun onLog(line: String?) {
|
||||
android.util.Log.i(G1ArmFlashFlow.TAG, line ?: "")
|
||||
post { it.onLog(line) }
|
||||
}
|
||||
|
||||
override fun onProgress(
|
||||
lens: String?,
|
||||
componentIndex: Int,
|
||||
componentCount: Int,
|
||||
blockIndex: Int,
|
||||
blockCount: Int,
|
||||
bytesSent: Long,
|
||||
bytesTotal: Long,
|
||||
) = post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
|
||||
|
||||
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
|
||||
|
||||
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
|
||||
},
|
||||
)
|
||||
|
||||
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
|
||||
this.listener = listener
|
||||
}
|
||||
|
||||
fun start() {
|
||||
synchronized(lock) {
|
||||
if (worker != null) {
|
||||
return
|
||||
}
|
||||
val w = Thread({ flow.run() }, "faceclaw-g1-flash")
|
||||
worker = w
|
||||
w.start()
|
||||
}
|
||||
}
|
||||
|
||||
fun cancel() {
|
||||
flow.cancel()
|
||||
}
|
||||
|
||||
fun close() {
|
||||
cancel()
|
||||
try {
|
||||
link.close()
|
||||
} catch (ignored: Exception) {
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** The phone's pairing with an address these glasses no longer use. */
|
||||
@JvmStatic
|
||||
fun forgetPairing(context: Context, address: String?) {
|
||||
if (address == null) return
|
||||
val manager = context.applicationContext.getSystemService(Context.BLUETOOTH_SERVICE) as android.bluetooth.BluetoothManager
|
||||
FaceclawBleManager.removeBond(manager.adapter.getRemoteDevice(address))
|
||||
}
|
||||
}
|
||||
|
||||
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) {
|
||||
mainHandler.post {
|
||||
val current = listener
|
||||
if (current != null) {
|
||||
action(current)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -104,7 +104,9 @@ class FaceclawVoiceController(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
override fun createPacketDecoder(): AudioPacketDecoder =
|
||||
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
|
||||
else AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
|
||||
override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES)
|
||||
|
||||
|
||||
@@ -167,6 +167,7 @@ class FontFileRenderer private constructor() {
|
||||
* [width u16][height u16]
|
||||
* [width*height coverage bytes, gamma-mapped]
|
||||
* An ink-free glyph (space) has width = height = 0 but a valid advance.
|
||||
* A color glyph (emoji) carries its ColorGlyphArt shade as its coverage.
|
||||
* Empty array when the font cannot be loaded or the render fails.
|
||||
*/
|
||||
@JvmStatic
|
||||
@@ -196,11 +197,16 @@ class FontFileRenderer private constructor() {
|
||||
}
|
||||
val penX = pad - bounds.left
|
||||
val baselineY = pad - bounds.top
|
||||
val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ALPHA_8)
|
||||
// Drawn in color: an emoji the system fallback supplies from a color font
|
||||
// would keep only its silhouette in an alpha-only bitmap. ColorGlyphArt shades
|
||||
// color glyphs and passes white ones through as their alpha.
|
||||
val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bitmap)
|
||||
canvas.drawText(text, penX.toFloat(), baselineY.toFloat(), paint)
|
||||
val pixels = alpha8Pixels(bitmap, bw, bh)
|
||||
val argb = IntArray(bw * bh)
|
||||
bitmap.getPixels(argb, 0, bw, 0, 0, bw, bh)
|
||||
bitmap.recycle()
|
||||
val pixels = ColorGlyphArt.coverage(argb)
|
||||
|
||||
// Trim to the tight ink box; bearing/inkTop become pen- and
|
||||
// line-relative inside the shared packer.
|
||||
|
||||
@@ -108,6 +108,64 @@ class ImageFileLoader private constructor() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Longest side decodeRgbaFromBytes decodes at; larger images are subsampled. */
|
||||
private const val MAX_RGBA_SIDE = 4096
|
||||
|
||||
/**
|
||||
* Decode encoded image bytes (JPEG, PNG, WebP, GIF's first frame, BMP)
|
||||
* to [widthLo, widthHi, heightLo, heightHi, R, G, B, A, ...] at native
|
||||
* size, not premultiplied, so a transparent pixel keeps its colour;
|
||||
* empty when undecodable. The EvenHub host converts these to grey the
|
||||
* way the Even app does (app/apps/evenhub/image-pipeline.ts).
|
||||
*/
|
||||
@JvmStatic
|
||||
fun decodeRgbaFromBytes(buffer: java.nio.ByteBuffer?): ByteArray {
|
||||
if (buffer == null) return ByteArray(0)
|
||||
try {
|
||||
val source = buffer.duplicate()
|
||||
source.rewind()
|
||||
val data = ByteArray(source.remaining())
|
||||
source.get(data)
|
||||
if (data.isEmpty()) return ByteArray(0)
|
||||
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
|
||||
BitmapFactory.decodeByteArray(data, 0, data.size, bounds)
|
||||
if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return ByteArray(0)
|
||||
var sample = 1
|
||||
while (maxOf(bounds.outWidth, bounds.outHeight) / sample > MAX_RGBA_SIDE) sample *= 2
|
||||
val options = BitmapFactory.Options().apply {
|
||||
inPreferredConfig = Bitmap.Config.ARGB_8888
|
||||
inPremultiplied = false
|
||||
inSampleSize = sample
|
||||
}
|
||||
val bitmap = BitmapFactory.decodeByteArray(data, 0, data.size, options) ?: return ByteArray(0)
|
||||
val width = bitmap.width
|
||||
val height = bitmap.height
|
||||
val argb = IntArray(width * height)
|
||||
bitmap.getPixels(argb, 0, width, 0, 0, width, height)
|
||||
bitmap.recycle()
|
||||
val packet = ByteArray(4 + argb.size * 4)
|
||||
packet[0] = (width and 0xff).toByte()
|
||||
packet[1] = ((width shr 8) and 0xff).toByte()
|
||||
packet[2] = (height and 0xff).toByte()
|
||||
packet[3] = ((height shr 8) and 0xff).toByte()
|
||||
for (i in argb.indices) {
|
||||
val pixel = argb[i]
|
||||
val o = 4 + i * 4
|
||||
packet[o] = (pixel shr 16).toByte()
|
||||
packet[o + 1] = (pixel shr 8).toByte()
|
||||
packet[o + 2] = pixel.toByte()
|
||||
packet[o + 3] = (pixel ushr 24).toByte()
|
||||
}
|
||||
return packet
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "RGBA image decode failed", e)
|
||||
return ByteArray(0)
|
||||
} catch (e: OutOfMemoryError) {
|
||||
Log.w(TAG, "RGBA image decode failed", e)
|
||||
return ByteArray(0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale a bitmap to fit within maxWidth x maxHeight (preserving aspect,
|
||||
* never upscaling) and convert it to the grayscale packet format.
|
||||
|
||||
Reference in new issue
Block a user