Merge upstream main preserving custom feature integrations

This commit is contained in:
MasterControl committed 2026-11-10 17:08:15 +00:00
commit 6150a1ac57
228 files changed
+16844 -1388

No files matched your search

+3
View File
@@ -19,6 +19,9 @@ dependencies {
// app's captions (translation models download on demand, per language).
implementation 'com.google.mlkit:language-id:17.0.6'
implementation 'com.google.mlkit:translate:17.0.3'
// Pure-Java Opus decoder for the RayNeo iO's microphone (16 kHz mono Opus over
// BLE); the same one Glassly uses. See AndroidSpeechEngines.OpusDecoderAdapter.
implementation 'io.github.jaredmdobson:concentus:1.0.1'
}
def faceclawSherpaOnnxVersion = "1.13.0"
@@ -46,6 +46,15 @@ class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLi
override fun isBonded(address: String): Boolean = bleManager.isBonded(address)
override fun requestBond(address: String) = bleManager.requestBond(address)
override fun removeBond(address: String): Boolean = bleManager.removeBond(address)
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean =
bleManager.hasCharacteristic(address, characteristicUuid)
override fun refreshServices(address: String, timeoutMs: Int): Boolean = bleManager.refreshServices(address, timeoutMs)
override fun prepareBenchmarkLink(address: String, mode: Int) = bleManager.prepareBenchmarkLink(address, mode)
override fun recordDisplayFrameSent() = FaceclawBleManager.recordDisplayFrameSent()
@@ -97,6 +97,44 @@ internal object AndroidSpeechEngines {
override fun close() = decoder.close()
}
/**
* The RayNeo iO's microphone: one 16 kHz mono Opus packet per BLE message (its remote
* audio stream, GlassesSessionIo), decoded with Concentus. No G2 DSP metadata; packets
* carry no counter, so "missing" stays 0 and a lost one is just a gap.
*/
class OpusDecoderAdapter : AudioPacketDecoder {
private val decoder = io.github.jaredmdobson.OpusDecoder(RayNeoIoProtocol.VOICE_SAMPLE_RATE, 1)
private var packets = 0L
private var errors = 0L
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
if (packet == null || packet.isEmpty() || pcmOut == null) return 0
return try {
val count = decoder.decode(packet, 0, packet.size, pcmOut, 0, pcmOut.size, false)
packets++
count
} catch (e: io.github.jaredmdobson.OpusException) {
errors++
if (errors == 1L || errors % 100 == 0L) {
android.util.Log.w("FaceclawVoice", "Opus decode error ($errors so far): ${e.message}")
}
0
}
}
/** 120 ms, Opus's longest frame. */
override val maxSamplesPerPacket: Int get() = RayNeoIoProtocol.VOICE_SAMPLE_RATE * 120 / 1000
override val hasFrameMeta: Boolean get() = false
override val lastAngleDegrees: Int get() = 0
override val lastSsr: Int get() = 0
override val realPackets: Long get() = packets
override val duplicatePackets: Long get() = 0
override val missingPackets: Long get() = 0
override val decodeErrors: Long get() = errors
override fun close() = decoder.resetState()
}
/**
* Open the phone's own microphone at the pipeline's native format (16 kHz
* mono PCM16), or null when it cannot start — the permission is missing
@@ -48,6 +48,17 @@ class AndroidStockLink(context: Context) : StockLink, FaceclawBleListener {
else -> BondState.UNKNOWN
}
override fun negotiatedMtu(address: String): Int = bleManager.getNegotiatedMtu(address)
override fun requestBond(address: String) = bleManager.requestBond(address)
override fun removeBond(address: String): Boolean = bleManager.removeBond(address)
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean =
bleManager.hasCharacteristic(address, characteristicUuid)
override fun refreshServices(address: String, timeoutMs: Int): Boolean = bleManager.refreshServices(address, timeoutMs)
override fun disconnect(address: String) = bleManager.disconnect(address)
override fun close() = bleManager.close()
@@ -189,6 +189,28 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision)
/**
* Drive a RayNeo iO (constructed with its address as both arms and no ring). The phone's
* protocol identity is derived from ANDROID_ID so it survives reinstalls: the glasses bind
* to the first identity that authenticates and refuse others until put in pairing mode.
*/
fun useRayNeoIo() {
val androidId = try {
android.provider.Settings.Secure.getString(appContext.contentResolver, android.provider.Settings.Secure.ANDROID_ID)
} catch (_: Exception) {
null
}
val seed = "faceclaw-rayneo-phone-id:" + (androidId ?: android.os.Build.FINGERPRINT)
val phoneId = java.security.MessageDigest.getInstance("SHA-256").digest(seed.toByteArray(Charsets.UTF_8)).copyOf(6)
val model = android.os.Build.MODEL ?: "Android"
core.useRayNeoIo(phoneId, model, android.os.Build.MANUFACTURER ?: "Android", model)
}
fun isRayNeoIo(): Boolean = core.isRayNeoIo()
/** Drive an Even G1 on g1v0 (two arms, as a G2 pair's addresses). */
fun useG1() = core.useG1()
fun start() {
if (core.start()) {
activeInstance = this
@@ -235,6 +257,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled)
/** See [GlassesSessionCore.setVoiceWakeEnabled]. */
fun setVoiceWakeEnabled(enabled: Boolean) = core.setVoiceWakeEnabled(enabled)
fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs)
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq)
@@ -27,6 +27,11 @@ import java.util.concurrent.atomic.AtomicLong
class FaceclawBleManager(context: Context) {
companion object {
private const val TAG = "FaceclawBle"
/** Not public API, but long stable, and the only way short of sending the user to Settings. */
fun removeBond(device: BluetoothDevice): Boolean =
device.bondState == BluetoothDevice.BOND_NONE || device.javaClass.getMethod("removeBond").invoke(device) == true
private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200)
// Gap after each LE 2M PHY request before checking whether it landed and asking
@@ -160,6 +165,36 @@ class FaceclawBleManager(context: Context) {
}
}
/**
* Remove this phone's half of a pairing the other side no longer has. False when the platform refused.
*/
fun removeBond(address: String?): Boolean {
if (address == null || address.trim().isEmpty()) {
return false
}
return try {
removeBond(bluetoothAdapter.getRemoteDevice(address))
} catch (t: Throwable) {
Log.w(TAG, "removeBond failed: $address", t)
false
}
}
/** Start bonding, without waiting: the OS may ask the user, and [isBonded] tells when it is done. */
fun requestBond(address: String?) {
if (address == null || address.trim().isEmpty()) {
return
}
try {
val device = bluetoothAdapter.getRemoteDevice(address)
if (device.bondState == BluetoothDevice.BOND_NONE) {
Log.i(TAG, "requestBond " + address + ": " + device.createBond())
}
} catch (t: Throwable) {
Log.w(TAG, "requestBond failed: " + address, t)
}
}
/**
* Android's bond state for this address (BluetoothDevice.BOND_NONE /
* BOND_BONDING / BOND_BONDED), or -1 when it cannot be determined.
@@ -364,6 +399,31 @@ class FaceclawBleManager(context: Context) {
return status == BluetoothGatt.GATT_SUCCESS
}
/** After discovery: whether the service table has this characteristic. */
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean {
synchronized(bluetoothApiLock) {
val gatt = links[address]?.gatt ?: return false
val target = UUID.fromString(characteristicUuid)
return gatt.services.any { it.getCharacteristic(target) != null }
}
}
/**
* Forget the service table Android remembers for a bonded device and read it again. A G1 arm keeps its address across an
* install that changes its services, and a cached table would make new firmware look like the old one.
*/
fun refreshServices(address: String, timeoutMs: Int): Boolean {
synchronized(bluetoothApiLock) {
val gatt = links[address]?.gatt ?: return false
try {
gatt.javaClass.getMethod("refresh").invoke(gatt)
} catch (t: Throwable) {
Log.w(TAG, "refresh failed: $address", t)
}
}
return discoverServices(address, timeoutMs)
}
fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean {
synchronized(bluetoothApiLock) {
val gatt = requireGatt(address)
@@ -129,13 +129,21 @@ class FaceclawDeviceDiscovery(context: Context) {
&& manufacturerData[0] == 0x45.toByte() && manufacturerData[1] == 0x52.toByte()) {
return true
}
// RayNeo iO: manufacturer company 0x5716 ([07][MAC]...) outside pairing mode;
// in pairing mode only the "RayNeo iO-xxxx" name identifies it.
if (manufacturerData != null && manufacturerData.size >= 2
&& manufacturerData[0] == 0x16.toByte() && manufacturerData[1] == 0x57.toByte()) {
return true
}
if (name == null) {
return false
}
val upper = name.uppercase()
// A bare "R1" substring also matches unrelated hardware ("Oppo Enco
// R1"), and TS only classifies rings from the stock name prefix.
return upper.contains("G2") || upper.startsWith("EVEN R1")
// Even G1: its stock name, or g1v0's G1CFW_L_XXXXXX (before 0.5.66: G1v0_L, G1v0S_R).
return upper.contains("G2") || upper.startsWith("EVEN R1") || upper.startsWith("RAYNEO") ||
upper.startsWith("EVEN G1") || upper.startsWith("G1CFW_") || upper.startsWith("G1V0")
}
private fun buildJson(
@@ -0,0 +1,106 @@
package com.faceclaw.app
import android.content.Context
import android.os.Handler
import android.os.Looper
/**
* Android host for the shared [G1ArmFlashFlow]: owns the GATT link and the worker thread for one G1 arm's install, its
* restore to Even's firmware, or a question about what it runs, and delivers listener callbacks on the main thread. For app/native/g1-flasher.ts.
*/
class FaceclawG1Flasher(
context: Context,
address: String?,
side: String?,
firmwarePath: String?,
netCorePath: String?,
mode: String?,
connectWindowMs: Int,
) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
@Volatile
private var listener: FaceclawFirmwareFlasherListener? = null
@Volatile
private var worker: Thread? = null
private val flow =
G1ArmFlashFlow(
link,
address ?: "",
side ?: "",
firmwarePath ?: "",
netCorePath ?: "",
G1ArmFlashFlow.mode(mode ?: ""),
connectWindowMs,
object : FaceclawFirmwareFlasherListener {
override fun onLog(line: String?) {
android.util.Log.i(G1ArmFlashFlow.TAG, line ?: "")
post { it.onLog(line) }
}
override fun onProgress(
lens: String?,
componentIndex: Int,
componentCount: Int,
blockIndex: Int,
blockCount: Int,
bytesSent: Long,
bytesTotal: Long,
) = post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
},
)
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
this.listener = listener
}
fun start() {
synchronized(lock) {
if (worker != null) {
return
}
val w = Thread({ flow.run() }, "faceclaw-g1-flash")
worker = w
w.start()
}
}
fun cancel() {
flow.cancel()
}
fun close() {
cancel()
try {
link.close()
} catch (ignored: Exception) {
}
}
companion object {
/** The phone's pairing with an address these glasses no longer use. */
@JvmStatic
fun forgetPairing(context: Context, address: String?) {
if (address == null) return
val manager = context.applicationContext.getSystemService(Context.BLUETOOTH_SERVICE) as android.bluetooth.BluetoothManager
FaceclawBleManager.removeBond(manager.adapter.getRemoteDevice(address))
}
}
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
action(current)
}
}
}
}
@@ -104,7 +104,9 @@ class FaceclawVoiceController(context: Context) {
}
}
override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter()
override fun createPacketDecoder(): AudioPacketDecoder =
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
else AndroidSpeechEngines.Lc3DecoderAdapter()
override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES)
@@ -167,6 +167,7 @@ class FontFileRenderer private constructor() {
* [width u16][height u16]
* [width*height coverage bytes, gamma-mapped]
* An ink-free glyph (space) has width = height = 0 but a valid advance.
* A color glyph (emoji) carries its ColorGlyphArt shade as its coverage.
* Empty array when the font cannot be loaded or the render fails.
*/
@JvmStatic
@@ -196,11 +197,16 @@ class FontFileRenderer private constructor() {
}
val penX = pad - bounds.left
val baselineY = pad - bounds.top
val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ALPHA_8)
// Drawn in color: an emoji the system fallback supplies from a color font
// would keep only its silhouette in an alpha-only bitmap. ColorGlyphArt shades
// color glyphs and passes white ones through as their alpha.
val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ARGB_8888)
val canvas = Canvas(bitmap)
canvas.drawText(text, penX.toFloat(), baselineY.toFloat(), paint)
val pixels = alpha8Pixels(bitmap, bw, bh)
val argb = IntArray(bw * bh)
bitmap.getPixels(argb, 0, bw, 0, 0, bw, bh)
bitmap.recycle()
val pixels = ColorGlyphArt.coverage(argb)
// Trim to the tight ink box; bearing/inkTop become pen- and
// line-relative inside the shared packer.
@@ -108,6 +108,64 @@ class ImageFileLoader private constructor() {
}
}
/** Longest side decodeRgbaFromBytes decodes at; larger images are subsampled. */
private const val MAX_RGBA_SIDE = 4096
/**
* Decode encoded image bytes (JPEG, PNG, WebP, GIF's first frame, BMP)
* to [widthLo, widthHi, heightLo, heightHi, R, G, B, A, ...] at native
* size, not premultiplied, so a transparent pixel keeps its colour;
* empty when undecodable. The EvenHub host converts these to grey the
* way the Even app does (app/apps/evenhub/image-pipeline.ts).
*/
@JvmStatic
fun decodeRgbaFromBytes(buffer: java.nio.ByteBuffer?): ByteArray {
if (buffer == null) return ByteArray(0)
try {
val source = buffer.duplicate()
source.rewind()
val data = ByteArray(source.remaining())
source.get(data)
if (data.isEmpty()) return ByteArray(0)
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(data, 0, data.size, bounds)
if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return ByteArray(0)
var sample = 1
while (maxOf(bounds.outWidth, bounds.outHeight) / sample > MAX_RGBA_SIDE) sample *= 2
val options = BitmapFactory.Options().apply {
inPreferredConfig = Bitmap.Config.ARGB_8888
inPremultiplied = false
inSampleSize = sample
}
val bitmap = BitmapFactory.decodeByteArray(data, 0, data.size, options) ?: return ByteArray(0)
val width = bitmap.width
val height = bitmap.height
val argb = IntArray(width * height)
bitmap.getPixels(argb, 0, width, 0, 0, width, height)
bitmap.recycle()
val packet = ByteArray(4 + argb.size * 4)
packet[0] = (width and 0xff).toByte()
packet[1] = ((width shr 8) and 0xff).toByte()
packet[2] = (height and 0xff).toByte()
packet[3] = ((height shr 8) and 0xff).toByte()
for (i in argb.indices) {
val pixel = argb[i]
val o = 4 + i * 4
packet[o] = (pixel shr 16).toByte()
packet[o + 1] = (pixel shr 8).toByte()
packet[o + 2] = pixel.toByte()
packet[o + 3] = (pixel ushr 24).toByte()
}
return packet
} catch (e: Exception) {
Log.w(TAG, "RGBA image decode failed", e)
return ByteArray(0)
} catch (e: OutOfMemoryError) {
Log.w(TAG, "RGBA image decode failed", e)
return ByteArray(0)
}
}
/**
* Scale a bitmap to fit within maxWidth x maxHeight (preserving aspect,
* never upscaling) and convert it to the grayscale packet format.