forked from Blendman974/faceclaw
Merge branch 'main' into glanceboard-tilt-angle-and-glance-down
This commit is contained in:
commit
37d81d0308
416 files changed
+34629
-3885
No files matched your search
@@ -19,6 +19,9 @@ dependencies {
|
||||
// app's captions (translation models download on demand, per language).
|
||||
implementation 'com.google.mlkit:language-id:17.0.6'
|
||||
implementation 'com.google.mlkit:translate:17.0.3'
|
||||
// Pure-Java Opus decoder for the RayNeo iO's microphone (16 kHz mono Opus over
|
||||
// BLE); the same one Glassly uses. See AndroidSpeechEngines.OpusDecoderAdapter.
|
||||
implementation 'io.github.jaredmdobson:concentus:1.0.1'
|
||||
}
|
||||
|
||||
def faceclawSherpaOnnxVersion = "1.13.0"
|
||||
@@ -261,15 +264,21 @@ if (faceclawVersionFile.exists()) {
|
||||
faceclawVersionName = faceclawVersionMatch.group(1)
|
||||
}
|
||||
}
|
||||
def faceclawBuildRevision = System.getenv("FACECLAW_ANDROID_BUILD_REVISION") ?: "0"
|
||||
if (!(faceclawBuildRevision ==~ /[0-4]/)) {
|
||||
throw new GradleException("FACECLAW_ANDROID_BUILD_REVISION must be an integer from 0 to 4")
|
||||
}
|
||||
def faceclawVersionCode = {
|
||||
// Prerelease labels belong in versionName, not the numeric versionCode.
|
||||
def parts = (faceclawVersionName.split('-')[0].tokenize('.') + ["0", "0"]).take(3)*.toInteger()
|
||||
return parts[0] * 10000 + parts[1] * 100 + parts[2]
|
||||
// Last digit: build revision * 2, plus form factor (phone 0, Wear OS 1).
|
||||
// Play requires distinct version codes even on separate form-factor tracks.
|
||||
return (parts[0] * 10000 + parts[1] * 100 + parts[2]) * 10 + faceclawBuildRevision.toInteger() * 2
|
||||
}()
|
||||
|
||||
android {
|
||||
compileSdkVersion 35
|
||||
buildToolsVersion "35"
|
||||
compileSdkVersion 36
|
||||
buildToolsVersion "36.0.0"
|
||||
// ndkVersion ""
|
||||
|
||||
sourceSets.main {
|
||||
@@ -279,7 +288,7 @@ android {
|
||||
|
||||
defaultConfig {
|
||||
minSdkVersion 24
|
||||
targetSdkVersion 35
|
||||
targetSdkVersion 36
|
||||
ndk {
|
||||
// The NativeScript platform build.gradle has already added all four
|
||||
// ABIs by the time app.gradle is applied, and abiFilters is additive,
|
||||
|
||||
@@ -46,6 +46,15 @@ class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLi
|
||||
|
||||
override fun isBonded(address: String): Boolean = bleManager.isBonded(address)
|
||||
|
||||
override fun requestBond(address: String) = bleManager.requestBond(address)
|
||||
|
||||
override fun removeBond(address: String): Boolean = bleManager.removeBond(address)
|
||||
|
||||
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean =
|
||||
bleManager.hasCharacteristic(address, characteristicUuid)
|
||||
|
||||
override fun refreshServices(address: String, timeoutMs: Int): Boolean = bleManager.refreshServices(address, timeoutMs)
|
||||
|
||||
override fun prepareBenchmarkLink(address: String, mode: Int) = bleManager.prepareBenchmarkLink(address, mode)
|
||||
|
||||
override fun recordDisplayFrameSent() = FaceclawBleManager.recordDisplayFrameSent()
|
||||
|
||||
@@ -94,6 +94,44 @@ internal object AndroidSpeechEngines {
|
||||
override fun close() = decoder.close()
|
||||
}
|
||||
|
||||
/**
|
||||
* The RayNeo iO's microphone: one 16 kHz mono Opus packet per BLE message (its remote
|
||||
* audio stream, GlassesSessionIo), decoded with Concentus. No G2 DSP metadata; packets
|
||||
* carry no counter, so "missing" stays 0 and a lost one is just a gap.
|
||||
*/
|
||||
class OpusDecoderAdapter : AudioPacketDecoder {
|
||||
private val decoder = io.github.jaredmdobson.OpusDecoder(RayNeoIoProtocol.VOICE_SAMPLE_RATE, 1)
|
||||
private var packets = 0L
|
||||
private var errors = 0L
|
||||
|
||||
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
|
||||
if (packet == null || packet.isEmpty() || pcmOut == null) return 0
|
||||
return try {
|
||||
val count = decoder.decode(packet, 0, packet.size, pcmOut, 0, pcmOut.size, false)
|
||||
packets++
|
||||
count
|
||||
} catch (e: io.github.jaredmdobson.OpusException) {
|
||||
errors++
|
||||
if (errors == 1L || errors % 100 == 0L) {
|
||||
android.util.Log.w("FaceclawVoice", "Opus decode error ($errors so far): ${e.message}")
|
||||
}
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/** 120 ms, Opus's longest frame. */
|
||||
override val maxSamplesPerPacket: Int get() = RayNeoIoProtocol.VOICE_SAMPLE_RATE * 120 / 1000
|
||||
override val hasFrameMeta: Boolean get() = false
|
||||
override val lastAngleDegrees: Int get() = 0
|
||||
override val lastSsr: Int get() = 0
|
||||
override val realPackets: Long get() = packets
|
||||
override val duplicatePackets: Long get() = 0
|
||||
override val missingPackets: Long get() = 0
|
||||
override val decodeErrors: Long get() = errors
|
||||
|
||||
override fun close() = decoder.resetState()
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the phone's own microphone at the pipeline's native format (16 kHz
|
||||
* mono PCM16), or null when it cannot start — the permission is missing
|
||||
|
||||
@@ -48,6 +48,17 @@ class AndroidStockLink(context: Context) : StockLink, FaceclawBleListener {
|
||||
else -> BondState.UNKNOWN
|
||||
}
|
||||
|
||||
override fun negotiatedMtu(address: String): Int = bleManager.getNegotiatedMtu(address)
|
||||
|
||||
override fun requestBond(address: String) = bleManager.requestBond(address)
|
||||
|
||||
override fun removeBond(address: String): Boolean = bleManager.removeBond(address)
|
||||
|
||||
override fun hasCharacteristic(address: String, characteristicUuid: String): Boolean =
|
||||
bleManager.hasCharacteristic(address, characteristicUuid)
|
||||
|
||||
override fun refreshServices(address: String, timeoutMs: Int): Boolean = bleManager.refreshServices(address, timeoutMs)
|
||||
|
||||
override fun disconnect(address: String) = bleManager.disconnect(address)
|
||||
|
||||
override fun close() = bleManager.close()
|
||||
|
||||
@@ -189,6 +189,28 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision)
|
||||
|
||||
/**
|
||||
* Drive a RayNeo iO (constructed with its address as both arms and no ring). The phone's
|
||||
* protocol identity is derived from ANDROID_ID so it survives reinstalls: the glasses bind
|
||||
* to the first identity that authenticates and refuse others until put in pairing mode.
|
||||
*/
|
||||
fun useRayNeoIo() {
|
||||
val androidId = try {
|
||||
android.provider.Settings.Secure.getString(appContext.contentResolver, android.provider.Settings.Secure.ANDROID_ID)
|
||||
} catch (_: Exception) {
|
||||
null
|
||||
}
|
||||
val seed = "faceclaw-rayneo-phone-id:" + (androidId ?: android.os.Build.FINGERPRINT)
|
||||
val phoneId = java.security.MessageDigest.getInstance("SHA-256").digest(seed.toByteArray(Charsets.UTF_8)).copyOf(6)
|
||||
val model = android.os.Build.MODEL ?: "Android"
|
||||
core.useRayNeoIo(phoneId, model, android.os.Build.MANUFACTURER ?: "Android", model)
|
||||
}
|
||||
|
||||
fun isRayNeoIo(): Boolean = core.isRayNeoIo()
|
||||
|
||||
/** Drive an Even G1 on g1v0 (two arms, as a G2 pair's addresses). */
|
||||
fun useG1() = core.useG1()
|
||||
|
||||
fun start() {
|
||||
if (core.start()) {
|
||||
activeInstance = this
|
||||
@@ -235,6 +257,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun setFaceclawWakeLeaseEnabled(enabled: Boolean): Boolean = core.setFaceclawWakeLeaseEnabled(enabled)
|
||||
|
||||
/** See [GlassesSessionCore.setVoiceWakeEnabled]. */
|
||||
fun setVoiceWakeEnabled(enabled: Boolean) = core.setVoiceWakeEnabled(enabled)
|
||||
|
||||
fun awaitEvenHubSessionReady(timeoutMs: Int): Boolean = core.awaitEvenHubSessionReady(timeoutMs)
|
||||
|
||||
fun setImuReportEnabled(enable: Boolean, reportFrq: Int) = core.setImuReportEnabled(enable, reportFrq)
|
||||
@@ -304,6 +329,13 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
core.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
|
||||
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.addKeyboardEventListener(listener)
|
||||
|
||||
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
|
||||
|
||||
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean =
|
||||
core.sendFaceclawKeyboardControl(record, label)
|
||||
|
||||
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener)
|
||||
|
||||
fun stopG2AudioForwarding() = core.stopG2AudioForwarding()
|
||||
|
||||
@@ -27,6 +27,11 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class FaceclawBleManager(context: Context) {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawBle"
|
||||
|
||||
/** Not public API, but long stable, and the only way short of sending the user to Settings. */
|
||||
fun removeBond(device: BluetoothDevice): Boolean =
|
||||
device.bondState == BluetoothDevice.BOND_NONE || device.javaClass.getMethod("removeBond").invoke(device) == true
|
||||
|
||||
private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200)
|
||||
|
||||
// Gap after each LE 2M PHY request before checking whether it landed and asking
|
||||
@@ -160,6 +165,36 @@ class FaceclawBleManager(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove this phone's half of a pairing the other side no longer has. False when the platform refused.
|
||||
*/
|
||||
fun removeBond(address: String?): Boolean {
|
||||
if (address == null || address.trim().isEmpty()) {
|
||||
return false
|
||||
}
|
||||
return try {
|
||||
removeBond(bluetoothAdapter.getRemoteDevice(address))
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "removeBond failed: $address", t)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/** Start bonding, without waiting: the OS may ask the user, and [isBonded] tells when it is done. */
|
||||
fun requestBond(address: String?) {
|
||||
if (address == null || address.trim().isEmpty()) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
val device = bluetoothAdapter.getRemoteDevice(address)
|
||||
if (device.bondState == BluetoothDevice.BOND_NONE) {
|
||||
Log.i(TAG, "requestBond " + address + ": " + device.createBond())
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "requestBond failed: " + address, t)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Android's bond state for this address (BluetoothDevice.BOND_NONE /
|
||||
* BOND_BONDING / BOND_BONDED), or -1 when it cannot be determined.
|
||||
@@ -364,6 +399,31 @@ class FaceclawBleManager(context: Context) {
|
||||
return status == BluetoothGatt.GATT_SUCCESS
|
||||
}
|
||||
|
||||
/** After discovery: whether the service table has this characteristic. */
|
||||
fun hasCharacteristic(address: String, characteristicUuid: String): Boolean {
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = links[address]?.gatt ?: return false
|
||||
val target = UUID.fromString(characteristicUuid)
|
||||
return gatt.services.any { it.getCharacteristic(target) != null }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Forget the service table Android remembers for a bonded device and read it again. A G1 arm keeps its address across an
|
||||
* install that changes its services, and a cached table would make new firmware look like the old one.
|
||||
*/
|
||||
fun refreshServices(address: String, timeoutMs: Int): Boolean {
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = links[address]?.gatt ?: return false
|
||||
try {
|
||||
gatt.javaClass.getMethod("refresh").invoke(gatt)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "refresh failed: $address", t)
|
||||
}
|
||||
}
|
||||
return discoverServices(address, timeoutMs)
|
||||
}
|
||||
|
||||
fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean {
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = requireGatt(address)
|
||||
|
||||
@@ -129,13 +129,21 @@ class FaceclawDeviceDiscovery(context: Context) {
|
||||
&& manufacturerData[0] == 0x45.toByte() && manufacturerData[1] == 0x52.toByte()) {
|
||||
return true
|
||||
}
|
||||
// RayNeo iO: manufacturer company 0x5716 ([07][MAC]...) outside pairing mode;
|
||||
// in pairing mode only the "RayNeo iO-xxxx" name identifies it.
|
||||
if (manufacturerData != null && manufacturerData.size >= 2
|
||||
&& manufacturerData[0] == 0x16.toByte() && manufacturerData[1] == 0x57.toByte()) {
|
||||
return true
|
||||
}
|
||||
if (name == null) {
|
||||
return false
|
||||
}
|
||||
val upper = name.uppercase()
|
||||
// A bare "R1" substring also matches unrelated hardware ("Oppo Enco
|
||||
// R1"), and TS only classifies rings from the stock name prefix.
|
||||
return upper.contains("G2") || upper.startsWith("EVEN R1")
|
||||
// Even G1: its stock name, or g1v0's G1CFW_L_XXXXXX (before 0.5.66: G1v0_L, G1v0S_R).
|
||||
return upper.contains("G2") || upper.startsWith("EVEN R1") || upper.startsWith("RAYNEO") ||
|
||||
upper.startsWith("EVEN G1") || upper.startsWith("G1CFW_") || upper.startsWith("G1V0")
|
||||
}
|
||||
|
||||
private fun buildJson(
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
|
||||
/**
|
||||
* Android host for the shared [G1ArmFlashFlow]: owns the GATT link and the worker thread for one G1 arm's install, its
|
||||
* restore to Even's firmware, or a question about what it runs, and delivers listener callbacks on the main thread. For app/native/g1-flasher.ts.
|
||||
*/
|
||||
class FaceclawG1Flasher(
|
||||
context: Context,
|
||||
address: String?,
|
||||
side: String?,
|
||||
firmwarePath: String?,
|
||||
netCorePath: String?,
|
||||
mode: String?,
|
||||
connectWindowMs: Int,
|
||||
) {
|
||||
private val link = AndroidStockLink(context.applicationContext)
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val lock = Any()
|
||||
|
||||
@Volatile
|
||||
private var listener: FaceclawFirmwareFlasherListener? = null
|
||||
|
||||
@Volatile
|
||||
private var worker: Thread? = null
|
||||
|
||||
private val flow =
|
||||
G1ArmFlashFlow(
|
||||
link,
|
||||
address ?: "",
|
||||
side ?: "",
|
||||
firmwarePath ?: "",
|
||||
netCorePath ?: "",
|
||||
G1ArmFlashFlow.mode(mode ?: ""),
|
||||
connectWindowMs,
|
||||
object : FaceclawFirmwareFlasherListener {
|
||||
override fun onLog(line: String?) {
|
||||
android.util.Log.i(G1ArmFlashFlow.TAG, line ?: "")
|
||||
post { it.onLog(line) }
|
||||
}
|
||||
|
||||
override fun onProgress(
|
||||
lens: String?,
|
||||
componentIndex: Int,
|
||||
componentCount: Int,
|
||||
blockIndex: Int,
|
||||
blockCount: Int,
|
||||
bytesSent: Long,
|
||||
bytesTotal: Long,
|
||||
) = post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
|
||||
|
||||
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
|
||||
|
||||
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
|
||||
},
|
||||
)
|
||||
|
||||
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
|
||||
this.listener = listener
|
||||
}
|
||||
|
||||
fun start() {
|
||||
synchronized(lock) {
|
||||
if (worker != null) {
|
||||
return
|
||||
}
|
||||
val w = Thread({ flow.run() }, "faceclaw-g1-flash")
|
||||
worker = w
|
||||
w.start()
|
||||
}
|
||||
}
|
||||
|
||||
fun cancel() {
|
||||
flow.cancel()
|
||||
}
|
||||
|
||||
fun close() {
|
||||
cancel()
|
||||
try {
|
||||
link.close()
|
||||
} catch (ignored: Exception) {
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** The phone's pairing with an address these glasses no longer use. */
|
||||
@JvmStatic
|
||||
fun forgetPairing(context: Context, address: String?) {
|
||||
if (address == null) return
|
||||
val manager = context.applicationContext.getSystemService(Context.BLUETOOTH_SERVICE) as android.bluetooth.BluetoothManager
|
||||
FaceclawBleManager.removeBond(manager.adapter.getRemoteDevice(address))
|
||||
}
|
||||
}
|
||||
|
||||
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) {
|
||||
mainHandler.post {
|
||||
val current = listener
|
||||
if (current != null) {
|
||||
action(current)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,24 @@ class FaceclawVoiceController(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
override fun createPacketDecoder(): AudioPacketDecoder = AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
override fun micArrayLink(): MicArrayLink? {
|
||||
val currentCommunicator = communicator ?: return null
|
||||
return object : MicArrayLink {
|
||||
override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean =
|
||||
currentCommunicator.startG2AudioForwarding(listener)
|
||||
|
||||
override fun stopForwarding() = currentCommunicator.stopG2AudioForwarding()
|
||||
|
||||
override fun isForwarding(): Boolean = currentCommunicator.isAudioCaptureActive()
|
||||
|
||||
override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
currentCommunicator.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
}
|
||||
}
|
||||
|
||||
override fun createPacketDecoder(): AudioPacketDecoder =
|
||||
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
|
||||
else AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
|
||||
override fun openPhoneMic(): PcmAudioSource? = AndroidSpeechEngines.openPhoneMic(PHONE_MIC_CHUNK_SAMPLES)
|
||||
|
||||
@@ -134,12 +151,15 @@ class FaceclawVoiceController(context: Context) {
|
||||
override fun release() {}
|
||||
}
|
||||
}
|
||||
|
||||
override fun recordingSink(): RecordingSink = RecordingSink { pcmBytes, sampleRate -> saveRecording(pcmBytes, sampleRate) }
|
||||
}
|
||||
|
||||
private val session = VoiceCaptureSession(host)
|
||||
|
||||
// The recording of this controller's live captures (see startRecording). It outlives any one
|
||||
// capture, and is only handed to the session while recording is enabled.
|
||||
private var recording: MicRecordingWriter? = null
|
||||
private var recordingEnabled = true
|
||||
|
||||
fun setListener(listener: FaceclawVoiceControllerListener?) {
|
||||
session.setListener(listener)
|
||||
}
|
||||
@@ -153,11 +173,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setUsePhoneMic(usePhoneMic)
|
||||
}
|
||||
|
||||
/** When true, the decoded mic PCM for each session is saved as a WAV. */
|
||||
fun setSaveRecordings(saveRecordings: Boolean) {
|
||||
session.setSaveRecordings(saveRecordings)
|
||||
}
|
||||
|
||||
/**
|
||||
* Which on-device model {@link #start}("onboard") should load: "whisper"
|
||||
* selects the second on-device model (sherpa-onnx offline Whisper); any
|
||||
@@ -206,6 +221,88 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setBeamFilter(enabled, centerDeg, halfWidthDeg)
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture glasses sessions from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the stock mixed stream. [configureRecord] is the CONFIGURE
|
||||
* record to arm both temples with; null goes back to the mixed stream. Must be set before
|
||||
* {@link #start}.
|
||||
*/
|
||||
fun setMicArray(configureRecord: ByteArray?) {
|
||||
session.setMicArray(configureRecord)
|
||||
}
|
||||
|
||||
/**
|
||||
* Have the next {@link #start} play the recording at [path] (a .mka file) from [startMs] in
|
||||
* place of any microphone; null goes back to the microphones. Progress arrives through the
|
||||
* listener's onReplay.
|
||||
*/
|
||||
fun setReplay(path: String?, startMs: Long) {
|
||||
session.setReplay(path, startMs)
|
||||
}
|
||||
|
||||
/** Continue the running replay from [positionMs] into its recording. */
|
||||
fun seekReplay(positionMs: Long) {
|
||||
session.seekReplay(positionMs)
|
||||
}
|
||||
|
||||
fun setReplayPaused(paused: Boolean) {
|
||||
session.setReplayPaused(paused)
|
||||
}
|
||||
|
||||
/**
|
||||
* Record live captures to a new Matroska audio file at [path], from now until
|
||||
* {@link #stopRecording}, across however many captures run in between: every source's audio
|
||||
* as it arrives, before any processing. [configJson] describes the capture and is stored in
|
||||
* the file. Returns false when the file cannot be created.
|
||||
*/
|
||||
@Synchronized
|
||||
fun startRecording(path: String, configJson: String?): Boolean {
|
||||
stopRecording()
|
||||
return try {
|
||||
val parent = File(path).parentFile
|
||||
if (parent != null && !parent.exists()) parent.mkdirs()
|
||||
recording = MicRecordingWriter(openFileOutput(path), configJson, System.currentTimeMillis())
|
||||
session.setRecorder(if (recordingEnabled) recording else null)
|
||||
true
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "could not create recording $path", t)
|
||||
recording = null
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether captures are written to the recording right now: off while the capture running
|
||||
* isn't one that is to be recorded.
|
||||
*/
|
||||
@Synchronized
|
||||
fun setRecordingEnabled(enabled: Boolean) {
|
||||
recordingEnabled = enabled
|
||||
session.setRecorder(if (enabled) recording else null)
|
||||
}
|
||||
|
||||
/** Whether a recording is open and still being written (false once writing to it failed). */
|
||||
@Synchronized
|
||||
fun isRecording(): Boolean = recording?.failed == false
|
||||
|
||||
/**
|
||||
* Note a change in what is being captured (a JSON object) at the recording's current
|
||||
* position; the audio after it is placed afresh on the recording's timeline.
|
||||
*/
|
||||
@Synchronized
|
||||
fun noteRecordingEvent(json: String) {
|
||||
recording?.event(json)
|
||||
}
|
||||
|
||||
/** Complete the recording's file. Returns its length in ms, or -1 when none was open. */
|
||||
@Synchronized
|
||||
fun stopRecording(): Long {
|
||||
val current = recording ?: return -1
|
||||
recording = null
|
||||
session.setRecorder(null)
|
||||
return current.finish()
|
||||
}
|
||||
|
||||
fun start(requestedMode: String?) {
|
||||
session.start(requestedMode, 0)
|
||||
}
|
||||
@@ -231,29 +328,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
stop()
|
||||
}
|
||||
|
||||
/** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */
|
||||
private fun saveRecording(pcmBytes: ByteArray, sampleRate: Int) {
|
||||
try {
|
||||
val dir = File(appContext.getExternalFilesDir(null), "voice-recordings")
|
||||
if (!dir.exists() && !dir.mkdirs()) {
|
||||
Log.w(TAG, "could not create voice-recordings dir")
|
||||
return
|
||||
}
|
||||
val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US)
|
||||
.format(java.util.Date())
|
||||
val file = File(dir, "voice-$stamp.wav")
|
||||
java.io.FileOutputStream(file).use { fos ->
|
||||
fos.write(BinaryEncoding.wavHeader(pcmBytes.size, sampleRate, 1))
|
||||
fos.write(pcmBytes)
|
||||
}
|
||||
Log.i(TAG, "saved voice recording " + file.absolutePath
|
||||
+ " samples=" + (pcmBytes.size / 2)
|
||||
+ " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / sampleRate))
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to save voice recording", t)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The on-device model directory for the given kind, populated by the
|
||||
* download flow in asr-model.ts (releases before 0.5.0 copied the
|
||||
|
||||
@@ -167,6 +167,7 @@ class FontFileRenderer private constructor() {
|
||||
* [width u16][height u16]
|
||||
* [width*height coverage bytes, gamma-mapped]
|
||||
* An ink-free glyph (space) has width = height = 0 but a valid advance.
|
||||
* A color glyph (emoji) carries its ColorGlyphArt shade as its coverage.
|
||||
* Empty array when the font cannot be loaded or the render fails.
|
||||
*/
|
||||
@JvmStatic
|
||||
@@ -196,11 +197,16 @@ class FontFileRenderer private constructor() {
|
||||
}
|
||||
val penX = pad - bounds.left
|
||||
val baselineY = pad - bounds.top
|
||||
val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ALPHA_8)
|
||||
// Drawn in color: an emoji the system fallback supplies from a color font
|
||||
// would keep only its silhouette in an alpha-only bitmap. ColorGlyphArt shades
|
||||
// color glyphs and passes white ones through as their alpha.
|
||||
val bitmap = Bitmap.createBitmap(bw, bh, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bitmap)
|
||||
canvas.drawText(text, penX.toFloat(), baselineY.toFloat(), paint)
|
||||
val pixels = alpha8Pixels(bitmap, bw, bh)
|
||||
val argb = IntArray(bw * bh)
|
||||
bitmap.getPixels(argb, 0, bw, 0, 0, bw, bh)
|
||||
bitmap.recycle()
|
||||
val pixels = ColorGlyphArt.coverage(argb)
|
||||
|
||||
// Trim to the tight ink box; bearing/inkTop become pen- and
|
||||
// line-relative inside the shared packer.
|
||||
|
||||
@@ -108,6 +108,64 @@ class ImageFileLoader private constructor() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Longest side decodeRgbaFromBytes decodes at; larger images are subsampled. */
|
||||
private const val MAX_RGBA_SIDE = 4096
|
||||
|
||||
/**
|
||||
* Decode encoded image bytes (JPEG, PNG, WebP, GIF's first frame, BMP)
|
||||
* to [widthLo, widthHi, heightLo, heightHi, R, G, B, A, ...] at native
|
||||
* size, not premultiplied, so a transparent pixel keeps its colour;
|
||||
* empty when undecodable. The EvenHub host converts these to grey the
|
||||
* way the Even app does (app/apps/evenhub/image-pipeline.ts).
|
||||
*/
|
||||
@JvmStatic
|
||||
fun decodeRgbaFromBytes(buffer: java.nio.ByteBuffer?): ByteArray {
|
||||
if (buffer == null) return ByteArray(0)
|
||||
try {
|
||||
val source = buffer.duplicate()
|
||||
source.rewind()
|
||||
val data = ByteArray(source.remaining())
|
||||
source.get(data)
|
||||
if (data.isEmpty()) return ByteArray(0)
|
||||
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
|
||||
BitmapFactory.decodeByteArray(data, 0, data.size, bounds)
|
||||
if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return ByteArray(0)
|
||||
var sample = 1
|
||||
while (maxOf(bounds.outWidth, bounds.outHeight) / sample > MAX_RGBA_SIDE) sample *= 2
|
||||
val options = BitmapFactory.Options().apply {
|
||||
inPreferredConfig = Bitmap.Config.ARGB_8888
|
||||
inPremultiplied = false
|
||||
inSampleSize = sample
|
||||
}
|
||||
val bitmap = BitmapFactory.decodeByteArray(data, 0, data.size, options) ?: return ByteArray(0)
|
||||
val width = bitmap.width
|
||||
val height = bitmap.height
|
||||
val argb = IntArray(width * height)
|
||||
bitmap.getPixels(argb, 0, width, 0, 0, width, height)
|
||||
bitmap.recycle()
|
||||
val packet = ByteArray(4 + argb.size * 4)
|
||||
packet[0] = (width and 0xff).toByte()
|
||||
packet[1] = ((width shr 8) and 0xff).toByte()
|
||||
packet[2] = (height and 0xff).toByte()
|
||||
packet[3] = ((height shr 8) and 0xff).toByte()
|
||||
for (i in argb.indices) {
|
||||
val pixel = argb[i]
|
||||
val o = 4 + i * 4
|
||||
packet[o] = (pixel shr 16).toByte()
|
||||
packet[o + 1] = (pixel shr 8).toByte()
|
||||
packet[o + 2] = pixel.toByte()
|
||||
packet[o + 3] = (pixel ushr 24).toByte()
|
||||
}
|
||||
return packet
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "RGBA image decode failed", e)
|
||||
return ByteArray(0)
|
||||
} catch (e: OutOfMemoryError) {
|
||||
Log.w(TAG, "RGBA image decode failed", e)
|
||||
return ByteArray(0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale a bitmap to fit within maxWidth x maxHeight (preserving aspect,
|
||||
* never upscaling) and convert it to the grayscale packet format.
|
||||
|
||||
@@ -62,7 +62,7 @@ class PreviewBitmapUtil private constructor() {
|
||||
/**
|
||||
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
|
||||
* alpha; black, which the glasses show as unlit, is transparent), for
|
||||
* showing glasses artwork on the phone's dark action bar.
|
||||
* showing glasses artwork in the phone UI, which tints it to suit.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun whiteIconFromGray(gray: ByteBuffer?, width: Int, height: Int): Bitmap {
|
||||
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Keyboard icon for the main page's tab-bar keyboard-dialog button (beside
|
||||
the mic button). Material Design "keyboard" glyph (Apache 2.0); base fill
|
||||
is overridden by tint-color in app.css. -->
|
||||
<!-- Keyboard icon for the main page's keyboard-dialog button (beside the mic
|
||||
button). Material Design "keyboard" glyph (Apache 2.0); base fill is
|
||||
overridden by tint-color in remote-controls.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Microphone icon for the main page's tab-bar wakeword button.
|
||||
<!-- Microphone icon for the main page's voice-input (wakeword) button.
|
||||
Converted from ~/Downloads/noun-microphone-1368757.svg (Noun Project
|
||||
icon #1368757); base fill is overridden by tint-color in app.css. -->
|
||||
icon #1368757); base fill is overridden by tint-color in remote-controls.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Gear icon for the main page's Settings button (beside the mic and keyboard
|
||||
buttons). Material Design "settings" glyph (Apache 2.0); base fill is
|
||||
overridden by tint-color in remote-controls.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:pathData="M19.14,12.94c0.04,-0.3 0.06,-0.61 0.06,-0.94c0,-0.32 -0.02,-0.64 -0.07,-0.94l2.03,-1.58c0.18,-0.14 0.23,-0.41 0.12,-0.61l-1.92,-3.32c-0.12,-0.22 -0.37,-0.29 -0.59,-0.22l-2.39,0.96c-0.5,-0.38 -1.03,-0.7 -1.62,-0.94L14.4,2.81c-0.04,-0.24 -0.24,-0.41 -0.48,-0.41h-3.84c-0.24,0 -0.43,0.17 -0.47,0.41L9.25,5.35C8.66,5.59 8.12,5.92 7.63,6.29L5.24,5.33c-0.22,-0.08 -0.47,0 -0.59,0.22L2.74,8.87C2.62,9.08 2.66,9.34 2.86,9.48l2.03,1.58C4.84,11.36 4.8,11.69 4.8,12s0.02,0.64 0.07,0.94l-2.03,1.58c-0.18,0.14 -0.23,0.41 -0.12,0.61l1.92,3.32c0.12,0.22 0.37,0.29 0.59,0.22l2.39,-0.96c0.5,0.38 1.03,0.7 1.62,0.94l0.36,2.54c0.05,0.24 0.24,0.41 0.48,0.41h3.84c0.24,0 0.44,-0.17 0.47,-0.41l0.36,-2.54c0.59,-0.24 1.13,-0.56 1.62,-0.94l2.39,0.96c0.22,0.08 0.47,0 0.59,-0.22l1.92,-3.32c0.12,-0.22 0.07,-0.47 -0.12,-0.61L19.14,12.94zM12,15.6c-1.98,0 -3.6,-1.62 -3.6,-3.6s1.62,-3.6 3.6,-3.6s3.6,1.62 3.6,3.6S13.98,15.6 12,15.6z" />
|
||||
</vector>
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_tab_keyboard@3x.png",
|
||||
"filename": "ic_control_keyboard@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
File renamed without changes.
|
Before Width: | Height: | Size: 590 B After Width: | Height: | Size: 590 B |
+1
-1
@@ -2,7 +2,7 @@
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_tab_mic@3x.png",
|
||||
"filename": "ic_control_mic@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
File renamed without changes.
|
Before Width: | Height: | Size: 1.2 KiB After Width: | Height: | Size: 1.2 KiB |
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_control_settings@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
"info": {
|
||||
"author": "xcode",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 1.7 KiB |
@@ -13,7 +13,7 @@
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Faceclaw</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>0.8.6</string>
|
||||
<string>0.9.0</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
@@ -57,6 +57,26 @@
|
||||
<string>Faceclaw connects to the terminal servers and developer app URLs you configure on your local network.</string>
|
||||
<key>UILaunchScreen</key>
|
||||
<dict/>
|
||||
<!-- Required: an app built with the iOS 27 SDK crashes at launch without the scene
|
||||
life cycle. @nativescript/core switches to it when it sees this key. Its side effect:
|
||||
the root page only loads once a scene connects, not on a background launch. -->
|
||||
<key>UIApplicationSceneManifest</key>
|
||||
<dict>
|
||||
<key>UIApplicationSupportsMultipleScenes</key>
|
||||
<false/>
|
||||
<key>UISceneConfigurations</key>
|
||||
<dict>
|
||||
<key>UIWindowSceneSessionRoleApplication</key>
|
||||
<array>
|
||||
<dict>
|
||||
<key>UISceneConfigurationName</key>
|
||||
<string>Default Configuration</string>
|
||||
<key>UISceneDelegateClassName</key>
|
||||
<string>SceneDelegate</string>
|
||||
</dict>
|
||||
</array>
|
||||
</dict>
|
||||
</dict>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
|
||||
@@ -5,7 +5,7 @@ CODE_SIGN_STYLE = Automatic;
|
||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||
// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+,
|
||||
// so it is weak-linked: the app still launches on older iOS and reports the model unavailable.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -framework WatchConnectivity -weak_framework FoundationModels
|
||||
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#import "FaceclawFontRenderer.h"
|
||||
#import <CoreText/CoreText.h>
|
||||
#import <CoreGraphics/CoreGraphics.h>
|
||||
#import <FaceclawKit/FaceclawKit.h>
|
||||
#import <math.h>
|
||||
|
||||
// Immutable font descriptors may be shared; layout objects never cross threads.
|
||||
@@ -42,8 +43,9 @@ static BOOL HasColorRun(CTLineRef line) {
|
||||
}
|
||||
// iOS falls back to Apple Color Emoji for default-text symbols the face lacks
|
||||
// (▶ U+25B6 in Roboto), where Android's minikin picks a text font; the
|
||||
// alpha-only raster keeps just the emoji's silhouette (▶ becomes a solid
|
||||
// rounded square). Request text presentation (U+FE0E) for each lone codepoint
|
||||
// emoji is a tile (▶ becomes a rounded square: a silhouette in the alpha-only
|
||||
// line rasters, a shaded tile in glyph cells), which reads worse than the
|
||||
// text symbol. Request text presentation (U+FE0E) for each lone codepoint
|
||||
// that fell back to a color font, wherever a text font can supply it. Returns
|
||||
// `text` itself when nothing changes.
|
||||
static NSString *PreferTextPresentation(CTFontRef font, NSString *text, CTLineRef line, BOOL glyphLayout) {
|
||||
@@ -78,18 +80,34 @@ static BOOL ValidText(NSString *text) { return text != nil && text.length <= 100
|
||||
static BOOL ValidRaster(NSInteger width, NSInteger height) {
|
||||
return width > 0 && height > 0 && width <= 65535 && height <= 65535 && width * height <= 4000000;
|
||||
}
|
||||
static void ConfigureTextContext(CGContextRef ctx) {
|
||||
CGContextSetShouldAntialias(ctx, true);
|
||||
CGContextSetShouldSmoothFonts(ctx, false); // Grayscale coverage, never LCD subpixel color.
|
||||
CGContextSetShouldSubpixelPositionFonts(ctx, true);
|
||||
CGContextSetShouldSubpixelQuantizeFonts(ctx, false);
|
||||
CGContextSetTextMatrix(ctx, CGAffineTransformIdentity);
|
||||
}
|
||||
static CGContextRef CreateContext(NSMutableData *pixels, NSInteger width, NSInteger height) CF_RETURNS_RETAINED {
|
||||
CGContextRef ctx = CGBitmapContextCreate(pixels.mutableBytes, width, height, 8, width, NULL, (CGBitmapInfo)kCGImageAlphaOnly);
|
||||
if (ctx) {
|
||||
CGContextSetShouldAntialias(ctx, true);
|
||||
CGContextSetShouldSmoothFonts(ctx, false); // Grayscale coverage, never LCD subpixel color.
|
||||
CGContextSetShouldSubpixelPositionFonts(ctx, true);
|
||||
CGContextSetShouldSubpixelQuantizeFonts(ctx, false);
|
||||
CGContextSetTextMatrix(ctx, CGAffineTransformIdentity);
|
||||
ConfigureTextContext(ctx);
|
||||
CGContextSetGrayFillColor(ctx, 1, 1);
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
// Premultiplied RGBA bytes, for a line with a color (emoji) run: an alpha-only raster keeps
|
||||
// just the emoji's silhouette, so its colors go to ColorGlyphArt to be shaded instead.
|
||||
static CGContextRef CreateColorContext(NSMutableData *pixels, NSInteger width, NSInteger height) CF_RETURNS_RETAINED {
|
||||
CGColorSpaceRef space = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
|
||||
CGContextRef ctx = CGBitmapContextCreate(pixels.mutableBytes, width, height, 8, width * 4, space,
|
||||
(CGBitmapInfo)kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big);
|
||||
CGColorSpaceRelease(space);
|
||||
if (ctx) {
|
||||
ConfigureTextContext(ctx);
|
||||
CGContextSetRGBFillColor(ctx, 1, 1, 1, 1);
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
static void U16(uint8_t *out, NSInteger value) { out[0] = value & 255; out[1] = (value >> 8) & 255; }
|
||||
static void GammaCopy(uint8_t *dst, const uint8_t *src, NSInteger count, double gamma) {
|
||||
if (!isfinite(gamma) || gamma <= 0) gamma = 1;
|
||||
@@ -150,10 +168,13 @@ static NSData *ImagePacket(NSData *pixels, NSInteger width, NSInteger height, do
|
||||
NSInteger bottom = floor(CGRectGetMinY(bounds)) - 2;
|
||||
NSInteger width = ceil(CGRectGetMaxX(bounds)) + 2 - left, height = top - bottom;
|
||||
if (!ValidRaster(width, height)) { CFRelease(line); CFRelease(font); return [NSData data]; }
|
||||
NSMutableData *pixels = [NSMutableData dataWithLength:width * height];
|
||||
CGContextRef ctx = CreateContext(pixels, width, height);
|
||||
BOOL color = HasColorRun(line);
|
||||
NSMutableData *raster = [NSMutableData dataWithLength:width * height * (color ? 4 : 1)];
|
||||
CGContextRef ctx = color ? CreateColorContext(raster, width, height) : CreateContext(raster, width, height);
|
||||
if (!ctx) { CFRelease(line); CFRelease(font); return [NSData data]; }
|
||||
CGContextSetTextPosition(ctx, -left, -bottom); CTLineDraw(line, ctx); CGContextRelease(ctx);
|
||||
NSData *pixels = color ? [[FaceclawKitIosColorGlyph shared] shadeRgba:raster] : raster;
|
||||
if ((NSInteger)pixels.length != width * height) { CFRelease(line); CFRelease(font); return [NSData data]; }
|
||||
const uint8_t *src = pixels.bytes;
|
||||
NSInteger x0 = width, y0 = height, x1 = -1, y1 = -1;
|
||||
for (NSInteger y = 0; y < height; y++) for (NSInteger x = 0; x < width; x++) if (src[y * width + x]) {
|
||||
|
||||
@@ -14,4 +14,10 @@
|
||||
+ (NSData *)bgraPacket:(CGImageRef)image maxSize:(NSInteger)maxSize;
|
||||
/** bgraPacket of encoded image bytes (JPEG, PNG, ...); nil when undecodable. */
|
||||
+ (NSData *)bgraPacketFromImageData:(NSData *)data maxSize:(NSInteger)maxSize;
|
||||
/**
|
||||
* [widthLo, widthHi, heightLo, heightHi, RGBA...] of encoded image bytes at native size
|
||||
* (at most 4096 px a side), first frame only; nil when undecodable. CoreGraphics
|
||||
* premultiplies, so transparent pixels come out black. For the EvenHub image pipeline.
|
||||
*/
|
||||
+ (NSData *)rgbaPacketFromImageData:(NSData *)data;
|
||||
@end
|
||||
@@ -158,4 +158,26 @@
|
||||
CGImageRelease(image);
|
||||
return packet;
|
||||
}
|
||||
+ (NSData *)rgbaPacketFromImageData:(NSData *)data {
|
||||
if (!data.length) return nil;
|
||||
CGImageSourceRef source = CGImageSourceCreateWithData((__bridge CFDataRef)data, NULL);
|
||||
if (!source) return nil;
|
||||
CGImageRef image = CGImageSourceCreateImageAtIndex(source, 0, NULL);
|
||||
CFRelease(source);
|
||||
if (!image) return nil;
|
||||
size_t width = CGImageGetWidth(image), height = CGImageGetHeight(image);
|
||||
if (width == 0 || height == 0 || width > 4096 || height > 4096) { CGImageRelease(image); return nil; }
|
||||
uint8_t dimensions[] = { width & 255, (width >> 8) & 255, height & 255, (height >> 8) & 255 };
|
||||
NSMutableData *packet = [NSMutableData dataWithBytes:dimensions length:4];
|
||||
[packet increaseLengthBy:width * height * 4];
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
CGContextRef context = CGBitmapContextCreate((uint8_t *)packet.mutableBytes + 4, width, height, 8, width * 4, space,
|
||||
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast);
|
||||
CGColorSpaceRelease(space);
|
||||
if (!context) { CGImageRelease(image); return nil; }
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, width, height), image);
|
||||
CGContextRelease(context);
|
||||
CGImageRelease(image);
|
||||
return packet;
|
||||
}
|
||||
@end
|
||||
@@ -0,0 +1,43 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
/**
|
||||
* Phone side of the Apple Watch remote: the single place that talks to
|
||||
* WatchConnectivity, so JS only ever sees path + JSON strings. The twin of the
|
||||
* Android FaceclawWearBridge; the messages are in wear/PROTOCOL.md.
|
||||
*
|
||||
* The session activates during launch, ahead of JS, because the watch can wake
|
||||
* this app in the background and messages arriving before a delegate exists
|
||||
* are lost. Until JS registers its handlers, messages are answered with
|
||||
* jsReady:NO so the watch can say "open Faceclaw on the phone".
|
||||
*
|
||||
* The public methods and both handlers belong to the main thread.
|
||||
*/
|
||||
@interface FaceclawWatchBridge : NSObject
|
||||
+ (instancetype)shared;
|
||||
/** WCSession works on this device (every iPhone; no iPad). */
|
||||
- (BOOL)isAvailable;
|
||||
/** A paired Apple Watch has the Faceclaw app. It needn't be running. */
|
||||
- (BOOL)isWatchAppAvailable;
|
||||
/**
|
||||
* Register JS. messageHandler gets every watch message; connectionHandler fires
|
||||
* when isWatchAppAvailable or the watch app's live reachability changes.
|
||||
*/
|
||||
- (void)setMessageHandler:(void (^)(NSString *path, NSString *json))messageHandler
|
||||
connectionHandler:(void (^)(void))connectionHandler;
|
||||
/**
|
||||
* Mirror the dashboard state: kept in the application context for the app's
|
||||
* next launch, and sent live too while it's in front. An unchanged state is
|
||||
* dropped unless `force`.
|
||||
*/
|
||||
- (void)publishState:(NSString *)json force:(BOOL)force;
|
||||
/** Send an event live. A sticky one also stays in the application context for a suspended app. */
|
||||
- (void)sendEvent:(NSString *)json sticky:(BOOL)sticky;
|
||||
/** A live message on any phone -> watch path; dropped if the watch app isn't in front. */
|
||||
- (void)sendToWatch:(NSString *)path json:(NSString *)json;
|
||||
/** Reply to a watch message; `seq` echoes the watch's sequence number. */
|
||||
- (void)sendAck:(double)seq ok:(BOOL)ok jsReady:(BOOL)jsReady message:(NSString *)message;
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
@@ -0,0 +1,272 @@
|
||||
#import "FaceclawWatchBridge.h"
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <WatchConnectivity/WatchConnectivity.h>
|
||||
|
||||
// Live messages are {path, json, ...}; the watch app's Protocol.swift mirrors these keys.
|
||||
static NSString *const kKeyPath = @"path";
|
||||
static NSString *const kKeyJson = @"json";
|
||||
static NSString *const kKeyUpdatedAt = @"updatedAt";
|
||||
static NSString *const kKeyEvent = @"event";
|
||||
static NSString *const kKeyEventAt = @"eventAt";
|
||||
static NSString *const kKeyAt = @"at";
|
||||
static NSString *const kKeyBattery = @"battery";
|
||||
|
||||
static NSString *const kPathState = @"/faceclaw/state";
|
||||
static NSString *const kPathStateRequest = @"/faceclaw/state/request";
|
||||
static NSString *const kPathBattery = @"/faceclaw/battery";
|
||||
static NSString *const kPathPress = @"/faceclaw/press";
|
||||
static NSString *const kPathAck = @"/faceclaw/ack";
|
||||
static NSString *const kPathEvent = @"/faceclaw/event";
|
||||
|
||||
/** State requests kept for a JS handler that isn't registered yet. */
|
||||
static const NSUInteger kPendingLimit = 16;
|
||||
static const NSTimeInterval kPendingMaxAge = 30;
|
||||
|
||||
static long long NowMs(void) {
|
||||
return (long long)(NSDate.date.timeIntervalSince1970 * 1000);
|
||||
}
|
||||
|
||||
static double ReadSeq(NSString *json) {
|
||||
NSData *data = [json dataUsingEncoding:NSUTF8StringEncoding];
|
||||
id parsed = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil;
|
||||
id seq = [parsed isKindOfClass:NSDictionary.class] ? parsed[@"seq"] : nil;
|
||||
return [seq isKindOfClass:NSNumber.class] ? [seq doubleValue] : 0;
|
||||
}
|
||||
|
||||
@interface FaceclawWatchPendingMessage : NSObject
|
||||
@property(nonatomic, copy) NSString *path;
|
||||
@property(nonatomic, copy) NSString *json;
|
||||
@property(nonatomic, strong) NSDate *receivedAt;
|
||||
@end
|
||||
@implementation FaceclawWatchPendingMessage
|
||||
@end
|
||||
|
||||
@interface FaceclawWatchBridge () <WCSessionDelegate>
|
||||
@property(nonatomic, copy, nullable) void (^messageHandler)(NSString *, NSString *);
|
||||
@property(nonatomic, copy, nullable) void (^connectionHandler)(void);
|
||||
@property(nonatomic, strong) NSMutableArray<FaceclawWatchPendingMessage *> *pending;
|
||||
@property(nonatomic, copy, nullable) NSString *lastState;
|
||||
@property(nonatomic) long long lastStateAt;
|
||||
@property(nonatomic, copy, nullable) NSString *heldEvent;
|
||||
@property(nonatomic) long long heldEventAt;
|
||||
/** Sends and context updates run here: they cost IPC, and the main thread also runs BLE. */
|
||||
@property(nonatomic, strong) dispatch_queue_t sendQueue;
|
||||
/** Latest context waiting for sendQueue; nil when none is queued. Guarded by @synchronized(self). */
|
||||
@property(nonatomic, copy, nullable) NSDictionary *queuedContext;
|
||||
@end
|
||||
|
||||
@implementation FaceclawWatchBridge
|
||||
|
||||
+ (void)load {
|
||||
// Registering here is what puts the delegate in place during launch, ahead of JS.
|
||||
[NSNotificationCenter.defaultCenter addObserverForName:UIApplicationDidFinishLaunchingNotification
|
||||
object:nil
|
||||
queue:NSOperationQueue.mainQueue
|
||||
usingBlock:^(NSNotification *note) {
|
||||
[[FaceclawWatchBridge shared] activate];
|
||||
}];
|
||||
}
|
||||
|
||||
+ (instancetype)shared {
|
||||
static FaceclawWatchBridge *instance;
|
||||
static dispatch_once_t once;
|
||||
dispatch_once(&once, ^{ instance = [FaceclawWatchBridge new]; });
|
||||
return instance;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) {
|
||||
_pending = [NSMutableArray new];
|
||||
_sendQueue = dispatch_queue_create("com.faceclaw.watch.send", DISPATCH_QUEUE_SERIAL);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)activate {
|
||||
if (!WCSession.isSupported) return;
|
||||
WCSession *session = WCSession.defaultSession;
|
||||
if (session.delegate == self) return;
|
||||
session.delegate = self;
|
||||
[session activateSession];
|
||||
}
|
||||
|
||||
#pragma mark - Public API (main thread)
|
||||
|
||||
- (BOOL)isAvailable {
|
||||
return WCSession.isSupported;
|
||||
}
|
||||
|
||||
- (BOOL)isWatchAppAvailable {
|
||||
if (!WCSession.isSupported) return NO;
|
||||
WCSession *session = WCSession.defaultSession;
|
||||
if (session.activationState != WCSessionActivationStateActivated) return NO;
|
||||
return session.isReachable || (session.isPaired && session.isWatchAppInstalled);
|
||||
}
|
||||
|
||||
- (void)setMessageHandler:(void (^)(NSString *, NSString *))messageHandler
|
||||
connectionHandler:(void (^)(void))connectionHandler {
|
||||
self.messageHandler = messageHandler;
|
||||
self.connectionHandler = connectionHandler;
|
||||
[self activate];
|
||||
NSArray<FaceclawWatchPendingMessage *> *replay = [self.pending copy];
|
||||
[self.pending removeAllObjects];
|
||||
for (FaceclawWatchPendingMessage *message in replay) {
|
||||
if (-message.receivedAt.timeIntervalSinceNow <= kPendingMaxAge) messageHandler(message.path, message.json);
|
||||
}
|
||||
// The watch's last battery report may have arrived while no one was listening.
|
||||
if (WCSession.isSupported && WCSession.defaultSession.activationState == WCSessionActivationStateActivated) {
|
||||
[self deliverBatteryFrom:WCSession.defaultSession.receivedApplicationContext];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)publishState:(NSString *)json force:(BOOL)force {
|
||||
if (!force && [json isEqualToString:self.lastState]) return;
|
||||
self.lastState = json;
|
||||
self.lastStateAt = NowMs();
|
||||
[self pushContext];
|
||||
[self sendLive:@{kKeyPath: kPathState, kKeyJson: json, kKeyUpdatedAt: @(self.lastStateAt)}];
|
||||
}
|
||||
|
||||
- (void)sendEvent:(NSString *)json sticky:(BOOL)sticky {
|
||||
long long at = NowMs();
|
||||
if (sticky) {
|
||||
self.heldEvent = json;
|
||||
self.heldEventAt = at;
|
||||
[self pushContext];
|
||||
}
|
||||
[self sendLive:@{kKeyPath: kPathEvent, kKeyJson: json, kKeyAt: @(at)}];
|
||||
}
|
||||
|
||||
- (void)sendToWatch:(NSString *)path json:(NSString *)json {
|
||||
[self sendLive:@{kKeyPath: path, kKeyJson: json}];
|
||||
}
|
||||
|
||||
- (void)sendAck:(double)seq ok:(BOOL)ok jsReady:(BOOL)jsReady message:(NSString *)message {
|
||||
NSDictionary *ack = @{@"seq": @(seq), @"ok": @(ok), @"jsReady": @(jsReady), @"message": message ?: @""};
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:ack options:0 error:nil];
|
||||
if (!data) return;
|
||||
[self sendToWatch:kPathAck json:[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]];
|
||||
}
|
||||
|
||||
#pragma mark - Sending
|
||||
|
||||
- (void)sendLive:(NSDictionary *)message {
|
||||
if (!WCSession.isSupported) return;
|
||||
dispatch_async(self.sendQueue, ^{
|
||||
WCSession *session = WCSession.defaultSession;
|
||||
if (session.activationState != WCSessionActivationStateActivated || !session.isReachable) return;
|
||||
[session sendMessage:message replyHandler:nil errorHandler:^(NSError *error) {
|
||||
NSLog(@"[FaceclawWatch] send %@ failed: %@", message[kKeyPath], error);
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
/** Queue the current state and held event as the application context; only the latest matters. */
|
||||
- (void)pushContext {
|
||||
if (!WCSession.isSupported) return;
|
||||
NSMutableDictionary *context = [NSMutableDictionary new];
|
||||
if (self.lastState) {
|
||||
context[kKeyJson] = self.lastState;
|
||||
context[kKeyUpdatedAt] = @(self.lastStateAt);
|
||||
}
|
||||
if (self.heldEvent) {
|
||||
context[kKeyEvent] = self.heldEvent;
|
||||
context[kKeyEventAt] = @(self.heldEventAt);
|
||||
}
|
||||
BOOL alreadyQueued;
|
||||
@synchronized (self) {
|
||||
alreadyQueued = self.queuedContext != nil;
|
||||
self.queuedContext = context;
|
||||
}
|
||||
if (alreadyQueued) return;
|
||||
dispatch_async(self.sendQueue, ^{
|
||||
NSDictionary *next;
|
||||
@synchronized (self) {
|
||||
next = self.queuedContext;
|
||||
self.queuedContext = nil;
|
||||
}
|
||||
WCSession *session = WCSession.defaultSession;
|
||||
if (!next || session.activationState != WCSessionActivationStateActivated) return;
|
||||
if (!session.isPaired || !session.isWatchAppInstalled) return;
|
||||
NSError *error = nil;
|
||||
if (![session updateApplicationContext:next error:&error]) {
|
||||
NSLog(@"[FaceclawWatch] application context update failed: %@", error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#pragma mark - Receiving
|
||||
|
||||
- (void)handleMessage:(NSString *)path json:(NSString *)json {
|
||||
if (self.messageHandler) {
|
||||
self.messageHandler(path, json);
|
||||
return;
|
||||
}
|
||||
// A battery report is status and a touch-down only matters live; neither expects an ack.
|
||||
if ([path isEqualToString:kPathBattery] || [path isEqualToString:kPathPress]) return;
|
||||
// Gestures and commands are refused rather than replayed: the watch reports them as failed,
|
||||
// and running them later would act on whatever the glasses happen to show by then.
|
||||
if ([path isEqualToString:kPathStateRequest]) {
|
||||
NSDate *now = NSDate.date;
|
||||
while (self.pending.count > 0 && [now timeIntervalSinceDate:self.pending.firstObject.receivedAt] > kPendingMaxAge) {
|
||||
[self.pending removeObjectAtIndex:0];
|
||||
}
|
||||
if (self.pending.count >= kPendingLimit) [self.pending removeObjectAtIndex:0];
|
||||
FaceclawWatchPendingMessage *message = [FaceclawWatchPendingMessage new];
|
||||
message.path = path;
|
||||
message.json = json;
|
||||
message.receivedAt = now;
|
||||
[self.pending addObject:message];
|
||||
}
|
||||
[self sendAck:ReadSeq(json) ok:NO jsReady:NO message:@"Faceclaw is not running on the phone. Open it to connect the glasses."];
|
||||
}
|
||||
|
||||
/** The watch leaves its battery in the application context while it can't message us live. */
|
||||
- (void)deliverBatteryFrom:(NSDictionary<NSString *, id> *)context {
|
||||
id json = context[kKeyBattery];
|
||||
if ([json isKindOfClass:NSString.class] && self.messageHandler) self.messageHandler(kPathBattery, json);
|
||||
}
|
||||
|
||||
- (void)notifyConnection {
|
||||
if (self.connectionHandler) self.connectionHandler();
|
||||
}
|
||||
|
||||
#pragma mark - WCSessionDelegate (any thread)
|
||||
|
||||
- (void)session:(WCSession *)session activationDidCompleteWithState:(WCSessionActivationState)state error:(NSError *)error {
|
||||
if (error) NSLog(@"[FaceclawWatch] activation failed: %@", error);
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
// Anything published before activation couldn't be delivered.
|
||||
if (self.lastState || self.heldEvent) [self pushContext];
|
||||
[self deliverBatteryFrom:session.receivedApplicationContext];
|
||||
[self notifyConnection];
|
||||
});
|
||||
}
|
||||
|
||||
- (void)sessionDidBecomeInactive:(WCSession *)session {}
|
||||
|
||||
- (void)sessionDidDeactivate:(WCSession *)session {
|
||||
// The user switched watches: the session must be reactivated for the new one.
|
||||
[session activateSession];
|
||||
}
|
||||
|
||||
- (void)sessionWatchStateDidChange:(WCSession *)session {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ [self notifyConnection]; });
|
||||
}
|
||||
|
||||
- (void)sessionReachabilityDidChange:(WCSession *)session {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ [self notifyConnection]; });
|
||||
}
|
||||
|
||||
- (void)session:(WCSession *)session didReceiveMessage:(NSDictionary<NSString *, id> *)message {
|
||||
id path = message[kKeyPath], json = message[kKeyJson];
|
||||
if (![path isKindOfClass:NSString.class]) return;
|
||||
NSString *body = [json isKindOfClass:NSString.class] ? json : @"{}";
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ [self handleMessage:path json:body]; });
|
||||
}
|
||||
|
||||
- (void)session:(WCSession *)session didReceiveApplicationContext:(NSDictionary<NSString *, id> *)context {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ [self deliverBatteryFrom:context]; });
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -14,5 +14,6 @@ module FaceclawGraphics {
|
||||
header "FaceclawLoopbackRedirect.h"
|
||||
header "FaceclawWebSignIn.h"
|
||||
header "FaceclawResourceUsage.h"
|
||||
header "FaceclawWatchBridge.h"
|
||||
export *
|
||||
}
|
||||
Reference in new issue
Block a user