forked from Blendman974/faceclaw
[verified] merge upstream 25cb255b into all-features
This commit is contained in:
commit
ffcdae660d
169 files changed
+16201
-597
No files matched your search
@@ -63,6 +63,8 @@ native/kotlin/.kotlin/
|
||||
native/kotlin/build/
|
||||
native/kotlin/shared/build/
|
||||
native/kotlin/local.properties
|
||||
# Go SSH transport build outputs (scripts/go-ssh-build.cjs)
|
||||
native/ssh/build/
|
||||
native/kotlin/plugin/platforms/android/*.aar
|
||||
native/kotlin/plugin/platforms/ios/
|
||||
!native/kotlin/plugin/platforms/
|
||||
|
||||
@@ -23,6 +23,12 @@ and Mentra Labs, Inc. (MIT License): https://github.com/tetramo-labs/glassly and
|
||||
https://github.com/tetramo-labs/glassly-rayneo-cfw. The iO wire codecs in
|
||||
native/kotlin/shared/.../rayneo/RayNeoIoProtocol.kt follow their protocol work.
|
||||
|
||||
Bluetooth keyboard support on the G2 uses the keyboard bridge Tetramo, Inc. wrote
|
||||
for their Glassly custom firmware (https://github.com/tetramo-labs/g2flash).
|
||||
Faceclaw's client for it, in app/g2/keyboard/ and app/apps/terminal/terminal-keys.ts,
|
||||
follows the reference client in that repository and the keyboard handling in the
|
||||
Glassly app (MIT License): https://github.com/tetramo-labs/glassly.
|
||||
|
||||
iPhone notification icons are Pebble's notification and timeline icons, from
|
||||
https://github.com/pebble-dev/iconography: the original Pebble Technology icons
|
||||
and Rebble community additions. They are licensed under the Apache License 2.0;
|
||||
|
||||
@@ -292,6 +292,13 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
/** See [GlassesSessionCore.getRingStatus]. */
|
||||
fun getRingStatus(): String = core.getRingStatus()
|
||||
|
||||
/** See [GlassesSessionCore.setHeadUpTrigger]. */
|
||||
fun setHeadUpTrigger(degrees: Int, originalSwitch: Int, originalAngle: Int) =
|
||||
core.setHeadUpTrigger(degrees, originalSwitch, originalAngle)
|
||||
|
||||
/** See [GlassesSessionCore.getHeadUpStatus]. */
|
||||
fun getHeadUpStatus(): String = core.getHeadUpStatus()
|
||||
|
||||
/** See [GlassesSessionCore.setGlassesRingLink]. */
|
||||
fun setGlassesRingLink(action: String, fallbackAddress: String?, fallbackName: String?): Boolean =
|
||||
core.setGlassesRingLink(action, fallbackAddress, fallbackName)
|
||||
@@ -324,6 +331,13 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
core.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
|
||||
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.addKeyboardEventListener(listener)
|
||||
|
||||
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
|
||||
|
||||
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean =
|
||||
core.sendFaceclawKeyboardControl(record, label)
|
||||
|
||||
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener)
|
||||
|
||||
fun stopG2AudioForwarding() = core.stopG2AudioForwarding()
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.ClipData
|
||||
import android.content.ClipboardManager
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.security.keystore.KeyGenParameterSpec
|
||||
import android.security.keystore.KeyProperties
|
||||
import android.security.keystore.StrongBoxUnavailableException
|
||||
import android.util.Base64
|
||||
import android.util.Log
|
||||
import com.faceclaw.ssh.faceclawssh.Channel
|
||||
import com.faceclaw.ssh.faceclawssh.Faceclawssh
|
||||
import com.faceclaw.ssh.faceclawssh.Listener
|
||||
import com.faceclaw.ssh.faceclawssh.Signer
|
||||
import java.security.KeyPairGenerator
|
||||
import java.security.KeyStore
|
||||
import java.security.PrivateKey
|
||||
import java.security.Signature
|
||||
import java.security.spec.ECGenParameterSpec
|
||||
import javax.crypto.Cipher
|
||||
import javax.crypto.KeyGenerator
|
||||
import javax.crypto.SecretKey
|
||||
import javax.crypto.spec.GCMParameterSpec
|
||||
|
||||
/**
|
||||
* One SSH channel for the TypeScript side (app/native/ssh.ts), backed by the
|
||||
* gomobile-bound Go transport in native/ssh. `spec` is the Go openSpec JSON.
|
||||
* Events arrive from Go goroutines and are posted to the Looper of the
|
||||
* thread that constructed this object (a JS isolate's own thread), like
|
||||
* FaceclawWebSocket.
|
||||
*/
|
||||
class FaceclawSshChannel(spec: String, useKey: Boolean, listener: FaceclawSshListener) {
|
||||
private val handler = Handler(Looper.myLooper() ?: Looper.getMainLooper())
|
||||
private val channel: Channel
|
||||
|
||||
init {
|
||||
val signer = if (useKey) FaceclawSshKeys.signer() else null
|
||||
channel = Faceclawssh.open(spec, signer, object : Listener {
|
||||
override fun onEvent(kind: String?, payload: String?) {
|
||||
handler.post {
|
||||
try {
|
||||
listener.onEvent(kind, payload)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "listener onEvent($kind) failed", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fun send(text: String): Boolean = channel.send(text)
|
||||
|
||||
fun close() = channel.close()
|
||||
|
||||
private companion object {
|
||||
const val TAG = "FaceclawSsh"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The device's SSH client key and the Terminal app's saved SSH passwords.
|
||||
*
|
||||
* The client key is an ECDSA P-256 key generated inside Android Keystore
|
||||
* (StrongBox when the device has one) and never exportable: only its public
|
||||
* half leaves the keystore, as an authorized_keys line. Passwords are
|
||||
* encrypted with an AES-GCM key that also lives in Keystore, and stored in a
|
||||
* private SharedPreferences file kept apart from the settings store (which is
|
||||
* exported by the settings port).
|
||||
*/
|
||||
object FaceclawSshKeys {
|
||||
private const val TAG = "FaceclawSsh"
|
||||
private const val CLIENT_KEY_ALIAS = "faceclaw-ssh-client"
|
||||
private const val SECRET_KEY_ALIAS = "faceclaw-ssh-secrets"
|
||||
private const val SECRETS_PREFS = "faceclaw_ssh_secrets"
|
||||
private const val GCM_TAG_BITS = 128
|
||||
|
||||
private fun keyStore(): KeyStore = KeyStore.getInstance("AndroidKeyStore").apply { load(null) }
|
||||
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun hasClientKey(): Boolean = keyStore().containsAlias(CLIENT_KEY_ALIAS)
|
||||
|
||||
/** Generate the client key if it doesn't exist yet. */
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun ensureClientKey() {
|
||||
if (keyStore().containsAlias(CLIENT_KEY_ALIAS)) return
|
||||
fun spec(strongBox: Boolean): KeyGenParameterSpec {
|
||||
val builder = KeyGenParameterSpec.Builder(CLIENT_KEY_ALIAS, KeyProperties.PURPOSE_SIGN)
|
||||
.setAlgorithmParameterSpec(ECGenParameterSpec("secp256r1"))
|
||||
.setDigests(KeyProperties.DIGEST_SHA256)
|
||||
if (strongBox && Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) builder.setIsStrongBoxBacked(true)
|
||||
return builder.build()
|
||||
}
|
||||
val generator = KeyPairGenerator.getInstance(KeyProperties.KEY_ALGORITHM_EC, "AndroidKeyStore")
|
||||
try {
|
||||
generator.initialize(spec(strongBox = true))
|
||||
generator.generateKeyPair()
|
||||
} catch (e: Exception) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && e is StrongBoxUnavailableException) {
|
||||
Log.i(TAG, "StrongBox unavailable; generating the SSH key in the TEE")
|
||||
} else {
|
||||
Log.w(TAG, "StrongBox key generation failed; retrying in the TEE", e)
|
||||
}
|
||||
generator.initialize(spec(strongBox = false))
|
||||
generator.generateKeyPair()
|
||||
}
|
||||
Log.i(TAG, "generated SSH client key")
|
||||
}
|
||||
|
||||
/** X.509 SubjectPublicKeyInfo DER of the client key, or null if none. */
|
||||
@JvmStatic
|
||||
fun publicKey(): ByteArray? = keyStore().getCertificate(CLIENT_KEY_ALIAS)?.publicKey?.encoded
|
||||
|
||||
/** The client key's authorized_keys line (generating the key if needed). */
|
||||
@JvmStatic
|
||||
fun authorizedKey(comment: String): String {
|
||||
ensureClientKey()
|
||||
return Faceclawssh.authorizedKey(publicKey(), comment)
|
||||
}
|
||||
|
||||
/** The client key's SHA256 fingerprint (generating the key if needed). */
|
||||
@JvmStatic
|
||||
fun fingerprint(): String {
|
||||
ensureClientKey()
|
||||
return Faceclawssh.fingerprint(publicKey())
|
||||
}
|
||||
|
||||
/** A Go Signer over the client key, generating the key if needed. */
|
||||
@JvmStatic
|
||||
fun signer(): Signer? = try {
|
||||
ensureClientKey()
|
||||
val publicKey = publicKey() ?: throw IllegalStateException("client key has no certificate")
|
||||
object : Signer {
|
||||
override fun publicKey(): ByteArray = publicKey
|
||||
override fun sign(data: ByteArray?): ByteArray {
|
||||
val privateKey = keyStore().getKey(CLIENT_KEY_ALIAS, null) as PrivateKey
|
||||
return Signature.getInstance("SHA256withECDSA").run {
|
||||
initSign(privateKey)
|
||||
update(data ?: ByteArray(0))
|
||||
sign()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "SSH client key unavailable", t)
|
||||
null
|
||||
}
|
||||
|
||||
private fun secretKey(): SecretKey {
|
||||
val store = keyStore()
|
||||
(store.getKey(SECRET_KEY_ALIAS, null) as? SecretKey)?.let { return it }
|
||||
val generator = KeyGenerator.getInstance(KeyProperties.KEY_ALGORITHM_AES, "AndroidKeyStore")
|
||||
generator.init(
|
||||
KeyGenParameterSpec.Builder(SECRET_KEY_ALIAS, KeyProperties.PURPOSE_ENCRYPT or KeyProperties.PURPOSE_DECRYPT)
|
||||
.setBlockModes(KeyProperties.BLOCK_MODE_GCM)
|
||||
.setEncryptionPaddings(KeyProperties.ENCRYPTION_PADDING_NONE)
|
||||
.setKeySize(256)
|
||||
.build()
|
||||
)
|
||||
return generator.generateKey()
|
||||
}
|
||||
|
||||
private fun prefs(context: Context) =
|
||||
context.applicationContext.getSharedPreferences(SECRETS_PREFS, Context.MODE_PRIVATE)
|
||||
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun setSecret(context: Context, id: String, value: String) {
|
||||
val cipher = Cipher.getInstance("AES/GCM/NoPadding")
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey())
|
||||
val sealed = cipher.iv + cipher.doFinal(value.toByteArray(Charsets.UTF_8))
|
||||
prefs(context).edit().putString(id, Base64.encodeToString(sealed, Base64.NO_WRAP)).apply()
|
||||
}
|
||||
|
||||
/** The saved secret, or "" if none (or it can no longer be decrypted). */
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun getSecret(context: Context, id: String): String {
|
||||
val encoded = prefs(context).getString(id, null) ?: return ""
|
||||
return try {
|
||||
val sealed = Base64.decode(encoded, Base64.NO_WRAP)
|
||||
val cipher = Cipher.getInstance("AES/GCM/NoPadding")
|
||||
val ivLength = 12
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey(), GCMParameterSpec(GCM_TAG_BITS, sealed, 0, ivLength))
|
||||
String(cipher.doFinal(sealed, ivLength, sealed.size - ivLength), Charsets.UTF_8)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "saved SSH secret unreadable; dropping it", t)
|
||||
prefs(context).edit().remove(id).apply()
|
||||
""
|
||||
}
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun removeSecret(context: Context, id: String) {
|
||||
prefs(context).edit().remove(id).apply()
|
||||
}
|
||||
|
||||
/** Put text on the phone's clipboard (from any thread). */
|
||||
@JvmStatic
|
||||
fun copyToClipboard(context: Context, label: String, text: String) {
|
||||
val appContext = context.applicationContext
|
||||
Handler(Looper.getMainLooper()).post {
|
||||
val clipboard = appContext.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
|
||||
clipboard.setPrimaryClip(ClipData.newPlainText(label, text))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -105,6 +105,21 @@ class FaceclawVoiceController(context: Context) {
|
||||
}
|
||||
}
|
||||
|
||||
override fun micArrayLink(): MicArrayLink? {
|
||||
val currentCommunicator = communicator ?: return null
|
||||
return object : MicArrayLink {
|
||||
override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean =
|
||||
currentCommunicator.startG2AudioForwarding(listener)
|
||||
|
||||
override fun stopForwarding() = currentCommunicator.stopG2AudioForwarding()
|
||||
|
||||
override fun isForwarding(): Boolean = currentCommunicator.isAudioCaptureActive()
|
||||
|
||||
override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
currentCommunicator.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
}
|
||||
}
|
||||
|
||||
override fun createPacketDecoder(): AudioPacketDecoder =
|
||||
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
|
||||
else AndroidSpeechEngines.Lc3DecoderAdapter()
|
||||
@@ -131,6 +146,11 @@ class FaceclawVoiceController(context: Context) {
|
||||
|
||||
private val session = VoiceCaptureSession(host)
|
||||
|
||||
// The recording of this controller's live captures (see startRecording). It outlives any one
|
||||
// capture, and is only handed to the session while recording is enabled.
|
||||
private var recording: MicRecordingWriter? = null
|
||||
private var recordingEnabled = true
|
||||
|
||||
fun setListener(listener: FaceclawVoiceControllerListener?) {
|
||||
session.setListener(listener)
|
||||
}
|
||||
@@ -203,6 +223,88 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setBeamFilter(enabled, centerDeg, halfWidthDeg)
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture glasses sessions from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the stock mixed stream. [configureRecord] is the CONFIGURE
|
||||
* record to arm both temples with; null goes back to the mixed stream. Must be set before
|
||||
* {@link #start}.
|
||||
*/
|
||||
fun setMicArray(configureRecord: ByteArray?) {
|
||||
session.setMicArray(configureRecord)
|
||||
}
|
||||
|
||||
/**
|
||||
* Have the next {@link #start} play the recording at [path] (a .mka file) from [startMs] in
|
||||
* place of any microphone; null goes back to the microphones. Progress arrives through the
|
||||
* listener's onReplay.
|
||||
*/
|
||||
fun setReplay(path: String?, startMs: Long) {
|
||||
session.setReplay(path, startMs)
|
||||
}
|
||||
|
||||
/** Continue the running replay from [positionMs] into its recording. */
|
||||
fun seekReplay(positionMs: Long) {
|
||||
session.seekReplay(positionMs)
|
||||
}
|
||||
|
||||
fun setReplayPaused(paused: Boolean) {
|
||||
session.setReplayPaused(paused)
|
||||
}
|
||||
|
||||
/**
|
||||
* Record live captures to a new Matroska audio file at [path], from now until
|
||||
* {@link #stopRecording}, across however many captures run in between: every source's audio
|
||||
* as it arrives, before any processing. [configJson] describes the capture and is stored in
|
||||
* the file. Returns false when the file cannot be created.
|
||||
*/
|
||||
@Synchronized
|
||||
fun startRecording(path: String, configJson: String?): Boolean {
|
||||
stopRecording()
|
||||
return try {
|
||||
val parent = File(path).parentFile
|
||||
if (parent != null && !parent.exists()) parent.mkdirs()
|
||||
recording = MicRecordingWriter(openFileOutput(path), configJson, System.currentTimeMillis())
|
||||
session.setRecorder(if (recordingEnabled) recording else null)
|
||||
true
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "could not create recording $path", t)
|
||||
recording = null
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether captures are written to the recording right now: off while the capture running
|
||||
* isn't one that is to be recorded.
|
||||
*/
|
||||
@Synchronized
|
||||
fun setRecordingEnabled(enabled: Boolean) {
|
||||
recordingEnabled = enabled
|
||||
session.setRecorder(if (enabled) recording else null)
|
||||
}
|
||||
|
||||
/** Whether a recording is open and still being written (false once writing to it failed). */
|
||||
@Synchronized
|
||||
fun isRecording(): Boolean = recording?.failed == false
|
||||
|
||||
/**
|
||||
* Note a change in what is being captured (a JSON object) at the recording's current
|
||||
* position; the audio after it is placed afresh on the recording's timeline.
|
||||
*/
|
||||
@Synchronized
|
||||
fun noteRecordingEvent(json: String) {
|
||||
recording?.event(json)
|
||||
}
|
||||
|
||||
/** Complete the recording's file. Returns its length in ms, or -1 when none was open. */
|
||||
@Synchronized
|
||||
fun stopRecording(): Long {
|
||||
val current = recording ?: return -1
|
||||
recording = null
|
||||
session.setRecorder(null)
|
||||
return current.finish()
|
||||
}
|
||||
|
||||
fun start(requestedMode: String?) {
|
||||
session.start(requestedMode, 0)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/**
|
||||
* One SSH channel for the TypeScript side (app/native/ssh.ios.ts), backed by
|
||||
* the gomobile-bound Go transport in native/ssh. `spec` is the Go openSpec
|
||||
* JSON. Events arrive on Go threads and queue here; the owning JS isolate
|
||||
* drains them with takeEvents (a JSON array of {kind, payload}).
|
||||
*/
|
||||
@interface FaceclawSshChannel : NSObject
|
||||
- (instancetype)initWithSpec:(NSString *)spec useKey:(BOOL)useKey;
|
||||
- (NSString *)takeEvents;
|
||||
- (BOOL)send:(NSString *)text;
|
||||
- (void)close;
|
||||
@end
|
||||
|
||||
/**
|
||||
* The device's SSH client key (a non-exportable Secure Enclave P-256 key; a
|
||||
* plain keychain key on the simulator, which has no Secure Enclave) and the
|
||||
* Terminal app's saved SSH passwords (keychain generic passwords).
|
||||
*/
|
||||
@interface FaceclawSshKeys : NSObject
|
||||
/** "ecdsa-sha2-nistp256 AAAA... comment"; generates the key if needed. nil on failure. */
|
||||
+ (NSString *)authorizedKeyWithComment:(NSString *)comment;
|
||||
/** "SHA256:..." fingerprint; generates the key if needed. nil on failure. */
|
||||
+ (NSString *)fingerprint;
|
||||
+ (void)setSecret:(NSString *)value forId:(NSString *)secretId;
|
||||
/** The saved secret, or "" if none. */
|
||||
+ (NSString *)secretForId:(NSString *)secretId;
|
||||
+ (void)removeSecretForId:(NSString *)secretId;
|
||||
+ (void)copyToClipboard:(NSString *)text;
|
||||
@end
|
||||
@@ -0,0 +1,234 @@
|
||||
#import "FaceclawSsh.h"
|
||||
#import <Faceclawssh/Faceclawssh.h>
|
||||
#import <Security/Security.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
static NSString *const kClientKeyTag = @"com.faceclaw.ssh.client";
|
||||
static NSString *const kSecretService = @"com.faceclaw.ssh.secret";
|
||||
|
||||
#pragma mark - Client key
|
||||
|
||||
static NSData *ClientKeyTagData(void) {
|
||||
return [kClientKeyTag dataUsingEncoding:NSUTF8StringEncoding];
|
||||
}
|
||||
|
||||
static SecKeyRef CopyClientKey(void) {
|
||||
NSDictionary *query = @{
|
||||
(id)kSecClass: (id)kSecClassKey,
|
||||
(id)kSecAttrApplicationTag: ClientKeyTagData(),
|
||||
(id)kSecAttrKeyType: (id)kSecAttrKeyTypeECSECPrimeRandom,
|
||||
(id)kSecReturnRef: @YES,
|
||||
};
|
||||
CFTypeRef key = NULL;
|
||||
if (SecItemCopyMatching((__bridge CFDictionaryRef)query, &key) != errSecSuccess) return NULL;
|
||||
return (SecKeyRef)key;
|
||||
}
|
||||
|
||||
static SecKeyRef CreateClientKey(BOOL secureEnclave) {
|
||||
SecAccessControlRef access = SecAccessControlCreateWithFlags(
|
||||
kCFAllocatorDefault, kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly,
|
||||
secureEnclave ? kSecAccessControlPrivateKeyUsage : 0, NULL);
|
||||
if (!access) return NULL;
|
||||
NSMutableDictionary *attributes = [@{
|
||||
(id)kSecAttrKeyType: (id)kSecAttrKeyTypeECSECPrimeRandom,
|
||||
(id)kSecAttrKeySizeInBits: @256,
|
||||
(id)kSecPrivateKeyAttrs: @{
|
||||
(id)kSecAttrIsPermanent: @YES,
|
||||
(id)kSecAttrApplicationTag: ClientKeyTagData(),
|
||||
(id)kSecAttrAccessControl: (__bridge id)access,
|
||||
},
|
||||
} mutableCopy];
|
||||
if (secureEnclave) attributes[(id)kSecAttrTokenID] = (id)kSecAttrTokenIDSecureEnclave;
|
||||
CFErrorRef error = NULL;
|
||||
SecKeyRef key = SecKeyCreateRandomKey((__bridge CFDictionaryRef)attributes, &error);
|
||||
CFRelease(access);
|
||||
if (!key && error) {
|
||||
NSLog(@"FaceclawSsh: key generation failed (secure enclave %d): %@", secureEnclave, (__bridge NSError *)error);
|
||||
CFRelease(error);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
/** The client key, generating it on first use. Caller releases. */
|
||||
static SecKeyRef CopyOrCreateClientKey(void) {
|
||||
@synchronized([FaceclawSshKeys class]) {
|
||||
SecKeyRef key = CopyClientKey();
|
||||
if (key) return key;
|
||||
key = CreateClientKey(YES);
|
||||
#if TARGET_OS_SIMULATOR
|
||||
if (!key) key = CreateClientKey(NO);
|
||||
#endif
|
||||
return key;
|
||||
}
|
||||
}
|
||||
|
||||
/** Uncompressed X9.62 public point (0x04 || X || Y). */
|
||||
static NSData *PublicKeyData(SecKeyRef key) {
|
||||
SecKeyRef publicKey = SecKeyCopyPublicKey(key);
|
||||
if (!publicKey) return nil;
|
||||
NSData *data = (__bridge_transfer NSData *)SecKeyCopyExternalRepresentation(publicKey, NULL);
|
||||
CFRelease(publicKey);
|
||||
return data;
|
||||
}
|
||||
|
||||
@interface FaceclawSshSigner : NSObject <FaceclawsshSigner>
|
||||
@end
|
||||
|
||||
@implementation FaceclawSshSigner {
|
||||
SecKeyRef _key;
|
||||
NSData *_publicKey;
|
||||
}
|
||||
- (instancetype)initWithKey:(SecKeyRef)key {
|
||||
if ((self = [super init])) {
|
||||
_key = (SecKeyRef)CFRetain(key);
|
||||
_publicKey = PublicKeyData(key);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
- (void)dealloc {
|
||||
if (_key) CFRelease(_key);
|
||||
}
|
||||
- (NSData *)publicKey {
|
||||
return _publicKey;
|
||||
}
|
||||
- (NSData *)sign:(NSData *)data error:(NSError **)error {
|
||||
CFErrorRef cfError = NULL;
|
||||
NSData *signature = (__bridge_transfer NSData *)SecKeyCreateSignature(
|
||||
_key, kSecKeyAlgorithmECDSASignatureMessageX962SHA256, (__bridge CFDataRef)(data ?: [NSData data]), &cfError);
|
||||
if (!signature) {
|
||||
if (error) *error = (__bridge_transfer NSError *)cfError;
|
||||
else if (cfError) CFRelease(cfError);
|
||||
}
|
||||
return signature;
|
||||
}
|
||||
@end
|
||||
|
||||
#pragma mark - Channel
|
||||
|
||||
@interface FaceclawSshEventQueue : NSObject <FaceclawsshListener>
|
||||
@property(nonatomic, strong) NSMutableArray *events;
|
||||
@property(nonatomic) BOOL ended;
|
||||
@end
|
||||
|
||||
@implementation FaceclawSshEventQueue
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) _events = [NSMutableArray array];
|
||||
return self;
|
||||
}
|
||||
- (void)onEvent:(NSString *)kind payload:(NSString *)payload {
|
||||
@synchronized(self) {
|
||||
if (_ended) return;
|
||||
[_events addObject:@{@"kind": kind ?: @"", @"payload": payload ?: @""}];
|
||||
}
|
||||
}
|
||||
@end
|
||||
|
||||
@implementation FaceclawSshChannel {
|
||||
FaceclawsshChannel *_channel;
|
||||
FaceclawSshEventQueue *_queue;
|
||||
}
|
||||
|
||||
- (instancetype)initWithSpec:(NSString *)spec useKey:(BOOL)useKey {
|
||||
if ((self = [super init])) {
|
||||
_queue = [FaceclawSshEventQueue new];
|
||||
FaceclawSshSigner *signer = nil;
|
||||
if (useKey) {
|
||||
SecKeyRef key = CopyOrCreateClientKey();
|
||||
if (key) {
|
||||
signer = [[FaceclawSshSigner alloc] initWithKey:key];
|
||||
CFRelease(key);
|
||||
}
|
||||
}
|
||||
_channel = FaceclawsshOpen(spec, signer, _queue);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (NSString *)takeEvents {
|
||||
NSArray *events;
|
||||
@synchronized(_queue) {
|
||||
if (_queue.events.count == 0) return @"[]";
|
||||
events = [_queue.events copy];
|
||||
[_queue.events removeAllObjects];
|
||||
}
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:events options:0 error:nil];
|
||||
return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"[]";
|
||||
}
|
||||
|
||||
- (BOOL)send:(NSString *)text {
|
||||
return [_channel send:text];
|
||||
}
|
||||
|
||||
- (void)close {
|
||||
@synchronized(_queue) {
|
||||
_queue.ended = YES;
|
||||
[_queue.events removeAllObjects];
|
||||
}
|
||||
[_channel close];
|
||||
}
|
||||
@end
|
||||
|
||||
#pragma mark - Keys and secrets
|
||||
|
||||
@implementation FaceclawSshKeys
|
||||
|
||||
+ (NSString *)authorizedKeyWithComment:(NSString *)comment {
|
||||
SecKeyRef key = CopyOrCreateClientKey();
|
||||
if (!key) return nil;
|
||||
NSData *publicKey = PublicKeyData(key);
|
||||
CFRelease(key);
|
||||
NSError *error = nil;
|
||||
NSString *line = FaceclawsshAuthorizedKey(publicKey, comment ?: @"", &error);
|
||||
if (error) NSLog(@"FaceclawSsh: authorizedKey failed: %@", error);
|
||||
return line;
|
||||
}
|
||||
|
||||
+ (NSString *)fingerprint {
|
||||
SecKeyRef key = CopyOrCreateClientKey();
|
||||
if (!key) return nil;
|
||||
NSData *publicKey = PublicKeyData(key);
|
||||
CFRelease(key);
|
||||
NSError *error = nil;
|
||||
NSString *fingerprint = FaceclawsshFingerprint(publicKey, &error);
|
||||
if (error) NSLog(@"FaceclawSsh: fingerprint failed: %@", error);
|
||||
return fingerprint;
|
||||
}
|
||||
|
||||
+ (NSDictionary *)secretQuery:(NSString *)secretId {
|
||||
return @{
|
||||
(id)kSecClass: (id)kSecClassGenericPassword,
|
||||
(id)kSecAttrService: kSecretService,
|
||||
(id)kSecAttrAccount: secretId ?: @"",
|
||||
};
|
||||
}
|
||||
|
||||
+ (void)setSecret:(NSString *)value forId:(NSString *)secretId {
|
||||
NSDictionary *query = [self secretQuery:secretId];
|
||||
SecItemDelete((__bridge CFDictionaryRef)query);
|
||||
NSMutableDictionary *item = [query mutableCopy];
|
||||
item[(id)kSecValueData] = [(value ?: @"") dataUsingEncoding:NSUTF8StringEncoding];
|
||||
item[(id)kSecAttrAccessible] = (id)kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly;
|
||||
OSStatus status = SecItemAdd((__bridge CFDictionaryRef)item, NULL);
|
||||
if (status != errSecSuccess) NSLog(@"FaceclawSsh: saving secret failed: %d", (int)status);
|
||||
}
|
||||
|
||||
+ (NSString *)secretForId:(NSString *)secretId {
|
||||
NSMutableDictionary *query = [[self secretQuery:secretId] mutableCopy];
|
||||
query[(id)kSecReturnData] = @YES;
|
||||
query[(id)kSecMatchLimit] = (id)kSecMatchLimitOne;
|
||||
CFTypeRef result = NULL;
|
||||
if (SecItemCopyMatching((__bridge CFDictionaryRef)query, &result) != errSecSuccess) return @"";
|
||||
NSData *data = (__bridge_transfer NSData *)result;
|
||||
return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"";
|
||||
}
|
||||
|
||||
+ (void)removeSecretForId:(NSString *)secretId {
|
||||
SecItemDelete((__bridge CFDictionaryRef)[self secretQuery:secretId]);
|
||||
}
|
||||
|
||||
+ (void)copyToClipboard:(NSString *)text {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
UIPasteboard.generalPasteboard.string = text ?: @"";
|
||||
});
|
||||
}
|
||||
@end
|
||||
@@ -2,6 +2,7 @@ module FaceclawGraphics {
|
||||
header "FaceclawGraphics.h"
|
||||
header "FaceclawFontRenderer.h"
|
||||
header "FaceclawSocket.h"
|
||||
header "FaceclawSsh.h"
|
||||
header "FaceclawLc3Decoder.h"
|
||||
header "FaceclawSpeech.h"
|
||||
header "FaceclawLocation.h"
|
||||
|
||||
@@ -80,8 +80,10 @@ import it into the new version (see [config transfer](scripts/config-transfer.md
|
||||
* **Integration with Android notifications**: shows the same notification
|
||||
icons your phone does, and popups when notifications arrive.
|
||||
* **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex
|
||||
CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on
|
||||
the glasses, and send them inputs with the voice assistant.
|
||||
CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), or attach to
|
||||
tmux sessions over SSH (no server-side setup beyond tmux 3.2+), view them
|
||||
on the glasses, and send them inputs with the voice assistant. g2mirror
|
||||
can also be tunnelled over SSH instead of a VPN.
|
||||
* **Media player controls** including playlist and media library navigation,
|
||||
compatible with most Android media players.
|
||||
* **Turn-by-turn directions** (requires a Mapbox API token).
|
||||
|
||||
@@ -55,8 +55,10 @@ export type GlanceboardProvider = {
|
||||
isEnabled: () => boolean;
|
||||
/** Whether a single tap shows the board (for tapTimeoutMs). */
|
||||
showOnTap: () => boolean;
|
||||
/** How long a tap or head-tilt keeps the board up. */
|
||||
/** How long a tap keeps the board up (and a head-tilt, when its head-down is unlikely to come). */
|
||||
tapTimeoutMs: () => number;
|
||||
/** How long a head-tilt showing may last without a head-down (null = no limit). */
|
||||
tiltTimeoutMs: () => number | null;
|
||||
/** Whether a long-press holds the board up until released. */
|
||||
showOnLongPress: () => boolean;
|
||||
/** Whether the head-tilt wake goes to the board rather than the regular UI. */
|
||||
@@ -187,6 +189,8 @@ export async function launchWorkerAppWindow(
|
||||
matchExistingBy?: "appId" | "windowId";
|
||||
/** Deliver watch swipes raw instead of the fallback (see WorkerWindowSpec). */
|
||||
acceptsDirectional?: boolean;
|
||||
/** Deliver hardware-keyboard keys as keys (see WorkerWindowSpec). */
|
||||
acceptsKeys?: boolean;
|
||||
},
|
||||
): Promise<void> {
|
||||
const existing = shell
|
||||
@@ -207,6 +211,7 @@ export async function launchWorkerAppWindow(
|
||||
icon: options.icon,
|
||||
focus: true,
|
||||
acceptsDirectional: options.acceptsDirectional,
|
||||
acceptsKeys: options.acceptsKeys,
|
||||
});
|
||||
ctx.appendLog(`launched ${options.windowId}`);
|
||||
}
|
||||
@@ -312,6 +312,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import { hasCalendarPermission } from "../../native/calendar-permissions";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import type { Viewport } from "../../ui/viewport";
|
||||
import { listingDayMs, spanLabel, spansDays } from "./event-days";
|
||||
|
||||
/** How far a row's text sits inside its selection box; the title lines up with it. */
|
||||
const ROW_TEXT_INSET = 10;
|
||||
@@ -143,14 +144,20 @@ export class CalendarLayer implements Layer {
|
||||
function buildEventRows(font: UiFont, events: CalendarEvent[]): EventRow[] {
|
||||
const rows: EventRow[] = [];
|
||||
let previousDayKey = "";
|
||||
const nowMs = Date.now();
|
||||
for (const event of events) {
|
||||
const dayKey = dayKeyOf(event.startMs);
|
||||
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(event.startMs);
|
||||
// An event still going since an earlier day is listed under today, not its first day.
|
||||
const listedMs = listingDayMs(event, nowMs);
|
||||
const dayKey = dayKeyOf(listedMs);
|
||||
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(listedMs);
|
||||
previousDayKey = dayKey;
|
||||
|
||||
const lines: string[] = [];
|
||||
const timeLabel = event.allDay ? "All day" : formatEventTime(event.startMs);
|
||||
lines.push(`${timeLabel} ${event.title || "(untitled)"}`);
|
||||
if (spansDays(event)) {
|
||||
lines.push(spanLabel(event));
|
||||
}
|
||||
if (event.location) {
|
||||
lines.push(event.location);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Which day an event is listed under, and how a multi-day span reads. Pure (no
|
||||
* native or UI imports) so it can be unit-tested.
|
||||
*/
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
export type DayEvent = { startMs: number; endMs: number; allDay: boolean };
|
||||
|
||||
function midnight(timestampMs: number): number {
|
||||
const date = new Date(timestampMs);
|
||||
return new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
|
||||
}
|
||||
|
||||
/** The last moment the event covers: all-day ends are exclusive (the midnight after its last day). */
|
||||
function lastMomentMs(event: DayEvent): number {
|
||||
return event.allDay ? Math.max(event.startMs, event.endMs - 1) : Math.max(event.startMs, event.endMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* The day to list an event under: its start, or today for one that began on an
|
||||
* earlier day and is still going. Without this, the third day of a three-day
|
||||
* event shows under its first day and never under today.
|
||||
*/
|
||||
export function listingDayMs(event: DayEvent, nowMs: number): number {
|
||||
return Math.max(midnight(event.startMs), Math.min(midnight(nowMs), midnight(lastMomentMs(event))));
|
||||
}
|
||||
|
||||
/** Whether the event covers more than one calendar day. */
|
||||
export function spansDays(event: DayEvent): boolean {
|
||||
return midnight(lastMomentMs(event)) > midnight(event.startMs);
|
||||
}
|
||||
|
||||
/** "Mon 16 – Wed 18 Nov", or with both months when they differ ("Mon 30 Nov – Tue 1 Dec"). */
|
||||
export function spanLabel(event: DayEvent): string {
|
||||
const start = new Date(event.startMs);
|
||||
const last = new Date(lastMomentMs(event));
|
||||
const day = (date: Date) => `${WEEKDAYS[date.getDay()]} ${date.getDate()}`;
|
||||
return start.getMonth() === last.getMonth() && start.getFullYear() === last.getFullYear()
|
||||
? `${day(start)} – ${day(last)} ${MONTHS[last.getMonth()]}`
|
||||
: `${day(start)} ${MONTHS[start.getMonth()]} – ${day(last)} ${MONTHS[last.getMonth()]}`;
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
import { addImuListener, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
|
||||
import { addImuListener, imuPitchDegrees, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
@@ -23,7 +23,8 @@ const BAR_RIGHT_INSET = 4;
|
||||
/**
|
||||
* Accelerometer demo: turns the IMU stream on while this page is visible (the
|
||||
* Developer window is foregrounded and the screen is on), off otherwise, and
|
||||
* shows the latest x/y/z reading with a simple per-axis bar.
|
||||
* shows the latest x/y/z reading (Faceclaw's frame, imu-types.ts) with a
|
||||
* simple per-axis bar, and the head's pitch.
|
||||
*
|
||||
* The IMU stream is experimental — the firmware may never populate it — so the
|
||||
* page also reports how many samples have arrived.
|
||||
@@ -92,7 +93,8 @@ export class AccelerometerDemoLayer implements Layer {
|
||||
const bodyX = padding.left + BODY_INDENT;
|
||||
|
||||
image.drawText(small, padding.left, top, "Accelerometer", 220);
|
||||
const status = this.enabled ? "IMU on" : "IMU off";
|
||||
const pitch = this.reading ? `pitch ${Math.round(imuPitchDegrees(this.reading))}° ` : "";
|
||||
const status = pitch + (this.enabled ? "IMU on" : "IMU off");
|
||||
image.drawText(small, right - small.measureText(status), top, status, 150);
|
||||
|
||||
if (this.reading) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo";
|
||||
import { AccelerometerDemoLayer } from "./accelerometer-demo";
|
||||
import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark";
|
||||
import { LightSensorDemoLayer } from "./light-sensor-demo";
|
||||
import { MouseTestLayer } from "./mouse-test";
|
||||
import { MicrophonesMonitorLayer } from "./microphones-monitor";
|
||||
import { ResourceUsageLayer } from "./resource-usage";
|
||||
import { RingStatusLayer } from "./ring-status";
|
||||
@@ -70,6 +71,13 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
label: "Microphones",
|
||||
onSelect: openMicrophones,
|
||||
},
|
||||
{
|
||||
// The touchpad or mouse of a keyboard paired through the glasses.
|
||||
label: "Mouse cursor",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new MouseTestLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "BLE bandwidth",
|
||||
onSelect: (ctx) => {
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import type { HidPointerSample } from "../../g2/keyboard/hid";
|
||||
import { keyboardService } from "../../g2/keyboard/service";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
|
||||
// How often to reconcile holding the pointer with window visibility.
|
||||
const RECONCILE_INTERVAL_MS = 400;
|
||||
/** A touchpad reports a hundred times a second; the page repaints at most this often. */
|
||||
const RENDER_INTERVAL_MS = 33;
|
||||
/** Cursor pixels per count the touchpad reports. */
|
||||
const POINTER_GAIN = 1;
|
||||
/** Left-click marks kept on the page. */
|
||||
const MAX_MARKS = 12;
|
||||
|
||||
/** The arrow, tip at its top left: '#' is the outline, '.' the fill. */
|
||||
const CURSOR = [
|
||||
"#",
|
||||
"##",
|
||||
"#.#",
|
||||
"#..#",
|
||||
"#...#",
|
||||
"#....#",
|
||||
"#.....#",
|
||||
"#......#",
|
||||
"#.......#",
|
||||
"#........#",
|
||||
"#.....#####",
|
||||
"#..#..#",
|
||||
"#.# #..#",
|
||||
"## #..#",
|
||||
"# #..#",
|
||||
" #..#",
|
||||
" ##",
|
||||
];
|
||||
|
||||
/**
|
||||
* Mouse cursor test: moves an arrow with the touchpad or mouse of the
|
||||
* keyboard paired through the glasses (app/g2/keyboard/), and shows its
|
||||
* buttons and wheel. The main UI has no cursor; this is where one is tried
|
||||
* out.
|
||||
*
|
||||
* While the page is visible it takes the pointer, so the buttons and wheel
|
||||
* show here instead of acting as select, back and scroll. A left click
|
||||
* leaves a mark where the arrow is and a right click clears them. Leave
|
||||
* with Esc, or a double tap on the ring or a temple.
|
||||
*/
|
||||
export class MouseTestLayer implements Layer {
|
||||
private x = -1;
|
||||
private y = -1;
|
||||
private size = { width: 0, height: 0 };
|
||||
private buttons = 0;
|
||||
private wheel = 0;
|
||||
private marks: Array<{ x: number; y: number }> = [];
|
||||
private samples = 0;
|
||||
private rate = 0;
|
||||
private rateWindowStart = Date.now();
|
||||
private removed = false;
|
||||
private release: (() => void) | null = null;
|
||||
private reconcileTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private renderTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly windowId: string,
|
||||
private readonly requestRender: () => void,
|
||||
) {
|
||||
this.reconcileTimer = setInterval(() => this.reconcile(), RECONCILE_INTERVAL_MS);
|
||||
this.reconcile();
|
||||
}
|
||||
|
||||
/** Hold the pointer only while the page can be seen; elsewhere its buttons go back to being keys. */
|
||||
private reconcile(): void {
|
||||
if (this.removed) return;
|
||||
const visible = shell.isWindowVisible(this.windowId);
|
||||
if (visible && !this.release) {
|
||||
this.release = keyboardService.takePointer((sample) => this.onSample(sample));
|
||||
} else if (!visible && this.release) {
|
||||
this.release();
|
||||
this.release = null;
|
||||
this.buttons = 0;
|
||||
}
|
||||
// The rate reads zero once the pointer has been still for a second.
|
||||
const now = Date.now();
|
||||
if (now - this.rateWindowStart >= 1000) {
|
||||
const rate = this.samples;
|
||||
this.samples = 0;
|
||||
this.rateWindowStart = now;
|
||||
if (rate !== this.rate) {
|
||||
this.rate = rate;
|
||||
this.scheduleRender();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private onSample(sample: HidPointerSample): void {
|
||||
if (this.removed) return;
|
||||
this.samples++;
|
||||
const { width, height } = this.size;
|
||||
if (width > 0 && height > 0) {
|
||||
if (this.x < 0) {
|
||||
this.x = width / 2;
|
||||
this.y = height / 2;
|
||||
}
|
||||
this.x = clamp(this.x + sample.dx * POINTER_GAIN, 0, width - 1);
|
||||
this.y = clamp(this.y + sample.dy * POINTER_GAIN, 0, height - 1);
|
||||
}
|
||||
const pressed = sample.buttons & ~this.buttons;
|
||||
this.buttons = sample.buttons;
|
||||
this.wheel += sample.wheel;
|
||||
if (pressed & 1 && this.x >= 0) {
|
||||
this.marks.push({ x: Math.round(this.x), y: Math.round(this.y) });
|
||||
if (this.marks.length > MAX_MARKS) this.marks.shift();
|
||||
}
|
||||
if (pressed & 2) this.marks = [];
|
||||
this.scheduleRender();
|
||||
}
|
||||
|
||||
private scheduleRender(): void {
|
||||
if (this.renderTimer !== null || this.removed) return;
|
||||
this.renderTimer = setTimeout(() => {
|
||||
this.renderTimer = null;
|
||||
if (!this.removed) this.requestRender();
|
||||
}, RENDER_INTERVAL_MS);
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
if (this.reconcileTimer !== null) clearInterval(this.reconcileTimer);
|
||||
this.reconcileTimer = null;
|
||||
if (this.renderTimer !== null) clearTimeout(this.renderTimer);
|
||||
this.renderTimer = null;
|
||||
this.release?.();
|
||||
this.release = null;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const { width, height, padding } = ctx.stack.getBaseSize();
|
||||
this.size = { width, height };
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const left = padding.left;
|
||||
const right = width - padding.right;
|
||||
const top = padding.top;
|
||||
const line = font.lineHeight + 4;
|
||||
|
||||
image.drawText(font, left, top, "Mouse cursor", 220);
|
||||
const snapshot = keyboardService.snapshot();
|
||||
const problem = snapshot.state !== "connected"
|
||||
? "No keyboard is connected."
|
||||
: !snapshot.pointer ? "This keyboard has no touchpad or mouse buttons." : null;
|
||||
if (problem) {
|
||||
image.drawText(font, left, top + 2 * line, problem, 190);
|
||||
image.drawText(font, left, top + 3 * line, "Pair one under Settings > Keyboard.", 120);
|
||||
return image;
|
||||
}
|
||||
|
||||
const position = this.x < 0 ? "move the pointer" : `${Math.round(this.x)}, ${Math.round(this.y)}`;
|
||||
image.drawText(font, right - font.measureText(position), top, position, 150);
|
||||
// Buttons: a box each, filled while held.
|
||||
let x = left;
|
||||
const boxY = top + line;
|
||||
for (const [index, label] of ["L", "M", "R"].entries()) {
|
||||
// HID numbers them left, right, middle; they sit left, middle, right.
|
||||
const held = !!(this.buttons & [1, 4, 2][index]!);
|
||||
const boxW = font.measureText(label) + 10;
|
||||
if (held) image.fillRect(x, boxY, boxW, font.lineHeight + 2, 200);
|
||||
else image.drawRect(x, boxY, boxW, font.lineHeight + 2, 90);
|
||||
image.drawText(font, x + 5, boxY + 1, label, held ? 1 : 170);
|
||||
x += boxW + 4;
|
||||
}
|
||||
image.drawText(font, x + 8, boxY + 1, `wheel ${this.wheel}`, 150);
|
||||
const rate = `${this.rate}/s`;
|
||||
image.drawText(font, right - font.measureText(rate), boxY + 1, rate, 110);
|
||||
const hint = "Left click marks, right click clears. Esc or double tap: back";
|
||||
image.drawText(font, left, height - padding.bottom - font.lineHeight, hint, 90);
|
||||
|
||||
for (const mark of this.marks) {
|
||||
image.drawLine(mark.x - 4, mark.y, mark.x + 4, mark.y, 140);
|
||||
image.drawLine(mark.x, mark.y - 4, mark.x, mark.y + 4, 140);
|
||||
}
|
||||
if (this.x >= 0) drawCursor(image, Math.round(this.x), Math.round(this.y));
|
||||
return image;
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function drawCursor(image: GrayImage, x: number, y: number): void {
|
||||
for (const [row, pixels] of CURSOR.entries()) {
|
||||
for (let col = 0; col < pixels.length; col++) {
|
||||
const pixel = pixels[col];
|
||||
if (pixel === " ") continue;
|
||||
const px = x + col;
|
||||
const py = y + row;
|
||||
if (px < 0 || py < 0 || px >= image.width || py >= image.height) continue;
|
||||
image.setPixel(px, py, pixel === "#" ? 255 : 70);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
|
||||
import { isFontFile } from "../../native/font-files";
|
||||
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
|
||||
import { isDecodableImageFile } from "../../native/image-files";
|
||||
import { isMicRecordingFile } from "../../native/mic-recording";
|
||||
import { isPlayableVideoFile } from "../../native/video-player";
|
||||
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
|
||||
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
|
||||
@@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & {
|
||||
openFontWindow: (title: string, path: string) => void;
|
||||
/** Open a video file's player (settings panel first) as its own shell window. */
|
||||
openVideoWindow: (title: string, path: string) => void;
|
||||
/** Replay a recording (.mka) through Transcribe, in a window of its own. */
|
||||
openRecordingInTranscribe: (title: string, path: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow
|
||||
const browser = new FileBrowserLayer({
|
||||
isSupportedFile: (name) =>
|
||||
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
|
||||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))),
|
||||
// The browser handles double-click itself (up a level), so it is not
|
||||
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
|
||||
onLeave: () => shell.yieldFocusToSidebar(),
|
||||
@@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isMicRecordingFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
label: "Open in Transcribe",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
options.openRecordingInTranscribe(entry.name, entry.path);
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
if (isPlayableVideoFile(entry.name)) {
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
FILES_WINDOW_ID,
|
||||
} from "./files-app";
|
||||
import { createVideoPlayerWindow } from "./video-window";
|
||||
import { openRecordingInTranscribe } from "../transcribe";
|
||||
|
||||
// Document windows are closeable and non-singleton; the serial keeps their
|
||||
// windowIds unique.
|
||||
@@ -91,6 +92,9 @@ const filesApp: AppDefinition = {
|
||||
},
|
||||
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
|
||||
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
|
||||
openRecordingInTranscribe: (title, path) => {
|
||||
openRecordingInTranscribe(ctx, path, title);
|
||||
},
|
||||
}),
|
||||
),
|
||||
// A document arriving via Android's Share intent opens as its own window.
|
||||
|
||||
@@ -305,6 +305,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
break;
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -236,6 +236,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
case "navigation-sensors":
|
||||
case "screen":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from
|
||||
import { uiFontOverridePickerMenuItem } from "../../ui/font-picker";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { drawSelectionHighlight, isMenuItemDisabled, menuHeadingItem, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { liveHeadPitchFooter } from "../../g2/head-angle";
|
||||
import { headUpControlsAvailable } from "../../native/glasses-controls";
|
||||
import { Menu } from "../../ui/menu-core";
|
||||
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
|
||||
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
@@ -20,7 +22,9 @@ import {
|
||||
glanceLayout,
|
||||
glanceLayoutSetting,
|
||||
glanceShowLinesSetting,
|
||||
glanceAngleSetting,
|
||||
glanceShowOnHeadTiltSetting,
|
||||
glanceTiltTimeoutSetting,
|
||||
glanceShowOnLongPressSetting,
|
||||
glanceSlotChoiceLabel,
|
||||
glanceSlotSettings,
|
||||
@@ -319,25 +323,45 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
|
||||
}
|
||||
|
||||
/**
|
||||
* Settings: layout, sleep gestures, duration, slot lines, widget font, and
|
||||
* stereo depth, then a row per widget on the board for its own settings.
|
||||
* Settings, in four sections: Layout (the slot grid), Trigger (the sleep
|
||||
* gestures that show the board, and for how long), Display (slot lines,
|
||||
* widget font, stereo depth) and Cards (a row per widget on the board, for
|
||||
* its own settings; left out while the board is empty).
|
||||
*/
|
||||
function openGlanceSettings(ctx: LayerContext): void {
|
||||
const items = (): MenuItem[] => [
|
||||
// The layout decides which slots (and so which widgets) are on the board.
|
||||
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
|
||||
enumSettingMenuItem(glanceTapDurationSetting),
|
||||
toggleSettingMenuItem(glanceShowOnLongPressSetting),
|
||||
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
|
||||
toggleSettingMenuItem(glanceShowLinesSetting),
|
||||
uiFontOverridePickerMenuItem(glanceFont, {
|
||||
title: "Glanceboard font",
|
||||
description:
|
||||
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
|
||||
}),
|
||||
enumSettingMenuItem(glanceDepthSetting),
|
||||
...widgetSettingsMenuItems(),
|
||||
];
|
||||
const items = (): MenuItem[] => {
|
||||
const cards = widgetSettingsMenuItems();
|
||||
return [
|
||||
menuHeadingItem("Layout"),
|
||||
// The layout decides which slots (and so which widgets) are on the board.
|
||||
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
|
||||
|
||||
menuHeadingItem("Trigger"),
|
||||
enumSettingMenuItem(glanceTapDurationSetting),
|
||||
toggleSettingMenuItem(glanceShowOnLongPressSetting),
|
||||
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
|
||||
enumSettingMenuItem(glanceTiltTimeoutSetting),
|
||||
// Only Android can change the glasses' head-up setting (glasses-controls.ts).
|
||||
// The controller's settings listener sends a new angle to the glasses; the
|
||||
// picker shows the head's current angle to choose by.
|
||||
...(headUpControlsAvailable()
|
||||
? [enumSettingMenuItem(glanceAngleSetting, {
|
||||
pickerFooter: liveHeadPitchFooter(() => shell.isWindowVisible(GLANCEBOARD_WINDOW_ID)),
|
||||
})]
|
||||
: []),
|
||||
|
||||
menuHeadingItem("Display"),
|
||||
toggleSettingMenuItem(glanceShowLinesSetting),
|
||||
uiFontOverridePickerMenuItem(glanceFont, {
|
||||
title: "Glanceboard font",
|
||||
description:
|
||||
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
|
||||
}),
|
||||
enumSettingMenuItem(glanceDepthSetting),
|
||||
|
||||
...(cards.length ? [menuHeadingItem("Cards"), ...cards] : []),
|
||||
];
|
||||
};
|
||||
const layer = new MenuLayer("Glanceboard settings", items(), PAGE_MENU_LAYOUT);
|
||||
ctx.stack.push(layer);
|
||||
}
|
||||
|
||||
@@ -49,8 +49,9 @@ export function glanceLayout(): GlanceLayout {
|
||||
|
||||
/**
|
||||
* Whether sleep-time gestures show the board at all. Off restores the plain
|
||||
* behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt
|
||||
* wakes the regular UI), double-tap wakes.
|
||||
* behaviour: a tap or hold while asleep does nothing, a head-tilt wakes the
|
||||
* regular UI, double-tap wakes; and the head-up angle goes back to the glasses'
|
||||
* own.
|
||||
*/
|
||||
export const glanceboardEnabledSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-enabled",
|
||||
@@ -85,9 +86,10 @@ function formatTapDuration(value: GlanceTapDuration): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* How long a tap (or head-tilt) keeps the board up; each further tap restarts
|
||||
* it. "off" means a single tap does not show the board at all; a head-tilt
|
||||
* then uses the default duration.
|
||||
* How long a tap keeps the board up; each further tap restarts it. "off"
|
||||
* means a single tap does not show the board at all. A head-tilt showing
|
||||
* lasts until the head comes back down, falling back to this duration (the
|
||||
* default one when "off") where the head-down is unlikely to arrive.
|
||||
*/
|
||||
export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>({
|
||||
id: "glanceboard-tap-duration",
|
||||
@@ -97,7 +99,7 @@ export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>
|
||||
values: TAP_DURATION_VALUES,
|
||||
formatValue: formatTapDuration,
|
||||
description:
|
||||
"How long a single tap (or a head-tilt) keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
|
||||
"How long a single tap keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
|
||||
});
|
||||
|
||||
/** Whether a single tap while asleep shows the board. */
|
||||
@@ -119,15 +121,71 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({
|
||||
description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.",
|
||||
});
|
||||
|
||||
/** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */
|
||||
/** The head-tilt wake shows the board (until the head comes back down) instead of the regular UI. */
|
||||
export const glanceShowOnHeadTiltSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-show-on-head-tilt",
|
||||
label: "Show on head tilt",
|
||||
storageKey: "glanceboard.showOnHeadTilt",
|
||||
defaultValue: true,
|
||||
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI.",
|
||||
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI. It stays up until you look back down.",
|
||||
});
|
||||
|
||||
export type GlanceTiltTimeout = "60s" | "30s" | "15s" | "10s" | "never";
|
||||
|
||||
/**
|
||||
* How long a head-tilt showing may last if the head-down never comes (the
|
||||
* board normally hides when the head comes back down). "never" removes the
|
||||
* limit; a double-tap still dismisses the board.
|
||||
*/
|
||||
export const glanceTiltTimeoutSetting = new ConfigSettingEnum<GlanceTiltTimeout>({
|
||||
id: "glanceboard-head-tilt-timeout",
|
||||
label: "Head tilt timeout",
|
||||
storageKey: "glanceboard.headTiltTimeout",
|
||||
defaultValue: "60s",
|
||||
values: ["60s", "30s", "15s", "10s", "never"],
|
||||
formatValue: (value) => (value === "never" ? "Never" : value.replace("s", " s")),
|
||||
description:
|
||||
"The longest a head-tilt keeps the Glanceboard up when looking back down is not detected. Never: it stays until you look down or double-tap. " +
|
||||
"When looking down is unlikely to register (a low Head tilt angle, or the display was suspended), the tap duration applies if shorter.",
|
||||
});
|
||||
|
||||
/** The Head tilt timeout in ms, or null for none. */
|
||||
export function glanceTiltTimeoutMs(): number | null {
|
||||
const value = glanceTiltTimeoutSetting.get();
|
||||
return value === "never" ? null : Number.parseInt(value, 10) * 1000;
|
||||
}
|
||||
|
||||
const GLANCE_ANGLE_VALUES = ["glasses", "5", "10", "15", "20", "25", "30", "35", "40", "45", "50", "55", "60"] as const;
|
||||
export type GlanceAngle = (typeof GLANCE_ANGLE_VALUES)[number];
|
||||
|
||||
/**
|
||||
* How far the head must tilt up for a head-tilt wake, as the glasses' own
|
||||
* pitch reading (degrees). "glasses" keeps the glasses' own setting (from the
|
||||
* Even app), writing back their original values if Faceclaw changed them. A
|
||||
* number is written to the glasses' head-up setting on every connection while
|
||||
* the Glanceboard is enabled (head-angle.ts). Android only.
|
||||
*/
|
||||
export const glanceAngleSetting = new ConfigSettingEnum<GlanceAngle>({
|
||||
id: "glanceboard-head-tilt-angle",
|
||||
label: "Head tilt angle",
|
||||
storageKey: "glanceboard.headTiltAngle",
|
||||
defaultValue: "glasses",
|
||||
values: GLANCE_ANGLE_VALUES,
|
||||
formatValue: (value) => (value === "glasses" ? "Glasses default" : `${value}°`),
|
||||
description:
|
||||
"How far you tilt your head up to wake the display; the list shows your head's angle now. Glasses default uses the glasses' own setting from the Even app. " +
|
||||
"The board hides when your head drops 20° below this angle; under 25°, looking straight ahead may not count, so it hides after the tap duration instead.",
|
||||
});
|
||||
|
||||
/**
|
||||
* The trigger Faceclaw should maintain, in degrees: the Head tilt angle while
|
||||
* the Glanceboard is enabled, else 0 (the glasses' own setting).
|
||||
*/
|
||||
export function glanceAngleTriggerDegrees(): number {
|
||||
const value = glanceAngleSetting.get();
|
||||
return value === "glasses" || !glanceboardEnabledSetting.get() ? 0 : Number(value);
|
||||
}
|
||||
|
||||
const DEPTH_VALUES = ["-64", "-48", "-32", "-16", "0", "16", "32", "48", "64"] as const;
|
||||
export type GlanceDepth = (typeof DEPTH_VALUES)[number];
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
glanceShowOnLongPressSetting,
|
||||
glanceShowOnTap,
|
||||
glanceTapTimeoutMs,
|
||||
glanceTiltTimeoutMs,
|
||||
} from "./glanceboard-settings";
|
||||
|
||||
const glanceboardApp: AppDefinition = {
|
||||
@@ -24,6 +25,7 @@ const glanceboardApp: AppDefinition = {
|
||||
isEnabled: () => glanceboardEnabledSetting.get(),
|
||||
showOnTap: glanceShowOnTap,
|
||||
tapTimeoutMs: glanceTapTimeoutMs,
|
||||
tiltTimeoutMs: glanceTiltTimeoutMs,
|
||||
showOnLongPress: () => glanceShowOnLongPressSetting.get(),
|
||||
showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(),
|
||||
depth: glanceDepth,
|
||||
|
||||
@@ -50,16 +50,26 @@ export const musicShowAlbumArtSetting = new ConfigSettingBoolean({
|
||||
"Show the album art beside the track details. When off, the title, artist, album and progress bar use the full width of the card.",
|
||||
});
|
||||
|
||||
export const musicShowIdleNoteSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-music-show-idle-note",
|
||||
label: "Show note when idle",
|
||||
storageKey: "glanceboard.music.showIdleNote",
|
||||
defaultValue: true,
|
||||
description: "Show a dim music note on the card while nothing is playing. When off, the card is left blank.",
|
||||
});
|
||||
|
||||
/** The music card's submenu in Glanceboard > Settings. */
|
||||
export function musicSettingsMenuItems(): MenuItem[] {
|
||||
return [toggleSettingMenuItem(musicShowAlbumArtSetting)];
|
||||
return [toggleSettingMenuItem(musicShowAlbumArtSetting), toggleSettingMenuItem(musicShowIdleNoteSetting)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Now playing: album art (unless turned off), title/artist/album and a
|
||||
* progress bar for the active media session (Apple Music on iOS), the Music
|
||||
* app's header at card size. The media bridge runs for the whole session; the
|
||||
* widget subscribes and ticks once a second only while something is playing.
|
||||
* app's header at card size. With nothing playing the card holds a dim note,
|
||||
* or nothing with that turned off. The media bridge runs for the whole
|
||||
* session; the widget subscribes and ticks once a second only while
|
||||
* something is playing.
|
||||
*/
|
||||
export class MusicWidget implements GlanceWidget {
|
||||
private media: MediaControllerState = mediaControllerBridge.snapshot();
|
||||
@@ -108,8 +118,8 @@ export class MusicWidget implements GlanceWidget {
|
||||
image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140);
|
||||
return;
|
||||
}
|
||||
// Nothing playing: a dim note, centred, and nothing else.
|
||||
const note = idleNoteImage();
|
||||
// Nothing playing: a dim note, centred, and nothing else; or a blank card.
|
||||
const note = musicShowIdleNoteSetting.get() ? idleNoteImage() : null;
|
||||
if (note) image.drawImage(note, ((image.width - note.width) / 2) | 0, ((image.height - note.height) / 2) | 0);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
|
||||
}
|
||||
|
||||
/**
|
||||
* The configuration a capture of the temples' own microphones arms them
|
||||
* with: `mics` of 2 is one microphone per temple (the front one), 4 is both
|
||||
* of each. The front end and direction detection follow Microphone setup
|
||||
* (the Microphones app), except that both of a temple's microphones only
|
||||
* come through the codec front end. Always raw PCM16 at 16 kHz, which is
|
||||
* what the firmware streams and transcription takes.
|
||||
*/
|
||||
export function micArrayCaptureConfig(mics: 2 | 4): MicConfig {
|
||||
const base = loadMicConfig();
|
||||
return {
|
||||
...base,
|
||||
source: mics === 4 ? "codec" : base.source,
|
||||
channelMask: mics === 4 ? 0b11 : 0b01,
|
||||
codec: "raw",
|
||||
format: "pcm16",
|
||||
sampleRateHz: 16_000,
|
||||
armHardware: true,
|
||||
};
|
||||
}
|
||||
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
|
||||
@@ -272,6 +272,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -370,6 +370,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
|
||||
handleInput(window, message.event as InputEvent, message.frameId);
|
||||
break;
|
||||
case "key":
|
||||
// Only windows opened with acceptsKeys are sent keys.
|
||||
break;
|
||||
case "text-input":
|
||||
if (editing) {
|
||||
// Voice input as an alternative to the phone keyboard.
|
||||
|
||||
@@ -433,6 +433,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "text-input":
|
||||
case "key":
|
||||
case "tool-call":
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -227,6 +227,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
|
||||
handleInput(window, message.event as InputEvent, message.frameId);
|
||||
break;
|
||||
case "key":
|
||||
// Only windows opened with acceptsKeys are sent keys.
|
||||
break;
|
||||
case "text-input":
|
||||
if (editingSetting) {
|
||||
// Voice input as an alternative to the phone keyboard.
|
||||
|
||||
@@ -424,6 +424,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
|
||||
handleInput(window, message.event as InputEvent, message.frameId);
|
||||
break;
|
||||
case "key":
|
||||
// Only windows opened with acceptsKeys are sent keys.
|
||||
break;
|
||||
case "text-input":
|
||||
if (window && window.windowId === message.windowId) handleTextInput(message.text.trim());
|
||||
break;
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
|
||||
/**
|
||||
* Terminal-app connection registry: the g2mirror hosts the app knows about,
|
||||
* stored as JSON under one settings key so every isolate sees the same list
|
||||
* and change notifications ride the ordinary settings-store channel.
|
||||
* Terminal-app connection registry: the hosts the app knows about, stored as
|
||||
* JSON under one settings key so every isolate sees the same list and change
|
||||
* notifications ride the ordinary settings-store channel.
|
||||
*
|
||||
* A connection is entered as a g2mirror:// connection string (the same form
|
||||
* g2mirror-view accepts): g2mirror://<token>@<host>[:port] for plain
|
||||
* websockets (default port 8737), g2mirrors://... for TLS (default 443).
|
||||
* A connection is entered as a connection string:
|
||||
* - g2mirror://<token>@<host>[:port]: a g2mirror server reached directly
|
||||
* (the same form g2mirror-view accepts) over a plain websocket (default
|
||||
* port 8737); g2mirrors://... for TLS (default 443).
|
||||
* - ssh://<user>@<host>[:port]: tmux on an SSH host (default port 22).
|
||||
* - ssh://<user>@<host>[:port]?g2mirror=<token>[@<host>[:port]]: a g2mirror
|
||||
* server tunnelled over SSH; the address after the token is where the
|
||||
* server listens as seen from the SSH host (default 127.0.0.1:8737).
|
||||
* SSH connections pin the host's key (hostKeys, trusted on first connection
|
||||
* after the user checks its fingerprint); their passwords live in the
|
||||
* platform keystore (app/native/ssh), not here.
|
||||
*/
|
||||
|
||||
export const TERMINAL_CONNECTIONS_KEY = "terminal.connections";
|
||||
@@ -25,6 +33,8 @@ export type TerminalConnection = {
|
||||
serverName: string | null;
|
||||
/** Auto-connect while the Terminal app is open. Toggled by Connect/Disconnect. */
|
||||
enabled: boolean;
|
||||
/** SSH connections: trusted host keys (SSH wire format, base64). */
|
||||
hostKeys: string[];
|
||||
};
|
||||
|
||||
export type ParsedConnectionString = {
|
||||
@@ -54,8 +64,79 @@ export function parseConnectionString(url: string): ParsedConnectionString | nul
|
||||
return { secure, host, port, authToken };
|
||||
}
|
||||
|
||||
/** The host[:port] part of a connection string; "" if it doesn't parse. */
|
||||
export type ParsedSshConnection = {
|
||||
user: string;
|
||||
host: string;
|
||||
port: number;
|
||||
} & (
|
||||
| { mode: "tmux" }
|
||||
| {
|
||||
mode: "g2mirror";
|
||||
authToken: string;
|
||||
/** Where g2mirror-server listens, from the SSH host's point of view. */
|
||||
targetHost: string;
|
||||
targetPort: number;
|
||||
}
|
||||
);
|
||||
|
||||
function decodeComponent(text: string): string {
|
||||
try {
|
||||
return decodeURIComponent(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
/** Parse an ssh://user@host[:port][?g2mirror=token[@host[:port]]] string; null if malformed. */
|
||||
export function parseSshConnectionString(url: string): ParsedSshConnection | null {
|
||||
const match = /^ssh:\/\/([^@\s/?]+)@(\[[^\]\s]+\]|[^:/@\s?]+)(?::(\d+))?\/?(?:\?(.*))?$/.exec(url.trim());
|
||||
if (!match) return null;
|
||||
const user = decodeComponent(match[1]!);
|
||||
const host = match[2]!.replace(/^\[(.*)\]$/, "$1");
|
||||
const port = match[3] ? parseInt(match[3], 10) : 22;
|
||||
if (!user || !(port >= 1 && port <= 65535)) return null;
|
||||
const params = new Map<string, string>();
|
||||
for (const part of (match[4] ?? "").split("&").filter(Boolean)) {
|
||||
const equals = part.indexOf("=");
|
||||
if (equals < 0) return null;
|
||||
params.set(part.slice(0, equals), decodeComponent(part.slice(equals + 1)));
|
||||
}
|
||||
for (const key of params.keys()) {
|
||||
if (key !== "g2mirror") return null;
|
||||
}
|
||||
const g2mirror = params.get("g2mirror");
|
||||
if (g2mirror === undefined) return { user, host, port, mode: "tmux" };
|
||||
// token[@host[:port]], split at the last "@" so the token may contain one.
|
||||
const at = g2mirror.lastIndexOf("@");
|
||||
const authToken = at < 0 ? g2mirror : g2mirror.slice(0, at);
|
||||
const target = at < 0 ? "" : g2mirror.slice(at + 1);
|
||||
const targetMatch = /^(\[[^\]\s]+\]|[^:\s]*)(?::(\d+))?$/.exec(target);
|
||||
if (!authToken || !targetMatch) return null;
|
||||
const targetHost = targetMatch[1] || "127.0.0.1";
|
||||
const targetPort = targetMatch[2] ? parseInt(targetMatch[2], 10) : 8737;
|
||||
if (!(targetPort >= 1 && targetPort <= 65535)) return null;
|
||||
return { user, host, port, mode: "g2mirror", authToken, targetHost, targetPort };
|
||||
}
|
||||
|
||||
export type ParsedAnyConnection =
|
||||
| ({ kind: "g2mirror" } & ParsedConnectionString)
|
||||
| ({ kind: "ssh" } & ParsedSshConnection);
|
||||
|
||||
/** Parse any supported connection string; null if it's none of them. */
|
||||
export function parseAnyConnectionString(url: string): ParsedAnyConnection | null {
|
||||
const direct = parseConnectionString(url);
|
||||
if (direct) return { kind: "g2mirror", ...direct };
|
||||
const ssh = parseSshConnectionString(url);
|
||||
return ssh ? { kind: "ssh", ...ssh } : null;
|
||||
}
|
||||
|
||||
/** The [user@]host[:port] part of a connection string (never a token); "" if it doesn't parse. */
|
||||
export function connectionStringHostLabel(url: string): string {
|
||||
const ssh = parseSshConnectionString(url);
|
||||
if (ssh) {
|
||||
const host = ssh.host.includes(":") ? `[${ssh.host}]` : ssh.host;
|
||||
return `${ssh.user}@${host}${ssh.port === 22 ? "" : `:${ssh.port}`}`;
|
||||
}
|
||||
const parsed = parseConnectionString(url);
|
||||
if (!parsed) return "";
|
||||
const defaultPort = parsed.secure ? 443 : 8737;
|
||||
@@ -79,6 +160,7 @@ export function loadConnections(): TerminalConnection[] {
|
||||
url: String(item?.url ?? ""),
|
||||
serverName: typeof item?.serverName === "string" && item.serverName ? item.serverName : null,
|
||||
enabled: item?.enabled !== false,
|
||||
hostKeys: Array.isArray(item?.hostKeys) ? item.hostKeys.filter((key: unknown) => typeof key === "string") : [],
|
||||
}))
|
||||
.filter((connection) => connection.id.length > 0 && connection.url.length > 0);
|
||||
} catch {
|
||||
|
||||
@@ -14,6 +14,8 @@ const terminalApp: AppDefinition = {
|
||||
// Session windows stay independent: launching reopens the hub even
|
||||
// while terminal sessions are open.
|
||||
matchExistingBy: "windowId",
|
||||
// The hub window can show a session too, so it takes keys like one.
|
||||
acceptsKeys: true,
|
||||
}),
|
||||
};
|
||||
|
||||
|
||||
@@ -16,11 +16,17 @@
|
||||
* Each view has its own websocket connection (the protocol allows one attached
|
||||
* session per connection) and its own xterm emulator.
|
||||
*
|
||||
* Connections are g2mirror://<token>@<host>[:port] strings (g2mirrors:// for
|
||||
* TLS), stored as JSON in the settings store (see connections.ts). Each
|
||||
* enabled connection gets a control websocket that backs the session list and
|
||||
* carries unsolicited bell/title notifications; control connections live as
|
||||
* long as the worker so bells keep flowing even if the hub window is closed.
|
||||
* Connections are connection strings stored as JSON in the settings store
|
||||
* (see connections.ts): g2mirror://<token>@<host>[:port] for a g2mirror
|
||||
* server (g2mirrors:// for TLS), ssh://<user>@<host> for tmux over SSH
|
||||
* (TmuxClient), or ssh://<user>@<host>?g2mirror=<token> for a g2mirror server
|
||||
* tunnelled over SSH. Each enabled connection gets a control client that
|
||||
* backs the session list and carries bell/title/activity notifications;
|
||||
* control connections live as long as the worker so bells keep flowing even
|
||||
* if the hub window is closed. SSH connections that need the user (an
|
||||
* unverified or changed host key, a password, an unauthorized key) record
|
||||
* the failure on the control (sshIssue) and offer a page that resolves it
|
||||
* instead of retrying.
|
||||
* The host's server_name (from the init reply) is cached per connection and
|
||||
* used as its display name, falling back to the connection string's
|
||||
* host[:port] before the first successful handshake.
|
||||
@@ -55,7 +61,24 @@ import {
|
||||
import * as frameTimings from "../../native/frame-timings";
|
||||
import { getActiveDisplay } from "../../native/active-display";
|
||||
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { G2MirrorClient, type G2MirrorClientOptions, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
|
||||
import type { KeyEvent } from "../../g2/keyboard/keys";
|
||||
import { encodeTerminalKey, terminalScrollbackKey } from "./terminal-keys";
|
||||
import { G2MirrorClient, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
|
||||
import type { TerminalClient } from "./terminal-client";
|
||||
import { TmuxClient, type OpenLineChannel } from "./tmux-client";
|
||||
import {
|
||||
copyToPhoneClipboard,
|
||||
describeSshFailure,
|
||||
getSshSecret,
|
||||
openSshChannel,
|
||||
removeSshSecret,
|
||||
setSshSecret,
|
||||
sshClientKey,
|
||||
type SshEndpoint,
|
||||
type SshFailure,
|
||||
type SshHostKey,
|
||||
} from "../../native/ssh";
|
||||
import { wrapText } from "../../graphics/textwrap";
|
||||
import { onSettingsStoreChanged } from "../../native/settings-store";
|
||||
import { clamp } from "../../util/numeric-util";
|
||||
import {
|
||||
@@ -64,12 +87,22 @@ import {
|
||||
terminalDisplayLayoutSetting,
|
||||
terminalLaunchPresetsSetting,
|
||||
terminalNewConnectionSetting,
|
||||
terminalSshPasswordSetting,
|
||||
terminalWakeOnBellSetting,
|
||||
textSettingMenuItem,
|
||||
toggleSettingMenuItem,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { terminalFontPickerMenuItem } from "../../ui/font-picker";
|
||||
import { connectionDisplayName, loadConnections, parseConnectionString, saveConnections, TERMINAL_CONNECTIONS_KEY, updateConnection, type TerminalConnection } from "./connections";
|
||||
import {
|
||||
connectionDisplayName,
|
||||
connectionStringHostLabel,
|
||||
loadConnections,
|
||||
parseAnyConnectionString,
|
||||
saveConnections,
|
||||
TERMINAL_CONNECTIONS_KEY,
|
||||
updateConnection,
|
||||
type TerminalConnection,
|
||||
} from "./connections";
|
||||
import { TerminalEmulator } from "./terminal-emulator";
|
||||
import { drawSubmenuIndicator, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
@@ -112,6 +145,10 @@ const RECONNECT_MAX_DELAY_MS = 60_000;
|
||||
const NEW_CONNECTION_DRAFT_KEY = "terminal.newConnectionDraft";
|
||||
// Storage key of terminalLaunchPresetsSetting, edited from the window menu.
|
||||
const LAUNCH_PRESETS_KEY = "terminal.launchPresets";
|
||||
// Storage key of terminalSshPasswordSetting (the password page's draft).
|
||||
const SSH_PASSWORD_DRAFT_KEY = "terminal.sshPasswordDraft";
|
||||
// Comment on this device's SSH public key in authorized_keys.
|
||||
const SSH_KEY_COMMENT = "faceclaw";
|
||||
|
||||
type BaseWindow = {
|
||||
windowId: string;
|
||||
@@ -148,10 +185,24 @@ type BaseWindow = {
|
||||
|
||||
/**
|
||||
* What the hub window is showing: the session list, the Manage Connections
|
||||
* section, or the Add-connection screen (type the g2mirror:// string on the
|
||||
* phone, click to save).
|
||||
* section, the Add-connection screen (type the connection string on the
|
||||
* phone, click to save), or a page of text with a click action (verify a
|
||||
* host key, enter a password, show this device's SSH key).
|
||||
*/
|
||||
type HubMode = "sessions" | "connections" | "add";
|
||||
type HubMode = "sessions" | "connections" | "add" | "page";
|
||||
|
||||
type HubPage = {
|
||||
title: string;
|
||||
/** Body text, wrapped to the window; re-read on every paint. */
|
||||
body: () => string;
|
||||
/** Bottom hint line. */
|
||||
hint: string;
|
||||
onClick?: () => void;
|
||||
/** Double-click. */
|
||||
onBack: () => void;
|
||||
/** Runs whenever the page goes away (back, click, window closed). */
|
||||
onLeave?: () => void;
|
||||
};
|
||||
|
||||
type HubWindow = BaseWindow & {
|
||||
kind: "hub";
|
||||
@@ -168,6 +219,8 @@ type HubWindow = BaseWindow & {
|
||||
sessionOrder: string[];
|
||||
/** Parse-failure message shown on the Add-connection screen. */
|
||||
addError: string;
|
||||
/** The page shown in mode "page". */
|
||||
page: HubPage | null;
|
||||
};
|
||||
|
||||
type ViewWindow = BaseWindow & {
|
||||
@@ -186,7 +239,7 @@ type ViewWindow = BaseWindow & {
|
||||
label: string;
|
||||
/** Sidebar-icon character (">3"); "" if every glyph was taken at open time. */
|
||||
glyph: string;
|
||||
client: G2MirrorClient;
|
||||
client: TerminalClient;
|
||||
emulator: TerminalEmulator;
|
||||
receivedData: boolean;
|
||||
attachRequested: boolean;
|
||||
@@ -220,8 +273,8 @@ type PendingView = {
|
||||
socket: string;
|
||||
label: string;
|
||||
glyph: string;
|
||||
/** Client options captured when the view was requested from a live control. */
|
||||
options: G2MirrorClientOptions;
|
||||
/** How to make the view's client, captured when it was requested from a live control. */
|
||||
options: ClientRecipe;
|
||||
};
|
||||
|
||||
const windows = new Map<string, TerminalWindow>();
|
||||
@@ -302,7 +355,13 @@ let screenOn = true;
|
||||
*/
|
||||
type ControlConnection = {
|
||||
config: TerminalConnection;
|
||||
client: G2MirrorClient | null;
|
||||
client: TerminalClient | null;
|
||||
/**
|
||||
* An SSH failure only the user can fix (host key to verify, password or
|
||||
* key needed). Shown on the host's heading with a row that opens the fix;
|
||||
* reconnects wait for it. Cleared on a successful connection.
|
||||
*/
|
||||
sshIssue: SshFailure | null;
|
||||
state: G2MirrorState | null;
|
||||
unsubscribers: Array<() => void>;
|
||||
reconnectTimer: ReturnType<typeof setTimeout> | null;
|
||||
@@ -435,6 +494,13 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "key": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
window.focused = message.focused;
|
||||
handleKey(window, message.event as KeyEvent);
|
||||
break;
|
||||
}
|
||||
case "render": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
@@ -504,7 +570,8 @@ onSettingsStoreChanged((key) => {
|
||||
reportIdle();
|
||||
return;
|
||||
case NEW_CONNECTION_DRAFT_KEY:
|
||||
// Live keystrokes from the phone editor; repaint the add screen.
|
||||
case SSH_PASSWORD_DRAFT_KEY:
|
||||
// Live keystrokes from the phone editor; repaint the add/password screen.
|
||||
renderHubWindows();
|
||||
return;
|
||||
case "terminal.autoReconnect":
|
||||
@@ -578,6 +645,7 @@ function openWindow(windowId: string, surfaceId: string, title: string, viewport
|
||||
list: null,
|
||||
sessionOrder: [],
|
||||
addError: "",
|
||||
page: null,
|
||||
});
|
||||
// The hub carries the app's assistant tools; declare them once it exists.
|
||||
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
|
||||
@@ -610,6 +678,10 @@ function closeWindow(windowId: string): void {
|
||||
// Window closed mid-add: shut the phone editor down.
|
||||
endAddConnection(window);
|
||||
}
|
||||
if (window.kind === "hub" && window.mode === "page") {
|
||||
// Likewise for a page's phone editor (the password draft).
|
||||
closeHubPage(window, "sessions");
|
||||
}
|
||||
// Likewise for a setting being edited from the window menu.
|
||||
window.menu?.close();
|
||||
windows.delete(windowId);
|
||||
@@ -627,17 +699,120 @@ function closeWindow(windowId: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/** Client options for a connection string; null if the string doesn't parse. */
|
||||
function clientOptionsFor(connection: TerminalConnection): G2MirrorClientOptions | null {
|
||||
const parsed = parseConnectionString(connection.url);
|
||||
/**
|
||||
* How to make clients for a connection: "g2mirror" dials a g2mirror server
|
||||
* directly, "ssh-g2mirror" reaches one through an SSH tunnel, "tmux" runs
|
||||
* tmux control mode over SSH.
|
||||
*/
|
||||
type ClientRecipe = {
|
||||
kind: "g2mirror" | "ssh-g2mirror" | "tmux";
|
||||
/**
|
||||
* `role` "control" watches bells (tmux keeps the bell bookkeeping for that
|
||||
* client only). `onSshFailure` hears why an SSH channel failed.
|
||||
*/
|
||||
create(cols: number, rows: number, role: "control" | "view", onSshFailure?: (failure: SshFailure) => void): TerminalClient;
|
||||
};
|
||||
|
||||
/** Client recipe for a connection string; null if the string doesn't parse. */
|
||||
function clientOptionsFor(connection: TerminalConnection): ClientRecipe | null {
|
||||
const parsed = parseAnyConnectionString(connection.url);
|
||||
if (!parsed) return null;
|
||||
if (parsed.kind === "g2mirror") {
|
||||
const { secure, host, port, authToken } = parsed;
|
||||
return {
|
||||
kind: "g2mirror",
|
||||
create: (cols, rows) => new G2MirrorClient({ secure, host, port, authToken, deviceName: DEVICE_NAME, cols, rows }),
|
||||
};
|
||||
}
|
||||
const connectionId = connection.id;
|
||||
const label = connectionStringHostLabel(connection.url);
|
||||
// Read at every channel open, so a newly trusted key or saved password
|
||||
// applies to the next (re)connect, views included.
|
||||
const endpoint = (): SshEndpoint => ({
|
||||
host: parsed.host,
|
||||
port: parsed.port,
|
||||
user: parsed.user,
|
||||
hostKeys: loadConnections().find((candidate) => candidate.id === connectionId)?.hostKeys ?? [],
|
||||
password: getSshSecret(sshPasswordSecretId(connectionId)),
|
||||
useKey: true,
|
||||
});
|
||||
if (parsed.mode === "g2mirror") {
|
||||
const { authToken, targetHost, targetPort } = parsed;
|
||||
const urlHost = targetHost.includes(":") && !targetHost.startsWith("[") ? `[${targetHost}]` : targetHost;
|
||||
const url = `ws://${urlHost}:${targetPort}/`;
|
||||
return {
|
||||
kind: "ssh-g2mirror",
|
||||
create: (cols, rows, _role, onSshFailure) =>
|
||||
new G2MirrorClient({
|
||||
secure: false,
|
||||
host: targetHost,
|
||||
port: targetPort,
|
||||
authToken,
|
||||
deviceName: DEVICE_NAME,
|
||||
cols,
|
||||
rows,
|
||||
transport: {
|
||||
label: `${label} over SSH`,
|
||||
open: (listener) => {
|
||||
const channel = openSshChannel(endpoint(), { type: "websocket", url }, {
|
||||
onOpen: () => listener.onOpen(),
|
||||
onText: (text) => listener.onTextMessage(text),
|
||||
onClosed: (info) => listener.onClosed(info.code ?? 1000, info.reason ?? ""),
|
||||
onFailure: (failure) => {
|
||||
onSshFailure?.(failure);
|
||||
listener.onFailure(describeSshFailure(failure));
|
||||
},
|
||||
});
|
||||
return { sendText: (text) => channel.send(text), sendBinary: () => false, close: () => channel.close() };
|
||||
},
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
...parsed,
|
||||
deviceName: DEVICE_NAME,
|
||||
...viewGrid(),
|
||||
kind: "tmux",
|
||||
create: (cols, rows, role, onSshFailure) => {
|
||||
const openChannel: OpenLineChannel = (command, listener) =>
|
||||
openSshChannel(endpoint(), { type: "exec", command }, {
|
||||
onOpen: () => listener.onOpen(),
|
||||
onText: (line) => listener.onLine(line),
|
||||
onStderr: (line) => listener.onStderr(line),
|
||||
onClosed: (info) => listener.onClosed(info.exitStatus ?? null),
|
||||
onFailure: (failure) => {
|
||||
onSshFailure?.(failure);
|
||||
listener.onFailure(describeSshFailure(failure));
|
||||
},
|
||||
});
|
||||
return new TmuxClient({ openChannel, label, cols, rows, stateKey: role === "control" ? connectionId : "" });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sshPasswordSecretId(connectionId: string): string {
|
||||
return `terminal.password.${connectionId}`;
|
||||
}
|
||||
|
||||
/** Failures that retrying can't fix: the user has to verify a key or supply credentials. */
|
||||
function sshFailureNeedsUser(failure: SshFailure): boolean {
|
||||
return failure.kind === "hostkey-unknown" || failure.kind === "hostkey-mismatch" || failure.kind === "auth";
|
||||
}
|
||||
|
||||
/**
|
||||
* A control's SSH channel failed. If the user has to act, remember why, and
|
||||
* if they're looking at the hub's lists, open the page that fixes it.
|
||||
*/
|
||||
function noteSshFailure(control: ControlConnection, failure: SshFailure): void {
|
||||
if (!sshFailureNeedsUser(failure) || controls.get(control.config.id) !== control || !control.client) return;
|
||||
control.sshIssue = failure;
|
||||
renderHubWindows();
|
||||
for (const window of windows.values()) {
|
||||
if (window.kind === "hub" && window.foreground && (window.mode === "sessions" || window.mode === "connections")) {
|
||||
openSshIssuePage(window, control);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile the live control clients with the stored connection list: start
|
||||
* enabled connections, stop disabled or removed ones, restart ones whose
|
||||
@@ -663,6 +838,7 @@ function syncControlsFromSettings(): void {
|
||||
control = {
|
||||
config: connection,
|
||||
client: null,
|
||||
sshIssue: null,
|
||||
state: null,
|
||||
unsubscribers: [],
|
||||
reconnectTimer: null,
|
||||
@@ -684,12 +860,13 @@ function syncControlsFromSettings(): void {
|
||||
|
||||
function startControl(control: ControlConnection): void {
|
||||
stopControl(control);
|
||||
const options = clientOptionsFor(control.config);
|
||||
if (!options) {
|
||||
const recipe = clientOptionsFor(control.config);
|
||||
if (!recipe) {
|
||||
control.state = null;
|
||||
return;
|
||||
}
|
||||
const client = new G2MirrorClient(options);
|
||||
const grid = viewGrid();
|
||||
const client = recipe.create(grid.cols, grid.rows, "control", (failure) => noteSshFailure(control, failure));
|
||||
const connectionId = control.config.id;
|
||||
control.client = client;
|
||||
control.state = client.state();
|
||||
@@ -703,6 +880,7 @@ function startControl(control: ControlConnection): void {
|
||||
control.state = state;
|
||||
if (state.phase === "connected") {
|
||||
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
|
||||
control.sshIssue = null;
|
||||
maybeCacheServerName(control, state.serverName);
|
||||
}
|
||||
if (state.phase === "failed") scheduleControlReconnect(control);
|
||||
@@ -777,6 +955,8 @@ function scheduleControlReconnect(control: ControlConnection): void {
|
||||
if (!terminalAutoReconnectSetting.get()) return;
|
||||
if (windows.size === 0 && !hasTerminalBackgroundWork()) return;
|
||||
if (!control.config.enabled) return;
|
||||
// Waiting on the user (host key, password) or on the host (no usable tmux).
|
||||
if (control.sshIssue || control.state?.retryable === false) return;
|
||||
const delayMs = control.reconnectDelayMs;
|
||||
control.reconnectDelayMs = Math.min(control.reconnectDelayMs * 2, RECONNECT_MAX_DELAY_MS);
|
||||
control.reconnectTimer = setTimeout(() => {
|
||||
@@ -943,7 +1123,7 @@ function createViewWindow(
|
||||
const cell = getTerminalFontConfig();
|
||||
const gridCols = Math.floor(viewport.width / cell.cellWidth);
|
||||
const gridRows = Math.floor(viewport.height / cell.cellHeight);
|
||||
const client = new G2MirrorClient({ ...view.options, cols: gridCols, rows: gridRows });
|
||||
const client = view.options.create(gridCols, gridRows, "view");
|
||||
const window: ViewWindow = {
|
||||
kind: "view",
|
||||
hub: null,
|
||||
@@ -1238,6 +1418,11 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
|
||||
renderAndSubmit(window, frameId);
|
||||
return;
|
||||
}
|
||||
if (window.kind === "hub" && window.mode === "page") {
|
||||
window.page?.onBack();
|
||||
renderAndSubmit(window, frameId);
|
||||
return;
|
||||
}
|
||||
if (window.kind === "hub" && window.mode === "connections") {
|
||||
setHubMode(window, "sessions");
|
||||
renderAndSubmit(window, frameId);
|
||||
@@ -1260,6 +1445,35 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
|
||||
frameTimings.finishFrame(frameId, "discarded: terminal view ignored input");
|
||||
}
|
||||
|
||||
/**
|
||||
* A key from a hardware keyboard (the glasses' keyboard bridge) that no
|
||||
* keybinding claimed. A session with nothing over it gets the key as its
|
||||
* terminal would: Escape and the arrows belong to the program, and the way
|
||||
* out is Escape with Shift, Ctrl or Alt, which never arrives here (it is a
|
||||
* built-in keybinding for "back"). Anywhere else (the hub's lists, an open
|
||||
* menu) the key is handed back, to act as navigation.
|
||||
*/
|
||||
function handleKey(window: TerminalWindow, event: KeyEvent): void {
|
||||
if (window.kind === "view" && !window.menu?.isOpen()) {
|
||||
activeViewId = window.windowId;
|
||||
const page = terminalScrollbackKey(event);
|
||||
if (page !== null) {
|
||||
handleViewScroll(window, page, 0);
|
||||
return;
|
||||
}
|
||||
const bytes = encodeTerminalKey(event, window.emulator.applicationCursorKeys());
|
||||
if (bytes === null) return;
|
||||
// Typing follows the live screen, as in any terminal emulator.
|
||||
if (window.scrollTop !== null) {
|
||||
window.scrollTop = null;
|
||||
scheduleRender(window);
|
||||
}
|
||||
window.client.sendInput(bytes);
|
||||
return;
|
||||
}
|
||||
post({ type: "key-unhandled", windowId: window.windowId, event });
|
||||
}
|
||||
|
||||
/** Top visible absolute line index when following the live bottom. */
|
||||
function followTop(window: ViewWindow): number {
|
||||
return window.historyNext + window.emulator.bufferLength() - window.gridRows;
|
||||
@@ -1386,6 +1600,154 @@ function setHubMode(window: HubWindow, mode: HubMode): void {
|
||||
window.addError = "";
|
||||
}
|
||||
|
||||
function openHubPage(window: HubWindow, page: HubPage): void {
|
||||
if (window.mode === "add") endAddConnection(window);
|
||||
if (window.mode === "page") window.page?.onLeave?.();
|
||||
window.page = page;
|
||||
window.mode = "page";
|
||||
scheduleRender(window);
|
||||
}
|
||||
|
||||
/** Leave the current page (if any) for `mode`. */
|
||||
function closeHubPage(window: HubWindow, mode: HubMode): void {
|
||||
const page = window.page;
|
||||
window.page = null;
|
||||
page?.onLeave?.();
|
||||
setHubMode(window, mode);
|
||||
scheduleRender(window);
|
||||
}
|
||||
|
||||
/** The list a page returns to: the one it was opened from. */
|
||||
function hubListMode(window: HubWindow): HubMode {
|
||||
return window.mode === "connections" ? "connections" : "sessions";
|
||||
}
|
||||
|
||||
function openSshIssuePage(window: HubWindow, control: ControlConnection): void {
|
||||
const issue = control.sshIssue;
|
||||
if (!issue) return;
|
||||
if (issue.kind === "hostkey-unknown" && issue.hostKey) {
|
||||
openHostKeyPage(window, control, issue.hostKey);
|
||||
} else if (issue.kind === "hostkey-mismatch" && issue.hostKey) {
|
||||
openHostKeyChangedPage(window, control, issue.hostKey);
|
||||
} else if (issue.kind === "auth" && issue.passwordAllowed) {
|
||||
openPasswordPage(window, control, Boolean(issue.passwordTried));
|
||||
} else if (issue.kind === "auth") {
|
||||
openClientKeyPage(window, `${connectionDisplayName(control.config)} didn't accept this device's key (and doesn't take passwords).\n`);
|
||||
}
|
||||
}
|
||||
|
||||
function hostKeyTypeName(type: string): string {
|
||||
if (type === "ssh-ed25519") return "ED25519";
|
||||
if (type.startsWith("ecdsa-")) return "ECDSA";
|
||||
if (type === "ssh-rsa") return "RSA";
|
||||
return type;
|
||||
}
|
||||
|
||||
function hostKeyFile(type: string): string {
|
||||
if (type === "ssh-ed25519") return "/etc/ssh/ssh_host_ed25519_key.pub";
|
||||
if (type.startsWith("ecdsa-")) return "/etc/ssh/ssh_host_ecdsa_key.pub";
|
||||
return "/etc/ssh/ssh_host_rsa_key.pub";
|
||||
}
|
||||
|
||||
/** First connection to an SSH host: show the key's fingerprint; click trusts it. */
|
||||
function openHostKeyPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
|
||||
const returnMode = hubListMode(window);
|
||||
openHubPage(window, {
|
||||
title: "Verify host key",
|
||||
body: () =>
|
||||
`First connection to ${connectionStringHostLabel(control.config.url)}. Its ${hostKeyTypeName(hostKey.type)} host key fingerprint is:\n` +
|
||||
`${hostKey.fingerprint}\n\n` +
|
||||
`To check it, run this on the host and compare:\nssh-keygen -lf ${hostKeyFile(hostKey.type)}`,
|
||||
hint: `click trust and connect ${GESTURE_DOUBLE_CLICK} cancel`,
|
||||
onClick: () => {
|
||||
const hostKeys = [...control.config.hostKeys.filter((key) => key !== hostKey.key), hostKey.key];
|
||||
control.config.hostKeys = hostKeys;
|
||||
updateConnection(control.config.id, { hostKeys });
|
||||
control.sshIssue = null;
|
||||
closeHubPage(window, returnMode);
|
||||
connectControl(control);
|
||||
},
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
});
|
||||
}
|
||||
|
||||
function openHostKeyChangedPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
|
||||
const returnMode = hubListMode(window);
|
||||
openHubPage(window, {
|
||||
title: "HOST KEY CHANGED",
|
||||
body: () =>
|
||||
`${connectionStringHostLabel(control.config.url)} presented a different host key than the one trusted before: ` +
|
||||
`${hostKeyTypeName(hostKey.type)} ${hostKey.fingerprint}\n\n` +
|
||||
"Someone may be intercepting the connection, so Faceclaw won't connect. If the host was reinstalled or its keys " +
|
||||
'were regenerated, use "Forget host key" on this connection in Manage Connections, then verify the new fingerprint.',
|
||||
hint: `${GESTURE_DOUBLE_CLICK} back`,
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
});
|
||||
}
|
||||
|
||||
/** Ask for the host's password on the phone (masked editor); click saves it and connects. */
|
||||
function openPasswordPage(window: HubWindow, control: ControlConnection, rejected: boolean): void {
|
||||
const returnMode = hubListMode(window);
|
||||
terminalSshPasswordSetting.set("");
|
||||
post({ type: "start-text-setting-edit", settingId: terminalSshPasswordSetting.id });
|
||||
openHubPage(window, {
|
||||
title: "SSH password",
|
||||
body: () => {
|
||||
const typed = terminalSshPasswordSetting.get();
|
||||
return (
|
||||
(rejected ? "The host refused the saved password.\n" : "") +
|
||||
`Type the password for ${connectionStringHostLabel(control.config.url)} in the phone app, then click to connect. ` +
|
||||
"It is saved encrypted on this phone.\n\n" +
|
||||
(typed ? "*".repeat(Math.min(typed.length, 32)) : "(nothing typed yet)")
|
||||
);
|
||||
},
|
||||
hint: `click connect ${GESTURE_DOUBLE_CLICK} cancel`,
|
||||
onClick: () => {
|
||||
const password = terminalSshPasswordSetting.get();
|
||||
if (!password) return;
|
||||
setSshSecret(sshPasswordSecretId(control.config.id), password);
|
||||
control.sshIssue = null;
|
||||
closeHubPage(window, returnMode);
|
||||
connectControl(control);
|
||||
},
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
onLeave: () => {
|
||||
terminalSshPasswordSetting.set("");
|
||||
post({ type: "end-text-setting-edit" });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** This device's SSH public key (the private half never leaves the keystore); click copies it. */
|
||||
function openClientKeyPage(window: HubWindow, intro = ""): void {
|
||||
const returnMode = hubListMode(window);
|
||||
let key: { authorizedKey: string; fingerprint: string } | null = null;
|
||||
let error = "";
|
||||
try {
|
||||
key = sshClientKey(SSH_KEY_COMMENT);
|
||||
} catch (caught) {
|
||||
error = String((caught as Error)?.message ?? caught);
|
||||
}
|
||||
let copied = false;
|
||||
openHubPage(window, {
|
||||
title: "This device's SSH key",
|
||||
body: () =>
|
||||
key
|
||||
? `${intro}Add this line to ~/.ssh/authorized_keys on each host to log in with this phone's key:\n` +
|
||||
`${key.authorizedKey}\n${key.fingerprint}` +
|
||||
(copied ? "\n\nCopied to the phone's clipboard." : "")
|
||||
: `Couldn't create this device's key: ${error}`,
|
||||
hint: key ? `click copy to phone clipboard ${GESTURE_DOUBLE_CLICK} back` : `${GESTURE_DOUBLE_CLICK} back`,
|
||||
onClick: key
|
||||
? () => {
|
||||
copyToPhoneClipboard("SSH public key", key!.authorizedKey);
|
||||
copied = true;
|
||||
}
|
||||
: undefined,
|
||||
onBack: () => closeHubPage(window, returnMode),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One control connection's sessions in this hub window's display order:
|
||||
* most-recently-updated first as of when the window became visible, with
|
||||
@@ -1441,6 +1803,7 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
];
|
||||
const multiHost = controls.size > 1;
|
||||
const canLaunch = launchPresetNames().length > 0;
|
||||
const isTmux = (control: ControlConnection) => clientOptionsFor(control.config)?.kind === "tmux";
|
||||
// Every configured host is listed under its own heading, in stored order,
|
||||
// whatever its connection state: a connected host lists its sessions, any
|
||||
// other shows its state on the heading plus a Connect row (unless it is
|
||||
@@ -1450,10 +1813,17 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
const name = connectionDisplayName(control.config);
|
||||
if (!isControlConnected(control)) {
|
||||
items.push({ label: `${name} (${controlStatusWord(control)})`, heading: true });
|
||||
if (control.sshIssue && control.config.enabled) {
|
||||
items.push({ label: sshIssueAction(control.sshIssue), onSelect: () => openSshIssuePage(window, control) });
|
||||
continue;
|
||||
}
|
||||
const connecting = control.config.enabled && control.state?.phase === "connecting";
|
||||
if (!connecting && clientOptionsFor(control.config)) {
|
||||
// A failure retrying can't fix (tmux missing or too old) says why;
|
||||
// selecting it retries once the host is fixed.
|
||||
const unfixable = control.config.enabled && control.state?.retryable === false;
|
||||
items.push({
|
||||
label: control.reconnectTimer ? "Connect now" : "Connect",
|
||||
label: unfixable ? control.state!.status : control.reconnectTimer ? "Connect now" : "Connect",
|
||||
onSelect: () => connectControl(control),
|
||||
});
|
||||
}
|
||||
@@ -1462,7 +1832,8 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
items.push({
|
||||
label: name,
|
||||
heading: true,
|
||||
onSelect: canLaunch ? () => launchOnHost(window, control) : undefined,
|
||||
// tmux hosts can always start a shell, presets or not.
|
||||
onSelect: canLaunch || isTmux(control) ? () => launchOnHost(window, control) : undefined,
|
||||
});
|
||||
const sessions = orderedSessions(window, control);
|
||||
for (const session of sessions) {
|
||||
@@ -1478,7 +1849,11 @@ function hubSessionItems(window: HubWindow): HubItem[] {
|
||||
}
|
||||
if (!sessions.length) {
|
||||
items.push({
|
||||
label: multiHost ? "(no live sessions)" : "(no live sessions; run g2mirror <command>)",
|
||||
label: isTmux(control)
|
||||
? "(no tmux sessions; select the host to start one)"
|
||||
: multiHost
|
||||
? "(no live sessions)"
|
||||
: "(no live sessions; run g2mirror <command>)",
|
||||
onSelect: () => control.client?.listSessions(),
|
||||
});
|
||||
}
|
||||
@@ -1499,6 +1874,10 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
|
||||
label: "Add connection",
|
||||
onSelect: () => beginAddConnection(window),
|
||||
},
|
||||
{
|
||||
label: "This device's SSH key",
|
||||
onSelect: () => openClientKeyPage(window),
|
||||
},
|
||||
{
|
||||
label: "Done",
|
||||
onSelect: () => setHubMode(window, "sessions"),
|
||||
@@ -1509,6 +1888,13 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
|
||||
|
||||
function controlStatusWord(control: ControlConnection): string {
|
||||
if (!control.config.enabled) return "off";
|
||||
const issue = control.sshIssue;
|
||||
if (issue) {
|
||||
if (issue.kind === "hostkey-unknown") return "verify host key";
|
||||
if (issue.kind === "hostkey-mismatch") return "HOST KEY CHANGED";
|
||||
if (issue.passwordTried) return "password rejected";
|
||||
return issue.passwordAllowed ? "password needed" : "key not authorized";
|
||||
}
|
||||
switch (control.state?.phase) {
|
||||
case "connected":
|
||||
case "attached":
|
||||
@@ -1541,6 +1927,32 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
|
||||
connectControl(control);
|
||||
},
|
||||
},
|
||||
...(control.config.hostKeys.length
|
||||
? [
|
||||
{
|
||||
label: "Forget host key",
|
||||
onSelect: (ctx: { stack: { pop(): unknown } }) => {
|
||||
ctx.stack.pop();
|
||||
control.config.hostKeys = [];
|
||||
updateConnection(control.config.id, { hostKeys: [] });
|
||||
control.sshIssue = null;
|
||||
if (control.config.enabled) startControl(control);
|
||||
renderHubWindows();
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(isSshConnection(control) && getSshSecret(sshPasswordSecretId(control.config.id))
|
||||
? [
|
||||
{
|
||||
label: "Forget saved password",
|
||||
onSelect: (ctx: { stack: { pop(): unknown } }) => {
|
||||
ctx.stack.pop();
|
||||
removeSshSecret(sshPasswordSecretId(control.config.id));
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
label: `Remove ${name}`,
|
||||
onSelect: (ctx) => {
|
||||
@@ -1558,6 +1970,11 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
|
||||
windowMenu(window).open(items);
|
||||
}
|
||||
|
||||
function isSshConnection(control: ControlConnection): boolean {
|
||||
const kind = clientOptionsFor(control.config)?.kind;
|
||||
return kind === "tmux" || kind === "ssh-g2mirror";
|
||||
}
|
||||
|
||||
/** Enable and (re)connect now; also the manual retry path, so reset backoff. */
|
||||
function connectControl(control: ControlConnection): void {
|
||||
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
|
||||
@@ -1578,6 +1995,7 @@ function disconnectControl(control: ControlConnection): void {
|
||||
|
||||
function removeControl(control: ControlConnection): void {
|
||||
stopControl(control);
|
||||
removeSshSecret(sshPasswordSecretId(control.config.id));
|
||||
controls.delete(control.config.id);
|
||||
saveConnections(loadConnections().filter((connection) => connection.id !== control.config.id));
|
||||
renderHubWindows();
|
||||
@@ -1586,7 +2004,7 @@ function removeControl(control: ControlConnection): void {
|
||||
/**
|
||||
* Enter the Add-connection screen: clear the draft and ask the shell to open
|
||||
* the phone text editor on it. The user types (or voice-inputs) the
|
||||
* g2mirror:// string, then clicks to save or double-clicks to cancel.
|
||||
* connection string, then clicks to save or double-clicks to cancel.
|
||||
*/
|
||||
function beginAddConnection(window: HubWindow): void {
|
||||
terminalNewConnectionSetting.set("");
|
||||
@@ -1596,15 +2014,15 @@ function beginAddConnection(window: HubWindow): void {
|
||||
|
||||
function saveAddConnection(window: HubWindow): void {
|
||||
const url = terminalNewConnectionSetting.get();
|
||||
if (!parseConnectionString(url)) {
|
||||
if (!parseAnyConnectionString(url)) {
|
||||
window.addError = url
|
||||
? "Not a g2mirror://token@host connection string."
|
||||
: "Enter a g2mirror:// connection string first.";
|
||||
? "Not an ssh://user@host or g2mirror://token@host string."
|
||||
: "Enter a connection string first.";
|
||||
scheduleRender(window);
|
||||
return;
|
||||
}
|
||||
const connections = loadConnections();
|
||||
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true });
|
||||
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true, hostKeys: [] });
|
||||
saveConnections(connections);
|
||||
endAddConnection(window);
|
||||
setHubMode(window, "connections");
|
||||
@@ -1634,6 +2052,15 @@ function handleHubInput(window: HubWindow, event: InputEvent, frameId: number):
|
||||
frameTimings.finishFrame(frameId, "discarded: terminal add-connection ignored input");
|
||||
return;
|
||||
}
|
||||
if (window.mode === "page") {
|
||||
if (event.type === "click" && window.page?.onClick) {
|
||||
window.page.onClick();
|
||||
renderAndSubmit(window, frameId);
|
||||
return;
|
||||
}
|
||||
frameTimings.finishFrame(frameId, "discarded: terminal page ignored input");
|
||||
return;
|
||||
}
|
||||
const list = hubMenu(window);
|
||||
list.setItems(hubItems(window));
|
||||
switch (event.type) {
|
||||
@@ -1800,6 +2227,7 @@ function openViewWindow(control: ControlConnection, socket: string, label: strin
|
||||
focus: true,
|
||||
// Default to tall sessions; the app display setting can override this.
|
||||
heightMode: "max",
|
||||
acceptsKeys: true,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1820,6 +2248,11 @@ function launchPresetNames(): string[] {
|
||||
*/
|
||||
function launchOnHost(window: HubWindow, control: ControlConnection): void {
|
||||
const presets = launchPresetNames();
|
||||
if (presets.length === 0) {
|
||||
// A tmux host with no presets configured: a new session running the login shell.
|
||||
launchFromUi(control, "shell");
|
||||
return;
|
||||
}
|
||||
if (presets.length === 1) {
|
||||
launchFromUi(control, presets[0]!);
|
||||
return;
|
||||
@@ -1864,6 +2297,9 @@ function paintHub(window: HubWindow): GrayImage {
|
||||
if (window.mode === "add") {
|
||||
return paintAddConnection(window);
|
||||
}
|
||||
if (window.mode === "page" && window.page) {
|
||||
return paintHubPage(window, window.page);
|
||||
}
|
||||
const { viewportWidth: width, viewportHeight: height, padding } = window;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const font = chromeFont();
|
||||
@@ -1878,7 +2314,7 @@ function paintHub(window: HubWindow): GrayImage {
|
||||
|
||||
let listTop = header.listTop;
|
||||
if (window.mode === "sessions" && controls.size === 0) {
|
||||
image.drawText(font, header.titleX, listTop, "Add a g2mirror:// connection to get started, see:", 150);
|
||||
image.drawText(font, header.titleX, listTop, "Add an ssh:// (tmux) or g2mirror:// connection to get started, see:", 150);
|
||||
image.drawText(font, header.titleX, listTop + step, "https://github.com/jimrandomh/g2mirror", 190);
|
||||
listTop += 2 * step + 6;
|
||||
}
|
||||
@@ -1920,8 +2356,8 @@ function paintAddConnection(window: HubWindow): GrayImage {
|
||||
image.drawText(font, padding.left, padding.top, "Add connection", 220);
|
||||
const bodyX = padding.left + PAGE_BODY_INDENT;
|
||||
const instructionsY = padding.top + 10 + step;
|
||||
image.drawText(font, bodyX, instructionsY, "Type the g2mirror://token@host string in the", 170);
|
||||
image.drawText(font, bodyX, instructionsY + step, "phone app (or use voice input from the menu).", 170);
|
||||
image.drawText(font, bodyX, instructionsY, "Type ssh://user@host (tmux) or g2mirror://token@host in", 170);
|
||||
image.drawText(font, bodyX, instructionsY + step, "the phone app (or use voice input from the menu).", 170);
|
||||
const draftY = instructionsY + 3 * step;
|
||||
const draft = terminalNewConnectionSetting.get();
|
||||
image.drawText(font, bodyX, draftY, truncateLabel(draft || "(empty)", 52), 220);
|
||||
@@ -1932,6 +2368,34 @@ function paintAddConnection(window: HubWindow): GrayImage {
|
||||
return image;
|
||||
}
|
||||
|
||||
/** A page of text: title at the top padding, wrapped body, hint on the bottom padding. */
|
||||
function paintHubPage(window: HubWindow, page: HubPage): GrayImage {
|
||||
const { viewportWidth: width, viewportHeight: height, padding } = window;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const font = chromeFont();
|
||||
const step = lineStep(font);
|
||||
image.drawText(font, padding.left, padding.top, page.title, 220);
|
||||
const bodyX = padding.left + PAGE_BODY_INDENT;
|
||||
const hintY = height - padding.bottom - font.lineHeight;
|
||||
const lines = wrapText(font, page.body(), Math.max(1, width - padding.right - bodyX), { breakLongWords: true });
|
||||
let y = padding.top + 10 + step;
|
||||
for (const line of lines) {
|
||||
if (y + step > hintY) break;
|
||||
image.drawText(font, bodyX, y, line, 190);
|
||||
y += step;
|
||||
}
|
||||
image.drawText(font, bodyX, hintY, page.hint, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
/** The action row under a host whose SSH connection needs the user. */
|
||||
function sshIssueAction(issue: SshFailure): string {
|
||||
if (issue.kind === "hostkey-unknown") return "Verify host key...";
|
||||
if (issue.kind === "hostkey-mismatch") return "Host key changed!";
|
||||
if (issue.passwordAllowed) return issue.passwordTried ? "Re-enter password..." : "Enter password...";
|
||||
return "Show this device's SSH key...";
|
||||
}
|
||||
|
||||
function truncateLabel(text: string, maxLength: number): string {
|
||||
return text.length <= maxLength ? text : `${text.slice(0, Math.max(0, maxLength - 3))}...`;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import type {
|
||||
G2MirrorHistoryReply,
|
||||
G2MirrorPhase,
|
||||
G2MirrorSession,
|
||||
G2MirrorState,
|
||||
} from "../../native/terminal-session-types";
|
||||
|
||||
/**
|
||||
* What the Terminal worker needs from a session backend. G2MirrorClient (a
|
||||
* g2mirror server, directly or tunnelled over SSH) and TmuxClient (tmux
|
||||
* control mode over SSH) both provide it. The model is g2mirror's: a client
|
||||
* starts "connecting", reaches "connected" (it can list and launch sessions,
|
||||
* and reports bells/activity/titles for them), and can then attach to one
|
||||
* session, view it (snapshot + live output), page its scrollback, and type
|
||||
* into it.
|
||||
*/
|
||||
export interface TerminalClient {
|
||||
state(): G2MirrorState;
|
||||
onStateChange(listener: (state: G2MirrorState) => void): () => void;
|
||||
/** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */
|
||||
onTerminalData(listener: (data: Uint8Array, kind: "snapshot" | "output") => void): () => void;
|
||||
onSessionAttached(listener: (command: string) => void): () => void;
|
||||
onSessionDetached(listener: (reason: string) => void): () => void;
|
||||
onBell(listener: (socket: string, lastBellAtMs: number) => void): () => void;
|
||||
onActivity(listener: (socket: string, lastOutputAtMs: number) => void): () => void;
|
||||
onTitle(listener: (socket: string, title: string) => void): () => void;
|
||||
/** Fired after a snapshot's terminal data, with its scrollback splice point. */
|
||||
onSnapshot(listener: (historyNext: number, historyOldest: number) => void): () => void;
|
||||
onHistoryLines(listener: (reply: G2MirrorHistoryReply) => void): () => void;
|
||||
setViewport(cols: number, rows: number): void;
|
||||
start(): void;
|
||||
stop(): void;
|
||||
listSessions(): void;
|
||||
launchSession(preset: string): Promise<string>;
|
||||
connectSession(socket: string): void;
|
||||
disconnectSession(): void;
|
||||
view(): void;
|
||||
unview(): void;
|
||||
requestHistory(before: number, limit?: number): void;
|
||||
sendInput(text: string): void;
|
||||
submitInput(text: string): void;
|
||||
}
|
||||
|
||||
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
|
||||
@@ -82,4 +82,9 @@ export class TerminalEmulator {
|
||||
cursorCol(): number {
|
||||
return this.term.buffer.active.cursorX;
|
||||
}
|
||||
|
||||
/** DECCKM: the session asked for application cursor keys (ESC O A rather than ESC [ A). */
|
||||
applicationCursorKeys(): boolean {
|
||||
return this.term.modes.applicationCursorKeysMode;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/**
|
||||
* What a terminal emulator sends for a key from a hardware keyboard, by the
|
||||
* classic xterm conventions: Ctrl chords become control characters, Alt
|
||||
* prefixes ESC, and modified cursor and function keys carry the xterm
|
||||
* modifier parameter (1 + shift + 2*alt + 4*ctrl). Follows the Glassly
|
||||
* Terminal miniapp's encoder (terminal/keys.ts, MIT).
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import type { KeyEvent } from "../../g2/keyboard/keys";
|
||||
|
||||
const ESC = "\x1b";
|
||||
|
||||
/** Keys xterm encodes as `ESC [ <n> ~`, with a modifier as `;m`. */
|
||||
const TILDE_KEYS: Record<string, number> = {
|
||||
Insert: 2,
|
||||
Delete: 3,
|
||||
PageUp: 5,
|
||||
PageDown: 6,
|
||||
F5: 15,
|
||||
F6: 17,
|
||||
F7: 18,
|
||||
F8: 19,
|
||||
F9: 20,
|
||||
F10: 21,
|
||||
F11: 23,
|
||||
F12: 24,
|
||||
};
|
||||
|
||||
/** Keys with a single-letter final byte: SS3 in application cursor mode, CSI otherwise. */
|
||||
const LETTER_KEYS: Record<string, string> = {
|
||||
ArrowUp: "A",
|
||||
ArrowDown: "B",
|
||||
ArrowRight: "C",
|
||||
ArrowLeft: "D",
|
||||
Home: "H",
|
||||
End: "F",
|
||||
};
|
||||
|
||||
/** F1-F4 are always SS3 (`ESC O P`), or CSI with a modifier. */
|
||||
const SS3_KEYS: Record<string, string> = { F1: "P", F2: "Q", F3: "R", F4: "S" };
|
||||
|
||||
/** Ctrl with a punctuation or digit key, per xterm (^[ ^\ ^] ^^ ^_ ^? ^@). */
|
||||
const CONTROL_PUNCTUATION: Record<string, string> = {
|
||||
"[": "\x1b",
|
||||
"\\": "\x1c",
|
||||
"]": "\x1d",
|
||||
"^": "\x1e",
|
||||
"6": "\x1e",
|
||||
"_": "\x1f",
|
||||
"-": "\x1f",
|
||||
"?": "\x7f",
|
||||
"/": "\x7f",
|
||||
"8": "\x7f",
|
||||
"@": "\x00",
|
||||
"2": "\x00",
|
||||
};
|
||||
|
||||
/**
|
||||
* The text to write to the terminal for one key event, or null when the key
|
||||
* sends nothing: releases, bare modifiers, Meta chords, and keys a terminal
|
||||
* has no bytes for (volume, media). `appCursor` is DECCKM, which the emulator
|
||||
* tracks from the session's output.
|
||||
*/
|
||||
export function encodeTerminalKey(event: KeyEvent, appCursor: boolean): string | null {
|
||||
if (event.phase !== "down") return null;
|
||||
const { key, modifiers } = event;
|
||||
if (modifiers.meta) return null;
|
||||
const modifierParam = 1 + (modifiers.shift ? 1 : 0) + (modifiers.alt ? 2 : 0) + (modifiers.ctrl ? 4 : 0);
|
||||
const modified = (prefix: string, final: string): string | null =>
|
||||
modifierParam > 1 ? `${ESC}[${prefix};${modifierParam}${final}` : null;
|
||||
|
||||
switch (key) {
|
||||
case "Enter":
|
||||
return modifiers.alt ? `${ESC}\r` : "\r";
|
||||
case "Escape":
|
||||
return ESC;
|
||||
case "Tab":
|
||||
return modifiers.shift ? `${ESC}[Z` : "\t";
|
||||
case "Backspace":
|
||||
if (modifiers.alt) return `${ESC}\x7f`;
|
||||
return modifiers.ctrl ? "\x08" : "\x7f";
|
||||
case " ":
|
||||
return modifiers.ctrl ? "\x00" : modifiers.alt ? `${ESC} ` : " ";
|
||||
}
|
||||
|
||||
const letter = LETTER_KEYS[key];
|
||||
if (letter) return modified("1", letter) ?? (appCursor ? `${ESC}O${letter}` : `${ESC}[${letter}`);
|
||||
const tilde = TILDE_KEYS[key];
|
||||
if (tilde) return modified(String(tilde), "~") ?? `${ESC}[${tilde}~`;
|
||||
const ss3 = SS3_KEYS[key];
|
||||
if (ss3) return modified("1", ss3) ?? `${ESC}O${ss3}`;
|
||||
|
||||
if (modifiers.ctrl) {
|
||||
// From the unshifted key: ^A..^Z, then the punctuation set.
|
||||
const base = key.length === 1 ? key.toLowerCase() : "";
|
||||
let char: string | undefined;
|
||||
if (base >= "a" && base <= "z") char = String.fromCharCode(base.charCodeAt(0) - 96);
|
||||
else char = CONTROL_PUNCTUATION[key];
|
||||
if (char === undefined) return null;
|
||||
return modifiers.alt ? `${ESC}${char}` : char;
|
||||
}
|
||||
|
||||
// With Alt held the translator leaves `text` out (it is a chord, not
|
||||
// typing); a terminal still wants ESC and the character.
|
||||
const text = event.text ?? (modifiers.alt && key.length === 1 ? key : undefined);
|
||||
if (text === undefined) return null;
|
||||
return modifiers.alt ? `${ESC}${text}` : text;
|
||||
}
|
||||
|
||||
/** Shift+PageUp / Shift+PageDown page the emulator's own scrollback, as in xterm. */
|
||||
export function terminalScrollbackKey(event: KeyEvent): -1 | 1 | null {
|
||||
if (event.phase !== "down") return null;
|
||||
const { shift, ctrl, alt, meta } = event.modifiers;
|
||||
if (!shift || ctrl || alt || meta) return null;
|
||||
if (event.key === "PageUp") return -1;
|
||||
if (event.key === "PageDown") return 1;
|
||||
return null;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,32 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app";
|
||||
import { type AppContext, type AppDefinition } from "../app-definition";
|
||||
import {
|
||||
createTranscribeAppWindow,
|
||||
createTranscribeReplayWindow,
|
||||
TRANSCRIBE_SURFACE_ID,
|
||||
TRANSCRIBE_WINDOW_ID,
|
||||
} from "./transcribe-app";
|
||||
|
||||
// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique.
|
||||
let nextReplaySerial = 1;
|
||||
|
||||
/**
|
||||
* Open a recording (.mka) in Transcribe's replay mode, as a window of its
|
||||
* own. Returns false where recordings can't be replayed.
|
||||
*/
|
||||
export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean {
|
||||
if (!ctx.actions.openVoiceReplay) return false;
|
||||
const windowId = `transcribe:replay:${nextReplaySerial++}`;
|
||||
void ctx
|
||||
.launchInProcessApp(windowId, `window:${windowId}`, (options) => {
|
||||
const window = createTranscribeReplayWindow(windowId, path, name, options);
|
||||
if (!window) throw new Error("recordings can't be replayed here");
|
||||
return window;
|
||||
})
|
||||
.catch((error) => {
|
||||
ctx.appendLog(`transcribe replay window failed: ${error}`);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
const transcribeApp: AppDefinition = {
|
||||
appId: "transcribe",
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */
|
||||
export function formatReplayTime(ms: number, withHours = false): string {
|
||||
const total = Math.max(0, Math.floor(ms / 1000));
|
||||
const hours = Math.floor(total / 3600);
|
||||
const minutes = Math.floor((total % 3600) / 60);
|
||||
const seconds = String(total % 60).padStart(2, "0");
|
||||
if (hours === 0 && !withHours) return `${minutes}:${seconds}`;
|
||||
return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`;
|
||||
}
|
||||
|
||||
/** The time picker's fields, in display and click order. */
|
||||
export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const;
|
||||
export type TimeField = (typeof TIME_FIELDS)[number];
|
||||
|
||||
const FIELD_SECONDS: Record<TimeField, number> = { hours: 3600, minutes: 60, seconds: 1 };
|
||||
|
||||
/**
|
||||
* Step a position (whole seconds) by `delta` of a field's unit, carrying
|
||||
* into the neighboring fields, within [0, maxSeconds].
|
||||
*/
|
||||
export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number {
|
||||
return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field]));
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext, PaintBelow } from "../../ui/layers";
|
||||
import { CONTEXT_MENU_DIM } from "../../ui/menu";
|
||||
import { centeredTextY } from "../../ui/metrics";
|
||||
import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time";
|
||||
|
||||
const PADDING = 12;
|
||||
const FIELD_GAP = 6;
|
||||
const FIELD_PAD_X = 6;
|
||||
/** Fill and outline of the active field (the Timers and date pickers' field style). */
|
||||
const ACTIVE_FILL = 35;
|
||||
const ACTIVE_STROKE = 130;
|
||||
|
||||
export type TimePickerOptions = {
|
||||
title: string;
|
||||
/** Where the picker starts, and the furthest it goes, in ms. */
|
||||
initialMs: number;
|
||||
maxMs: number;
|
||||
onPick: (positionMs: number) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* A modal position picker for a recording: hours (when the recording has
|
||||
* any), minutes and seconds, over the dimmed content beneath. Input follows
|
||||
* the date picker's: scroll adjusts the active field (up is later, carrying
|
||||
* into the fields beside it), click moves to the next field and on the last
|
||||
* one picks, double-click steps back a field and on the first one cancels.
|
||||
* A watch's swipes move between fields and adjust.
|
||||
*/
|
||||
export class TimePickerLayer implements Layer {
|
||||
readonly acceptsDirectional = true;
|
||||
readonly dimUnderneath = CONTEXT_MENU_DIM;
|
||||
private readonly fields: readonly TimeField[];
|
||||
private readonly maxSeconds: number;
|
||||
private seconds: number;
|
||||
private field = 0;
|
||||
|
||||
constructor(private readonly options: TimePickerOptions) {
|
||||
this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000));
|
||||
this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000)));
|
||||
this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1);
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent, ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "swipe-up":
|
||||
this.adjust(1);
|
||||
return;
|
||||
case "scroll-down":
|
||||
case "swipe-down":
|
||||
this.adjust(-1);
|
||||
return;
|
||||
case "swipe-right":
|
||||
this.field = Math.min(this.fields.length - 1, this.field + 1);
|
||||
return;
|
||||
case "swipe-left":
|
||||
this.field = Math.max(0, this.field - 1);
|
||||
return;
|
||||
case "click":
|
||||
if (this.field < this.fields.length - 1) {
|
||||
this.field++;
|
||||
} else {
|
||||
ctx.stack.pop();
|
||||
this.options.onPick(this.seconds * 1000);
|
||||
}
|
||||
return;
|
||||
case "double-click":
|
||||
if (this.field > 0) this.field--;
|
||||
else ctx.stack.pop();
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private adjust(delta: number): void {
|
||||
this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds);
|
||||
}
|
||||
|
||||
private hints(field: number): string {
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, "adjust"],
|
||||
[GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"],
|
||||
[GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"],
|
||||
]);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
const image = paintBelow();
|
||||
const small = getDefaultSmallFont();
|
||||
const medium = getDefaultMediumFont();
|
||||
const focused = ctx.stack.isFocused();
|
||||
const base = ctx.stack.getBaseSize();
|
||||
|
||||
const values = [
|
||||
String(Math.floor(this.seconds / 3600)),
|
||||
String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"),
|
||||
String(this.seconds % 60).padStart(2, "0"),
|
||||
].slice(TIME_FIELDS.length - this.fields.length);
|
||||
// Sized for the widest value a field can show, so the dialog holds still while the time changes.
|
||||
const fieldWidths = this.fields.map(
|
||||
(field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X,
|
||||
);
|
||||
const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1);
|
||||
const boxHeight = medium.lineHeight + 6;
|
||||
const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`;
|
||||
const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index))));
|
||||
const width = Math.min(
|
||||
base.width,
|
||||
Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING,
|
||||
);
|
||||
const titleHeight = small.lineHeight + 12;
|
||||
const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight;
|
||||
const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10);
|
||||
const x = Math.max(0, Math.floor((base.width - width) / 2));
|
||||
const y = Math.max(0, Math.floor((base.height - height) / 2));
|
||||
|
||||
// Fill 1, not 0: 0 is the transparent color key on the shell surface.
|
||||
image.fillRoundedRect(x, y, width, height, 1);
|
||||
image.drawRoundedRect(x, y, width, height, 72);
|
||||
image.drawText(small, x + PADDING, y + 8, this.options.title, 220);
|
||||
|
||||
const top = y + 8 + titleHeight;
|
||||
let fieldX = x + PADDING;
|
||||
for (let index = 0; index < this.fields.length; index++) {
|
||||
const fieldWidth = fieldWidths[index]!;
|
||||
const active = index === this.field;
|
||||
if (active) {
|
||||
if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6);
|
||||
image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6);
|
||||
}
|
||||
const value = values[index]!;
|
||||
image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2),
|
||||
centeredTextY(medium, top, boxHeight), value, active ? 250 : 170);
|
||||
const caption = this.fields[index]!;
|
||||
image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2),
|
||||
top + boxHeight + 2, caption, active ? 190 : 110);
|
||||
fieldX += fieldWidth + FIELD_GAP;
|
||||
}
|
||||
image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190);
|
||||
image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110);
|
||||
return image;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
import { withRecordingIcon } from "../../ui/listening-header";
|
||||
import { TimePickerLayer } from "./time-picker";
|
||||
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
||||
import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
YieldAtRootLayer,
|
||||
@@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage {
|
||||
}
|
||||
|
||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||
/** Begin continuous mic capture (kept on while the window is open). */
|
||||
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void;
|
||||
stopContinuousVoiceCapture: () => void;
|
||||
};
|
||||
|
||||
@@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
// One timestamp per session names both the audio recording and a saved transcript.
|
||||
const stamp = transcriptTimestamp();
|
||||
const layer = new TranscribeLayer(stamp);
|
||||
// At launch, and again when the Microphones setting changes: a running
|
||||
// capture moves over to the microphones now asked for.
|
||||
const startCapture = () =>
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() });
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId: TRANSCRIBE_WINDOW_ID,
|
||||
@@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
},
|
||||
// The controller starts and stops the recording as the setting changes.
|
||||
toggleSettingMenuItem(saveRecordingSetting),
|
||||
enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }),
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
@@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
app.requestRender();
|
||||
});
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
startCapture();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||
return app;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe in replay mode: a window of its own that transcribes a
|
||||
* recording (a .mka file from Save Recording) in place of the microphones,
|
||||
* in real time and through the same pipeline. It neither uses the mic nor
|
||||
* shows the mic tray icon, so it runs beside a live Transcribe window. The
|
||||
* header shows the position in the recording; a tap pauses and resumes; the
|
||||
* menu jumps to a time.
|
||||
*/
|
||||
export function createTranscribeReplayWindow(
|
||||
windowId: string,
|
||||
path: string,
|
||||
name: string,
|
||||
options: InProcessAppOptions,
|
||||
): InProcessWindow | null {
|
||||
const replay = options.actions.openVoiceReplay?.(path) ?? null;
|
||||
if (!replay) return null;
|
||||
// Names a saved transcript after the recording it came from.
|
||||
const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name);
|
||||
const app = createInProcessWindow({
|
||||
appId: "transcribe",
|
||||
windowId,
|
||||
title: name,
|
||||
iconLetter: "Tr",
|
||||
icon: "mic",
|
||||
closeable: true,
|
||||
// Relaunching Transcribe after a restart would open the microphone, not this recording.
|
||||
restoresApp: false,
|
||||
// Like a playing video: something to watch, not to touch.
|
||||
keepsScreenOn: () => replay.state === "playing",
|
||||
actions: options.actions,
|
||||
baseLayer: new YieldAtRootLayer(layer),
|
||||
menuItems: () => [
|
||||
{
|
||||
label: "Jump to time",
|
||||
// The length isn't known until the recording has been opened.
|
||||
disabled: () => replay.durationMs <= 0,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(
|
||||
new TimePickerLayer({
|
||||
title: "Jump to time",
|
||||
initialMs: replay.positionMs,
|
||||
maxMs: replay.durationMs,
|
||||
onPick: (positionMs) => {
|
||||
replay.jumpTo(positionMs);
|
||||
app.requestRender();
|
||||
},
|
||||
}),
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: replay.state === "paused" ? "Resume" : "Pause",
|
||||
disabled: () => replay.state !== "playing" && replay.state !== "paused",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.setPaused(replay.state === "playing");
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Restart from 0:00",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
replay.jumpTo(0);
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Save",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.saveTranscript();
|
||||
},
|
||||
},
|
||||
],
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
layer.onRemoved();
|
||||
replay.close();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
// Started first, so the layer picks its opening status up as it subscribes.
|
||||
replay.start();
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(windowId));
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings";
|
||||
import { type MicCount } from "../../ui/layers";
|
||||
import { micControlSupported } from "../microphones/mic-control";
|
||||
|
||||
export type TranscribeMics = "1" | "2" | "4";
|
||||
|
||||
const MICS_LABELS: Record<TranscribeMics, string> = {
|
||||
"1": "1 (mixed)",
|
||||
"2": "2 (one per temple)",
|
||||
"4": "4 (two per temple)",
|
||||
};
|
||||
|
||||
/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */
|
||||
export function separateMicsSupported(): boolean {
|
||||
return glassesTypeSetting.get() === "g2" && micControlSupported();
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones Transcribe captures. With more than
|
||||
* one, the temples stream their own microphones (each temple on its own
|
||||
* clock) instead of the glasses' mixed stream; a recording keeps every one,
|
||||
* and what is transcribed is a mix made on the phone.
|
||||
*/
|
||||
export const transcribeMicsSetting = new ConfigSettingEnum<TranscribeMics>({
|
||||
id: "transcribe-mics",
|
||||
label: "Microphones",
|
||||
storageKey: "transcribe.mics",
|
||||
defaultValue: "1",
|
||||
values: ["1", "2", "4"],
|
||||
formatValue: (value) => MICS_LABELS[value],
|
||||
isDisabled: (value) => value !== "1" && !separateMicsSupported(),
|
||||
description:
|
||||
"Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.",
|
||||
});
|
||||
|
||||
/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */
|
||||
export function transcribeMicCount(): MicCount {
|
||||
if (!separateMicsSupported()) return 1;
|
||||
return Number(transcribeMicsSetting.get()) as MicCount;
|
||||
}
|
||||
@@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { formatReplayTime } from "./replay-time";
|
||||
import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model";
|
||||
|
||||
/**
|
||||
* Live transcription view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
|
||||
* it, and saves it to Downloads from the app menu.
|
||||
*
|
||||
* Given a replay, it transcribes that recording instead of the microphone:
|
||||
* the header shows the position in the recording, a tap pauses and resumes,
|
||||
* and each jump leaves its time as a line in the transcript.
|
||||
*/
|
||||
export class TranscribeLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer {
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private unsubscribeReplay: (() => void) | null = null;
|
||||
private readonly source: VoiceEventSource;
|
||||
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
/**
|
||||
* `stamp` names the saved transcript, matching the session's audio
|
||||
* recording. `replay` is the recording to transcribe in place of the
|
||||
* microphone; `replayName` is what to call it.
|
||||
*/
|
||||
constructor(
|
||||
private readonly stamp: string = transcriptTimestamp(),
|
||||
private readonly replay: VoiceReplay | null = null,
|
||||
private readonly replayName = "",
|
||||
) {
|
||||
this.source = replay ? replay.events : voiceControlBridge;
|
||||
if (replay) this.status = "Opening the recording...";
|
||||
}
|
||||
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
/** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.meter.start(requestRender, isVisible, this.source);
|
||||
this.unsubscribeTranscript = this.source.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
|
||||
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
||||
this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause());
|
||||
// The current status arrives while subscribing, before the window has a
|
||||
// surface to render to; the window paints it when it is first shown.
|
||||
let subscribing = true;
|
||||
this.unsubscribeStatus = this.source.onStatus((state) => {
|
||||
this.status = state.status;
|
||||
this.listening = state.listening;
|
||||
this.detail = state.detail;
|
||||
this.saveNotice = "";
|
||||
requestRender();
|
||||
if (!subscribing) requestRender();
|
||||
});
|
||||
subscribing = false;
|
||||
const replay = this.replay;
|
||||
if (replay) {
|
||||
let shownSecond = -1;
|
||||
let shownState = replay.state;
|
||||
this.unsubscribeReplay = replay.onChange((jumpedToMs) => {
|
||||
if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`);
|
||||
// The clock ticks in whole seconds; repaint when what it shows changed.
|
||||
const second = Math.floor(replay.positionMs / 1000);
|
||||
if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return;
|
||||
shownSecond = second;
|
||||
shownState = replay.state;
|
||||
if (isVisible()) requestRender();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */
|
||||
private replayStatus(replay: VoiceReplay): string {
|
||||
if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status;
|
||||
const withHours = replay.durationMs >= 3_600_000;
|
||||
const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`;
|
||||
if (replay.state === "paused") return `${clock} · Paused`;
|
||||
if (replay.state === "ended") return `${clock} · Finished`;
|
||||
// Anything but plain listening is worth showing: the model loading, a provider reconnecting.
|
||||
return this.listening ? clock : `${clock} · ${this.status}`;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
@@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer {
|
||||
const viewport = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(viewport.width, viewport.height, 0);
|
||||
|
||||
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const replay = this.replay;
|
||||
const status = replay
|
||||
? this.replayStatus(replay)
|
||||
: this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||
const playing = !replay || replay.state === "playing";
|
||||
const bodyTop = paintListeningHeader(
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
|
||||
voiceControlBridge.recording,
|
||||
image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding,
|
||||
this.source.recording,
|
||||
);
|
||||
const step = lineStep(font) + 2;
|
||||
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const placeholder = replay ? `Replaying ${this.replayName}` : "Listening...";
|
||||
const text = this.transcript.text || (this.listening ? placeholder : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth);
|
||||
|
||||
const firstLine = this.scroll.layout(wrapped.length, layout.pageLines);
|
||||
@@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer {
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "click" && this.replay) {
|
||||
this.replay.setPaused(this.replay.state === "playing");
|
||||
ctx.actions.requestRender();
|
||||
return;
|
||||
}
|
||||
if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribeReplay?.();
|
||||
this.unsubscribeReplay = null;
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -36,6 +36,18 @@ export class TranscriptModel {
|
||||
if (this.live) this.breakAfterLive = true;
|
||||
else if (this.finalized) this.separator = "\n";
|
||||
}
|
||||
|
||||
/**
|
||||
* A line of its own (where a replay jumped to), ending whatever was in
|
||||
* progress: what follows it starts a new paragraph.
|
||||
*/
|
||||
mark(label: string): void {
|
||||
const before = this.text;
|
||||
this.finalized = before ? `${before}\n${label}` : label;
|
||||
this.live = "";
|
||||
this.separator = "\n";
|
||||
this.breakAfterLive = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** An absolute history position, with an explicit follow-tail mode. */
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Date and calendar-event text for the assistant's tool results. Pure (no
|
||||
* native imports) so it can be unit-tested.
|
||||
*/
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
export type FormattableEvent = {
|
||||
title: string;
|
||||
startMs: number;
|
||||
endMs: number;
|
||||
allDay: boolean;
|
||||
location?: string;
|
||||
};
|
||||
|
||||
export function formatEvent(event: FormattableEvent): string {
|
||||
const location = event.location ? ` @ ${event.location}` : "";
|
||||
return `- ${event.title || "(untitled)"} — ${formatEventSpan(event)}${location}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* When an event happens, end included: a multi-day all-day event names its last
|
||||
* day (all-day ends are exclusive, at the midnight after it), a timed event its
|
||||
* end time. Without the end, an event that started yesterday and runs through
|
||||
* tomorrow reads as yesterday's.
|
||||
*/
|
||||
export function formatEventSpan(event: Pick<FormattableEvent, "startMs" | "endMs" | "allDay">): string {
|
||||
const start = new Date(event.startMs);
|
||||
if (event.allDay) {
|
||||
const lastDay = new Date(Math.max(event.startMs, event.endMs - 1));
|
||||
return sameDay(start, lastDay)
|
||||
? `${formatDate(start)} (all day)`
|
||||
: `${formatDate(start)} – ${formatDate(lastDay)} (all day)`;
|
||||
}
|
||||
if (!(event.endMs > event.startMs)) return formatDateTime(start);
|
||||
const end = new Date(event.endMs);
|
||||
return sameDay(start, end)
|
||||
? `${formatDateTime(start)} – ${formatTime(end)}`
|
||||
: `${formatDateTime(start)} – ${formatDateTime(end)}`;
|
||||
}
|
||||
|
||||
export function formatDateTime(date: Date): string {
|
||||
return `${formatDate(date)}, ${formatTime(date)}`;
|
||||
}
|
||||
|
||||
export function formatDate(date: Date): string {
|
||||
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
|
||||
}
|
||||
|
||||
export function formatTime(date: Date): string {
|
||||
let hours = date.getHours();
|
||||
const minutes = date.getMinutes().toString().padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
}
|
||||
|
||||
function sameDay(a: Date, b: Date): boolean {
|
||||
return a.getFullYear() === b.getFullYear() && a.getMonth() === b.getMonth() && a.getDate() === b.getDate();
|
||||
}
|
||||
@@ -1,10 +1,11 @@
|
||||
import { readUpcomingEventsAsync, type CalendarEvent } from "../native/calendar";
|
||||
import { readUpcomingEventsAsync } from "../native/calendar";
|
||||
import { hasCalendarPermission } from "../native/calendar-permissions";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { spotifyService } from "../native/spotify";
|
||||
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
|
||||
import { formatDateTime, formatEvent } from "./date-format";
|
||||
|
||||
/**
|
||||
* Registers the always-available system tools into the registry. Called once at
|
||||
@@ -12,9 +13,6 @@ import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registr
|
||||
* media, notifications); nothing here needs an app window to be open.
|
||||
*/
|
||||
|
||||
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
|
||||
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
|
||||
let registered = false;
|
||||
|
||||
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
|
||||
@@ -244,35 +242,12 @@ function describeNowPlaying(): string {
|
||||
return "Nothing is currently playing.";
|
||||
}
|
||||
|
||||
function formatEvent(event: CalendarEvent): string {
|
||||
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
|
||||
const location = event.location ? ` @ ${event.location}` : "";
|
||||
return `- ${event.title || "(untitled)"} — ${when}${location}`;
|
||||
}
|
||||
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string; customText?: string[] }): string {
|
||||
const title = notification.title || notification.customText?.join(" ") || "(no title)";
|
||||
const body = notification.text ? `: ${notification.text}` : "";
|
||||
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
|
||||
}
|
||||
|
||||
function formatDateTime(date: Date): string {
|
||||
return `${formatDate(date)}, ${formatTime(date)}`;
|
||||
}
|
||||
|
||||
function formatDate(date: Date): string {
|
||||
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
|
||||
}
|
||||
|
||||
function formatTime(date: Date): string {
|
||||
let hours = date.getHours();
|
||||
const minutes = date.getMinutes().toString().padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
}
|
||||
|
||||
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
|
||||
const num = typeof value === "number" ? value : Number(value);
|
||||
if (!Number.isFinite(num)) return fallback;
|
||||
|
||||
@@ -20,9 +20,14 @@ import { isPhoneDoNotDisturbEnabled } from "../native/phone-do-not-disturb";
|
||||
import { watchNotificationListenerBinding } from "../native/notification-access";
|
||||
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
|
||||
import { defaultRingName, setGlassesRingLink } from "../native/ring-status";
|
||||
import { applyGlanceAngle, glassesHeadUpTrigger, syncGlanceAngleOriginal } from "./head-angle";
|
||||
import { glanceAngleTriggerDegrees, glanceTapTimeoutMs } from "../apps/glanceboard/glanceboard-settings";
|
||||
import { GLANCE_TILT_MAX_MS, glanceTiltMaxMs } from "./glance-state";
|
||||
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
|
||||
import { grayImageToPreviewSource } from "../native/gray-image-preview";
|
||||
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from "./firmware-compat";
|
||||
import { keyboardService } from "./keyboard/service";
|
||||
import type { KeyEvent } from "./keyboard/keys";
|
||||
import { hasExtractedEvenHubFonts } from "./firmware-builder";
|
||||
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy";
|
||||
import { WearRemote, type WearRemoteInputKind } from "./wear-remote";
|
||||
@@ -54,7 +59,7 @@ import { type GlanceEvent } from "./glance-state";
|
||||
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
|
||||
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { VoiceReplay, voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceActivity } from "../ui/shell/voice-activity";
|
||||
import { GrayImage } from "../graphics/image";
|
||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||
@@ -77,7 +82,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, windowEdges, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { sameEdges } from "../ui/viewport";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type MicCount, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, getHeadUpPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
@@ -99,6 +104,8 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
|
||||
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
|
||||
import { wearerVerificationOptions } from "../apps/microphones/speakers";
|
||||
import { micSession } from "../apps/microphones/mic-session";
|
||||
import { micArrayCaptureConfig, micControlSupported } from "../apps/microphones/mic-control";
|
||||
import { encodeMicControl } from "../apps/microphones/mic-protocol";
|
||||
import {
|
||||
chargingPreviewMessage,
|
||||
glassesDisplayAsleep,
|
||||
@@ -335,6 +342,7 @@ class DashboardController {
|
||||
private evenHubResumePromise: Promise<boolean> | null = null;
|
||||
/** Faceclaw's firmware at the required revision reported in; its extensions can be used. */
|
||||
private customFirmwareConfirmed = false;
|
||||
private lastGlanceAngleDegrees = glanceAngleTriggerDegrees();
|
||||
private faceclawWakeLeaseState: boolean | null = null;
|
||||
/** Ring connection mode the current session was started with (it applies per connection). */
|
||||
private ringModeAtConnect: RingConnectionMode = "glasses";
|
||||
@@ -412,6 +420,7 @@ class DashboardController {
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
|
||||
this.startContinuousVoiceCapture(client, options),
|
||||
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
|
||||
openVoiceReplay: (path: string) => this.openVoiceReplay(path),
|
||||
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
|
||||
};
|
||||
this.sharedActions = sharedActions;
|
||||
@@ -461,6 +470,7 @@ class DashboardController {
|
||||
if (on) this.glance.dismiss();
|
||||
this.handleScreenStateChanged(on);
|
||||
if (on) this.requestShellRender();
|
||||
if (on) keyboardService.retryNow();
|
||||
this.emit();
|
||||
},
|
||||
});
|
||||
@@ -494,6 +504,7 @@ class DashboardController {
|
||||
this.syncAssistantBridgeIfChanged();
|
||||
this.retargetContinuousCapture();
|
||||
this.syncLockScreenSettingIfChanged();
|
||||
this.applyGlanceAngleIfChanged();
|
||||
this.syncContinuousRecording();
|
||||
// A Preview color or scaling change should show on the mirror at once,
|
||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||
@@ -515,6 +526,29 @@ class DashboardController {
|
||||
assistantAvailable: () => shell.isAssistantAvailable(),
|
||||
assistant: text => shell.sendToAssistant(text),
|
||||
});
|
||||
// A keyboard paired through the glasses (CFW keyboard bridge). The
|
||||
// service decides what each key means (app/g2/keyboard/key-router.ts);
|
||||
// this is how it reaches the display and the input pipeline.
|
||||
keyboardService.setHost({
|
||||
ready: () => this.phase === "connected",
|
||||
locked: () => this.glassesLocked,
|
||||
wake: () => {
|
||||
void this.handleInputEvent({
|
||||
kind: "display-wake",
|
||||
containerName: "",
|
||||
eventType: OsEventTypeList.DOUBLE_CLICK_EVENT,
|
||||
eventSource: EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL,
|
||||
systemExitReasonCode: 0,
|
||||
frameId: 0,
|
||||
}).catch((error) => this.appendLog(`keyboard wake failed: ${this.formatError(error)}`));
|
||||
},
|
||||
gesture: (gesture, origin) => this.injectSyntheticRingInput(gesture, origin),
|
||||
log: (line) => this.appendLog(line),
|
||||
// A keyboard that is switched off would otherwise keep the glasses'
|
||||
// radio looking for it around the clock. One still connected when the
|
||||
// display sleeps stays connected, and its next key wakes the display.
|
||||
shouldReconnect: () => this.phase === "connected" && shell.isScreenOn(),
|
||||
});
|
||||
this.wearRemote = new WearRemote({
|
||||
apps: launchableApps,
|
||||
injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"),
|
||||
@@ -1676,6 +1710,8 @@ class DashboardController {
|
||||
if (mappedPhase === "connected") {
|
||||
this.syncEvenHubScreenOffSetting();
|
||||
this.ensureWearStateTracking();
|
||||
// The session reapplies it on reconnects; restate it for a new session.
|
||||
applyGlanceAngle();
|
||||
this.maybeWakeAfterChargingReconnect();
|
||||
} else if (mappedPhase !== "charging") {
|
||||
this.cancelEvenHubSuspendTimer();
|
||||
@@ -1784,6 +1820,7 @@ class DashboardController {
|
||||
this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`);
|
||||
});
|
||||
this.ensureWearStateTracking();
|
||||
this.syncKeyboardLink();
|
||||
}
|
||||
// The shared session halts on incompatible firmware by itself
|
||||
// ("incompatible-firmware" phase); this only explains why.
|
||||
@@ -1837,6 +1874,7 @@ class DashboardController {
|
||||
this.requestShellRender();
|
||||
this.updateCompositePreview();
|
||||
this.updateConnectedForegroundNotification();
|
||||
syncGlanceAngleOriginal();
|
||||
}, SHELL_REFRESH_INTERVAL_MS);
|
||||
this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS);
|
||||
this.screenTimeoutTimer = setInterval(() => {
|
||||
@@ -2274,6 +2312,30 @@ class DashboardController {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* How many of the glasses' microphones the continuous-capture windows call
|
||||
* for (the most any asks for), where the glasses can stream them
|
||||
* separately: G2 with the microphone channel in use. Otherwise 1, the
|
||||
* mixed stream.
|
||||
*/
|
||||
private continuousMics(): MicCount {
|
||||
if (glassesTypeSetting.get() !== "g2" || !micControlSupported()) return 1;
|
||||
let mics: MicCount = 1;
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if ((options.mics ?? 1) > mics) mics = options.mics!;
|
||||
}
|
||||
return mics;
|
||||
}
|
||||
|
||||
/**
|
||||
* A recording as a capture of its own (see VoiceReplay), transcribed with
|
||||
* the provider and processing settings a microphone capture would get.
|
||||
* "My voice only" is left out: a recording is of whoever spoke.
|
||||
*/
|
||||
private openVoiceReplay(path: string): VoiceReplay {
|
||||
return new VoiceReplay(path, () => ({ ...this.voiceCaptureOptions(null), speakerVerification: undefined }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Continuous capture's claim on the mic (the Microphones app and EvenHub
|
||||
* mic apps stand down while it is held) follows whether the capture really
|
||||
@@ -2299,9 +2361,26 @@ class DashboardController {
|
||||
// Otherwise use the shared transcription-language preference.
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
// Separate microphones only come from connected glasses.
|
||||
const mics = communicator ? this.continuousMics() : 1;
|
||||
const micConfig = mics === 1 ? null : micArrayCaptureConfig(mics);
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
micArray: micConfig ? { configure: encodeMicControl("configure", micConfig) } : null,
|
||||
// Stored in a recording of the capture (see Save Recording).
|
||||
captureInfo: {
|
||||
glasses: communicator ? glassesModelName() : null,
|
||||
mics,
|
||||
micConfig: micConfig && {
|
||||
source: micConfig.source,
|
||||
channelMask: micConfig.channelMask,
|
||||
codec: micConfig.codec,
|
||||
format: micConfig.format,
|
||||
sampleRateHz: micConfig.sampleRateHz,
|
||||
directionDetection: micConfig.beamform,
|
||||
},
|
||||
},
|
||||
provider,
|
||||
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
@@ -2484,6 +2563,16 @@ class DashboardController {
|
||||
frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventLabel(event.kind, event.eventType)}`);
|
||||
let frameOwned = false;
|
||||
try {
|
||||
if (event.kind === "sys-event" && event.eventType === OsEventTypeList.HEAD_DOWN_EVENT) {
|
||||
// Only the Glanceboard cares (it ends a head-tilt showing); never
|
||||
// shell input, so it can't count as activity or wake anything.
|
||||
const glanceEvent = this.glance.eventForGesture("head-down");
|
||||
if (glanceEvent) {
|
||||
await this.glance.handleEvent(glanceEvent, frameId);
|
||||
frameOwned = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const inputEvent = rawInputEventToInputEvent(event);
|
||||
if (!acceptInput(inputEvent)) return;
|
||||
// The gesture, plus which app is on screen and whether input goes to it,
|
||||
@@ -2602,15 +2691,49 @@ class DashboardController {
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-send the head-tilt angle when it changed (the setting, from either settings UI, or the Glanceboard toggle). */
|
||||
private applyGlanceAngleIfChanged(): void {
|
||||
const degrees = glanceAngleTriggerDegrees();
|
||||
if (degrees === this.lastGlanceAngleDegrees) return;
|
||||
this.lastGlanceAngleDegrees = degrees;
|
||||
applyGlanceAngle();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tell the keyboard bridge whether the session can host a keyboard: a G2
|
||||
* on firmware with the bridge, connected and in use. While the glasses
|
||||
* charge in their case the link stays up but nobody is typing, so the
|
||||
* firmware is told to let the keyboard go.
|
||||
*/
|
||||
private syncKeyboardLink(): void {
|
||||
const up =
|
||||
this.phase === "connected" &&
|
||||
this.customFirmwareConfirmed &&
|
||||
glassesTypeSetting.get() === "g2";
|
||||
// Charging, or a disconnect the user asked for: the link is still up, so
|
||||
// the DISABLE gets out ahead of everything the teardown sends.
|
||||
if (up) keyboardService.linkReady();
|
||||
else keyboardService.linkLost(this.phase === "charging" || this.phase === "disconnecting");
|
||||
}
|
||||
|
||||
/**
|
||||
* The Glanceboard event a sleep-time input means, or null when the shell
|
||||
* should see the input as usual. A display-wake carries the head-tilt
|
||||
* gesture (HEAD_UP_EVENT) as a press; the double-tap display-wake stays the
|
||||
* regular UI's wake.
|
||||
* gesture (HEAD_UP_EVENT) as a tilt, which lasts until the head-down; the
|
||||
* double-tap display-wake stays the regular UI's wake.
|
||||
*/
|
||||
private glanceEventFor(inputEvent: InputEvent, event: RawInputEvent): GlanceEvent | null {
|
||||
if (inputEvent.type === "display-wake") {
|
||||
return event.eventType === OsEventTypeList.HEAD_UP_EVENT ? this.glance.eventForGesture("head-tilt") : null;
|
||||
if (event.eventType !== OsEventTypeList.HEAD_UP_EVENT) return null;
|
||||
const glanceEvent = this.glance.eventForGesture("head-tilt");
|
||||
if (glanceEvent?.type !== "tilt") return glanceEvent;
|
||||
const maxMs = glanceTiltMaxMs({
|
||||
capMs: glanceEvent.maxMs === undefined ? GLANCE_TILT_MAX_MS : glanceEvent.maxMs,
|
||||
resumedSession: this.evenHubSessionSuspended || this.evenHubResumePromise !== null,
|
||||
triggerDegrees: glassesHeadUpTrigger() ?? (glanceAngleTriggerDegrees() || null),
|
||||
tapTimeoutMs: glanceTapTimeoutMs(),
|
||||
});
|
||||
return { type: "tilt", maxMs };
|
||||
}
|
||||
return this.glance.eventForGesture(inputEvent.type);
|
||||
}
|
||||
@@ -2709,6 +2832,7 @@ class DashboardController {
|
||||
}
|
||||
},
|
||||
endTextSettingEdit: () => this.endTextSettingEdit(),
|
||||
unhandledKey: (event) => keyboardService.handleUnhandledKey(event as KeyEvent),
|
||||
});
|
||||
this.appHosts.set(appId, host);
|
||||
return host;
|
||||
@@ -2742,10 +2866,11 @@ class DashboardController {
|
||||
if (!this.openAppsRestored || this.suppressOpenAppsPersist) return;
|
||||
const open: string[] = [];
|
||||
for (const window of shell.getWindows()) {
|
||||
if (window.appId === "launcher") continue;
|
||||
if (window.appId === "launcher" || window.restoresApp === false) continue;
|
||||
if (!open.includes(window.appId)) open.push(window.appId);
|
||||
}
|
||||
savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null });
|
||||
const foreground = shell.foregroundWindow();
|
||||
savePersistedOpenApps({ open, foreground: foreground && foreground.restoresApp !== false ? foreground.appId : null });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -3077,6 +3202,7 @@ class DashboardController {
|
||||
private setPhase(phase: ConnectionPhase): void {
|
||||
if (this.phase === phase) return;
|
||||
this.phase = phase;
|
||||
this.syncKeyboardLink();
|
||||
if (phase !== "connected" && phase !== "charging") {
|
||||
shell.setBatteryLevels({ ring: null, ringCharging: null });
|
||||
}
|
||||
|
||||
@@ -68,6 +68,23 @@ export function extractG2Mac(mfg: Uint8Array): string | null {
|
||||
return humanReadableAddressFromWireBytes(mfg.subarray(16, 22));
|
||||
}
|
||||
|
||||
/**
|
||||
* G2 manufacturer data (hex) carrying `serial` and `address`, for an arm known
|
||||
* only from a saved pairing: what `extractG2Serial` and `extractG2Mac` read
|
||||
* back, with the serial zero-padded and no flag byte. Null when the serial is
|
||||
* not up to 14 printable ASCII characters or the address is not a MAC.
|
||||
*/
|
||||
export function g2ManufacturerData(serial: string, address: string): string | null {
|
||||
const wire = wireBytesFromHumanReadableAddress(address);
|
||||
if (!wire || !/^[\x20-\x7e]{1,14}$/.test(serial)) return null;
|
||||
const bytes = new Uint8Array(22);
|
||||
bytes[0] = 0x45;
|
||||
bytes[1] = 0x52;
|
||||
for (let i = 0; i < serial.length; i++) bytes[2 + i] = serial.charCodeAt(i);
|
||||
bytes.set(wire, 16);
|
||||
return bytesToHex(bytes);
|
||||
}
|
||||
|
||||
/** Preserve the final G2 manufacturer-data byte without guessing its meaning. */
|
||||
export function extractG2AdvertisementFlag(mfg: Uint8Array): number | null {
|
||||
if (mfg.length < 23 || !hasEvenManufacturerSignature(mfg)) return null;
|
||||
|
||||
@@ -79,6 +79,12 @@ export const OsEventTypeList = {
|
||||
HEAD_UP_EVENT: 12,
|
||||
/** Faceclaw/18: R1 touch-down, before click/hold/slide recognition. */
|
||||
RING_PRESS_EVENT: 14,
|
||||
/**
|
||||
* Faceclaw/38: the IMU head-down (the head back below the head-up
|
||||
* threshold minus 20 degrees), forwarded while a page is up. Ends a
|
||||
* head-tilt Glanceboard; nothing else uses it.
|
||||
*/
|
||||
HEAD_DOWN_EVENT: 15,
|
||||
} as const;
|
||||
|
||||
export const OsEventTypeName: Record<number, string> = {
|
||||
@@ -96,4 +102,5 @@ export const OsEventTypeName: Record<number, string> = {
|
||||
11: "SHORT_THEN_LONG_PRESS_EVENT",
|
||||
12: "HEAD_UP_EVENT",
|
||||
14: "RING_PRESS_EVENT",
|
||||
15: "HEAD_DOWN_EVENT",
|
||||
};
|
||||
@@ -26,7 +26,7 @@ export type FirmwareInfo = {
|
||||
};
|
||||
|
||||
/** The Faceclaw firmware revision this build of the app needs. */
|
||||
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 38;
|
||||
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 40;
|
||||
|
||||
const FACECLAW_PREFIX = "Faceclaw/";
|
||||
const LEGACY_PREFIX = "EVENCFW";
|
||||
|
||||
+101
-35
@@ -19,12 +19,42 @@ export type FirmwarePatchSet = {
|
||||
export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"base": "g2_2.3.0.24.bin",
|
||||
"baseSha256": "187ccf2bcc5c17a212106e8a376745511e8289c4232b634a7ea94b9bf25a0979",
|
||||
"outputSha256": "4aac023f49ec6965186d9106e328a516f20a2915ab400d561f662f984023a432",
|
||||
"outputSha256": "2a31417f04cec16ecd4e4c3e575d89ce77fb3c5386b47cdb49b9753aaefb1c32",
|
||||
"patches": [
|
||||
{
|
||||
"offset": 1379947,
|
||||
"old": "0ef009fa",
|
||||
"new": "08f3a6fc",
|
||||
"desc": "DM callback allocation: retain keyboard lifecycle on queue exhaustion"
|
||||
},
|
||||
{
|
||||
"offset": 1380027,
|
||||
"old": "0ef0e1f9",
|
||||
"new": "08f304fc",
|
||||
"desc": "ATT callback allocation: expose lost keyboard events as relay gaps"
|
||||
},
|
||||
{
|
||||
"offset": 2649417,
|
||||
"old": "0a681279",
|
||||
"new": "d2f155fc",
|
||||
"desc": "SMP Pairing I/O capability: KeyboardDisplay only for owned keyboard"
|
||||
},
|
||||
{
|
||||
"offset": 2651811,
|
||||
"old": "0a681279",
|
||||
"new": "d1f1a8ff",
|
||||
"desc": "SMP Pairing I/O capability: KeyboardDisplay only for owned keyboard"
|
||||
},
|
||||
{
|
||||
"offset": 1385975,
|
||||
"old": "38b584b0",
|
||||
"new": "06f3c2bf",
|
||||
"desc": "BLE app dispatcher: isolate explicitly enabled keyboard connection"
|
||||
},
|
||||
{
|
||||
"offset": 1045797,
|
||||
"old": "38b584b0",
|
||||
"new": "5cf371bb",
|
||||
"new": "5bf309ba",
|
||||
"desc": "ring receiver entry: timestamped unfiltered reports under framebuffer lease"
|
||||
},
|
||||
{
|
||||
@@ -36,49 +66,49 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 1437923,
|
||||
"old": "fef788ff",
|
||||
"new": "fdf2a2f8",
|
||||
"new": "fbf23aff",
|
||||
"desc": "cfw_ancs_event"
|
||||
},
|
||||
{
|
||||
"offset": 1437951,
|
||||
"old": "fef7e8fb",
|
||||
"new": "fcf2d0fd",
|
||||
"new": "fbf268fc",
|
||||
"desc": "cfw_ancs_open"
|
||||
},
|
||||
{
|
||||
"offset": 1437969,
|
||||
"old": "fef7e7fb",
|
||||
"new": "fcf2ebfd",
|
||||
"new": "fbf283fc",
|
||||
"desc": "cfw_ancs_close"
|
||||
},
|
||||
{
|
||||
"offset": 1388387,
|
||||
"old": "39694d00",
|
||||
"new": "25597d00",
|
||||
"new": "55467d00",
|
||||
"desc": "connection-bound ANCS EUS write callback"
|
||||
},
|
||||
{
|
||||
"offset": 971497,
|
||||
"old": "fff717f8",
|
||||
"new": "6ef3a1fc",
|
||||
"new": "6df339fb",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 971845,
|
||||
"old": "fef769ff",
|
||||
"new": "6ef3f3fb",
|
||||
"new": "6df38bfa",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 979579,
|
||||
"old": "fdf74ef8",
|
||||
"new": "6cf3d8fc",
|
||||
"new": "6bf370fb",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 1428895,
|
||||
"old": "f8f76cfd",
|
||||
"new": "fbf250fe",
|
||||
"new": "f9f2d0fb",
|
||||
"desc": "bl cfw_receive_packet (private SID-f0 probe before TPL reassembly)"
|
||||
},
|
||||
{
|
||||
@@ -102,7 +132,7 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 1058287,
|
||||
"old": "fff754fb",
|
||||
"new": "59f342fb",
|
||||
"new": "58f3daf9",
|
||||
"desc": "bl ble_classify_link (only a <= 11.25 ms / latency-0 link is 'already fast')"
|
||||
},
|
||||
{
|
||||
@@ -111,148 +141,184 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"new": "5ff43332",
|
||||
"desc": "reserve final 1 KiB of primary TLSF arena for CFW context anchor"
|
||||
},
|
||||
{
|
||||
"offset": 1130241,
|
||||
"old": "5ff44d22",
|
||||
"new": "5ff40622",
|
||||
"desc": "LVGL arena size 0xcd000 -> 0x86000 (fixed CFW buffers follow it)"
|
||||
},
|
||||
{
|
||||
"offset": 2697223,
|
||||
"old": "9c212020",
|
||||
"new": "9c192320",
|
||||
"desc": "EvenHub arena base 0x2020219c -> 0x2023199c"
|
||||
},
|
||||
{
|
||||
"offset": 2697133,
|
||||
"old": "5ff4e121",
|
||||
"new": "5ff48221",
|
||||
"desc": "EvenHub arena clear length 0x70800 -> 0x41000"
|
||||
},
|
||||
{
|
||||
"offset": 2697149,
|
||||
"old": "5ff4e121",
|
||||
"new": "5ff48221",
|
||||
"desc": "EvenHub TLSF pool size 0x70800 -> 0x41000"
|
||||
},
|
||||
{
|
||||
"offset": 2245731,
|
||||
"old": "5ff4e125",
|
||||
"new": "5ff48225",
|
||||
"desc": "EvenHub heap total in the stock usage log 0x70800 -> 0x41000"
|
||||
},
|
||||
{
|
||||
"offset": 4394495,
|
||||
"old": "9b212020",
|
||||
"new": "9bb11b20",
|
||||
"desc": "MPU region 6 (non-cacheable) ends with the shrunk LVGL arena"
|
||||
},
|
||||
{
|
||||
"offset": 1248487,
|
||||
"old": "d4f712fb",
|
||||
"new": "29f344fd",
|
||||
"new": "27f3a0f9",
|
||||
"desc": "bl settings_send_wrapper (append caps field 100)"
|
||||
},
|
||||
{
|
||||
"offset": 1246183,
|
||||
"old": "f5f7e2f8",
|
||||
"new": "29f3e8fe",
|
||||
"new": "27f3befb",
|
||||
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
|
||||
},
|
||||
{
|
||||
"offset": 975219,
|
||||
"old": "02f083fc",
|
||||
"new": "6bf34aff",
|
||||
"new": "69f320fc",
|
||||
"desc": "bl faceclaw_display_start @ 0x467d68 (fail-open wake takeover)"
|
||||
},
|
||||
{
|
||||
"offset": 975411,
|
||||
"old": "02f023fc",
|
||||
"new": "6bf3b8ff",
|
||||
"new": "69f38efc",
|
||||
"desc": "bl faceclaw_display_start_headup @ 0x467e28 (fail-open wake takeover)"
|
||||
},
|
||||
{
|
||||
"offset": 975103,
|
||||
"old": "07f085fb",
|
||||
"new": "6df3a6fc",
|
||||
"new": "6cf33efb",
|
||||
"desc": "bl headup_gate (idle input gate: tap/long/release -> field-102 events, stock result kept)"
|
||||
},
|
||||
{
|
||||
"offset": 830549,
|
||||
"old": "19f03efb",
|
||||
"new": "90f37bfe",
|
||||
"new": "8ff313fd",
|
||||
"desc": "bl gesture_ring_press_mode (ring touch-down 14, head up 12 / down 15; stock dispatch preserved)"
|
||||
},
|
||||
{
|
||||
"offset": 830795,
|
||||
"old": "1ff071ff",
|
||||
"new": "90f334fe",
|
||||
"new": "8ff3ccfc",
|
||||
"desc": "bl gesture_press (owned event 9 before stock menu gate)"
|
||||
},
|
||||
{
|
||||
"offset": 830887,
|
||||
"old": "1ff043ff",
|
||||
"new": "90f32efe",
|
||||
"new": "8ff3c6fc",
|
||||
"desc": "bl gesture_short_long (Faceclaw lease gates event 11 vs stock Menu path)"
|
||||
},
|
||||
{
|
||||
"offset": 831763,
|
||||
"old": "1ff08dfd",
|
||||
"new": "90f364fc",
|
||||
"new": "8ff3fcfa",
|
||||
"desc": "bl gesture_release (owned event 10 before stock menu gate)"
|
||||
},
|
||||
{
|
||||
"offset": 1570209,
|
||||
"old": "7fb50600",
|
||||
"new": "dbf2d5b8",
|
||||
"new": "d8f231bd",
|
||||
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
|
||||
},
|
||||
{
|
||||
"offset": 1051545,
|
||||
"old": "f6f787f9",
|
||||
"new": "55f3c9f8",
|
||||
"new": "54f359fe",
|
||||
"desc": "bl display_copy_hook @ 0x47a78e (640x480 direct framebuffer)"
|
||||
},
|
||||
{
|
||||
"offset": 1051861,
|
||||
"old": "f6f7e9f8",
|
||||
"new": "55f32bf8",
|
||||
"new": "54f3bbfd",
|
||||
"desc": "bl display_copy_hook @ 0x47a8ca (640x480 direct framebuffer)"
|
||||
},
|
||||
{
|
||||
"offset": 1262237,
|
||||
"old": "d9f7d0fd",
|
||||
"new": "25f3bffd",
|
||||
"new": "23f3edfa",
|
||||
"desc": "bl faceclaw_send_wear_event @ 0x4ade92 (outside onboarding)"
|
||||
},
|
||||
{
|
||||
"offset": 1262337,
|
||||
"old": "d9f79efd",
|
||||
"new": "25f38dfd",
|
||||
"new": "23f3bbfa",
|
||||
"desc": "bl faceclaw_send_wear_event @ 0x4adef6 (outside onboarding)"
|
||||
},
|
||||
{
|
||||
"offset": 1309405,
|
||||
"old": "67f02dff",
|
||||
"new": "19f35bfe",
|
||||
"new": "17f397fb",
|
||||
"desc": "bl compass_decode_capture (sample-matched GAF diagnostics)"
|
||||
},
|
||||
{
|
||||
"offset": 1307865,
|
||||
"old": "fff7c9fb",
|
||||
"new": "1af3ddf9",
|
||||
"new": "17f319ff",
|
||||
"desc": "bl compass_report_event (stock UI + heading with diagnostics over BLE)"
|
||||
},
|
||||
{
|
||||
"offset": 1300937,
|
||||
"old": "00210170",
|
||||
"new": "1bf3bbfe",
|
||||
"new": "19f3f7fb",
|
||||
"desc": "bl compass_preserve_accuracy (retain magnetic calibration under Faceclaw framebuffer lease)"
|
||||
},
|
||||
{
|
||||
"offset": 4537963,
|
||||
"old": "",
|
||||
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f004fd002800f0cb814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f042fc002800f009814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8948242f6f76ac0f2440ab8f1000fbff35f8f4ad00af19a0240464ff0ff314ff0ff379047002840f090811198002861d010980778a7f1170002285bd811980c2840f07d81109c6078a178084300f07781e0780138022800f27281a279e379667a42ea0322237a207942ea034242ea0663a27ae67a6179192f42ea062200f0de81172f40f0f481002a40f05981fe2800f0b086812800f0a386002840f0b68651ea030040f04c8100f000bf04460020d047e8b104f5257151e8008fb8f1000f0cd1bff35f8f00bf41e80002002a00f0e58051e8008fb8f1000ff5d00021bff32f8f0029bff35f8f00f0d480804692e74ff0ff372ae147f6df40c0f24f00cdf83880804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0f8fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bce6012120e70020109f42f6f76addf83880c0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2924dc0b2926d01c2924d03ce00f9012a803f045ff4ff0ff3701249df85100002812a818bf4ff0ff3704f05ff8054644b99df8500028b94af21f30c0f247004c3080470f98002818bf056114e09c2901d08b2918d168b14af21f31c0f247010746884797f8681121b997f824133846109f49b14ff0ff370af5807140468847384647b0bde8f08f0f9012a8344003f007ff387800f07f061e2e05d11199384607f094fe0746b9e71d2e00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a6f115010129e1d90f2e2cdd142e40f31082152e00f05d83162e00f069831f2e40f02c841198012840f0fc8438781f2840f0f88445f64d55c0f24605a847012841f0828140f24c71c0f2010105f5e2777944e02029220023b8470746002818bf4ff0ff3766e7092e00f3fe80052e00f03e82072e40f0028400274ff0010947f67f764ff07f080d943ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf0Line truncated
|
||||
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff784ff47e740027012647f67f754ff07f0a1191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0ad101684ef66d62ccf2ff02914203d14068b0f57e7f4ed04ff47e7002f0f9f9002800f027816ff0070142180831b1f57c7f177257729772d772177357739773d773f2d14ef66d61ccf2ff01016044608673a0f8c85180f8c7a1a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001f494599d1bff35f8f4be8000121b15be8001f4945f8d08fe74ff47e7a00254ff0010947f67f777f2407e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0ad101684ef66d62ccf2ff02914203d14168b1f57e7f50d04ff47e7002f07ff9002800f0ad806ff0070142180831b1f57c7f157255729572d572157355739573d573f2d14ef66d61ccf2ff010160c0f804a080f80e90a0f8c87180f8c741a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001fb14297d1bff35f8f4be8000121b15be8001fb142f8d08de7d0f8984242f6f76ac0f2440abff35f8fdcb30af19a0220464ff0ff314ff0ff379047002840f02e81119800284ed010980778a7f11700022848d811980c2821d1109d6878a97808431cd0e8780138022818d8aa79eb796e7a42ea03222b7a287942ea034242ea0663aa7aee7a6979192f42ea062200f07f81172f40f09581002a00f04e834ff0ff37f8e004460020d047c8b104f5267151e8004f54b9bff35f8f41e80002002a00f0998051e8004f002cf6d00021bff32f8f0029bff35f8f00f089800446a5e74ff0ff37dbe047f6df40c0f24f000e94804700274ff47e79012647f67f744ff07f0a13978df8507006e048f2a351c0f24501bff32f8f8847dbf80050bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0ad101684ef66d62ccf2ff02914203d14168b1f57e7f57d04ff47e7002f095f8002850d06ff0070100bf42180831b1f57c7f177257729772d772177357739773d773f2d14ef66d61ccf2ff010160c0f804908673a0f8c84180f8c7a1a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001fa94298d1bff35f8f00bf4be8000159b15be8001fa942f8d08de70af5a97188471ae701216be70020109f42f6f76ac0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0518bf01258a291edc0b2920d01c291ed03ae00f9012a801f06dfe4ff0ff37012612a801f037ff054646b99df8500028b94af21f30c0f247004c3080470f98002818bf056117e09c2901d08b291cd180b14af21f31c0f247010646884796f86811bff35f8f29b996f824133046bff35f8f51b14ff0ff370e9c0af5807120468847384647b0bde8f08f0f9012a804ea050601f030fe387800f07f041e2c05d11199384606f01ffe0746bae71d2c00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66d63ccf2ff039a420bd14a68b2f57e7f04bf002281f8a12303e0a4f115010129e1d90f2c2cdd142c40f3ad81152c00f0c882162c00f0d2821f2c40f095831198012840f0648438781f2840f0608445f64d54c0f24604a047012841f05e8041f2de01c0f2010104f5e2777944e02029220023b8470746002818bf4ff0ff3767e7092c00f3b880052c00f0ee81072c40f06b830d9600264ff0010947f67f777f243ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf00f087bc182f4ff0ff3708bf00297ff463af002b7ff460af352800f37083002800f05f84012800f06b84312840f05d84082abff451af00f06cbc48f2a351c0f24501bff32f8f8847dbf80050bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66d62ccf2ff02914204d14168b1f57e7f00f077854ff47e7001f01dff002800f06f856ff0070142180831b1f57c7f167256729672d672167356739673d673f2d14ef66d61ccf2ff0101604ff47e71416080f80e90a0f8c87180f8c741a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001fa94295d1bff35f8f4be80001002900f028855be8001fa942f6d089e70a2c00f0bb810b2c40f0b382dbf800500320cff680702840b0f1005fbff35f8f40f0808728684ef66d61ccf2ff01884240f0788768680027b0f57e7f7ff48daed5f8900342f68d49c0f244090028a5f8a07318bfc8470021a5f8d013c5f8cc13c5f8701185f86c11bff35f8f85f86811686c85f86b11c5f87411c5f82013c5f88813a5f88c13a5f8481040b1c847686c09f166018847002804bf002068644bf63940c0f251008047dbf800400320cff680702040b0f1005fbff35f8f46d120684ef66Line truncated
|
||||
"desc": "append injected blobs to main-app payload"
|
||||
},
|
||||
{
|
||||
"offset": 779123,
|
||||
"old": "805a3900",
|
||||
"new": "68703a00",
|
||||
"new": "10783a00",
|
||||
"desc": "main-app subheader payload size (ps)"
|
||||
},
|
||||
{
|
||||
"offset": 152,
|
||||
"old": "005b3900",
|
||||
"new": "e8703a00",
|
||||
"new": "90783a00",
|
||||
"desc": "main-app TOC entry size (ps + 128)"
|
||||
},
|
||||
{
|
||||
"offset": 779243,
|
||||
"old": "805a3904",
|
||||
"new": "68703a04",
|
||||
"new": "10783a04",
|
||||
"desc": "main-app preamble length (low 24 bits)"
|
||||
},
|
||||
{
|
||||
"offset": 779247,
|
||||
"old": "effdbcea",
|
||||
"new": "0b0c48b0",
|
||||
"new": "adc5378e",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
|
||||
},
|
||||
{
|
||||
"offset": 156,
|
||||
"old": "c0c6322a",
|
||||
"new": "52937d21",
|
||||
"new": "0c2e1c3e",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
|
||||
},
|
||||
{
|
||||
"offset": 779127,
|
||||
"old": "c0c6322a",
|
||||
"new": "52937d21",
|
||||
"new": "0c2e1c3e",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -82,7 +82,7 @@ export class GlanceHost {
|
||||
* The glance event a sleep-time gesture means under the board's settings,
|
||||
* or null when the shell should see the input as usual: nothing while the
|
||||
* board is disabled, no press from a tap when "Show on tap" is disabled,
|
||||
* no hold when "Show on long press" is off, no press from a head-tilt when
|
||||
* no hold when "Show on long press" is off, no tilt from a head-tilt when
|
||||
* "Show on head tilt" is off (it then wakes the regular UI). A release
|
||||
* always passes so a hold in progress can end.
|
||||
*/
|
||||
@@ -92,7 +92,8 @@ export class GlanceHost {
|
||||
if (gesture === "click" && !provider.showOnTap()) return null;
|
||||
if ((gesture === "long-press" || gesture === "short-then-long-press") && !provider.showOnLongPress()) return null;
|
||||
if (gesture === "head-tilt" && !provider.showOnHeadTilt()) return null;
|
||||
return glanceEventForGesture(gesture, this.isVisible());
|
||||
const event = glanceEventForGesture(gesture, this.isVisible());
|
||||
return event?.type === "tilt" ? { type: "tilt", maxMs: provider.tiltTimeoutMs() } : event;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+82
-12
@@ -1,9 +1,10 @@
|
||||
/**
|
||||
* The Glanceboard's visibility rules, as a pure reducer so they can be tested
|
||||
* without the display plumbing. The board is an alternate sleep-time display:
|
||||
* while the shell is asleep a press (single tap, or a head-tilt wake) shows it
|
||||
* for a short while, a hold (long-press or short-then-long-press) shows it until release,
|
||||
* and a double-tap dismisses it in favour of the regular UI. GlanceHost feeds
|
||||
* while the shell is asleep a press (single tap) shows it for a short while, a
|
||||
* head-tilt wake shows it until the head comes back down, a hold (long-press or
|
||||
* short-then-long-press) shows it until release, and a double-tap dismisses it
|
||||
* in favour of the regular UI. GlanceHost feeds
|
||||
* these events in and applies the resulting visibility to the compositor.
|
||||
*
|
||||
* No NativeScript imports: this module is compiled and run under plain node
|
||||
@@ -13,17 +14,67 @@
|
||||
/** Fallback press timeout when the host supplies none; each further press restarts it. */
|
||||
export const GLANCE_TIMEOUT_MS = 3000;
|
||||
|
||||
/**
|
||||
* Default cap for a head-tilt showing: the board normally stays up until the
|
||||
* head comes back down (as the stock dashboard does), but a lost head-down
|
||||
* event should not leave it on indefinitely. The user's Head tilt timeout
|
||||
* setting replaces it (and may remove it).
|
||||
*/
|
||||
export const GLANCE_TILT_MAX_MS = 60_000;
|
||||
|
||||
/**
|
||||
* The firmware reports head-down once the pitch falls 20° below the head-up
|
||||
* trigger (a fixed hysteresis in DRV_IMUCheckHeadUpEvent). Under this trigger,
|
||||
* looking level again does not get below that, so the head-down may never come.
|
||||
*/
|
||||
export const GLANCE_LOW_TILT_TRIGGER_DEGREES = 25;
|
||||
|
||||
/**
|
||||
* How long a head-tilt showing may last without a head-down: capMs (the Head
|
||||
* tilt timeout; null = no limit), except when the head-down is unlikely to
|
||||
* arrive, where the tap duration is used if shorter: a tilt that woke a
|
||||
* suspended EvenHub session (the head-down is only forwarded once the page is
|
||||
* back, so a quick glance can end before that), or a trigger too low for a
|
||||
* level gaze to count as looking down. triggerDegrees is the glasses'
|
||||
* trigger, or null when not known.
|
||||
*/
|
||||
export function glanceTiltMaxMs(options: {
|
||||
capMs: number | null;
|
||||
resumedSession: boolean;
|
||||
triggerDegrees: number | null;
|
||||
tapTimeoutMs: number;
|
||||
}): number | null {
|
||||
const lowTrigger =
|
||||
options.triggerDegrees !== null && options.triggerDegrees > 0 && options.triggerDegrees < GLANCE_LOW_TILT_TRIGGER_DEGREES;
|
||||
if (!options.resumedSession && !lowTrigger) return options.capMs;
|
||||
return options.capMs === null ? options.tapTimeoutMs : Math.min(options.capMs, options.tapTimeoutMs);
|
||||
}
|
||||
|
||||
export type GlanceState = {
|
||||
visible: boolean;
|
||||
/** A long-press is holding the board up; the timer does not run meanwhile. */
|
||||
holding: boolean;
|
||||
/**
|
||||
* A head-tilt is holding the board up: it stays until the firmware reports
|
||||
* the head back down (or tiltMaxMs passes).
|
||||
*/
|
||||
tilted: boolean;
|
||||
/** The current head-tilt's cap (0 when not tilted, null for none); a hold ending mid-tilt re-arms it. */
|
||||
tiltMaxMs: number | null;
|
||||
/** When the board hides on its own (epoch ms), or null while held / hidden. */
|
||||
hideAtMs: number | null;
|
||||
};
|
||||
|
||||
export type GlanceEvent =
|
||||
/** Single tap or head-tilt wake: show, and (re)start the auto-hide timer. */
|
||||
/** Single tap: show, and (re)start the auto-hide timer. */
|
||||
| { type: "press" }
|
||||
/**
|
||||
* Head-tilt (IMU head-up): show until the head comes back down, or until
|
||||
* maxMs passes (default GLANCE_TILT_MAX_MS; null = no limit).
|
||||
*/
|
||||
| { type: "tilt"; maxMs?: number | null }
|
||||
/** The IMU reported the head back down: ends a head-tilt showing. */
|
||||
| { type: "head-down" }
|
||||
/** Long-press: show and keep showing until the release. */
|
||||
| { type: "hold" }
|
||||
/** The long-press ended. */
|
||||
@@ -33,7 +84,7 @@ export type GlanceEvent =
|
||||
/** Double-tap (or any other wake of the regular UI): hide now. */
|
||||
| { type: "dismiss" };
|
||||
|
||||
export const GLANCE_HIDDEN: GlanceState = Object.freeze({ visible: false, holding: false, hideAtMs: null });
|
||||
export const GLANCE_HIDDEN: GlanceState = Object.freeze({ visible: false, holding: false, tilted: false, tiltMaxMs: 0, hideAtMs: null });
|
||||
|
||||
export function reduceGlance(
|
||||
state: GlanceState,
|
||||
@@ -43,14 +94,26 @@ export function reduceGlance(
|
||||
): GlanceState {
|
||||
switch (event.type) {
|
||||
case "press":
|
||||
// A tap during a hold changes nothing: the hold decides when it ends.
|
||||
if (state.holding) return state;
|
||||
return { visible: true, holding: false, hideAtMs: nowMs + timeoutMs };
|
||||
// A tap during a hold or a head-tilt changes nothing: those decide when it ends.
|
||||
if (state.holding || state.tilted) return state;
|
||||
return { visible: true, holding: false, tilted: false, tiltMaxMs: 0, hideAtMs: nowMs + timeoutMs };
|
||||
case "tilt": {
|
||||
const maxMs = event.maxMs === undefined ? GLANCE_TILT_MAX_MS : event.maxMs;
|
||||
return { visible: true, holding: state.holding, tilted: true, tiltMaxMs: maxMs, hideAtMs: state.holding ? null : tiltHideAt(nowMs, maxMs) };
|
||||
}
|
||||
case "head-down":
|
||||
// Only a head-tilt showing ends with the head; a tap's timer or a hold keeps running.
|
||||
if (!state.tilted) return state;
|
||||
return state.holding ? { ...state, tilted: false, tiltMaxMs: 0 } : GLANCE_HIDDEN;
|
||||
case "hold":
|
||||
return { visible: true, holding: true, hideAtMs: null };
|
||||
return { visible: true, holding: true, tilted: state.tilted, tiltMaxMs: state.tiltMaxMs, hideAtMs: null };
|
||||
case "release":
|
||||
// Releases delivered late or after a disabled hold gesture mean nothing here.
|
||||
return state.holding ? GLANCE_HIDDEN : state;
|
||||
if (!state.holding) return state;
|
||||
// Head still up: the board stays, as if the tilt had just shown it.
|
||||
return state.tilted
|
||||
? { visible: true, holding: false, tilted: true, tiltMaxMs: state.tiltMaxMs, hideAtMs: tiltHideAt(nowMs, state.tiltMaxMs) }
|
||||
: GLANCE_HIDDEN;
|
||||
case "timeout":
|
||||
if (!state.visible || state.holding || state.hideAtMs === null || nowMs < state.hideAtMs) return state;
|
||||
return GLANCE_HIDDEN;
|
||||
@@ -59,6 +122,10 @@ export function reduceGlance(
|
||||
}
|
||||
}
|
||||
|
||||
function tiltHideAt(nowMs: number, maxMs: number | null): number | null {
|
||||
return maxMs === null ? null : nowMs + maxMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which glance event, if any, a shell input gesture means while the screen is
|
||||
* off. A double-tap only concerns the board while it is showing (it then
|
||||
@@ -66,13 +133,16 @@ export function reduceGlance(
|
||||
* already handles or ignores while asleep.
|
||||
*/
|
||||
export function glanceEventForGesture(
|
||||
gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | (string & {}),
|
||||
gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | "head-down" | (string & {}),
|
||||
boardVisible: boolean,
|
||||
): GlanceEvent | null {
|
||||
switch (gesture) {
|
||||
case "click":
|
||||
case "head-tilt":
|
||||
return { type: "press" };
|
||||
case "head-tilt":
|
||||
return { type: "tilt" };
|
||||
case "head-down":
|
||||
return boardVisible ? { type: "head-down" } : null;
|
||||
case "long-press":
|
||||
case "short-then-long-press":
|
||||
return { type: "hold" };
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { addImuListener, imuPitchDegrees, setImuReportEnabled, type ImuReading } from "../native/imu";
|
||||
import { getHeadUpStatus, setHeadUpTrigger, type HeadUpStatus } from "../native/glasses-controls";
|
||||
import { getStringSetting, setStringSetting } from "../native/settings-store";
|
||||
import { glanceAngleTriggerDegrees } from "../apps/glanceboard/glanceboard-settings";
|
||||
|
||||
// Same ImuReportPace code the accelerometer demo uses (not literal Hz).
|
||||
const IMU_REPORT_PACE = 200;
|
||||
/** How often the live reading checks visibility and repaints a changed value. */
|
||||
const LIVE_TICK_MS = 250;
|
||||
/** Weight of each new sample in the smoothed pitch. */
|
||||
const LIVE_SMOOTHING = 0.3;
|
||||
/** A reading older than this (or none this long after the stream started) is shown as missing. */
|
||||
const LIVE_STALE_MS = 3_000;
|
||||
|
||||
/**
|
||||
* The head's pitch, live, for the Head tilt angle picker. The IMU report
|
||||
* stream runs while `visible()` holds (checked every tick, so it stops while
|
||||
* the screen is off or the window is in the background) and until stop().
|
||||
* text() is the footer line; a changed value repaints at most once a tick.
|
||||
*/
|
||||
class LiveHeadPitch {
|
||||
private pitch: number | null = null;
|
||||
private lastSampleAt = 0;
|
||||
private streaming = false;
|
||||
private streamingSince = 0;
|
||||
private shown = "";
|
||||
private readonly unsubscribe: () => void;
|
||||
private readonly timer: ReturnType<typeof setInterval>;
|
||||
|
||||
constructor(
|
||||
private readonly visible: () => boolean,
|
||||
private readonly repaint: () => void,
|
||||
) {
|
||||
this.unsubscribe = addImuListener((reading) => this.onReading(reading));
|
||||
this.timer = setInterval(() => this.tick(), LIVE_TICK_MS);
|
||||
this.tick();
|
||||
}
|
||||
|
||||
text(): string {
|
||||
return this.shown;
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
clearInterval(this.timer);
|
||||
this.unsubscribe();
|
||||
if (this.streaming) setImuReportEnabled(false, IMU_REPORT_PACE);
|
||||
this.streaming = false;
|
||||
}
|
||||
|
||||
private onReading(reading: ImuReading): void {
|
||||
// Glasses temples only (1 right, 3 left, 0 unspecified); the ring is 2.
|
||||
if (!this.streaming || reading.source === 2) return;
|
||||
if (!(Math.hypot(reading.x, reading.y, reading.z) > 1e-6)) return;
|
||||
const pitch = imuPitchDegrees(reading);
|
||||
this.pitch = this.pitch === null ? pitch : this.pitch + LIVE_SMOOTHING * (pitch - this.pitch);
|
||||
this.lastSampleAt = Date.now();
|
||||
}
|
||||
|
||||
private tick(): void {
|
||||
const now = Date.now();
|
||||
const visible = this.visible();
|
||||
if (visible !== this.streaming) {
|
||||
setImuReportEnabled(visible, IMU_REPORT_PACE);
|
||||
this.streaming = visible;
|
||||
this.streamingSince = now;
|
||||
this.pitch = null;
|
||||
}
|
||||
const text = this.format(now);
|
||||
if (text !== this.shown) {
|
||||
this.shown = text;
|
||||
this.repaint();
|
||||
}
|
||||
}
|
||||
|
||||
private format(now: number): string {
|
||||
if (this.pitch !== null && now - this.lastSampleAt < LIVE_STALE_MS) return `Head now: ${Math.round(this.pitch)}°`;
|
||||
return now - Math.max(this.streamingSince, this.lastSampleAt) < LIVE_STALE_MS ? "Head now: …" : "Head now: no data";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Head tilt angle picker's footer: the head's current pitch, read while
|
||||
* the picker is open and `visible()` holds (enumSettingMenuItem pickerFooter).
|
||||
*/
|
||||
export function liveHeadPitchFooter(visible: () => boolean): { text: () => string; open: (repaint: () => void) => () => void } {
|
||||
let reader: LiveHeadPitch | null = null;
|
||||
return {
|
||||
text: () => reader?.text() ?? "",
|
||||
open: (repaint) => {
|
||||
const opened = new LiveHeadPitch(visible, repaint);
|
||||
reader = opened;
|
||||
return () => {
|
||||
opened.stop();
|
||||
if (reader === opened) reader = null;
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// The glasses' own head-up switch and angle from before Faceclaw first changed
|
||||
// them, as "switch,angle", or "" when none is held. Kept here (not only in the
|
||||
// session) so "Glasses default" can still restore them after an app restart.
|
||||
const ORIGINAL_KEY = "glanceboard.headTiltOriginal";
|
||||
|
||||
function savedOriginal(): { switch: number; angle: number } | null {
|
||||
const [sw, angle] = getStringSetting(ORIGINAL_KEY, "").split(",").map(Number);
|
||||
return (sw === 0 || sw === 1) && Number.isInteger(angle) ? { switch: sw, angle } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Push the wanted trigger to the session (on change and on each connect):
|
||||
* the Head tilt angle while the Glanceboard is enabled, else none, which
|
||||
* writes back the glasses' own values if Faceclaw changed them.
|
||||
*/
|
||||
export function applyGlanceAngle(): boolean {
|
||||
const sent = setHeadUpTrigger(glanceAngleTriggerDegrees(), savedOriginal());
|
||||
syncGlanceAngleOriginal();
|
||||
return sent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the persisted original in step with the session: save the values the
|
||||
* session captured before its first write, and drop them once a read shows
|
||||
* them restored. Called on apply, from the controller's periodic refresh, and
|
||||
* when the settings row draws.
|
||||
*/
|
||||
export function syncGlanceAngleOriginal(status: HeadUpStatus | null = getHeadUpStatus()): void {
|
||||
if (!status) return;
|
||||
const saved = savedOriginal();
|
||||
if (!saved) {
|
||||
if (status.originalSwitch === 0 || status.originalSwitch === 1) {
|
||||
setStringSetting(ORIGINAL_KEY, `${status.originalSwitch},${status.originalAngle}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (
|
||||
glanceAngleTriggerDegrees() === 0 &&
|
||||
status.desired < 0 &&
|
||||
!status.restorePending &&
|
||||
status.switch === saved.switch &&
|
||||
status.angle === saved.angle
|
||||
) {
|
||||
setStringSetting(ORIGINAL_KEY, "");
|
||||
}
|
||||
}
|
||||
|
||||
/** The glasses' current head-up trigger (angle + calibration), 0 when off, or null when not read yet. */
|
||||
export function glassesHeadUpTrigger(status: HeadUpStatus | null = getHeadUpStatus()): number | null {
|
||||
if (!status || status.switch < 0) return null;
|
||||
return status.switch === 1 ? status.angle + status.calibration : 0;
|
||||
}
|
||||
@@ -29,6 +29,8 @@ import { OsEventTypeList } from './events'
|
||||
import { loadDeviceAddresses } from './device-addresses'
|
||||
import { updateGlassesPresence } from './glasses-presence'
|
||||
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
|
||||
import { keyboardService } from './keyboard/service'
|
||||
import type { KeyEvent } from './keyboard/keys'
|
||||
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy'
|
||||
import { WearRemote, type WearRemoteInputKind } from './wear-remote'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
@@ -238,6 +240,7 @@ export class IosPreviewController {
|
||||
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
|
||||
this.requestShellRender()
|
||||
this.wearRemote?.schedulePublish()
|
||||
if (on) keyboardService.retryNow()
|
||||
},
|
||||
})
|
||||
const launcher = createLauncherWindow({
|
||||
@@ -276,6 +279,28 @@ export class IosPreviewController {
|
||||
assistantAvailable: () => shell.isAssistantAvailable(),
|
||||
assistant: text => shell.sendToAssistant(text),
|
||||
})
|
||||
// A keyboard paired through the glasses (the CFW keyboard bridge). The service decides what
|
||||
// each key means (app/g2/keyboard/key-router.ts); its gestures share the controller queue
|
||||
// with physical input, like the watch's.
|
||||
const queueKeyboardInput = (run: () => Promise<void>): Promise<void> => {
|
||||
const task = this.inputQueue.then(() => { this.onActivity(); return run() })
|
||||
this.inputQueue = task.catch(error => this.fail(error))
|
||||
return task
|
||||
}
|
||||
keyboardService.setHost({
|
||||
ready: () => this.state.phase === 'connected',
|
||||
locked: () => this.glassesLocked,
|
||||
wake: () => { void queueKeyboardInput(() => this.receiveInput(makeInputEvent({ type: 'display-wake' }))).catch(() => {}) },
|
||||
gesture: (gesture, origin) => queueKeyboardInput(async () => {
|
||||
await this.receiveInput(makeInputEvent({ type: gesture, source: origin } as InputEventPayload))
|
||||
// A discrete hold includes its release.
|
||||
if (gesture === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: origin }))
|
||||
}),
|
||||
log: line => this.logBluetooth(line),
|
||||
// Only while the display is on: a keyboard that is switched off would otherwise keep the
|
||||
// glasses' radio looking for it around the clock.
|
||||
shouldReconnect: () => this.state.phase === 'connected' && shell.isScreenOn(),
|
||||
})
|
||||
// The header's EvenHub phone-UI button hides while that UI is showing.
|
||||
onPhoneShownChanged(() => this.syncEvenHubPhoneUi())
|
||||
registerSystemTools()
|
||||
@@ -816,6 +841,7 @@ export class IosPreviewController {
|
||||
},
|
||||
setSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible),
|
||||
removeSurface: id => this.removeWindowSurface(id),
|
||||
unhandledKey: event => keyboardService.handleUnhandledKey(event as KeyEvent),
|
||||
submitPixels: (id, pixels, width, height, draws) => {
|
||||
void this.submitWorkerPixels(id, pixels, width, height, draws).catch(error => this.fail(error))
|
||||
},
|
||||
@@ -837,6 +863,18 @@ export class IosPreviewController {
|
||||
this.state = { ...this.state, phase, status }
|
||||
this.logBluetooth(status)
|
||||
this.emitState()
|
||||
this.syncKeyboardLink()
|
||||
}
|
||||
/**
|
||||
* Tell the keyboard bridge whether the session can host a keyboard: a G2 on firmware with
|
||||
* the bridge, connected. On a disconnect the user asked for, the link is still up when the
|
||||
* phase turns, so the firmware is told to let the keyboard go first.
|
||||
*/
|
||||
private syncKeyboardLink(): void {
|
||||
const up = !!this.communicator && this.state.phase === 'connected' && this.firmwareCompatible
|
||||
&& glassesTypeSetting.get() === 'g2'
|
||||
if (up) keyboardService.linkReady()
|
||||
else keyboardService.linkLost(this.state.phase === 'disconnecting')
|
||||
}
|
||||
/** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */
|
||||
private syncPresence(): void {
|
||||
@@ -956,6 +994,7 @@ export class IosPreviewController {
|
||||
const firmwareWarning = firmwareIncompatibilityMessage(info) ?? ''
|
||||
if (firmwareWarning && firmwareWarning !== this.state.firmwareWarning) this.logBluetooth(`Firmware compatibility warning: ${firmwareWarning}`)
|
||||
this.firmwareCompatible = !firmwareWarning && hasCompatibleFirmware(info)
|
||||
this.syncKeyboardLink()
|
||||
this.state = { ...this.state, leftVersion: info.leftVersion || this.state.leftVersion,
|
||||
rightVersion: info.rightVersion || this.state.rightVersion, capabilities: info.extension || this.state.capabilities,
|
||||
firmwareWarning }
|
||||
@@ -1020,6 +1059,13 @@ export class IosPreviewController {
|
||||
private handleRingEvent(input: RawInputEvent): void {
|
||||
this.inputQueue = this.inputQueue.then(async () => {
|
||||
if (this.state.phase !== 'connected') return
|
||||
if (input.kind === 'sys-event' && input.eventType === OsEventTypeList.HEAD_DOWN_EVENT) {
|
||||
// As on Android: only the Glanceboard uses it (it ends a head-tilt
|
||||
// showing); it never reaches the shell or counts as activity.
|
||||
const glanceEvent = this.glance.eventForGesture('head-down')
|
||||
if (glanceEvent) await this.glance.handleEvent(glanceEvent, 0)
|
||||
return
|
||||
}
|
||||
this.onActivity()
|
||||
if (input.ringInput) this.state = { ...this.state, ring: true }
|
||||
await this.receiveInput(rawInputEventToInputEvent(input),
|
||||
@@ -1118,8 +1164,8 @@ export class IosPreviewController {
|
||||
/**
|
||||
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
||||
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
||||
* Recording is on.
|
||||
* on Android. `recordAs` names a recording (.mka) of the audio in Downloads
|
||||
* when Save Recording is on.
|
||||
*/
|
||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||
this.continuousClients.set(client, options)
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
/**
|
||||
* What keys from a hardware keyboard do, apart from typing: the built-in
|
||||
* defaults, the user's custom keybindings, and how the two combine.
|
||||
*
|
||||
* A key becomes one of the gestures the ring and the watch already have, so
|
||||
* nothing downstream knows about keyboards. Three layers, first match wins:
|
||||
*
|
||||
* 1. Custom keybindings (Settings > Keyboard > Keybindings). These apply
|
||||
* everywhere, including where the focused window takes keys for itself
|
||||
* (a terminal session), since that is the point of binding a remote's
|
||||
* few buttons.
|
||||
* 2. Built-in bindings that also apply there: Escape with Shift, Ctrl or
|
||||
* Alt goes back, which is the way out of a terminal session; and a
|
||||
* mouse's buttons and wheel, which are never typed.
|
||||
* 3. Built-in navigation, only for keys no window took: the arrows are the
|
||||
* watch's four swipes, Enter selects, Escape goes back.
|
||||
*
|
||||
* A keyboard's touchpad or mouse arrives here as keys too: "MouseButton1"
|
||||
* (left), "MouseButton2" (right), "MouseButton3" (middle) and so on, and
|
||||
* "WheelUp" / "WheelDown" / "WheelLeft" / "WheelRight" (see pointer.ts).
|
||||
*
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import type { KeyEvent } from "./keys";
|
||||
|
||||
/** A gesture a key can stand for, named as the input pipeline names it. */
|
||||
export type KeyGesture =
|
||||
| "swipe-up"
|
||||
| "swipe-down"
|
||||
| "swipe-left"
|
||||
| "swipe-right"
|
||||
| "scroll-up"
|
||||
| "scroll-down"
|
||||
| "click"
|
||||
| "double-click"
|
||||
| "long-press"
|
||||
| "short-then-long-press";
|
||||
|
||||
/**
|
||||
* What a custom keybinding does. "none" turns off the key's built-in
|
||||
* navigation meaning; it does not stop the key being typed into a window
|
||||
* that takes keys.
|
||||
*/
|
||||
export type KeyAction = KeyGesture | "none";
|
||||
|
||||
/** The actions, in the order the picker lists them. */
|
||||
export const KEY_ACTIONS: readonly { action: KeyAction; label: string }[] = [
|
||||
{ action: "click", label: "Select" },
|
||||
{ action: "double-click", label: "Back" },
|
||||
{ action: "swipe-up", label: "Swipe up" },
|
||||
{ action: "swipe-down", label: "Swipe down" },
|
||||
{ action: "swipe-left", label: "Swipe left" },
|
||||
{ action: "swipe-right", label: "Swipe right" },
|
||||
{ action: "scroll-up", label: "Scroll up" },
|
||||
{ action: "scroll-down", label: "Scroll down" },
|
||||
{ action: "long-press", label: "System menu" },
|
||||
{ action: "short-then-long-press", label: "App menu" },
|
||||
{ action: "none", label: "Nothing" },
|
||||
];
|
||||
|
||||
export function keyActionLabel(action: KeyAction): string {
|
||||
return KEY_ACTIONS.find((entry) => entry.action === action)?.label ?? action;
|
||||
}
|
||||
|
||||
function isKeyAction(value: unknown): value is KeyAction {
|
||||
return KEY_ACTIONS.some((entry) => entry.action === value);
|
||||
}
|
||||
|
||||
/** A physical key with the modifiers held: what a keybinding is keyed on. */
|
||||
export type KeyChord = { code: string; shift: boolean; ctrl: boolean; alt: boolean; meta: boolean };
|
||||
export type KeyBinding = { chord: KeyChord; action: KeyAction };
|
||||
|
||||
const MODIFIER_CODES = new Set([
|
||||
"ControlLeft", "ControlRight", "ShiftLeft", "ShiftRight", "AltLeft", "AltRight", "MetaLeft", "MetaRight",
|
||||
]);
|
||||
|
||||
/** Shift, Ctrl, Alt and Meta themselves: they qualify a chord rather than being one. */
|
||||
export function isModifierCode(code: string): boolean {
|
||||
return MODIFIER_CODES.has(code);
|
||||
}
|
||||
|
||||
export function chordOf(event: KeyEvent): KeyChord {
|
||||
const { shift, ctrl, alt, meta } = event.modifiers;
|
||||
return { code: event.code, shift, ctrl, alt, meta };
|
||||
}
|
||||
|
||||
/** Canonical text of a chord ("Ctrl+Shift+Escape"): the stored form, and the lookup key. */
|
||||
export function chordId(chord: KeyChord): string {
|
||||
return [chord.ctrl && "Ctrl", chord.alt && "Alt", chord.shift && "Shift", chord.meta && "Meta", chord.code]
|
||||
.filter(Boolean).join("+");
|
||||
}
|
||||
|
||||
/** The inverse of chordId, or null for anything it could not have produced. */
|
||||
export function parseChordId(text: string): KeyChord | null {
|
||||
const parts = text.split("+");
|
||||
const code = parts.pop() ?? "";
|
||||
if (!/^[A-Za-z][A-Za-z0-9]{0,39}$/.test(code) || isModifierCode(code)) return null;
|
||||
const chord: KeyChord = { code, shift: false, ctrl: false, alt: false, meta: false };
|
||||
for (const part of parts) {
|
||||
if (part === "Ctrl" && !chord.ctrl) chord.ctrl = true;
|
||||
else if (part === "Alt" && !chord.alt) chord.alt = true;
|
||||
else if (part === "Shift" && !chord.shift) chord.shift = true;
|
||||
else if (part === "Meta" && !chord.meta) chord.meta = true;
|
||||
else return null;
|
||||
}
|
||||
return chord;
|
||||
}
|
||||
|
||||
const CODE_LABELS: Record<string, string> = {
|
||||
ArrowUp: "Up", ArrowDown: "Down", ArrowLeft: "Left", ArrowRight: "Right",
|
||||
Escape: "Esc", PageUp: "Page Up", PageDown: "Page Down", CapsLock: "Caps Lock",
|
||||
PrintScreen: "Print Screen", ScrollLock: "Scroll Lock", NumLock: "Num Lock", ContextMenu: "Menu",
|
||||
Minus: "-", Equal: "=", BracketLeft: "[", BracketRight: "]", Backslash: "\\", IntlBackslash: "\\",
|
||||
IntlHash: "#", Semicolon: ";", Quote: "'", Backquote: "`", Comma: ",", Period: ".", Slash: "/",
|
||||
AudioVolumeUp: "Volume Up", AudioVolumeDown: "Volume Down", AudioVolumeMute: "Mute",
|
||||
MediaPlayPause: "Play/Pause", MediaPlay: "Play", MediaPause: "Pause", MediaStop: "Stop",
|
||||
MediaTrackNext: "Next Track", MediaTrackPrevious: "Previous Track",
|
||||
BrightnessUp: "Brightness Up", BrightnessDown: "Brightness Down",
|
||||
BrowserSearch: "Search", BrowserHome: "Browser Home", BrowserBack: "Browser Back", BrowserForward: "Browser Forward",
|
||||
LaunchMail: "Mail", LaunchApplication1: "App 1", LaunchApplication2: "App 2",
|
||||
MouseButton1: "Left button", MouseButton2: "Right button", MouseButton3: "Middle button",
|
||||
WheelUp: "Wheel up", WheelDown: "Wheel down", WheelLeft: "Wheel left", WheelRight: "Wheel right",
|
||||
};
|
||||
|
||||
/** A key as a person would name it. Unknown keys keep their HID usage number, which is all there is. */
|
||||
function codeLabel(code: string): string {
|
||||
const known = CODE_LABELS[code];
|
||||
if (known) return known;
|
||||
let match = /^Key([A-Z])$/.exec(code);
|
||||
if (match) return match[1]!;
|
||||
match = /^Digit([0-9])$/.exec(code);
|
||||
if (match) return match[1]!;
|
||||
match = /^Numpad(.+)$/.exec(code);
|
||||
if (match) return `Numpad ${match[1]}`;
|
||||
match = /^MouseButton([0-9]+)$/.exec(code);
|
||||
if (match) return `Mouse button ${match[1]}`;
|
||||
match = /^Usage([0-9a-f]+)$/.exec(code);
|
||||
if (match) return `Key 0x${match[1]}`;
|
||||
match = /^Consumer([0-9a-f]+)$/.exec(code);
|
||||
if (match) return `Media key 0x${match[1]}`;
|
||||
return code;
|
||||
}
|
||||
|
||||
/** A chord for display: "Shift+Esc", "Left", "Volume Up". */
|
||||
export function describeChord(chord: KeyChord): string {
|
||||
return [chord.ctrl && "Ctrl", chord.alt && "Alt", chord.shift && "Shift", chord.meta && "Meta", codeLabel(chord.code)]
|
||||
.filter(Boolean).join("+");
|
||||
}
|
||||
|
||||
/** The most custom keybindings kept; far more than a keyboard has uses for. */
|
||||
export const MAX_KEY_BINDINGS = 64;
|
||||
|
||||
/**
|
||||
* Custom keybindings from their stored JSON. Lenient: whatever doesn't make
|
||||
* sense is dropped, and a later entry for a chord replaces an earlier one.
|
||||
*/
|
||||
export function parseKeyBindings(raw: string): KeyBinding[] {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = raw ? JSON.parse(raw) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
const bindings = new Map<string, KeyBinding>();
|
||||
for (const entry of parsed) {
|
||||
if (!entry || typeof entry !== "object") continue;
|
||||
const { key, action } = entry as { key?: unknown; action?: unknown };
|
||||
const chord = typeof key === "string" ? parseChordId(key) : null;
|
||||
if (!chord || !isKeyAction(action)) continue;
|
||||
bindings.set(chordId(chord), { chord, action });
|
||||
if (bindings.size >= MAX_KEY_BINDINGS) break;
|
||||
}
|
||||
return [...bindings.values()];
|
||||
}
|
||||
|
||||
export function serializeKeyBindings(bindings: readonly KeyBinding[]): string {
|
||||
return JSON.stringify(bindings.map((binding) => ({ key: chordId(binding.chord), action: binding.action })));
|
||||
}
|
||||
|
||||
/** Custom keybindings as the lookup resolveKey takes. */
|
||||
export function keyBindingMap(bindings: readonly KeyBinding[]): Map<string, KeyAction> {
|
||||
return new Map(bindings.map((binding) => [chordId(binding.chord), binding.action]));
|
||||
}
|
||||
|
||||
/** What a mouse does unless bound otherwise: the ring's tap, double tap, hold and scroll. */
|
||||
const MOUSE_DEFAULTS: Record<string, KeyGesture | undefined> = {
|
||||
MouseButton1: "click",
|
||||
MouseButton2: "double-click",
|
||||
MouseButton3: "long-press",
|
||||
WheelUp: "scroll-up",
|
||||
WheelDown: "scroll-down",
|
||||
};
|
||||
|
||||
/**
|
||||
* The built-in meaning of a key press, or null when it has none.
|
||||
* `overridesTyping`: it holds even where the focused window takes keys.
|
||||
* Built-ins go by the logical key, so both Enter keys select.
|
||||
*/
|
||||
export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overridesTyping: boolean } | null {
|
||||
// A mouse's buttons and wheel are the ring's gestures: tap, double tap,
|
||||
// hold, and scroll. Nothing types them, so they hold everywhere, whatever
|
||||
// is held on the keyboard.
|
||||
const mouse = MOUSE_DEFAULTS[event.key];
|
||||
if (mouse) return { action: mouse, overridesTyping: true };
|
||||
const { shift, ctrl, alt, meta } = event.modifiers;
|
||||
if (meta) return null;
|
||||
if (event.key === "Escape") {
|
||||
// Plain Escape belongs to whatever takes keys (it is a keystroke in a
|
||||
// terminal); with a modifier it always means "back out of here".
|
||||
return { action: "double-click", overridesTyping: shift || ctrl || alt };
|
||||
}
|
||||
if (ctrl || alt) return null;
|
||||
switch (event.key) {
|
||||
case "ArrowUp":
|
||||
return { action: "swipe-up", overridesTyping: false };
|
||||
case "ArrowDown":
|
||||
return { action: "swipe-down", overridesTyping: false };
|
||||
case "ArrowLeft":
|
||||
return { action: "swipe-left", overridesTyping: false };
|
||||
case "ArrowRight":
|
||||
return { action: "swipe-right", overridesTyping: false };
|
||||
case "Enter":
|
||||
return { action: "click", overridesTyping: false };
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** The built-ins, for showing to the user. Keep in step with defaultKeyAction. */
|
||||
export const DEFAULT_KEYS_HELP =
|
||||
"Arrow keys: swipe up, down, left and right, as on the watch. Enter: select. Esc: back.\n\n" +
|
||||
"In a terminal session those keys are typed instead, and Shift+Esc (or Ctrl+Esc, Alt+Esc) goes back.\n\n" +
|
||||
"Mouse or touchpad: the left button selects, the right button goes back, the middle button opens the system menu, " +
|
||||
"and the wheel scrolls. These work in a terminal session too.\n\n" +
|
||||
"A custom keybinding replaces these for its key and works everywhere, terminals included. " +
|
||||
"\"Nothing\" turns a key's meaning here off without stopping it being typed.";
|
||||
|
||||
/**
|
||||
* Whether holding the key repeats the gesture. Moving up and down a list
|
||||
* does. Sideways swipes don't: depending on the layout they enter and leave
|
||||
* things, and a held key would flap between the two. Selecting, backing out
|
||||
* and menus act once per press.
|
||||
*/
|
||||
export function gestureRepeats(gesture: KeyGesture): boolean {
|
||||
return gesture === "scroll-up" || gesture === "scroll-down" || gesture === "swipe-up" || gesture === "swipe-down";
|
||||
}
|
||||
|
||||
export type KeyResolution = {
|
||||
/** The key is spoken for: don't offer it to the focused window. */
|
||||
consume: boolean;
|
||||
/**
|
||||
* The gesture to perform: at once when `consume`, otherwise only if no
|
||||
* window takes the key. Null for none (including a held key's repeats of
|
||||
* a gesture that doesn't repeat).
|
||||
*/
|
||||
gesture: KeyGesture | null;
|
||||
};
|
||||
|
||||
/** What one key press (or auto-repeat) means, given the custom keybindings. */
|
||||
export function resolveKey(event: KeyEvent, custom: ReadonlyMap<string, KeyAction>): KeyResolution {
|
||||
const repeatable = (gesture: KeyGesture): KeyGesture | null =>
|
||||
event.repeat && !gestureRepeats(gesture) ? null : gesture;
|
||||
const bound = custom.get(chordId(chordOf(event)));
|
||||
if (bound !== undefined && bound !== "none") return { consume: true, gesture: repeatable(bound) };
|
||||
// "none" takes the built-in meaning away and leaves the key to be typed.
|
||||
if (bound === "none") return { consume: false, gesture: null };
|
||||
const builtIn = defaultKeyAction(event);
|
||||
if (!builtIn) return { consume: false, gesture: null };
|
||||
return { consume: builtIn.overridesTyping, gesture: repeatable(builtIn.action) };
|
||||
}
|
||||
|
||||
export type GestureOrigin = "ring" | "watch";
|
||||
|
||||
/**
|
||||
* Which device a keyboard's gesture is presented as. Swipes are the watch's
|
||||
* and scrolls the ring's; the two schemes differ in places (an icon grid
|
||||
* moves by cell for the watch, by row and then cell for the ring). A tap, a
|
||||
* double tap or a hold exists in both, so it goes with whichever the
|
||||
* keyboard last moved with: arrow keys make the keyboard a watch, a remote
|
||||
* bound to scroll makes it a ring. Before any movement, the watch.
|
||||
*/
|
||||
export class GestureScheme {
|
||||
private scheme: GestureOrigin = "watch";
|
||||
|
||||
originFor(gesture: KeyGesture): GestureOrigin {
|
||||
if (gesture.startsWith("swipe-")) this.scheme = "watch";
|
||||
else if (gesture.startsWith("scroll-")) this.scheme = "ring";
|
||||
return this.scheme;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,566 @@
|
||||
/**
|
||||
* The phone side of the CFW keyboard bridge for one glasses session: the
|
||||
* client, the remembered keyboard, reconnecting it, and what the UI shows.
|
||||
* The session hands it field-131 records and a way to send controls; the
|
||||
* rest stays in here. Modelled on the Glassly app's G2KeyboardBridge (MIT).
|
||||
*
|
||||
* The firmware keeps nothing across a reboot (it comes back with the bridge
|
||||
* off), and holds the keyboard's connection only while told to. So the
|
||||
* remembered keyboard is brought back whenever the glasses link comes up and
|
||||
* whenever the keyboard drops, backing off while it stays away. The
|
||||
* discovered handle mapping is good for one connection and is never stored.
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import { KeyboardClient, KeyboardClientError, type KeyboardStatus } from "./client";
|
||||
import type { HidKeyEvent, HidPointerSample } from "./hid";
|
||||
import {
|
||||
KeyboardDm,
|
||||
KeyboardError,
|
||||
mergeKeyboardAdverts,
|
||||
parseKeyboardAddress,
|
||||
type KeyboardAdvert,
|
||||
} from "./protocol";
|
||||
|
||||
export type SavedKeyboard = { address: string; addressType: number; name: string };
|
||||
|
||||
export type KeyboardBridgeState =
|
||||
/** No glasses link that can host a keyboard. */
|
||||
| "unavailable"
|
||||
| "idle"
|
||||
| "scanning"
|
||||
| "connecting"
|
||||
| "pairing"
|
||||
| "configuring"
|
||||
| "connected"
|
||||
| "error";
|
||||
|
||||
/** A pairing step waiting on the user. */
|
||||
export type KeyboardPairingPrompt =
|
||||
/** Type this on the keyboard, then Enter. */
|
||||
| { kind: "display"; pin: number }
|
||||
/** Confirm both sides show this value (answerCompare). */
|
||||
| { kind: "compare"; value: number }
|
||||
/** The keyboard shows a code to enter here (answerPasskey). */
|
||||
| { kind: "entry" };
|
||||
|
||||
export type KeyboardBridgeSnapshot = {
|
||||
state: KeyboardBridgeState;
|
||||
saved: SavedKeyboard | null;
|
||||
/** Name of the keyboard being connected or in use. */
|
||||
activeName: string;
|
||||
error: string;
|
||||
pairing: KeyboardPairingPrompt | null;
|
||||
/** Advertisers from the last scan, HID devices first, strongest first. */
|
||||
scanResults: KeyboardAdvert[];
|
||||
/** A user-started scan or connect is running. */
|
||||
busy: boolean;
|
||||
/** The remembered keyboard is away and the bridge keeps trying to bring it back. */
|
||||
waiting: boolean;
|
||||
/** The connected keyboard has a touchpad or mouse buttons. */
|
||||
pointer: boolean;
|
||||
};
|
||||
|
||||
export type KeyboardBridgeHost = {
|
||||
/** Queue a control record on the live session; false when there is none. */
|
||||
send(record: Uint8Array): boolean;
|
||||
loadSaved(): SavedKeyboard | null;
|
||||
storeSaved(saved: SavedKeyboard | null): void;
|
||||
/** Key transitions from the keyboard (including forced releases). */
|
||||
onKeys(events: HidKeyEvent[]): void;
|
||||
/** A report from the keyboard's touchpad or mouse (including a forced release of its buttons). */
|
||||
onPointer(sample: HidPointerSample): void;
|
||||
/** The keyboard stopped being usable; drop held-key state without releases. */
|
||||
onKeyboardGone(): void;
|
||||
/** Something in snapshot() changed. */
|
||||
onChanged(): void;
|
||||
/**
|
||||
* A line for the log. `routine` marks the ones that repeat for as long as
|
||||
* a remembered keyboard stays away, which only a verbose log wants.
|
||||
*/
|
||||
log(line: string, routine?: boolean): void;
|
||||
/** A fresh six-digit pairing passkey from a good random source. */
|
||||
randomPasskey?(): number;
|
||||
/**
|
||||
* Whether looking for an absent keyboard is worthwhile now. Each attempt
|
||||
* has the glasses' radio listening for the keyboard for up to half a
|
||||
* minute, so the owner limits it to when the glasses are in use, and calls
|
||||
* retryNow() when that starts. Default: always.
|
||||
*/
|
||||
shouldReconnect?(): boolean;
|
||||
};
|
||||
|
||||
/** Delay before each automatic attempt after a failure; the last one repeats. */
|
||||
const RECONNECT_BACKOFF_MS = [2_000, 5_000, 10_000, 20_000, 30_000];
|
||||
/** After the glasses link comes up, before the first attempt. */
|
||||
const RESUME_DELAY_MS = 1_000;
|
||||
/** After the keyboard drops (it may just be going to sleep). */
|
||||
const DROP_RETRY_DELAY_MS = 2_000;
|
||||
const RELEASE_POLLS = 20;
|
||||
const RELEASE_POLL_MS = 150;
|
||||
|
||||
export class KeyboardBridge {
|
||||
private client: KeyboardClient | null = null;
|
||||
private available = false;
|
||||
private state: KeyboardBridgeState = "unavailable";
|
||||
private saved: SavedKeyboard | null;
|
||||
private activeName = "";
|
||||
private error = "";
|
||||
private pairing: KeyboardPairingPrompt | null = null;
|
||||
private scanResults: KeyboardAdvert[] = [];
|
||||
/** A user-started scan/connect holds the bridge. */
|
||||
private busy = false;
|
||||
/** The keyboard is connected, encrypted and subscribed. */
|
||||
private ready = false;
|
||||
private userDisconnected = false;
|
||||
/** The keyboard was let go while the glasses were out of reach; tell the firmware when they are back. */
|
||||
private releaseOnLink = false;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private reconnectAttempt = 0;
|
||||
/** Bumped whenever the client is replaced; work started for an older one stops quietly. */
|
||||
private generation = 0;
|
||||
|
||||
constructor(private readonly host: KeyboardBridgeHost) {
|
||||
this.saved = host.loadSaved();
|
||||
}
|
||||
|
||||
snapshot(): KeyboardBridgeSnapshot {
|
||||
return {
|
||||
state: this.state,
|
||||
saved: this.saved,
|
||||
activeName: this.activeName,
|
||||
error: this.error,
|
||||
pairing: this.pairing,
|
||||
scanResults: sortAdverts(this.scanResults),
|
||||
busy: this.busy,
|
||||
waiting: this.available && !!this.saved && !this.ready && !this.userDisconnected && !this.busy,
|
||||
pointer: this.ready && this.client?.configuration?.pointer === true,
|
||||
};
|
||||
}
|
||||
|
||||
/** True while key events can arrive. */
|
||||
isConnected(): boolean {
|
||||
return this.ready;
|
||||
}
|
||||
|
||||
// ---------- Link lifecycle (driven by the session owner) ----------
|
||||
|
||||
/** The glasses link is up on firmware with the bridge: bring the remembered keyboard back. */
|
||||
linkReady(): void {
|
||||
if (this.available) return;
|
||||
this.available = true;
|
||||
this.userDisconnected = false;
|
||||
this.reconnectAttempt = 0;
|
||||
this.error = "";
|
||||
const client = this.replaceClient();
|
||||
this.setState("idle");
|
||||
if (this.saved && !this.releaseOnLink) {
|
||||
this.scheduleReconnect(RESUME_DELAY_MS);
|
||||
} else if (this.releaseOnLink && client) {
|
||||
this.userDisconnected = !!this.saved;
|
||||
client.query().then((status) => {
|
||||
if (status.enabled || status.conn) return client.disable();
|
||||
}).catch(() => {});
|
||||
}
|
||||
this.releaseOnLink = false;
|
||||
}
|
||||
|
||||
/** The glasses link is gone. Only the remembered keyboard survives. */
|
||||
linkLost(): void {
|
||||
if (!this.available && !this.client) return;
|
||||
this.available = false;
|
||||
this.cancelReconnect();
|
||||
this.generation++;
|
||||
this.client?.close();
|
||||
this.client = null;
|
||||
this.busy = false;
|
||||
this.pairing = null;
|
||||
this.error = "";
|
||||
this.dropKeyboard();
|
||||
this.setState("unavailable");
|
||||
}
|
||||
|
||||
/**
|
||||
* The app is about to close the glasses link on purpose: tell the firmware
|
||||
* to let the keyboard go, so it isn't left attached to glasses nobody is
|
||||
* listening to. Best effort; nothing waits for the answer.
|
||||
*/
|
||||
linkClosing(): void {
|
||||
const client = this.client;
|
||||
if (!client || !this.available || !client.status.enabled) return;
|
||||
client.disable().catch(() => {});
|
||||
}
|
||||
|
||||
/** One field-131 payload from the session. */
|
||||
accept(payload: Uint8Array): void {
|
||||
this.client?.accept(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* The keyboard may be reachable now, or is wanted now (the display woke):
|
||||
* if it is away and the bridge is waiting out a backoff, try at once.
|
||||
*/
|
||||
retryNow(): void {
|
||||
if (!this.available || !this.saved || this.ready || this.userDisconnected || this.busy) return;
|
||||
// No timer means an attempt is already in flight.
|
||||
if (this.reconnectTimer === null) return;
|
||||
this.reconnectAttempt = 0;
|
||||
this.scheduleReconnect(0);
|
||||
}
|
||||
|
||||
// ---------- Commands (the settings UI) ----------
|
||||
|
||||
/** Scan for about five seconds; results accumulate in the snapshot as they arrive. */
|
||||
async scan(): Promise<KeyboardAdvert[]> {
|
||||
const client = this.beginUserOperation();
|
||||
const generation = this.generation;
|
||||
this.scanResults = [];
|
||||
this.error = "";
|
||||
this.setState("scanning");
|
||||
try {
|
||||
await this.releaseLink(client);
|
||||
await client.enable();
|
||||
await client.scan();
|
||||
if (generation === this.generation) this.setState("idle");
|
||||
} catch (e) {
|
||||
if (generation === this.generation) this.fail(e);
|
||||
throw friendly(e);
|
||||
} finally {
|
||||
if (generation === this.generation) this.endUserOperation();
|
||||
}
|
||||
return sortAdverts(this.scanResults);
|
||||
}
|
||||
|
||||
/** Connect, pair and subscribe a keyboard, then remember it. */
|
||||
async connect(keyboard: SavedKeyboard): Promise<void> {
|
||||
if (!parseKeyboardAddress(keyboard.address) || keyboard.addressType < 0 || keyboard.addressType > 3) {
|
||||
throw new KeyboardClientError("Invalid keyboard address");
|
||||
}
|
||||
const client = this.beginUserOperation();
|
||||
const generation = this.generation;
|
||||
this.userDisconnected = false;
|
||||
try {
|
||||
await this.connectWith(client, generation, keyboard, false);
|
||||
} finally {
|
||||
if (generation === this.generation) this.endUserOperation();
|
||||
}
|
||||
}
|
||||
|
||||
/** Try the remembered keyboard again now. */
|
||||
async reconnect(): Promise<void> {
|
||||
if (!this.saved) throw new KeyboardClientError("No remembered keyboard");
|
||||
await this.connect(this.saved);
|
||||
}
|
||||
|
||||
/** Drop the keyboard but keep it remembered. No automatic reconnect until the next explicit connect or glasses session. */
|
||||
async disconnect(): Promise<void> {
|
||||
this.userDisconnected = true;
|
||||
this.cancelReconnect();
|
||||
const client = this.replaceClient();
|
||||
// A scan or connect this interrupts no longer owns the bridge.
|
||||
this.busy = false;
|
||||
this.dropKeyboard();
|
||||
this.pairing = null;
|
||||
this.error = "";
|
||||
if (!client) {
|
||||
this.releaseOnLink = true;
|
||||
this.setState("unavailable");
|
||||
return;
|
||||
}
|
||||
this.setState("idle");
|
||||
try {
|
||||
await client.query();
|
||||
await client.disable();
|
||||
} catch (e) {
|
||||
this.host.log(`keyboard disconnect: ${messageOf(e)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Forget the remembered keyboard. The glasses keep their bond with it. */
|
||||
async forget(): Promise<void> {
|
||||
this.saved = null;
|
||||
this.host.storeSaved(null);
|
||||
await this.disconnect();
|
||||
this.scanResults = [];
|
||||
this.activeName = "";
|
||||
this.host.onChanged();
|
||||
}
|
||||
|
||||
/** Answer a "compare" pairing prompt. */
|
||||
async answerCompare(accept: boolean): Promise<void> {
|
||||
if (!this.client || this.pairing?.kind !== "compare") throw new KeyboardClientError("No pairing comparison is pending");
|
||||
this.pairing = null;
|
||||
this.host.onChanged();
|
||||
await this.client.confirm(accept);
|
||||
}
|
||||
|
||||
/** Answer an "entry" pairing prompt with the code the keyboard shows. */
|
||||
async answerPasskey(pin: number): Promise<void> {
|
||||
if (!this.client || this.pairing?.kind !== "entry") throw new KeyboardClientError("No pairing code is pending");
|
||||
this.pairing = null;
|
||||
this.host.onChanged();
|
||||
await this.client.authenticate(pin);
|
||||
}
|
||||
|
||||
// ---------- Internals ----------
|
||||
|
||||
private beginUserOperation(): KeyboardClient {
|
||||
if (!this.available) throw new KeyboardClientError("Connect to G2 glasses running Faceclaw firmware first");
|
||||
if (this.busy) throw new KeyboardClientError("Another keyboard operation is in progress");
|
||||
// An automatic attempt may be mid-flight (a sleeping keyboard holds one
|
||||
// for up to 30 s): a fresh client fails it at once and takes over.
|
||||
this.cancelReconnect();
|
||||
const client = this.replaceClient()!;
|
||||
this.dropKeyboard();
|
||||
this.pairing = null;
|
||||
this.busy = true;
|
||||
return client;
|
||||
}
|
||||
|
||||
private endUserOperation(): void {
|
||||
this.busy = false;
|
||||
this.host.onChanged();
|
||||
}
|
||||
|
||||
/** Close the current client and, while the link is up, start a fresh one. */
|
||||
private replaceClient(): KeyboardClient | null {
|
||||
this.generation++;
|
||||
this.client?.close();
|
||||
this.client = null;
|
||||
if (!this.available) return null;
|
||||
const generation = this.generation;
|
||||
const live = () => generation === this.generation;
|
||||
const client = new KeyboardClient(
|
||||
(record) => live() && this.host.send(record),
|
||||
{
|
||||
keys: (events) => { if (live() && this.ready) this.host.onKeys(events); },
|
||||
pointer: (sample) => { if (live() && this.ready) this.host.onPointer(sample); },
|
||||
advert: (advert) => { if (live()) this.addAdvert(advert); },
|
||||
status: (status) => { if (live()) this.onStatus(status); },
|
||||
dm: (event, status) => { if (live()) this.onDm(event, status); },
|
||||
passkeyDisplay: (pin) => { if (live()) this.prompt({ kind: "display", pin }); },
|
||||
passkeyRequest: () => { if (live()) this.prompt({ kind: "entry" }); },
|
||||
compareRequest: (value) => { if (live()) this.prompt({ kind: "compare", value }); },
|
||||
servicesChanged: () => { if (live()) this.onServicesChanged(); },
|
||||
error: (message) => { if (live()) this.host.log(`keyboard: ${message}`); },
|
||||
log: (line) => { if (live()) this.host.log(`keyboard setup: ${line}`); },
|
||||
},
|
||||
{ randomPasskey: this.host.randomPasskey ? () => this.host.randomPasskey!() : undefined },
|
||||
);
|
||||
this.client = client;
|
||||
return client;
|
||||
}
|
||||
|
||||
private async connectWith(
|
||||
client: KeyboardClient,
|
||||
generation: number,
|
||||
keyboard: SavedKeyboard,
|
||||
resume: boolean,
|
||||
): Promise<void> {
|
||||
const live = () => generation === this.generation;
|
||||
this.error = "";
|
||||
this.activeName = keyboard.name;
|
||||
this.setState("connecting");
|
||||
try {
|
||||
const status = await client.query();
|
||||
if (resume && status.enabled && status.connected && status.encrypted) {
|
||||
// The glasses kept the keyboard through a phone-side drop (or an app
|
||||
// restart): only the handle mapping is missing.
|
||||
this.host.log("keyboard still connected to the glasses; rediscovering");
|
||||
} else {
|
||||
await this.releaseLink(client);
|
||||
await client.enable();
|
||||
await client.connect(keyboard.addressType, parseKeyboardAddress(keyboard.address)!);
|
||||
}
|
||||
if (!live()) return;
|
||||
this.pairing = null;
|
||||
this.setState("configuring");
|
||||
const config = await client.configure();
|
||||
if (!live()) return;
|
||||
this.ready = true;
|
||||
this.reconnectAttempt = 0;
|
||||
// The list was for choosing; it is stale from here on.
|
||||
this.scanResults = [];
|
||||
if (!sameKeyboard(this.saved, keyboard)) {
|
||||
this.saved = { ...keyboard };
|
||||
this.host.storeSaved(this.saved);
|
||||
}
|
||||
this.host.log(
|
||||
`keyboard ready: ${keyboard.name || keyboard.address}, ${config.watched.length} of ` +
|
||||
`${config.reports.filter((r) => r.type === 1).length} input reports subscribed` +
|
||||
`${config.pointer ? ", with a pointer" : ""}, epoch ${config.epoch}`);
|
||||
this.setState("connected");
|
||||
} catch (e) {
|
||||
if (!live()) return;
|
||||
this.ready = false;
|
||||
this.pairing = null;
|
||||
if (resume) {
|
||||
// An automatic attempt: a keyboard that is asleep or out of range is
|
||||
// the ordinary case, not an error to put in front of the user.
|
||||
this.host.log(`keyboard reconnect: ${messageOf(friendly(e))}`, true);
|
||||
this.setState("idle");
|
||||
} else {
|
||||
this.fail(e);
|
||||
}
|
||||
throw friendly(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The firmware refuses SCAN and CONNECT while it still owns a keyboard
|
||||
* slot, and an interrupted connect keeps initiating until the close lands.
|
||||
* DISABLE is epoch-checked and a fresh client hasn't seen the epoch yet,
|
||||
* so QUERY first (it isn't, and its answer carries the epoch), then wait
|
||||
* for the slot to clear.
|
||||
*/
|
||||
private async releaseLink(client: KeyboardClient): Promise<void> {
|
||||
const status = await client.query();
|
||||
if (!status.enabled && status.conn === 0) return;
|
||||
await client.disable();
|
||||
for (let i = 0; i < RELEASE_POLLS; i++) {
|
||||
if ((await client.query()).conn === 0) return;
|
||||
await delay(RELEASE_POLL_MS);
|
||||
}
|
||||
throw new KeyboardClientError("The glasses are still closing the previous keyboard link; try again in a moment");
|
||||
}
|
||||
|
||||
private cancelReconnect(): void {
|
||||
if (this.reconnectTimer !== null) clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
|
||||
private scheduleReconnect(delayMs: number): void {
|
||||
this.cancelReconnect();
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
void this.attemptReconnect();
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
private async attemptReconnect(): Promise<void> {
|
||||
const saved = this.saved;
|
||||
const client = this.client;
|
||||
if (!this.available || !client || !saved || this.userDisconnected || this.ready || this.busy) return;
|
||||
if (this.host.shouldReconnect && !this.host.shouldReconnect()) {
|
||||
this.parkFirmware(client);
|
||||
this.scheduleReconnect(RECONNECT_BACKOFF_MS[RECONNECT_BACKOFF_MS.length - 1]!);
|
||||
return;
|
||||
}
|
||||
const generation = this.generation;
|
||||
try {
|
||||
await this.connectWith(client, generation, saved, true);
|
||||
} catch {
|
||||
if (generation !== this.generation || !this.available || this.userDisconnected) return;
|
||||
// Keyboards sleep, and the ring's own scan can hold the controller
|
||||
// (BUSY): keep trying, less often, while the glasses stay up.
|
||||
const backoff = RECONNECT_BACKOFF_MS[Math.min(this.reconnectAttempt, RECONNECT_BACKOFF_MS.length - 1)]!;
|
||||
this.reconnectAttempt++;
|
||||
this.scheduleReconnect(backoff);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* While it is enabled the firmware bridge runs a 20 ms timer and reports
|
||||
* its status every second, keyboard or no keyboard. When nothing will be
|
||||
* tried for a while, switch it off; the next attempt enables it again.
|
||||
*/
|
||||
private parkFirmware(client: KeyboardClient): void {
|
||||
if (!client.status.enabled) return;
|
||||
client.disable().catch(() => {});
|
||||
}
|
||||
|
||||
private onStatus(status: KeyboardStatus): void {
|
||||
if (this.ready && (!status.connected || !status.encrypted)) this.keyboardDropped("status");
|
||||
else if (this.state === "connecting" && status.connected && !status.encrypted) this.setState("pairing");
|
||||
}
|
||||
|
||||
private onDm(event: number, status: number): void {
|
||||
if (event === KeyboardDm.connClose || event === KeyboardDm.pairFailed || event === KeyboardDm.encryptFailed) {
|
||||
this.host.log(`keyboard DM event 0x${event.toString(16)} status ${status}`);
|
||||
if (this.ready) this.keyboardDropped("dm");
|
||||
}
|
||||
}
|
||||
|
||||
private onServicesChanged(): void {
|
||||
this.host.log("keyboard services changed; rediscovering");
|
||||
this.dropKeyboard();
|
||||
this.setState("idle");
|
||||
if (this.saved && !this.userDisconnected && !this.busy) this.scheduleReconnect(0);
|
||||
}
|
||||
|
||||
private keyboardDropped(reason: string): void {
|
||||
this.host.log(`keyboard disconnected (${reason})`);
|
||||
this.dropKeyboard();
|
||||
this.setState("idle");
|
||||
if (this.saved && !this.userDisconnected && !this.busy) this.scheduleReconnect(DROP_RETRY_DELAY_MS);
|
||||
}
|
||||
|
||||
/** The keyboard is no longer usable: tell the host once. */
|
||||
private dropKeyboard(): void {
|
||||
if (!this.ready) return;
|
||||
this.ready = false;
|
||||
this.host.onKeyboardGone();
|
||||
}
|
||||
|
||||
private addAdvert(advert: KeyboardAdvert): void {
|
||||
const index = this.scanResults.findIndex(
|
||||
(a) => a.address === advert.address && a.addressType === advert.addressType);
|
||||
if (index < 0) {
|
||||
this.scanResults.push(advert);
|
||||
} else {
|
||||
const merged = mergeKeyboardAdverts(this.scanResults[index]!, advert);
|
||||
const old = this.scanResults[index]!;
|
||||
if (merged.name === old.name && merged.hid === old.hid && merged.rssi === old.rssi) return;
|
||||
this.scanResults[index] = merged;
|
||||
}
|
||||
this.host.onChanged();
|
||||
}
|
||||
|
||||
private prompt(pairing: KeyboardPairingPrompt): void {
|
||||
this.pairing = pairing;
|
||||
this.setState("pairing");
|
||||
}
|
||||
|
||||
private setState(next: KeyboardBridgeState): void {
|
||||
this.state = next;
|
||||
if (next !== "pairing" && next !== "connecting") this.pairing = null;
|
||||
this.host.onChanged();
|
||||
}
|
||||
|
||||
private fail(error: unknown): void {
|
||||
this.error = messageOf(friendly(error));
|
||||
this.host.log(`keyboard error: ${this.error}`);
|
||||
this.setState("error");
|
||||
}
|
||||
}
|
||||
|
||||
function sameKeyboard(a: SavedKeyboard | null, b: SavedKeyboard): boolean {
|
||||
return !!a && a.address === b.address && a.addressType === b.addressType && a.name === b.name;
|
||||
}
|
||||
|
||||
function sortAdverts(adverts: readonly KeyboardAdvert[]): KeyboardAdvert[] {
|
||||
return [...adverts].sort((a, b) => Number(b.hid) - Number(a.hid) || b.rssi - a.rssi);
|
||||
}
|
||||
|
||||
/**
|
||||
* Firmware BUSY nearly always means the ring's own scan or reconnect holds
|
||||
* the radio; say that rather than the raw control error.
|
||||
*/
|
||||
function friendly(error: unknown): Error {
|
||||
if (error instanceof KeyboardClientError && error.code === KeyboardError.busy) {
|
||||
return new KeyboardClientError(
|
||||
"The glasses' radio is busy (usually the ring reconnecting). Wait a few seconds and try again.", error.code);
|
||||
}
|
||||
if (error instanceof KeyboardClientError && error.code === KeyboardError.noSpace) {
|
||||
return new KeyboardClientError(
|
||||
"The glasses have no room for another connection or pairing.", error.code);
|
||||
}
|
||||
return error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,754 @@
|
||||
/**
|
||||
* Transport-independent client for the CFW keyboard bridge: enable, scan,
|
||||
* connect and pair a keyboard through the glasses, discover its HID service
|
||||
* over the relayed ATT operations, subscribe its input reports, and decode
|
||||
* them into key transitions. From the reference client in g2flash demos/
|
||||
* (keyboard-client.ts), with the fixes its use in the Glassly app turned up.
|
||||
*
|
||||
* The firmware keeps almost no policy: it owns the connection and the bond,
|
||||
* and forwards one ATT request at a time. Everything HID lives here. Handle
|
||||
* numbers found by configure() are only good for one connection epoch.
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import {
|
||||
HidKeys,
|
||||
parseReportMap,
|
||||
reportHasKeys,
|
||||
reportHasPointer,
|
||||
reportPointer,
|
||||
type HidKeyEvent,
|
||||
type HidPointerSample,
|
||||
type HidReport,
|
||||
} from "./hid";
|
||||
import {
|
||||
KeyboardDm,
|
||||
KeyboardError,
|
||||
KeyboardKind,
|
||||
KeyboardOp,
|
||||
KeyboardRecords,
|
||||
keyboardControl,
|
||||
keyboardErrorName,
|
||||
le16,
|
||||
parseKeyboardAdvert,
|
||||
u16,
|
||||
u32,
|
||||
type KeyboardAdvert,
|
||||
type KeyboardRecord,
|
||||
} from "./protocol";
|
||||
|
||||
/** Queue one field-130 control record; false when the glasses link can't take it now. */
|
||||
export type KeyboardTransport = (record: Uint8Array) => boolean;
|
||||
|
||||
export type KeyboardStatus = {
|
||||
enabled: boolean;
|
||||
/** The firmware's connection id for the keyboard, 0 when it owns none. */
|
||||
conn: number;
|
||||
connected: boolean;
|
||||
encrypted: boolean;
|
||||
scanning: boolean;
|
||||
};
|
||||
|
||||
export type KeyboardClientEvents = {
|
||||
/** Key transitions from one input report (or the releases a gap forces). */
|
||||
keys?(events: HidKeyEvent[]): void;
|
||||
/**
|
||||
* One report from the keyboard's touchpad or mouse: the buttons now held
|
||||
* (across all its pointer reports) and the movement and wheel since the
|
||||
* last one. Also sent with everything zero when a gap forces a release.
|
||||
*/
|
||||
pointer?(sample: HidPointerSample): void;
|
||||
advert?(advert: KeyboardAdvert): void;
|
||||
status?(status: KeyboardStatus): void;
|
||||
/** The stream may have lost records; held keys were released. */
|
||||
gap?(): void;
|
||||
/** Device-manager event from the glasses' BLE host (KeyboardDm). */
|
||||
dm?(event: number, status: number): void;
|
||||
/** The keyboard shows a six-digit passkey; answer with authenticate(). */
|
||||
passkeyRequest?(): void;
|
||||
/** We chose a passkey and answered with it; the user types it on the keyboard, then Enter. */
|
||||
passkeyDisplay?(pin: number): void;
|
||||
/** Numeric comparison: answer with confirm() once the user has compared. */
|
||||
compareRequest?(value: number): void;
|
||||
/** The keyboard's GATT table changed; the mapping is void until configure() runs again. */
|
||||
servicesChanged?(): void;
|
||||
error?(message: string): void;
|
||||
/** Setup diagnostics: the keyboard's attribute table and what was made of it. Never key contents. */
|
||||
log?(line: string): void;
|
||||
};
|
||||
|
||||
export type KeyboardReport = {
|
||||
/** Handle of the HID service declaration the report belongs to. */
|
||||
service: number;
|
||||
handle: number;
|
||||
id: number;
|
||||
/** 1 input, 2 output, 3 feature. */
|
||||
type: number;
|
||||
ccc?: number;
|
||||
/** From the characteristic's declaration; undefined when the keyboard wouldn't say (see declaredProperties). */
|
||||
properties?: number;
|
||||
map: Uint8Array;
|
||||
};
|
||||
|
||||
export type KeyboardConfiguration = {
|
||||
epoch: number;
|
||||
/** Every report found, input or not. */
|
||||
reports: KeyboardReport[];
|
||||
/** Handles of the input reports subscribed: those that carry keys, or a pointer's buttons and movement. */
|
||||
watched: number[];
|
||||
/** Whether any of them is a pointer's (a touchpad, mouse buttons, a wheel). */
|
||||
pointer: boolean;
|
||||
serviceChanged?: number;
|
||||
};
|
||||
|
||||
export type KeyboardClientOptions = {
|
||||
/** A fresh six-digit passkey for display pairing. Default: Math.random (pass a stronger source). */
|
||||
randomPasskey?: () => number;
|
||||
};
|
||||
|
||||
export class KeyboardClientError extends Error {
|
||||
/** The firmware's control error code (KeyboardError), when that is what failed. */
|
||||
readonly code: number | undefined;
|
||||
/** The keyboard's ATT error code, when it refused a read or write. */
|
||||
readonly attStatus: number | undefined;
|
||||
constructor(message: string, code?: number, attStatus?: number) {
|
||||
super(message);
|
||||
this.name = "KeyboardClientError";
|
||||
this.code = code;
|
||||
this.attStatus = attStatus;
|
||||
}
|
||||
}
|
||||
|
||||
type Attribute = { handle: number; uuid: number | string };
|
||||
type Pending = {
|
||||
op: number;
|
||||
resolve: (r: KeyboardRecord) => void;
|
||||
reject: (e: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
type Waiter = { test: () => boolean; resolve: () => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> };
|
||||
type Input = { report: HidReport; id: number; service: number; keys: boolean; pointer: boolean };
|
||||
|
||||
const COMMAND_TIMEOUT_MS = 32_000;
|
||||
const MAX_PENDING = 8;
|
||||
const MAX_ATTRIBUTES = 512;
|
||||
const MAX_VALUE = 512;
|
||||
const MAX_WATCHES = 16;
|
||||
|
||||
const UUID_PRIMARY_SERVICE = 0x2800;
|
||||
const UUID_SECONDARY_SERVICE = 0x2801;
|
||||
const UUID_CHARACTERISTIC = 0x2803;
|
||||
const UUID_CCC = 0x2902;
|
||||
const UUID_EXTERNAL_REPORT_REFERENCE = 0x2907;
|
||||
const UUID_REPORT_REFERENCE = 0x2908;
|
||||
const UUID_HID_SERVICE = 0x1812;
|
||||
const UUID_SERVICE_CHANGED = 0x2a05;
|
||||
const UUID_REPORT_MAP = 0x2a4b;
|
||||
const UUID_REPORT = 0x2a4d;
|
||||
const UUID_PROTOCOL_MODE = 0x2a4e;
|
||||
|
||||
/** ATT error codes that end a paged read rather than failing it. */
|
||||
const ATT_INVALID_OFFSET = 7;
|
||||
const ATT_NOT_LONG = 11;
|
||||
const ATT_NOT_FOUND = 10;
|
||||
|
||||
/**
|
||||
* ATT UUIDs are little endian. A 128-bit UUID on the Bluetooth base collapses
|
||||
* to its 16-bit form; vendor UUIDs stay as hex strings.
|
||||
*/
|
||||
export function attributeUuid(bytes: Uint8Array): number | string {
|
||||
if (bytes.length === 2) return u16(bytes);
|
||||
if (bytes.length !== 16) throw new KeyboardClientError("Invalid ATT UUID length");
|
||||
const base = [0xfb, 0x34, 0x9b, 0x5f, 0x80, 0, 0, 0x80, 0, 0x10, 0, 0];
|
||||
if (base.every((v, i) => bytes[i] === v) && bytes[14] === 0 && bytes[15] === 0) return u16(bytes, 12);
|
||||
return Array.from(bytes, (x) => x.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
export class KeyboardClient {
|
||||
/** The firmware's connection generation; every control but QUERY/ENABLE must carry the live one. */
|
||||
epoch = 0;
|
||||
status: KeyboardStatus = { enabled: false, conn: 0, connected: false, encrypted: false, scanning: false };
|
||||
configuration: KeyboardConfiguration | undefined;
|
||||
|
||||
private serial = 0;
|
||||
private closed = false;
|
||||
/** The link under connect() went away (close, failed open, failed pairing). */
|
||||
private linkLost = false;
|
||||
private readonly pending = new Map<number, Pending>();
|
||||
private readonly waiters = new Set<Waiter>();
|
||||
private readonly keys = new HidKeys();
|
||||
/** Pointer buttons held, by the report that says so (as HidKeys does for keys). */
|
||||
private readonly pointerButtons = new Map<number, number>();
|
||||
private inputs = new Map<number, Input>();
|
||||
private readonly records = new KeyboardRecords(() => this.gap());
|
||||
private readonly randomPasskey: () => number;
|
||||
|
||||
constructor(
|
||||
private readonly transport: KeyboardTransport,
|
||||
private readonly events: KeyboardClientEvents = {},
|
||||
options: KeyboardClientOptions = {},
|
||||
) {
|
||||
this.randomPasskey = options.randomPasskey ?? (() => Math.floor(Math.random() * 1_000_000));
|
||||
}
|
||||
|
||||
// ---------- Inbound ----------
|
||||
|
||||
/** Feed one field-131 payload, in arrival order. */
|
||||
accept(data: Uint8Array): void {
|
||||
if (this.closed) return;
|
||||
const r = this.records.push(data);
|
||||
if (!r) return;
|
||||
this.epoch = r.epoch;
|
||||
if (r.kind === KeyboardKind.status && r.data.length === 10) {
|
||||
this.status = {
|
||||
enabled: !!r.data[0], conn: r.data[1]!, connected: !!r.data[2], encrypted: !!r.data[3], scanning: !!r.data[4],
|
||||
};
|
||||
if (!this.status.enabled || !this.status.connected || !this.status.encrypted) this.release();
|
||||
this.events.status?.(this.status);
|
||||
}
|
||||
const pending = this.pending.get(r.request);
|
||||
if (pending && (r.kind === KeyboardKind.status || r.kind === KeyboardKind.att)) {
|
||||
this.pending.delete(r.request);
|
||||
clearTimeout(pending.timer);
|
||||
if (r.kind === KeyboardKind.status && r.code) {
|
||||
pending.reject(new KeyboardClientError(
|
||||
`Keyboard command ${opName(pending.op)}: firmware ${keyboardErrorName(r.code)}`, r.code));
|
||||
} else {
|
||||
pending.resolve(r);
|
||||
}
|
||||
}
|
||||
if (r.kind === KeyboardKind.dm) this.acceptDm(r);
|
||||
else if (r.kind === KeyboardKind.input) this.acceptInput(r);
|
||||
else if (r.kind === KeyboardKind.advert) {
|
||||
const advert = parseKeyboardAdvert(r.data);
|
||||
if (advert) this.events.advert?.(advert);
|
||||
}
|
||||
this.checkWaiters();
|
||||
}
|
||||
|
||||
private acceptDm(r: KeyboardRecord): void {
|
||||
const event = r.code & 255;
|
||||
const status = r.code >>> 8;
|
||||
if (event === KeyboardDm.connClose || event === KeyboardDm.pairFailed || event === KeyboardDm.encryptFailed ||
|
||||
(event === KeyboardDm.connOpen && status !== 0)) {
|
||||
this.linkLost = true;
|
||||
this.gap();
|
||||
this.inputs.clear();
|
||||
this.configuration = undefined;
|
||||
}
|
||||
if (event === KeyboardDm.authRequest && r.data.length === 2) {
|
||||
if (r.data[0]) {
|
||||
this.command(KeyboardOp.disconnect).catch(() => {});
|
||||
this.events.error?.("This keyboard only pairs out of band, which isn't supported.");
|
||||
} else if (r.data[1]) {
|
||||
// KeyboardDisplay on our side: choose the passkey, show it, and
|
||||
// answer at once; the user types it on the keyboard.
|
||||
const pin = Math.min(999_999, Math.max(0, Math.floor(this.randomPasskey())));
|
||||
this.events.passkeyDisplay?.(pin);
|
||||
this.authenticate(pin).catch((e) => this.events.error?.(messageOf(e)));
|
||||
} else {
|
||||
this.events.passkeyRequest?.();
|
||||
}
|
||||
}
|
||||
if (event === KeyboardDm.compare && r.data.length === 4) this.events.compareRequest?.(u32(r.data));
|
||||
this.events.dm?.(event, status);
|
||||
}
|
||||
|
||||
private acceptInput(r: KeyboardRecord): void {
|
||||
const changed = this.configuration?.serviceChanged;
|
||||
if (changed !== undefined && r.handle === changed) {
|
||||
this.gap();
|
||||
this.inputs.clear();
|
||||
this.configuration = undefined;
|
||||
this.events.servicesChanged?.();
|
||||
return;
|
||||
}
|
||||
const input = this.inputs.get(r.handle);
|
||||
if (!input || this.configuration?.epoch !== r.epoch) {
|
||||
this.gap();
|
||||
return;
|
||||
}
|
||||
if (input.keys) {
|
||||
try {
|
||||
const events = this.keys.update(r.handle, input.report, r.data);
|
||||
if (events.length) this.events.keys?.(events);
|
||||
} catch (e) {
|
||||
this.release();
|
||||
this.events.error?.(messageOf(e));
|
||||
}
|
||||
}
|
||||
if (input.pointer) {
|
||||
const sample = reportPointer(input.report, r.data);
|
||||
this.pointerButtons.set(r.handle, sample.buttons);
|
||||
this.events.pointer?.({ ...sample, buttons: this.heldButtons() });
|
||||
}
|
||||
}
|
||||
|
||||
private heldButtons(): number {
|
||||
let buttons = 0;
|
||||
for (const held of this.pointerButtons.values()) buttons |= held;
|
||||
return buttons;
|
||||
}
|
||||
|
||||
/** Nothing is held any more, as far as anyone can tell: say so. */
|
||||
private release(): void {
|
||||
const events = this.keys.reset();
|
||||
if (events.length) this.events.keys?.(events);
|
||||
const buttons = this.heldButtons();
|
||||
this.pointerButtons.clear();
|
||||
if (buttons) this.events.pointer?.({ buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 });
|
||||
}
|
||||
|
||||
private gap(): void {
|
||||
this.release();
|
||||
this.events.gap?.();
|
||||
for (const [id, p] of this.pending) {
|
||||
if (p.op < KeyboardOp.find) continue;
|
||||
clearTimeout(p.timer);
|
||||
this.pending.delete(id);
|
||||
p.reject(new KeyboardClientError("Keyboard stream gap during ATT transaction"));
|
||||
}
|
||||
}
|
||||
|
||||
/** The phone's link to the glasses dropped: release held keys and fail everything in flight. */
|
||||
lostPhoneLink(): void {
|
||||
this.records.reset();
|
||||
this.cancel("Glasses link lost");
|
||||
}
|
||||
|
||||
/** Stop for good; later accept() calls are ignored. */
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.lostPhoneLink();
|
||||
this.closed = true;
|
||||
}
|
||||
|
||||
private cancel(reason: string): void {
|
||||
for (const p of this.pending.values()) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(new KeyboardClientError(reason));
|
||||
}
|
||||
this.pending.clear();
|
||||
for (const w of this.waiters) {
|
||||
clearTimeout(w.timer);
|
||||
w.reject(new KeyboardClientError(reason));
|
||||
}
|
||||
this.waiters.clear();
|
||||
}
|
||||
|
||||
private checkWaiters(): void {
|
||||
for (const w of [...this.waiters]) {
|
||||
let done = false;
|
||||
try {
|
||||
done = w.test();
|
||||
} catch (e) {
|
||||
this.waiters.delete(w);
|
||||
clearTimeout(w.timer);
|
||||
w.reject(e instanceof Error ? e : new KeyboardClientError(String(e)));
|
||||
continue;
|
||||
}
|
||||
if (!done) continue;
|
||||
this.waiters.delete(w);
|
||||
clearTimeout(w.timer);
|
||||
w.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve once `test` holds; it is re-evaluated after every accepted record. */
|
||||
private waitFor(test: () => boolean, timeoutMs: number, what: string): Promise<void> {
|
||||
if (this.closed) return Promise.reject(new KeyboardClientError("Keyboard client closed"));
|
||||
if (test()) return Promise.resolve();
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const waiter: Waiter = {
|
||||
test, resolve, reject,
|
||||
timer: setTimeout(() => {
|
||||
this.waiters.delete(waiter);
|
||||
reject(new KeyboardClientError(`Timed out waiting for ${what}`));
|
||||
}, timeoutMs),
|
||||
};
|
||||
this.waiters.add(waiter);
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- Commands ----------
|
||||
|
||||
/** Send one control and await the STATUS or ATT record that answers it. */
|
||||
command(op: number, body: readonly number[] = []): Promise<KeyboardRecord> {
|
||||
if (this.closed) return Promise.reject(new KeyboardClientError("Keyboard client closed"));
|
||||
if (this.pending.size >= MAX_PENDING) return Promise.reject(new KeyboardClientError("Too many keyboard commands"));
|
||||
const request = this.serial = this.serial % 65535 + 1;
|
||||
return new Promise<KeyboardRecord>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(request);
|
||||
reject(new KeyboardClientError(`Keyboard command ${opName(op)} timed out`));
|
||||
}, COMMAND_TIMEOUT_MS);
|
||||
this.pending.set(request, { op, resolve, reject, timer });
|
||||
let queued = false;
|
||||
try {
|
||||
queued = this.transport(keyboardControl(op, request, this.epoch, body));
|
||||
} catch {
|
||||
queued = false;
|
||||
}
|
||||
if (!queued) {
|
||||
clearTimeout(timer);
|
||||
this.pending.delete(request);
|
||||
reject(new KeyboardClientError("The glasses aren't ready for keyboard commands"));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async enable(): Promise<void> {
|
||||
await this.command(KeyboardOp.enable);
|
||||
}
|
||||
|
||||
/** Disconnect the keyboard and stop the bridge; the bond stays. */
|
||||
async disable(): Promise<void> {
|
||||
await this.command(KeyboardOp.disable);
|
||||
this.release();
|
||||
}
|
||||
|
||||
/** Ask for the firmware's state. Also the way to learn the live epoch: QUERY isn't epoch-checked. */
|
||||
async query(): Promise<KeyboardStatus> {
|
||||
await this.command(KeyboardOp.query);
|
||||
return this.status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Active scan for about five seconds; adverts arrive through events.advert.
|
||||
* The firmware only clears `scanning` once the stock scanner is idle again,
|
||||
* and a ring looking for its lens can hold that well past the window, so a
|
||||
* lingering flag is not an error: the adverts already delivered stand.
|
||||
*/
|
||||
async scan(): Promise<void> {
|
||||
await this.command(KeyboardOp.scan);
|
||||
try {
|
||||
await this.waitFor(() => !this.status.scanning, 8000, "the scan to finish");
|
||||
} catch (e) {
|
||||
if (this.closed) throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/** Connect, then pair unless the link comes up encrypted. `address` is LSB first. */
|
||||
async connect(addressType: number, address: Uint8Array): Promise<void> {
|
||||
if (address.length !== 6 || !Number.isInteger(addressType) || addressType < 0 || addressType > 3) {
|
||||
throw new KeyboardClientError("Invalid BLE address");
|
||||
}
|
||||
this.release();
|
||||
this.inputs.clear();
|
||||
this.configuration = undefined;
|
||||
this.linkLost = false;
|
||||
await this.command(KeyboardOp.connect, [addressType, ...address]);
|
||||
await this.waitFor(() => this.status.connected || this.linkLost, 32_000, "the keyboard to connect");
|
||||
if (!this.status.connected) throw new KeyboardClientError("The keyboard didn't answer");
|
||||
if (!this.status.encrypted) {
|
||||
// The keyboard's own security request may have started pairing already.
|
||||
// BUSY is not success: the encrypted status below still decides.
|
||||
try {
|
||||
await this.command(KeyboardOp.pair);
|
||||
} catch (e) {
|
||||
if (!(e instanceof KeyboardClientError) || e.code !== KeyboardError.busy) throw e;
|
||||
}
|
||||
await this.waitFor(() => this.status.encrypted || this.linkLost, 65_000, "pairing to finish");
|
||||
if (!this.status.encrypted) throw new KeyboardClientError("Pairing with the keyboard failed");
|
||||
}
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
await this.command(KeyboardOp.disconnect);
|
||||
this.release();
|
||||
}
|
||||
|
||||
/** Answer a pending passkey prompt. */
|
||||
async authenticate(pin: number): Promise<void> {
|
||||
if (!Number.isInteger(pin) || pin < 0 || pin > 999_999) throw new KeyboardClientError("Invalid pairing passkey");
|
||||
await this.command(KeyboardOp.auth, [pin & 255, (pin >>> 8) & 255, pin >>> 16]);
|
||||
}
|
||||
|
||||
/** Answer a pending numeric comparison. */
|
||||
async confirm(accept: boolean): Promise<void> {
|
||||
await this.command(KeyboardOp.compare, [accept ? 1 : 0]);
|
||||
}
|
||||
|
||||
private async att(op: number, body: readonly number[]): Promise<KeyboardRecord> {
|
||||
const r = await this.command(op, body);
|
||||
if (r.kind !== KeyboardKind.att) throw new KeyboardClientError("Expected keyboard ATT response");
|
||||
return r;
|
||||
}
|
||||
|
||||
/** Read a whole attribute: Read, then Read Blob until a short or empty reply. Never assumes the MTU. */
|
||||
async read(handle: number): Promise<Uint8Array> {
|
||||
const parts: number[] = [];
|
||||
for (;;) {
|
||||
const r = await this.att(KeyboardOp.read, [...le16(handle), ...le16(parts.length)]);
|
||||
const status = r.code >>> 8;
|
||||
if (parts.length && (status === ATT_INVALID_OFFSET || status === ATT_NOT_LONG)) break;
|
||||
if (status) throw new KeyboardClientError(`ATT read ${handle}: ${status}`, undefined, status);
|
||||
if (!r.data.length) break;
|
||||
for (const b of r.data) parts.push(b);
|
||||
if (parts.length > MAX_VALUE) throw new KeyboardClientError("Attribute exceeds 512 bytes");
|
||||
if (parts.length === MAX_VALUE) break;
|
||||
}
|
||||
return Uint8Array.from(parts);
|
||||
}
|
||||
|
||||
/** Read a short attribute once (a Report Reference, a declaration, a service UUID). */
|
||||
private async shortRead(handle: number): Promise<Uint8Array> {
|
||||
const r = await this.att(KeyboardOp.read, [...le16(handle), 0, 0]);
|
||||
if (r.code >>> 8) throw new KeyboardClientError(`ATT read ${handle}: ${r.code >>> 8}`, undefined, r.code >>> 8);
|
||||
return r.data;
|
||||
}
|
||||
|
||||
/** Write 1..18 bytes (an output report such as the LEDs, a CCC, the protocol mode). */
|
||||
async write(handle: number, value: Uint8Array, withoutResponse = false): Promise<void> {
|
||||
if (!value.length || value.length > 18) throw new KeyboardClientError("Keyboard write supports 1..18 bytes");
|
||||
const r = await this.att(KeyboardOp.write, [...le16(handle), withoutResponse ? 1 : 0, value.length, ...value]);
|
||||
if (r.code >>> 8) throw new KeyboardClientError(`ATT write ${handle}: ${r.code >>> 8}`, undefined, r.code >>> 8);
|
||||
}
|
||||
|
||||
private async attributes(): Promise<Attribute[]> {
|
||||
const attrs: Attribute[] = [];
|
||||
let start = 1;
|
||||
while (start <= 65535) {
|
||||
const r = await this.att(KeyboardOp.find, [...le16(start), 255, 255]);
|
||||
const status = r.code >>> 8;
|
||||
if (status === ATT_NOT_FOUND) break; // the normal end of the table
|
||||
if (status) throw new KeyboardClientError(`ATT Find Information: ${status}`);
|
||||
const width = r.data[0] === 1 ? 4 : r.data[0] === 2 ? 18 : 0;
|
||||
if (!width || r.data.length <= 1 || (r.data.length - 1) % width) {
|
||||
throw new KeyboardClientError("Malformed Find Information response");
|
||||
}
|
||||
for (let i = 1; i < r.data.length; i += width) {
|
||||
const handle = u16(r.data, i);
|
||||
if (handle < start || attrs.length >= MAX_ATTRIBUTES) throw new KeyboardClientError("Invalid or excessive GATT handles");
|
||||
attrs.push({ handle, uuid: attributeUuid(r.data.subarray(i + 2, i + width)) });
|
||||
start = handle + 1;
|
||||
}
|
||||
}
|
||||
return attrs;
|
||||
}
|
||||
|
||||
/**
|
||||
* The properties byte of a characteristic, from its declaration, or
|
||||
* undefined when the keyboard won't say. A GATT client normally learns
|
||||
* declarations through Read By Type, which the bridge doesn't relay, so
|
||||
* this reads the declaration attribute itself. Most stacks answer that
|
||||
* with the full declaration. Some keep only the properties byte there and
|
||||
* build the rest for Read By Type alone (older Telink SDKs, common in
|
||||
* small keyboards), and some refuse the read. None of that is a reason to
|
||||
* turn a keyboard away: the report's handle and UUID are already known.
|
||||
*/
|
||||
private async declaredProperties(attrs: readonly Attribute[], a: Attribute): Promise<number | undefined> {
|
||||
const decl = attrs.filter((d) => d.handle < a.handle && d.uuid === UUID_CHARACTERISTIC).pop();
|
||||
if (!decl) return undefined;
|
||||
let declaration: Uint8Array;
|
||||
try {
|
||||
declaration = await this.shortRead(decl.handle);
|
||||
} catch (e) {
|
||||
const status = attRefusal(e);
|
||||
this.events.log?.(`declaration ${decl.handle} of ${a.handle}: read refused (ATT ${status})`);
|
||||
return undefined;
|
||||
}
|
||||
if (declaration.length === 5 || declaration.length === 19) {
|
||||
if (u16(declaration, 1) === a.handle && attributeUuid(declaration.subarray(3)) === a.uuid) return declaration[0]!;
|
||||
this.events.log?.(`declaration ${decl.handle} of ${a.handle}: describes something else (${hex(declaration)})`);
|
||||
return undefined;
|
||||
}
|
||||
if (declaration.length === 1) return declaration[0]!;
|
||||
this.events.log?.(`declaration ${decl.handle} of ${a.handle}: unexpected value (${hex(declaration)})`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover every HID service, its Report Map and each report's Report
|
||||
* Reference; select Report Protocol and subscribe the input reports that
|
||||
* carry keys or a pointer's buttons and movement. Needs an encrypted link.
|
||||
* No handle is assumed.
|
||||
*
|
||||
* Only what input needs is required of the device: a Report Map that
|
||||
* parses, and at least one such input report with a Report Reference and
|
||||
* a CCC. A report that can't be placed is skipped and logged, as is
|
||||
* anything wrong with an external report (the battery level, usually).
|
||||
*/
|
||||
async configure(): Promise<KeyboardConfiguration> {
|
||||
const epoch = this.epoch;
|
||||
const log = (line: string): void => this.events.log?.(line);
|
||||
// A new discovery replaces any earlier mapping, whether or not it succeeds.
|
||||
this.release();
|
||||
this.inputs.clear();
|
||||
this.configuration = undefined;
|
||||
const attrs = await this.attributes();
|
||||
log(`gatt ${attrs.map((a) => `${a.handle}=${typeof a.uuid === "number" ? a.uuid.toString(16) : a.uuid}`).join(" ")}`);
|
||||
const reports: KeyboardReport[] = [];
|
||||
const watched: KeyboardReport[] = [];
|
||||
let pointerFound = false;
|
||||
const skipped: string[] = [];
|
||||
const descriptorsFor = (handle: number): Attribute[] => {
|
||||
const end = attrs.find((a) => a.handle > handle &&
|
||||
(a.uuid === UUID_PRIMARY_SERVICE || a.uuid === UUID_SECONDARY_SERVICE || a.uuid === UUID_CHARACTERISTIC))?.handle ?? 65536;
|
||||
return attrs.filter((a) => a.handle > handle && a.handle < end);
|
||||
};
|
||||
/** Why a report was left out; the link going away is not a reason to carry on. */
|
||||
const skip = (a: Attribute, reason: string): void => {
|
||||
skipped.push(`${a.handle}: ${reason}`);
|
||||
log(`report ${a.handle} skipped: ${reason}`);
|
||||
};
|
||||
const addReport = async (service: number, a: Attribute, map: Uint8Array, parsed: Map<number, HidReport>) => {
|
||||
if (reports.some((r) => r.handle === a.handle)) return skip(a, "already listed by another HID service");
|
||||
const descriptors = descriptorsFor(a.handle);
|
||||
const ref = descriptors.find((d) => d.uuid === UUID_REPORT_REFERENCE);
|
||||
if (!ref) return skip(a, "no Report Reference descriptor");
|
||||
let reference: Uint8Array;
|
||||
try {
|
||||
reference = await this.shortRead(ref.handle);
|
||||
} catch (e) {
|
||||
return skip(a, `Report Reference ${ref.handle} unreadable (ATT ${attRefusal(e)})`);
|
||||
}
|
||||
if (reference.length !== 2) return skip(a, `Report Reference ${ref.handle} is ${hex(reference) || "empty"}`);
|
||||
const id = reference[0]!;
|
||||
const type = reference[1]!;
|
||||
if (type < 1 || type > 3) return skip(a, `Report Reference type ${type}`);
|
||||
const ccc = descriptors.find((d) => d.uuid === UUID_CCC)?.handle;
|
||||
const properties = await this.declaredProperties(attrs, a);
|
||||
const report: KeyboardReport = { service, handle: a.handle, id, type, ccc, properties, map };
|
||||
if (type === 1) {
|
||||
const layout = parsed.get(id);
|
||||
if (!layout) return skip(a, `input report id ${id} is not in the Report Map`);
|
||||
if (ccc === undefined || (properties !== undefined && !(properties & 0x10))) return skip(a, "input report can't notify");
|
||||
const keys = reportHasKeys(layout);
|
||||
const pointer = reportHasPointer(layout);
|
||||
if (keys || pointer) watched.push(report);
|
||||
else log(`report ${a.handle} (id ${id}) carries no keys or pointer; not subscribed`);
|
||||
if (pointer) {
|
||||
pointerFound = true;
|
||||
log(`report ${a.handle} (id ${id}) is a pointer's${keys ? ", with keys" : ""}`);
|
||||
}
|
||||
}
|
||||
reports.push(report);
|
||||
log(`report ${a.handle}: id ${id} type ${type} ccc ${ccc ?? "-"} properties ${properties ?? "?"}`);
|
||||
};
|
||||
for (let i = 0; i < attrs.length; i++) {
|
||||
const service = attrs[i]!;
|
||||
if (service.uuid !== UUID_PRIMARY_SERVICE) continue;
|
||||
if (attributeUuid(await this.shortRead(service.handle)) !== UUID_HID_SERVICE) continue;
|
||||
const end = attrs.slice(i + 1).find((a) => a.uuid === UUID_PRIMARY_SERVICE || a.uuid === UUID_SECONDARY_SERVICE)?.handle ?? 65536;
|
||||
const members = attrs.filter((a) => a.handle > service.handle && a.handle < end);
|
||||
const mapAttr = members.find((a) => a.uuid === UUID_REPORT_MAP);
|
||||
if (!mapAttr) throw new KeyboardClientError(`HID service ${service.handle} has no Report Map`);
|
||||
const map = await this.read(mapAttr.handle);
|
||||
log(`report map ${mapAttr.handle}: ${hex(map)}`);
|
||||
const parsed = parseReportMap(map);
|
||||
for (const mode of members.filter((a) => a.uuid === UUID_PROTOCOL_MODE)) {
|
||||
try {
|
||||
await this.write(mode.handle, Uint8Array.of(1), true);
|
||||
} catch (e) {
|
||||
// Report Protocol is the default after a connection; a keyboard
|
||||
// that won't take the write is almost certainly in it already.
|
||||
log(`protocol mode ${mode.handle}: write refused (ATT ${attRefusal(e)})`);
|
||||
}
|
||||
}
|
||||
for (const a of members.filter((m) => m.uuid === UUID_REPORT)) await addReport(service.handle, a, map, parsed);
|
||||
// HOGP lets a Report Map refer to reports in other services (commonly Battery).
|
||||
for (const ext of descriptorsFor(mapAttr.handle).filter((a) => a.uuid === UUID_EXTERNAL_REPORT_REFERENCE)) {
|
||||
let value: Uint8Array;
|
||||
try {
|
||||
value = await this.shortRead(ext.handle);
|
||||
} catch (e) {
|
||||
log(`external report reference ${ext.handle} skipped: unreadable (ATT ${attRefusal(e)})`);
|
||||
continue;
|
||||
}
|
||||
if (value.length !== 2 && value.length !== 16) {
|
||||
log(`external report reference ${ext.handle} skipped: value ${hex(value) || "empty"}`);
|
||||
continue;
|
||||
}
|
||||
const uuid = attributeUuid(value);
|
||||
for (const a of attrs.filter((m) => m.uuid === uuid && (m.handle < service.handle || m.handle >= end))) {
|
||||
await addReport(service.handle, a, map, parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.closed || this.epoch !== epoch) throw new KeyboardClientError("The keyboard link changed during setup");
|
||||
if (!watched.length) {
|
||||
throw new KeyboardClientError(reports.length || skipped.length
|
||||
? `No usable keyboard input report (${skipped.join("; ") || "none carries keys or a pointer"})`
|
||||
: "This device has no HID service");
|
||||
}
|
||||
if (watched.length > MAX_WATCHES) throw new KeyboardClientError("Keyboard has more than 16 input reports");
|
||||
const changed = attrs.find((a) => a.uuid === UUID_SERVICE_CHANGED);
|
||||
const changedCcc = changed && descriptorsFor(changed.handle).find((a) => a.uuid === UUID_CCC);
|
||||
const watches = watched.map((r) => r.handle);
|
||||
if (changed && changedCcc && watches.length < MAX_WATCHES) watches.push(changed.handle);
|
||||
const serviceChanged = changed && changedCcc && watches.includes(changed.handle) ? changed.handle : undefined;
|
||||
const config: KeyboardConfiguration = {
|
||||
epoch: this.epoch, reports, watched: watched.map((r) => r.handle), pointer: pointerFound, serviceChanged,
|
||||
};
|
||||
this.restore(config);
|
||||
try {
|
||||
await this.command(KeyboardOp.watch, [watches.length, ...watches.flatMap(le16)]);
|
||||
if (serviceChanged !== undefined) {
|
||||
try {
|
||||
await this.write(changedCcc!.handle, Uint8Array.of(2, 0));
|
||||
} catch (e) {
|
||||
log(`service changed ${changedCcc!.handle}: subscription refused (ATT ${attRefusal(e)})`);
|
||||
}
|
||||
}
|
||||
for (const r of watched) await this.write(r.ccc!, Uint8Array.of(1, 0));
|
||||
} catch (e) {
|
||||
this.inputs.clear();
|
||||
this.configuration = undefined;
|
||||
this.release();
|
||||
throw e;
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Install a mapping found by configure(). It is only good while the glasses
|
||||
* keep the same connection (the same epoch): after a keyboard reconnect or
|
||||
* a firmware boot, discover again.
|
||||
*/
|
||||
restore(config: KeyboardConfiguration): void {
|
||||
if (!config.reports.length || config.reports.length > 64) throw new KeyboardClientError("Invalid keyboard configuration");
|
||||
const inputs = new Map<number, Input>();
|
||||
for (const r of config.reports) {
|
||||
if (r.type !== 1 || !config.watched.includes(r.handle)) continue;
|
||||
if (!Number.isInteger(r.handle) || r.handle < 1 || r.handle > 65535 || inputs.has(r.handle)) {
|
||||
throw new KeyboardClientError("Invalid restored Report handle");
|
||||
}
|
||||
if (!r.map.length || r.map.length > MAX_VALUE) throw new KeyboardClientError("Invalid restored Report Map");
|
||||
const report = parseReportMap(r.map).get(r.id);
|
||||
if (!report) throw new KeyboardClientError("Restored Report ID missing");
|
||||
inputs.set(r.handle, { report, id: r.id, service: r.service, keys: reportHasKeys(report), pointer: reportHasPointer(report) });
|
||||
}
|
||||
if (!inputs.size || inputs.size > MAX_WATCHES) throw new KeyboardClientError("Invalid restored input count");
|
||||
this.release();
|
||||
this.inputs = inputs;
|
||||
this.configuration = config;
|
||||
}
|
||||
|
||||
/** Output reports of the configured keyboard (the LED report is the usual one). */
|
||||
outputReports(): KeyboardReport[] {
|
||||
return this.configuration?.reports.filter((r) => r.type === 2) ?? [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The ATT error code when `error` is the keyboard refusing a read or write.
|
||||
* Anything else (the link dropping, a timeout, a stream gap) is rethrown:
|
||||
* those end the setup, where a refusal only costs the one attribute.
|
||||
*/
|
||||
function attRefusal(error: unknown): number {
|
||||
if (error instanceof KeyboardClientError && error.attStatus !== undefined) return error.attStatus;
|
||||
throw error;
|
||||
}
|
||||
|
||||
function hex(bytes: Uint8Array): string {
|
||||
return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
||||
}
|
||||
|
||||
function opName(op: number): string {
|
||||
for (const [name, value] of Object.entries(KeyboardOp)) if (value === op) return name;
|
||||
return String(op);
|
||||
}
|
||||
|
||||
function messageOf(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* Bounded HID report-descriptor decoder for the keyboard bridge: turns a
|
||||
* keyboard's Report Map into the layout of each input report, and input
|
||||
* reports into the set of keys held. From the reference client in g2flash
|
||||
* demos/ (keyboard-hid.ts).
|
||||
*
|
||||
* HID over GATT carries the Report ID out of band (in each characteristic's
|
||||
* Report Reference descriptor), so byte zero of a report value is data: it is
|
||||
* never stripped as USB HID would. The keyboard/keypad page (7) and the
|
||||
* consumer page (12) produce keys. A mouse's or touchpad's report (buttons,
|
||||
* relative X/Y, wheel) is read by reportPointer.
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
|
||||
export type HidUsage = { page: number; usage: number };
|
||||
export type HidKeyEvent = HidUsage & { down: boolean };
|
||||
|
||||
type Global = { page: number; min: number; max: number; size: number; count: number; id: number };
|
||||
type Field = {
|
||||
bit: number;
|
||||
size: number;
|
||||
count: number;
|
||||
variable: boolean;
|
||||
/** Reports a change since the last report (a mouse's axes and wheel), not a state. */
|
||||
relative: boolean;
|
||||
min: number;
|
||||
max: number;
|
||||
usages: HidUsage[];
|
||||
range?: [HidUsage, HidUsage];
|
||||
};
|
||||
export type HidReport = { id: number; bits: number; fields: Field[] };
|
||||
|
||||
const HID_PAGE_KEYBOARD = 7;
|
||||
const HID_PAGE_CONSUMER = 12;
|
||||
const REPORT_MAP_MAX = 512;
|
||||
|
||||
function fail(reason: string): never {
|
||||
throw new Error(`Unsupported or malformed HID descriptor: ${reason}`);
|
||||
}
|
||||
|
||||
const usageKey = (u: HidUsage): string => `${u.page}:${u.usage}`;
|
||||
|
||||
/** Input reports by Report ID (0 when the descriptor uses none). Throws on anything out of bounds. */
|
||||
export function parseReportMap(bytes: Uint8Array): Map<number, HidReport> {
|
||||
if (!bytes.length || bytes.length > REPORT_MAP_MAX) fail("report map must contain 1..512 bytes");
|
||||
const reports = new Map<number, HidReport>();
|
||||
let g: Global = { page: 0, min: 0, max: 0, size: 0, count: 0, id: 0 };
|
||||
const stack: Global[] = [];
|
||||
let usages: HidUsage[] = [];
|
||||
let lo: HidUsage | undefined;
|
||||
let hi: HidUsage | undefined;
|
||||
let depth = 0;
|
||||
const usage = (v: number, n: number): HidUsage =>
|
||||
n === 4 ? { page: v >>> 16, usage: v & 65535 } : { page: g.page, usage: v };
|
||||
for (let p = 0; p < bytes.length;) {
|
||||
const prefix = bytes[p++]!;
|
||||
if (prefix === 0xfe) fail("long items");
|
||||
const n = [0, 1, 2, 4][prefix & 3]!;
|
||||
if (p + n > bytes.length) fail("truncated item");
|
||||
let v = 0;
|
||||
for (let j = 0; j < n; j++) v += bytes[p++]! * 2 ** (8 * j);
|
||||
const signed = n && v >= 2 ** (n * 8 - 1) ? v - 2 ** (n * 8) : v;
|
||||
const type = (prefix >> 2) & 3;
|
||||
const tag = prefix >> 4;
|
||||
if (type === 1) {
|
||||
switch (tag) {
|
||||
case 0: g.page = v; break;
|
||||
case 1: g.min = signed; break;
|
||||
case 2: g.max = g.min < 0 ? signed : v; break;
|
||||
case 7: g.size = v; break;
|
||||
case 8: if (!v || v > 255) fail("Report ID"); g.id = v; break;
|
||||
case 9: g.count = v; break;
|
||||
case 10: if (stack.length >= 8) fail("global stack overflow"); stack.push({ ...g }); break;
|
||||
case 11: if (!stack.length) fail("global stack underflow"); g = stack.pop()!; break;
|
||||
case 3: case 4: case 5: case 6: break; // physical range, unit and exponent
|
||||
default: fail("unknown global item");
|
||||
}
|
||||
} else if (type === 2) {
|
||||
if (tag === 0) { if (usages.length >= 512) fail("too many usages"); usages.push(usage(v, n)); }
|
||||
else if (tag === 1) lo = usage(v, n);
|
||||
else if (tag === 2) hi = usage(v, n);
|
||||
else if (tag === 10) fail("usage delimiters");
|
||||
else if (tag > 9) fail("unknown local item");
|
||||
} else if (type === 0) {
|
||||
if (tag === 10) { if (++depth > 32) fail("collection depth"); }
|
||||
else if (tag === 12) { if (--depth < 0) fail("collection underflow"); }
|
||||
else if (tag === 8) {
|
||||
if (!g.size || g.size > 32 || !g.count || g.count > 512) fail("field dimensions");
|
||||
let r = reports.get(g.id);
|
||||
if (!r) {
|
||||
if (reports.size >= 16) fail("too many input reports");
|
||||
r = { id: g.id, bits: 0, fields: [] };
|
||||
reports.set(g.id, r);
|
||||
}
|
||||
const field: Field = {
|
||||
bit: r.bits, size: g.size, count: g.count, variable: !!(v & 2), relative: !!(v & 4),
|
||||
min: g.min, max: g.max, usages: [...usages],
|
||||
};
|
||||
if (lo || hi) {
|
||||
if (!lo || !hi || lo.page !== hi.page || hi.usage < lo.usage) fail("usage range");
|
||||
field.range = [lo!, hi!];
|
||||
}
|
||||
r.bits += g.size * g.count;
|
||||
if (r.bits > 4096) fail("report exceeds 512 bytes");
|
||||
// Constant fields are padding.
|
||||
if (!(v & 1)) r.fields.push(field);
|
||||
} else if (tag !== 9 && tag !== 11) fail("unknown main item");
|
||||
usages = [];
|
||||
lo = hi = undefined;
|
||||
} else fail("reserved item type");
|
||||
}
|
||||
if (depth || stack.length) fail("unbalanced collection/global stack");
|
||||
if (reports.has(0) && reports.size > 1) fail("mixed numbered and unnumbered input reports");
|
||||
if (!reports.size) fail("no input reports");
|
||||
return reports;
|
||||
}
|
||||
|
||||
function usageAt(f: Field, index: number): HidUsage | undefined {
|
||||
if (index < 0) return undefined;
|
||||
if (index < f.usages.length) return f.usages[index];
|
||||
const rangeIndex = index - f.usages.length;
|
||||
if (f.range && rangeIndex <= f.range[1].usage - f.range[0].usage) {
|
||||
return { page: f.range[0].page, usage: f.range[0].usage + rangeIndex };
|
||||
}
|
||||
// Variable items reuse their last usage when Report Count exceeds the usages.
|
||||
if (f.variable) return f.range?.[1] ?? f.usages[f.usages.length - 1];
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an input report can ever say a key is held. False for a pointer's
|
||||
* report (a trackpad or mouse on a combined device), whose movement would
|
||||
* otherwise be relayed, and thrown away, a hundred times a second.
|
||||
*/
|
||||
export function reportHasKeys(report: HidReport): boolean {
|
||||
const isKeyPage = (u: HidUsage): boolean => u.page === HID_PAGE_KEYBOARD || u.page === HID_PAGE_CONSUMER;
|
||||
return report.fields.some((f) => !f.relative && (f.usages.some(isKeyPage) || (f.range !== undefined && isKeyPage(f.range[0]))));
|
||||
}
|
||||
|
||||
/** One field's `index`th value, sign-extended when the field is signed. Bits past the data read as zero. */
|
||||
function fieldValue(f: Field, index: number, data: Uint8Array): number {
|
||||
let value = 0;
|
||||
for (let j = 0; j < f.size; j++) {
|
||||
const bit = f.bit + index * f.size + j;
|
||||
if ((data[bit >>> 3] ?? 0) & (1 << (bit & 7))) value += 2 ** j;
|
||||
}
|
||||
if (f.min < 0 && value >= 2 ** (f.size - 1)) value -= 2 ** f.size;
|
||||
return value;
|
||||
}
|
||||
|
||||
/** What one report from a mouse or touchpad says. */
|
||||
export type HidPointerSample = {
|
||||
/** Buttons held, bit 0 for button 1 (left), bit 1 for button 2 (right), bit 2 for button 3 (middle). */
|
||||
buttons: number;
|
||||
/** Movement since the last report, in the device's counts; +x right, +y down. */
|
||||
dx: number;
|
||||
dy: number;
|
||||
/** Wheel detents since the last report; positive is away from the user ("up"). */
|
||||
wheel: number;
|
||||
/** Horizontal wheel (AC Pan); positive is right. */
|
||||
pan: number;
|
||||
};
|
||||
|
||||
const HID_PAGE_GENERIC_DESKTOP = 1;
|
||||
const HID_PAGE_BUTTON = 9;
|
||||
const USAGE_X = 0x30;
|
||||
const USAGE_Y = 0x31;
|
||||
const USAGE_WHEEL = 0x38;
|
||||
const USAGE_AC_PAN = 0x238;
|
||||
/** Buttons past this are not tracked; a mouse with more is rare and has nothing to bind them to anyway. */
|
||||
const MAX_POINTER_BUTTONS = 8;
|
||||
|
||||
/** Which part of a pointer sample a field's usage feeds, if any. */
|
||||
function pointerRole(f: Field, u: HidUsage | undefined): "button" | "dx" | "dy" | "wheel" | "pan" | null {
|
||||
if (!u || !f.variable) return null;
|
||||
if (u.page === HID_PAGE_BUTTON) return !f.relative && u.usage >= 1 && u.usage <= MAX_POINTER_BUTTONS ? "button" : null;
|
||||
if (!f.relative) return null; // an absolute X/Y is a touchscreen or a digitizer, not a mouse
|
||||
if (u.page === HID_PAGE_GENERIC_DESKTOP) {
|
||||
if (u.usage === USAGE_X) return "dx";
|
||||
if (u.usage === USAGE_Y) return "dy";
|
||||
if (u.usage === USAGE_WHEEL) return "wheel";
|
||||
}
|
||||
if (u.page === HID_PAGE_CONSUMER && u.usage === USAGE_AC_PAN) return "pan";
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Whether an input report carries a mouse's buttons, movement or wheel. */
|
||||
export function reportHasPointer(report: HidReport): boolean {
|
||||
return report.fields.some((f) => {
|
||||
for (let i = 0; i < f.count; i++) if (pointerRole(f, usageAt(f, i))) return true;
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a mouse or touchpad report. Lenient about length, unlike reportKeys:
|
||||
* some devices leave the wheel bytes off when there is nothing to say, and a
|
||||
* missing byte is zero either way.
|
||||
*/
|
||||
export function reportPointer(report: HidReport, data: Uint8Array): HidPointerSample {
|
||||
const sample: HidPointerSample = { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 };
|
||||
for (const f of report.fields) {
|
||||
for (let i = 0; i < f.count; i++) {
|
||||
const u = usageAt(f, i);
|
||||
const role = pointerRole(f, u);
|
||||
if (!role) continue;
|
||||
const value = fieldValue(f, i, data);
|
||||
if (value < f.min || value > f.max) continue; // out of range: no reading
|
||||
if (role === "button") {
|
||||
if (value) sample.buttons |= 1 << (u!.usage - 1);
|
||||
} else {
|
||||
sample[role] += value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return sample;
|
||||
}
|
||||
|
||||
/** The keys one input report says are held. Throws on a length mismatch or a rollover report. */
|
||||
export function reportKeys(report: HidReport, data: Uint8Array): HidUsage[] {
|
||||
if (data.length !== Math.ceil(report.bits / 8)) throw new Error("HID input length does not match Report Map");
|
||||
const keys = new Map<string, HidUsage>();
|
||||
for (const f of report.fields) {
|
||||
// A relative field is a movement, not a key that is down.
|
||||
if (f.relative) continue;
|
||||
for (let i = 0; i < f.count; i++) {
|
||||
const value = fieldValue(f, i, data);
|
||||
if (value < f.min || value > f.max) continue; // null state
|
||||
const u = usageAt(f, f.variable ? i : value - f.min);
|
||||
if (!u || (f.variable && value === 0) || !u.usage) continue;
|
||||
if (u.page === HID_PAGE_KEYBOARD && u.usage <= 3) throw new Error("HID keyboard rollover/error report");
|
||||
if (u.page === HID_PAGE_KEYBOARD || u.page === HID_PAGE_CONSUMER) keys.set(usageKey(u), u);
|
||||
}
|
||||
}
|
||||
return [...keys.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Held keys across a keyboard's input reports. Ownership is per report
|
||||
* characteristic, then unioned: releasing a key in one report cannot release
|
||||
* a modifier another report still holds.
|
||||
*/
|
||||
export class HidKeys {
|
||||
private reports = new Map<number, HidUsage[]>();
|
||||
private held = new Map<string, HidUsage>();
|
||||
|
||||
/** Apply one report from characteristic `handle`; returns the transitions it caused. */
|
||||
update(handle: number, report: HidReport, data: Uint8Array): HidKeyEvent[] {
|
||||
this.reports.set(handle, reportKeys(report, data));
|
||||
return this.diff();
|
||||
}
|
||||
|
||||
/** Release everything (a stream gap, a dropped link). */
|
||||
reset(): HidKeyEvent[] {
|
||||
this.reports.clear();
|
||||
return this.diff();
|
||||
}
|
||||
|
||||
private diff(): HidKeyEvent[] {
|
||||
const next = new Map<string, HidUsage>();
|
||||
for (const values of this.reports.values()) for (const u of values) next.set(usageKey(u), u);
|
||||
const events: HidKeyEvent[] = [];
|
||||
for (const [k, u] of this.held) if (!next.has(k)) events.push({ ...u, down: false });
|
||||
for (const [k, u] of next) if (!this.held.has(k)) events.push({ ...u, down: true });
|
||||
this.held = next;
|
||||
return events;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Where a key from the hardware keyboard goes: to a keybinding being set up,
|
||||
* to a gesture it is bound to, to the window with input focus, or to the
|
||||
* built-in navigation. The meanings are in bindings.ts; this is the order of
|
||||
* asking, and the bookkeeping around it. Shared by the Android and iOS
|
||||
* controllers through KeyRouterHost.
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import {
|
||||
GestureScheme,
|
||||
chordOf,
|
||||
isModifierCode,
|
||||
resolveKey,
|
||||
type GestureOrigin,
|
||||
type KeyAction,
|
||||
type KeyChord,
|
||||
type KeyGesture,
|
||||
} from "./bindings";
|
||||
import type { KeyEvent } from "./keys";
|
||||
|
||||
export type KeyRouterHost = {
|
||||
/** Connected to glasses that are in use. */
|
||||
ready(): boolean;
|
||||
locked(): boolean;
|
||||
screenOn(): boolean;
|
||||
/** Turn a dark display on, and nothing else. */
|
||||
wake(): void;
|
||||
/** Offer a key to the window with input focus; true when that window takes keys. */
|
||||
offerToWindow(event: KeyEvent): boolean;
|
||||
/** Perform a gesture as if it came from the ring or the watch. */
|
||||
gesture(gesture: KeyGesture, origin: GestureOrigin): Promise<void>;
|
||||
/** The custom keybindings, by chord id. */
|
||||
bindings(): ReadonlyMap<string, KeyAction>;
|
||||
/** A key was used for something other than a gesture or typing (count it against the screen timeout). */
|
||||
noteActivity(): void;
|
||||
log(line: string): void;
|
||||
now?(): number;
|
||||
};
|
||||
|
||||
/**
|
||||
* A held key repeats twenty times a second, and a wheel or a two-finger
|
||||
* scroll reports faster still: more than a list can repaint over the
|
||||
* glasses' link. Those are dropped while a gesture is still being handled,
|
||||
* and closer together than this.
|
||||
*/
|
||||
const REPEAT_MIN_INTERVAL_MS = 100;
|
||||
|
||||
export class KeyRouter {
|
||||
private capture: ((chord: KeyChord) => void) | null = null;
|
||||
/** Keys whose press was spoken for, so their release is too. */
|
||||
private readonly consumed = new Set<string>();
|
||||
private readonly scheme = new GestureScheme();
|
||||
private gesturesInFlight = 0;
|
||||
private lastGestureAtMs = 0;
|
||||
|
||||
constructor(private readonly host: KeyRouterHost) {}
|
||||
|
||||
/** One key transition from the keyboard. */
|
||||
handleKey(event: KeyEvent): void {
|
||||
const host = this.host;
|
||||
if (!host.ready()) return;
|
||||
if (host.locked() || !host.screenOn()) {
|
||||
// A dark display takes no typing: the first press only wakes it, as
|
||||
// the ring's double tap would, and a locked one stays locked.
|
||||
if (event.phase === "down" && !event.repeat && !host.screenOn()) host.wake();
|
||||
return;
|
||||
}
|
||||
if (this.capture) {
|
||||
this.captureKey(event);
|
||||
return;
|
||||
}
|
||||
if (event.phase === "up") {
|
||||
if (!this.consumed.delete(event.code)) host.offerToWindow(event);
|
||||
return;
|
||||
}
|
||||
const resolution = resolveKey(event, host.bindings());
|
||||
const droppable = event.repeat || event.burst === true;
|
||||
if (resolution.consume) {
|
||||
this.consumed.add(event.code);
|
||||
if (resolution.gesture) this.perform(resolution.gesture, droppable);
|
||||
return;
|
||||
}
|
||||
if (host.offerToWindow(event)) return;
|
||||
if (resolution.gesture) this.perform(resolution.gesture, droppable);
|
||||
}
|
||||
|
||||
/**
|
||||
* A key the focused window was offered and had no use for (a terminal
|
||||
* showing its session list rather than a session): it gets the built-in
|
||||
* navigation meaning after all.
|
||||
*/
|
||||
handleUnhandledKey(event: KeyEvent): void {
|
||||
const host = this.host;
|
||||
if (event.phase !== "down" || this.capture || !host.ready() || host.locked() || !host.screenOn()) return;
|
||||
const resolution = resolveKey(event, host.bindings());
|
||||
if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat || event.burst === true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take the next key press instead of acting on it (setting up a
|
||||
* keybinding). Modifiers held with it are part of the chord; pressed alone
|
||||
* they are waited through. Returns a function that calls the capture off.
|
||||
*/
|
||||
captureNextKey(onChord: (chord: KeyChord) => void): () => void {
|
||||
this.capture = onChord;
|
||||
return () => {
|
||||
if (this.capture === onChord) this.capture = null;
|
||||
};
|
||||
}
|
||||
|
||||
/** The keyboard went away: nothing is held any more. */
|
||||
reset(): void {
|
||||
this.consumed.clear();
|
||||
}
|
||||
|
||||
private captureKey(event: KeyEvent): void {
|
||||
this.host.noteActivity();
|
||||
if (event.phase === "up") {
|
||||
this.consumed.delete(event.code);
|
||||
return;
|
||||
}
|
||||
if (event.repeat || isModifierCode(event.code)) return;
|
||||
const capture = this.capture!;
|
||||
this.capture = null;
|
||||
this.consumed.add(event.code);
|
||||
capture(chordOf(event));
|
||||
}
|
||||
|
||||
private perform(gesture: KeyGesture, droppable: boolean): void {
|
||||
const now = this.host.now ? this.host.now() : Date.now();
|
||||
if (droppable && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return;
|
||||
this.lastGestureAtMs = now;
|
||||
void this.run(gesture, this.scheme.originFor(gesture));
|
||||
}
|
||||
|
||||
private async run(gesture: KeyGesture, origin: GestureOrigin): Promise<void> {
|
||||
this.gesturesInFlight++;
|
||||
try {
|
||||
await this.host.gesture(gesture, origin);
|
||||
} catch (error) {
|
||||
this.host.log(`keyboard gesture ${gesture} failed: ${error instanceof Error ? error.message : String(error)}`);
|
||||
} finally {
|
||||
this.gesturesInFlight--;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
/**
|
||||
* Key events as the rest of Faceclaw sees them, and the translation from the
|
||||
* keyboard bridge's raw HID transitions: layout (US only for now), modifier
|
||||
* and Caps Lock state, and typematic repeat, which a BLE keyboard never sends
|
||||
* (its report just stays asserted). The translation tables follow the Glassly
|
||||
* app's (GlassesKeyboardInput.ts, MIT).
|
||||
* No platform imports: this runs under node in the tests, and in app workers.
|
||||
*/
|
||||
import type { HidKeyEvent } from "./hid";
|
||||
|
||||
export type KeyModifiers = { shift: boolean; ctrl: boolean; alt: boolean; meta: boolean };
|
||||
|
||||
/**
|
||||
* One key transition, shaped like a DOM KeyboardEvent: `key` is the logical
|
||||
* key with the layout applied ("a", "A", "Enter", "ArrowUp"), `code` the
|
||||
* physical key ("KeyA"), `text` what a press types (absent for releases, for
|
||||
* keys that type nothing, and for Ctrl/Alt/Meta chords, which are commands).
|
||||
*/
|
||||
export type KeyEvent = {
|
||||
phase: "down" | "up";
|
||||
key: string;
|
||||
code: string;
|
||||
text?: string;
|
||||
modifiers: KeyModifiers;
|
||||
/** A synthesized auto-repeat of a held key. */
|
||||
repeat: boolean;
|
||||
/**
|
||||
* One of a stream that can arrive faster than it can be acted on (a wheel
|
||||
* turning): whoever acts on it may drop some rather than fall behind.
|
||||
*/
|
||||
burst?: boolean;
|
||||
/** Date.now() when the phone saw the transition. */
|
||||
timestampMs: number;
|
||||
};
|
||||
|
||||
const HID_PAGE_KEYBOARD = 7;
|
||||
const HID_PAGE_CONSUMER = 12;
|
||||
/** A mouse's buttons, which arrive here as keys: usage 1 is the left button, 2 the right, 3 the middle. */
|
||||
export const HID_PAGE_BUTTON = 9;
|
||||
const USAGE_CAPS_LOCK = 57;
|
||||
|
||||
/** [unshifted, shifted, code] for US-layout punctuation on the keyboard page. */
|
||||
const PUNCTUATION: Record<number, [string, string, string]> = {
|
||||
45: ["-", "_", "Minus"],
|
||||
46: ["=", "+", "Equal"],
|
||||
47: ["[", "{", "BracketLeft"],
|
||||
48: ["]", "}", "BracketRight"],
|
||||
49: ["\\", "|", "Backslash"],
|
||||
50: ["#", "~", "IntlHash"],
|
||||
51: [";", ":", "Semicolon"],
|
||||
52: ["'", '"', "Quote"],
|
||||
53: ["`", "~", "Backquote"],
|
||||
54: [",", "<", "Comma"],
|
||||
55: [".", ">", "Period"],
|
||||
56: ["/", "?", "Slash"],
|
||||
100: ["\\", "|", "IntlBackslash"],
|
||||
};
|
||||
|
||||
const DIGITS = "1234567890";
|
||||
const SHIFTED_DIGITS = "!@#$%^&*()";
|
||||
|
||||
/** [key, code] for keys that type nothing. */
|
||||
const NAMED: Record<number, [string, string]> = {
|
||||
40: ["Enter", "Enter"],
|
||||
41: ["Escape", "Escape"],
|
||||
42: ["Backspace", "Backspace"],
|
||||
43: ["Tab", "Tab"],
|
||||
57: ["CapsLock", "CapsLock"],
|
||||
70: ["PrintScreen", "PrintScreen"],
|
||||
71: ["ScrollLock", "ScrollLock"],
|
||||
72: ["Pause", "Pause"],
|
||||
73: ["Insert", "Insert"],
|
||||
74: ["Home", "Home"],
|
||||
75: ["PageUp", "PageUp"],
|
||||
76: ["Delete", "Delete"],
|
||||
77: ["End", "End"],
|
||||
78: ["PageDown", "PageDown"],
|
||||
79: ["ArrowRight", "ArrowRight"],
|
||||
80: ["ArrowLeft", "ArrowLeft"],
|
||||
81: ["ArrowDown", "ArrowDown"],
|
||||
82: ["ArrowUp", "ArrowUp"],
|
||||
83: ["NumLock", "NumLock"],
|
||||
88: ["Enter", "NumpadEnter"],
|
||||
101: ["ContextMenu", "ContextMenu"],
|
||||
102: ["Power", "Power"],
|
||||
224: ["Control", "ControlLeft"],
|
||||
225: ["Shift", "ShiftLeft"],
|
||||
226: ["Alt", "AltLeft"],
|
||||
227: ["Meta", "MetaLeft"],
|
||||
228: ["Control", "ControlRight"],
|
||||
229: ["Shift", "ShiftRight"],
|
||||
230: ["Alt", "AltRight"],
|
||||
231: ["Meta", "MetaRight"],
|
||||
};
|
||||
|
||||
/** [key and text, code] for the numeric keypad's operators. */
|
||||
const NUMPAD: Record<number, [string, string]> = {
|
||||
84: ["/", "NumpadDivide"],
|
||||
85: ["*", "NumpadMultiply"],
|
||||
86: ["-", "NumpadSubtract"],
|
||||
87: ["+", "NumpadAdd"],
|
||||
99: [".", "NumpadDecimal"],
|
||||
103: ["=", "NumpadEqual"],
|
||||
};
|
||||
|
||||
const CONSUMER: Record<number, string> = {
|
||||
0xe9: "AudioVolumeUp",
|
||||
0xea: "AudioVolumeDown",
|
||||
0xe2: "AudioVolumeMute",
|
||||
0xcd: "MediaPlayPause",
|
||||
0xb0: "MediaPlay",
|
||||
0xb1: "MediaPause",
|
||||
0xb5: "MediaTrackNext",
|
||||
0xb6: "MediaTrackPrevious",
|
||||
0xb7: "MediaStop",
|
||||
0x6f: "BrightnessUp",
|
||||
0x70: "BrightnessDown",
|
||||
0x221: "BrowserSearch",
|
||||
0x223: "BrowserHome",
|
||||
0x224: "BrowserBack",
|
||||
0x225: "BrowserForward",
|
||||
0x18a: "LaunchMail",
|
||||
0x192: "LaunchApplication2",
|
||||
0x194: "LaunchApplication1",
|
||||
};
|
||||
|
||||
const MODIFIER_USAGES: Record<number, keyof KeyModifiers> = {
|
||||
224: "ctrl", 228: "ctrl", 225: "shift", 229: "shift", 226: "alt", 230: "alt", 227: "meta", 231: "meta",
|
||||
};
|
||||
|
||||
type Translated = { key: string; code: string; text?: string };
|
||||
|
||||
/** One HID usage under the US layout. Caps Lock only affects letters. */
|
||||
export function translateUsage(page: number, usage: number, shift: boolean, capsLock: boolean): Translated {
|
||||
if (page === HID_PAGE_KEYBOARD) {
|
||||
if (usage >= 4 && usage <= 29) {
|
||||
const lower = String.fromCharCode(97 + usage - 4);
|
||||
const key = shift !== capsLock ? lower.toUpperCase() : lower;
|
||||
return { key, code: `Key${lower.toUpperCase()}`, text: key };
|
||||
}
|
||||
if (usage >= 30 && usage <= 39) {
|
||||
const index = usage - 30;
|
||||
const key = shift ? SHIFTED_DIGITS[index]! : DIGITS[index]!;
|
||||
return { key, code: `Digit${DIGITS[index]}`, text: key };
|
||||
}
|
||||
if (usage === 44) return { key: " ", code: "Space", text: " " };
|
||||
const punctuation = PUNCTUATION[usage];
|
||||
if (punctuation) {
|
||||
const key = shift ? punctuation[1] : punctuation[0];
|
||||
return { key, code: punctuation[2], text: key };
|
||||
}
|
||||
if (usage >= 58 && usage <= 69) return { key: `F${usage - 57}`, code: `F${usage - 57}` };
|
||||
if (usage >= 104 && usage <= 115) return { key: `F${usage - 91}`, code: `F${usage - 91}` };
|
||||
if (usage >= 89 && usage <= 97) {
|
||||
const digit = String(usage - 88);
|
||||
return { key: digit, code: `Numpad${digit}`, text: digit };
|
||||
}
|
||||
if (usage === 98) return { key: "0", code: "Numpad0", text: "0" };
|
||||
const numpad = NUMPAD[usage];
|
||||
if (numpad) return { key: numpad[0], code: numpad[1], text: numpad[0] };
|
||||
const named = NAMED[usage];
|
||||
if (named) return { key: named[0], code: named[1] };
|
||||
return { key: "Unidentified", code: `Usage${usage.toString(16).padStart(2, "0")}` };
|
||||
}
|
||||
if (page === HID_PAGE_CONSUMER) {
|
||||
const name = CONSUMER[usage];
|
||||
return name ? { key: name, code: name } : { key: "Unidentified", code: `Consumer${usage.toString(16)}` };
|
||||
}
|
||||
if (page === HID_PAGE_BUTTON) return { key: `MouseButton${usage}`, code: `MouseButton${usage}` };
|
||||
return { key: "Unidentified", code: `Page${page.toString(16)}Usage${usage.toString(16)}` };
|
||||
}
|
||||
|
||||
export type KeyTranslatorOptions = {
|
||||
emit: (event: KeyEvent) => void;
|
||||
now?: () => number;
|
||||
setTimer?: (callback: () => void, delayMs: number) => unknown;
|
||||
clearTimer?: (handle: unknown) => void;
|
||||
/** Hold time before the first repeat, and the gap between repeats. */
|
||||
repeatDelayMs?: number;
|
||||
repeatIntervalMs?: number;
|
||||
};
|
||||
|
||||
const REPEAT_DELAY_MS = 400;
|
||||
const REPEAT_INTERVAL_MS = 50;
|
||||
|
||||
/**
|
||||
* Raw HID transitions in, KeyEvents out. Modifier state comes from the same
|
||||
* stream, and the bridge reports a release for every key when the stream may
|
||||
* have lost something, so a missed key-up cannot leave a modifier stuck or a
|
||||
* key repeating forever.
|
||||
*/
|
||||
export class KeyTranslator {
|
||||
private readonly emitEvent: (event: KeyEvent) => void;
|
||||
private readonly now: () => number;
|
||||
private readonly setTimer: (callback: () => void, delayMs: number) => unknown;
|
||||
private readonly clearTimer: (handle: unknown) => void;
|
||||
private readonly repeatDelayMs: number;
|
||||
private readonly repeatIntervalMs: number;
|
||||
private readonly held = new Set<string>();
|
||||
private modifiers: KeyModifiers = { shift: false, ctrl: false, alt: false, meta: false };
|
||||
private capsLock = false;
|
||||
private repeatTimer: unknown = null;
|
||||
private repeating: { page: number; usage: number } | null = null;
|
||||
|
||||
constructor(options: KeyTranslatorOptions) {
|
||||
this.emitEvent = options.emit;
|
||||
this.now = options.now ?? (() => Date.now());
|
||||
this.setTimer = options.setTimer ?? ((callback, delayMs) => setTimeout(callback, delayMs));
|
||||
this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as ReturnType<typeof setTimeout>));
|
||||
this.repeatDelayMs = options.repeatDelayMs ?? REPEAT_DELAY_MS;
|
||||
this.repeatIntervalMs = options.repeatIntervalMs ?? REPEAT_INTERVAL_MS;
|
||||
}
|
||||
|
||||
/** Apply the transitions one input report produced, in order. */
|
||||
handle(events: readonly HidKeyEvent[]): void {
|
||||
for (const event of events) this.handleOne(event);
|
||||
}
|
||||
|
||||
private handleOne(event: HidKeyEvent): void {
|
||||
const id = `${event.page}:${event.usage}`;
|
||||
const modifier = event.page === HID_PAGE_KEYBOARD ? MODIFIER_USAGES[event.usage] : undefined;
|
||||
if (event.down) {
|
||||
if (this.held.has(id)) return;
|
||||
this.held.add(id);
|
||||
if (modifier) this.modifiers[modifier] = true;
|
||||
if (event.page === HID_PAGE_KEYBOARD && event.usage === USAGE_CAPS_LOCK) this.capsLock = !this.capsLock;
|
||||
this.emit("down", event.page, event.usage, false);
|
||||
// The newest non-modifier key is the one that repeats, as on a PC. A
|
||||
// mouse button neither repeats nor takes the repeat from a held key.
|
||||
if (!modifier && event.page !== HID_PAGE_BUTTON) this.startRepeat(event.page, event.usage);
|
||||
} else {
|
||||
if (!this.held.delete(id)) return;
|
||||
if (modifier) this.modifiers[modifier] = this.otherHeld(event.usage, modifier);
|
||||
if (this.repeating?.page === event.page && this.repeating.usage === event.usage) this.stopRepeat();
|
||||
this.emit("up", event.page, event.usage, false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A press and release in one, for input that has no "held" (a wheel's
|
||||
* detent): `name` is both the key and the code, e.g. "WheelUp". Carries
|
||||
* the modifiers held on the keyboard, like any key.
|
||||
*/
|
||||
pulse(name: string): void {
|
||||
const timestampMs = this.now();
|
||||
for (const phase of ["down", "up"] as const) {
|
||||
this.emitEvent({ phase, key: name, code: name, modifiers: { ...this.modifiers }, repeat: false, burst: true, timestampMs });
|
||||
}
|
||||
}
|
||||
|
||||
/** The keyboard went away: forget everything without emitting releases. */
|
||||
reset(): void {
|
||||
this.stopRepeat();
|
||||
this.held.clear();
|
||||
this.modifiers = { shift: false, ctrl: false, alt: false, meta: false };
|
||||
this.capsLock = false;
|
||||
}
|
||||
|
||||
/** Whether the same modifier's key on the other side is still down. */
|
||||
private otherHeld(released: number, modifier: keyof KeyModifiers): boolean {
|
||||
for (const [usage, name] of Object.entries(MODIFIER_USAGES)) {
|
||||
if (name === modifier && Number(usage) !== released && this.held.has(`${HID_PAGE_KEYBOARD}:${usage}`)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private emit(phase: "down" | "up", page: number, usage: number, repeat: boolean): void {
|
||||
const translated = translateUsage(page, usage, this.modifiers.shift, this.capsLock);
|
||||
const event: KeyEvent = {
|
||||
phase,
|
||||
key: translated.key,
|
||||
code: translated.code,
|
||||
modifiers: { ...this.modifiers },
|
||||
repeat,
|
||||
timestampMs: this.now(),
|
||||
};
|
||||
if (phase === "down" && translated.text !== undefined &&
|
||||
!this.modifiers.ctrl && !this.modifiers.alt && !this.modifiers.meta) {
|
||||
event.text = translated.text;
|
||||
}
|
||||
this.emitEvent(event);
|
||||
}
|
||||
|
||||
private startRepeat(page: number, usage: number): void {
|
||||
this.stopRepeat();
|
||||
this.repeating = { page, usage };
|
||||
this.repeatTimer = this.setTimer(() => this.repeatTick(), this.repeatDelayMs);
|
||||
}
|
||||
|
||||
private repeatTick(): void {
|
||||
const key = this.repeating;
|
||||
this.repeatTimer = null;
|
||||
if (!key || !this.held.has(`${key.page}:${key.usage}`)) {
|
||||
this.repeating = null;
|
||||
return;
|
||||
}
|
||||
this.emit("down", key.page, key.usage, true);
|
||||
this.repeatTimer = this.setTimer(() => this.repeatTick(), this.repeatIntervalMs);
|
||||
}
|
||||
|
||||
private stopRepeat(): void {
|
||||
if (this.repeatTimer !== null) this.clearTimer(this.repeatTimer);
|
||||
this.repeatTimer = null;
|
||||
this.repeating = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* A keyboard's touchpad or mouse, as the rest of the keyboard support sees
|
||||
* it. Its buttons and wheel become keys ("MouseButton1", "WheelUp"), so
|
||||
* they get default meanings and custom keybindings like any key
|
||||
* (bindings.ts). Its movement goes nowhere unless something has taken the
|
||||
* pointer (the cursor test page): there is no cursor in the main UI.
|
||||
* No platform imports: this runs under node in the tests.
|
||||
*/
|
||||
import type { HidKeyEvent, HidPointerSample } from "./hid";
|
||||
import { HID_PAGE_BUTTON } from "./keys";
|
||||
|
||||
/** The names a wheel's detents arrive under, as keys. */
|
||||
export type WheelKey = "WheelUp" | "WheelDown" | "WheelLeft" | "WheelRight";
|
||||
|
||||
export type PointerInputHost = {
|
||||
/** Button transitions, as HID usages on the Button page. */
|
||||
buttons(events: HidKeyEvent[]): void;
|
||||
/** One wheel report in a direction, however many detents it claimed. */
|
||||
wheel(key: WheelKey): void;
|
||||
};
|
||||
|
||||
export class PointerInput {
|
||||
/** The buttons the host has been told are down. */
|
||||
private reported = 0;
|
||||
/** The buttons actually down, as of the last sample. */
|
||||
private held = 0;
|
||||
/** Buttons that went down while the pointer was taken: not keys until they have come up. */
|
||||
private ignored = 0;
|
||||
private grab: ((sample: HidPointerSample) => void) | null = null;
|
||||
|
||||
constructor(private readonly host: PointerInputHost) {}
|
||||
|
||||
/** One report from the pointer (see KeyboardClientEvents.pointer). */
|
||||
handle(sample: HidPointerSample): void {
|
||||
this.held = sample.buttons;
|
||||
if (this.grab) {
|
||||
this.grab(sample);
|
||||
return;
|
||||
}
|
||||
this.ignored &= sample.buttons;
|
||||
this.report(sample.buttons & ~this.ignored);
|
||||
// One step per report: a touchpad's two-finger scroll sends many small
|
||||
// ones, and the gesture they turn into is rate-limited downstream.
|
||||
if (sample.wheel) this.host.wheel(sample.wheel > 0 ? "WheelUp" : "WheelDown");
|
||||
if (sample.pan) this.host.wheel(sample.pan > 0 ? "WheelRight" : "WheelLeft");
|
||||
}
|
||||
|
||||
/**
|
||||
* Take the pointer: every sample goes to `listener`, movement included,
|
||||
* and its buttons and wheel stop acting as keys. Buttons held at that
|
||||
* moment are released first, so nothing is left down. Returns a function
|
||||
* that gives the pointer back.
|
||||
*/
|
||||
take(listener: (sample: HidPointerSample) => void): () => void {
|
||||
this.report(0);
|
||||
this.grab = listener;
|
||||
return () => {
|
||||
if (this.grab !== listener) return;
|
||||
this.grab = null;
|
||||
// A button still down was pressed for the listener, not as a key: its
|
||||
// release is nobody's business either.
|
||||
this.ignored = this.held;
|
||||
};
|
||||
}
|
||||
|
||||
/** The keyboard went away: nothing is held, and nobody is told. */
|
||||
reset(): void {
|
||||
this.reported = 0;
|
||||
this.held = 0;
|
||||
this.ignored = 0;
|
||||
}
|
||||
|
||||
private report(buttons: number): void {
|
||||
const changed = buttons ^ this.reported;
|
||||
if (!changed) return;
|
||||
const events: HidKeyEvent[] = [];
|
||||
for (let bit = 0; bit < 8; bit++) {
|
||||
if (changed & (1 << bit)) events.push({ page: HID_PAGE_BUTTON, usage: bit + 1, down: !!(buttons & (1 << bit)) });
|
||||
}
|
||||
this.reported = buttons;
|
||||
this.host.buttons(events);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
/**
|
||||
* Wire format of the CFW keyboard bridge (Faceclaw/39). The master temple acts
|
||||
* as a BLE central for one HID-over-GATT keyboard; the phone drives that link
|
||||
* over private settings fields and gets the keyboard's raw input reports back.
|
||||
* The firmware side is g2flash patches/keyboard_core.h, documented in its
|
||||
* docs/keyboard.md. This follows the reference client in g2flash demos/
|
||||
* (keyboard-protocol.ts), which came with the firmware from tetramo-labs.
|
||||
*
|
||||
* Phone -> glasses, settings field 130 (wrapped by the native session):
|
||||
* ['K','B',1,op, request LE16, epoch LE16, body...] at most 80 bytes
|
||||
* Glasses -> phone, settings field 131:
|
||||
* ['K','B',1,kind, sequence LE32, epoch LE16, request LE16, code LE16,
|
||||
* handle LE16, total LE16, offset LE16, timestamp LE32, fragment...]
|
||||
*
|
||||
* All integers are little endian. No platform imports: this runs under node
|
||||
* in the tests.
|
||||
*/
|
||||
|
||||
export const KEYBOARD_PROTOCOL_VERSION = 1;
|
||||
|
||||
/** Fixed part of a field-131 record, before its fragment. */
|
||||
export const KEYBOARD_RECORD_HEADER = 24;
|
||||
/** Most body bytes one record carries; longer bodies are fragmented. */
|
||||
export const KEYBOARD_RECORD_CHUNK = 96;
|
||||
/** Largest reassembled body (an ATT value or a HID report). */
|
||||
export const KEYBOARD_BODY_MAX = 512;
|
||||
/** Largest control body after the 8-byte prefix. */
|
||||
export const KEYBOARD_CONTROL_BODY_MAX = 72;
|
||||
|
||||
export const KeyboardOp = {
|
||||
query: 0,
|
||||
enable: 1,
|
||||
disable: 2,
|
||||
scan: 3,
|
||||
connect: 4,
|
||||
disconnect: 5,
|
||||
pair: 6,
|
||||
auth: 7,
|
||||
compare: 8,
|
||||
find: 9,
|
||||
read: 10,
|
||||
write: 11,
|
||||
watch: 12,
|
||||
} as const;
|
||||
export type KeyboardOpCode = (typeof KeyboardOp)[keyof typeof KeyboardOp];
|
||||
|
||||
export const KeyboardKind = {
|
||||
status: 1,
|
||||
att: 2,
|
||||
input: 3,
|
||||
dm: 4,
|
||||
advert: 5,
|
||||
} as const;
|
||||
|
||||
/** Control error codes, carried in a STATUS record's `code`. */
|
||||
export const KeyboardError = {
|
||||
ok: 0,
|
||||
badRequest: 1,
|
||||
disabled: 2,
|
||||
busy: 3,
|
||||
staleEpoch: 4,
|
||||
noSpace: 5,
|
||||
notConnected: 6,
|
||||
notEncrypted: 7,
|
||||
timeout: 8,
|
||||
noMemory: 9,
|
||||
} as const;
|
||||
|
||||
export function keyboardErrorName(code: number): string {
|
||||
switch (code) {
|
||||
case KeyboardError.ok: return "ok";
|
||||
case KeyboardError.badRequest: return "bad request";
|
||||
case KeyboardError.disabled: return "keyboard bridge disabled";
|
||||
case KeyboardError.busy: return "busy";
|
||||
case KeyboardError.staleEpoch: return "stale epoch";
|
||||
case KeyboardError.noSpace: return "no connection or bond space";
|
||||
case KeyboardError.notConnected: return "not connected";
|
||||
case KeyboardError.notEncrypted: return "not encrypted";
|
||||
case KeyboardError.timeout: return "timeout";
|
||||
case KeyboardError.noMemory: return "allocation failure";
|
||||
default: return `error ${code}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Cordio device-manager events the bridge relays (a DM record's `code & 0xff`). */
|
||||
export const KeyboardDm = {
|
||||
connOpen: 0x27,
|
||||
connClose: 0x28,
|
||||
pairComplete: 0x2a,
|
||||
pairFailed: 0x2b,
|
||||
encrypted: 0x2c,
|
||||
encryptFailed: 0x2d,
|
||||
authRequest: 0x2e,
|
||||
compare: 0x35,
|
||||
} as const;
|
||||
|
||||
/** ATT client events (an ATT record's `code & 0xff`; the status is `code >> 8`). */
|
||||
export const KeyboardAtt = {
|
||||
notification: 13,
|
||||
indication: 14,
|
||||
} as const;
|
||||
|
||||
export type KeyboardRecord = {
|
||||
kind: number;
|
||||
sequence: number;
|
||||
epoch: number;
|
||||
request: number;
|
||||
code: number;
|
||||
handle: number;
|
||||
/** Glasses uptime in ms when the record was queued. */
|
||||
timestamp: number;
|
||||
data: Uint8Array;
|
||||
};
|
||||
|
||||
export const u16 = (p: Uint8Array, off = 0): number => p[off]! | (p[off + 1]! << 8);
|
||||
export const u32 = (p: Uint8Array, off = 0): number =>
|
||||
(p[off]! | (p[off + 1]! << 8) | (p[off + 2]! << 16) | (p[off + 3]! << 24)) >>> 0;
|
||||
export const le16 = (v: number): number[] => [v & 255, (v >>> 8) & 255];
|
||||
|
||||
/** A field-130 control record. The session wraps it in the settings package. */
|
||||
export function keyboardControl(op: number, request: number, epoch: number, body: readonly number[] = []): Uint8Array {
|
||||
if (!Number.isInteger(op) || op < 0 || op > KeyboardOp.watch || body.length > KEYBOARD_CONTROL_BODY_MAX) {
|
||||
throw new Error("Keyboard control bounds");
|
||||
}
|
||||
return Uint8Array.from([75, 66, KEYBOARD_PROTOCOL_VERSION, op, ...le16(request), ...le16(epoch), ...body]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded, ordered reassembly of field-131 records. `onGap` fires whenever
|
||||
* the stream may have lost something (a skipped sequence, an epoch change, a
|
||||
* malformed or orphaned fragment): the caller then releases every held key
|
||||
* and fails any setup transaction in flight. The firmware's once-a-second
|
||||
* status record makes a lost final release visible even when no further keys
|
||||
* arrive.
|
||||
*/
|
||||
export class KeyboardRecords {
|
||||
private epoch: number | undefined;
|
||||
private last: number | undefined;
|
||||
private fragment: { record: KeyboardRecord; total: number; used: number } | undefined;
|
||||
|
||||
constructor(private readonly onGap: () => void) {}
|
||||
|
||||
/** Forget the stream position (the phone link dropped); counts as a gap. */
|
||||
reset(): void {
|
||||
this.epoch = undefined;
|
||||
this.last = undefined;
|
||||
this.fragment = undefined;
|
||||
this.onGap();
|
||||
}
|
||||
|
||||
/** Feed one field-131 payload; returns a complete record, or undefined. */
|
||||
push(p: Uint8Array): KeyboardRecord | undefined {
|
||||
const invalid = (): undefined => {
|
||||
this.fragment = undefined;
|
||||
this.onGap();
|
||||
return undefined;
|
||||
};
|
||||
if (
|
||||
p.length < KEYBOARD_RECORD_HEADER || p.length > KEYBOARD_RECORD_HEADER + KEYBOARD_RECORD_CHUNK ||
|
||||
p[0] !== 75 || p[1] !== 66 || p[2] !== KEYBOARD_PROTOCOL_VERSION
|
||||
) {
|
||||
return invalid();
|
||||
}
|
||||
const r: KeyboardRecord = {
|
||||
kind: p[3]!,
|
||||
sequence: u32(p, 4),
|
||||
epoch: u16(p, 8),
|
||||
request: u16(p, 10),
|
||||
code: u16(p, 12),
|
||||
handle: u16(p, 14),
|
||||
timestamp: u32(p, 20),
|
||||
data: new Uint8Array(0),
|
||||
};
|
||||
const total = u16(p, 16);
|
||||
const offset = u16(p, 18);
|
||||
const n = p.length - KEYBOARD_RECORD_HEADER;
|
||||
if (r.kind < KeyboardKind.status || r.kind > KeyboardKind.advert || total > KEYBOARD_BODY_MAX ||
|
||||
offset + n > total || (total > 0 && n === 0)) {
|
||||
return invalid();
|
||||
}
|
||||
if (this.epoch !== r.epoch) {
|
||||
this.onGap();
|
||||
this.fragment = undefined;
|
||||
this.last = undefined;
|
||||
this.epoch = r.epoch;
|
||||
}
|
||||
if (offset === 0) {
|
||||
if (this.last !== undefined) {
|
||||
const distance = (r.sequence - this.last) >>> 0;
|
||||
if (distance === 0 || distance >= 0x80000000) return undefined; // duplicate or old delivery
|
||||
}
|
||||
if (this.fragment || (this.last !== undefined && r.sequence !== ((this.last + 1) >>> 0))) this.onGap();
|
||||
this.last = r.sequence;
|
||||
this.fragment = { record: { ...r, data: new Uint8Array(total) }, total, used: 0 };
|
||||
}
|
||||
const f = this.fragment;
|
||||
if (!f || f.used !== offset || f.total !== total || f.record.sequence !== r.sequence ||
|
||||
f.record.kind !== r.kind || f.record.request !== r.request || f.record.code !== r.code ||
|
||||
f.record.handle !== r.handle || f.record.timestamp !== r.timestamp) {
|
||||
return invalid();
|
||||
}
|
||||
f.record.data.set(p.subarray(KEYBOARD_RECORD_HEADER), offset);
|
||||
f.used += n;
|
||||
if (f.used !== total) return undefined;
|
||||
this.fragment = undefined;
|
||||
return f.record;
|
||||
}
|
||||
}
|
||||
|
||||
/** BLE address types as the controller reports them. */
|
||||
export const KEYBOARD_ADDRESS_TYPES = ["public", "random", "public-identity", "random-identity"] as const;
|
||||
|
||||
/** One advertiser seen during a scan (an ADVERT record's body, parsed). */
|
||||
export type KeyboardAdvert = {
|
||||
addressType: number;
|
||||
/** "aa:bb:cc:dd:ee:ff", most significant byte first. */
|
||||
address: string;
|
||||
rssi: number;
|
||||
name: string;
|
||||
/** Advertises the HID service, or a HID appearance. */
|
||||
hid: boolean;
|
||||
};
|
||||
|
||||
/** ADVERT body: addrType, addr[6] (LSB first), RSSI, event type, AD length, AD structures. */
|
||||
export function parseKeyboardAdvert(body: Uint8Array): KeyboardAdvert | null {
|
||||
if (body.length < 10 || body[0]! > 3) return null;
|
||||
let name = "";
|
||||
let hid = false;
|
||||
for (let i = 10; i < body.length;) {
|
||||
const n = body[i++]!;
|
||||
if (n === 0 || i + n > body.length) break;
|
||||
const type = body[i]!;
|
||||
if (type === 8 || type === 9) name = decodeUtf8(body.subarray(i + 1, i + n));
|
||||
// 16-bit service lists (incomplete / complete): 0x1812 is HID.
|
||||
if (type === 2 || type === 3) {
|
||||
for (let j = i + 1; j + 1 < i + n; j += 2) if (body[j] === 0x12 && body[j + 1] === 0x18) hid = true;
|
||||
}
|
||||
// Appearance 0x03C0..0x03FF is the HID category (0x03C1 keyboard); some
|
||||
// keyboards advertise that without the service list.
|
||||
if (type === 0x19 && n >= 3 && body[i + 2] === 0x03 && body[i + 1]! >= 0xc0) hid = true;
|
||||
i += n;
|
||||
}
|
||||
return {
|
||||
addressType: body[0]!,
|
||||
address: formatKeyboardAddress(body.subarray(1, 7)),
|
||||
rssi: (body[7]! << 24) >> 24,
|
||||
name,
|
||||
hid,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a later record for the same advertiser into an earlier one. An active
|
||||
* scan reports the advertisement and its scan response separately and the
|
||||
* name usually only rides in the latter: keep a name once seen, HID once
|
||||
* seen, and the best signal.
|
||||
*/
|
||||
export function mergeKeyboardAdverts(a: KeyboardAdvert, b: KeyboardAdvert): KeyboardAdvert {
|
||||
return { ...a, rssi: Math.max(a.rssi, b.rssi), name: a.name || b.name, hid: a.hid || b.hid };
|
||||
}
|
||||
|
||||
/** Controller byte order (LSB first) to the usual "aa:bb:cc:dd:ee:ff". */
|
||||
export function formatKeyboardAddress(lsbFirst: Uint8Array): string {
|
||||
return Array.from(lsbFirst).reverse().map((b) => b.toString(16).padStart(2, "0")).join(":");
|
||||
}
|
||||
|
||||
/** "aa:bb:cc:dd:ee:ff" to the controller's LSB-first bytes, or null if malformed. */
|
||||
export function parseKeyboardAddress(text: string): Uint8Array | null {
|
||||
if (!/^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/i.test(text)) return null;
|
||||
return Uint8Array.from(text.split(":").reverse().map((part) => parseInt(part, 16)));
|
||||
}
|
||||
|
||||
/** Lenient UTF-8 decode (advertised names); no TextDecoder in every runtime this loads in. */
|
||||
function decodeUtf8(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (let i = 0; i < bytes.length;) {
|
||||
const b = bytes[i++]!;
|
||||
let code = b;
|
||||
let extra = 0;
|
||||
if (b >= 0xf0 && b < 0xf8) { code = b & 0x07; extra = 3; }
|
||||
else if (b >= 0xe0) { code = b & 0x0f; extra = 2; }
|
||||
else if (b >= 0xc0) { code = b & 0x1f; extra = 1; }
|
||||
else if (b >= 0x80) { out += "�"; continue; }
|
||||
if (i + extra > bytes.length) { out += "�"; break; }
|
||||
let valid = true;
|
||||
for (let k = 0; k < extra; k++) {
|
||||
const c = bytes[i + k]!;
|
||||
if ((c & 0xc0) !== 0x80) { valid = false; break; }
|
||||
code = (code << 6) | (c & 0x3f);
|
||||
}
|
||||
if (!valid || code > 0x10ffff) { out += "�"; continue; }
|
||||
i += extra;
|
||||
if (code === 0) continue;
|
||||
out += String.fromCodePoint(code);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
/**
|
||||
* The keyboard bridge as the app uses it: one KeyboardBridge for the life of
|
||||
* the app, fed by whichever glasses session is live, its remembered keyboard
|
||||
* and the custom keybindings kept in the settings store, and its key
|
||||
* transitions turned into KeyEvents and routed. The platform's controller
|
||||
* reports the link coming and going and performs the gestures keys stand
|
||||
* for; the Settings app drives pairing through `bridge` and edits the
|
||||
* keybindings.
|
||||
*/
|
||||
import { addKeyboardEventListener, keyboardLinkSession, sendKeyboardControlRecord } from "../../native/keyboard-link";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import {
|
||||
MAX_KEY_BINDINGS,
|
||||
chordId,
|
||||
keyBindingMap,
|
||||
parseKeyBindings,
|
||||
serializeKeyBindings,
|
||||
type GestureOrigin,
|
||||
type KeyAction,
|
||||
type KeyBinding,
|
||||
type KeyChord,
|
||||
type KeyGesture,
|
||||
} from "./bindings";
|
||||
import { KeyboardBridge, type KeyboardBridgeSnapshot, type SavedKeyboard } from "./bridge";
|
||||
import type { HidPointerSample } from "./hid";
|
||||
import { KeyRouter } from "./key-router";
|
||||
import { KeyTranslator, type KeyEvent } from "./keys";
|
||||
import { PointerInput } from "./pointer";
|
||||
|
||||
declare const java: any;
|
||||
declare const arc4random_uniform: ((bound: number) => number) | undefined;
|
||||
|
||||
/** JSON of the remembered keyboard ({ address, addressType, name }), or "" for none. */
|
||||
const SAVED_KEYBOARD_KEY = "keyboard.saved";
|
||||
/** JSON list of the custom keybindings ({ key, action }); see bindings.ts. */
|
||||
const KEY_BINDINGS_KEY = "keyboard.bindings";
|
||||
|
||||
/** What the platform's controller does for the keyboard. */
|
||||
export type KeyboardServiceHost = {
|
||||
/** Connected to glasses that are in use (not charging in their case). */
|
||||
ready(): boolean;
|
||||
locked(): boolean;
|
||||
/** Turn a dark display on, and nothing else (a locked display stays locked). */
|
||||
wake(): void;
|
||||
/** Perform a gesture as if it came from the ring or the watch. */
|
||||
gesture(gesture: KeyGesture, origin: GestureOrigin): Promise<void>;
|
||||
/** Routine lines only (failed background reconnects); the rest go to the device log directly. */
|
||||
log(line: string): void;
|
||||
/** Whether an absent keyboard is worth looking for now (see KeyboardBridgeHost). */
|
||||
shouldReconnect(): boolean;
|
||||
};
|
||||
|
||||
function loadSavedKeyboard(): SavedKeyboard | null {
|
||||
try {
|
||||
const raw = getStringSetting(SAVED_KEYBOARD_KEY, "");
|
||||
if (!raw) return null;
|
||||
const value = JSON.parse(raw);
|
||||
if (typeof value?.address !== "string" || typeof value?.addressType !== "number") return null;
|
||||
return { address: value.address, addressType: value.addressType, name: typeof value.name === "string" ? value.name : "" };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Passkey entry only protects the pairing if the passkey can't be guessed. */
|
||||
function randomPasskey(): number {
|
||||
try {
|
||||
if (global.isAndroid) return new java.security.SecureRandom().nextInt(1_000_000);
|
||||
if (typeof arc4random_uniform === "function") return arc4random_uniform(1_000_000);
|
||||
} catch {
|
||||
// Fall through to the weaker source rather than failing the pairing.
|
||||
}
|
||||
return Math.floor(Math.random() * 1_000_000);
|
||||
}
|
||||
|
||||
class KeyboardService {
|
||||
private host: KeyboardServiceHost | null = null;
|
||||
private session: unknown = null;
|
||||
private offEvents: (() => void) | null = null;
|
||||
private readonly listeners = new Set<() => void>();
|
||||
private bindings: KeyBinding[] = parseKeyBindings(getStringSetting(KEY_BINDINGS_KEY, ""));
|
||||
private bindingMap = keyBindingMap(this.bindings);
|
||||
|
||||
private readonly router = new KeyRouter({
|
||||
ready: () => this.host?.ready() ?? false,
|
||||
locked: () => this.host?.locked() ?? false,
|
||||
screenOn: () => shell.isScreenOn(),
|
||||
wake: () => this.host?.wake(),
|
||||
offerToWindow: (event) => shell.receiveKeyEvent(event),
|
||||
gesture: (gesture, origin) => this.host?.gesture(gesture, origin) ?? Promise.resolve(),
|
||||
bindings: () => this.bindingMap,
|
||||
noteActivity: () => shell.noteUserActivity(),
|
||||
log: (line) => console.log(`[keyboard] ${line}`),
|
||||
});
|
||||
private readonly translator = new KeyTranslator({ emit: (event) => this.router.handleKey(event) });
|
||||
/** The touchpad's buttons and wheel go in as keys; see pointer.ts. */
|
||||
private readonly pointer = new PointerInput({
|
||||
buttons: (events) => this.translator.handle(events),
|
||||
wheel: (key) => this.translator.pulse(key),
|
||||
});
|
||||
|
||||
readonly bridge = new KeyboardBridge({
|
||||
send: (record) => {
|
||||
const session = this.session;
|
||||
if (!session) return false;
|
||||
try {
|
||||
return sendKeyboardControlRecord(session, record, `op=${record[3]}`);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
loadSaved: loadSavedKeyboard,
|
||||
storeSaved: (saved) => setStringSetting(SAVED_KEYBOARD_KEY, saved ? JSON.stringify(saved) : ""),
|
||||
onKeys: (events) => this.translator.handle(events),
|
||||
onPointer: (sample) => this.pointer.handle(sample),
|
||||
onKeyboardGone: () => {
|
||||
this.translator.reset();
|
||||
this.pointer.reset();
|
||||
this.router.reset();
|
||||
},
|
||||
onChanged: () => {
|
||||
for (const listener of [...this.listeners]) listener();
|
||||
},
|
||||
// Pairing and setup are rare and are what a failure report needs, so
|
||||
// they always reach the device log. Nothing here ever includes a key.
|
||||
log: (line, routine) => {
|
||||
if (routine) this.host?.log(line);
|
||||
else console.log(`[keyboard] ${line}`);
|
||||
},
|
||||
randomPasskey,
|
||||
shouldReconnect: () => this.host?.shouldReconnect() ?? true,
|
||||
});
|
||||
|
||||
setHost(host: KeyboardServiceHost): void {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
snapshot(): KeyboardBridgeSnapshot {
|
||||
return this.bridge.snapshot();
|
||||
}
|
||||
|
||||
/** Called on every bridge state change. Returns an unsubscribe function. */
|
||||
subscribe(listener: () => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/** The glasses came into use (the display woke): look for an absent keyboard now. */
|
||||
retryNow(): void {
|
||||
this.bridge.retryNow();
|
||||
}
|
||||
|
||||
/** A G2 session on firmware with the bridge is up. Idempotent. */
|
||||
linkReady(): void {
|
||||
const session = keyboardLinkSession();
|
||||
if (!session) return;
|
||||
if (session !== this.session) {
|
||||
// The native session object is rebuilt across reconnects.
|
||||
this.offEvents?.();
|
||||
this.session = session;
|
||||
this.offEvents = addKeyboardEventListener(session, (body) => this.bridge.accept(body));
|
||||
}
|
||||
this.bridge.linkReady();
|
||||
}
|
||||
|
||||
/**
|
||||
* The session can't host the keyboard any more. `release` asks the firmware
|
||||
* to let the keyboard go first, for when the link itself stays up (the
|
||||
* glasses went into their case, or the app is disconnecting on purpose).
|
||||
* Idempotent.
|
||||
*/
|
||||
linkLost(release = false): void {
|
||||
if (release) this.bridge.linkClosing();
|
||||
this.bridge.linkLost();
|
||||
this.offEvents?.();
|
||||
this.offEvents = null;
|
||||
this.session = null;
|
||||
}
|
||||
|
||||
/** A key a worker's window was sent and had no use for (the "key-unhandled" reply). */
|
||||
handleUnhandledKey(event: KeyEvent): void {
|
||||
this.router.handleUnhandledKey(event);
|
||||
}
|
||||
|
||||
// ---------- Custom keybindings ----------
|
||||
|
||||
/** The custom keybindings, in the order they were added. */
|
||||
keyBindings(): readonly KeyBinding[] {
|
||||
return this.bindings;
|
||||
}
|
||||
|
||||
/** Bind a chord, replacing what it was bound to. False when the list is full. */
|
||||
setKeyBinding(chord: KeyChord, action: KeyAction): boolean {
|
||||
const id = chordId(chord);
|
||||
const index = this.bindings.findIndex((binding) => chordId(binding.chord) === id);
|
||||
if (index < 0 && this.bindings.length >= MAX_KEY_BINDINGS) return false;
|
||||
const next = [...this.bindings];
|
||||
if (index < 0) next.push({ chord, action });
|
||||
else next[index] = { chord, action };
|
||||
this.storeBindings(next);
|
||||
return true;
|
||||
}
|
||||
|
||||
removeKeyBinding(chord: KeyChord): void {
|
||||
const id = chordId(chord);
|
||||
this.storeBindings(this.bindings.filter((binding) => chordId(binding.chord) !== id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Take the next key pressed on the keyboard instead of acting on it. Returns
|
||||
* a function that calls the capture off.
|
||||
*/
|
||||
captureNextKey(onChord: (chord: KeyChord) => void): () => void {
|
||||
return this.router.captureNextKey(onChord);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take the keyboard's touchpad or mouse: every report goes to `listener`,
|
||||
* movement included, and its buttons and wheel stop acting as keys until
|
||||
* the returned function is called. For the cursor test page; the main UI
|
||||
* has no cursor.
|
||||
*/
|
||||
takePointer(listener: (sample: HidPointerSample) => void): () => void {
|
||||
return this.pointer.take(listener);
|
||||
}
|
||||
|
||||
private storeBindings(bindings: KeyBinding[]): void {
|
||||
this.bindings = bindings;
|
||||
this.bindingMap = keyBindingMap(bindings);
|
||||
setStringSetting(KEY_BINDINGS_KEY, serializeKeyBindings(bindings));
|
||||
}
|
||||
}
|
||||
|
||||
export const keyboardService = new KeyboardService();
|
||||
@@ -3,11 +3,18 @@ import type { DiscoveryEvents } from './device-discovery'
|
||||
import type { RawAdvertisement } from '../g2/even-advertisement'
|
||||
export { buildAddressSet } from './device-discovery-common'
|
||||
|
||||
/**
|
||||
* How often a scan re-reads which arms the system is connected to. Each read refreshes
|
||||
* their rows, which age out like advertisements once the link drops and they go quiet.
|
||||
*/
|
||||
const CONNECTED_ARMS_POLL_MS = 2000
|
||||
|
||||
export class DeviceDiscoveryBridge {
|
||||
private ble = iosBluetooth()
|
||||
private off: (() => void) | null = null
|
||||
private generation = 0
|
||||
private cancelCollection: (() => void) | null = null
|
||||
private connectedArmsTimer: ReturnType<typeof setInterval> | null = null
|
||||
get bluetoothEnabled(): boolean { return this.ble.state === 5 }
|
||||
get scanning(): boolean { return !!this.off && this.ble.scanning }
|
||||
startScan(events: DiscoveryEvents): boolean {
|
||||
@@ -21,16 +28,27 @@ export class DeviceDiscoveryBridge {
|
||||
void this.ble.startScan(24 * 60 * 60 * 1000).catch(error => {
|
||||
if (generation === this.generation) events.onScanFailed?.(-1, String(error.message ?? error))
|
||||
})
|
||||
// A connected arm doesn't advertise (iOS holds the right arm's ANCS link open after a
|
||||
// session ends), so the scan alone would show those glasses one-armed.
|
||||
const reportConnectedArms = () => { for (const device of this.ble.systemConnectedDevices()) events.onAdvertisement(device) }
|
||||
reportConnectedArms()
|
||||
this.connectedArmsTimer = setInterval(reportConnectedArms, CONNECTED_ARMS_POLL_MS)
|
||||
return true
|
||||
}
|
||||
stopScan(): void {
|
||||
++this.generation; this.cancelCollection?.()
|
||||
if (this.connectedArmsTimer) { clearInterval(this.connectedArmsTimer); this.connectedArmsTimer = null }
|
||||
if (this.off) { this.off(); this.off = null; this.ble.stopScan() }
|
||||
}
|
||||
// CoreBluetooth does not expose the phone's bond list. Fresh advertisements
|
||||
// supply both identity and the UUID mapping that iOS needs for connections.
|
||||
// supply both identity and the UUID mapping that iOS needs for connections;
|
||||
// startScan() reports the arms the system is connected to.
|
||||
emitBondedDevices(): void {}
|
||||
async getBondedCandidates(): Promise<RawAdvertisement[]> { return [...this.ble.devices.values()] }
|
||||
async getBondedCandidates(): Promise<RawAdvertisement[]> {
|
||||
const devices = new Map(this.ble.devices)
|
||||
for (const device of this.ble.systemConnectedDevices()) if (!devices.has(device.address)) devices.set(device.address, device)
|
||||
return [...devices.values()]
|
||||
}
|
||||
async scanCandidates(timeoutMs = 6000): Promise<RawAdvertisement[]> {
|
||||
const generation = this.generation
|
||||
await this.ble.ensureReady()
|
||||
|
||||
@@ -19,6 +19,7 @@ declare const FaceclawKitFaceclawCompassListener: any;
|
||||
declare const FaceclawKitFaceclawImuListener: any;
|
||||
declare const FaceclawKitIosAmbientLightListener: any;
|
||||
declare const FaceclawKitIosMicStatusListener: any;
|
||||
declare const FaceclawKitIosKeyboardEventListener: any;
|
||||
|
||||
|
||||
export type CommunicatorPhase =
|
||||
@@ -198,6 +199,13 @@ class MicStatusListener extends NSObject {
|
||||
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class KeyboardEventListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosKeyboardEventListener];
|
||||
handler!: (body: Uint8Array) => void;
|
||||
onNsDataKeyboardEventBody(body: NSData): void { this.handler(fromData(body)); }
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS twin of the Android communicator bridge: the same public API over
|
||||
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
|
||||
@@ -803,6 +811,19 @@ export class FaceclawCommunicatorBridge {
|
||||
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
|
||||
}
|
||||
|
||||
/** Raw field-131 keyboard-bridge records, in arrival order, on the main queue. */
|
||||
addKeyboardEventListener(listener: (body: Uint8Array) => void): () => void {
|
||||
const proxy = KeyboardEventListener.new() as KeyboardEventListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addKeyboardEventListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeKeyboardEventListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Queue a CFW keyboard-bridge control for the right temple; false when nothing was queued. */
|
||||
sendFaceclawKeyboardControl(record: Uint8Array, label: string): boolean {
|
||||
return Boolean(this.communicator.sendFaceclawKeyboardControlRecordLabel(toData(new Uint8Array(record)), label));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every audio-characteristic packet, whatever enabled the mics, on the
|
||||
* main queue. Passive: sends nothing and doesn't affect capture.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { openSocket } from "./socket";
|
||||
import { openSocket, type SocketConnection, type SocketListener } from "./socket";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState } from "./terminal-session-types";
|
||||
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
@@ -19,54 +20,7 @@ const LAUNCH_REPLY_TIMEOUT_MS = 10_000;
|
||||
// otherwise treat the "\r" as a pasted newline instead of the Enter key.
|
||||
const SUBMIT_DELAY_MS = 150;
|
||||
|
||||
export type G2MirrorSession = {
|
||||
socket: string;
|
||||
pid: number;
|
||||
cwdHint: string;
|
||||
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
|
||||
lastBellAt: number | null;
|
||||
/**
|
||||
* Unix epoch ms (host clock) of the app's last output, or null if none
|
||||
* observed. Fresh to within the wrapper's 2s activity reporting interval
|
||||
* while we're connected; compare against other host timestamps, not the
|
||||
* phone clock (see the worker's local-receive-time activity tracking).
|
||||
*/
|
||||
lastOutputAt: number | null;
|
||||
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
|
||||
title: string | null;
|
||||
};
|
||||
|
||||
export type G2MirrorPhase =
|
||||
| "idle"
|
||||
| "connecting"
|
||||
| "connected" // handshake accepted; can list/attach
|
||||
| "attached" // relaying to one session
|
||||
| "failed";
|
||||
|
||||
export type G2MirrorState = {
|
||||
phase: G2MirrorPhase;
|
||||
status: string;
|
||||
sessions: G2MirrorSession[];
|
||||
attachedCommand: string;
|
||||
/**
|
||||
* server_name from the init success reply: a human-readable name for the
|
||||
* machine the server runs on. "" until the handshake succeeds (or when an
|
||||
* older server omits the field).
|
||||
*/
|
||||
serverName: string;
|
||||
};
|
||||
|
||||
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
|
||||
export type G2MirrorHistoryReply = {
|
||||
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
|
||||
start: number;
|
||||
/** Oldest index still retained by the wrapper (older requests return nothing). */
|
||||
oldest: number;
|
||||
/** Current splice point (next line to be archived). */
|
||||
next: number;
|
||||
/** Plain text, ANSI-stripped, oldest-to-newest. */
|
||||
lines: string[];
|
||||
};
|
||||
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
|
||||
|
||||
export type G2MirrorClientOptions = {
|
||||
/** TLS (g2mirrors:// → wss) vs plain (g2mirror:// → ws). */
|
||||
@@ -77,6 +31,12 @@ export type G2MirrorClientOptions = {
|
||||
deviceName: string;
|
||||
cols: number;
|
||||
rows: number;
|
||||
/**
|
||||
* Opens the websocket some other way than dialing host:port directly
|
||||
* (e.g. through an SSH tunnel); `label` then names the endpoint in status
|
||||
* lines instead of host:port.
|
||||
*/
|
||||
transport?: { open: (listener: SocketListener) => SocketConnection; label: string };
|
||||
};
|
||||
|
||||
type TerminalDataKind = "snapshot" | "output";
|
||||
@@ -180,7 +140,8 @@ export class G2MirrorClient {
|
||||
this.stopped = false;
|
||||
const generation = ++this.connectionGeneration;
|
||||
const url = `${this.options.secure ? "wss" : "ws"}://${this.options.host}:${this.options.port}`;
|
||||
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
|
||||
const transport = this.options.transport;
|
||||
this.setState("connecting", `Connecting to ${transport ? transport.label : `${this.options.host}:${this.options.port}`}...`);
|
||||
this.listenerProxy = {
|
||||
onOpen: () => {
|
||||
if (this.stopped || generation !== this.connectionGeneration) return;
|
||||
@@ -208,7 +169,7 @@ export class G2MirrorClient {
|
||||
},
|
||||
};
|
||||
try {
|
||||
this.ws = openSocket(url, this.listenerProxy);
|
||||
this.ws = transport ? transport.open(this.listenerProxy!) : openSocket(url, this.listenerProxy);
|
||||
} catch (error) {
|
||||
this.ws = null;
|
||||
this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`);
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Head-up (look-up wake) trigger on the active glasses session
|
||||
* (GlassesSessionCore.setHeadUpTrigger / getHeadUpStatus). Android only for
|
||||
* now (headUpControlsAvailable): on iOS, or with no session, the setter does
|
||||
* nothing and the getter returns null.
|
||||
*/
|
||||
|
||||
/** Whether this platform can change the glasses' head-up setting at all. */
|
||||
export function headUpControlsAvailable(): boolean {
|
||||
return Boolean(global.isAndroid);
|
||||
}
|
||||
|
||||
/** GlassesSessionCore.getHeadUpStatus. switch -1 = not read on this connection. */
|
||||
export type HeadUpStatus = {
|
||||
switch: number;
|
||||
angle: number;
|
||||
calibration: number;
|
||||
/** The trigger Faceclaw is maintaining, or -1 when it leaves the glasses' own setting. */
|
||||
desired: number;
|
||||
readAgeMs: number;
|
||||
/** The glasses' own switch and angle from before Faceclaw changed them (switch -1 = none held). */
|
||||
originalSwitch: number;
|
||||
originalAngle: number;
|
||||
/** Those originals are being written back. */
|
||||
restorePending: boolean;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the session to keep the head-up trigger at `degrees` (1..90); 0 stops
|
||||
* managing it and writes back `original` (or what the session saved itself)
|
||||
* if Faceclaw changed the glasses' values.
|
||||
*/
|
||||
export function setHeadUpTrigger(degrees: number, original: { switch: number; angle: number } | null = null): boolean {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return false;
|
||||
try {
|
||||
active.setHeadUpTrigger(Math.round(degrees), original ? original.switch : -1, original ? Math.round(original.angle) : 0);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn(`setHeadUpTrigger failed: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function getHeadUpStatus(): HeadUpStatus | null {
|
||||
const active = activeCommunicator();
|
||||
if (!active) return null;
|
||||
try {
|
||||
return JSON.parse(String(active.getHeadUpStatus())) as HeadUpStatus;
|
||||
} catch (error) {
|
||||
console.warn(`getHeadUpStatus failed: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,23 @@
|
||||
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
|
||||
|
||||
/**
|
||||
* An accelerometer sample in g, in Faceclaw's frame for every glasses model
|
||||
* (the G2's own; the session rotates other models' samples into it, see
|
||||
* ImuOrientation.kt): +x forward, +y to the wearer's left, +z up through the
|
||||
* top of the head. At rest it is the "up" direction: head level reads
|
||||
* (0, 0, 1), looking 45° up (0.71, 0, 0.71), the head tilted 45° to the right
|
||||
* (0, 0.71, 0.71).
|
||||
*/
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
|
||||
/**
|
||||
* How far the head is tilted up from level, in degrees (negative: down), from
|
||||
* a reading taken at rest. Rolling the head to the side does not change it.
|
||||
*/
|
||||
export function imuPitchDegrees(reading: { x: number; y: number; z: number }): number {
|
||||
return (Math.atan2(reading.x, Math.hypot(reading.y, reading.z)) * 180) / Math.PI;
|
||||
}
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { ApplicationSettings } from '@nativescript/core'
|
||||
import { loadPairedGlassesIdentity } from '../g2/device-addresses'
|
||||
import { g2ManufacturerData } from '../g2/even-advertisement'
|
||||
import { serialKey } from '../g2/glasses-hardware-identity'
|
||||
import { identifyIosPeripheral, type IosAdvertisement, type IosDevice } from '../g2/ios-peripheral-identity'
|
||||
import { hexToBytes } from '../util/hex-util'
|
||||
import { kotlinListener } from './kotlin-listener.ios'
|
||||
@@ -10,6 +13,13 @@ export const BLE_STATE_UNSUPPORTED = 2, BLE_STATE_UNAUTHORIZED = 3, BLE_STATE_PO
|
||||
/** CBManagerAuthorization.allowedAlways. */
|
||||
const BLE_AUTHORIZATION_ALLOWED = 3
|
||||
|
||||
/**
|
||||
* Per address, the peripheral identifier and side last heard there and (since 2026-11-16)
|
||||
* the advertisement itself, from which an arm that has stopped advertising is rebuilt.
|
||||
*/
|
||||
const PERIPHERAL_KEY_PREFIX = 'ios.ble.peripheral.'
|
||||
type RememberedPeripheral = { identifier: string; role: string; name?: string; manufacturerData?: string }
|
||||
|
||||
export type BluetoothEvent =
|
||||
| { kind: 'state'; state: number; authorization: number }
|
||||
| { kind: 'advertisement'; identifier: string; name: string; manufacturerData: string; rssi: number; connectable: boolean }
|
||||
@@ -28,6 +38,8 @@ export class IosBluetooth {
|
||||
private readonly native = FaceclawKitIosBleCentral.alloc().initWithPlatform(FaceclawKitIosProtocolPlatform.shared)
|
||||
private readonly listeners = new Set<(event: BluetoothEvent) => void>()
|
||||
readonly devices = new Map<string, IosDevice>()
|
||||
/** The record last written per settings key, so repeated advertisements don't rewrite it. */
|
||||
private readonly rememberedValues = new Map<string, string>()
|
||||
private scanTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private scanGeneration = 0
|
||||
private readonly scanListener: object
|
||||
@@ -47,12 +59,68 @@ export class IosBluetooth {
|
||||
const device = identifyIosPeripheral(raw)
|
||||
if (device) {
|
||||
this.devices.set(device.address, device)
|
||||
ApplicationSettings.setString(`ios.ble.peripheral.${device.address}`, JSON.stringify({ identifier: device.identifier, role: device.role }))
|
||||
this.remember(device)
|
||||
this.emit({ kind: 'device', device })
|
||||
}
|
||||
},
|
||||
})
|
||||
}
|
||||
/** Persists what `device` advertised; skips the write when nothing changed (scans report duplicates). */
|
||||
private remember(device: IosDevice): void {
|
||||
const key = `${PERIPHERAL_KEY_PREFIX}${device.address}`
|
||||
const record: RememberedPeripheral = { identifier: device.identifier, role: device.role, name: device.name, manufacturerData: device.manufacturerData }
|
||||
const value = JSON.stringify(record)
|
||||
if (this.rememberedValues.get(key) === value) return
|
||||
this.rememberedValues.set(key, value)
|
||||
ApplicationSettings.setString(key, value)
|
||||
}
|
||||
/**
|
||||
* Even arms the system is connected to, rebuilt from what they last advertised here. A
|
||||
* connected arm doesn't advertise, and iOS keeps the right arm's ANCS link open after a
|
||||
* session ends (even after pairing other glasses), so a scan alone hears only the left
|
||||
* arm. An arm this phone has never heard advertise can't be rebuilt: its MAC and the
|
||||
* pair's serial come only from the advertisement. The device has no signal strength.
|
||||
*/
|
||||
systemConnectedDevices(): IosDevice[] {
|
||||
if (this.state !== BLE_STATE_POWERED_ON) return []
|
||||
let connected: any
|
||||
try {
|
||||
connected = this.native.systemConnectedG2Arms()
|
||||
} catch (error) {
|
||||
// Scanning still works without this; don't let it take the pairing page down.
|
||||
console.log(`[IosBluetooth] systemConnectedG2Arms failed: ${error}`)
|
||||
return []
|
||||
}
|
||||
if (!connected?.count) return []
|
||||
const records = new Map<string, { address: string; record: RememberedPeripheral }>()
|
||||
for (const key of ApplicationSettings.getAllKeys()) {
|
||||
if (!key.startsWith(PERIPHERAL_KEY_PREFIX)) continue
|
||||
try {
|
||||
const record = JSON.parse(ApplicationSettings.getString(key, '{}')) as RememberedPeripheral
|
||||
if (typeof record.identifier === 'string') records.set(record.identifier.toUpperCase(), { address: key.slice(PERIPHERAL_KEY_PREFIX.length), record })
|
||||
} catch { /* Malformed: a fresh advertisement rewrites it. */ }
|
||||
}
|
||||
const identity = loadPairedGlassesIdentity()
|
||||
const devices: IosDevice[] = []
|
||||
for (let i = 0; i < connected.count; i++) {
|
||||
const peripheral = connected.objectAtIndex(i)
|
||||
const identifier = String(peripheral.identifier)
|
||||
const remembered = records.get(identifier)
|
||||
if (!remembered) continue
|
||||
const { address, record } = remembered
|
||||
let name = record.name ?? ''
|
||||
let manufacturerData = record.manufacturerData ?? ''
|
||||
if (!manufacturerData && identity && (address === identity.leftAddress || address === identity.rightAddress)) {
|
||||
// Remembered before advertisements were: the saved pairing supplies the serial.
|
||||
const side = address === identity.leftAddress ? 'left' : 'right'
|
||||
name ||= side === 'left' ? identity.leftName : identity.rightName
|
||||
manufacturerData = g2ManufacturerData(serialKey(identity.serial) ?? '', address) ?? ''
|
||||
}
|
||||
const device = identifyIosPeripheral({ identifier, name: name || String(peripheral.name ?? ''), manufacturerData, rssi: 0, connectable: true })
|
||||
if (device?.address === address && device.role === record.role) devices.push({ ...device, rssi: null, source: 'paired' })
|
||||
}
|
||||
return devices
|
||||
}
|
||||
onEvent(listener: (event: BluetoothEvent) => void): () => void {
|
||||
this.listeners.add(listener); return () => { this.listeners.delete(listener) }
|
||||
}
|
||||
@@ -111,7 +179,7 @@ export class IosBluetooth {
|
||||
const live = this.devices.get(address)
|
||||
if (live && live.role !== role) throw new Error(`${address} advertises as ${live.role}, not ${role}. Check Configure devices.`)
|
||||
try {
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
|
||||
if (saved.role === role && typeof saved.identifier === 'string') result[role] = saved.identifier
|
||||
} catch { /* A fresh scan can recover malformed settings. */ }
|
||||
}
|
||||
@@ -138,13 +206,15 @@ export class IosBluetooth {
|
||||
/** The peripheral last heard at this address, if it was on this side. */
|
||||
peripheralIdentifier(address: string, role: string): string | null {
|
||||
try {
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
|
||||
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
|
||||
return saved.role === role && typeof saved.identifier === 'string' ? saved.identifier : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
forget(address: string): void { ApplicationSettings.remove(`ios.ble.peripheral.${address}`); this.devices.delete(address) }
|
||||
forget(address: string): void {
|
||||
ApplicationSettings.remove(`${PERIPHERAL_KEY_PREFIX}${address}`); this.rememberedValues.delete(`${PERIPHERAL_KEY_PREFIX}${address}`); this.devices.delete(address)
|
||||
}
|
||||
}
|
||||
let instance: IosBluetooth | null = null
|
||||
export function iosBluetooth(): IosBluetooth { return instance ??= new IosBluetooth() }
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/** The iOS twin of keyboard-link.ts, over the running shared-Kotlin session. */
|
||||
|
||||
export function keyboardLinkSession(): unknown {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
|
||||
return (session as FaceclawCommunicatorBridge).sendFaceclawKeyboardControl(record, label);
|
||||
}
|
||||
|
||||
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
|
||||
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
|
||||
const unsubscribe = (session as FaceclawCommunicatorBridge).addKeyboardEventListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { toJavaBytes } from "./cloud-stt";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Platform plumbing for the CFW keyboard bridge (app/g2/keyboard/): the running
|
||||
* session to send field-130 controls on and to hear field-131 records from.
|
||||
* The session is opaque and is a different object after a reconnect.
|
||||
*/
|
||||
|
||||
export function keyboardLinkSession(): unknown {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
|
||||
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
|
||||
return (session as any).sendFaceclawKeyboardControl(toJavaBytes(record), label) !== false;
|
||||
}
|
||||
|
||||
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
|
||||
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
|
||||
const communicator = session as any;
|
||||
const proxy = new com.faceclaw.app.FaceclawKeyboardEventListener({
|
||||
onKeyboardEvent: (body: any) => listener(toUint8Array(body)),
|
||||
});
|
||||
communicator.addKeyboardEventListener(proxy);
|
||||
return () => {
|
||||
try {
|
||||
communicator.removeKeyboardEventListener(proxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
|
||||
// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt).
|
||||
declare const FaceclawKitIosMicRecordings: any
|
||||
|
||||
/**
|
||||
* Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in
|
||||
* Documents/Downloads while it is captured, through the shared Kotlin writer
|
||||
* that Android's captures are recorded with. The file is written about a
|
||||
* second at a time, so a crash leaves everything up to the last second.
|
||||
*/
|
||||
export class MicRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/**
|
||||
* A recorder for Downloads/<name>.mka (a -2, -3... suffix if taken), or null.
|
||||
* `info` describes the capture and is stored in the file.
|
||||
*/
|
||||
static open(name: string, info: Record<string, unknown> = {}): MicRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.mka`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`)
|
||||
const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info))
|
||||
return recorder ? new MicRecorder(recorder, path) : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.recorder || !pcm.length) return
|
||||
// The handle is the buffer's start: a view into a larger buffer is copied out first.
|
||||
const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice()
|
||||
try {
|
||||
// Copied by the recorder before this returns.
|
||||
this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength))
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
this.close()
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
try { this.recorder?.finish() } catch { /* already closed */ }
|
||||
this.recorder = null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
|
||||
/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */
|
||||
export const RECORDING_EXTENSION = ".mka";
|
||||
|
||||
export function isMicRecordingFile(name: string): boolean {
|
||||
return name.toLowerCase().endsWith(RECORDING_EXTENSION);
|
||||
}
|
||||
|
||||
/**
|
||||
* The path for a new recording in the public Downloads directory:
|
||||
* Downloads/<name>.mka, with a -2, -3... suffix if that is taken. Null when
|
||||
* Downloads isn't available. The file itself is created by whoever records.
|
||||
*/
|
||||
export function newRecordingPath(name: string): string | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`);
|
||||
return String(file.getAbsolutePath());
|
||||
} catch (error) {
|
||||
console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Platform-independent half of the SSH transport bindings (ssh.ts /
|
||||
* ssh.ios.ts): the types the app uses and the decoding of the Go
|
||||
* transport's events (native/ssh/faceclawssh.go, Listener).
|
||||
*/
|
||||
|
||||
/** Where and as whom to connect. */
|
||||
export type SshEndpoint = {
|
||||
host: string;
|
||||
port: number;
|
||||
user: string;
|
||||
/** Pinned host keys (SSH wire format, base64). Empty: first connection reports hostkey-unknown. */
|
||||
hostKeys: string[];
|
||||
/** Saved password, or "" to try only the device key. */
|
||||
password: string;
|
||||
/** Offer the device's client key. */
|
||||
useKey: boolean;
|
||||
};
|
||||
|
||||
export type SshChannelSpec =
|
||||
| {
|
||||
/** Run a command on the host; text is exchanged a line at a time. */
|
||||
type: "exec";
|
||||
command: string;
|
||||
}
|
||||
| {
|
||||
/** WebSocket to a URL resolved on the host's side (e.g. ws://127.0.0.1:8737/). */
|
||||
type: "websocket";
|
||||
url: string;
|
||||
};
|
||||
|
||||
export type SshHostKey = {
|
||||
/** SSH key type, e.g. "ssh-ed25519". */
|
||||
type: string;
|
||||
/** OpenSSH-style "SHA256:..." fingerprint. */
|
||||
fingerprint: string;
|
||||
/** SSH wire format, base64: the value to pin in SshEndpoint.hostKeys. */
|
||||
key: string;
|
||||
};
|
||||
|
||||
export type SshFailureKind =
|
||||
| "hostkey-unknown"
|
||||
| "hostkey-mismatch"
|
||||
| "auth"
|
||||
| "network"
|
||||
| "channel"
|
||||
| "config"
|
||||
| "internal";
|
||||
|
||||
export type SshFailure = {
|
||||
kind: SshFailureKind;
|
||||
message: string;
|
||||
/** For hostkey-*: the key the host presented. */
|
||||
hostKey?: SshHostKey;
|
||||
/** For auth: the host takes passwords / we sent one / we offered our key. */
|
||||
passwordAllowed?: boolean;
|
||||
passwordTried?: boolean;
|
||||
keyTried?: boolean;
|
||||
};
|
||||
|
||||
export type SshClosed = {
|
||||
/** Exec channels: the command's exit status (and signal, if killed). */
|
||||
exitStatus?: number;
|
||||
signal?: string;
|
||||
/** WebSocket channels: the close frame's code and reason. */
|
||||
code?: number;
|
||||
reason?: string;
|
||||
};
|
||||
|
||||
export type SshChannelListener = {
|
||||
onOpen(): void;
|
||||
onText(text: string): void;
|
||||
onStderr?(line: string): void;
|
||||
/** Terminal: the channel ended normally. */
|
||||
onClosed(info: SshClosed): void;
|
||||
/** Terminal: the channel couldn't open or was lost. */
|
||||
onFailure(failure: SshFailure): void;
|
||||
};
|
||||
|
||||
export type SshChannel = {
|
||||
/** Queue a websocket text frame / an exec stdin line. False once closed. */
|
||||
send(text: string): boolean;
|
||||
/** Flush queued sends, then end the channel. No events follow. */
|
||||
close(): void;
|
||||
};
|
||||
|
||||
export type SshClientKey = {
|
||||
/** "ecdsa-sha2-nistp256 AAAA... comment", for ~/.ssh/authorized_keys. */
|
||||
authorizedKey: string;
|
||||
fingerprint: string;
|
||||
};
|
||||
|
||||
/** Build the Go transport's openSpec JSON. */
|
||||
export function encodeSshSpec(endpoint: SshEndpoint, channel: SshChannelSpec): string {
|
||||
return JSON.stringify({
|
||||
host: endpoint.host,
|
||||
port: endpoint.port,
|
||||
user: endpoint.user,
|
||||
password: endpoint.password,
|
||||
hostKeys: endpoint.hostKeys,
|
||||
useKey: endpoint.useKey,
|
||||
channel,
|
||||
});
|
||||
}
|
||||
|
||||
/** Route one transport event to the listener. Unknown kinds are ignored. */
|
||||
export function dispatchSshEvent(listener: SshChannelListener, kind: string, payload: string): void {
|
||||
switch (kind) {
|
||||
case "open":
|
||||
listener.onOpen();
|
||||
return;
|
||||
case "text":
|
||||
listener.onText(payload);
|
||||
return;
|
||||
case "stderr":
|
||||
listener.onStderr?.(payload);
|
||||
return;
|
||||
case "closed":
|
||||
listener.onClosed(parseJson(payload) ?? {});
|
||||
return;
|
||||
case "failure": {
|
||||
const failure = parseJson(payload);
|
||||
listener.onFailure(
|
||||
failure && typeof failure.kind === "string"
|
||||
? failure
|
||||
: { kind: "internal", message: `bad failure payload: ${payload}` },
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseJson(text: string): any {
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Human summary of a failure for status lines. */
|
||||
export function describeSshFailure(failure: SshFailure): string {
|
||||
switch (failure.kind) {
|
||||
case "hostkey-unknown":
|
||||
return "Host key not yet trusted.";
|
||||
case "hostkey-mismatch":
|
||||
return "HOST KEY CHANGED; refusing to connect.";
|
||||
case "auth":
|
||||
return failure.message || "Authentication failed.";
|
||||
default:
|
||||
return failure.message || failure.kind;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import {
|
||||
dispatchSshEvent,
|
||||
encodeSshSpec,
|
||||
type SshChannel,
|
||||
type SshChannelListener,
|
||||
type SshChannelSpec,
|
||||
type SshClientKey,
|
||||
type SshEndpoint,
|
||||
} from "./ssh-common";
|
||||
|
||||
export * from "./ssh-common";
|
||||
|
||||
declare const FaceclawSshChannel: any;
|
||||
declare const FaceclawSshKeys: any;
|
||||
|
||||
// Like socket.ios.ts: Go delivers events on its own threads, so the native
|
||||
// side queues them and this isolate drains the queue on a timer.
|
||||
const POLL_INTERVAL_MS = 25;
|
||||
|
||||
/** iOS SSH transport: FaceclawSshChannel (ObjC) over the gomobile-bound Go package in native/ssh. */
|
||||
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
|
||||
const native = FaceclawSshChannel.alloc().initWithSpecUseKey(encodeSshSpec(endpoint, channel), endpoint.useKey);
|
||||
let ended = false;
|
||||
const stop = () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
clearInterval(timer);
|
||||
native.close();
|
||||
};
|
||||
const timer = setInterval(() => {
|
||||
const events = JSON.parse(String(native.takeEvents())) as Array<{ kind: string; payload: string }>;
|
||||
for (const event of events) {
|
||||
if (ended) break;
|
||||
if (event.kind === "closed" || event.kind === "failure") {
|
||||
// Terminal events: stop polling, then deliver.
|
||||
ended = true;
|
||||
clearInterval(timer);
|
||||
}
|
||||
dispatchSshEvent(listener, event.kind, event.payload);
|
||||
}
|
||||
}, POLL_INTERVAL_MS);
|
||||
return {
|
||||
send: (text) => !ended && native.send(text) === true,
|
||||
close: stop,
|
||||
};
|
||||
}
|
||||
|
||||
export function sshClientKey(comment: string): SshClientKey {
|
||||
const authorizedKey = FaceclawSshKeys.authorizedKeyWithComment(comment);
|
||||
const fingerprint = FaceclawSshKeys.fingerprint();
|
||||
if (!authorizedKey || !fingerprint) throw new Error("Could not create the SSH client key.");
|
||||
return { authorizedKey: String(authorizedKey), fingerprint: String(fingerprint) };
|
||||
}
|
||||
|
||||
export function getSshSecret(id: string): string {
|
||||
return String(FaceclawSshKeys.secretForId(id) ?? "");
|
||||
}
|
||||
|
||||
export function setSshSecret(id: string, value: string): void {
|
||||
FaceclawSshKeys.setSecretForId(value, id);
|
||||
}
|
||||
|
||||
export function removeSshSecret(id: string): void {
|
||||
FaceclawSshKeys.removeSecretForId(id);
|
||||
}
|
||||
|
||||
export function copyToPhoneClipboard(_label: string, text: string): void {
|
||||
FaceclawSshKeys.copyToClipboard(text);
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import {
|
||||
dispatchSshEvent,
|
||||
encodeSshSpec,
|
||||
type SshChannel,
|
||||
type SshChannelListener,
|
||||
type SshChannelSpec,
|
||||
type SshClientKey,
|
||||
type SshEndpoint,
|
||||
} from "./ssh-common";
|
||||
|
||||
export * from "./ssh-common";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Android SSH transport: FaceclawSshChannel (Kotlin) over the gomobile-bound
|
||||
* Go package in native/ssh. Callable from the main isolate or a worker;
|
||||
* events arrive on the calling isolate's thread.
|
||||
*/
|
||||
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
|
||||
let closed = false;
|
||||
const proxy = new com.faceclaw.app.FaceclawSshListener({
|
||||
onEvent: (kind: string, payload: string) => {
|
||||
if (closed) return;
|
||||
dispatchSshEvent(listener, String(kind), payload == null ? "" : String(payload));
|
||||
},
|
||||
});
|
||||
const native = new com.faceclaw.app.FaceclawSshChannel(encodeSshSpec(endpoint, channel), endpoint.useKey, proxy);
|
||||
return {
|
||||
send: (text) => !closed && native.send(text) === true,
|
||||
close: () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
native.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function context(): any {
|
||||
return Utils.android.getApplicationContext();
|
||||
}
|
||||
|
||||
/** The device's SSH client key (generated on first use). */
|
||||
export function sshClientKey(comment: string): SshClientKey {
|
||||
return {
|
||||
authorizedKey: String(com.faceclaw.app.FaceclawSshKeys.authorizedKey(comment)),
|
||||
fingerprint: String(com.faceclaw.app.FaceclawSshKeys.fingerprint()),
|
||||
};
|
||||
}
|
||||
|
||||
export function getSshSecret(id: string): string {
|
||||
return String(com.faceclaw.app.FaceclawSshKeys.getSecret(context(), id) ?? "");
|
||||
}
|
||||
|
||||
export function setSshSecret(id: string, value: string): void {
|
||||
com.faceclaw.app.FaceclawSshKeys.setSecret(context(), id, value);
|
||||
}
|
||||
|
||||
export function removeSshSecret(id: string): void {
|
||||
com.faceclaw.app.FaceclawSshKeys.removeSecret(context(), id);
|
||||
}
|
||||
|
||||
export function copyToPhoneClipboard(label: string, text: string): void {
|
||||
com.faceclaw.app.FaceclawSshKeys.copyToClipboard(context(), label, text);
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Session-list and connection-state types shared by the Terminal app's
|
||||
* backends (g2mirror-client.ts, apps/terminal/tmux-client.ts). Kept free of
|
||||
* platform imports so the tmux client can be unit-tested under node.
|
||||
*/
|
||||
|
||||
export type G2MirrorSession = {
|
||||
socket: string;
|
||||
pid: number;
|
||||
cwdHint: string;
|
||||
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
|
||||
lastBellAt: number | null;
|
||||
/**
|
||||
* Unix epoch ms (host clock) of the app's last output, or null if none
|
||||
* observed. Fresh to within the wrapper's 2s activity reporting interval
|
||||
* while we're connected; compare against other host timestamps, not the
|
||||
* phone clock (see the worker's local-receive-time activity tracking).
|
||||
*/
|
||||
lastOutputAt: number | null;
|
||||
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
|
||||
title: string | null;
|
||||
};
|
||||
|
||||
export type G2MirrorPhase =
|
||||
| "idle"
|
||||
| "connecting"
|
||||
| "connected" // handshake accepted; can list/attach
|
||||
| "attached" // relaying to one session
|
||||
| "failed";
|
||||
|
||||
export type G2MirrorState = {
|
||||
phase: G2MirrorPhase;
|
||||
status: string;
|
||||
sessions: G2MirrorSession[];
|
||||
attachedCommand: string;
|
||||
/**
|
||||
* server_name from the init success reply: a human-readable name for the
|
||||
* machine the server runs on. "" until the handshake succeeds (or when an
|
||||
* older server omits the field).
|
||||
*/
|
||||
serverName: string;
|
||||
/**
|
||||
* False when reconnecting can't help until the user changes something on
|
||||
* the host (e.g. tmux missing or too old). Absent means retryable.
|
||||
*/
|
||||
retryable?: boolean;
|
||||
};
|
||||
|
||||
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
|
||||
export type G2MirrorHistoryReply = {
|
||||
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
|
||||
start: number;
|
||||
/** Oldest index still retained by the wrapper (older requests return nothing). */
|
||||
oldest: number;
|
||||
/** Current splice point (next line to be archived). */
|
||||
next: number;
|
||||
/** Plain text, ANSI-stripped, oldest-to-newest. */
|
||||
lines: string[];
|
||||
};
|
||||
@@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
|
||||
import type { ReconnectingSttClient } from './reconnecting-stt'
|
||||
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
|
||||
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
|
||||
import { WavRecorder } from './wav-recorder.ios'
|
||||
import { MicRecorder } from './mic-recorder.ios'
|
||||
import { pcmLevel } from '../util/audio-level'
|
||||
|
||||
declare const FaceclawSpeech: any
|
||||
@@ -57,11 +57,11 @@ export class IosVoiceControlBridge {
|
||||
private continuous = false
|
||||
// Shown instead of the native "ended" status once a drain the bridge started completes.
|
||||
private drainStatus: string | null = null
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
private recordingName = ''
|
||||
private recorder: WavRecorder | null = null
|
||||
private recorder: MicRecorder | null = null
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false
|
||||
private readonly recordingListeners = new Set<() => void>()
|
||||
@@ -209,7 +209,7 @@ export class IosVoiceControlBridge {
|
||||
/** Where a listening continuous capture gets its audio; null when none is listening. */
|
||||
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or ''. Only cloud captures supply PCM.
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
@@ -232,7 +232,10 @@ export class IosVoiceControlBridge {
|
||||
private record(pcm: Uint8Array): void {
|
||||
if (!this.recording) return
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName)
|
||||
this.recorder = MicRecorder.open(this.recordingName, {
|
||||
recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios',
|
||||
source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1,
|
||||
})
|
||||
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
|
||||
}
|
||||
|
||||
+319
-27
@@ -1,10 +1,10 @@
|
||||
import type { OpenAiSttModel } from "./openai-stt";
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt";
|
||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||
import { newRecordingPath } from "./mic-recording";
|
||||
import { SpeechPauseDetector } from "./speech-pause";
|
||||
import { WavRecorder } from "./wav-recorder";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { pcmLevel } from "../util/audio-level";
|
||||
|
||||
@@ -86,6 +86,49 @@ export type PushToTalkOptions = {
|
||||
* frame). Null/absent leaves the mic omnidirectional.
|
||||
*/
|
||||
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
|
||||
/**
|
||||
* Capture from the temples' own microphones (the custom firmware's
|
||||
* mic_control stream) instead of the glasses' mixed stream. Each temple
|
||||
* streams its microphones on its own link and clock; the phone mixes them
|
||||
* into the mono feed that is transcribed. Null/absent keeps the mixed
|
||||
* stream; ignored with usePhoneMic.
|
||||
*/
|
||||
micArray?: MicArrayCapture | null;
|
||||
/**
|
||||
* Play this recording (a .mka file from Save Recording) in real time in
|
||||
* place of any microphone. The capture needs no glasses, and ends by
|
||||
* itself when the recording does.
|
||||
*/
|
||||
replay?: { path: string; startMs: number } | null;
|
||||
/** What the caller knows of the capture, for the description stored in a recording of it. */
|
||||
captureInfo?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type MicArrayCapture = {
|
||||
/** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */
|
||||
configure: Uint8Array;
|
||||
};
|
||||
|
||||
/** Progress of a capture that replays a recording; positions are ms into it. */
|
||||
export type VoiceReplayEvent = {
|
||||
/**
|
||||
* `start` once the file is open, `position` about once per second played,
|
||||
* `seek` when a jump took effect, `end` when the recording ran out, and
|
||||
* `error` when it could not be opened (the status says why).
|
||||
*/
|
||||
kind: "start" | "position" | "seek" | "end" | "error";
|
||||
positionMs: number;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */
|
||||
export type VoiceEventSource = {
|
||||
onStatus(listener: (state: VoiceControlState) => void): () => void;
|
||||
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void;
|
||||
onSpeechPause(listener: () => void): () => void;
|
||||
onMicLevel(listener: (level: number) => void): () => void;
|
||||
readonly recording: boolean;
|
||||
readonly recordingFailed: boolean;
|
||||
};
|
||||
|
||||
/** Who currently wants the mic running. */
|
||||
@@ -132,9 +175,13 @@ export class FaceclawVoiceControlBridge {
|
||||
private captureEndpointing = false;
|
||||
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
|
||||
private cloudClient: CloudSttClient | null = null;
|
||||
// What the running STT stream transcribes with (transcriptionKey); a holder
|
||||
// asking for something else restarts the stream rather than sharing it.
|
||||
// What the running STT stream captures from and transcribes with
|
||||
// (streamKey); a holder asking for something else restarts the stream
|
||||
// rather than sharing it.
|
||||
private streamProvider: string | null = null;
|
||||
// The options of the running stream, for the description of a recording of it.
|
||||
private streamOptions: PushToTalkOptions | null = null;
|
||||
private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>();
|
||||
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
|
||||
// controller runs in the decode-only "cloud" mode and every decoded PCM
|
||||
// frame is broadcast to these listeners. STT capture preempts it — a live
|
||||
@@ -152,11 +199,14 @@ export class FaceclawVoiceControlBridge {
|
||||
// the bridge swallow it.
|
||||
private verificationActive = false;
|
||||
private verificationRejected = false;
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||
// and it is created by the first audio, so a failed start leaves no file.
|
||||
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
|
||||
// set: every source's chunks as they arrive, before any processing, written
|
||||
// by the controller. The file outlives a capture, so a restarted one keeps
|
||||
// adding to it, and it is created by the first audio, so a failed start
|
||||
// leaves no file.
|
||||
private recordingName = "";
|
||||
private recorder: WavRecorder | null = null;
|
||||
// The file is open in the controller.
|
||||
private recordingOpen = false;
|
||||
// The file could not be created: one attempt per recording, not one per packet.
|
||||
private recordingFailure = false;
|
||||
private readonly recordingListeners = new Set<() => void>();
|
||||
@@ -253,16 +303,55 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||
* End the continuous capture as an utterance: stop it and have the
|
||||
* transcript of its last audio come as a final. For a capture whose audio
|
||||
* ran out by itself (a replay).
|
||||
*/
|
||||
finishContinuousCapture(): void {
|
||||
this.releaseCapture("continuous", true);
|
||||
}
|
||||
|
||||
/** Progress of a capture that replays a recording (PushToTalkOptions.replay). */
|
||||
onReplay(listener: (event: VoiceReplayEvent) => void): () => void {
|
||||
this.replayListeners.add(listener);
|
||||
return () => this.replayListeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Continue the running replay from `positionMs` into its recording. */
|
||||
seekReplay(positionMs: number): void {
|
||||
this.controller?.seekReplay(Math.max(0, Math.round(positionMs)));
|
||||
}
|
||||
|
||||
/** Hold the running replay where it is, or let it go on. */
|
||||
setReplayPaused(paused: boolean): void {
|
||||
this.controller?.setReplayPaused(paused);
|
||||
}
|
||||
|
||||
/** Whether a capture is running and still has audio coming (a replay stops being so at its end). */
|
||||
isCaptureLive(): boolean {
|
||||
return this.micIsLive();
|
||||
}
|
||||
|
||||
/**
|
||||
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
|
||||
* until called with another name or "".
|
||||
*/
|
||||
recordTo(name: string): void {
|
||||
if (name === this.recordingName) return;
|
||||
this.recorder?.close();
|
||||
this.recorder = null;
|
||||
this.closeRecording();
|
||||
this.setRecording(name, false);
|
||||
}
|
||||
|
||||
private closeRecording(): void {
|
||||
if (!this.recordingOpen) return;
|
||||
this.recordingOpen = false;
|
||||
try {
|
||||
this.controller?.stopRecording();
|
||||
} catch (error) {
|
||||
console.warn(`Voice: closing the recording failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||
get recording(): boolean {
|
||||
return this.recordingName !== "" && !this.recordingFailure;
|
||||
@@ -288,18 +377,48 @@ export class FaceclawVoiceControlBridge {
|
||||
for (const listener of Array.from(this.recordingListeners)) listener();
|
||||
}
|
||||
|
||||
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
|
||||
private record(pcm: any): void {
|
||||
if (!this.recording) return;
|
||||
if (!this.recorder) {
|
||||
this.recorder = WavRecorder.open(this.recordingName);
|
||||
if (!this.recorder) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
console.log(`Voice: recording audio to ${this.recorder.path}`);
|
||||
/**
|
||||
* Audio arrived for a continuous capture: open the recording asked for, if
|
||||
* it isn't open yet. The controller writes it from then on, as the audio
|
||||
* arrives on its own thread.
|
||||
*/
|
||||
private ensureRecording(): void {
|
||||
if (!this.recording || this.recordingOpen) return;
|
||||
const path = newRecordingPath(this.recordingName);
|
||||
let opened = false;
|
||||
try {
|
||||
opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture())));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: could not create the recording: ${error}`);
|
||||
}
|
||||
this.recorder.append(pcm);
|
||||
if (!opened) {
|
||||
this.setRecording(this.recordingName, true);
|
||||
return;
|
||||
}
|
||||
this.recordingOpen = true;
|
||||
console.log(`Voice: recording audio to ${path}`);
|
||||
}
|
||||
|
||||
/** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */
|
||||
private syncRecordingEnabled(): void {
|
||||
this.controller?.setRecordingEnabled(this.captureHolders.has("continuous"));
|
||||
}
|
||||
|
||||
/** The running capture, as the JSON stored in a recording of it. */
|
||||
private describeCapture(): Record<string, unknown> {
|
||||
const options = this.streamOptions;
|
||||
return {
|
||||
recording: this.recordingName,
|
||||
startedAt: new Date().toISOString(),
|
||||
source: options?.usePhoneMic ? "phone" : "glasses",
|
||||
micArray: Boolean(options?.micArray) && !options?.usePhoneMic,
|
||||
transcription: { provider: options?.provider ?? "", language: options?.language ?? "" },
|
||||
processing: {
|
||||
noiseSuppression: Boolean(options?.noiseSuppression),
|
||||
beamFilter: options?.beamFilter ?? null,
|
||||
},
|
||||
...options?.captureInfo,
|
||||
};
|
||||
}
|
||||
|
||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||
@@ -366,6 +485,8 @@ export class FaceclawVoiceControlBridge {
|
||||
// preview-mode capture may have left on the shared controller.
|
||||
this.controller?.setUsePhoneMic(false);
|
||||
this.controller?.setEndpointing(false);
|
||||
this.controller?.setMicArray(null);
|
||||
this.controller?.setReplay(null, 0);
|
||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||
// unprocessed stream and run their own beam/suppression processing.
|
||||
this.controller?.setNoiseSuppression(false);
|
||||
@@ -401,18 +522,33 @@ export class FaceclawVoiceControlBridge {
|
||||
const micLive = this.micIsLive();
|
||||
this.suspendedHolders.delete(holder);
|
||||
this.captureHolders.add(holder);
|
||||
if (micLive && this.streamProvider === transcriptionKey(options)) {
|
||||
if (micLive && this.streamProvider === streamKey(options)) {
|
||||
// Mic already running; the new holder just shares the existing stream
|
||||
// (transcripts are already broadcast to its listeners).
|
||||
this.syncRecordingEnabled();
|
||||
return;
|
||||
}
|
||||
// Any leftover capture is dead, or transcribes with another provider than
|
||||
// Any leftover capture is dead, or captures or transcribes otherwise than
|
||||
// this holder needs; drop it before starting the new one.
|
||||
if (this.started) this.teardownCapture();
|
||||
this.streamProvider = transcriptionKey(options);
|
||||
this.streamProvider = streamKey(options);
|
||||
this.streamOptions = options;
|
||||
this.ensureController();
|
||||
this.syncRecordingEnabled();
|
||||
// A recording under way carries on into this capture: note what changed.
|
||||
if (this.recordingOpen) {
|
||||
try {
|
||||
this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() }));
|
||||
} catch (error) {
|
||||
console.warn(`Voice: recording event failed: ${error}`);
|
||||
}
|
||||
}
|
||||
this.controller?.setCommunicator(options.communicator);
|
||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||
this.controller?.setMicArray(
|
||||
options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null,
|
||||
);
|
||||
this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0)));
|
||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||
this.captureEndpointing = Boolean(options.endpointing);
|
||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||
@@ -437,7 +573,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.speechPause.reset();
|
||||
// Replaces whatever the last capture left behind ("Listening...", an
|
||||
// error, the permission prompt) until the controller reports progress.
|
||||
this.setStatus("Starting microphone...");
|
||||
this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone...");
|
||||
// A previous push-to-talk commit may still be awaiting its final result.
|
||||
this.cloudClient?.stop();
|
||||
const cloudClient = this.createCloudClient(options);
|
||||
@@ -485,6 +621,7 @@ export class FaceclawVoiceControlBridge {
|
||||
if (commit) this.cloudClient?.finish();
|
||||
return;
|
||||
}
|
||||
this.syncRecordingEnabled();
|
||||
if (this.captureHolders.size > 0) {
|
||||
// Another holder still wants the mic; keep it running.
|
||||
return;
|
||||
@@ -509,6 +646,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.captureHolders.clear();
|
||||
this.suspendedHolders.clear();
|
||||
this.suspendedRaw = false;
|
||||
this.syncRecordingEnabled();
|
||||
this.teardownCapture();
|
||||
this.setStatus("Voice control stopped.");
|
||||
}
|
||||
@@ -526,6 +664,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.suspendedHolders.add(holder);
|
||||
}
|
||||
this.captureHolders.clear();
|
||||
this.syncRecordingEnabled();
|
||||
this.suspendedRaw = this.rawActive;
|
||||
this.teardownCapture();
|
||||
if (this.suspendedHolders.size > 0) {
|
||||
@@ -583,6 +722,7 @@ export class FaceclawVoiceControlBridge {
|
||||
private teardownCapture(): void {
|
||||
this.rawActive = false;
|
||||
this.streamProvider = null;
|
||||
this.streamOptions = null;
|
||||
if (global.isAndroid) {
|
||||
this.controller?.stop();
|
||||
}
|
||||
@@ -616,7 +756,7 @@ export class FaceclawVoiceControlBridge {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
// Push-to-talk on a capture of its own is not recorded.
|
||||
if (this.captureHolders.has("continuous")) this.record(pcm);
|
||||
if (this.captureHolders.has("continuous")) this.ensureRecording();
|
||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||
for (const listener of this.speechPauseListeners) listener();
|
||||
}
|
||||
@@ -647,6 +787,10 @@ export class FaceclawVoiceControlBridge {
|
||||
);
|
||||
}
|
||||
},
|
||||
onReplay: (kind: string, positionMs: number, durationMs: number) => {
|
||||
const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) };
|
||||
for (const listener of Array.from(this.replayListeners)) listener(event);
|
||||
},
|
||||
});
|
||||
this.controller.setListener(this.listenerProxy);
|
||||
}
|
||||
@@ -683,6 +827,21 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture reads and transcribes with. A running capture is shared only
|
||||
* by holders asking for the same: the provider and language, the microphones
|
||||
* (the mixed stream, or the temples' own under a given configuration), or the
|
||||
* recording it replays.
|
||||
*/
|
||||
function streamKey(options: PushToTalkOptions): string {
|
||||
const source = options.replay
|
||||
? `replay:${options.replay.path}`
|
||||
: options.micArray && !options.usePhoneMic
|
||||
? `array:${Array.from(options.micArray.configure).join(",")}`
|
||||
: "mixed";
|
||||
return `${transcriptionKey(options)}|${source}`;
|
||||
}
|
||||
|
||||
function toJavaFloats(values: Float32Array): any {
|
||||
const javaFloats = Array.create("float", values.length);
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
@@ -692,3 +851,136 @@ function toJavaFloats(values: Float32Array): any {
|
||||
}
|
||||
|
||||
export const voiceControlBridge = new FaceclawVoiceControlBridge();
|
||||
|
||||
export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed";
|
||||
|
||||
/**
|
||||
* A recording played back as a capture of its own: transcribed by the same
|
||||
* pipeline as the microphone, with the same settings, on a bridge and native
|
||||
* controller separate from the microphone's. Nothing here touches the mic,
|
||||
* so voice input and a live Transcribe window carry on beside a replay.
|
||||
*/
|
||||
export class VoiceReplay {
|
||||
private readonly bridge = new FaceclawVoiceControlBridge();
|
||||
private readonly listeners = new Set<(jumpedToMs?: number) => void>();
|
||||
private readonly offReplay: () => void;
|
||||
private position = 0;
|
||||
private duration = 0;
|
||||
private current: VoiceReplayState = "opening";
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* `options` gives the capture options (provider, keys, processing) each
|
||||
* time the replay starts, so a restart picks up changed settings.
|
||||
*/
|
||||
constructor(readonly path: string, private readonly options: () => PushToTalkOptions) {
|
||||
this.offReplay = this.bridge.onReplay((event) => this.handle(event));
|
||||
}
|
||||
|
||||
/** The replay's status, transcript and level events. */
|
||||
get events(): VoiceEventSource {
|
||||
return this.bridge;
|
||||
}
|
||||
|
||||
/** Milliseconds into the recording. */
|
||||
get positionMs(): number {
|
||||
return this.position;
|
||||
}
|
||||
|
||||
/** The recording's length in ms; 0 until it has been opened. */
|
||||
get durationMs(): number {
|
||||
return this.duration;
|
||||
}
|
||||
|
||||
get state(): VoiceReplayState {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
/**
|
||||
* The position, duration or state changed. `jumpedToMs` is given when
|
||||
* playback moved there other than by playing on: a jump, or a start
|
||||
* part-way in. It comes after the transcript of what played before.
|
||||
*/
|
||||
onChange(listener: (jumpedToMs?: number) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
/** Play from `positionMs` into the recording. */
|
||||
start(positionMs = 0): void {
|
||||
if (this.closed) return;
|
||||
this.position = Math.max(0, positionMs);
|
||||
this.set("opening");
|
||||
this.bridge.startContinuousCapture({
|
||||
...this.options(),
|
||||
communicator: null,
|
||||
usePhoneMic: false,
|
||||
micArray: null,
|
||||
replay: { path: this.path, startMs: this.position },
|
||||
});
|
||||
}
|
||||
|
||||
/** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */
|
||||
jumpTo(positionMs: number): void {
|
||||
if (this.closed) return;
|
||||
const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs);
|
||||
if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) {
|
||||
this.bridge.seekReplay(target);
|
||||
this.position = target;
|
||||
this.notify();
|
||||
return;
|
||||
}
|
||||
this.start(target);
|
||||
}
|
||||
|
||||
setPaused(paused: boolean): void {
|
||||
if (this.closed || (this.current !== "playing" && this.current !== "paused")) return;
|
||||
this.bridge.setReplayPaused(paused);
|
||||
this.set(paused ? "paused" : "playing");
|
||||
}
|
||||
|
||||
/** Stop for good and release the capture. */
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
this.offReplay();
|
||||
this.listeners.clear();
|
||||
this.bridge.stop();
|
||||
}
|
||||
|
||||
private handle(event: VoiceReplayEvent): void {
|
||||
this.duration = event.durationMs;
|
||||
switch (event.kind) {
|
||||
case "start":
|
||||
this.position = event.positionMs;
|
||||
this.current = "playing";
|
||||
this.notify(event.positionMs > 0 ? event.positionMs : undefined);
|
||||
return;
|
||||
case "seek":
|
||||
this.position = event.positionMs;
|
||||
this.notify(event.positionMs);
|
||||
return;
|
||||
case "end":
|
||||
this.position = event.durationMs;
|
||||
// Its last audio is still to be transcribed (a cloud provider's final).
|
||||
this.bridge.finishContinuousCapture();
|
||||
this.set("ended");
|
||||
return;
|
||||
case "error":
|
||||
this.set("failed");
|
||||
return;
|
||||
default:
|
||||
this.position = event.positionMs;
|
||||
this.notify();
|
||||
}
|
||||
}
|
||||
|
||||
private set(state: VoiceReplayState): void {
|
||||
this.current = state;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
private notify(jumpedToMs?: number): void {
|
||||
for (const listener of Array.from(this.listeners)) listener(jumpedToMs);
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
|
||||
export function wavHeader(dataBytes: number): Uint8Array {
|
||||
const header = new Uint8Array(44);
|
||||
const view = new DataView(header.buffer);
|
||||
const ascii = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
|
||||
};
|
||||
ascii(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataBytes, true);
|
||||
ascii(8, "WAVE");
|
||||
ascii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true); // PCM
|
||||
view.setUint16(22, 1, true); // mono
|
||||
view.setUint32(24, 16000, true);
|
||||
view.setUint32(28, 32000, true); // byte rate
|
||||
view.setUint16(32, 2, true); // block align
|
||||
view.setUint16(34, 16, true); // bits per sample
|
||||
ascii(36, "data");
|
||||
view.setUint32(40, dataBytes, true);
|
||||
return header;
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { knownFolders, path as paths } from '@nativescript/core'
|
||||
import { wavHeader } from './wav-header'
|
||||
|
||||
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
|
||||
const FLUSH_BYTES = 32000
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
|
||||
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private pending: Uint8Array[] = []
|
||||
private pendingBytes = 0
|
||||
private dataBytes = 0
|
||||
|
||||
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
const fm = NSFileManager.defaultManager
|
||||
const dir = paths.join(knownFolders.documents().path, 'Downloads')
|
||||
try {
|
||||
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
|
||||
let path = paths.join(dir, `${name}.wav`)
|
||||
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
|
||||
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
|
||||
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
|
||||
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
|
||||
return new WavRecorder(handle, path)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
append(pcm: Uint8Array): void {
|
||||
if (!this.handle || !pcm.length) return
|
||||
this.pending.push(pcm.slice())
|
||||
this.pendingBytes += pcm.length
|
||||
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.flush()
|
||||
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
|
||||
this.handle = null
|
||||
}
|
||||
|
||||
private flush(): void {
|
||||
const handle = this.handle
|
||||
if (!handle || !this.pendingBytes) return
|
||||
const chunk = new Uint8Array(this.pendingBytes)
|
||||
let offset = 0
|
||||
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
|
||||
this.pending = []
|
||||
this.pendingBytes = 0
|
||||
try {
|
||||
// NativeScript throws a failed call's NSError (storage full, file removed).
|
||||
handle.writeDataError(toData(chunk))
|
||||
handle.seekToOffsetError(0)
|
||||
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
|
||||
handle.seekToEndReturningOffsetError(null)
|
||||
this.dataBytes += chunk.length
|
||||
return
|
||||
} catch {
|
||||
// Stop recording rather than retry.
|
||||
}
|
||||
try { handle.closeAndReturnError() } catch { /* ignore */ }
|
||||
this.handle = null
|
||||
}
|
||||
}
|
||||
|
||||
function toData(bytes: Uint8Array): NSData {
|
||||
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
declare const android: any;
|
||||
declare const java: any;
|
||||
declare const com: any;
|
||||
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
/**
|
||||
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
|
||||
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
|
||||
* native recorder keeps the header's sizes current, so a crash leaves a
|
||||
* playable file of everything up to the last second.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private constructor(private recorder: any, readonly path: string) {}
|
||||
|
||||
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||
static open(name: string): WavRecorder | null {
|
||||
try {
|
||||
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||
);
|
||||
downloads.mkdirs();
|
||||
let file = new java.io.File(downloads, `${name}.wav`);
|
||||
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
|
||||
const path = String(file.getAbsolutePath());
|
||||
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
|
||||
} catch (error) {
|
||||
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
|
||||
append(pcm: any): void {
|
||||
this.recorder?.append(pcm);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.recorder?.finish();
|
||||
this.recorder = null;
|
||||
}
|
||||
}
|
||||
@@ -1063,7 +1063,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
|
||||
storageKey: "developer.useMicControl",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
|
||||
"Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.",
|
||||
});
|
||||
|
||||
export const showBleBandwidthSetting = new ConfigSettingBoolean({
|
||||
@@ -1171,7 +1171,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({
|
||||
label: "Save Recording",
|
||||
storageKey: "voice.saveRecording",
|
||||
defaultValue: false,
|
||||
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
|
||||
description: global.isIOS
|
||||
? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them."
|
||||
: "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.",
|
||||
});
|
||||
|
||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
@@ -1481,11 +1483,28 @@ export const terminalNewConnectionSetting = new ConfigSettingString({
|
||||
label: "New connection",
|
||||
storageKey: "terminal.newConnectionDraft",
|
||||
defaultValue: "",
|
||||
editorTitle: "g2mirror connection string (g2mirror://token@host)",
|
||||
editorTitle: "Connection (ssh://user@host or g2mirror://token@host)",
|
||||
glassesEditTitle: "Add connection",
|
||||
normalize: (value) => (value ?? "").replace(/[\x00-\x1f]+/g, "").trim(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Staging buffer for the Terminal app's SSH password prompt: the worker opens
|
||||
* the phone's (masked) text editor on it, then moves what was typed into the
|
||||
* platform keystore (app/native/ssh) and clears it as soon as the user
|
||||
* confirms or cancels on the glasses. Not listed in the Settings app.
|
||||
*/
|
||||
export const terminalSshPasswordSetting = new ConfigSettingString({
|
||||
id: "terminal-ssh-password",
|
||||
label: "SSH password",
|
||||
storageKey: "terminal.sshPasswordDraft",
|
||||
defaultValue: "",
|
||||
inputKind: "password",
|
||||
editorTitle: "SSH password",
|
||||
glassesEditTitle: "SSH password",
|
||||
normalize: (value) => (value ?? "").replace(/[\r\n]+/g, ""),
|
||||
});
|
||||
|
||||
/**
|
||||
* Staging buffer for the T3 Code app's Pair-environment screen: the phone
|
||||
* text editor types the `t3 pair` link into it, and the app reads it back
|
||||
@@ -1865,9 +1884,18 @@ type SettingsMenuOptions<T> = {
|
||||
label?: string
|
||||
}
|
||||
|
||||
type EnumSettingMenuOptions<T> = SettingsMenuOptions<T> & {
|
||||
/**
|
||||
* A live line under the picker's values, such as a sensor reading to pick
|
||||
* by. open() runs as the picker opens, with a function that repaints it,
|
||||
* and returns what to run when the picker closes.
|
||||
*/
|
||||
pickerFooter?: { text: () => string; open: (repaint: () => void) => () => void }
|
||||
}
|
||||
|
||||
export function enumSettingMenuItem<TValue extends string, TId extends string = string>(
|
||||
setting: ConfigSettingEnum<TValue, TId>,
|
||||
opts?: SettingsMenuOptions<TValue>
|
||||
opts?: EnumSettingMenuOptions<TValue>
|
||||
): MenuItem {
|
||||
const label = opts?.label ?? setting.label;
|
||||
return {
|
||||
@@ -1895,7 +1923,11 @@ export function enumSettingMenuItem<TValue extends string, TId extends string =
|
||||
);
|
||||
},
|
||||
}));
|
||||
openModalMenu(ctx, label, items, Math.max(0, setting.values.indexOf(current)));
|
||||
const footer = opts?.pickerFooter;
|
||||
openModalMenu(ctx, label, items, Math.max(0, setting.values.indexOf(current)), {
|
||||
footer: footer?.text,
|
||||
onClosed: footer?.open(() => ctx.actions.requestRender()),
|
||||
});
|
||||
},
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, setting.displayValue(setting.get()));
|
||||
|
||||
@@ -112,6 +112,8 @@ import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from ".
|
||||
import { uiFontPickerMenuItem } from "../font-picker";
|
||||
import { NotificationFilterLayer } from "../notification-filter";
|
||||
import { compatibleModelDiscoveryMenuItem } from "../openai-compatible-model-picker";
|
||||
import { keyboardSettingsMenuItem } from "../keyboard-settings";
|
||||
import { keybindingsMenuItem } from "../keybinding-settings";
|
||||
import { isVeniceApiBaseUrl } from "../../native/openai-chat";
|
||||
|
||||
/** The Settings section the Notifications app's "Settings" menu item opens. */
|
||||
@@ -266,6 +268,14 @@ function settingsSections(): SettingsSection[] {
|
||||
],
|
||||
renderDetail: renderWatchStatus,
|
||||
},
|
||||
// A Bluetooth keyboard paired with the glasses themselves (the G2's
|
||||
// custom firmware hosts it; app/g2/keyboard/). Hidden rather than left
|
||||
// out: the Settings window can be open across a change of glasses.
|
||||
{
|
||||
label: "Keyboard",
|
||||
items: [keyboardSettingsMenuItem(), keybindingsMenuItem()],
|
||||
hidden: () => glassesTypeSetting.get() !== "g2",
|
||||
},
|
||||
{
|
||||
label: "Developer",
|
||||
items: [
|
||||
|
||||
@@ -28,6 +28,13 @@ export type SettingsSection = {
|
||||
height: number;
|
||||
ctx: LayerContext;
|
||||
}) => number | void;
|
||||
/**
|
||||
* Leave the section out of the list while this returns true. Re-read on
|
||||
* every paint and input, for a section that only applies with some other
|
||||
* setting (the paired glasses' type): the Settings window outlives changes
|
||||
* to those.
|
||||
*/
|
||||
hidden?: () => boolean;
|
||||
};
|
||||
|
||||
const LEFT_W = 150;
|
||||
@@ -66,7 +73,7 @@ export class SettingsPanelLayer implements Layer {
|
||||
|
||||
constructor(private readonly sections: SettingsSection[]) {
|
||||
this.leftMenu = new Menu<SettingsSection>({
|
||||
items: sections,
|
||||
items: sections.filter((section) => !section.hidden?.()),
|
||||
wrap: true,
|
||||
rowGap: ROW_GAP,
|
||||
highlight: { radius: 6 },
|
||||
@@ -110,9 +117,28 @@ export class SettingsPanelLayer implements Layer {
|
||||
return this.leftMenu.selectedItem ?? this.sections[0]!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bring the left column in line with which sections are hidden now. The
|
||||
* highlight stays on its section; if that one just went away it moves to
|
||||
* the list's start, with focus back in the left column.
|
||||
*/
|
||||
private syncSections(): void {
|
||||
const visible = this.sections.filter((section) => !section.hidden?.());
|
||||
const shown = this.leftMenu.items;
|
||||
if (visible.length === shown.length && visible.every((section, index) => section === shown[index])) return;
|
||||
const selected = this.leftMenu.selectedItem;
|
||||
const index = selected ? visible.indexOf(selected) : -1;
|
||||
this.leftMenu.setItems(visible, Math.max(0, index));
|
||||
if (index < 0) {
|
||||
this.focus = "left";
|
||||
this.resetRight();
|
||||
}
|
||||
}
|
||||
|
||||
/** Select a left-column section by label (deep link, e.g. from an app's menu). */
|
||||
focusSection(label: string): void {
|
||||
const index = this.sections.findIndex((section) => section.label === label);
|
||||
this.syncSections();
|
||||
const index = this.leftMenu.items.findIndex((section) => section.label === label);
|
||||
if (index < 0) return;
|
||||
this.leftMenu.select(index);
|
||||
this.focus = "left";
|
||||
@@ -125,6 +151,7 @@ export class SettingsPanelLayer implements Layer {
|
||||
const { width, height, padding } = viewport;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const appFocused = ctx.stack.isFocused();
|
||||
this.syncSections();
|
||||
const section = this.section();
|
||||
const rightItems = section.items;
|
||||
this.rightMenu.setItems(rightItems);
|
||||
@@ -230,6 +257,7 @@ export class SettingsPanelLayer implements Layer {
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
this.syncSections();
|
||||
const scroll = scrollEventFor(event);
|
||||
if (this.focus === "left") {
|
||||
if (scroll) {
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { getDefaultSmallFont } from "../graphics/ui-fonts";
|
||||
import {
|
||||
DEFAULT_KEYS_HELP,
|
||||
KEY_ACTIONS,
|
||||
chordId,
|
||||
describeChord,
|
||||
keyActionLabel,
|
||||
type KeyChord,
|
||||
} from "../g2/keyboard/bindings";
|
||||
import { keyboardService } from "../g2/keyboard/service";
|
||||
import { TextViewerLayer } from "../apps/files/text-viewer";
|
||||
import type { LayerContext } from "./layers";
|
||||
import { drawRightValueMenuItem, MenuLayer, openModalMenu, submenuItem, TextPageLayer, type MenuItem } from "./menu";
|
||||
|
||||
/**
|
||||
* Settings > Keyboard > Keybindings: the custom keybindings (a key, with
|
||||
* any modifiers held, to a gesture), which come before the built-in
|
||||
* meanings in app/g2/keyboard/bindings.ts. Adding one asks for the key on
|
||||
* the keyboard itself, so it works for a remote whose buttons send who
|
||||
* knows what.
|
||||
*/
|
||||
|
||||
/** Waits for the key to bind. Double tap backs out, as from any page. */
|
||||
class KeyCaptureLayer extends TextPageLayer {
|
||||
private cancel: (() => void) | null = null;
|
||||
|
||||
constructor(private readonly onChord: (chord: KeyChord, ctx: LayerContext) => void) {
|
||||
super(
|
||||
"Add keybinding",
|
||||
"Press the key to bind, on the keyboard: a key, a mouse button, or a turn of the wheel. Hold Shift, Ctrl or Alt with it to bind the combination.",
|
||||
);
|
||||
}
|
||||
|
||||
open(ctx: LayerContext): void {
|
||||
ctx.stack.push(this);
|
||||
this.cancel = keyboardService.captureNextKey((chord) => {
|
||||
this.cancel = null;
|
||||
ctx.stack.popThrough(this);
|
||||
this.onChord(chord, ctx);
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.cancel?.();
|
||||
this.cancel = null;
|
||||
}
|
||||
}
|
||||
|
||||
class KeybindingsLayer extends MenuLayer {
|
||||
constructor(viewport: { width: number; height: number }) {
|
||||
super("Keybindings", [], {
|
||||
x: "center",
|
||||
y: 8,
|
||||
width: Math.min(320, viewport.width - 40),
|
||||
minHeight: Math.min(160, viewport.height - 40),
|
||||
maxHeight: Math.min(240, viewport.height - 40),
|
||||
depth: 4,
|
||||
});
|
||||
}
|
||||
|
||||
open(ctx: LayerContext): void {
|
||||
this.updateRows();
|
||||
ctx.stack.push(this);
|
||||
}
|
||||
|
||||
/** Rebuild the rows, with the highlight on `selectId`'s keybinding when given. */
|
||||
private updateRows(selectId?: string): void {
|
||||
const bindings = keyboardService.keyBindings();
|
||||
const rows: MenuItem[] = bindings.map((binding) => {
|
||||
const label = describeChord(binding.chord);
|
||||
return {
|
||||
label,
|
||||
onSelect: (ctx: LayerContext) => this.pickAction(ctx, binding.chord),
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, keyActionLabel(binding.action));
|
||||
},
|
||||
};
|
||||
});
|
||||
rows.push(
|
||||
{ label: "Add keybinding", onSelect: (ctx) => this.add(ctx) },
|
||||
// Scrolls, where a page would cut the text off in a short window.
|
||||
submenuItem("Default keys", (ctx) => ctx.stack.push(new TextViewerLayer(DEFAULT_KEYS_HELP, "Default keys"))),
|
||||
);
|
||||
const selected = selectId === undefined ? -1 : bindings.findIndex((binding) => chordId(binding.chord) === selectId);
|
||||
if (selected >= 0) this.setItems(rows, selected);
|
||||
else this.setItems(rows);
|
||||
}
|
||||
|
||||
private add(ctx: LayerContext): void {
|
||||
if (!keyboardService.bridge.isConnected()) {
|
||||
ctx.stack.push(new TextPageLayer(
|
||||
"Add keybinding",
|
||||
"No keyboard is connected. Pair or reconnect one under Bluetooth keyboard, then come back: the key to bind is read from the keyboard.",
|
||||
));
|
||||
return;
|
||||
}
|
||||
new KeyCaptureLayer((chord, captureCtx) => this.pickAction(captureCtx, chord)).open(ctx);
|
||||
}
|
||||
|
||||
/** Choose what `chord` does: for a new keybinding, or to change or remove one. */
|
||||
private pickAction(ctx: LayerContext, chord: KeyChord): void {
|
||||
const id = chordId(chord);
|
||||
const current = keyboardService.keyBindings().find((binding) => chordId(binding.chord) === id)?.action;
|
||||
const done = (inner: LayerContext): void => {
|
||||
inner.stack.pop();
|
||||
this.updateRows(id);
|
||||
inner.actions.requestRender();
|
||||
};
|
||||
const rows: MenuItem[] = KEY_ACTIONS.map((entry) => ({
|
||||
label: entry.action === current ? `${entry.label} *` : entry.label,
|
||||
onSelect: (inner: LayerContext) => {
|
||||
if (!keyboardService.setKeyBinding(chord, entry.action)) {
|
||||
inner.stack.pop();
|
||||
inner.stack.push(new TextPageLayer("Keybindings", "There is no room for another keybinding. Remove one first."));
|
||||
return;
|
||||
}
|
||||
done(inner);
|
||||
},
|
||||
}));
|
||||
if (current !== undefined) {
|
||||
rows.push({
|
||||
label: "Remove keybinding",
|
||||
onSelect: (inner) => {
|
||||
keyboardService.removeKeyBinding(chord);
|
||||
done(inner);
|
||||
},
|
||||
});
|
||||
}
|
||||
const selected = Math.max(0, KEY_ACTIONS.findIndex((entry) => entry.action === current));
|
||||
openModalMenu(ctx, describeChord(chord), rows, selected);
|
||||
}
|
||||
}
|
||||
|
||||
/** The Settings row that opens the keybindings menu. */
|
||||
export function keybindingsMenuItem(): MenuItem {
|
||||
return {
|
||||
label: "Keybindings",
|
||||
description:
|
||||
"Choose what keys do: arrows, Enter, Esc and a mouse's buttons and wheel navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.",
|
||||
onSelect: (ctx) => new KeybindingsLayer(ctx.stack.getBaseSize()).open(ctx),
|
||||
render: ({ image, x, y, width }) => {
|
||||
const count = keyboardService.keyBindings().length;
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "Keybindings", count ? `${count} custom` : "Default");
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import { getDefaultSmallFont } from "../graphics/ui-fonts";
|
||||
import type { KeyboardBridgeSnapshot } from "../g2/keyboard/bridge";
|
||||
import type { KeyboardAdvert } from "../g2/keyboard/protocol";
|
||||
import { keyboardService } from "../g2/keyboard/service";
|
||||
import { ConfigSettingString, textSettingMenuItem } from "./dashboard-settings";
|
||||
import type { LayerContext } from "./layers";
|
||||
import { drawRightValueMenuItem, MenuLayer, type MenuItem } from "./menu";
|
||||
|
||||
/**
|
||||
* Settings > Keyboard: pairing a Bluetooth keyboard with the glasses (the
|
||||
* CFW keyboard bridge, app/g2/keyboard/). The keyboard pairs with the
|
||||
* glasses, not the phone, so all of it happens here, on the glasses: scan,
|
||||
* pick, and for a keyboard that wants one, the passkey to type on it.
|
||||
*/
|
||||
|
||||
/** The code a keyboard with a display shows, typed on the phone. Scratch, not a preference. */
|
||||
const pairingCodeSetting = new ConfigSettingString({
|
||||
id: "keyboardPairingCode",
|
||||
storageKey: "keyboard.pairingCode",
|
||||
label: "Code on keyboard",
|
||||
defaultValue: "",
|
||||
normalize: (value) => (value ?? "").replace(/\D/g, "").slice(0, 6),
|
||||
});
|
||||
|
||||
const sixDigits = (value: number): string => {
|
||||
const text = String(value).padStart(6, "0");
|
||||
return `${text.slice(0, 3)} ${text.slice(3)}`;
|
||||
};
|
||||
|
||||
function advertLabel(advert: KeyboardAdvert): string {
|
||||
const name = advert.name.trim() || advert.address;
|
||||
return name.length > 30 ? `${name.slice(0, 27)}...` : name;
|
||||
}
|
||||
|
||||
/** One line saying where the keyboard stands, for the Settings row and the menu's first row. */
|
||||
export function keyboardStatusText(snapshot: KeyboardBridgeSnapshot): string {
|
||||
const name = snapshot.activeName || snapshot.saved?.name || "keyboard";
|
||||
switch (snapshot.state) {
|
||||
case "unavailable":
|
||||
return snapshot.saved ? "Glasses not connected" : "Not set up";
|
||||
case "scanning":
|
||||
return "Scanning...";
|
||||
case "connecting":
|
||||
return `Connecting to ${name}...`;
|
||||
case "pairing":
|
||||
return `Pairing with ${name}...`;
|
||||
case "configuring":
|
||||
return `Setting up ${name}...`;
|
||||
case "connected":
|
||||
return `Connected: ${name}`;
|
||||
case "error":
|
||||
return snapshot.error || "Failed";
|
||||
default:
|
||||
if (snapshot.waiting) return `Waiting for ${name}`;
|
||||
return snapshot.saved ? `${name} (disconnected)` : "Not set up";
|
||||
}
|
||||
}
|
||||
|
||||
/** The short form for the right edge of the Settings row. */
|
||||
function keyboardStatusBrief(snapshot: KeyboardBridgeSnapshot): string {
|
||||
switch (snapshot.state) {
|
||||
case "connected":
|
||||
return "Connected";
|
||||
case "scanning":
|
||||
return "Scanning";
|
||||
case "connecting":
|
||||
case "pairing":
|
||||
case "configuring":
|
||||
return "Connecting";
|
||||
case "error":
|
||||
return "Failed";
|
||||
case "unavailable":
|
||||
return snapshot.saved ? "Offline" : "Off";
|
||||
default:
|
||||
return snapshot.waiting ? "Waiting" : snapshot.saved ? "Disconnected" : "Off";
|
||||
}
|
||||
}
|
||||
|
||||
const info = (label: string): MenuItem => ({ label, disabled: true, onSelect: () => {} });
|
||||
|
||||
class KeyboardSettingsLayer extends MenuLayer {
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private showOthers = false;
|
||||
/** What kind of rows are showing; a change moves the highlight to the first row that does something. */
|
||||
private shape = "";
|
||||
|
||||
constructor(viewport: { width: number; height: number }) {
|
||||
super("Bluetooth keyboard", [], {
|
||||
x: "center",
|
||||
y: 8,
|
||||
width: Math.min(320, viewport.width - 40),
|
||||
minHeight: Math.min(160, viewport.height - 40),
|
||||
maxHeight: Math.min(240, viewport.height - 40),
|
||||
depth: 4,
|
||||
});
|
||||
}
|
||||
|
||||
open(ctx: LayerContext): void {
|
||||
ctx.stack.push(this);
|
||||
this.updateRows();
|
||||
this.unsubscribe = keyboardService.subscribe(() => {
|
||||
this.updateRows();
|
||||
ctx.actions.requestRender();
|
||||
});
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
/** Run a bridge command; its failure shows up in the snapshot, which repaints the rows. */
|
||||
private run(action: () => Promise<unknown>): void {
|
||||
void action().catch(() => {});
|
||||
}
|
||||
|
||||
private updateRows(): void {
|
||||
const snapshot = keyboardService.snapshot();
|
||||
const bridge = keyboardService.bridge;
|
||||
const rows: MenuItem[] = [info(keyboardStatusText(snapshot))];
|
||||
let shape: string = snapshot.state;
|
||||
|
||||
const scanRow = (label: string): MenuItem => ({
|
||||
label,
|
||||
disabled: snapshot.busy,
|
||||
onSelect: () => {
|
||||
this.showOthers = false;
|
||||
this.run(() => bridge.scan());
|
||||
},
|
||||
});
|
||||
|
||||
if (snapshot.state === "unavailable") {
|
||||
rows.push(info("Connect G2 glasses to pair one."));
|
||||
if (snapshot.saved) rows.push({ label: `Forget ${snapshot.saved.name || "keyboard"}`, onSelect: () => this.run(() => bridge.forget()) });
|
||||
} else if (snapshot.pairing) {
|
||||
const pairing = snapshot.pairing;
|
||||
shape = `pairing:${pairing.kind}`;
|
||||
if (pairing.kind === "display") {
|
||||
rows.push(info(`Type ${sixDigits(pairing.pin)} on the keyboard,`), info("then press Enter."));
|
||||
} else if (pairing.kind === "compare") {
|
||||
rows.push(
|
||||
info(`Does the keyboard show ${sixDigits(pairing.value)}?`),
|
||||
{ label: "Yes, pair", onSelect: () => this.run(() => bridge.answerCompare(true)) },
|
||||
{ label: "No", onSelect: () => this.run(() => bridge.answerCompare(false)) },
|
||||
);
|
||||
} else {
|
||||
rows.push(
|
||||
info("Enter the code the keyboard shows."),
|
||||
textSettingMenuItem(pairingCodeSetting),
|
||||
{
|
||||
label: "Pair with this code",
|
||||
disabled: () => pairingCodeSetting.get().length !== 6,
|
||||
onSelect: () => {
|
||||
const code = pairingCodeSetting.get();
|
||||
pairingCodeSetting.set("");
|
||||
this.run(() => bridge.answerPasskey(parseInt(code, 10)));
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
rows.push({ label: "Cancel pairing", onSelect: () => this.run(() => bridge.disconnect()) });
|
||||
} else if (snapshot.state === "connecting" || snapshot.state === "pairing" || snapshot.state === "configuring") {
|
||||
rows.push({ label: "Cancel", onSelect: () => this.run(() => bridge.disconnect()) });
|
||||
} else if (snapshot.state === "scanning" || (snapshot.scanResults.length > 0 && snapshot.state !== "connected")) {
|
||||
shape = "results";
|
||||
const hid = snapshot.scanResults.filter((advert) => advert.hid);
|
||||
const others = snapshot.scanResults.filter((advert) => !advert.hid);
|
||||
const pick = (advert: KeyboardAdvert): MenuItem => ({
|
||||
label: advertLabel(advert),
|
||||
disabled: snapshot.busy,
|
||||
onSelect: () => {
|
||||
this.run(() => bridge.connect({ address: advert.address, addressType: advert.addressType, name: advert.name.trim() || advert.address }));
|
||||
},
|
||||
});
|
||||
rows.push(...hid.map(pick));
|
||||
if (!hid.length && snapshot.state !== "scanning") rows.push(info("No keyboards found."), info("Put it in pairing mode."));
|
||||
if (others.length && this.showOthers) rows.push(...others.map(pick));
|
||||
else if (others.length) {
|
||||
rows.push({
|
||||
label: `Show ${others.length} other device${others.length === 1 ? "" : "s"}`,
|
||||
onSelect: (ctx) => {
|
||||
this.showOthers = true;
|
||||
this.updateRows();
|
||||
ctx.actions.requestRender();
|
||||
},
|
||||
});
|
||||
}
|
||||
if (snapshot.state !== "scanning") rows.push(scanRow("Scan again"));
|
||||
if (snapshot.saved && snapshot.state !== "scanning") {
|
||||
rows.push({ label: `Reconnect ${snapshot.saved.name || "keyboard"}`, disabled: snapshot.busy, onSelect: () => this.run(() => bridge.reconnect()) });
|
||||
}
|
||||
} else if (snapshot.state === "connected") {
|
||||
rows.push(
|
||||
{ label: "Disconnect", onSelect: () => this.run(() => bridge.disconnect()) },
|
||||
{ label: "Forget this keyboard", onSelect: () => this.run(() => bridge.forget()) },
|
||||
scanRow("Pair a different keyboard"),
|
||||
);
|
||||
} else if (snapshot.saved) {
|
||||
shape = snapshot.waiting ? "waiting" : "saved";
|
||||
rows.push({ label: "Reconnect now", disabled: snapshot.busy, onSelect: () => this.run(() => bridge.reconnect()) });
|
||||
if (snapshot.waiting) rows.push({ label: "Stop reconnecting", onSelect: () => this.run(() => bridge.disconnect()) });
|
||||
rows.push(
|
||||
{ label: "Forget this keyboard", onSelect: () => this.run(() => bridge.forget()) },
|
||||
scanRow("Pair a different keyboard"),
|
||||
);
|
||||
} else {
|
||||
shape = "empty";
|
||||
rows.push(info("Put the keyboard in pairing mode,"), info("then scan."), scanRow("Scan for keyboards"));
|
||||
}
|
||||
|
||||
const firstAction = rows.findIndex((row) => row.disabled !== true);
|
||||
if (shape !== this.shape) {
|
||||
this.shape = shape;
|
||||
this.setItems(rows, Math.max(0, firstAction));
|
||||
} else {
|
||||
this.setItems(rows);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The Settings row that opens the keyboard menu. */
|
||||
export function keyboardSettingsMenuItem(): MenuItem {
|
||||
return {
|
||||
label: "Bluetooth keyboard",
|
||||
description:
|
||||
"Pair a Bluetooth keyboard with the glasses. It types into Terminal sessions, and its arrow keys, Enter and Escape work everywhere as scroll, select and back.",
|
||||
onSelect: (ctx) => new KeyboardSettingsLayer(ctx.stack.getBaseSize()).open(ctx),
|
||||
render: ({ image, x, y, width }) => {
|
||||
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "Bluetooth keyboard", keyboardStatusBrief(keyboardService.snapshot()));
|
||||
},
|
||||
};
|
||||
}
|
||||
+19
-1
@@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size";
|
||||
import { dimPlanes, type Plane } from "../graphics/plane";
|
||||
import { spanCurrent } from "../native/frame-timings";
|
||||
import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings";
|
||||
import { type VoiceReplay } from "../native/voice-control";
|
||||
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
|
||||
import { toViewport, type Viewport, type ViewportSize } from "./viewport";
|
||||
|
||||
@@ -13,8 +14,17 @@ export type TextSettingsEditToggle = {
|
||||
|
||||
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
|
||||
export type ContinuousCaptureOptions = {
|
||||
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
|
||||
/** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */
|
||||
recordAs?: string;
|
||||
/**
|
||||
* How many of the glasses' microphones to capture: 1 is their own mixed
|
||||
* stream; 2 (one per temple) and 4 (both of each temple) stream the
|
||||
* microphones separately through the custom firmware, for recording and
|
||||
* array processing. While any client asks for more than one, every capture
|
||||
* uses the most asked for. Glasses without the microphone channel, and the
|
||||
* phone's microphone, give 1 whatever is asked.
|
||||
*/
|
||||
mics?: MicCount;
|
||||
/**
|
||||
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
||||
* or "" for the provider to detect it), instead of the Transcription
|
||||
@@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = {
|
||||
|
||||
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
|
||||
|
||||
export type MicCount = 1 | 2 | 4;
|
||||
|
||||
export type LayerActions = {
|
||||
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
|
||||
requestRender: () => void;
|
||||
@@ -55,6 +67,12 @@ export type LayerActions = {
|
||||
*/
|
||||
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise<void> | void;
|
||||
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
|
||||
/**
|
||||
* Open a recording (.mka) for playing back through transcription in place
|
||||
* of the microphone; the caller starts and closes it. Absent, or null,
|
||||
* where recordings can't be replayed.
|
||||
*/
|
||||
openVoiceReplay?: (path: string) => VoiceReplay | null;
|
||||
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
|
||||
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
|
||||
import { truncateText } from "../graphics/textwrap";
|
||||
import { voiceControlBridge } from "../native/voice-control";
|
||||
import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control";
|
||||
import { lineStep } from "./metrics";
|
||||
import type { Edges } from "./viewport";
|
||||
|
||||
@@ -68,8 +68,9 @@ export class MicLevelMeter {
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
|
||||
/** `source` is the capture whose level to show: the microphone's unless given. */
|
||||
start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void {
|
||||
this.unsubscribe = source.onMicLevel((level) => {
|
||||
this.peak = Math.max(this.peak, level);
|
||||
});
|
||||
this.timer = setInterval(() => {
|
||||
|
||||
+58
-6
@@ -39,6 +39,8 @@ const PAGE_TITLE_OFFSET_Y = 2;
|
||||
const PAGE_TITLE_GAP = 4;
|
||||
/** Room a page menu keeps right of its rows for the scrollbar. */
|
||||
const PAGE_SCROLLBAR_GUTTER = 6;
|
||||
/** A heading row's text: dimmer than a row's, brighter than a disabled row's. */
|
||||
const MENU_HEADING_VALUE = 90;
|
||||
|
||||
export type MenuLayout = {
|
||||
/** Stereo depth of the menu surface, including the selected row. */
|
||||
@@ -57,7 +59,7 @@ export type MenuLayout = {
|
||||
* Dim hint line pinned to the bottom edge inside the box, below the items
|
||||
* (gesture help such as "●— system menu"). Takes a line off the item area.
|
||||
*/
|
||||
footer?: string;
|
||||
footer?: MenuFooter;
|
||||
/** Brightness factor for everything beneath the menu (see Layer.dimUnderneath). Default: none. */
|
||||
dimUnderneath?: number;
|
||||
/**
|
||||
@@ -83,9 +85,16 @@ export type MenuPageLayout = {
|
||||
/** Width of the rows. Default (and at most): the window's, inside its side padding. */
|
||||
width?: number;
|
||||
/** Dim hint line pinned above the window's bottom padding, below the rows. */
|
||||
footer?: string;
|
||||
footer?: MenuFooter;
|
||||
};
|
||||
|
||||
/** A menu's footer line: fixed text, or read on every paint (a live value). */
|
||||
export type MenuFooter = string | (() => string);
|
||||
|
||||
function footerText(footer: MenuFooter): string {
|
||||
return typeof footer === "function" ? footer() : footer;
|
||||
}
|
||||
|
||||
export type MenuItemRenderArgs = {
|
||||
image: GrayImage;
|
||||
x: number;
|
||||
@@ -104,6 +113,12 @@ export type MenuItem = {
|
||||
description?: string;
|
||||
/** Disabled rows stay visible but render dimly and ignore clicks. */
|
||||
disabled?: boolean | (() => boolean);
|
||||
/**
|
||||
* A label for the group of rows after it (see menuHeadingItem). MenuLayer's
|
||||
* highlight skips it; a list that doesn't know headings shows it as a
|
||||
* disabled row.
|
||||
*/
|
||||
heading?: boolean;
|
||||
onSelect: (ctx: LayerContext, menu: MenuLayer) => Promise<void> | void;
|
||||
render?: (args: MenuItemRenderArgs) => void;
|
||||
};
|
||||
@@ -207,6 +222,22 @@ export function submenuItem(
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A heading for the rows that follow it, splitting a long menu into sections:
|
||||
* dim capitals in the rows' text column, never selected.
|
||||
*/
|
||||
export function menuHeadingItem(label: string): MenuItem {
|
||||
return {
|
||||
label,
|
||||
heading: true,
|
||||
disabled: true,
|
||||
onSelect: () => {},
|
||||
render: ({ image, x, y, text }) => {
|
||||
image.drawText(getDefaultSmallFont(), x, y + LIST_ROW_TEXT_INSET, text.toUpperCase(), MENU_HEADING_VALUE);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function drawToggleMenuItem(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
@@ -274,6 +305,7 @@ export class MenuLayer implements Layer {
|
||||
wrap: true,
|
||||
rowGap: 1,
|
||||
getHeight: () => listRowHeight(getDefaultSmallFont()),
|
||||
isSelectable: (item) => !item.heading,
|
||||
draw: ({ image, item, x, y, width, height, selected }) => {
|
||||
const font = getDefaultSmallFont();
|
||||
const disabled = isMenuItemDisabled(item);
|
||||
@@ -361,7 +393,7 @@ export class MenuLayer implements Layer {
|
||||
image.drawText(font, x + 12, y + 8, this.title, 220);
|
||||
}
|
||||
if (layout.footer) {
|
||||
image.drawText(font, x + 22, y + height - MENU_BODY_PADDING - font.lineHeight, layout.footer, 110);
|
||||
image.drawText(font, x + 22, y + height - MENU_BODY_PADDING - font.lineHeight, footerText(layout.footer), 110);
|
||||
}
|
||||
|
||||
const bodyY = y + chromeTop;
|
||||
@@ -398,7 +430,7 @@ export class MenuLayer implements Layer {
|
||||
let fixedBottom = height - padding.bottom;
|
||||
if (layout.footer) {
|
||||
const footerY = height - padding.bottom - font.lineHeight;
|
||||
image.drawText(font, x + MENU_ROW_TEXT_INSET, footerY, layout.footer, 110);
|
||||
image.drawText(font, x + MENU_ROW_TEXT_INSET, footerY, footerText(layout.footer), 110);
|
||||
bottom = fixedBottom = footerY - MENU_FOOTER_GAP;
|
||||
}
|
||||
if (bottom <= top) return image;
|
||||
@@ -432,17 +464,26 @@ export class MenuLayer implements Layer {
|
||||
}
|
||||
}
|
||||
|
||||
export type ModalMenuOptions = {
|
||||
/** A dim line under the items (see MenuLayout.footer). */
|
||||
footer?: MenuFooter;
|
||||
/** Runs when the menu leaves the stack, however it closes. */
|
||||
onClosed?: () => void;
|
||||
};
|
||||
|
||||
/** Open a centered, bordered menu over the current layer. */
|
||||
export function openModalMenu(
|
||||
ctx: LayerContext,
|
||||
title: string,
|
||||
items: MenuItem[],
|
||||
initialSelectedIndex = 0,
|
||||
options: ModalMenuOptions = {},
|
||||
): void {
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const font = getDefaultSmallFont();
|
||||
const menuWidth = Math.min(320, width - 40);
|
||||
const naturalHeight = menuTitleHeight(font) + 2 * MENU_BODY_PADDING + items.length * listRowHeight(font);
|
||||
const footerHeight = options.footer ? MENU_FOOTER_GAP + lineStep(font) : 0;
|
||||
const naturalHeight = menuTitleHeight(font) + 2 * MENU_BODY_PADDING + items.length * listRowHeight(font) + footerHeight;
|
||||
// 20px clear above and below, or 8 in a short window (the RayNeo iO's),
|
||||
// where those rows are worth more than the margin.
|
||||
const marginY = height < SHORT_WINDOW_HEIGHT ? 8 : 20;
|
||||
@@ -455,8 +496,19 @@ export function openModalMenu(
|
||||
maxHeight: menuHeight,
|
||||
// Floats nearer than the content it opens over, like the window and system menus.
|
||||
depth: 4,
|
||||
footer: options.footer,
|
||||
};
|
||||
ctx.stack.push(new MenuLayer(title, items, layout).selectItem(initialSelectedIndex));
|
||||
ctx.stack.push(new ModalMenuLayer(title, items, layout, options.onClosed).selectItem(initialSelectedIndex));
|
||||
}
|
||||
|
||||
class ModalMenuLayer extends MenuLayer {
|
||||
constructor(title: string, items: MenuItem[], layout: MenuLayout, private readonly onClosed?: () => void) {
|
||||
super(title, items, layout);
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.onClosed?.();
|
||||
}
|
||||
}
|
||||
|
||||
export function isMenuItemDisabled(item: MenuItem): boolean {
|
||||
|
||||
@@ -50,6 +50,8 @@ export type InProcessWindowOptions = {
|
||||
setScreenOn?: ShellWindow["setScreenOn"];
|
||||
/** See ShellWindow.keepsScreenOn. */
|
||||
keepsScreenOn?: ShellWindow["keepsScreenOn"];
|
||||
/** See ShellWindow.restoresApp. */
|
||||
restoresApp?: boolean;
|
||||
/** The window came to or left the foreground (after its surface visibility follows). */
|
||||
onForegroundChanged?: (foreground: boolean) => void;
|
||||
/** Shared actions; requestRender is rebound to this window's render. */
|
||||
@@ -179,6 +181,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
|
||||
holdToTalk: options.holdToTalk,
|
||||
isVoiceCapturing: options.isVoiceCapturing,
|
||||
keepsScreenOn: options.keepsScreenOn,
|
||||
restoresApp: options.restoresApp,
|
||||
setScreenOn: options.setScreenOn,
|
||||
heightMode,
|
||||
hasAppMenu: () => appMenuItems().length > 0,
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
isWatchInput,
|
||||
makeInputEvent,
|
||||
} from "../gestures";
|
||||
import type { KeyEvent } from "../../g2/keyboard/keys";
|
||||
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
|
||||
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
|
||||
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
|
||||
@@ -158,6 +159,13 @@ export type ShellWindow = {
|
||||
* draws this window's top bar.
|
||||
*/
|
||||
heightMode: WindowHeightMode;
|
||||
/**
|
||||
* False for a window that must not bring its app back when Faceclaw
|
||||
* restarts (the controller's open-apps restore relaunches apps, not
|
||||
* windows): one that shows something a plain launch of its app would not,
|
||||
* where the relaunch would do something else in its place.
|
||||
*/
|
||||
restoresApp?: boolean;
|
||||
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
|
||||
close?: () => void;
|
||||
drawIcon: ShellChromeWindow["drawIcon"];
|
||||
@@ -192,6 +200,15 @@ export type ShellWindow = {
|
||||
* only windows that consume typed text (the terminal) implement it.
|
||||
*/
|
||||
receiveTextInput?: (text: string, options?: { submit?: boolean }) => void;
|
||||
/**
|
||||
* Take a key from a hardware keyboard (app/g2/keyboard) as a key. Optional:
|
||||
* only windows that give keys a meaning of their own (the terminal)
|
||||
* implement it. They get the keys no keybinding claimed while they have
|
||||
* input focus, presses and releases alike; the rest of the time, and for
|
||||
* every other window, the navigation keys act as gestures (see
|
||||
* app/g2/keyboard/bindings.ts).
|
||||
*/
|
||||
handleKeyEvent?: (event: KeyEvent) => void;
|
||||
/** Foreground state changed: this window's surface is (not) the visible one. */
|
||||
setForeground?: (foreground: boolean) => void;
|
||||
/**
|
||||
@@ -1671,6 +1688,21 @@ class Shell {
|
||||
this.foregroundWindow()?.receiveTextInput?.(text, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Offer a hardware-keyboard key to the window that has input focus. True
|
||||
* when that window takes keys (ShellWindow.handleKeyEvent) and got this
|
||||
* one; false when the key is the caller's to interpret: the screen is off,
|
||||
* the sidebar or a shell overlay has focus, or the window has no use for
|
||||
* keys. Typing counts as activity for the screen timeout.
|
||||
*/
|
||||
receiveKeyEvent(event: KeyEvent): boolean {
|
||||
const window = this.inputTargetWindow();
|
||||
if (!window?.handleKeyEvent) return false;
|
||||
if (event.phase === "down") this.lastInputAtMs = Date.now();
|
||||
window.handleKeyEvent(event);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the voice dialog aimed at the foreground window. Called when the
|
||||
* user picks Voice input from the system menu (or an app asks via a
|
||||
|
||||
@@ -28,6 +28,8 @@ export type WorkerAppMessage =
|
||||
| { type: "close-window"; windowId: string }
|
||||
| { type: "input"; windowId: string; event: unknown; frameId: number; focused: boolean }
|
||||
| { type: "text-input"; windowId: string; text: string; submit?: boolean }
|
||||
/** A hardware-keyboard key (a KeyEvent), for windows opened with acceptsKeys. */
|
||||
| { type: "key"; windowId: string; event: unknown; focused: boolean }
|
||||
| { type: "render"; windowId: string; focused: boolean }
|
||||
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
|
||||
/**
|
||||
@@ -56,6 +58,16 @@ export type WorkerAppReply =
|
||||
type: "worker-ready";
|
||||
}
|
||||
| { type: "yield-focus"; windowId: string }
|
||||
| {
|
||||
/**
|
||||
* A "key" message the window had no use for as a key (a terminal
|
||||
* showing its session list rather than a session). The host gives it
|
||||
* the keyboard's navigation meaning instead.
|
||||
*/
|
||||
type: "key-unhandled";
|
||||
windowId: string;
|
||||
event: unknown;
|
||||
}
|
||||
| {
|
||||
/** Foreground and focus one of the app's existing windows. */
|
||||
type: "focus-window";
|
||||
@@ -81,6 +93,8 @@ export type WorkerAppReply =
|
||||
focus?: boolean;
|
||||
/** Window height: the standard 288px band ("min", default) or full screen ("max"). */
|
||||
heightMode?: WindowHeightMode;
|
||||
/** Send the window hardware-keyboard keys (see WorkerWindowSpec). */
|
||||
acceptsKeys?: boolean;
|
||||
}
|
||||
| {
|
||||
/**
|
||||
@@ -196,6 +210,13 @@ export type WorkerWindowSpec = {
|
||||
* of all four directions in every window state (see ShellWindow).
|
||||
*/
|
||||
acceptsDirectional?: boolean;
|
||||
/**
|
||||
* Deliver hardware-keyboard keys to the worker as "key" messages while the
|
||||
* window has input focus (see ShellWindow.handleKeyEvent). The worker
|
||||
* answers "key-unhandled" for a key it doesn't want as a key just then,
|
||||
* which gets the keyboard's navigation meaning instead.
|
||||
*/
|
||||
acceptsKeys?: boolean;
|
||||
};
|
||||
|
||||
export type WorkerAppHostOptions = {
|
||||
@@ -223,6 +244,8 @@ export type WorkerAppHostOptions = {
|
||||
startTextSettingEdit: (settingId: string) => void;
|
||||
/** Close the phone text editor opened by startTextSettingEdit. */
|
||||
endTextSettingEdit: () => void;
|
||||
/** A key a window was sent and handed back (the "key-unhandled" reply); a KeyEvent. */
|
||||
unhandledKey?: (event: unknown) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -312,6 +335,10 @@ export class WorkerAppHost {
|
||||
shell.yieldFocusToSidebar();
|
||||
}
|
||||
break;
|
||||
case "key-unhandled":
|
||||
// Stale once focus has moved on: the key was meant for that window.
|
||||
if (shell.isWindowFocused(message.windowId)) this.options.unhandledKey?.(message.event);
|
||||
break;
|
||||
case "focus-window":
|
||||
if (this.openWindows.has(message.windowId)) {
|
||||
shell.focusWindow(message.windowId);
|
||||
@@ -335,6 +362,7 @@ export class WorkerAppHost {
|
||||
iconGlyph: message.iconGlyph,
|
||||
focus: message.focus,
|
||||
heightMode: message.heightMode,
|
||||
acceptsKeys: message.acceptsKeys,
|
||||
});
|
||||
break;
|
||||
case "set-height-mode":
|
||||
@@ -522,6 +550,13 @@ export class WorkerAppHost {
|
||||
receiveTextInput: (text, options) => {
|
||||
this.post({ type: "text-input", windowId: spec.windowId, text, ...(options?.submit === undefined ? {} : { submit: options.submit }) });
|
||||
},
|
||||
...(spec.acceptsKeys
|
||||
? {
|
||||
handleKeyEvent: (event) => {
|
||||
this.post({ type: "key", windowId: spec.windowId, event, focused: shell.isWindowFocused(spec.windowId) });
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
setForeground: (foreground) => {
|
||||
this.options.setSurfaceVisible(surfaceId, foreground);
|
||||
if (foreground) {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
const { buildKotlin, checkAndroidRuntime } = require('../../scripts/kotlin-build.cjs');
|
||||
const { buildGoSsh } = require('../../scripts/go-ssh-build.cjs');
|
||||
|
||||
module.exports = function (hookArgs) {
|
||||
// NativeScript parses hooks with an older JavaScript parser; avoid optional chaining.
|
||||
const { platform, release } = hookArgs.prepareData;
|
||||
if (platform === 'android') checkAndroidRuntime();
|
||||
return buildKotlin(platform, release ? 'release' : 'debug');
|
||||
return buildKotlin(platform, release ? 'release' : 'debug').then(function () {
|
||||
return buildGoSsh(platform);
|
||||
});
|
||||
};
|
||||
@@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
|
||||
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
|
||||
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
|
||||
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
|
||||
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
|
||||
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
|
||||
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
|
||||
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
|
||||
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
|
||||
|
||||
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
|
||||
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
|
||||
|
||||
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
|
||||
|
||||
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
|
||||
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
|
||||
override var size = 0L
|
||||
private set
|
||||
|
||||
override fun write(bytes: ByteArray, offset: Int, length: Int) {
|
||||
file.write(bytes, offset, length)
|
||||
size += length
|
||||
}
|
||||
|
||||
override fun writeAt(position: Long, bytes: ByteArray) {
|
||||
require(position >= 0 && position + bytes.size <= size)
|
||||
file.seek(position)
|
||||
file.write(bytes)
|
||||
file.seek(size)
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
|
||||
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
|
||||
private val file = java.io.RandomAccessFile(path, "r")
|
||||
override val length = file.length()
|
||||
|
||||
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
|
||||
file.seek(position)
|
||||
var total = 0
|
||||
while (total < length) {
|
||||
val read = file.read(buffer, offset + total, length - total)
|
||||
if (read <= 0) break
|
||||
total += read
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
override fun close() = file.close()
|
||||
}
|
||||
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
|
||||
expect fun deleteFile(path: String): Boolean
|
||||
|
||||
expect fun renameFile(from: String, to: String): Boolean
|
||||
|
||||
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
|
||||
interface SeekableOutput {
|
||||
/** Bytes appended so far. */
|
||||
val size: Long
|
||||
|
||||
/** Appends [length] bytes of [bytes] from [offset]. */
|
||||
fun write(bytes: ByteArray, offset: Int, length: Int)
|
||||
|
||||
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
|
||||
fun writeAt(position: Long, bytes: ByteArray)
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** A file read at arbitrary positions without loading it whole. */
|
||||
interface SeekableInput {
|
||||
val length: Long
|
||||
|
||||
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
|
||||
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
|
||||
|
||||
fun close()
|
||||
}
|
||||
|
||||
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
|
||||
expect fun openFileOutput(path: String): SeekableOutput
|
||||
|
||||
/** Opens [path] for reading. Throws when it cannot. */
|
||||
expect fun openFileInput(path: String): SeekableInput
|
||||
@@ -0,0 +1,764 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
|
||||
/**
|
||||
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
|
||||
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
|
||||
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
|
||||
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
|
||||
*
|
||||
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
|
||||
* about a second each, cues and the segment size written when the recording finishes. A
|
||||
* recording cut short by a crash is still a valid live-style stream of everything up to its last
|
||||
* complete cluster, and the reader takes it as such.
|
||||
*/
|
||||
internal object MatroskaIds {
|
||||
const val EBML = 0x1A45DFA3
|
||||
const val EBML_VERSION = 0x4286
|
||||
const val EBML_READ_VERSION = 0x42F7
|
||||
const val EBML_MAX_ID_LENGTH = 0x42F2
|
||||
const val EBML_MAX_SIZE_LENGTH = 0x42F3
|
||||
const val DOC_TYPE = 0x4282
|
||||
const val DOC_TYPE_VERSION = 0x4287
|
||||
const val DOC_TYPE_READ_VERSION = 0x4285
|
||||
const val VOID = 0xEC
|
||||
const val SEGMENT = 0x18538067
|
||||
const val SEEK_HEAD = 0x114D9B74
|
||||
const val SEEK = 0x4DBB
|
||||
const val SEEK_ID = 0x53AB
|
||||
const val SEEK_POSITION = 0x53AC
|
||||
const val INFO = 0x1549A966
|
||||
const val TIMESTAMP_SCALE = 0x2AD7B1
|
||||
const val DURATION = 0x4489
|
||||
const val DATE_UTC = 0x4461
|
||||
const val TITLE = 0x7BA9
|
||||
const val MUXING_APP = 0x4D80
|
||||
const val WRITING_APP = 0x5741
|
||||
const val TRACKS = 0x1654AE6B
|
||||
const val TRACK_ENTRY = 0xAE
|
||||
const val TRACK_NUMBER = 0xD7
|
||||
const val TRACK_UID = 0x73C5
|
||||
const val TRACK_TYPE = 0x83
|
||||
const val FLAG_DEFAULT = 0x88
|
||||
const val FLAG_LACING = 0x9C
|
||||
const val NAME = 0x536E
|
||||
const val LANGUAGE = 0x22B59C
|
||||
const val CODEC_ID = 0x86
|
||||
const val AUDIO = 0xE1
|
||||
const val SAMPLING_FREQUENCY = 0xB5
|
||||
const val CHANNELS = 0x9F
|
||||
const val BIT_DEPTH = 0x6264
|
||||
const val ATTACHMENTS = 0x1941A469
|
||||
const val ATTACHED_FILE = 0x61A7
|
||||
const val FILE_NAME = 0x466E
|
||||
const val FILE_MIME_TYPE = 0x4660
|
||||
const val FILE_DATA = 0x465C
|
||||
const val FILE_UID = 0x46AE
|
||||
const val CLUSTER = 0x1F43B675
|
||||
const val TIMESTAMP = 0xE7
|
||||
const val SIMPLE_BLOCK = 0xA3
|
||||
const val BLOCK_GROUP = 0xA0
|
||||
const val BLOCK = 0xA1
|
||||
const val BLOCK_DURATION = 0x9B
|
||||
const val CUES = 0x1C53BB6B
|
||||
const val CUE_POINT = 0xBB
|
||||
const val CUE_TIME = 0xB3
|
||||
const val CUE_TRACK_POSITIONS = 0xB7
|
||||
const val CUE_TRACK = 0xF7
|
||||
const val CUE_CLUSTER_POSITION = 0xF1
|
||||
}
|
||||
|
||||
/** Builds EBML elements into a byte buffer. */
|
||||
internal class EbmlWriter(capacity: Int = 256) {
|
||||
private val sink = ByteSink(capacity)
|
||||
|
||||
fun size(): Int = sink.size()
|
||||
|
||||
fun bytes(): ByteArray = sink.toByteArray()
|
||||
|
||||
/** An element ID, written as given: its length marker is part of the value. */
|
||||
fun id(id: Int) {
|
||||
if (id > 0xFFFFFF) sink.write(id ushr 24)
|
||||
if (id > 0xFFFF) sink.write(id ushr 16)
|
||||
if (id > 0xFF) sink.write(id ushr 8)
|
||||
sink.write(id)
|
||||
}
|
||||
|
||||
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
|
||||
fun dataSize(size: Long, width: Int = 0) {
|
||||
var bytes = width
|
||||
if (bytes == 0) {
|
||||
bytes = 1
|
||||
// The all-ones value of each width means "unknown size".
|
||||
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
|
||||
}
|
||||
val marked = size or (1L shl (7 * bytes))
|
||||
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun uint(id: Int, value: Long) {
|
||||
var bytes = 1
|
||||
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
|
||||
id(id)
|
||||
dataSize(bytes.toLong())
|
||||
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
/** A fixed eight-byte integer (dates). */
|
||||
fun int64(id: Int, value: Long) {
|
||||
id(id)
|
||||
dataSize(8)
|
||||
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
|
||||
}
|
||||
|
||||
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
|
||||
|
||||
fun string(id: Int, text: String) {
|
||||
val bytes = text.encodeToByteArray()
|
||||
binary(id, bytes, 0, bytes.size)
|
||||
}
|
||||
|
||||
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
|
||||
id(id)
|
||||
dataSize(length.toLong())
|
||||
sink.write(bytes, offset, length)
|
||||
}
|
||||
|
||||
fun raw(bytes: ByteArray) = sink.write(bytes)
|
||||
|
||||
fun raw(value: Int) = sink.write(value)
|
||||
|
||||
fun master(id: Int, build: EbmlWriter.() -> Unit) {
|
||||
val child = EbmlWriter()
|
||||
child.build()
|
||||
id(id)
|
||||
dataSize(child.size().toLong())
|
||||
sink.write(child.bytes())
|
||||
}
|
||||
|
||||
companion object {
|
||||
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
|
||||
fun padded(element: ByteArray, reserved: Int): ByteArray? {
|
||||
val filler = reserved - element.size
|
||||
// A Void element is at least two bytes.
|
||||
if (filler < 0 || filler == 1) return null
|
||||
val out = element.copyOf(reserved)
|
||||
if (filler == 0) return out
|
||||
val at = element.size
|
||||
out[at] = MatroskaIds.VOID.toByte()
|
||||
if (filler < 9) {
|
||||
out[at + 1] = (0x80 or (filler - 2)).toByte()
|
||||
} else {
|
||||
val size = (filler - 9).toLong()
|
||||
out[at + 1] = 0x01
|
||||
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
|
||||
|
||||
/**
|
||||
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
|
||||
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
|
||||
* milliseconds from the start of the file. Not thread-safe.
|
||||
*/
|
||||
class MkaWriter(
|
||||
private val output: SeekableOutput,
|
||||
title: String?,
|
||||
/** When the recording began (Unix epoch ms), for the file's date. */
|
||||
dateUtcMs: Long,
|
||||
attachments: List<MkaAttachment>,
|
||||
) {
|
||||
companion object {
|
||||
const val CODEC_PCM = "A_PCM/INT/LIT"
|
||||
const val CODEC_TEXT = "S_TEXT/UTF8"
|
||||
const val TRACK_AUDIO = 2
|
||||
const val TRACK_SUBTITLE = 0x11
|
||||
const val APP_NAME = "Faceclaw"
|
||||
private const val SEEK_RESERVED = 128
|
||||
private const val TRACKS_RESERVED = 4096
|
||||
private const val CLUSTER_SPAN_MS = 1000L
|
||||
private const val CLUSTER_MAX_BYTES = 1 shl 20
|
||||
// Block timestamps are signed 16-bit offsets from their cluster's.
|
||||
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
|
||||
private const val DURATION_PATCH_CLUSTERS = 5
|
||||
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
|
||||
private const val DATE_EPOCH_MS = 978_307_200_000L
|
||||
}
|
||||
|
||||
private class Track(
|
||||
val number: Int,
|
||||
val type: Int,
|
||||
val name: String,
|
||||
val sampleRate: Int,
|
||||
val channels: Int,
|
||||
val preferred: Boolean,
|
||||
)
|
||||
|
||||
private class Cue(val clusterPosition: Long, val timeMs: Long)
|
||||
|
||||
private val tracks = ArrayList<Track>()
|
||||
private val cues = ArrayList<Cue>()
|
||||
private val segmentSizePosition: Long
|
||||
private val segmentStart: Long
|
||||
private val seekPosition: Long
|
||||
private val infoPosition: Long
|
||||
private val durationPosition: Long
|
||||
private val tracksPosition: Long
|
||||
private val attachmentsPosition: Long
|
||||
private var cluster: EbmlWriter? = null
|
||||
private var clusterBaseMs = 0L
|
||||
private var clustersSincePatch = 0
|
||||
private var finished = false
|
||||
private val hasAttachments = attachments.isNotEmpty()
|
||||
|
||||
/** The end of the latest audio or text written, in ms from the start of the file. */
|
||||
var durationMs = 0L
|
||||
private set
|
||||
|
||||
init {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.EBML) {
|
||||
uint(MatroskaIds.EBML_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_READ_VERSION, 1)
|
||||
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
|
||||
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
|
||||
string(MatroskaIds.DOC_TYPE, "matroska")
|
||||
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
|
||||
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
|
||||
}
|
||||
head.id(MatroskaIds.SEGMENT)
|
||||
segmentSizePosition = head.size().toLong()
|
||||
// Unknown size (a live stream) until finish() knows it.
|
||||
head.raw(0x01)
|
||||
repeat(7) { head.raw(0xFF) }
|
||||
segmentStart = head.size().toLong()
|
||||
append(head.bytes())
|
||||
|
||||
seekPosition = output.size
|
||||
infoPosition = seekPosition + SEEK_RESERVED
|
||||
val info = EbmlWriter()
|
||||
info.master(MatroskaIds.INFO) {
|
||||
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
|
||||
string(MatroskaIds.MUXING_APP, APP_NAME)
|
||||
string(MatroskaIds.WRITING_APP, APP_NAME)
|
||||
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
|
||||
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
|
||||
// Last, so its eight bytes are the element's final ones.
|
||||
float64(MatroskaIds.DURATION, 0.0)
|
||||
}
|
||||
durationPosition = infoPosition + info.size() - 8
|
||||
tracksPosition = infoPosition + info.size()
|
||||
attachmentsPosition = tracksPosition + TRACKS_RESERVED
|
||||
|
||||
append(seekHead(-1))
|
||||
append(info.bytes())
|
||||
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
|
||||
if (attachments.isNotEmpty()) {
|
||||
val files = EbmlWriter()
|
||||
files.master(MatroskaIds.ATTACHMENTS) {
|
||||
attachments.forEachIndexed { index, file ->
|
||||
master(MatroskaIds.ATTACHED_FILE) {
|
||||
string(MatroskaIds.FILE_NAME, file.name)
|
||||
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
|
||||
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
|
||||
uint(MatroskaIds.FILE_UID, index + 1L)
|
||||
}
|
||||
}
|
||||
}
|
||||
append(files.bytes())
|
||||
}
|
||||
}
|
||||
|
||||
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
|
||||
|
||||
private fun seekHead(cuesPosition: Long): ByteArray {
|
||||
val head = EbmlWriter()
|
||||
head.master(MatroskaIds.SEEK_HEAD) {
|
||||
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
|
||||
val idBytes = EbmlWriter()
|
||||
idBytes.id(id)
|
||||
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
|
||||
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
|
||||
}
|
||||
entry(MatroskaIds.INFO, infoPosition)
|
||||
entry(MatroskaIds.TRACKS, tracksPosition)
|
||||
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
|
||||
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
|
||||
}
|
||||
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
|
||||
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
|
||||
* the one players pick by default.
|
||||
*/
|
||||
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
|
||||
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
|
||||
|
||||
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
|
||||
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
|
||||
|
||||
private fun addTrack(track: Track): Int {
|
||||
check(!finished) { "recording already finished" }
|
||||
// A block's track number is written in one byte.
|
||||
if (track.number > 126) return -1
|
||||
tracks.add(track)
|
||||
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
|
||||
if (region == null) {
|
||||
tracks.remove(track)
|
||||
return -1
|
||||
}
|
||||
output.writeAt(tracksPosition, region)
|
||||
return track.number
|
||||
}
|
||||
|
||||
private fun listedTracks(): List<Track> =
|
||||
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
|
||||
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
|
||||
tracks.filter { it.type != TRACK_AUDIO }
|
||||
|
||||
private fun tracksElement(): ByteArray {
|
||||
val listed = listedTracks()
|
||||
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
|
||||
val out = EbmlWriter(1024)
|
||||
out.master(MatroskaIds.TRACKS) {
|
||||
for (track in listed) {
|
||||
master(MatroskaIds.TRACK_ENTRY) {
|
||||
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_UID, track.number.toLong())
|
||||
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
|
||||
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
|
||||
uint(MatroskaIds.FLAG_LACING, 0)
|
||||
string(MatroskaIds.NAME, track.name)
|
||||
string(MatroskaIds.LANGUAGE, "und")
|
||||
if (track.type == TRACK_AUDIO) {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_PCM)
|
||||
master(MatroskaIds.AUDIO) {
|
||||
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
|
||||
uint(MatroskaIds.CHANNELS, track.channels.toLong())
|
||||
uint(MatroskaIds.BIT_DEPTH, 16)
|
||||
}
|
||||
} else {
|
||||
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.bytes()
|
||||
}
|
||||
|
||||
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
|
||||
|
||||
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
|
||||
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
|
||||
val track = track(trackNumber)
|
||||
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
|
||||
if (length <= 0) return
|
||||
val block = openCluster(timeMs)
|
||||
block.id(MatroskaIds.SIMPLE_BLOCK)
|
||||
block.dataSize(4L + length)
|
||||
blockHeader(block, trackNumber, timeMs, 0x80)
|
||||
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
|
||||
val frames = length / (2 * track.channels)
|
||||
noteEnd(timeMs + frames * 1000L / track.sampleRate)
|
||||
}
|
||||
|
||||
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
|
||||
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
|
||||
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
|
||||
val bytes = text.encodeToByteArray()
|
||||
val block = openCluster(timeMs)
|
||||
block.master(MatroskaIds.BLOCK_GROUP) {
|
||||
id(MatroskaIds.BLOCK)
|
||||
dataSize(4L + bytes.size)
|
||||
blockHeader(this, trackNumber, timeMs, 0)
|
||||
raw(bytes)
|
||||
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
|
||||
}
|
||||
noteEnd(timeMs + maxOf(1L, durationMs))
|
||||
}
|
||||
|
||||
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
|
||||
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
|
||||
block.raw(0x80 or trackNumber)
|
||||
block.raw(relative shr 8)
|
||||
block.raw(relative)
|
||||
block.raw(flags)
|
||||
}
|
||||
|
||||
private fun noteEnd(endMs: Long) {
|
||||
if (endMs > durationMs) durationMs = endMs
|
||||
}
|
||||
|
||||
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
|
||||
private fun openCluster(timeMs: Long): EbmlWriter {
|
||||
check(!finished) { "recording already finished" }
|
||||
val time = maxOf(0L, timeMs)
|
||||
val open = cluster
|
||||
if (open != null) {
|
||||
val relative = time - clusterBaseMs
|
||||
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
|
||||
flushCluster()
|
||||
}
|
||||
clusterBaseMs = time
|
||||
return EbmlWriter(64 * 1024).also { cluster = it }
|
||||
}
|
||||
|
||||
private fun flushCluster() {
|
||||
val body = cluster ?: return
|
||||
cluster = null
|
||||
if (body.size() == 0) return
|
||||
val timestamp = EbmlWriter(16)
|
||||
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
|
||||
val header = EbmlWriter(32)
|
||||
header.id(MatroskaIds.CLUSTER)
|
||||
header.dataSize(timestamp.size().toLong() + body.size())
|
||||
header.raw(timestamp.bytes())
|
||||
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
|
||||
append(header.bytes())
|
||||
append(body.bytes())
|
||||
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
|
||||
}
|
||||
|
||||
private fun patchDuration() {
|
||||
clustersSincePatch = 0
|
||||
val value = durationMs.toDouble().toRawBits()
|
||||
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
|
||||
}
|
||||
|
||||
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
|
||||
fun finish() {
|
||||
if (finished) return
|
||||
try {
|
||||
flushCluster()
|
||||
finished = true
|
||||
val cuesPosition = output.size
|
||||
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
|
||||
if (cues.isNotEmpty()) {
|
||||
val index = EbmlWriter(cues.size * 16 + 16)
|
||||
index.master(MatroskaIds.CUES) {
|
||||
for (cue in cues) {
|
||||
master(MatroskaIds.CUE_POINT) {
|
||||
uint(MatroskaIds.CUE_TIME, cue.timeMs)
|
||||
master(MatroskaIds.CUE_TRACK_POSITIONS) {
|
||||
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
|
||||
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
append(index.bytes())
|
||||
output.writeAt(seekPosition, seekHead(cuesPosition))
|
||||
}
|
||||
patchDuration()
|
||||
val size = EbmlWriter(8)
|
||||
size.dataSize(output.size - segmentStart, 8)
|
||||
output.writeAt(segmentSizePosition, size.bytes())
|
||||
} finally {
|
||||
finished = true
|
||||
output.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class MkaTrackInfo(
|
||||
@JvmField val number: Int,
|
||||
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
|
||||
@JvmField val type: Int,
|
||||
@JvmField val codecId: String,
|
||||
@JvmField val name: String,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val bitDepth: Int,
|
||||
) {
|
||||
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
|
||||
}
|
||||
|
||||
class MkaBlock(
|
||||
@JvmField val track: Int,
|
||||
@JvmField val timeMs: Long,
|
||||
@JvmField val data: ByteArray,
|
||||
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
|
||||
@JvmField val durationMs: Long,
|
||||
)
|
||||
|
||||
/**
|
||||
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
|
||||
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
|
||||
* IllegalArgumentException for anything that is not such a file.
|
||||
*/
|
||||
class MkaReader(private val input: SeekableInput) {
|
||||
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
|
||||
|
||||
private val clusters = ArrayList<ClusterRef>()
|
||||
private val trackList = ArrayList<MkaTrackInfo>()
|
||||
private val attachmentList = ArrayList<MkaAttachment>()
|
||||
private var timestampScale = 1_000_000L
|
||||
private var infoDurationMs = 0L
|
||||
private var nextCluster = 0
|
||||
private var blocks: List<MkaBlock> = emptyList()
|
||||
private var blockIndex = 0
|
||||
private var skipBeforeMs = 0L
|
||||
|
||||
val tracks: List<MkaTrackInfo> get() = trackList
|
||||
val attachments: List<MkaAttachment> get() = attachmentList
|
||||
|
||||
/** The end of the last audio or text in the file, in ms. */
|
||||
val durationMs: Long
|
||||
|
||||
init {
|
||||
scan()
|
||||
var end = infoDurationMs
|
||||
if (clusters.isNotEmpty()) {
|
||||
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
|
||||
}
|
||||
durationMs = end
|
||||
}
|
||||
|
||||
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
|
||||
|
||||
/** Continue from the first block at or after [timeMs]. */
|
||||
fun seekTo(timeMs: Long) {
|
||||
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
|
||||
nextCluster = maxOf(0, index)
|
||||
blocks = emptyList()
|
||||
blockIndex = 0
|
||||
skipBeforeMs = timeMs
|
||||
}
|
||||
|
||||
/** The next block, or null at the end of the file. */
|
||||
fun next(): MkaBlock? {
|
||||
while (true) {
|
||||
while (blockIndex >= blocks.size) {
|
||||
if (nextCluster >= clusters.size) return null
|
||||
blocks = readCluster(clusters[nextCluster++])
|
||||
blockIndex = 0
|
||||
}
|
||||
val block = blocks[blockIndex++]
|
||||
if (block.timeMs >= skipBeforeMs) return block
|
||||
}
|
||||
}
|
||||
|
||||
fun close() = input.close()
|
||||
|
||||
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
|
||||
|
||||
private fun blockSpanMs(block: MkaBlock): Long {
|
||||
if (block.durationMs >= 0) return block.durationMs
|
||||
val track = track(block.track) ?: return 0
|
||||
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
|
||||
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
|
||||
}
|
||||
|
||||
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
|
||||
private class Header(val id: Int, val size: Long, val length: Int)
|
||||
|
||||
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
|
||||
if (at >= end) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val idLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (idLength > 4 || at + idLength >= end) return null
|
||||
var id = 0
|
||||
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
|
||||
val sizeAt = at + idLength
|
||||
val sizeFirst = bytes[sizeAt].toInt() and 0xff
|
||||
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
|
||||
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
|
||||
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
|
||||
var allOnes = size == (0xff shr sizeLength).toLong()
|
||||
for (index in 1 until sizeLength) {
|
||||
val byte = bytes[sizeAt + index].toInt() and 0xff
|
||||
if (byte != 0xff) allOnes = false
|
||||
size = (size shl 8) or byte.toLong()
|
||||
}
|
||||
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
|
||||
}
|
||||
|
||||
private fun readAt(position: Long, length: Int): ByteArray {
|
||||
val buffer = ByteArray(length)
|
||||
val read = input.readAt(position, buffer, 0, length)
|
||||
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
|
||||
}
|
||||
|
||||
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
|
||||
var value = 0L
|
||||
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
|
||||
return value
|
||||
}
|
||||
|
||||
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
|
||||
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
|
||||
8 -> Double.fromBits(uint(bytes, at, 8))
|
||||
else -> 0.0
|
||||
}
|
||||
|
||||
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
|
||||
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
|
||||
var at = start
|
||||
while (true) {
|
||||
val child = header(bytes, at, end) ?: return
|
||||
val payload = at + child.length
|
||||
if (child.size < 0 || child.size > end - payload) return
|
||||
visit(child.id, payload, child.size.toInt())
|
||||
at = payload + child.size.toInt()
|
||||
}
|
||||
}
|
||||
|
||||
private fun scan() {
|
||||
val length = input.length
|
||||
val start = readAt(0, 64)
|
||||
val ebml = header(start, 0, start.size)
|
||||
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
|
||||
var position = ebml.length + ebml.size
|
||||
val segment = readAt(position, 12).let { header(it, 0, it.size) }
|
||||
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
|
||||
position += segment.length
|
||||
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
|
||||
while (position < end) {
|
||||
val head = readAt(position, 12)
|
||||
val element = header(head, 0, head.size) ?: break
|
||||
val payload = position + element.length
|
||||
if (element.id == MatroskaIds.CLUSTER) {
|
||||
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
|
||||
// A recording cut short ends in a partial cluster; its complete blocks still count.
|
||||
val available = minOf(element.size, end - payload)
|
||||
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
|
||||
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
|
||||
} else if (element.size < 0) {
|
||||
break
|
||||
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
|
||||
when (element.id) {
|
||||
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
|
||||
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
|
||||
}
|
||||
}
|
||||
if (element.size < 0) break
|
||||
position = payload + element.size
|
||||
}
|
||||
}
|
||||
|
||||
private fun clusterTime(position: Long, size: Int): Long {
|
||||
val head = readAt(position, minOf(size, 16))
|
||||
val first = header(head, 0, head.size)
|
||||
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
|
||||
return uint(head, first.length, first.size.toInt())
|
||||
}
|
||||
val whole = readAt(position, size)
|
||||
var time = 0L
|
||||
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
|
||||
return time
|
||||
}
|
||||
|
||||
private fun parseInfo(bytes: ByteArray) {
|
||||
var duration = 0.0
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
|
||||
MatroskaIds.DURATION -> duration = float(bytes, at, size)
|
||||
}
|
||||
}
|
||||
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
|
||||
}
|
||||
|
||||
private fun parseTracks(bytes: ByteArray) {
|
||||
trackList.clear()
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.TRACK_ENTRY) {
|
||||
var number = 0
|
||||
var type = 0
|
||||
var codec = ""
|
||||
var name = ""
|
||||
var rate = 8000.0
|
||||
var channels = 1
|
||||
var depth = 0
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
|
||||
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
|
||||
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
|
||||
when (audio) {
|
||||
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
|
||||
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
|
||||
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val audio = type == MkaWriter.TRACK_AUDIO
|
||||
trackList.add(
|
||||
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun parseAttachments(bytes: ByteArray) {
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
if (id == MatroskaIds.ATTACHED_FILE) {
|
||||
var name = ""
|
||||
var mime = ""
|
||||
var data = ByteArray(0)
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
|
||||
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
|
||||
}
|
||||
}
|
||||
attachmentList.add(MkaAttachment(name, mime, data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
|
||||
val bytes = readAt(cluster.position, cluster.size)
|
||||
val out = ArrayList<MkaBlock>()
|
||||
var clusterRaw = cluster.rawTime
|
||||
children(bytes, 0, bytes.size) { id, at, size ->
|
||||
when (id) {
|
||||
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
|
||||
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
|
||||
MatroskaIds.BLOCK_GROUP -> {
|
||||
var blockAt = -1
|
||||
var blockSize = 0
|
||||
var duration = -1L
|
||||
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
|
||||
when (field) {
|
||||
MatroskaIds.BLOCK -> {
|
||||
blockAt = fieldAt
|
||||
blockSize = fieldSize
|
||||
}
|
||||
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
|
||||
}
|
||||
}
|
||||
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
|
||||
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
|
||||
if (size < 4) return null
|
||||
val first = bytes[at].toInt() and 0xff
|
||||
val trackLength = first.countLeadingZeroBits() - 24 + 1
|
||||
if (trackLength > 2 || size < trackLength + 3) return null
|
||||
var track = first and (0xff shr trackLength)
|
||||
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
|
||||
val head = at + trackLength
|
||||
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
|
||||
val flags = bytes[head + 2].toInt() and 0xff
|
||||
if (flags and 0x06 != 0) return null
|
||||
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmField
|
||||
import kotlin.jvm.JvmStatic
|
||||
|
||||
/**
|
||||
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
|
||||
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
|
||||
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
|
||||
* contract is the header of g2flash patches/mic_control.c).
|
||||
*/
|
||||
object MicArrayProtocol {
|
||||
const val VERSION = 1
|
||||
const val HEADER_BYTES = 21
|
||||
const val OP_CONFIGURE = 1
|
||||
const val OP_STOP = 3
|
||||
const val OP_RENEW = 4
|
||||
/** How long an armed temple streams without a CONFIGURE or RENEW. */
|
||||
const val LEASE_MS = 90_000L
|
||||
const val FLAG_TRUNCATED = 0x80
|
||||
const val CODEC_RAW = 1
|
||||
const val FORMAT_PCM16 = 0
|
||||
|
||||
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
|
||||
class Frame(
|
||||
/** Low 16 bits of the temple's frame counter. */
|
||||
@JvmField val sequence: Int,
|
||||
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
|
||||
@JvmField val deviceTickMs: Long,
|
||||
@JvmField val channels: Int,
|
||||
@JvmField val sampleRate: Int,
|
||||
@JvmField val format: Int,
|
||||
@JvmField val codec: Int,
|
||||
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
|
||||
@JvmField val angleDegrees: Int,
|
||||
@JvmField val ssr: Int,
|
||||
/** The firmware cut the payload short of the samples it captured. */
|
||||
@JvmField val truncated: Boolean,
|
||||
@JvmField val payloadOffset: Int,
|
||||
@JvmField val payloadLength: Int,
|
||||
)
|
||||
|
||||
/** A control record with no payload: ['M','C', version, op]. */
|
||||
@JvmStatic
|
||||
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
|
||||
|
||||
@JvmStatic
|
||||
fun isConfigure(record: ByteArray?): Boolean =
|
||||
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
|
||||
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
|
||||
|
||||
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
|
||||
@JvmStatic
|
||||
fun decodeFrame(data: ByteArray): Frame? {
|
||||
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
|
||||
return null
|
||||
}
|
||||
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
|
||||
return Frame(
|
||||
sequence = u16(data, 4),
|
||||
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
|
||||
channels = data[10].toInt() and 0xff,
|
||||
sampleRate = u16(data, 11) * 100,
|
||||
format = data[13].toInt() and 0xff,
|
||||
codec = data[14].toInt() and 0xff,
|
||||
angleDegrees = u16(data, 15).toShort().toInt(),
|
||||
ssr = u16(data, 17).toShort().toInt(),
|
||||
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
|
||||
payloadOffset = HEADER_BYTES,
|
||||
payloadLength = length,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
|
||||
* payload isn't raw 16-bit PCM or holds none.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
|
||||
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
|
||||
val frames = frame.payloadLength / (2 * frame.channels)
|
||||
if (frames <= 0) return null
|
||||
val samples = ShortArray(frames * frame.channels)
|
||||
var at = frame.payloadOffset
|
||||
for (index in samples.indices) {
|
||||
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
|
||||
at += 2
|
||||
}
|
||||
return samples
|
||||
}
|
||||
|
||||
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
|
||||
}
|
||||
Loaded 100 of 169 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user