Merge pull request 'chore: synchroniser main avec upstream 25cb255b' (#25) from MasterControl/faceclaw:sync/upstream-main-20261119 into main

Reviewed-on: #25
This commit was merged in pull request #25.
This commit is contained in:
Blendman974 committed 2026-11-19 22:03:23 +00:00
commit 597d38318e
167 files changed
+15885 -526

No files matched your search

+2
View File
@@ -58,6 +58,8 @@ native/kotlin/.kotlin/
native/kotlin/build/
native/kotlin/shared/build/
native/kotlin/local.properties
# Go SSH transport build outputs (scripts/go-ssh-build.cjs)
native/ssh/build/
native/kotlin/plugin/platforms/android/*.aar
native/kotlin/plugin/platforms/ios/
!native/kotlin/plugin/platforms/
+6
View File
@@ -23,6 +23,12 @@ and Mentra Labs, Inc. (MIT License): https://github.com/tetramo-labs/glassly and
https://github.com/tetramo-labs/glassly-rayneo-cfw. The iO wire codecs in
native/kotlin/shared/.../rayneo/RayNeoIoProtocol.kt follow their protocol work.
Bluetooth keyboard support on the G2 uses the keyboard bridge Tetramo, Inc. wrote
for their Glassly custom firmware (https://github.com/tetramo-labs/g2flash).
Faceclaw's client for it, in app/g2/keyboard/ and app/apps/terminal/terminal-keys.ts,
follows the reference client in that repository and the keyboard handling in the
Glassly app (MIT License): https://github.com/tetramo-labs/glassly.
iPhone notification icons are Pebble's notification and timeline icons, from
https://github.com/pebble-dev/iconography: the original Pebble Technology icons
and Rebble community additions. They are licensed under the Apache License 2.0;
@@ -290,6 +290,13 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
/** See [GlassesSessionCore.getRingStatus]. */
fun getRingStatus(): String = core.getRingStatus()
/** See [GlassesSessionCore.setHeadUpTrigger]. */
fun setHeadUpTrigger(degrees: Int, originalSwitch: Int, originalAngle: Int) =
core.setHeadUpTrigger(degrees, originalSwitch, originalAngle)
/** See [GlassesSessionCore.getHeadUpStatus]. */
fun getHeadUpStatus(): String = core.getHeadUpStatus()
/** See [GlassesSessionCore.setGlassesRingLink]. */
fun setGlassesRingLink(action: String, fallbackAddress: String?, fallbackName: String?): Boolean =
core.setGlassesRingLink(action, fallbackAddress, fallbackName)
@@ -322,6 +329,13 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
core.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
fun addKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.addKeyboardEventListener(listener)
fun removeKeyboardEventListener(listener: FaceclawKeyboardEventListener?) = core.removeKeyboardEventListener(listener)
fun sendFaceclawKeyboardControl(record: ByteArray?, label: String): Boolean =
core.sendFaceclawKeyboardControl(record, label)
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener)
fun stopG2AudioForwarding() = core.stopG2AudioForwarding()
@@ -0,0 +1,212 @@
package com.faceclaw.app
import android.content.ClipData
import android.content.ClipboardManager
import android.content.Context
import android.os.Build
import android.os.Handler
import android.os.Looper
import android.security.keystore.KeyGenParameterSpec
import android.security.keystore.KeyProperties
import android.security.keystore.StrongBoxUnavailableException
import android.util.Base64
import android.util.Log
import com.faceclaw.ssh.faceclawssh.Channel
import com.faceclaw.ssh.faceclawssh.Faceclawssh
import com.faceclaw.ssh.faceclawssh.Listener
import com.faceclaw.ssh.faceclawssh.Signer
import java.security.KeyPairGenerator
import java.security.KeyStore
import java.security.PrivateKey
import java.security.Signature
import java.security.spec.ECGenParameterSpec
import javax.crypto.Cipher
import javax.crypto.KeyGenerator
import javax.crypto.SecretKey
import javax.crypto.spec.GCMParameterSpec
/**
* One SSH channel for the TypeScript side (app/native/ssh.ts), backed by the
* gomobile-bound Go transport in native/ssh. `spec` is the Go openSpec JSON.
* Events arrive from Go goroutines and are posted to the Looper of the
* thread that constructed this object (a JS isolate's own thread), like
* FaceclawWebSocket.
*/
class FaceclawSshChannel(spec: String, useKey: Boolean, listener: FaceclawSshListener) {
private val handler = Handler(Looper.myLooper() ?: Looper.getMainLooper())
private val channel: Channel
init {
val signer = if (useKey) FaceclawSshKeys.signer() else null
channel = Faceclawssh.open(spec, signer, object : Listener {
override fun onEvent(kind: String?, payload: String?) {
handler.post {
try {
listener.onEvent(kind, payload)
} catch (t: Throwable) {
Log.w(TAG, "listener onEvent($kind) failed", t)
}
}
}
})
}
fun send(text: String): Boolean = channel.send(text)
fun close() = channel.close()
private companion object {
const val TAG = "FaceclawSsh"
}
}
/**
* The device's SSH client key and the Terminal app's saved SSH passwords.
*
* The client key is an ECDSA P-256 key generated inside Android Keystore
* (StrongBox when the device has one) and never exportable: only its public
* half leaves the keystore, as an authorized_keys line. Passwords are
* encrypted with an AES-GCM key that also lives in Keystore, and stored in a
* private SharedPreferences file kept apart from the settings store (which is
* exported by the settings port).
*/
object FaceclawSshKeys {
private const val TAG = "FaceclawSsh"
private const val CLIENT_KEY_ALIAS = "faceclaw-ssh-client"
private const val SECRET_KEY_ALIAS = "faceclaw-ssh-secrets"
private const val SECRETS_PREFS = "faceclaw_ssh_secrets"
private const val GCM_TAG_BITS = 128
private fun keyStore(): KeyStore = KeyStore.getInstance("AndroidKeyStore").apply { load(null) }
@JvmStatic
@Synchronized
fun hasClientKey(): Boolean = keyStore().containsAlias(CLIENT_KEY_ALIAS)
/** Generate the client key if it doesn't exist yet. */
@JvmStatic
@Synchronized
fun ensureClientKey() {
if (keyStore().containsAlias(CLIENT_KEY_ALIAS)) return
fun spec(strongBox: Boolean): KeyGenParameterSpec {
val builder = KeyGenParameterSpec.Builder(CLIENT_KEY_ALIAS, KeyProperties.PURPOSE_SIGN)
.setAlgorithmParameterSpec(ECGenParameterSpec("secp256r1"))
.setDigests(KeyProperties.DIGEST_SHA256)
if (strongBox && Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) builder.setIsStrongBoxBacked(true)
return builder.build()
}
val generator = KeyPairGenerator.getInstance(KeyProperties.KEY_ALGORITHM_EC, "AndroidKeyStore")
try {
generator.initialize(spec(strongBox = true))
generator.generateKeyPair()
} catch (e: Exception) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && e is StrongBoxUnavailableException) {
Log.i(TAG, "StrongBox unavailable; generating the SSH key in the TEE")
} else {
Log.w(TAG, "StrongBox key generation failed; retrying in the TEE", e)
}
generator.initialize(spec(strongBox = false))
generator.generateKeyPair()
}
Log.i(TAG, "generated SSH client key")
}
/** X.509 SubjectPublicKeyInfo DER of the client key, or null if none. */
@JvmStatic
fun publicKey(): ByteArray? = keyStore().getCertificate(CLIENT_KEY_ALIAS)?.publicKey?.encoded
/** The client key's authorized_keys line (generating the key if needed). */
@JvmStatic
fun authorizedKey(comment: String): String {
ensureClientKey()
return Faceclawssh.authorizedKey(publicKey(), comment)
}
/** The client key's SHA256 fingerprint (generating the key if needed). */
@JvmStatic
fun fingerprint(): String {
ensureClientKey()
return Faceclawssh.fingerprint(publicKey())
}
/** A Go Signer over the client key, generating the key if needed. */
@JvmStatic
fun signer(): Signer? = try {
ensureClientKey()
val publicKey = publicKey() ?: throw IllegalStateException("client key has no certificate")
object : Signer {
override fun publicKey(): ByteArray = publicKey
override fun sign(data: ByteArray?): ByteArray {
val privateKey = keyStore().getKey(CLIENT_KEY_ALIAS, null) as PrivateKey
return Signature.getInstance("SHA256withECDSA").run {
initSign(privateKey)
update(data ?: ByteArray(0))
sign()
}
}
}
} catch (t: Throwable) {
Log.w(TAG, "SSH client key unavailable", t)
null
}
private fun secretKey(): SecretKey {
val store = keyStore()
(store.getKey(SECRET_KEY_ALIAS, null) as? SecretKey)?.let { return it }
val generator = KeyGenerator.getInstance(KeyProperties.KEY_ALGORITHM_AES, "AndroidKeyStore")
generator.init(
KeyGenParameterSpec.Builder(SECRET_KEY_ALIAS, KeyProperties.PURPOSE_ENCRYPT or KeyProperties.PURPOSE_DECRYPT)
.setBlockModes(KeyProperties.BLOCK_MODE_GCM)
.setEncryptionPaddings(KeyProperties.ENCRYPTION_PADDING_NONE)
.setKeySize(256)
.build()
)
return generator.generateKey()
}
private fun prefs(context: Context) =
context.applicationContext.getSharedPreferences(SECRETS_PREFS, Context.MODE_PRIVATE)
@JvmStatic
@Synchronized
fun setSecret(context: Context, id: String, value: String) {
val cipher = Cipher.getInstance("AES/GCM/NoPadding")
cipher.init(Cipher.ENCRYPT_MODE, secretKey())
val sealed = cipher.iv + cipher.doFinal(value.toByteArray(Charsets.UTF_8))
prefs(context).edit().putString(id, Base64.encodeToString(sealed, Base64.NO_WRAP)).apply()
}
/** The saved secret, or "" if none (or it can no longer be decrypted). */
@JvmStatic
@Synchronized
fun getSecret(context: Context, id: String): String {
val encoded = prefs(context).getString(id, null) ?: return ""
return try {
val sealed = Base64.decode(encoded, Base64.NO_WRAP)
val cipher = Cipher.getInstance("AES/GCM/NoPadding")
val ivLength = 12
cipher.init(Cipher.DECRYPT_MODE, secretKey(), GCMParameterSpec(GCM_TAG_BITS, sealed, 0, ivLength))
String(cipher.doFinal(sealed, ivLength, sealed.size - ivLength), Charsets.UTF_8)
} catch (t: Throwable) {
Log.w(TAG, "saved SSH secret unreadable; dropping it", t)
prefs(context).edit().remove(id).apply()
""
}
}
@JvmStatic
@Synchronized
fun removeSecret(context: Context, id: String) {
prefs(context).edit().remove(id).apply()
}
/** Put text on the phone's clipboard (from any thread). */
@JvmStatic
fun copyToClipboard(context: Context, label: String, text: String) {
val appContext = context.applicationContext
Handler(Looper.getMainLooper()).post {
val clipboard = appContext.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
clipboard.setPrimaryClip(ClipData.newPlainText(label, text))
}
}
}
@@ -114,6 +114,21 @@ class FaceclawVoiceController(context: Context) {
}
}
override fun micArrayLink(): MicArrayLink? {
val currentCommunicator = communicator ?: return null
return object : MicArrayLink {
override fun startForwarding(listener: FaceclawAudioPacketListener): Boolean =
currentCommunicator.startG2AudioForwarding(listener)
override fun stopForwarding() = currentCommunicator.stopG2AudioForwarding()
override fun isForwarding(): Boolean = currentCommunicator.isAudioCaptureActive()
override fun sendControl(record: ByteArray, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
currentCommunicator.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
}
}
override fun createPacketDecoder(): AudioPacketDecoder =
if (communicator?.isRayNeoIo() == true) AndroidSpeechEngines.OpusDecoderAdapter()
else AndroidSpeechEngines.Lc3DecoderAdapter()
@@ -140,6 +155,11 @@ class FaceclawVoiceController(context: Context) {
private val session = VoiceCaptureSession(host)
// The recording of this controller's live captures (see startRecording). It outlives any one
// capture, and is only handed to the session while recording is enabled.
private var recording: MicRecordingWriter? = null
private var recordingEnabled = true
fun setListener(listener: FaceclawVoiceControllerListener?) {
session.setListener(listener)
}
@@ -201,6 +221,88 @@ class FaceclawVoiceController(context: Context) {
session.setBeamFilter(enabled, centerDeg, halfWidthDeg)
}
/**
* Capture glasses sessions from the temples' own microphones (the custom firmware's
* mic_control stream) instead of the stock mixed stream. [configureRecord] is the CONFIGURE
* record to arm both temples with; null goes back to the mixed stream. Must be set before
* {@link #start}.
*/
fun setMicArray(configureRecord: ByteArray?) {
session.setMicArray(configureRecord)
}
/**
* Have the next {@link #start} play the recording at [path] (a .mka file) from [startMs] in
* place of any microphone; null goes back to the microphones. Progress arrives through the
* listener's onReplay.
*/
fun setReplay(path: String?, startMs: Long) {
session.setReplay(path, startMs)
}
/** Continue the running replay from [positionMs] into its recording. */
fun seekReplay(positionMs: Long) {
session.seekReplay(positionMs)
}
fun setReplayPaused(paused: Boolean) {
session.setReplayPaused(paused)
}
/**
* Record live captures to a new Matroska audio file at [path], from now until
* {@link #stopRecording}, across however many captures run in between: every source's audio
* as it arrives, before any processing. [configJson] describes the capture and is stored in
* the file. Returns false when the file cannot be created.
*/
@Synchronized
fun startRecording(path: String, configJson: String?): Boolean {
stopRecording()
return try {
val parent = File(path).parentFile
if (parent != null && !parent.exists()) parent.mkdirs()
recording = MicRecordingWriter(openFileOutput(path), configJson, System.currentTimeMillis())
session.setRecorder(if (recordingEnabled) recording else null)
true
} catch (t: Throwable) {
Log.w(TAG, "could not create recording $path", t)
recording = null
false
}
}
/**
* Whether captures are written to the recording right now: off while the capture running
* isn't one that is to be recorded.
*/
@Synchronized
fun setRecordingEnabled(enabled: Boolean) {
recordingEnabled = enabled
session.setRecorder(if (enabled) recording else null)
}
/** Whether a recording is open and still being written (false once writing to it failed). */
@Synchronized
fun isRecording(): Boolean = recording?.failed == false
/**
* Note a change in what is being captured (a JSON object) at the recording's current
* position; the audio after it is placed afresh on the recording's timeline.
*/
@Synchronized
fun noteRecordingEvent(json: String) {
recording?.event(json)
}
/** Complete the recording's file. Returns its length in ms, or -1 when none was open. */
@Synchronized
fun stopRecording(): Long {
val current = recording ?: return -1
recording = null
session.setRecorder(null)
return current.finish()
}
fun start(requestedMode: String?) {
session.start(requestedMode, 0)
}
+31
View File
@@ -0,0 +1,31 @@
#import <Foundation/Foundation.h>
/**
* One SSH channel for the TypeScript side (app/native/ssh.ios.ts), backed by
* the gomobile-bound Go transport in native/ssh. `spec` is the Go openSpec
* JSON. Events arrive on Go threads and queue here; the owning JS isolate
* drains them with takeEvents (a JSON array of {kind, payload}).
*/
@interface FaceclawSshChannel : NSObject
- (instancetype)initWithSpec:(NSString *)spec useKey:(BOOL)useKey;
- (NSString *)takeEvents;
- (BOOL)send:(NSString *)text;
- (void)close;
@end
/**
* The device's SSH client key (a non-exportable Secure Enclave P-256 key; a
* plain keychain key on the simulator, which has no Secure Enclave) and the
* Terminal app's saved SSH passwords (keychain generic passwords).
*/
@interface FaceclawSshKeys : NSObject
/** "ecdsa-sha2-nistp256 AAAA... comment"; generates the key if needed. nil on failure. */
+ (NSString *)authorizedKeyWithComment:(NSString *)comment;
/** "SHA256:..." fingerprint; generates the key if needed. nil on failure. */
+ (NSString *)fingerprint;
+ (void)setSecret:(NSString *)value forId:(NSString *)secretId;
/** The saved secret, or "" if none. */
+ (NSString *)secretForId:(NSString *)secretId;
+ (void)removeSecretForId:(NSString *)secretId;
+ (void)copyToClipboard:(NSString *)text;
@end
+234
View File
@@ -0,0 +1,234 @@
#import "FaceclawSsh.h"
#import <Faceclawssh/Faceclawssh.h>
#import <Security/Security.h>
#import <UIKit/UIKit.h>
static NSString *const kClientKeyTag = @"com.faceclaw.ssh.client";
static NSString *const kSecretService = @"com.faceclaw.ssh.secret";
#pragma mark - Client key
static NSData *ClientKeyTagData(void) {
return [kClientKeyTag dataUsingEncoding:NSUTF8StringEncoding];
}
static SecKeyRef CopyClientKey(void) {
NSDictionary *query = @{
(id)kSecClass: (id)kSecClassKey,
(id)kSecAttrApplicationTag: ClientKeyTagData(),
(id)kSecAttrKeyType: (id)kSecAttrKeyTypeECSECPrimeRandom,
(id)kSecReturnRef: @YES,
};
CFTypeRef key = NULL;
if (SecItemCopyMatching((__bridge CFDictionaryRef)query, &key) != errSecSuccess) return NULL;
return (SecKeyRef)key;
}
static SecKeyRef CreateClientKey(BOOL secureEnclave) {
SecAccessControlRef access = SecAccessControlCreateWithFlags(
kCFAllocatorDefault, kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly,
secureEnclave ? kSecAccessControlPrivateKeyUsage : 0, NULL);
if (!access) return NULL;
NSMutableDictionary *attributes = [@{
(id)kSecAttrKeyType: (id)kSecAttrKeyTypeECSECPrimeRandom,
(id)kSecAttrKeySizeInBits: @256,
(id)kSecPrivateKeyAttrs: @{
(id)kSecAttrIsPermanent: @YES,
(id)kSecAttrApplicationTag: ClientKeyTagData(),
(id)kSecAttrAccessControl: (__bridge id)access,
},
} mutableCopy];
if (secureEnclave) attributes[(id)kSecAttrTokenID] = (id)kSecAttrTokenIDSecureEnclave;
CFErrorRef error = NULL;
SecKeyRef key = SecKeyCreateRandomKey((__bridge CFDictionaryRef)attributes, &error);
CFRelease(access);
if (!key && error) {
NSLog(@"FaceclawSsh: key generation failed (secure enclave %d): %@", secureEnclave, (__bridge NSError *)error);
CFRelease(error);
}
return key;
}
/** The client key, generating it on first use. Caller releases. */
static SecKeyRef CopyOrCreateClientKey(void) {
@synchronized([FaceclawSshKeys class]) {
SecKeyRef key = CopyClientKey();
if (key) return key;
key = CreateClientKey(YES);
#if TARGET_OS_SIMULATOR
if (!key) key = CreateClientKey(NO);
#endif
return key;
}
}
/** Uncompressed X9.62 public point (0x04 || X || Y). */
static NSData *PublicKeyData(SecKeyRef key) {
SecKeyRef publicKey = SecKeyCopyPublicKey(key);
if (!publicKey) return nil;
NSData *data = (__bridge_transfer NSData *)SecKeyCopyExternalRepresentation(publicKey, NULL);
CFRelease(publicKey);
return data;
}
@interface FaceclawSshSigner : NSObject <FaceclawsshSigner>
@end
@implementation FaceclawSshSigner {
SecKeyRef _key;
NSData *_publicKey;
}
- (instancetype)initWithKey:(SecKeyRef)key {
if ((self = [super init])) {
_key = (SecKeyRef)CFRetain(key);
_publicKey = PublicKeyData(key);
}
return self;
}
- (void)dealloc {
if (_key) CFRelease(_key);
}
- (NSData *)publicKey {
return _publicKey;
}
- (NSData *)sign:(NSData *)data error:(NSError **)error {
CFErrorRef cfError = NULL;
NSData *signature = (__bridge_transfer NSData *)SecKeyCreateSignature(
_key, kSecKeyAlgorithmECDSASignatureMessageX962SHA256, (__bridge CFDataRef)(data ?: [NSData data]), &cfError);
if (!signature) {
if (error) *error = (__bridge_transfer NSError *)cfError;
else if (cfError) CFRelease(cfError);
}
return signature;
}
@end
#pragma mark - Channel
@interface FaceclawSshEventQueue : NSObject <FaceclawsshListener>
@property(nonatomic, strong) NSMutableArray *events;
@property(nonatomic) BOOL ended;
@end
@implementation FaceclawSshEventQueue
- (instancetype)init {
if ((self = [super init])) _events = [NSMutableArray array];
return self;
}
- (void)onEvent:(NSString *)kind payload:(NSString *)payload {
@synchronized(self) {
if (_ended) return;
[_events addObject:@{@"kind": kind ?: @"", @"payload": payload ?: @""}];
}
}
@end
@implementation FaceclawSshChannel {
FaceclawsshChannel *_channel;
FaceclawSshEventQueue *_queue;
}
- (instancetype)initWithSpec:(NSString *)spec useKey:(BOOL)useKey {
if ((self = [super init])) {
_queue = [FaceclawSshEventQueue new];
FaceclawSshSigner *signer = nil;
if (useKey) {
SecKeyRef key = CopyOrCreateClientKey();
if (key) {
signer = [[FaceclawSshSigner alloc] initWithKey:key];
CFRelease(key);
}
}
_channel = FaceclawsshOpen(spec, signer, _queue);
}
return self;
}
- (NSString *)takeEvents {
NSArray *events;
@synchronized(_queue) {
if (_queue.events.count == 0) return @"[]";
events = [_queue.events copy];
[_queue.events removeAllObjects];
}
NSData *data = [NSJSONSerialization dataWithJSONObject:events options:0 error:nil];
return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"[]";
}
- (BOOL)send:(NSString *)text {
return [_channel send:text];
}
- (void)close {
@synchronized(_queue) {
_queue.ended = YES;
[_queue.events removeAllObjects];
}
[_channel close];
}
@end
#pragma mark - Keys and secrets
@implementation FaceclawSshKeys
+ (NSString *)authorizedKeyWithComment:(NSString *)comment {
SecKeyRef key = CopyOrCreateClientKey();
if (!key) return nil;
NSData *publicKey = PublicKeyData(key);
CFRelease(key);
NSError *error = nil;
NSString *line = FaceclawsshAuthorizedKey(publicKey, comment ?: @"", &error);
if (error) NSLog(@"FaceclawSsh: authorizedKey failed: %@", error);
return line;
}
+ (NSString *)fingerprint {
SecKeyRef key = CopyOrCreateClientKey();
if (!key) return nil;
NSData *publicKey = PublicKeyData(key);
CFRelease(key);
NSError *error = nil;
NSString *fingerprint = FaceclawsshFingerprint(publicKey, &error);
if (error) NSLog(@"FaceclawSsh: fingerprint failed: %@", error);
return fingerprint;
}
+ (NSDictionary *)secretQuery:(NSString *)secretId {
return @{
(id)kSecClass: (id)kSecClassGenericPassword,
(id)kSecAttrService: kSecretService,
(id)kSecAttrAccount: secretId ?: @"",
};
}
+ (void)setSecret:(NSString *)value forId:(NSString *)secretId {
NSDictionary *query = [self secretQuery:secretId];
SecItemDelete((__bridge CFDictionaryRef)query);
NSMutableDictionary *item = [query mutableCopy];
item[(id)kSecValueData] = [(value ?: @"") dataUsingEncoding:NSUTF8StringEncoding];
item[(id)kSecAttrAccessible] = (id)kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly;
OSStatus status = SecItemAdd((__bridge CFDictionaryRef)item, NULL);
if (status != errSecSuccess) NSLog(@"FaceclawSsh: saving secret failed: %d", (int)status);
}
+ (NSString *)secretForId:(NSString *)secretId {
NSMutableDictionary *query = [[self secretQuery:secretId] mutableCopy];
query[(id)kSecReturnData] = @YES;
query[(id)kSecMatchLimit] = (id)kSecMatchLimitOne;
CFTypeRef result = NULL;
if (SecItemCopyMatching((__bridge CFDictionaryRef)query, &result) != errSecSuccess) return @"";
NSData *data = (__bridge_transfer NSData *)result;
return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"";
}
+ (void)removeSecretForId:(NSString *)secretId {
SecItemDelete((__bridge CFDictionaryRef)[self secretQuery:secretId]);
}
+ (void)copyToClipboard:(NSString *)text {
dispatch_async(dispatch_get_main_queue(), ^{
UIPasteboard.generalPasteboard.string = text ?: @"";
});
}
@end
+1
View File
@@ -2,6 +2,7 @@ module FaceclawGraphics {
header "FaceclawGraphics.h"
header "FaceclawFontRenderer.h"
header "FaceclawSocket.h"
header "FaceclawSsh.h"
header "FaceclawLc3Decoder.h"
header "FaceclawSpeech.h"
header "FaceclawLocation.h"
+4 -2
View File
@@ -80,8 +80,10 @@ import it into the new version (see [config transfer](scripts/config-transfer.md
* **Integration with Android notifications**: shows the same notification
icons your phone does, and popups when notifications arrive.
* **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex
CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on
the glasses, and send them inputs with the voice assistant.
CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), or attach to
tmux sessions over SSH (no server-side setup beyond tmux 3.2+), view them
on the glasses, and send them inputs with the voice assistant. g2mirror
can also be tunnelled over SSH instead of a VPN.
* **Media player controls** including playlist and media library navigation,
compatible with most Android media players.
* **Turn-by-turn directions** (requires a Mapbox API token).
+6 -1
View File
@@ -55,8 +55,10 @@ export type GlanceboardProvider = {
isEnabled: () => boolean;
/** Whether a single tap shows the board (for tapTimeoutMs). */
showOnTap: () => boolean;
/** How long a tap or head-tilt keeps the board up. */
/** How long a tap keeps the board up (and a head-tilt, when its head-down is unlikely to come). */
tapTimeoutMs: () => number;
/** How long a head-tilt showing may last without a head-down (null = no limit). */
tiltTimeoutMs: () => number | null;
/** Whether a long-press holds the board up until released. */
showOnLongPress: () => boolean;
/** Whether the head-tilt wake goes to the board rather than the regular UI. */
@@ -187,6 +189,8 @@ export async function launchWorkerAppWindow(
matchExistingBy?: "appId" | "windowId";
/** Deliver watch swipes raw instead of the fallback (see WorkerWindowSpec). */
acceptsDirectional?: boolean;
/** Deliver hardware-keyboard keys as keys (see WorkerWindowSpec). */
acceptsKeys?: boolean;
},
): Promise<void> {
const existing = shell
@@ -207,6 +211,7 @@ export async function launchWorkerAppWindow(
icon: options.icon,
focus: true,
acceptsDirectional: options.acceptsDirectional,
acceptsKeys: options.acceptsKeys,
});
ctx.appendLog(`launched ${options.windowId}`);
}
+1
View File
@@ -312,6 +312,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+9 -2
View File
@@ -9,6 +9,7 @@ import { hasCalendarPermission } from "../../native/calendar-permissions";
import { type Layer, type LayerContext } from "../../ui/layers";
import { lineStep } from "../../ui/metrics";
import type { Viewport } from "../../ui/viewport";
import { listingDayMs, spanLabel, spansDays } from "./event-days";
/** How far a row's text sits inside its selection box; the title lines up with it. */
const ROW_TEXT_INSET = 10;
@@ -143,14 +144,20 @@ export class CalendarLayer implements Layer {
function buildEventRows(font: UiFont, events: CalendarEvent[]): EventRow[] {
const rows: EventRow[] = [];
let previousDayKey = "";
const nowMs = Date.now();
for (const event of events) {
const dayKey = dayKeyOf(event.startMs);
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(event.startMs);
// An event still going since an earlier day is listed under today, not its first day.
const listedMs = listingDayMs(event, nowMs);
const dayKey = dayKeyOf(listedMs);
const dayHeader = dayKey === previousDayKey ? null : dayHeaderLabel(listedMs);
previousDayKey = dayKey;
const lines: string[] = [];
const timeLabel = event.allDay ? "All day" : formatEventTime(event.startMs);
lines.push(`${timeLabel} ${event.title || "(untitled)"}`);
if (spansDays(event)) {
lines.push(spanLabel(event));
}
if (event.location) {
lines.push(event.location);
}
+43
View File
@@ -0,0 +1,43 @@
/**
* Which day an event is listed under, and how a multi-day span reads. Pure (no
* native or UI imports) so it can be unit-tested.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
export type DayEvent = { startMs: number; endMs: number; allDay: boolean };
function midnight(timestampMs: number): number {
const date = new Date(timestampMs);
return new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
}
/** The last moment the event covers: all-day ends are exclusive (the midnight after its last day). */
function lastMomentMs(event: DayEvent): number {
return event.allDay ? Math.max(event.startMs, event.endMs - 1) : Math.max(event.startMs, event.endMs);
}
/**
* The day to list an event under: its start, or today for one that began on an
* earlier day and is still going. Without this, the third day of a three-day
* event shows under its first day and never under today.
*/
export function listingDayMs(event: DayEvent, nowMs: number): number {
return Math.max(midnight(event.startMs), Math.min(midnight(nowMs), midnight(lastMomentMs(event))));
}
/** Whether the event covers more than one calendar day. */
export function spansDays(event: DayEvent): boolean {
return midnight(lastMomentMs(event)) > midnight(event.startMs);
}
/** "Mon 16 – Wed 18 Nov", or with both months when they differ ("Mon 30 Nov – Tue 1 Dec"). */
export function spanLabel(event: DayEvent): string {
const start = new Date(event.startMs);
const last = new Date(lastMomentMs(event));
const day = (date: Date) => `${WEEKDAYS[date.getDay()]} ${date.getDate()}`;
return start.getMonth() === last.getMonth() && start.getFullYear() === last.getFullYear()
? `${day(start)} – ${day(last)} ${MONTHS[last.getMonth()]}`
: `${day(start)} ${MONTHS[start.getMonth()]} – ${day(last)} ${MONTHS[last.getMonth()]}`;
}
+5 -3
View File
@@ -1,7 +1,7 @@
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import { clamp } from "../../util/numeric-util";
import { addImuListener, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
import { addImuListener, imuPitchDegrees, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu";
import { type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { shell } from "../../ui/shell/shell";
@@ -23,7 +23,8 @@ const BAR_RIGHT_INSET = 4;
/**
* Accelerometer demo: turns the IMU stream on while this page is visible (the
* Developer window is foregrounded and the screen is on), off otherwise, and
* shows the latest x/y/z reading with a simple per-axis bar.
* shows the latest x/y/z reading (Faceclaw's frame, imu-types.ts) with a
* simple per-axis bar, and the head's pitch.
*
* The IMU stream is experimental — the firmware may never populate it — so the
* page also reports how many samples have arrived.
@@ -92,7 +93,8 @@ export class AccelerometerDemoLayer implements Layer {
const bodyX = padding.left + BODY_INDENT;
image.drawText(small, padding.left, top, "Accelerometer", 220);
const status = this.enabled ? "IMU on" : "IMU off";
const pitch = this.reading ? `pitch ${Math.round(imuPitchDegrees(this.reading))}° ` : "";
const status = pitch + (this.enabled ? "IMU on" : "IMU off");
image.drawText(small, right - small.measureText(status), top, status, 150);
if (this.reading) {
+8
View File
@@ -6,6 +6,7 @@ import { BuzzerDemoLayer } from "./buzzer-demo";
import { AccelerometerDemoLayer } from "./accelerometer-demo";
import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark";
import { LightSensorDemoLayer } from "./light-sensor-demo";
import { MouseTestLayer } from "./mouse-test";
import { MicrophonesMonitorLayer } from "./microphones-monitor";
import { ResourceUsageLayer } from "./resource-usage";
import { RingStatusLayer } from "./ring-status";
@@ -70,6 +71,13 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
label: "Microphones",
onSelect: openMicrophones,
},
{
// The touchpad or mouse of a keyboard paired through the glasses.
label: "Mouse cursor",
onSelect: (ctx) => {
ctx.stack.push(new MouseTestLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
},
},
{
label: "BLE bandwidth",
onSelect: (ctx) => {
+206
View File
@@ -0,0 +1,206 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GrayImage } from "../../graphics/image";
import type { HidPointerSample } from "../../g2/keyboard/hid";
import { keyboardService } from "../../g2/keyboard/service";
import { type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { shell } from "../../ui/shell/shell";
import { clamp } from "../../util/numeric-util";
// How often to reconcile holding the pointer with window visibility.
const RECONCILE_INTERVAL_MS = 400;
/** A touchpad reports a hundred times a second; the page repaints at most this often. */
const RENDER_INTERVAL_MS = 33;
/** Cursor pixels per count the touchpad reports. */
const POINTER_GAIN = 1;
/** Left-click marks kept on the page. */
const MAX_MARKS = 12;
/** The arrow, tip at its top left: '#' is the outline, '.' the fill. */
const CURSOR = [
"#",
"##",
"#.#",
"#..#",
"#...#",
"#....#",
"#.....#",
"#......#",
"#.......#",
"#........#",
"#.....#####",
"#..#..#",
"#.# #..#",
"## #..#",
"# #..#",
" #..#",
" ##",
];
/**
* Mouse cursor test: moves an arrow with the touchpad or mouse of the
* keyboard paired through the glasses (app/g2/keyboard/), and shows its
* buttons and wheel. The main UI has no cursor; this is where one is tried
* out.
*
* While the page is visible it takes the pointer, so the buttons and wheel
* show here instead of acting as select, back and scroll. A left click
* leaves a mark where the arrow is and a right click clears them. Leave
* with Esc, or a double tap on the ring or a temple.
*/
export class MouseTestLayer implements Layer {
private x = -1;
private y = -1;
private size = { width: 0, height: 0 };
private buttons = 0;
private wheel = 0;
private marks: Array<{ x: number; y: number }> = [];
private samples = 0;
private rate = 0;
private rateWindowStart = Date.now();
private removed = false;
private release: (() => void) | null = null;
private reconcileTimer: ReturnType<typeof setInterval> | null = null;
private renderTimer: ReturnType<typeof setTimeout> | null = null;
constructor(
private readonly windowId: string,
private readonly requestRender: () => void,
) {
this.reconcileTimer = setInterval(() => this.reconcile(), RECONCILE_INTERVAL_MS);
this.reconcile();
}
/** Hold the pointer only while the page can be seen; elsewhere its buttons go back to being keys. */
private reconcile(): void {
if (this.removed) return;
const visible = shell.isWindowVisible(this.windowId);
if (visible && !this.release) {
this.release = keyboardService.takePointer((sample) => this.onSample(sample));
} else if (!visible && this.release) {
this.release();
this.release = null;
this.buttons = 0;
}
// The rate reads zero once the pointer has been still for a second.
const now = Date.now();
if (now - this.rateWindowStart >= 1000) {
const rate = this.samples;
this.samples = 0;
this.rateWindowStart = now;
if (rate !== this.rate) {
this.rate = rate;
this.scheduleRender();
}
}
}
private onSample(sample: HidPointerSample): void {
if (this.removed) return;
this.samples++;
const { width, height } = this.size;
if (width > 0 && height > 0) {
if (this.x < 0) {
this.x = width / 2;
this.y = height / 2;
}
this.x = clamp(this.x + sample.dx * POINTER_GAIN, 0, width - 1);
this.y = clamp(this.y + sample.dy * POINTER_GAIN, 0, height - 1);
}
const pressed = sample.buttons & ~this.buttons;
this.buttons = sample.buttons;
this.wheel += sample.wheel;
if (pressed & 1 && this.x >= 0) {
this.marks.push({ x: Math.round(this.x), y: Math.round(this.y) });
if (this.marks.length > MAX_MARKS) this.marks.shift();
}
if (pressed & 2) this.marks = [];
this.scheduleRender();
}
private scheduleRender(): void {
if (this.renderTimer !== null || this.removed) return;
this.renderTimer = setTimeout(() => {
this.renderTimer = null;
if (!this.removed) this.requestRender();
}, RENDER_INTERVAL_MS);
}
onRemoved(): void {
this.removed = true;
if (this.reconcileTimer !== null) clearInterval(this.reconcileTimer);
this.reconcileTimer = null;
if (this.renderTimer !== null) clearTimeout(this.renderTimer);
this.renderTimer = null;
this.release?.();
this.release = null;
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height, padding } = ctx.stack.getBaseSize();
this.size = { width, height };
const image = new GrayImage(width, height, 0);
const left = padding.left;
const right = width - padding.right;
const top = padding.top;
const line = font.lineHeight + 4;
image.drawText(font, left, top, "Mouse cursor", 220);
const snapshot = keyboardService.snapshot();
const problem = snapshot.state !== "connected"
? "No keyboard is connected."
: !snapshot.pointer ? "This keyboard has no touchpad or mouse buttons." : null;
if (problem) {
image.drawText(font, left, top + 2 * line, problem, 190);
image.drawText(font, left, top + 3 * line, "Pair one under Settings > Keyboard.", 120);
return image;
}
const position = this.x < 0 ? "move the pointer" : `${Math.round(this.x)}, ${Math.round(this.y)}`;
image.drawText(font, right - font.measureText(position), top, position, 150);
// Buttons: a box each, filled while held.
let x = left;
const boxY = top + line;
for (const [index, label] of ["L", "M", "R"].entries()) {
// HID numbers them left, right, middle; they sit left, middle, right.
const held = !!(this.buttons & [1, 4, 2][index]!);
const boxW = font.measureText(label) + 10;
if (held) image.fillRect(x, boxY, boxW, font.lineHeight + 2, 200);
else image.drawRect(x, boxY, boxW, font.lineHeight + 2, 90);
image.drawText(font, x + 5, boxY + 1, label, held ? 1 : 170);
x += boxW + 4;
}
image.drawText(font, x + 8, boxY + 1, `wheel ${this.wheel}`, 150);
const rate = `${this.rate}/s`;
image.drawText(font, right - font.measureText(rate), boxY + 1, rate, 110);
const hint = "Left click marks, right click clears. Esc or double tap: back";
image.drawText(font, left, height - padding.bottom - font.lineHeight, hint, 90);
for (const mark of this.marks) {
image.drawLine(mark.x - 4, mark.y, mark.x + 4, mark.y, 140);
image.drawLine(mark.x, mark.y - 4, mark.x, mark.y + 4, 140);
}
if (this.x >= 0) drawCursor(image, Math.round(this.x), Math.round(this.y));
return image;
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
if (event.type === "double-click") {
ctx.stack.pop();
}
}
}
function drawCursor(image: GrayImage, x: number, y: number): void {
for (const [row, pixels] of CURSOR.entries()) {
for (let col = 0; col < pixels.length; col++) {
const pixel = pixels[col];
if (pixel === " ") continue;
const px = x + col;
const py = y + row;
if (px < 0 || py < 0 || px >= image.width || py >= image.height) continue;
image.setPixel(px, py, pixel === "#" ? 255 : 70);
}
}
}
+15 -1
View File
@@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access";
import { isFontFile } from "../../native/font-files";
import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts";
import { isDecodableImageFile } from "../../native/image-files";
import { isMicRecordingFile } from "../../native/mic-recording";
import { isPlayableVideoFile } from "../../native/video-player";
import { readEvenHubPackageManifest } from "../evenhub/installed-apps";
import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog";
@@ -38,6 +39,8 @@ export type FilesAppOptions = InProcessAppOptions & {
openFontWindow: (title: string, path: string) => void;
/** Open a video file's player (settings panel first) as its own shell window. */
openVideoWindow: (title: string, path: string) => void;
/** Replay a recording (.mka) through Transcribe, in a window of its own. */
openRecordingInTranscribe: (title: string, path: string) => void;
};
/**
@@ -50,7 +53,7 @@ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow
const browser = new FileBrowserLayer({
isSupportedFile: (name) =>
TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) ||
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))),
(!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name) || isMicRecordingFile(name))),
// The browser handles double-click itself (up a level), so it is not
// wrapped in YieldAtRootLayer; it yields explicitly from the top level.
onLeave: () => shell.yieldFocusToSidebar(),
@@ -235,6 +238,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI
},
];
}
if (isMicRecordingFile(entry.name)) {
return [
{
label: "Open in Transcribe",
onSelect: (ctx) => {
ctx.stack.pop();
options.openRecordingInTranscribe(entry.name, entry.path);
},
},
];
}
if (isPlayableVideoFile(entry.name)) {
return [
{
+4
View File
@@ -14,6 +14,7 @@ import {
FILES_WINDOW_ID,
} from "./files-app";
import { createVideoPlayerWindow } from "./video-window";
import { openRecordingInTranscribe } from "../transcribe";
// Document windows are closeable and non-singleton; the serial keeps their
// windowIds unique.
@@ -91,6 +92,9 @@ const filesApp: AppDefinition = {
},
openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path),
openVideoWindow: (title, path) => openVideoWindow(ctx, title, path),
openRecordingInTranscribe: (title, path) => {
openRecordingInTranscribe(ctx, path, title);
},
}),
),
// A document arriving via Android's Share intent opens as its own window.
+1
View File
@@ -305,6 +305,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
break;
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+1
View File
@@ -236,6 +236,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
case "navigation-sensors":
case "screen":
case "text-input":
case "key":
case "tool-call":
break;
}
+29 -5
View File
@@ -5,7 +5,9 @@ import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from
import { uiFontOverridePickerMenuItem } from "../../ui/font-picker";
import { type InputEvent } from "../../ui/gestures";
import { type Layer, type LayerContext } from "../../ui/layers";
import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
import { drawSelectionHighlight, isMenuItemDisabled, menuHeadingItem, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
import { liveHeadPitchFooter } from "../../g2/head-angle";
import { headUpControlsAvailable } from "../../native/glasses-controls";
import { Menu } from "../../ui/menu-core";
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
@@ -20,7 +22,9 @@ import {
glanceLayout,
glanceLayoutSetting,
glanceShowLinesSetting,
glanceAngleSetting,
glanceShowOnHeadTiltSetting,
glanceTiltTimeoutSetting,
glanceShowOnLongPressSetting,
glanceSlotChoiceLabel,
glanceSlotSettings,
@@ -319,16 +323,34 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
}
/**
* Settings: layout, sleep gestures, duration, slot lines, widget font, and
* stereo depth, then a row per widget on the board for its own settings.
* Settings, in four sections: Layout (the slot grid), Trigger (the sleep
* gestures that show the board, and for how long), Display (slot lines,
* widget font, stereo depth) and Cards (a row per widget on the board, for
* its own settings; left out while the board is empty).
*/
function openGlanceSettings(ctx: LayerContext): void {
const items = (): MenuItem[] => [
const items = (): MenuItem[] => {
const cards = widgetSettingsMenuItems();
return [
menuHeadingItem("Layout"),
// The layout decides which slots (and so which widgets) are on the board.
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
menuHeadingItem("Trigger"),
enumSettingMenuItem(glanceTapDurationSetting),
toggleSettingMenuItem(glanceShowOnLongPressSetting),
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
enumSettingMenuItem(glanceTiltTimeoutSetting),
// Only Android can change the glasses' head-up setting (glasses-controls.ts).
// The controller's settings listener sends a new angle to the glasses; the
// picker shows the head's current angle to choose by.
...(headUpControlsAvailable()
? [enumSettingMenuItem(glanceAngleSetting, {
pickerFooter: liveHeadPitchFooter(() => shell.isWindowVisible(GLANCEBOARD_WINDOW_ID)),
})]
: []),
menuHeadingItem("Display"),
toggleSettingMenuItem(glanceShowLinesSetting),
uiFontOverridePickerMenuItem(glanceFont, {
title: "Glanceboard font",
@@ -336,8 +358,10 @@ function openGlanceSettings(ctx: LayerContext): void {
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
}),
enumSettingMenuItem(glanceDepthSetting),
...widgetSettingsMenuItems(),
...(cards.length ? [menuHeadingItem("Cards"), ...cards] : []),
];
};
const layer = new MenuLayer("Glanceboard settings", items(), PAGE_MENU_LAYOUT);
ctx.stack.push(layer);
}
+66 -8
View File
@@ -49,8 +49,9 @@ export function glanceLayout(): GlanceLayout {
/**
* Whether sleep-time gestures show the board at all. Off restores the plain
* behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt
* wakes the regular UI), double-tap wakes.
* behaviour: a tap or hold while asleep does nothing, a head-tilt wakes the
* regular UI, double-tap wakes; and the head-up angle goes back to the glasses'
* own.
*/
export const glanceboardEnabledSetting = new ConfigSettingBoolean({
id: "glanceboard-enabled",
@@ -85,9 +86,10 @@ function formatTapDuration(value: GlanceTapDuration): string {
}
/**
* How long a tap (or head-tilt) keeps the board up; each further tap restarts
* it. "off" means a single tap does not show the board at all; a head-tilt
* then uses the default duration.
* How long a tap keeps the board up; each further tap restarts it. "off"
* means a single tap does not show the board at all. A head-tilt showing
* lasts until the head comes back down, falling back to this duration (the
* default one when "off") where the head-down is unlikely to arrive.
*/
export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>({
id: "glanceboard-tap-duration",
@@ -97,7 +99,7 @@ export const glanceTapDurationSetting = new ConfigSettingEnum<GlanceTapDuration>
values: TAP_DURATION_VALUES,
formatValue: formatTapDuration,
description:
"How long a single tap (or a head-tilt) keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
"How long a single tap keeps the Glanceboard on screen. Disabled: a single tap does not show the Glanceboard.",
});
/** Whether a single tap while asleep shows the board. */
@@ -119,15 +121,71 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({
description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.",
});
/** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */
/** The head-tilt wake shows the board (until the head comes back down) instead of the regular UI. */
export const glanceShowOnHeadTiltSetting = new ConfigSettingBoolean({
id: "glanceboard-show-on-head-tilt",
label: "Show on head tilt",
storageKey: "glanceboard.showOnHeadTilt",
defaultValue: true,
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI.",
description: "Tilting your head up while asleep shows the Glanceboard instead of waking the regular UI. It stays up until you look back down.",
});
export type GlanceTiltTimeout = "60s" | "30s" | "15s" | "10s" | "never";
/**
* How long a head-tilt showing may last if the head-down never comes (the
* board normally hides when the head comes back down). "never" removes the
* limit; a double-tap still dismisses the board.
*/
export const glanceTiltTimeoutSetting = new ConfigSettingEnum<GlanceTiltTimeout>({
id: "glanceboard-head-tilt-timeout",
label: "Head tilt timeout",
storageKey: "glanceboard.headTiltTimeout",
defaultValue: "60s",
values: ["60s", "30s", "15s", "10s", "never"],
formatValue: (value) => (value === "never" ? "Never" : value.replace("s", " s")),
description:
"The longest a head-tilt keeps the Glanceboard up when looking back down is not detected. Never: it stays until you look down or double-tap. " +
"When looking down is unlikely to register (a low Head tilt angle, or the display was suspended), the tap duration applies if shorter.",
});
/** The Head tilt timeout in ms, or null for none. */
export function glanceTiltTimeoutMs(): number | null {
const value = glanceTiltTimeoutSetting.get();
return value === "never" ? null : Number.parseInt(value, 10) * 1000;
}
const GLANCE_ANGLE_VALUES = ["glasses", "5", "10", "15", "20", "25", "30", "35", "40", "45", "50", "55", "60"] as const;
export type GlanceAngle = (typeof GLANCE_ANGLE_VALUES)[number];
/**
* How far the head must tilt up for a head-tilt wake, as the glasses' own
* pitch reading (degrees). "glasses" keeps the glasses' own setting (from the
* Even app), writing back their original values if Faceclaw changed them. A
* number is written to the glasses' head-up setting on every connection while
* the Glanceboard is enabled (head-angle.ts). Android only.
*/
export const glanceAngleSetting = new ConfigSettingEnum<GlanceAngle>({
id: "glanceboard-head-tilt-angle",
label: "Head tilt angle",
storageKey: "glanceboard.headTiltAngle",
defaultValue: "glasses",
values: GLANCE_ANGLE_VALUES,
formatValue: (value) => (value === "glasses" ? "Glasses default" : `${value}°`),
description:
"How far you tilt your head up to wake the display; the list shows your head's angle now. Glasses default uses the glasses' own setting from the Even app. " +
"The board hides when your head drops 20° below this angle; under 25°, looking straight ahead may not count, so it hides after the tap duration instead.",
});
/**
* The trigger Faceclaw should maintain, in degrees: the Head tilt angle while
* the Glanceboard is enabled, else 0 (the glasses' own setting).
*/
export function glanceAngleTriggerDegrees(): number {
const value = glanceAngleSetting.get();
return value === "glasses" || !glanceboardEnabledSetting.get() ? 0 : Number(value);
}
const DEPTH_VALUES = ["-64", "-48", "-32", "-16", "0", "16", "32", "48", "64"] as const;
export type GlanceDepth = (typeof DEPTH_VALUES)[number];
+2
View File
@@ -9,6 +9,7 @@ import {
glanceShowOnLongPressSetting,
glanceShowOnTap,
glanceTapTimeoutMs,
glanceTiltTimeoutMs,
} from "./glanceboard-settings";
const glanceboardApp: AppDefinition = {
@@ -24,6 +25,7 @@ const glanceboardApp: AppDefinition = {
isEnabled: () => glanceboardEnabledSetting.get(),
showOnTap: glanceShowOnTap,
tapTimeoutMs: glanceTapTimeoutMs,
tiltTimeoutMs: glanceTiltTimeoutMs,
showOnLongPress: () => glanceShowOnLongPressSetting.get(),
showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(),
depth: glanceDepth,
+15 -5
View File
@@ -50,16 +50,26 @@ export const musicShowAlbumArtSetting = new ConfigSettingBoolean({
"Show the album art beside the track details. When off, the title, artist, album and progress bar use the full width of the card.",
});
export const musicShowIdleNoteSetting = new ConfigSettingBoolean({
id: "glanceboard-music-show-idle-note",
label: "Show note when idle",
storageKey: "glanceboard.music.showIdleNote",
defaultValue: true,
description: "Show a dim music note on the card while nothing is playing. When off, the card is left blank.",
});
/** The music card's submenu in Glanceboard > Settings. */
export function musicSettingsMenuItems(): MenuItem[] {
return [toggleSettingMenuItem(musicShowAlbumArtSetting)];
return [toggleSettingMenuItem(musicShowAlbumArtSetting), toggleSettingMenuItem(musicShowIdleNoteSetting)];
}
/**
* Now playing: album art (unless turned off), title/artist/album and a
* progress bar for the active media session (Apple Music on iOS), the Music
* app's header at card size. The media bridge runs for the whole session; the
* widget subscribes and ticks once a second only while something is playing.
* app's header at card size. With nothing playing the card holds a dim note,
* or nothing with that turned off. The media bridge runs for the whole
* session; the widget subscribes and ticks once a second only while
* something is playing.
*/
export class MusicWidget implements GlanceWidget {
private media: MediaControllerState = mediaControllerBridge.snapshot();
@@ -108,8 +118,8 @@ export class MusicWidget implements GlanceWidget {
image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140);
return;
}
// Nothing playing: a dim note, centred, and nothing else.
const note = idleNoteImage();
// Nothing playing: a dim note, centred, and nothing else; or a blank card.
const note = musicShowIdleNoteSetting.get() ? idleNoteImage() : null;
if (note) image.drawImage(note, ((image.width - note.width) / 2) | 0, ((image.height - note.height) / 2) | 0);
return;
}
+21
View File
@@ -56,6 +56,27 @@ export function saveMicConfig(config: MicConfig): void {
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
}
/**
* The configuration a capture of the temples' own microphones arms them
* with: `mics` of 2 is one microphone per temple (the front one), 4 is both
* of each. The front end and direction detection follow Microphone setup
* (the Microphones app), except that both of a temple's microphones only
* come through the codec front end. Always raw PCM16 at 16 kHz, which is
* what the firmware streams and transcription takes.
*/
export function micArrayCaptureConfig(mics: 2 | 4): MicConfig {
const base = loadMicConfig();
return {
...base,
source: mics === 4 ? "codec" : base.source,
channelMask: mics === 4 ? 0b11 : 0b01,
codec: "raw",
format: "pcm16",
sampleRateHz: 16_000,
armHardware: true,
};
}
class MicArrayController {
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
private statuses: MicArrayStatuses = { left: null, right: null };
@@ -272,6 +272,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+3
View File
@@ -370,6 +370,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
handleInput(window, message.event as InputEvent, message.frameId);
break;
case "key":
// Only windows opened with acceptsKeys are sent keys.
break;
case "text-input":
if (editing) {
// Voice input as an alternative to the phone keyboard.
+1
View File
@@ -433,6 +433,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
case "navigation-sensors":
case "text-input":
case "key":
case "tool-call":
break;
}
+3
View File
@@ -227,6 +227,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
handleInput(window, message.event as InputEvent, message.frameId);
break;
case "key":
// Only windows opened with acceptsKeys are sent keys.
break;
case "text-input":
if (editingSetting) {
// Voice input as an alternative to the phone keyboard.
+3
View File
@@ -424,6 +424,9 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
frameTimings.logFrame(message.frameId, `input received in ${message.windowId} worker`);
handleInput(window, message.event as InputEvent, message.frameId);
break;
case "key":
// Only windows opened with acceptsKeys are sent keys.
break;
case "text-input":
if (window && window.windowId === message.windowId) handleTextInput(message.text.trim());
break;
+89 -7
View File
@@ -1,13 +1,21 @@
import { getStringSetting, setStringSetting } from "../../native/settings-store";
/**
* Terminal-app connection registry: the g2mirror hosts the app knows about,
* stored as JSON under one settings key so every isolate sees the same list
* and change notifications ride the ordinary settings-store channel.
* Terminal-app connection registry: the hosts the app knows about, stored as
* JSON under one settings key so every isolate sees the same list and change
* notifications ride the ordinary settings-store channel.
*
* A connection is entered as a g2mirror:// connection string (the same form
* g2mirror-view accepts): g2mirror://<token>@<host>[:port] for plain
* websockets (default port 8737), g2mirrors://... for TLS (default 443).
* A connection is entered as a connection string:
* - g2mirror://<token>@<host>[:port]: a g2mirror server reached directly
* (the same form g2mirror-view accepts) over a plain websocket (default
* port 8737); g2mirrors://... for TLS (default 443).
* - ssh://<user>@<host>[:port]: tmux on an SSH host (default port 22).
* - ssh://<user>@<host>[:port]?g2mirror=<token>[@<host>[:port]]: a g2mirror
* server tunnelled over SSH; the address after the token is where the
* server listens as seen from the SSH host (default 127.0.0.1:8737).
* SSH connections pin the host's key (hostKeys, trusted on first connection
* after the user checks its fingerprint); their passwords live in the
* platform keystore (app/native/ssh), not here.
*/
export const TERMINAL_CONNECTIONS_KEY = "terminal.connections";
@@ -25,6 +33,8 @@ export type TerminalConnection = {
serverName: string | null;
/** Auto-connect while the Terminal app is open. Toggled by Connect/Disconnect. */
enabled: boolean;
/** SSH connections: trusted host keys (SSH wire format, base64). */
hostKeys: string[];
};
export type ParsedConnectionString = {
@@ -54,8 +64,79 @@ export function parseConnectionString(url: string): ParsedConnectionString | nul
return { secure, host, port, authToken };
}
/** The host[:port] part of a connection string; "" if it doesn't parse. */
export type ParsedSshConnection = {
user: string;
host: string;
port: number;
} & (
| { mode: "tmux" }
| {
mode: "g2mirror";
authToken: string;
/** Where g2mirror-server listens, from the SSH host's point of view. */
targetHost: string;
targetPort: number;
}
);
function decodeComponent(text: string): string {
try {
return decodeURIComponent(text);
} catch {
return text;
}
}
/** Parse an ssh://user@host[:port][?g2mirror=token[@host[:port]]] string; null if malformed. */
export function parseSshConnectionString(url: string): ParsedSshConnection | null {
const match = /^ssh:\/\/([^@\s/?]+)@(\[[^\]\s]+\]|[^:/@\s?]+)(?::(\d+))?\/?(?:\?(.*))?$/.exec(url.trim());
if (!match) return null;
const user = decodeComponent(match[1]!);
const host = match[2]!.replace(/^\[(.*)\]$/, "$1");
const port = match[3] ? parseInt(match[3], 10) : 22;
if (!user || !(port >= 1 && port <= 65535)) return null;
const params = new Map<string, string>();
for (const part of (match[4] ?? "").split("&").filter(Boolean)) {
const equals = part.indexOf("=");
if (equals < 0) return null;
params.set(part.slice(0, equals), decodeComponent(part.slice(equals + 1)));
}
for (const key of params.keys()) {
if (key !== "g2mirror") return null;
}
const g2mirror = params.get("g2mirror");
if (g2mirror === undefined) return { user, host, port, mode: "tmux" };
// token[@host[:port]], split at the last "@" so the token may contain one.
const at = g2mirror.lastIndexOf("@");
const authToken = at < 0 ? g2mirror : g2mirror.slice(0, at);
const target = at < 0 ? "" : g2mirror.slice(at + 1);
const targetMatch = /^(\[[^\]\s]+\]|[^:\s]*)(?::(\d+))?$/.exec(target);
if (!authToken || !targetMatch) return null;
const targetHost = targetMatch[1] || "127.0.0.1";
const targetPort = targetMatch[2] ? parseInt(targetMatch[2], 10) : 8737;
if (!(targetPort >= 1 && targetPort <= 65535)) return null;
return { user, host, port, mode: "g2mirror", authToken, targetHost, targetPort };
}
export type ParsedAnyConnection =
| ({ kind: "g2mirror" } & ParsedConnectionString)
| ({ kind: "ssh" } & ParsedSshConnection);
/** Parse any supported connection string; null if it's none of them. */
export function parseAnyConnectionString(url: string): ParsedAnyConnection | null {
const direct = parseConnectionString(url);
if (direct) return { kind: "g2mirror", ...direct };
const ssh = parseSshConnectionString(url);
return ssh ? { kind: "ssh", ...ssh } : null;
}
/** The [user@]host[:port] part of a connection string (never a token); "" if it doesn't parse. */
export function connectionStringHostLabel(url: string): string {
const ssh = parseSshConnectionString(url);
if (ssh) {
const host = ssh.host.includes(":") ? `[${ssh.host}]` : ssh.host;
return `${ssh.user}@${host}${ssh.port === 22 ? "" : `:${ssh.port}`}`;
}
const parsed = parseConnectionString(url);
if (!parsed) return "";
const defaultPort = parsed.secure ? 443 : 8737;
@@ -79,6 +160,7 @@ export function loadConnections(): TerminalConnection[] {
url: String(item?.url ?? ""),
serverName: typeof item?.serverName === "string" && item.serverName ? item.serverName : null,
enabled: item?.enabled !== false,
hostKeys: Array.isArray(item?.hostKeys) ? item.hostKeys.filter((key: unknown) => typeof key === "string") : [],
}))
.filter((connection) => connection.id.length > 0 && connection.url.length > 0);
} catch {
+2
View File
@@ -14,6 +14,8 @@ const terminalApp: AppDefinition = {
// Session windows stay independent: launching reopens the hub even
// while terminal sessions are open.
matchExistingBy: "windowId",
// The hub window can show a session too, so it takes keys like one.
acceptsKeys: true,
}),
};
+502 -38
View File
@@ -16,11 +16,17 @@
* Each view has its own websocket connection (the protocol allows one attached
* session per connection) and its own xterm emulator.
*
* Connections are g2mirror://<token>@<host>[:port] strings (g2mirrors:// for
* TLS), stored as JSON in the settings store (see connections.ts). Each
* enabled connection gets a control websocket that backs the session list and
* carries unsolicited bell/title notifications; control connections live as
* long as the worker so bells keep flowing even if the hub window is closed.
* Connections are connection strings stored as JSON in the settings store
* (see connections.ts): g2mirror://<token>@<host>[:port] for a g2mirror
* server (g2mirrors:// for TLS), ssh://<user>@<host> for tmux over SSH
* (TmuxClient), or ssh://<user>@<host>?g2mirror=<token> for a g2mirror server
* tunnelled over SSH. Each enabled connection gets a control client that
* backs the session list and carries bell/title/activity notifications;
* control connections live as long as the worker so bells keep flowing even
* if the hub window is closed. SSH connections that need the user (an
* unverified or changed host key, a password, an unauthorized key) record
* the failure on the control (sshIssue) and offer a page that resolves it
* instead of retrying.
* The host's server_name (from the init reply) is cached per connection and
* used as its display name, falling back to the connection string's
* host[:port] before the first successful handshake.
@@ -55,7 +61,24 @@ import {
import * as frameTimings from "../../native/frame-timings";
import { getActiveDisplay } from "../../native/active-display";
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
import { G2MirrorClient, type G2MirrorClientOptions, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
import type { KeyEvent } from "../../g2/keyboard/keys";
import { encodeTerminalKey, terminalScrollbackKey } from "./terminal-keys";
import { G2MirrorClient, type G2MirrorSession, type G2MirrorState } from "../../native/g2mirror-client";
import type { TerminalClient } from "./terminal-client";
import { TmuxClient, type OpenLineChannel } from "./tmux-client";
import {
copyToPhoneClipboard,
describeSshFailure,
getSshSecret,
openSshChannel,
removeSshSecret,
setSshSecret,
sshClientKey,
type SshEndpoint,
type SshFailure,
type SshHostKey,
} from "../../native/ssh";
import { wrapText } from "../../graphics/textwrap";
import { onSettingsStoreChanged } from "../../native/settings-store";
import { clamp } from "../../util/numeric-util";
import {
@@ -64,12 +87,22 @@ import {
terminalDisplayLayoutSetting,
terminalLaunchPresetsSetting,
terminalNewConnectionSetting,
terminalSshPasswordSetting,
terminalWakeOnBellSetting,
textSettingMenuItem,
toggleSettingMenuItem,
} from "../../ui/dashboard-settings";
import { terminalFontPickerMenuItem } from "../../ui/font-picker";
import { connectionDisplayName, loadConnections, parseConnectionString, saveConnections, TERMINAL_CONNECTIONS_KEY, updateConnection, type TerminalConnection } from "./connections";
import {
connectionDisplayName,
connectionStringHostLabel,
loadConnections,
parseAnyConnectionString,
saveConnections,
TERMINAL_CONNECTIONS_KEY,
updateConnection,
type TerminalConnection,
} from "./connections";
import { TerminalEmulator } from "./terminal-emulator";
import { drawSubmenuIndicator, submenuItem, type MenuItem } from "../../ui/menu";
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
@@ -112,6 +145,10 @@ const RECONNECT_MAX_DELAY_MS = 60_000;
const NEW_CONNECTION_DRAFT_KEY = "terminal.newConnectionDraft";
// Storage key of terminalLaunchPresetsSetting, edited from the window menu.
const LAUNCH_PRESETS_KEY = "terminal.launchPresets";
// Storage key of terminalSshPasswordSetting (the password page's draft).
const SSH_PASSWORD_DRAFT_KEY = "terminal.sshPasswordDraft";
// Comment on this device's SSH public key in authorized_keys.
const SSH_KEY_COMMENT = "faceclaw";
type BaseWindow = {
windowId: string;
@@ -148,10 +185,24 @@ type BaseWindow = {
/**
* What the hub window is showing: the session list, the Manage Connections
* section, or the Add-connection screen (type the g2mirror:// string on the
* phone, click to save).
* section, the Add-connection screen (type the connection string on the
* phone, click to save), or a page of text with a click action (verify a
* host key, enter a password, show this device's SSH key).
*/
type HubMode = "sessions" | "connections" | "add";
type HubMode = "sessions" | "connections" | "add" | "page";
type HubPage = {
title: string;
/** Body text, wrapped to the window; re-read on every paint. */
body: () => string;
/** Bottom hint line. */
hint: string;
onClick?: () => void;
/** Double-click. */
onBack: () => void;
/** Runs whenever the page goes away (back, click, window closed). */
onLeave?: () => void;
};
type HubWindow = BaseWindow & {
kind: "hub";
@@ -168,6 +219,8 @@ type HubWindow = BaseWindow & {
sessionOrder: string[];
/** Parse-failure message shown on the Add-connection screen. */
addError: string;
/** The page shown in mode "page". */
page: HubPage | null;
};
type ViewWindow = BaseWindow & {
@@ -186,7 +239,7 @@ type ViewWindow = BaseWindow & {
label: string;
/** Sidebar-icon character (">3"); "" if every glyph was taken at open time. */
glyph: string;
client: G2MirrorClient;
client: TerminalClient;
emulator: TerminalEmulator;
receivedData: boolean;
attachRequested: boolean;
@@ -220,8 +273,8 @@ type PendingView = {
socket: string;
label: string;
glyph: string;
/** Client options captured when the view was requested from a live control. */
options: G2MirrorClientOptions;
/** How to make the view's client, captured when it was requested from a live control. */
options: ClientRecipe;
};
const windows = new Map<string, TerminalWindow>();
@@ -302,7 +355,13 @@ let screenOn = true;
*/
type ControlConnection = {
config: TerminalConnection;
client: G2MirrorClient | null;
client: TerminalClient | null;
/**
* An SSH failure only the user can fix (host key to verify, password or
* key needed). Shown on the host's heading with a row that opens the fix;
* reconnects wait for it. Cleared on a successful connection.
*/
sshIssue: SshFailure | null;
state: G2MirrorState | null;
unsubscribers: Array<() => void>;
reconnectTimer: ReturnType<typeof setTimeout> | null;
@@ -435,6 +494,13 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
}
break;
}
case "key": {
const window = windows.get(message.windowId);
if (!window) break;
window.focused = message.focused;
handleKey(window, message.event as KeyEvent);
break;
}
case "render": {
const window = windows.get(message.windowId);
if (!window) break;
@@ -504,7 +570,8 @@ onSettingsStoreChanged((key) => {
reportIdle();
return;
case NEW_CONNECTION_DRAFT_KEY:
// Live keystrokes from the phone editor; repaint the add screen.
case SSH_PASSWORD_DRAFT_KEY:
// Live keystrokes from the phone editor; repaint the add/password screen.
renderHubWindows();
return;
case "terminal.autoReconnect":
@@ -578,6 +645,7 @@ function openWindow(windowId: string, surfaceId: string, title: string, viewport
list: null,
sessionOrder: [],
addError: "",
page: null,
});
// The hub carries the app's assistant tools; declare them once it exists.
post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS });
@@ -610,6 +678,10 @@ function closeWindow(windowId: string): void {
// Window closed mid-add: shut the phone editor down.
endAddConnection(window);
}
if (window.kind === "hub" && window.mode === "page") {
// Likewise for a page's phone editor (the password draft).
closeHubPage(window, "sessions");
}
// Likewise for a setting being edited from the window menu.
window.menu?.close();
windows.delete(windowId);
@@ -627,15 +699,118 @@ function closeWindow(windowId: string): void {
}
}
/** Client options for a connection string; null if the string doesn't parse. */
function clientOptionsFor(connection: TerminalConnection): G2MirrorClientOptions | null {
const parsed = parseConnectionString(connection.url);
if (!parsed) return null;
return {
...parsed,
deviceName: DEVICE_NAME,
...viewGrid(),
/**
* How to make clients for a connection: "g2mirror" dials a g2mirror server
* directly, "ssh-g2mirror" reaches one through an SSH tunnel, "tmux" runs
* tmux control mode over SSH.
*/
type ClientRecipe = {
kind: "g2mirror" | "ssh-g2mirror" | "tmux";
/**
* `role` "control" watches bells (tmux keeps the bell bookkeeping for that
* client only). `onSshFailure` hears why an SSH channel failed.
*/
create(cols: number, rows: number, role: "control" | "view", onSshFailure?: (failure: SshFailure) => void): TerminalClient;
};
/** Client recipe for a connection string; null if the string doesn't parse. */
function clientOptionsFor(connection: TerminalConnection): ClientRecipe | null {
const parsed = parseAnyConnectionString(connection.url);
if (!parsed) return null;
if (parsed.kind === "g2mirror") {
const { secure, host, port, authToken } = parsed;
return {
kind: "g2mirror",
create: (cols, rows) => new G2MirrorClient({ secure, host, port, authToken, deviceName: DEVICE_NAME, cols, rows }),
};
}
const connectionId = connection.id;
const label = connectionStringHostLabel(connection.url);
// Read at every channel open, so a newly trusted key or saved password
// applies to the next (re)connect, views included.
const endpoint = (): SshEndpoint => ({
host: parsed.host,
port: parsed.port,
user: parsed.user,
hostKeys: loadConnections().find((candidate) => candidate.id === connectionId)?.hostKeys ?? [],
password: getSshSecret(sshPasswordSecretId(connectionId)),
useKey: true,
});
if (parsed.mode === "g2mirror") {
const { authToken, targetHost, targetPort } = parsed;
const urlHost = targetHost.includes(":") && !targetHost.startsWith("[") ? `[${targetHost}]` : targetHost;
const url = `ws://${urlHost}:${targetPort}/`;
return {
kind: "ssh-g2mirror",
create: (cols, rows, _role, onSshFailure) =>
new G2MirrorClient({
secure: false,
host: targetHost,
port: targetPort,
authToken,
deviceName: DEVICE_NAME,
cols,
rows,
transport: {
label: `${label} over SSH`,
open: (listener) => {
const channel = openSshChannel(endpoint(), { type: "websocket", url }, {
onOpen: () => listener.onOpen(),
onText: (text) => listener.onTextMessage(text),
onClosed: (info) => listener.onClosed(info.code ?? 1000, info.reason ?? ""),
onFailure: (failure) => {
onSshFailure?.(failure);
listener.onFailure(describeSshFailure(failure));
},
});
return { sendText: (text) => channel.send(text), sendBinary: () => false, close: () => channel.close() };
},
},
}),
};
}
return {
kind: "tmux",
create: (cols, rows, role, onSshFailure) => {
const openChannel: OpenLineChannel = (command, listener) =>
openSshChannel(endpoint(), { type: "exec", command }, {
onOpen: () => listener.onOpen(),
onText: (line) => listener.onLine(line),
onStderr: (line) => listener.onStderr(line),
onClosed: (info) => listener.onClosed(info.exitStatus ?? null),
onFailure: (failure) => {
onSshFailure?.(failure);
listener.onFailure(describeSshFailure(failure));
},
});
return new TmuxClient({ openChannel, label, cols, rows, stateKey: role === "control" ? connectionId : "" });
},
};
}
function sshPasswordSecretId(connectionId: string): string {
return `terminal.password.${connectionId}`;
}
/** Failures that retrying can't fix: the user has to verify a key or supply credentials. */
function sshFailureNeedsUser(failure: SshFailure): boolean {
return failure.kind === "hostkey-unknown" || failure.kind === "hostkey-mismatch" || failure.kind === "auth";
}
/**
* A control's SSH channel failed. If the user has to act, remember why, and
* if they're looking at the hub's lists, open the page that fixes it.
*/
function noteSshFailure(control: ControlConnection, failure: SshFailure): void {
if (!sshFailureNeedsUser(failure) || controls.get(control.config.id) !== control || !control.client) return;
control.sshIssue = failure;
renderHubWindows();
for (const window of windows.values()) {
if (window.kind === "hub" && window.foreground && (window.mode === "sessions" || window.mode === "connections")) {
openSshIssuePage(window, control);
break;
}
}
}
/**
@@ -663,6 +838,7 @@ function syncControlsFromSettings(): void {
control = {
config: connection,
client: null,
sshIssue: null,
state: null,
unsubscribers: [],
reconnectTimer: null,
@@ -684,12 +860,13 @@ function syncControlsFromSettings(): void {
function startControl(control: ControlConnection): void {
stopControl(control);
const options = clientOptionsFor(control.config);
if (!options) {
const recipe = clientOptionsFor(control.config);
if (!recipe) {
control.state = null;
return;
}
const client = new G2MirrorClient(options);
const grid = viewGrid();
const client = recipe.create(grid.cols, grid.rows, "control", (failure) => noteSshFailure(control, failure));
const connectionId = control.config.id;
control.client = client;
control.state = client.state();
@@ -703,6 +880,7 @@ function startControl(control: ControlConnection): void {
control.state = state;
if (state.phase === "connected") {
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
control.sshIssue = null;
maybeCacheServerName(control, state.serverName);
}
if (state.phase === "failed") scheduleControlReconnect(control);
@@ -777,6 +955,8 @@ function scheduleControlReconnect(control: ControlConnection): void {
if (!terminalAutoReconnectSetting.get()) return;
if (windows.size === 0 && !hasTerminalBackgroundWork()) return;
if (!control.config.enabled) return;
// Waiting on the user (host key, password) or on the host (no usable tmux).
if (control.sshIssue || control.state?.retryable === false) return;
const delayMs = control.reconnectDelayMs;
control.reconnectDelayMs = Math.min(control.reconnectDelayMs * 2, RECONNECT_MAX_DELAY_MS);
control.reconnectTimer = setTimeout(() => {
@@ -943,7 +1123,7 @@ function createViewWindow(
const cell = getTerminalFontConfig();
const gridCols = Math.floor(viewport.width / cell.cellWidth);
const gridRows = Math.floor(viewport.height / cell.cellHeight);
const client = new G2MirrorClient({ ...view.options, cols: gridCols, rows: gridRows });
const client = view.options.create(gridCols, gridRows, "view");
const window: ViewWindow = {
kind: "view",
hub: null,
@@ -1238,6 +1418,11 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
renderAndSubmit(window, frameId);
return;
}
if (window.kind === "hub" && window.mode === "page") {
window.page?.onBack();
renderAndSubmit(window, frameId);
return;
}
if (window.kind === "hub" && window.mode === "connections") {
setHubMode(window, "sessions");
renderAndSubmit(window, frameId);
@@ -1260,6 +1445,35 @@ function handleInput(window: TerminalWindow, event: InputEvent, frameId: number)
frameTimings.finishFrame(frameId, "discarded: terminal view ignored input");
}
/**
* A key from a hardware keyboard (the glasses' keyboard bridge) that no
* keybinding claimed. A session with nothing over it gets the key as its
* terminal would: Escape and the arrows belong to the program, and the way
* out is Escape with Shift, Ctrl or Alt, which never arrives here (it is a
* built-in keybinding for "back"). Anywhere else (the hub's lists, an open
* menu) the key is handed back, to act as navigation.
*/
function handleKey(window: TerminalWindow, event: KeyEvent): void {
if (window.kind === "view" && !window.menu?.isOpen()) {
activeViewId = window.windowId;
const page = terminalScrollbackKey(event);
if (page !== null) {
handleViewScroll(window, page, 0);
return;
}
const bytes = encodeTerminalKey(event, window.emulator.applicationCursorKeys());
if (bytes === null) return;
// Typing follows the live screen, as in any terminal emulator.
if (window.scrollTop !== null) {
window.scrollTop = null;
scheduleRender(window);
}
window.client.sendInput(bytes);
return;
}
post({ type: "key-unhandled", windowId: window.windowId, event });
}
/** Top visible absolute line index when following the live bottom. */
function followTop(window: ViewWindow): number {
return window.historyNext + window.emulator.bufferLength() - window.gridRows;
@@ -1386,6 +1600,154 @@ function setHubMode(window: HubWindow, mode: HubMode): void {
window.addError = "";
}
function openHubPage(window: HubWindow, page: HubPage): void {
if (window.mode === "add") endAddConnection(window);
if (window.mode === "page") window.page?.onLeave?.();
window.page = page;
window.mode = "page";
scheduleRender(window);
}
/** Leave the current page (if any) for `mode`. */
function closeHubPage(window: HubWindow, mode: HubMode): void {
const page = window.page;
window.page = null;
page?.onLeave?.();
setHubMode(window, mode);
scheduleRender(window);
}
/** The list a page returns to: the one it was opened from. */
function hubListMode(window: HubWindow): HubMode {
return window.mode === "connections" ? "connections" : "sessions";
}
function openSshIssuePage(window: HubWindow, control: ControlConnection): void {
const issue = control.sshIssue;
if (!issue) return;
if (issue.kind === "hostkey-unknown" && issue.hostKey) {
openHostKeyPage(window, control, issue.hostKey);
} else if (issue.kind === "hostkey-mismatch" && issue.hostKey) {
openHostKeyChangedPage(window, control, issue.hostKey);
} else if (issue.kind === "auth" && issue.passwordAllowed) {
openPasswordPage(window, control, Boolean(issue.passwordTried));
} else if (issue.kind === "auth") {
openClientKeyPage(window, `${connectionDisplayName(control.config)} didn't accept this device's key (and doesn't take passwords).\n`);
}
}
function hostKeyTypeName(type: string): string {
if (type === "ssh-ed25519") return "ED25519";
if (type.startsWith("ecdsa-")) return "ECDSA";
if (type === "ssh-rsa") return "RSA";
return type;
}
function hostKeyFile(type: string): string {
if (type === "ssh-ed25519") return "/etc/ssh/ssh_host_ed25519_key.pub";
if (type.startsWith("ecdsa-")) return "/etc/ssh/ssh_host_ecdsa_key.pub";
return "/etc/ssh/ssh_host_rsa_key.pub";
}
/** First connection to an SSH host: show the key's fingerprint; click trusts it. */
function openHostKeyPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
const returnMode = hubListMode(window);
openHubPage(window, {
title: "Verify host key",
body: () =>
`First connection to ${connectionStringHostLabel(control.config.url)}. Its ${hostKeyTypeName(hostKey.type)} host key fingerprint is:\n` +
`${hostKey.fingerprint}\n\n` +
`To check it, run this on the host and compare:\nssh-keygen -lf ${hostKeyFile(hostKey.type)}`,
hint: `click trust and connect ${GESTURE_DOUBLE_CLICK} cancel`,
onClick: () => {
const hostKeys = [...control.config.hostKeys.filter((key) => key !== hostKey.key), hostKey.key];
control.config.hostKeys = hostKeys;
updateConnection(control.config.id, { hostKeys });
control.sshIssue = null;
closeHubPage(window, returnMode);
connectControl(control);
},
onBack: () => closeHubPage(window, returnMode),
});
}
function openHostKeyChangedPage(window: HubWindow, control: ControlConnection, hostKey: SshHostKey): void {
const returnMode = hubListMode(window);
openHubPage(window, {
title: "HOST KEY CHANGED",
body: () =>
`${connectionStringHostLabel(control.config.url)} presented a different host key than the one trusted before: ` +
`${hostKeyTypeName(hostKey.type)} ${hostKey.fingerprint}\n\n` +
"Someone may be intercepting the connection, so Faceclaw won't connect. If the host was reinstalled or its keys " +
'were regenerated, use "Forget host key" on this connection in Manage Connections, then verify the new fingerprint.',
hint: `${GESTURE_DOUBLE_CLICK} back`,
onBack: () => closeHubPage(window, returnMode),
});
}
/** Ask for the host's password on the phone (masked editor); click saves it and connects. */
function openPasswordPage(window: HubWindow, control: ControlConnection, rejected: boolean): void {
const returnMode = hubListMode(window);
terminalSshPasswordSetting.set("");
post({ type: "start-text-setting-edit", settingId: terminalSshPasswordSetting.id });
openHubPage(window, {
title: "SSH password",
body: () => {
const typed = terminalSshPasswordSetting.get();
return (
(rejected ? "The host refused the saved password.\n" : "") +
`Type the password for ${connectionStringHostLabel(control.config.url)} in the phone app, then click to connect. ` +
"It is saved encrypted on this phone.\n\n" +
(typed ? "*".repeat(Math.min(typed.length, 32)) : "(nothing typed yet)")
);
},
hint: `click connect ${GESTURE_DOUBLE_CLICK} cancel`,
onClick: () => {
const password = terminalSshPasswordSetting.get();
if (!password) return;
setSshSecret(sshPasswordSecretId(control.config.id), password);
control.sshIssue = null;
closeHubPage(window, returnMode);
connectControl(control);
},
onBack: () => closeHubPage(window, returnMode),
onLeave: () => {
terminalSshPasswordSetting.set("");
post({ type: "end-text-setting-edit" });
},
});
}
/** This device's SSH public key (the private half never leaves the keystore); click copies it. */
function openClientKeyPage(window: HubWindow, intro = ""): void {
const returnMode = hubListMode(window);
let key: { authorizedKey: string; fingerprint: string } | null = null;
let error = "";
try {
key = sshClientKey(SSH_KEY_COMMENT);
} catch (caught) {
error = String((caught as Error)?.message ?? caught);
}
let copied = false;
openHubPage(window, {
title: "This device's SSH key",
body: () =>
key
? `${intro}Add this line to ~/.ssh/authorized_keys on each host to log in with this phone's key:\n` +
`${key.authorizedKey}\n${key.fingerprint}` +
(copied ? "\n\nCopied to the phone's clipboard." : "")
: `Couldn't create this device's key: ${error}`,
hint: key ? `click copy to phone clipboard ${GESTURE_DOUBLE_CLICK} back` : `${GESTURE_DOUBLE_CLICK} back`,
onClick: key
? () => {
copyToPhoneClipboard("SSH public key", key!.authorizedKey);
copied = true;
}
: undefined,
onBack: () => closeHubPage(window, returnMode),
});
}
/**
* One control connection's sessions in this hub window's display order:
* most-recently-updated first as of when the window became visible, with
@@ -1441,6 +1803,7 @@ function hubSessionItems(window: HubWindow): HubItem[] {
];
const multiHost = controls.size > 1;
const canLaunch = launchPresetNames().length > 0;
const isTmux = (control: ControlConnection) => clientOptionsFor(control.config)?.kind === "tmux";
// Every configured host is listed under its own heading, in stored order,
// whatever its connection state: a connected host lists its sessions, any
// other shows its state on the heading plus a Connect row (unless it is
@@ -1450,10 +1813,17 @@ function hubSessionItems(window: HubWindow): HubItem[] {
const name = connectionDisplayName(control.config);
if (!isControlConnected(control)) {
items.push({ label: `${name} (${controlStatusWord(control)})`, heading: true });
if (control.sshIssue && control.config.enabled) {
items.push({ label: sshIssueAction(control.sshIssue), onSelect: () => openSshIssuePage(window, control) });
continue;
}
const connecting = control.config.enabled && control.state?.phase === "connecting";
if (!connecting && clientOptionsFor(control.config)) {
// A failure retrying can't fix (tmux missing or too old) says why;
// selecting it retries once the host is fixed.
const unfixable = control.config.enabled && control.state?.retryable === false;
items.push({
label: control.reconnectTimer ? "Connect now" : "Connect",
label: unfixable ? control.state!.status : control.reconnectTimer ? "Connect now" : "Connect",
onSelect: () => connectControl(control),
});
}
@@ -1462,7 +1832,8 @@ function hubSessionItems(window: HubWindow): HubItem[] {
items.push({
label: name,
heading: true,
onSelect: canLaunch ? () => launchOnHost(window, control) : undefined,
// tmux hosts can always start a shell, presets or not.
onSelect: canLaunch || isTmux(control) ? () => launchOnHost(window, control) : undefined,
});
const sessions = orderedSessions(window, control);
for (const session of sessions) {
@@ -1478,7 +1849,11 @@ function hubSessionItems(window: HubWindow): HubItem[] {
}
if (!sessions.length) {
items.push({
label: multiHost ? "(no live sessions)" : "(no live sessions; run g2mirror <command>)",
label: isTmux(control)
? "(no tmux sessions; select the host to start one)"
: multiHost
? "(no live sessions)"
: "(no live sessions; run g2mirror <command>)",
onSelect: () => control.client?.listSessions(),
});
}
@@ -1499,6 +1874,10 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
label: "Add connection",
onSelect: () => beginAddConnection(window),
},
{
label: "This device's SSH key",
onSelect: () => openClientKeyPage(window),
},
{
label: "Done",
onSelect: () => setHubMode(window, "sessions"),
@@ -1509,6 +1888,13 @@ function hubConnectionItems(window: HubWindow): HubItem[] {
function controlStatusWord(control: ControlConnection): string {
if (!control.config.enabled) return "off";
const issue = control.sshIssue;
if (issue) {
if (issue.kind === "hostkey-unknown") return "verify host key";
if (issue.kind === "hostkey-mismatch") return "HOST KEY CHANGED";
if (issue.passwordTried) return "password rejected";
return issue.passwordAllowed ? "password needed" : "key not authorized";
}
switch (control.state?.phase) {
case "connected":
case "attached":
@@ -1541,6 +1927,32 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
connectControl(control);
},
},
...(control.config.hostKeys.length
? [
{
label: "Forget host key",
onSelect: (ctx: { stack: { pop(): unknown } }) => {
ctx.stack.pop();
control.config.hostKeys = [];
updateConnection(control.config.id, { hostKeys: [] });
control.sshIssue = null;
if (control.config.enabled) startControl(control);
renderHubWindows();
},
},
]
: []),
...(isSshConnection(control) && getSshSecret(sshPasswordSecretId(control.config.id))
? [
{
label: "Forget saved password",
onSelect: (ctx: { stack: { pop(): unknown } }) => {
ctx.stack.pop();
removeSshSecret(sshPasswordSecretId(control.config.id));
},
},
]
: []),
{
label: `Remove ${name}`,
onSelect: (ctx) => {
@@ -1558,6 +1970,11 @@ function openConnectionActions(window: HubWindow, control: ControlConnection): v
windowMenu(window).open(items);
}
function isSshConnection(control: ControlConnection): boolean {
const kind = clientOptionsFor(control.config)?.kind;
return kind === "tmux" || kind === "ssh-g2mirror";
}
/** Enable and (re)connect now; also the manual retry path, so reset backoff. */
function connectControl(control: ControlConnection): void {
control.reconnectDelayMs = RECONNECT_MIN_DELAY_MS;
@@ -1578,6 +1995,7 @@ function disconnectControl(control: ControlConnection): void {
function removeControl(control: ControlConnection): void {
stopControl(control);
removeSshSecret(sshPasswordSecretId(control.config.id));
controls.delete(control.config.id);
saveConnections(loadConnections().filter((connection) => connection.id !== control.config.id));
renderHubWindows();
@@ -1586,7 +2004,7 @@ function removeControl(control: ControlConnection): void {
/**
* Enter the Add-connection screen: clear the draft and ask the shell to open
* the phone text editor on it. The user types (or voice-inputs) the
* g2mirror:// string, then clicks to save or double-clicks to cancel.
* connection string, then clicks to save or double-clicks to cancel.
*/
function beginAddConnection(window: HubWindow): void {
terminalNewConnectionSetting.set("");
@@ -1596,15 +2014,15 @@ function beginAddConnection(window: HubWindow): void {
function saveAddConnection(window: HubWindow): void {
const url = terminalNewConnectionSetting.get();
if (!parseConnectionString(url)) {
if (!parseAnyConnectionString(url)) {
window.addError = url
? "Not a g2mirror://token@host connection string."
: "Enter a g2mirror:// connection string first.";
? "Not an ssh://user@host or g2mirror://token@host string."
: "Enter a connection string first.";
scheduleRender(window);
return;
}
const connections = loadConnections();
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true });
connections.push({ id: `c${Date.now()}`, url, serverName: null, enabled: true, hostKeys: [] });
saveConnections(connections);
endAddConnection(window);
setHubMode(window, "connections");
@@ -1634,6 +2052,15 @@ function handleHubInput(window: HubWindow, event: InputEvent, frameId: number):
frameTimings.finishFrame(frameId, "discarded: terminal add-connection ignored input");
return;
}
if (window.mode === "page") {
if (event.type === "click" && window.page?.onClick) {
window.page.onClick();
renderAndSubmit(window, frameId);
return;
}
frameTimings.finishFrame(frameId, "discarded: terminal page ignored input");
return;
}
const list = hubMenu(window);
list.setItems(hubItems(window));
switch (event.type) {
@@ -1800,6 +2227,7 @@ function openViewWindow(control: ControlConnection, socket: string, label: strin
focus: true,
// Default to tall sessions; the app display setting can override this.
heightMode: "max",
acceptsKeys: true,
});
}
@@ -1820,6 +2248,11 @@ function launchPresetNames(): string[] {
*/
function launchOnHost(window: HubWindow, control: ControlConnection): void {
const presets = launchPresetNames();
if (presets.length === 0) {
// A tmux host with no presets configured: a new session running the login shell.
launchFromUi(control, "shell");
return;
}
if (presets.length === 1) {
launchFromUi(control, presets[0]!);
return;
@@ -1864,6 +2297,9 @@ function paintHub(window: HubWindow): GrayImage {
if (window.mode === "add") {
return paintAddConnection(window);
}
if (window.mode === "page" && window.page) {
return paintHubPage(window, window.page);
}
const { viewportWidth: width, viewportHeight: height, padding } = window;
const image = new GrayImage(width, height, 0);
const font = chromeFont();
@@ -1878,7 +2314,7 @@ function paintHub(window: HubWindow): GrayImage {
let listTop = header.listTop;
if (window.mode === "sessions" && controls.size === 0) {
image.drawText(font, header.titleX, listTop, "Add a g2mirror:// connection to get started, see:", 150);
image.drawText(font, header.titleX, listTop, "Add an ssh:// (tmux) or g2mirror:// connection to get started, see:", 150);
image.drawText(font, header.titleX, listTop + step, "https://github.com/jimrandomh/g2mirror", 190);
listTop += 2 * step + 6;
}
@@ -1920,8 +2356,8 @@ function paintAddConnection(window: HubWindow): GrayImage {
image.drawText(font, padding.left, padding.top, "Add connection", 220);
const bodyX = padding.left + PAGE_BODY_INDENT;
const instructionsY = padding.top + 10 + step;
image.drawText(font, bodyX, instructionsY, "Type the g2mirror://token@host string in the", 170);
image.drawText(font, bodyX, instructionsY + step, "phone app (or use voice input from the menu).", 170);
image.drawText(font, bodyX, instructionsY, "Type ssh://user@host (tmux) or g2mirror://token@host in", 170);
image.drawText(font, bodyX, instructionsY + step, "the phone app (or use voice input from the menu).", 170);
const draftY = instructionsY + 3 * step;
const draft = terminalNewConnectionSetting.get();
image.drawText(font, bodyX, draftY, truncateLabel(draft || "(empty)", 52), 220);
@@ -1932,6 +2368,34 @@ function paintAddConnection(window: HubWindow): GrayImage {
return image;
}
/** A page of text: title at the top padding, wrapped body, hint on the bottom padding. */
function paintHubPage(window: HubWindow, page: HubPage): GrayImage {
const { viewportWidth: width, viewportHeight: height, padding } = window;
const image = new GrayImage(width, height, 0);
const font = chromeFont();
const step = lineStep(font);
image.drawText(font, padding.left, padding.top, page.title, 220);
const bodyX = padding.left + PAGE_BODY_INDENT;
const hintY = height - padding.bottom - font.lineHeight;
const lines = wrapText(font, page.body(), Math.max(1, width - padding.right - bodyX), { breakLongWords: true });
let y = padding.top + 10 + step;
for (const line of lines) {
if (y + step > hintY) break;
image.drawText(font, bodyX, y, line, 190);
y += step;
}
image.drawText(font, bodyX, hintY, page.hint, 110);
return image;
}
/** The action row under a host whose SSH connection needs the user. */
function sshIssueAction(issue: SshFailure): string {
if (issue.kind === "hostkey-unknown") return "Verify host key...";
if (issue.kind === "hostkey-mismatch") return "Host key changed!";
if (issue.passwordAllowed) return issue.passwordTried ? "Re-enter password..." : "Enter password...";
return "Show this device's SSH key...";
}
function truncateLabel(text: string, maxLength: number): string {
return text.length <= maxLength ? text : `${text.slice(0, Math.max(0, maxLength - 3))}...`;
}
+44
View File
@@ -0,0 +1,44 @@
import type {
G2MirrorHistoryReply,
G2MirrorPhase,
G2MirrorSession,
G2MirrorState,
} from "../../native/terminal-session-types";
/**
* What the Terminal worker needs from a session backend. G2MirrorClient (a
* g2mirror server, directly or tunnelled over SSH) and TmuxClient (tmux
* control mode over SSH) both provide it. The model is g2mirror's: a client
* starts "connecting", reaches "connected" (it can list and launch sessions,
* and reports bells/activity/titles for them), and can then attach to one
* session, view it (snapshot + live output), page its scrollback, and type
* into it.
*/
export interface TerminalClient {
state(): G2MirrorState;
onStateChange(listener: (state: G2MirrorState) => void): () => void;
/** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */
onTerminalData(listener: (data: Uint8Array, kind: "snapshot" | "output") => void): () => void;
onSessionAttached(listener: (command: string) => void): () => void;
onSessionDetached(listener: (reason: string) => void): () => void;
onBell(listener: (socket: string, lastBellAtMs: number) => void): () => void;
onActivity(listener: (socket: string, lastOutputAtMs: number) => void): () => void;
onTitle(listener: (socket: string, title: string) => void): () => void;
/** Fired after a snapshot's terminal data, with its scrollback splice point. */
onSnapshot(listener: (historyNext: number, historyOldest: number) => void): () => void;
onHistoryLines(listener: (reply: G2MirrorHistoryReply) => void): () => void;
setViewport(cols: number, rows: number): void;
start(): void;
stop(): void;
listSessions(): void;
launchSession(preset: string): Promise<string>;
connectSession(socket: string): void;
disconnectSession(): void;
view(): void;
unview(): void;
requestHistory(before: number, limit?: number): void;
sendInput(text: string): void;
submitInput(text: string): void;
}
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
+5
View File
@@ -82,4 +82,9 @@ export class TerminalEmulator {
cursorCol(): number {
return this.term.buffer.active.cursorX;
}
/** DECCKM: the session asked for application cursor keys (ESC O A rather than ESC [ A). */
applicationCursorKeys(): boolean {
return this.term.modes.applicationCursorKeysMode;
}
}
+118
View File
@@ -0,0 +1,118 @@
/**
* What a terminal emulator sends for a key from a hardware keyboard, by the
* classic xterm conventions: Ctrl chords become control characters, Alt
* prefixes ESC, and modified cursor and function keys carry the xterm
* modifier parameter (1 + shift + 2*alt + 4*ctrl). Follows the Glassly
* Terminal miniapp's encoder (terminal/keys.ts, MIT).
* No platform imports: this runs under node in the tests.
*/
import type { KeyEvent } from "../../g2/keyboard/keys";
const ESC = "\x1b";
/** Keys xterm encodes as `ESC [ <n> ~`, with a modifier as `;m`. */
const TILDE_KEYS: Record<string, number> = {
Insert: 2,
Delete: 3,
PageUp: 5,
PageDown: 6,
F5: 15,
F6: 17,
F7: 18,
F8: 19,
F9: 20,
F10: 21,
F11: 23,
F12: 24,
};
/** Keys with a single-letter final byte: SS3 in application cursor mode, CSI otherwise. */
const LETTER_KEYS: Record<string, string> = {
ArrowUp: "A",
ArrowDown: "B",
ArrowRight: "C",
ArrowLeft: "D",
Home: "H",
End: "F",
};
/** F1-F4 are always SS3 (`ESC O P`), or CSI with a modifier. */
const SS3_KEYS: Record<string, string> = { F1: "P", F2: "Q", F3: "R", F4: "S" };
/** Ctrl with a punctuation or digit key, per xterm (^[ ^\ ^] ^^ ^_ ^? ^@). */
const CONTROL_PUNCTUATION: Record<string, string> = {
"[": "\x1b",
"\\": "\x1c",
"]": "\x1d",
"^": "\x1e",
"6": "\x1e",
"_": "\x1f",
"-": "\x1f",
"?": "\x7f",
"/": "\x7f",
"8": "\x7f",
"@": "\x00",
"2": "\x00",
};
/**
* The text to write to the terminal for one key event, or null when the key
* sends nothing: releases, bare modifiers, Meta chords, and keys a terminal
* has no bytes for (volume, media). `appCursor` is DECCKM, which the emulator
* tracks from the session's output.
*/
export function encodeTerminalKey(event: KeyEvent, appCursor: boolean): string | null {
if (event.phase !== "down") return null;
const { key, modifiers } = event;
if (modifiers.meta) return null;
const modifierParam = 1 + (modifiers.shift ? 1 : 0) + (modifiers.alt ? 2 : 0) + (modifiers.ctrl ? 4 : 0);
const modified = (prefix: string, final: string): string | null =>
modifierParam > 1 ? `${ESC}[${prefix};${modifierParam}${final}` : null;
switch (key) {
case "Enter":
return modifiers.alt ? `${ESC}\r` : "\r";
case "Escape":
return ESC;
case "Tab":
return modifiers.shift ? `${ESC}[Z` : "\t";
case "Backspace":
if (modifiers.alt) return `${ESC}\x7f`;
return modifiers.ctrl ? "\x08" : "\x7f";
case " ":
return modifiers.ctrl ? "\x00" : modifiers.alt ? `${ESC} ` : " ";
}
const letter = LETTER_KEYS[key];
if (letter) return modified("1", letter) ?? (appCursor ? `${ESC}O${letter}` : `${ESC}[${letter}`);
const tilde = TILDE_KEYS[key];
if (tilde) return modified(String(tilde), "~") ?? `${ESC}[${tilde}~`;
const ss3 = SS3_KEYS[key];
if (ss3) return modified("1", ss3) ?? `${ESC}O${ss3}`;
if (modifiers.ctrl) {
// From the unshifted key: ^A..^Z, then the punctuation set.
const base = key.length === 1 ? key.toLowerCase() : "";
let char: string | undefined;
if (base >= "a" && base <= "z") char = String.fromCharCode(base.charCodeAt(0) - 96);
else char = CONTROL_PUNCTUATION[key];
if (char === undefined) return null;
return modifiers.alt ? `${ESC}${char}` : char;
}
// With Alt held the translator leaves `text` out (it is a chord, not
// typing); a terminal still wants ESC and the character.
const text = event.text ?? (modifiers.alt && key.length === 1 ? key : undefined);
if (text === undefined) return null;
return modifiers.alt ? `${ESC}${text}` : text;
}
/** Shift+PageUp / Shift+PageDown page the emulator's own scrollback, as in xterm. */
export function terminalScrollbackKey(event: KeyEvent): -1 | 1 | null {
if (event.phase !== "down") return null;
const { shift, ctrl, alt, meta } = event.modifiers;
if (!shift || ctrl || alt || meta) return null;
if (event.key === "PageUp") return -1;
if (event.key === "PageDown") return 1;
return null;
}
File diff suppressed because it is too large. Load diff
+29 -2
View File
@@ -1,5 +1,32 @@
import { type AppDefinition } from "../app-definition";
import { createTranscribeAppWindow, TRANSCRIBE_SURFACE_ID, TRANSCRIBE_WINDOW_ID } from "./transcribe-app";
import { type AppContext, type AppDefinition } from "../app-definition";
import {
createTranscribeAppWindow,
createTranscribeReplayWindow,
TRANSCRIBE_SURFACE_ID,
TRANSCRIBE_WINDOW_ID,
} from "./transcribe-app";
// Replay windows are closeable and non-singleton; the serial keeps their windowIds unique.
let nextReplaySerial = 1;
/**
* Open a recording (.mka) in Transcribe's replay mode, as a window of its
* own. Returns false where recordings can't be replayed.
*/
export function openRecordingInTranscribe(ctx: AppContext, path: string, name: string): boolean {
if (!ctx.actions.openVoiceReplay) return false;
const windowId = `transcribe:replay:${nextReplaySerial++}`;
void ctx
.launchInProcessApp(windowId, `window:${windowId}`, (options) => {
const window = createTranscribeReplayWindow(windowId, path, name, options);
if (!window) throw new Error("recordings can't be replayed here");
return window;
})
.catch((error) => {
ctx.appendLog(`transcribe replay window failed: ${error}`);
});
return true;
}
const transcribeApp: AppDefinition = {
appId: "transcribe",
+23
View File
@@ -0,0 +1,23 @@
/** A position in a recording as m:ss, or h:mm:ss from an hour on (or when `withHours`). */
export function formatReplayTime(ms: number, withHours = false): string {
const total = Math.max(0, Math.floor(ms / 1000));
const hours = Math.floor(total / 3600);
const minutes = Math.floor((total % 3600) / 60);
const seconds = String(total % 60).padStart(2, "0");
if (hours === 0 && !withHours) return `${minutes}:${seconds}`;
return `${hours}:${String(minutes).padStart(2, "0")}:${seconds}`;
}
/** The time picker's fields, in display and click order. */
export const TIME_FIELDS = ["hours", "minutes", "seconds"] as const;
export type TimeField = (typeof TIME_FIELDS)[number];
const FIELD_SECONDS: Record<TimeField, number> = { hours: 3600, minutes: 60, seconds: 1 };
/**
* Step a position (whole seconds) by `delta` of a field's unit, carrying
* into the neighboring fields, within [0, maxSeconds].
*/
export function stepReplaySeconds(seconds: number, field: TimeField, delta: number, maxSeconds: number): number {
return Math.max(0, Math.min(maxSeconds, seconds + delta * FIELD_SECONDS[field]));
}
+147
View File
@@ -0,0 +1,147 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, gestureHints, type InputEvent } from "../../ui/gestures";
import type { Layer, LayerContext, PaintBelow } from "../../ui/layers";
import { CONTEXT_MENU_DIM } from "../../ui/menu";
import { centeredTextY } from "../../ui/metrics";
import { formatReplayTime, stepReplaySeconds, TIME_FIELDS, type TimeField } from "./replay-time";
const PADDING = 12;
const FIELD_GAP = 6;
const FIELD_PAD_X = 6;
/** Fill and outline of the active field (the Timers and date pickers' field style). */
const ACTIVE_FILL = 35;
const ACTIVE_STROKE = 130;
export type TimePickerOptions = {
title: string;
/** Where the picker starts, and the furthest it goes, in ms. */
initialMs: number;
maxMs: number;
onPick: (positionMs: number) => void;
};
/**
* A modal position picker for a recording: hours (when the recording has
* any), minutes and seconds, over the dimmed content beneath. Input follows
* the date picker's: scroll adjusts the active field (up is later, carrying
* into the fields beside it), click moves to the next field and on the last
* one picks, double-click steps back a field and on the first one cancels.
* A watch's swipes move between fields and adjust.
*/
export class TimePickerLayer implements Layer {
readonly acceptsDirectional = true;
readonly dimUnderneath = CONTEXT_MENU_DIM;
private readonly fields: readonly TimeField[];
private readonly maxSeconds: number;
private seconds: number;
private field = 0;
constructor(private readonly options: TimePickerOptions) {
this.maxSeconds = Math.max(0, Math.floor(options.maxMs / 1000));
this.seconds = Math.max(0, Math.min(this.maxSeconds, Math.floor(options.initialMs / 1000)));
this.fields = this.maxSeconds >= 3600 ? TIME_FIELDS : TIME_FIELDS.slice(1);
}
handleInput(event: InputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
case "swipe-up":
this.adjust(1);
return;
case "scroll-down":
case "swipe-down":
this.adjust(-1);
return;
case "swipe-right":
this.field = Math.min(this.fields.length - 1, this.field + 1);
return;
case "swipe-left":
this.field = Math.max(0, this.field - 1);
return;
case "click":
if (this.field < this.fields.length - 1) {
this.field++;
} else {
ctx.stack.pop();
this.options.onPick(this.seconds * 1000);
}
return;
case "double-click":
if (this.field > 0) this.field--;
else ctx.stack.pop();
return;
default:
return;
}
}
private adjust(delta: number): void {
this.seconds = stepReplaySeconds(this.seconds, this.fields[this.field]!, delta, this.maxSeconds);
}
private hints(field: number): string {
return gestureHints([
[GESTURE_SCROLL, "adjust"],
[GESTURE_CLICK, field < this.fields.length - 1 ? "next" : "jump"],
[GESTURE_DOUBLE_CLICK, field > 0 ? "back" : "cancel"],
]);
}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
const image = paintBelow();
const small = getDefaultSmallFont();
const medium = getDefaultMediumFont();
const focused = ctx.stack.isFocused();
const base = ctx.stack.getBaseSize();
const values = [
String(Math.floor(this.seconds / 3600)),
String(Math.floor((this.seconds % 3600) / 60)).padStart(this.fields.length === 3 ? 2 : 1, "0"),
String(this.seconds % 60).padStart(2, "0"),
].slice(TIME_FIELDS.length - this.fields.length);
// Sized for the widest value a field can show, so the dialog holds still while the time changes.
const fieldWidths = this.fields.map(
(field) => Math.max(medium.measureText("000"), small.measureText(field)) + 2 * FIELD_PAD_X,
);
const fieldsWidth = fieldWidths.reduce((sum, width) => sum + width, 0) + FIELD_GAP * (fieldWidths.length - 1);
const boxHeight = medium.lineHeight + 6;
const length = `of ${formatReplayTime(this.maxSeconds * 1000)}`;
const hintsWidth = Math.max(...this.fields.map((_, index) => small.measureText(this.hints(index))));
const width = Math.min(
base.width,
Math.max(fieldsWidth, hintsWidth, small.measureText(this.options.title), small.measureText(length)) + 2 * PADDING,
);
const titleHeight = small.lineHeight + 12;
const bodyHeight = boxHeight + 2 + small.lineHeight + 10 + small.lineHeight;
const height = Math.min(base.height, 8 + titleHeight + bodyHeight + 12 + small.lineHeight + 10);
const x = Math.max(0, Math.floor((base.width - width) / 2));
const y = Math.max(0, Math.floor((base.height - height) / 2));
// Fill 1, not 0: 0 is the transparent color key on the shell surface.
image.fillRoundedRect(x, y, width, height, 1);
image.drawRoundedRect(x, y, width, height, 72);
image.drawText(small, x + PADDING, y + 8, this.options.title, 220);
const top = y + 8 + titleHeight;
let fieldX = x + PADDING;
for (let index = 0; index < this.fields.length; index++) {
const fieldWidth = fieldWidths[index]!;
const active = index === this.field;
if (active) {
if (focused) image.fillRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_FILL, 6);
image.drawRoundedRect(fieldX, top, fieldWidth, boxHeight, ACTIVE_STROKE, 6);
}
const value = values[index]!;
image.drawText(medium, fieldX + Math.floor((fieldWidth - medium.measureText(value)) / 2),
centeredTextY(medium, top, boxHeight), value, active ? 250 : 170);
const caption = this.fields[index]!;
image.drawText(small, fieldX + Math.floor((fieldWidth - small.measureText(caption)) / 2),
top + boxHeight + 2, caption, active ? 190 : 110);
fieldX += fieldWidth + FIELD_GAP;
}
image.drawText(small, x + PADDING, top + bodyHeight - small.lineHeight, length, 190);
image.drawText(small, x + PADDING, y + height - 10 - small.lineHeight, this.hints(this.field), 110);
return image;
}
}
+101 -4
View File
@@ -1,8 +1,11 @@
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
import { voiceControlBridge } from "../../native/voice-control";
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { type ContinuousCaptureOptions } from "../../ui/layers";
import { withRecordingIcon } from "../../ui/listening-header";
import { TimePickerLayer } from "./time-picker";
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
import { transcribeMicCount, transcribeMicsSetting } from "./transcribe-settings";
import {
createInProcessWindow,
YieldAtRootLayer,
@@ -45,8 +48,8 @@ export function micTrayIcon(recording: boolean): GrayImage {
}
export type TranscribeAppOptions = InProcessAppOptions & {
/** Begin continuous mic capture (kept on while the window is open). */
startContinuousVoiceCapture: (options?: { recordAs?: string }) => void;
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
startContinuousVoiceCapture: (options?: ContinuousCaptureOptions) => void;
stopContinuousVoiceCapture: () => void;
};
@@ -59,6 +62,10 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
// One timestamp per session names both the audio recording and a saved transcript.
const stamp = transcriptTimestamp();
const layer = new TranscribeLayer(stamp);
// At launch, and again when the Microphones setting changes: a running
// capture moves over to the microphones now asked for.
const startCapture = () =>
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}`, mics: transcribeMicCount() });
const app = createInProcessWindow({
appId: "transcribe",
windowId: TRANSCRIBE_WINDOW_ID,
@@ -78,6 +85,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
},
// The controller starts and stops the recording as the setting changes.
toggleSettingMenuItem(saveRecordingSetting),
enumSettingMenuItem(transcribeMicsSetting, { onChange: startCapture }),
],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
@@ -97,7 +105,96 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
app.requestRender();
});
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
startCapture();
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
return app;
}
/**
* Transcribe in replay mode: a window of its own that transcribes a
* recording (a .mka file from Save Recording) in place of the microphones,
* in real time and through the same pipeline. It neither uses the mic nor
* shows the mic tray icon, so it runs beside a live Transcribe window. The
* header shows the position in the recording; a tap pauses and resumes; the
* menu jumps to a time.
*/
export function createTranscribeReplayWindow(
windowId: string,
path: string,
name: string,
options: InProcessAppOptions,
): InProcessWindow | null {
const replay = options.actions.openVoiceReplay?.(path) ?? null;
if (!replay) return null;
// Names a saved transcript after the recording it came from.
const layer = new TranscribeLayer(name.replace(/\.[^.]+$/, ""), replay, name);
const app = createInProcessWindow({
appId: "transcribe",
windowId,
title: name,
iconLetter: "Tr",
icon: "mic",
closeable: true,
// Relaunching Transcribe after a restart would open the microphone, not this recording.
restoresApp: false,
// Like a playing video: something to watch, not to touch.
keepsScreenOn: () => replay.state === "playing",
actions: options.actions,
baseLayer: new YieldAtRootLayer(layer),
menuItems: () => [
{
label: "Jump to time",
// The length isn't known until the recording has been opened.
disabled: () => replay.durationMs <= 0,
onSelect: (ctx) => {
ctx.stack.pop();
ctx.stack.push(
new TimePickerLayer({
title: "Jump to time",
initialMs: replay.positionMs,
maxMs: replay.durationMs,
onPick: (positionMs) => {
replay.jumpTo(positionMs);
app.requestRender();
},
}),
);
},
},
{
label: replay.state === "paused" ? "Resume" : "Pause",
disabled: () => replay.state !== "playing" && replay.state !== "paused",
onSelect: (ctx) => {
ctx.stack.pop();
replay.setPaused(replay.state === "playing");
},
},
{
label: "Restart from 0:00",
onSelect: (ctx) => {
ctx.stack.pop();
replay.jumpTo(0);
},
},
{
label: "Save",
onSelect: (ctx) => {
ctx.stack.pop();
layer.saveTranscript();
},
},
],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
layer.onRemoved();
replay.close();
options.onClosed();
},
});
// Started first, so the layer picks its opening status up as it subscribes.
replay.start();
layer.start(app.requestRender, () => shell.isWindowVisible(windowId));
return app;
}
@@ -0,0 +1,40 @@
import { ConfigSettingEnum, glassesTypeSetting } from "../../ui/dashboard-settings";
import { type MicCount } from "../../ui/layers";
import { micControlSupported } from "../microphones/mic-control";
export type TranscribeMics = "1" | "2" | "4";
const MICS_LABELS: Record<TranscribeMics, string> = {
"1": "1 (mixed)",
"2": "2 (one per temple)",
"4": "4 (two per temple)",
};
/** Whether the paired glasses can stream their microphones separately (G2, with the microphone channel in use). */
export function separateMicsSupported(): boolean {
return glassesTypeSetting.get() === "g2" && micControlSupported();
}
/**
* How many of the glasses' microphones Transcribe captures. With more than
* one, the temples stream their own microphones (each temple on its own
* clock) instead of the glasses' mixed stream; a recording keeps every one,
* and what is transcribed is a mix made on the phone.
*/
export const transcribeMicsSetting = new ConfigSettingEnum<TranscribeMics>({
id: "transcribe-mics",
label: "Microphones",
storageKey: "transcribe.mics",
defaultValue: "1",
values: ["1", "2", "4"],
formatValue: (value) => MICS_LABELS[value],
isDisabled: (value) => value !== "1" && !separateMicsSupported(),
description:
"Which of the glasses' microphones to capture. 1 is the glasses' own mixed stream. 2 and 4 stream each temple's microphones separately (custom firmware), so a saved recording has every microphone on its own channel; the transcript then comes from a mix made on the phone.",
});
/** The setting as a capture option; 1 where the glasses can't stream their microphones separately. */
export function transcribeMicCount(): MicCount {
if (!separateMicsSupported()) return 1;
return Number(transcribeMicsSetting.get()) as MicCount;
}
+72 -13
View File
@@ -3,15 +3,20 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { voiceControlBridge, type VoiceEventSource, type VoiceReplay } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
import { type InputEvent } from "../../ui/gestures";
import { formatReplayTime } from "./replay-time";
import { TranscriptModel, TranscriptScroll, transcriptLayout, wrapTranscribeText } from "./transcript-model";
/**
* Live transcription view. Continuous mic capture is owned by the app wrapper
* (createTranscribeAppWindow); this layer accumulates the transcript, shows
* it, and saves it to Downloads from the app menu.
*
* Given a replay, it transcribes that recording instead of the microphone:
* the header shows the position in the recording, a tap pauses and resumes,
* and each jump leaves its time as a line in the transcript.
*/
export class TranscribeLayer implements Layer {
private status = "Starting microphone...";
@@ -24,25 +29,67 @@ export class TranscribeLayer implements Layer {
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribePause: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
private unsubscribeReplay: (() => void) | null = null;
private readonly source: VoiceEventSource;
/** `stamp` names the saved transcript, matching the session's audio recording. */
constructor(private readonly stamp: string = transcriptTimestamp()) {}
/**
* `stamp` names the saved transcript, matching the session's audio
* recording. `replay` is the recording to transcribe in place of the
* microphone; `replayName` is what to call it.
*/
constructor(
private readonly stamp: string = transcriptTimestamp(),
private readonly replay: VoiceReplay | null = null,
private readonly replayName = "",
) {
this.source = replay ? replay.events : voiceControlBridge;
if (replay) this.status = "Opening the recording...";
}
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
/** `isVisible` says whether the window is on screen (the level meter and replay clock repaint only then). */
start(requestRender: () => void, isVisible: () => boolean): void {
this.meter.start(requestRender, isVisible);
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
this.meter.start(requestRender, isVisible, this.source);
this.unsubscribeTranscript = this.source.onTranscript((event) => {
this.transcript.accept(event);
requestRender();
});
this.unsubscribePause = voiceControlBridge.onSpeechPause(() => this.transcript.pause());
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.unsubscribePause = this.source.onSpeechPause(() => this.transcript.pause());
// The current status arrives while subscribing, before the window has a
// surface to render to; the window paints it when it is first shown.
let subscribing = true;
this.unsubscribeStatus = this.source.onStatus((state) => {
this.status = state.status;
this.listening = state.listening;
this.detail = state.detail;
this.saveNotice = "";
requestRender();
if (!subscribing) requestRender();
});
subscribing = false;
const replay = this.replay;
if (replay) {
let shownSecond = -1;
let shownState = replay.state;
this.unsubscribeReplay = replay.onChange((jumpedToMs) => {
if (jumpedToMs !== undefined) this.transcript.mark(`[${formatReplayTime(jumpedToMs)}]`);
// The clock ticks in whole seconds; repaint when what it shows changed.
const second = Math.floor(replay.positionMs / 1000);
if (jumpedToMs === undefined && second === shownSecond && replay.state === shownState) return;
shownSecond = second;
shownState = replay.state;
if (isVisible()) requestRender();
});
}
}
/** The header's status for a replay: where it is in the recording, and anything worth saying beside that. */
private replayStatus(replay: VoiceReplay): string {
if (replay.state === "failed" || (replay.state === "opening" && replay.durationMs === 0)) return this.status;
const withHours = replay.durationMs >= 3_600_000;
const clock = `${formatReplayTime(replay.positionMs, withHours)} / ${formatReplayTime(replay.durationMs, withHours)}`;
if (replay.state === "paused") return `${clock} · Paused`;
if (replay.state === "ended") return `${clock} · Finished`;
// Anything but plain listening is worth showing: the model loading, a provider reconnecting.
return this.listening ? clock : `${clock} · ${this.status}`;
}
paint(ctx: LayerContext): GrayImage {
@@ -50,14 +97,19 @@ export class TranscribeLayer implements Layer {
const viewport = ctx.stack.getBaseSize();
const image = new GrayImage(viewport.width, viewport.height, 0);
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
const replay = this.replay;
const status = replay
? this.replayStatus(replay)
: this.source.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
const playing = !replay || replay.state === "playing";
const bodyTop = paintListeningHeader(
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
voiceControlBridge.recording,
image, font, "Transcribe", this.saveNotice || status, this.listening && playing ? this.meter : null, viewport.padding,
this.source.recording,
);
const step = lineStep(font) + 2;
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
const placeholder = replay ? `Replaying ${this.replayName}` : "Listening...";
const text = this.transcript.text || (this.listening ? placeholder : this.detail);
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, layout.textWidth);
const firstLine = this.scroll.layout(wrapped.length, layout.pageLines);
@@ -82,6 +134,11 @@ export class TranscribeLayer implements Layer {
ctx.actions.requestRender();
return;
}
if (event.type === "click" && this.replay) {
this.replay.setPaused(this.replay.state === "playing");
ctx.actions.requestRender();
return;
}
if (event.type === "double-click") {
ctx.stack.pop();
}
@@ -89,6 +146,8 @@ export class TranscribeLayer implements Layer {
onRemoved(): void {
this.meter.stop();
this.unsubscribeReplay?.();
this.unsubscribeReplay = null;
this.unsubscribePause?.();
this.unsubscribePause = null;
this.unsubscribeTranscript?.();
+12
View File
@@ -36,6 +36,18 @@ export class TranscriptModel {
if (this.live) this.breakAfterLive = true;
else if (this.finalized) this.separator = "\n";
}
/**
* A line of its own (where a replay jumped to), ending whatever was in
* progress: what follows it starts a new paragraph.
*/
mark(label: string): void {
const before = this.text;
this.finalized = before ? `${before}\n${label}` : label;
this.live = "";
this.separator = "\n";
this.breakAfterLive = false;
}
}
/** An absolute history position, with an explicit follow-tail mode. */
+62
View File
@@ -0,0 +1,62 @@
/**
* Date and calendar-event text for the assistant's tool results. Pure (no
* native imports) so it can be unit-tested.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
export type FormattableEvent = {
title: string;
startMs: number;
endMs: number;
allDay: boolean;
location?: string;
};
export function formatEvent(event: FormattableEvent): string {
const location = event.location ? ` @ ${event.location}` : "";
return `- ${event.title || "(untitled)"} — ${formatEventSpan(event)}${location}`;
}
/**
* When an event happens, end included: a multi-day all-day event names its last
* day (all-day ends are exclusive, at the midnight after it), a timed event its
* end time. Without the end, an event that started yesterday and runs through
* tomorrow reads as yesterday's.
*/
export function formatEventSpan(event: Pick<FormattableEvent, "startMs" | "endMs" | "allDay">): string {
const start = new Date(event.startMs);
if (event.allDay) {
const lastDay = new Date(Math.max(event.startMs, event.endMs - 1));
return sameDay(start, lastDay)
? `${formatDate(start)} (all day)`
: `${formatDate(start)} – ${formatDate(lastDay)} (all day)`;
}
if (!(event.endMs > event.startMs)) return formatDateTime(start);
const end = new Date(event.endMs);
return sameDay(start, end)
? `${formatDateTime(start)} – ${formatTime(end)}`
: `${formatDateTime(start)} – ${formatDateTime(end)}`;
}
export function formatDateTime(date: Date): string {
return `${formatDate(date)}, ${formatTime(date)}`;
}
export function formatDate(date: Date): string {
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
}
export function formatTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
return `${hours}:${minutes} ${meridiem}`;
}
function sameDay(a: Date, b: Date): boolean {
return a.getFullYear() === b.getFullYear() && a.getMonth() === b.getMonth() && a.getDate() === b.getDate();
}
+2 -27
View File
@@ -1,10 +1,11 @@
import { readUpcomingEventsAsync, type CalendarEvent } from "../native/calendar";
import { readUpcomingEventsAsync } from "../native/calendar";
import { hasCalendarPermission } from "../native/calendar-permissions";
import { mediaControllerBridge } from "../native/media-controller";
import { spotifyService } from "../native/spotify";
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
import { shell } from "../ui/shell/shell";
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
import { formatDateTime, formatEvent } from "./date-format";
/**
* Registers the always-available system tools into the registry. Called once at
@@ -12,9 +13,6 @@ import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registr
* media, notifications); nothing here needs an app window to be open.
*/
const WEEKDAYS = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
let registered = false;
export function registerSystemTools(registry: ToolRegistry = toolRegistry): void {
@@ -244,35 +242,12 @@ function describeNowPlaying(): string {
return "Nothing is currently playing.";
}
function formatEvent(event: CalendarEvent): string {
const when = event.allDay ? `${formatDate(new Date(event.startMs))} (all day)` : formatDateTime(new Date(event.startMs));
const location = event.location ? ` @ ${event.location}` : "";
return `- ${event.title || "(untitled)"} — ${when}${location}`;
}
function formatNotification(notification: { key: string; appName: string; title: string; text: string; customText?: string[] }): string {
const title = notification.title || notification.customText?.join(" ") || "(no title)";
const body = notification.text ? `: ${notification.text}` : "";
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
}
function formatDateTime(date: Date): string {
return `${formatDate(date)}, ${formatTime(date)}`;
}
function formatDate(date: Date): string {
return `${WEEKDAYS[date.getDay()]} ${MONTHS[date.getMonth()]} ${date.getDate()}`;
}
function formatTime(date: Date): string {
let hours = date.getHours();
const minutes = date.getMinutes().toString().padStart(2, "0");
const meridiem = hours >= 12 ? "PM" : "AM";
hours = hours % 12;
if (hours === 0) hours = 12;
return `${hours}:${minutes} ${meridiem}`;
}
function clampNumber(value: unknown, min: number, max: number, fallback: number): number {
const num = typeof value === "number" ? value : Number(value);
if (!Number.isFinite(num)) return fallback;
+133 -7
View File
@@ -19,9 +19,14 @@ import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, onAndroidNotificationRe
import { watchNotificationListenerBinding } from "../native/notification-access";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { defaultRingName, setGlassesRingLink } from "../native/ring-status";
import { applyGlanceAngle, glassesHeadUpTrigger, syncGlanceAngleOriginal } from "./head-angle";
import { glanceAngleTriggerDegrees, glanceTapTimeoutMs } from "../apps/glanceboard/glanceboard-settings";
import { GLANCE_TILT_MAX_MS, glanceTiltMaxMs } from "./glance-state";
import { getBooleanSetting, setBooleanSetting } from "../native/settings-store";
import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from "./firmware-compat";
import { keyboardService } from "./keyboard/service";
import type { KeyEvent } from "./keyboard/keys";
import { hasExtractedEvenHubFonts } from "./firmware-builder";
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy";
import { WearRemote, type WearRemoteInputKind } from "./wear-remote";
@@ -53,7 +58,7 @@ import { type GlanceEvent } from "./glance-state";
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress";
import { beginRenderPass, endRenderPass } from "../util/render-freshness";
import { voiceControlBridge } from "../native/voice-control";
import { VoiceReplay, voiceControlBridge } from "../native/voice-control";
import { voiceActivity } from "../ui/shell/voice-activity";
import { GrayImage } from "../graphics/image";
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
@@ -76,7 +81,7 @@ import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-p
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, windowEdges, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { sameEdges } from "../ui/viewport";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type ContinuousCaptureOptions, type LayerActions, type MicCount, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
@@ -98,6 +103,8 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W
import { isInstalledPackagePresent } from "../apps/evenhub/updates";
import { wearerVerificationOptions } from "../apps/microphones/speakers";
import { micSession } from "../apps/microphones/mic-session";
import { micArrayCaptureConfig, micControlSupported } from "../apps/microphones/mic-control";
import { encodeMicControl } from "../apps/microphones/mic-protocol";
import {
chargingPreviewMessage,
glassesDisplayAsleep,
@@ -333,6 +340,7 @@ class DashboardController {
private evenHubResumePromise: Promise<boolean> | null = null;
/** Faceclaw's firmware at the required revision reported in; its extensions can be used. */
private customFirmwareConfirmed = false;
private lastGlanceAngleDegrees = glanceAngleTriggerDegrees();
private faceclawWakeLeaseState: boolean | null = null;
/** Ring connection mode the current session was started with (it applies per connection). */
private ringModeAtConnect: RingConnectionMode = "glasses";
@@ -410,6 +418,7 @@ class DashboardController {
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) =>
this.startContinuousVoiceCapture(client, options),
stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client),
openVoiceReplay: (path: string) => this.openVoiceReplay(path),
playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload),
};
this.sharedActions = sharedActions;
@@ -457,6 +466,7 @@ class DashboardController {
if (on) this.glance.dismiss();
this.handleScreenStateChanged(on);
if (on) this.requestShellRender();
if (on) keyboardService.retryNow();
this.emit();
},
});
@@ -488,6 +498,7 @@ class DashboardController {
if (!resized && !appsResized) this.applyWindowEdgesIfChanged();
this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged();
this.applyGlanceAngleIfChanged();
this.syncContinuousRecording();
// A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on
@@ -509,6 +520,29 @@ class DashboardController {
assistantAvailable: () => shell.isAssistantAvailable(),
assistant: text => shell.sendToAssistant(text),
});
// A keyboard paired through the glasses (CFW keyboard bridge). The
// service decides what each key means (app/g2/keyboard/key-router.ts);
// this is how it reaches the display and the input pipeline.
keyboardService.setHost({
ready: () => this.phase === "connected",
locked: () => this.glassesLocked,
wake: () => {
void this.handleInputEvent({
kind: "display-wake",
containerName: "",
eventType: OsEventTypeList.DOUBLE_CLICK_EVENT,
eventSource: EventSourceType.TOUCH_EVENT_FORM_DUMMY_NULL,
systemExitReasonCode: 0,
frameId: 0,
}).catch((error) => this.appendLog(`keyboard wake failed: ${this.formatError(error)}`));
},
gesture: (gesture, origin) => this.injectSyntheticRingInput(gesture, origin),
log: (line) => this.appendLog(line),
// A keyboard that is switched off would otherwise keep the glasses'
// radio looking for it around the clock. One still connected when the
// display sleeps stays connected, and its next key wakes the display.
shouldReconnect: () => this.phase === "connected" && shell.isScreenOn(),
});
this.wearRemote = new WearRemote({
apps: launchableApps,
injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"),
@@ -1655,6 +1689,8 @@ class DashboardController {
if (mappedPhase === "connected") {
this.syncEvenHubScreenOffSetting();
this.ensureWearStateTracking();
// The session reapplies it on reconnects; restate it for a new session.
applyGlanceAngle();
this.maybeWakeAfterChargingReconnect();
} else if (mappedPhase !== "charging") {
this.cancelEvenHubSuspendTimer();
@@ -1763,6 +1799,7 @@ class DashboardController {
this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`);
});
this.ensureWearStateTracking();
this.syncKeyboardLink();
}
// The shared session halts on incompatible firmware by itself
// ("incompatible-firmware" phase); this only explains why.
@@ -1816,6 +1853,7 @@ class DashboardController {
this.requestShellRender();
this.updateCompositePreview();
this.updateConnectedForegroundNotification();
syncGlanceAngleOriginal();
}, SHELL_REFRESH_INTERVAL_MS);
this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS);
this.screenTimeoutTimer = setInterval(() => {
@@ -2253,6 +2291,30 @@ class DashboardController {
return undefined;
}
/**
* How many of the glasses' microphones the continuous-capture windows call
* for (the most any asks for), where the glasses can stream them
* separately: G2 with the microphone channel in use. Otherwise 1, the
* mixed stream.
*/
private continuousMics(): MicCount {
if (glassesTypeSetting.get() !== "g2" || !micControlSupported()) return 1;
let mics: MicCount = 1;
for (const options of this.continuousClients.values()) {
if ((options.mics ?? 1) > mics) mics = options.mics!;
}
return mics;
}
/**
* A recording as a capture of its own (see VoiceReplay), transcribed with
* the provider and processing settings a microphone capture would get.
* "My voice only" is left out: a recording is of whoever spoke.
*/
private openVoiceReplay(path: string): VoiceReplay {
return new VoiceReplay(path, () => ({ ...this.voiceCaptureOptions(null), speakerVerification: undefined }));
}
/**
* Continuous capture's claim on the mic (the Microphones app and EvenHub
* mic apps stand down while it is held) follows whether the capture really
@@ -2278,9 +2340,26 @@ class DashboardController {
// Otherwise only Whisper server is told a language (its own setting).
const choice = this.transcriptionChoice();
const provider = choice?.provider ?? voiceProviderSetting.get();
// Separate microphones only come from connected glasses.
const mics = communicator ? this.continuousMics() : 1;
const micConfig = mics === 1 ? null : micArrayCaptureConfig(mics);
return {
communicator: communicator?.getNativeCommunicator() ?? null,
usePhoneMic: communicator === null,
micArray: micConfig ? { configure: encodeMicControl("configure", micConfig) } : null,
// Stored in a recording of the capture (see Save Recording).
captureInfo: {
glasses: communicator ? glassesModelName() : null,
mics,
micConfig: micConfig && {
source: micConfig.source,
channelMask: micConfig.channelMask,
codec: micConfig.codec,
format: micConfig.format,
sampleRateHz: micConfig.sampleRateHz,
directionDetection: micConfig.beamform,
},
},
provider,
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
@@ -2463,6 +2542,16 @@ class DashboardController {
frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventLabel(event.kind, event.eventType)}`);
let frameOwned = false;
try {
if (event.kind === "sys-event" && event.eventType === OsEventTypeList.HEAD_DOWN_EVENT) {
// Only the Glanceboard cares (it ends a head-tilt showing); never
// shell input, so it can't count as activity or wake anything.
const glanceEvent = this.glance.eventForGesture("head-down");
if (glanceEvent) {
await this.glance.handleEvent(glanceEvent, frameId);
frameOwned = true;
}
return;
}
const inputEvent = rawInputEventToInputEvent(event);
if (!acceptInput(inputEvent)) return;
// The gesture, plus which app is on screen and whether input goes to it,
@@ -2581,15 +2670,49 @@ class DashboardController {
}
}
/** Re-send the head-tilt angle when it changed (the setting, from either settings UI, or the Glanceboard toggle). */
private applyGlanceAngleIfChanged(): void {
const degrees = glanceAngleTriggerDegrees();
if (degrees === this.lastGlanceAngleDegrees) return;
this.lastGlanceAngleDegrees = degrees;
applyGlanceAngle();
}
/**
* Tell the keyboard bridge whether the session can host a keyboard: a G2
* on firmware with the bridge, connected and in use. While the glasses
* charge in their case the link stays up but nobody is typing, so the
* firmware is told to let the keyboard go.
*/
private syncKeyboardLink(): void {
const up =
this.phase === "connected" &&
this.customFirmwareConfirmed &&
glassesTypeSetting.get() === "g2";
// Charging, or a disconnect the user asked for: the link is still up, so
// the DISABLE gets out ahead of everything the teardown sends.
if (up) keyboardService.linkReady();
else keyboardService.linkLost(this.phase === "charging" || this.phase === "disconnecting");
}
/**
* The Glanceboard event a sleep-time input means, or null when the shell
* should see the input as usual. A display-wake carries the head-tilt
* gesture (HEAD_UP_EVENT) as a press; the double-tap display-wake stays the
* regular UI's wake.
* gesture (HEAD_UP_EVENT) as a tilt, which lasts until the head-down; the
* double-tap display-wake stays the regular UI's wake.
*/
private glanceEventFor(inputEvent: InputEvent, event: RawInputEvent): GlanceEvent | null {
if (inputEvent.type === "display-wake") {
return event.eventType === OsEventTypeList.HEAD_UP_EVENT ? this.glance.eventForGesture("head-tilt") : null;
if (event.eventType !== OsEventTypeList.HEAD_UP_EVENT) return null;
const glanceEvent = this.glance.eventForGesture("head-tilt");
if (glanceEvent?.type !== "tilt") return glanceEvent;
const maxMs = glanceTiltMaxMs({
capMs: glanceEvent.maxMs === undefined ? GLANCE_TILT_MAX_MS : glanceEvent.maxMs,
resumedSession: this.evenHubSessionSuspended || this.evenHubResumePromise !== null,
triggerDegrees: glassesHeadUpTrigger() ?? (glanceAngleTriggerDegrees() || null),
tapTimeoutMs: glanceTapTimeoutMs(),
});
return { type: "tilt", maxMs };
}
return this.glance.eventForGesture(inputEvent.type);
}
@@ -2688,6 +2811,7 @@ class DashboardController {
}
},
endTextSettingEdit: () => this.endTextSettingEdit(),
unhandledKey: (event) => keyboardService.handleUnhandledKey(event as KeyEvent),
});
this.appHosts.set(appId, host);
return host;
@@ -2721,10 +2845,11 @@ class DashboardController {
if (!this.openAppsRestored || this.suppressOpenAppsPersist) return;
const open: string[] = [];
for (const window of shell.getWindows()) {
if (window.appId === "launcher") continue;
if (window.appId === "launcher" || window.restoresApp === false) continue;
if (!open.includes(window.appId)) open.push(window.appId);
}
savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null });
const foreground = shell.foregroundWindow();
savePersistedOpenApps({ open, foreground: foreground && foreground.restoresApp !== false ? foreground.appId : null });
}
/**
@@ -3052,6 +3177,7 @@ class DashboardController {
private setPhase(phase: ConnectionPhase): void {
if (this.phase === phase) return;
this.phase = phase;
this.syncKeyboardLink();
if (phase !== "connected" && phase !== "charging") {
shell.setBatteryLevels({ ring: null, ringCharging: null });
}
+17
View File
@@ -68,6 +68,23 @@ export function extractG2Mac(mfg: Uint8Array): string | null {
return humanReadableAddressFromWireBytes(mfg.subarray(16, 22));
}
/**
* G2 manufacturer data (hex) carrying `serial` and `address`, for an arm known
* only from a saved pairing: what `extractG2Serial` and `extractG2Mac` read
* back, with the serial zero-padded and no flag byte. Null when the serial is
* not up to 14 printable ASCII characters or the address is not a MAC.
*/
export function g2ManufacturerData(serial: string, address: string): string | null {
const wire = wireBytesFromHumanReadableAddress(address);
if (!wire || !/^[\x20-\x7e]{1,14}$/.test(serial)) return null;
const bytes = new Uint8Array(22);
bytes[0] = 0x45;
bytes[1] = 0x52;
for (let i = 0; i < serial.length; i++) bytes[2 + i] = serial.charCodeAt(i);
bytes.set(wire, 16);
return bytesToHex(bytes);
}
/** Preserve the final G2 manufacturer-data byte without guessing its meaning. */
export function extractG2AdvertisementFlag(mfg: Uint8Array): number | null {
if (mfg.length < 23 || !hasEvenManufacturerSignature(mfg)) return null;
+7
View File
@@ -79,6 +79,12 @@ export const OsEventTypeList = {
HEAD_UP_EVENT: 12,
/** Faceclaw/18: R1 touch-down, before click/hold/slide recognition. */
RING_PRESS_EVENT: 14,
/**
* Faceclaw/38: the IMU head-down (the head back below the head-up
* threshold minus 20 degrees), forwarded while a page is up. Ends a
* head-tilt Glanceboard; nothing else uses it.
*/
HEAD_DOWN_EVENT: 15,
} as const;
export const OsEventTypeName: Record<number, string> = {
@@ -96,4 +102,5 @@ export const OsEventTypeName: Record<number, string> = {
11: "SHORT_THEN_LONG_PRESS_EVENT",
12: "HEAD_UP_EVENT",
14: "RING_PRESS_EVENT",
15: "HEAD_DOWN_EVENT",
};
+1 -1
View File
@@ -26,7 +26,7 @@ export type FirmwareInfo = {
};
/** The Faceclaw firmware revision this build of the app needs. */
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 38;
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 40;
const FACECLAW_PREFIX = "Faceclaw/";
const LEGACY_PREFIX = "EVENCFW";
+101 -35
View File
@@ -19,12 +19,42 @@ export type FirmwarePatchSet = {
export const CFW_PATCH_SET: FirmwarePatchSet = {
"base": "g2_2.3.0.24.bin",
"baseSha256": "187ccf2bcc5c17a212106e8a376745511e8289c4232b634a7ea94b9bf25a0979",
"outputSha256": "4aac023f49ec6965186d9106e328a516f20a2915ab400d561f662f984023a432",
"outputSha256": "2a31417f04cec16ecd4e4c3e575d89ce77fb3c5386b47cdb49b9753aaefb1c32",
"patches": [
{
"offset": 1379947,
"old": "0ef009fa",
"new": "08f3a6fc",
"desc": "DM callback allocation: retain keyboard lifecycle on queue exhaustion"
},
{
"offset": 1380027,
"old": "0ef0e1f9",
"new": "08f304fc",
"desc": "ATT callback allocation: expose lost keyboard events as relay gaps"
},
{
"offset": 2649417,
"old": "0a681279",
"new": "d2f155fc",
"desc": "SMP Pairing I/O capability: KeyboardDisplay only for owned keyboard"
},
{
"offset": 2651811,
"old": "0a681279",
"new": "d1f1a8ff",
"desc": "SMP Pairing I/O capability: KeyboardDisplay only for owned keyboard"
},
{
"offset": 1385975,
"old": "38b584b0",
"new": "06f3c2bf",
"desc": "BLE app dispatcher: isolate explicitly enabled keyboard connection"
},
{
"offset": 1045797,
"old": "38b584b0",
"new": "5cf371bb",
"new": "5bf309ba",
"desc": "ring receiver entry: timestamped unfiltered reports under framebuffer lease"
},
{
@@ -36,49 +66,49 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1437923,
"old": "fef788ff",
"new": "fdf2a2f8",
"new": "fbf23aff",
"desc": "cfw_ancs_event"
},
{
"offset": 1437951,
"old": "fef7e8fb",
"new": "fcf2d0fd",
"new": "fbf268fc",
"desc": "cfw_ancs_open"
},
{
"offset": 1437969,
"old": "fef7e7fb",
"new": "fcf2ebfd",
"new": "fbf283fc",
"desc": "cfw_ancs_close"
},
{
"offset": 1388387,
"old": "39694d00",
"new": "25597d00",
"new": "55467d00",
"desc": "connection-bound ANCS EUS write callback"
},
{
"offset": 971497,
"old": "fff717f8",
"new": "6ef3a1fc",
"new": "6df339fb",
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
},
{
"offset": 971845,
"old": "fef769ff",
"new": "6ef3f3fb",
"new": "6df38bfa",
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
},
{
"offset": 979579,
"old": "fdf74ef8",
"new": "6cf3d8fc",
"new": "6bf370fb",
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
},
{
"offset": 1428895,
"old": "f8f76cfd",
"new": "fbf250fe",
"new": "f9f2d0fb",
"desc": "bl cfw_receive_packet (private SID-f0 probe before TPL reassembly)"
},
{
@@ -102,7 +132,7 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1058287,
"old": "fff754fb",
"new": "59f342fb",
"new": "58f3daf9",
"desc": "bl ble_classify_link (only a <= 11.25 ms / latency-0 link is 'already fast')"
},
{
@@ -111,148 +141,184 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
"new": "5ff43332",
"desc": "reserve final 1 KiB of primary TLSF arena for CFW context anchor"
},
{
"offset": 1130241,
"old": "5ff44d22",
"new": "5ff40622",
"desc": "LVGL arena size 0xcd000 -> 0x86000 (fixed CFW buffers follow it)"
},
{
"offset": 2697223,
"old": "9c212020",
"new": "9c192320",
"desc": "EvenHub arena base 0x2020219c -> 0x2023199c"
},
{
"offset": 2697133,
"old": "5ff4e121",
"new": "5ff48221",
"desc": "EvenHub arena clear length 0x70800 -> 0x41000"
},
{
"offset": 2697149,
"old": "5ff4e121",
"new": "5ff48221",
"desc": "EvenHub TLSF pool size 0x70800 -> 0x41000"
},
{
"offset": 2245731,
"old": "5ff4e125",
"new": "5ff48225",
"desc": "EvenHub heap total in the stock usage log 0x70800 -> 0x41000"
},
{
"offset": 4394495,
"old": "9b212020",
"new": "9bb11b20",
"desc": "MPU region 6 (non-cacheable) ends with the shrunk LVGL arena"
},
{
"offset": 1248487,
"old": "d4f712fb",
"new": "29f344fd",
"new": "27f3a0f9",
"desc": "bl settings_send_wrapper (append caps field 100)"
},
{
"offset": 1246183,
"old": "f5f7e2f8",
"new": "29f3e8fe",
"new": "27f3befb",
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
},
{
"offset": 975219,
"old": "02f083fc",
"new": "6bf34aff",
"new": "69f320fc",
"desc": "bl faceclaw_display_start @ 0x467d68 (fail-open wake takeover)"
},
{
"offset": 975411,
"old": "02f023fc",
"new": "6bf3b8ff",
"new": "69f38efc",
"desc": "bl faceclaw_display_start_headup @ 0x467e28 (fail-open wake takeover)"
},
{
"offset": 975103,
"old": "07f085fb",
"new": "6df3a6fc",
"new": "6cf33efb",
"desc": "bl headup_gate (idle input gate: tap/long/release -> field-102 events, stock result kept)"
},
{
"offset": 830549,
"old": "19f03efb",
"new": "90f37bfe",
"new": "8ff313fd",
"desc": "bl gesture_ring_press_mode (ring touch-down 14, head up 12 / down 15; stock dispatch preserved)"
},
{
"offset": 830795,
"old": "1ff071ff",
"new": "90f334fe",
"new": "8ff3ccfc",
"desc": "bl gesture_press (owned event 9 before stock menu gate)"
},
{
"offset": 830887,
"old": "1ff043ff",
"new": "90f32efe",
"new": "8ff3c6fc",
"desc": "bl gesture_short_long (Faceclaw lease gates event 11 vs stock Menu path)"
},
{
"offset": 831763,
"old": "1ff08dfd",
"new": "90f364fc",
"new": "8ff3fcfa",
"desc": "bl gesture_release (owned event 10 before stock menu gate)"
},
{
"offset": 1570209,
"old": "7fb50600",
"new": "dbf2d5b8",
"new": "d8f231bd",
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
},
{
"offset": 1051545,
"old": "f6f787f9",
"new": "55f3c9f8",
"new": "54f359fe",
"desc": "bl display_copy_hook @ 0x47a78e (640x480 direct framebuffer)"
},
{
"offset": 1051861,
"old": "f6f7e9f8",
"new": "55f32bf8",
"new": "54f3bbfd",
"desc": "bl display_copy_hook @ 0x47a8ca (640x480 direct framebuffer)"
},
{
"offset": 1262237,
"old": "d9f7d0fd",
"new": "25f3bffd",
"new": "23f3edfa",
"desc": "bl faceclaw_send_wear_event @ 0x4ade92 (outside onboarding)"
},
{
"offset": 1262337,
"old": "d9f79efd",
"new": "25f38dfd",
"new": "23f3bbfa",
"desc": "bl faceclaw_send_wear_event @ 0x4adef6 (outside onboarding)"
},
{
"offset": 1309405,
"old": "67f02dff",
"new": "19f35bfe",
"new": "17f397fb",
"desc": "bl compass_decode_capture (sample-matched GAF diagnostics)"
},
{
"offset": 1307865,
"old": "fff7c9fb",
"new": "1af3ddf9",
"new": "17f319ff",
"desc": "bl compass_report_event (stock UI + heading with diagnostics over BLE)"
},
{
"offset": 1300937,
"old": "00210170",
"new": "1bf3bbfe",
"new": "19f3f7fb",
"desc": "bl compass_preserve_accuracy (retain magnetic calibration under Faceclaw framebuffer lease)"
},
{
"offset": 4537963,
"old": "",
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f004fd002800f0cb814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f042fc002800f009814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8948242f6f76ac0f2440ab8f1000fbff35f8f4ad00af19a0240464ff0ff314ff0ff379047002840f090811198002861d010980778a7f1170002285bd811980c2840f07d81109c6078a178084300f07781e0780138022800f27281a279e379667a42ea0322237a207942ea034242ea0663a27ae67a6179192f42ea062200f0de81172f40f0f481002a40f05981fe2800f0b086812800f0a386002840f0b68651ea030040f04c8100f000bf04460020d047e8b104f5257151e8008fb8f1000f0cd1bff35f8f00bf41e80002002a00f0e58051e8008fb8f1000ff5d00021bff32f8f0029bff35f8f00f0d480804692e74ff0ff372ae147f6df40c0f24f00cdf83880804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0f8fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bce6012120e70020109f42f6f76addf83880c0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2924dc0b2926d01c2924d03ce00f9012a803f045ff4ff0ff3701249df85100002812a818bf4ff0ff3704f05ff8054644b99df8500028b94af21f30c0f247004c3080470f98002818bf056114e09c2901d08b2918d168b14af21f31c0f247010746884797f8681121b997f824133846109f49b14ff0ff370af5807140468847384647b0bde8f08f0f9012a8344003f007ff387800f07f061e2e05d11199384607f094fe0746b9e71d2e00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a6f115010129e1d90f2e2cdd142e40f31082152e00f05d83162e00f069831f2e40f02c841198012840f0fc8438781f2840f0f88445f64d55c0f24605a847012841f0828140f24c71c0f2010105f5e2777944e02029220023b8470746002818bf4ff0ff3766e7092e00f3fe80052e00f03e82072e40f0028400274ff0010947f67f764ff07f080d943ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf0Line truncated
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff784ff47e740027012647f67f754ff07f0a1191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0ad101684ef66d62ccf2ff02914203d14068b0f57e7f4ed04ff47e7002f0f9f9002800f027816ff0070142180831b1f57c7f177257729772d772177357739773d773f2d14ef66d61ccf2ff01016044608673a0f8c85180f8c7a1a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001f494599d1bff35f8f4be8000121b15be8001f4945f8d08fe74ff47e7a00254ff0010947f67f777f2407e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0ad101684ef66d62ccf2ff02914203d14168b1f57e7f50d04ff47e7002f07ff9002800f0ad806ff0070142180831b1f57c7f157255729572d572157355739573d573f2d14ef66d61ccf2ff010160c0f804a080f80e90a0f8c87180f8c741a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001fb14297d1bff35f8f4be8000121b15be8001fb142f8d08de7d0f8984242f6f76ac0f2440abff35f8fdcb30af19a0220464ff0ff314ff0ff379047002840f02e81119800284ed010980778a7f11700022848d811980c2821d1109d6878a97808431cd0e8780138022818d8aa79eb796e7a42ea03222b7a287942ea034242ea0663aa7aee7a6979192f42ea062200f07f81172f40f09581002a00f04e834ff0ff37f8e004460020d047c8b104f5267151e8004f54b9bff35f8f41e80002002a00f0998051e8004f002cf6d00021bff32f8f0029bff35f8f00f089800446a5e74ff0ff37dbe047f6df40c0f24f000e94804700274ff47e79012647f67f744ff07f0a13978df8507006e048f2a351c0f24501bff32f8f8847dbf80050bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0ad101684ef66d62ccf2ff02914203d14168b1f57e7f57d04ff47e7002f095f8002850d06ff0070100bf42180831b1f57c7f177257729772d772177357739773d773f2d14ef66d61ccf2ff010160c0f804908673a0f8c84180f8c7a1a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001fa94298d1bff35f8f00bf4be8000159b15be8001fa942f8d08de70af5a97188471ae701216be70020109f42f6f76ac0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0518bf01258a291edc0b2920d01c291ed03ae00f9012a801f06dfe4ff0ff37012612a801f037ff054646b99df8500028b94af21f30c0f247004c3080470f98002818bf056117e09c2901d08b291cd180b14af21f31c0f247010646884796f86811bff35f8f29b996f824133046bff35f8f51b14ff0ff370e9c0af5807120468847384647b0bde8f08f0f9012a804ea050601f030fe387800f07f041e2c05d11199384606f01ffe0746bae71d2c00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66d63ccf2ff039a420bd14a68b2f57e7f04bf002281f8a12303e0a4f115010129e1d90f2c2cdd142c40f3ad81152c00f0c882162c00f0d2821f2c40f095831198012840f0648438781f2840f0608445f64d54c0f24604a047012841f05e8041f2de01c0f2010104f5e2777944e02029220023b8470746002818bf4ff0ff3767e7092c00f3b880052c00f0ee81072c40f06b830d9600264ff0010947f67f777f243ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf00f087bc182f4ff0ff3708bf00297ff463af002b7ff460af352800f37083002800f05f84012800f06b84312840f05d84082abff451af00f06cbc48f2a351c0f24501bff32f8f8847dbf80050bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66d62ccf2ff02914204d14168b1f57e7f00f077854ff47e7001f01dff002800f06f856ff0070142180831b1f57c7f167256729672d672167356739673d673f2d14ef66d61ccf2ff0101604ff47e71416080f80e90a0f8c87180f8c741a0f82080a0f82280a0f82480a0f82680a0f82880a0f82a80a0f82c80a0f82e80a0f83080a0f83280a0f83480a0f83680a0f83880a0f83a80a0f83c80a0f83e805be8001fa94295d1bff35f8f4be80001002900f028855be8001fa942f6d089e70a2c00f0bb810b2c40f0b382dbf800500320cff680702840b0f1005fbff35f8f40f0808728684ef66d61ccf2ff01884240f0788768680027b0f57e7f7ff48daed5f8900342f68d49c0f244090028a5f8a07318bfc8470021a5f8d013c5f8cc13c5f8701185f86c11bff35f8f85f86811686c85f86b11c5f87411c5f82013c5f88813a5f88c13a5f8481040b1c847686c09f166018847002804bf002068644bf63940c0f251008047dbf800400320cff680702040b0f1005fbff35f8f46d120684ef66Line truncated
"desc": "append injected blobs to main-app payload"
},
{
"offset": 779123,
"old": "805a3900",
"new": "68703a00",
"new": "10783a00",
"desc": "main-app subheader payload size (ps)"
},
{
"offset": 152,
"old": "005b3900",
"new": "e8703a00",
"new": "90783a00",
"desc": "main-app TOC entry size (ps + 128)"
},
{
"offset": 779243,
"old": "805a3904",
"new": "68703a04",
"new": "10783a04",
"desc": "main-app preamble length (low 24 bits)"
},
{
"offset": 779247,
"old": "effdbcea",
"new": "0b0c48b0",
"new": "adc5378e",
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
},
{
"offset": 156,
"old": "c0c6322a",
"new": "52937d21",
"new": "0c2e1c3e",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
},
{
"offset": 779127,
"old": "c0c6322a",
"new": "52937d21",
"new": "0c2e1c3e",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
}
]
+3 -2
View File
@@ -82,7 +82,7 @@ export class GlanceHost {
* The glance event a sleep-time gesture means under the board's settings,
* or null when the shell should see the input as usual: nothing while the
* board is disabled, no press from a tap when "Show on tap" is disabled,
* no hold when "Show on long press" is off, no press from a head-tilt when
* no hold when "Show on long press" is off, no tilt from a head-tilt when
* "Show on head tilt" is off (it then wakes the regular UI). A release
* always passes so a hold in progress can end.
*/
@@ -92,7 +92,8 @@ export class GlanceHost {
if (gesture === "click" && !provider.showOnTap()) return null;
if ((gesture === "long-press" || gesture === "short-then-long-press") && !provider.showOnLongPress()) return null;
if (gesture === "head-tilt" && !provider.showOnHeadTilt()) return null;
return glanceEventForGesture(gesture, this.isVisible());
const event = glanceEventForGesture(gesture, this.isVisible());
return event?.type === "tilt" ? { type: "tilt", maxMs: provider.tiltTimeoutMs() } : event;
}
/**
+82 -12
View File
@@ -1,9 +1,10 @@
/**
* The Glanceboard's visibility rules, as a pure reducer so they can be tested
* without the display plumbing. The board is an alternate sleep-time display:
* while the shell is asleep a press (single tap, or a head-tilt wake) shows it
* for a short while, a hold (long-press or short-then-long-press) shows it until release,
* and a double-tap dismisses it in favour of the regular UI. GlanceHost feeds
* while the shell is asleep a press (single tap) shows it for a short while, a
* head-tilt wake shows it until the head comes back down, a hold (long-press or
* short-then-long-press) shows it until release, and a double-tap dismisses it
* in favour of the regular UI. GlanceHost feeds
* these events in and applies the resulting visibility to the compositor.
*
* No NativeScript imports: this module is compiled and run under plain node
@@ -13,17 +14,67 @@
/** Fallback press timeout when the host supplies none; each further press restarts it. */
export const GLANCE_TIMEOUT_MS = 3000;
/**
* Default cap for a head-tilt showing: the board normally stays up until the
* head comes back down (as the stock dashboard does), but a lost head-down
* event should not leave it on indefinitely. The user's Head tilt timeout
* setting replaces it (and may remove it).
*/
export const GLANCE_TILT_MAX_MS = 60_000;
/**
* The firmware reports head-down once the pitch falls 20° below the head-up
* trigger (a fixed hysteresis in DRV_IMUCheckHeadUpEvent). Under this trigger,
* looking level again does not get below that, so the head-down may never come.
*/
export const GLANCE_LOW_TILT_TRIGGER_DEGREES = 25;
/**
* How long a head-tilt showing may last without a head-down: capMs (the Head
* tilt timeout; null = no limit), except when the head-down is unlikely to
* arrive, where the tap duration is used if shorter: a tilt that woke a
* suspended EvenHub session (the head-down is only forwarded once the page is
* back, so a quick glance can end before that), or a trigger too low for a
* level gaze to count as looking down. triggerDegrees is the glasses'
* trigger, or null when not known.
*/
export function glanceTiltMaxMs(options: {
capMs: number | null;
resumedSession: boolean;
triggerDegrees: number | null;
tapTimeoutMs: number;
}): number | null {
const lowTrigger =
options.triggerDegrees !== null && options.triggerDegrees > 0 && options.triggerDegrees < GLANCE_LOW_TILT_TRIGGER_DEGREES;
if (!options.resumedSession && !lowTrigger) return options.capMs;
return options.capMs === null ? options.tapTimeoutMs : Math.min(options.capMs, options.tapTimeoutMs);
}
export type GlanceState = {
visible: boolean;
/** A long-press is holding the board up; the timer does not run meanwhile. */
holding: boolean;
/**
* A head-tilt is holding the board up: it stays until the firmware reports
* the head back down (or tiltMaxMs passes).
*/
tilted: boolean;
/** The current head-tilt's cap (0 when not tilted, null for none); a hold ending mid-tilt re-arms it. */
tiltMaxMs: number | null;
/** When the board hides on its own (epoch ms), or null while held / hidden. */
hideAtMs: number | null;
};
export type GlanceEvent =
/** Single tap or head-tilt wake: show, and (re)start the auto-hide timer. */
/** Single tap: show, and (re)start the auto-hide timer. */
| { type: "press" }
/**
* Head-tilt (IMU head-up): show until the head comes back down, or until
* maxMs passes (default GLANCE_TILT_MAX_MS; null = no limit).
*/
| { type: "tilt"; maxMs?: number | null }
/** The IMU reported the head back down: ends a head-tilt showing. */
| { type: "head-down" }
/** Long-press: show and keep showing until the release. */
| { type: "hold" }
/** The long-press ended. */
@@ -33,7 +84,7 @@ export type GlanceEvent =
/** Double-tap (or any other wake of the regular UI): hide now. */
| { type: "dismiss" };
export const GLANCE_HIDDEN: GlanceState = Object.freeze({ visible: false, holding: false, hideAtMs: null });
export const GLANCE_HIDDEN: GlanceState = Object.freeze({ visible: false, holding: false, tilted: false, tiltMaxMs: 0, hideAtMs: null });
export function reduceGlance(
state: GlanceState,
@@ -43,14 +94,26 @@ export function reduceGlance(
): GlanceState {
switch (event.type) {
case "press":
// A tap during a hold changes nothing: the hold decides when it ends.
if (state.holding) return state;
return { visible: true, holding: false, hideAtMs: nowMs + timeoutMs };
// A tap during a hold or a head-tilt changes nothing: those decide when it ends.
if (state.holding || state.tilted) return state;
return { visible: true, holding: false, tilted: false, tiltMaxMs: 0, hideAtMs: nowMs + timeoutMs };
case "tilt": {
const maxMs = event.maxMs === undefined ? GLANCE_TILT_MAX_MS : event.maxMs;
return { visible: true, holding: state.holding, tilted: true, tiltMaxMs: maxMs, hideAtMs: state.holding ? null : tiltHideAt(nowMs, maxMs) };
}
case "head-down":
// Only a head-tilt showing ends with the head; a tap's timer or a hold keeps running.
if (!state.tilted) return state;
return state.holding ? { ...state, tilted: false, tiltMaxMs: 0 } : GLANCE_HIDDEN;
case "hold":
return { visible: true, holding: true, hideAtMs: null };
return { visible: true, holding: true, tilted: state.tilted, tiltMaxMs: state.tiltMaxMs, hideAtMs: null };
case "release":
// Releases delivered late or after a disabled hold gesture mean nothing here.
return state.holding ? GLANCE_HIDDEN : state;
if (!state.holding) return state;
// Head still up: the board stays, as if the tilt had just shown it.
return state.tilted
? { visible: true, holding: false, tilted: true, tiltMaxMs: state.tiltMaxMs, hideAtMs: tiltHideAt(nowMs, state.tiltMaxMs) }
: GLANCE_HIDDEN;
case "timeout":
if (!state.visible || state.holding || state.hideAtMs === null || nowMs < state.hideAtMs) return state;
return GLANCE_HIDDEN;
@@ -59,6 +122,10 @@ export function reduceGlance(
}
}
function tiltHideAt(nowMs: number, maxMs: number | null): number | null {
return maxMs === null ? null : nowMs + maxMs;
}
/**
* Which glance event, if any, a shell input gesture means while the screen is
* off. A double-tap only concerns the board while it is showing (it then
@@ -66,13 +133,16 @@ export function reduceGlance(
* already handles or ignores while asleep.
*/
export function glanceEventForGesture(
gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | (string & {}),
gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | "head-down" | (string & {}),
boardVisible: boolean,
): GlanceEvent | null {
switch (gesture) {
case "click":
case "head-tilt":
return { type: "press" };
case "head-tilt":
return { type: "tilt" };
case "head-down":
return boardVisible ? { type: "head-down" } : null;
case "long-press":
case "short-then-long-press":
return { type: "hold" };
+151
View File
@@ -0,0 +1,151 @@
import { addImuListener, imuPitchDegrees, setImuReportEnabled, type ImuReading } from "../native/imu";
import { getHeadUpStatus, setHeadUpTrigger, type HeadUpStatus } from "../native/glasses-controls";
import { getStringSetting, setStringSetting } from "../native/settings-store";
import { glanceAngleTriggerDegrees } from "../apps/glanceboard/glanceboard-settings";
// Same ImuReportPace code the accelerometer demo uses (not literal Hz).
const IMU_REPORT_PACE = 200;
/** How often the live reading checks visibility and repaints a changed value. */
const LIVE_TICK_MS = 250;
/** Weight of each new sample in the smoothed pitch. */
const LIVE_SMOOTHING = 0.3;
/** A reading older than this (or none this long after the stream started) is shown as missing. */
const LIVE_STALE_MS = 3_000;
/**
* The head's pitch, live, for the Head tilt angle picker. The IMU report
* stream runs while `visible()` holds (checked every tick, so it stops while
* the screen is off or the window is in the background) and until stop().
* text() is the footer line; a changed value repaints at most once a tick.
*/
class LiveHeadPitch {
private pitch: number | null = null;
private lastSampleAt = 0;
private streaming = false;
private streamingSince = 0;
private shown = "";
private readonly unsubscribe: () => void;
private readonly timer: ReturnType<typeof setInterval>;
constructor(
private readonly visible: () => boolean,
private readonly repaint: () => void,
) {
this.unsubscribe = addImuListener((reading) => this.onReading(reading));
this.timer = setInterval(() => this.tick(), LIVE_TICK_MS);
this.tick();
}
text(): string {
return this.shown;
}
stop(): void {
clearInterval(this.timer);
this.unsubscribe();
if (this.streaming) setImuReportEnabled(false, IMU_REPORT_PACE);
this.streaming = false;
}
private onReading(reading: ImuReading): void {
// Glasses temples only (1 right, 3 left, 0 unspecified); the ring is 2.
if (!this.streaming || reading.source === 2) return;
if (!(Math.hypot(reading.x, reading.y, reading.z) > 1e-6)) return;
const pitch = imuPitchDegrees(reading);
this.pitch = this.pitch === null ? pitch : this.pitch + LIVE_SMOOTHING * (pitch - this.pitch);
this.lastSampleAt = Date.now();
}
private tick(): void {
const now = Date.now();
const visible = this.visible();
if (visible !== this.streaming) {
setImuReportEnabled(visible, IMU_REPORT_PACE);
this.streaming = visible;
this.streamingSince = now;
this.pitch = null;
}
const text = this.format(now);
if (text !== this.shown) {
this.shown = text;
this.repaint();
}
}
private format(now: number): string {
if (this.pitch !== null && now - this.lastSampleAt < LIVE_STALE_MS) return `Head now: ${Math.round(this.pitch)}°`;
return now - Math.max(this.streamingSince, this.lastSampleAt) < LIVE_STALE_MS ? "Head now: …" : "Head now: no data";
}
}
/**
* The Head tilt angle picker's footer: the head's current pitch, read while
* the picker is open and `visible()` holds (enumSettingMenuItem pickerFooter).
*/
export function liveHeadPitchFooter(visible: () => boolean): { text: () => string; open: (repaint: () => void) => () => void } {
let reader: LiveHeadPitch | null = null;
return {
text: () => reader?.text() ?? "",
open: (repaint) => {
const opened = new LiveHeadPitch(visible, repaint);
reader = opened;
return () => {
opened.stop();
if (reader === opened) reader = null;
};
},
};
}
// The glasses' own head-up switch and angle from before Faceclaw first changed
// them, as "switch,angle", or "" when none is held. Kept here (not only in the
// session) so "Glasses default" can still restore them after an app restart.
const ORIGINAL_KEY = "glanceboard.headTiltOriginal";
function savedOriginal(): { switch: number; angle: number } | null {
const [sw, angle] = getStringSetting(ORIGINAL_KEY, "").split(",").map(Number);
return (sw === 0 || sw === 1) && Number.isInteger(angle) ? { switch: sw, angle } : null;
}
/**
* Push the wanted trigger to the session (on change and on each connect):
* the Head tilt angle while the Glanceboard is enabled, else none, which
* writes back the glasses' own values if Faceclaw changed them.
*/
export function applyGlanceAngle(): boolean {
const sent = setHeadUpTrigger(glanceAngleTriggerDegrees(), savedOriginal());
syncGlanceAngleOriginal();
return sent;
}
/**
* Keep the persisted original in step with the session: save the values the
* session captured before its first write, and drop them once a read shows
* them restored. Called on apply, from the controller's periodic refresh, and
* when the settings row draws.
*/
export function syncGlanceAngleOriginal(status: HeadUpStatus | null = getHeadUpStatus()): void {
if (!status) return;
const saved = savedOriginal();
if (!saved) {
if (status.originalSwitch === 0 || status.originalSwitch === 1) {
setStringSetting(ORIGINAL_KEY, `${status.originalSwitch},${status.originalAngle}`);
}
return;
}
if (
glanceAngleTriggerDegrees() === 0 &&
status.desired < 0 &&
!status.restorePending &&
status.switch === saved.switch &&
status.angle === saved.angle
) {
setStringSetting(ORIGINAL_KEY, "");
}
}
/** The glasses' current head-up trigger (angle + calibration), 0 when off, or null when not read yet. */
export function glassesHeadUpTrigger(status: HeadUpStatus | null = getHeadUpStatus()): number | null {
if (!status || status.switch < 0) return null;
return status.switch === 1 ? status.angle + status.calibration : 0;
}
+48 -2
View File
@@ -29,6 +29,8 @@ import { OsEventTypeList } from './events'
import { loadDeviceAddresses } from './device-addresses'
import { updateGlassesPresence } from './glasses-presence'
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
import { keyboardService } from './keyboard/service'
import type { KeyEvent } from './keyboard/keys'
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy'
import { WearRemote, type WearRemoteInputKind } from './wear-remote'
import { deviceAddressError } from './ios-peripheral-identity'
@@ -232,6 +234,7 @@ export class IosPreviewController {
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
this.requestShellRender()
this.wearRemote?.schedulePublish()
if (on) keyboardService.retryNow()
},
})
const launcher = createLauncherWindow({
@@ -270,6 +273,28 @@ export class IosPreviewController {
assistantAvailable: () => shell.isAssistantAvailable(),
assistant: text => shell.sendToAssistant(text),
})
// A keyboard paired through the glasses (the CFW keyboard bridge). The service decides what
// each key means (app/g2/keyboard/key-router.ts); its gestures share the controller queue
// with physical input, like the watch's.
const queueKeyboardInput = (run: () => Promise<void>): Promise<void> => {
const task = this.inputQueue.then(() => { this.onActivity(); return run() })
this.inputQueue = task.catch(error => this.fail(error))
return task
}
keyboardService.setHost({
ready: () => this.state.phase === 'connected',
locked: () => this.glassesLocked,
wake: () => { void queueKeyboardInput(() => this.receiveInput(makeInputEvent({ type: 'display-wake' }))).catch(() => {}) },
gesture: (gesture, origin) => queueKeyboardInput(async () => {
await this.receiveInput(makeInputEvent({ type: gesture, source: origin } as InputEventPayload))
// A discrete hold includes its release.
if (gesture === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: origin }))
}),
log: line => this.logBluetooth(line),
// Only while the display is on: a keyboard that is switched off would otherwise keep the
// glasses' radio looking for it around the clock.
shouldReconnect: () => this.state.phase === 'connected' && shell.isScreenOn(),
})
// The header's EvenHub phone-UI button hides while that UI is showing.
onPhoneShownChanged(() => this.syncEvenHubPhoneUi())
registerSystemTools()
@@ -809,6 +834,7 @@ export class IosPreviewController {
},
setSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible),
removeSurface: id => this.removeWindowSurface(id),
unhandledKey: event => keyboardService.handleUnhandledKey(event as KeyEvent),
submitPixels: (id, pixels, width, height, draws) => {
void this.submitWorkerPixels(id, pixels, width, height, draws).catch(error => this.fail(error))
},
@@ -830,6 +856,18 @@ export class IosPreviewController {
this.state = { ...this.state, phase, status }
this.logBluetooth(status)
this.emitState()
this.syncKeyboardLink()
}
/**
* Tell the keyboard bridge whether the session can host a keyboard: a G2 on firmware with
* the bridge, connected. On a disconnect the user asked for, the link is still up when the
* phase turns, so the firmware is told to let the keyboard go first.
*/
private syncKeyboardLink(): void {
const up = !!this.communicator && this.state.phase === 'connected' && this.firmwareCompatible
&& glassesTypeSetting.get() === 'g2'
if (up) keyboardService.linkReady()
else keyboardService.linkLost(this.state.phase === 'disconnecting')
}
/** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */
private syncPresence(): void {
@@ -949,6 +987,7 @@ export class IosPreviewController {
const firmwareWarning = firmwareIncompatibilityMessage(info) ?? ''
if (firmwareWarning && firmwareWarning !== this.state.firmwareWarning) this.logBluetooth(`Firmware compatibility warning: ${firmwareWarning}`)
this.firmwareCompatible = !firmwareWarning && hasCompatibleFirmware(info)
this.syncKeyboardLink()
this.state = { ...this.state, leftVersion: info.leftVersion || this.state.leftVersion,
rightVersion: info.rightVersion || this.state.rightVersion, capabilities: info.extension || this.state.capabilities,
firmwareWarning }
@@ -1013,6 +1052,13 @@ export class IosPreviewController {
private handleRingEvent(input: RawInputEvent): void {
this.inputQueue = this.inputQueue.then(async () => {
if (this.state.phase !== 'connected') return
if (input.kind === 'sys-event' && input.eventType === OsEventTypeList.HEAD_DOWN_EVENT) {
// As on Android: only the Glanceboard uses it (it ends a head-tilt
// showing); it never reaches the shell or counts as activity.
const glanceEvent = this.glance.eventForGesture('head-down')
if (glanceEvent) await this.glance.handleEvent(glanceEvent, 0)
return
}
this.onActivity()
if (input.ringInput) this.state = { ...this.state, ring: true }
await this.receiveInput(rawInputEventToInputEvent(input),
@@ -1111,8 +1157,8 @@ export class IosPreviewController {
/**
* Continuous capture on behalf of `client` (one id per window: Transcribe,
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
* Recording is on.
* on Android. `recordAs` names a recording (.mka) of the audio in Downloads
* when Save Recording is on.
*/
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options)
+291
View File
@@ -0,0 +1,291 @@
/**
* What keys from a hardware keyboard do, apart from typing: the built-in
* defaults, the user's custom keybindings, and how the two combine.
*
* A key becomes one of the gestures the ring and the watch already have, so
* nothing downstream knows about keyboards. Three layers, first match wins:
*
* 1. Custom keybindings (Settings > Keyboard > Keybindings). These apply
* everywhere, including where the focused window takes keys for itself
* (a terminal session), since that is the point of binding a remote's
* few buttons.
* 2. Built-in bindings that also apply there: Escape with Shift, Ctrl or
* Alt goes back, which is the way out of a terminal session; and a
* mouse's buttons and wheel, which are never typed.
* 3. Built-in navigation, only for keys no window took: the arrows are the
* watch's four swipes, Enter selects, Escape goes back.
*
* A keyboard's touchpad or mouse arrives here as keys too: "MouseButton1"
* (left), "MouseButton2" (right), "MouseButton3" (middle) and so on, and
* "WheelUp" / "WheelDown" / "WheelLeft" / "WheelRight" (see pointer.ts).
*
* No platform imports: this runs under node in the tests.
*/
import type { KeyEvent } from "./keys";
/** A gesture a key can stand for, named as the input pipeline names it. */
export type KeyGesture =
| "swipe-up"
| "swipe-down"
| "swipe-left"
| "swipe-right"
| "scroll-up"
| "scroll-down"
| "click"
| "double-click"
| "long-press"
| "short-then-long-press";
/**
* What a custom keybinding does. "none" turns off the key's built-in
* navigation meaning; it does not stop the key being typed into a window
* that takes keys.
*/
export type KeyAction = KeyGesture | "none";
/** The actions, in the order the picker lists them. */
export const KEY_ACTIONS: readonly { action: KeyAction; label: string }[] = [
{ action: "click", label: "Select" },
{ action: "double-click", label: "Back" },
{ action: "swipe-up", label: "Swipe up" },
{ action: "swipe-down", label: "Swipe down" },
{ action: "swipe-left", label: "Swipe left" },
{ action: "swipe-right", label: "Swipe right" },
{ action: "scroll-up", label: "Scroll up" },
{ action: "scroll-down", label: "Scroll down" },
{ action: "long-press", label: "System menu" },
{ action: "short-then-long-press", label: "App menu" },
{ action: "none", label: "Nothing" },
];
export function keyActionLabel(action: KeyAction): string {
return KEY_ACTIONS.find((entry) => entry.action === action)?.label ?? action;
}
function isKeyAction(value: unknown): value is KeyAction {
return KEY_ACTIONS.some((entry) => entry.action === value);
}
/** A physical key with the modifiers held: what a keybinding is keyed on. */
export type KeyChord = { code: string; shift: boolean; ctrl: boolean; alt: boolean; meta: boolean };
export type KeyBinding = { chord: KeyChord; action: KeyAction };
const MODIFIER_CODES = new Set([
"ControlLeft", "ControlRight", "ShiftLeft", "ShiftRight", "AltLeft", "AltRight", "MetaLeft", "MetaRight",
]);
/** Shift, Ctrl, Alt and Meta themselves: they qualify a chord rather than being one. */
export function isModifierCode(code: string): boolean {
return MODIFIER_CODES.has(code);
}
export function chordOf(event: KeyEvent): KeyChord {
const { shift, ctrl, alt, meta } = event.modifiers;
return { code: event.code, shift, ctrl, alt, meta };
}
/** Canonical text of a chord ("Ctrl+Shift+Escape"): the stored form, and the lookup key. */
export function chordId(chord: KeyChord): string {
return [chord.ctrl && "Ctrl", chord.alt && "Alt", chord.shift && "Shift", chord.meta && "Meta", chord.code]
.filter(Boolean).join("+");
}
/** The inverse of chordId, or null for anything it could not have produced. */
export function parseChordId(text: string): KeyChord | null {
const parts = text.split("+");
const code = parts.pop() ?? "";
if (!/^[A-Za-z][A-Za-z0-9]{0,39}$/.test(code) || isModifierCode(code)) return null;
const chord: KeyChord = { code, shift: false, ctrl: false, alt: false, meta: false };
for (const part of parts) {
if (part === "Ctrl" && !chord.ctrl) chord.ctrl = true;
else if (part === "Alt" && !chord.alt) chord.alt = true;
else if (part === "Shift" && !chord.shift) chord.shift = true;
else if (part === "Meta" && !chord.meta) chord.meta = true;
else return null;
}
return chord;
}
const CODE_LABELS: Record<string, string> = {
ArrowUp: "Up", ArrowDown: "Down", ArrowLeft: "Left", ArrowRight: "Right",
Escape: "Esc", PageUp: "Page Up", PageDown: "Page Down", CapsLock: "Caps Lock",
PrintScreen: "Print Screen", ScrollLock: "Scroll Lock", NumLock: "Num Lock", ContextMenu: "Menu",
Minus: "-", Equal: "=", BracketLeft: "[", BracketRight: "]", Backslash: "\\", IntlBackslash: "\\",
IntlHash: "#", Semicolon: ";", Quote: "'", Backquote: "`", Comma: ",", Period: ".", Slash: "/",
AudioVolumeUp: "Volume Up", AudioVolumeDown: "Volume Down", AudioVolumeMute: "Mute",
MediaPlayPause: "Play/Pause", MediaPlay: "Play", MediaPause: "Pause", MediaStop: "Stop",
MediaTrackNext: "Next Track", MediaTrackPrevious: "Previous Track",
BrightnessUp: "Brightness Up", BrightnessDown: "Brightness Down",
BrowserSearch: "Search", BrowserHome: "Browser Home", BrowserBack: "Browser Back", BrowserForward: "Browser Forward",
LaunchMail: "Mail", LaunchApplication1: "App 1", LaunchApplication2: "App 2",
MouseButton1: "Left button", MouseButton2: "Right button", MouseButton3: "Middle button",
WheelUp: "Wheel up", WheelDown: "Wheel down", WheelLeft: "Wheel left", WheelRight: "Wheel right",
};
/** A key as a person would name it. Unknown keys keep their HID usage number, which is all there is. */
function codeLabel(code: string): string {
const known = CODE_LABELS[code];
if (known) return known;
let match = /^Key([A-Z])$/.exec(code);
if (match) return match[1]!;
match = /^Digit([0-9])$/.exec(code);
if (match) return match[1]!;
match = /^Numpad(.+)$/.exec(code);
if (match) return `Numpad ${match[1]}`;
match = /^MouseButton([0-9]+)$/.exec(code);
if (match) return `Mouse button ${match[1]}`;
match = /^Usage([0-9a-f]+)$/.exec(code);
if (match) return `Key 0x${match[1]}`;
match = /^Consumer([0-9a-f]+)$/.exec(code);
if (match) return `Media key 0x${match[1]}`;
return code;
}
/** A chord for display: "Shift+Esc", "Left", "Volume Up". */
export function describeChord(chord: KeyChord): string {
return [chord.ctrl && "Ctrl", chord.alt && "Alt", chord.shift && "Shift", chord.meta && "Meta", codeLabel(chord.code)]
.filter(Boolean).join("+");
}
/** The most custom keybindings kept; far more than a keyboard has uses for. */
export const MAX_KEY_BINDINGS = 64;
/**
* Custom keybindings from their stored JSON. Lenient: whatever doesn't make
* sense is dropped, and a later entry for a chord replaces an earlier one.
*/
export function parseKeyBindings(raw: string): KeyBinding[] {
let parsed: unknown;
try {
parsed = raw ? JSON.parse(raw) : [];
} catch {
return [];
}
if (!Array.isArray(parsed)) return [];
const bindings = new Map<string, KeyBinding>();
for (const entry of parsed) {
if (!entry || typeof entry !== "object") continue;
const { key, action } = entry as { key?: unknown; action?: unknown };
const chord = typeof key === "string" ? parseChordId(key) : null;
if (!chord || !isKeyAction(action)) continue;
bindings.set(chordId(chord), { chord, action });
if (bindings.size >= MAX_KEY_BINDINGS) break;
}
return [...bindings.values()];
}
export function serializeKeyBindings(bindings: readonly KeyBinding[]): string {
return JSON.stringify(bindings.map((binding) => ({ key: chordId(binding.chord), action: binding.action })));
}
/** Custom keybindings as the lookup resolveKey takes. */
export function keyBindingMap(bindings: readonly KeyBinding[]): Map<string, KeyAction> {
return new Map(bindings.map((binding) => [chordId(binding.chord), binding.action]));
}
/** What a mouse does unless bound otherwise: the ring's tap, double tap, hold and scroll. */
const MOUSE_DEFAULTS: Record<string, KeyGesture | undefined> = {
MouseButton1: "click",
MouseButton2: "double-click",
MouseButton3: "long-press",
WheelUp: "scroll-up",
WheelDown: "scroll-down",
};
/**
* The built-in meaning of a key press, or null when it has none.
* `overridesTyping`: it holds even where the focused window takes keys.
* Built-ins go by the logical key, so both Enter keys select.
*/
export function defaultKeyAction(event: KeyEvent): { action: KeyGesture; overridesTyping: boolean } | null {
// A mouse's buttons and wheel are the ring's gestures: tap, double tap,
// hold, and scroll. Nothing types them, so they hold everywhere, whatever
// is held on the keyboard.
const mouse = MOUSE_DEFAULTS[event.key];
if (mouse) return { action: mouse, overridesTyping: true };
const { shift, ctrl, alt, meta } = event.modifiers;
if (meta) return null;
if (event.key === "Escape") {
// Plain Escape belongs to whatever takes keys (it is a keystroke in a
// terminal); with a modifier it always means "back out of here".
return { action: "double-click", overridesTyping: shift || ctrl || alt };
}
if (ctrl || alt) return null;
switch (event.key) {
case "ArrowUp":
return { action: "swipe-up", overridesTyping: false };
case "ArrowDown":
return { action: "swipe-down", overridesTyping: false };
case "ArrowLeft":
return { action: "swipe-left", overridesTyping: false };
case "ArrowRight":
return { action: "swipe-right", overridesTyping: false };
case "Enter":
return { action: "click", overridesTyping: false };
default:
return null;
}
}
/** The built-ins, for showing to the user. Keep in step with defaultKeyAction. */
export const DEFAULT_KEYS_HELP =
"Arrow keys: swipe up, down, left and right, as on the watch. Enter: select. Esc: back.\n\n" +
"In a terminal session those keys are typed instead, and Shift+Esc (or Ctrl+Esc, Alt+Esc) goes back.\n\n" +
"Mouse or touchpad: the left button selects, the right button goes back, the middle button opens the system menu, " +
"and the wheel scrolls. These work in a terminal session too.\n\n" +
"A custom keybinding replaces these for its key and works everywhere, terminals included. " +
"\"Nothing\" turns a key's meaning here off without stopping it being typed.";
/**
* Whether holding the key repeats the gesture. Moving up and down a list
* does. Sideways swipes don't: depending on the layout they enter and leave
* things, and a held key would flap between the two. Selecting, backing out
* and menus act once per press.
*/
export function gestureRepeats(gesture: KeyGesture): boolean {
return gesture === "scroll-up" || gesture === "scroll-down" || gesture === "swipe-up" || gesture === "swipe-down";
}
export type KeyResolution = {
/** The key is spoken for: don't offer it to the focused window. */
consume: boolean;
/**
* The gesture to perform: at once when `consume`, otherwise only if no
* window takes the key. Null for none (including a held key's repeats of
* a gesture that doesn't repeat).
*/
gesture: KeyGesture | null;
};
/** What one key press (or auto-repeat) means, given the custom keybindings. */
export function resolveKey(event: KeyEvent, custom: ReadonlyMap<string, KeyAction>): KeyResolution {
const repeatable = (gesture: KeyGesture): KeyGesture | null =>
event.repeat && !gestureRepeats(gesture) ? null : gesture;
const bound = custom.get(chordId(chordOf(event)));
if (bound !== undefined && bound !== "none") return { consume: true, gesture: repeatable(bound) };
// "none" takes the built-in meaning away and leaves the key to be typed.
if (bound === "none") return { consume: false, gesture: null };
const builtIn = defaultKeyAction(event);
if (!builtIn) return { consume: false, gesture: null };
return { consume: builtIn.overridesTyping, gesture: repeatable(builtIn.action) };
}
export type GestureOrigin = "ring" | "watch";
/**
* Which device a keyboard's gesture is presented as. Swipes are the watch's
* and scrolls the ring's; the two schemes differ in places (an icon grid
* moves by cell for the watch, by row and then cell for the ring). A tap, a
* double tap or a hold exists in both, so it goes with whichever the
* keyboard last moved with: arrow keys make the keyboard a watch, a remote
* bound to scroll makes it a ring. Before any movement, the watch.
*/
export class GestureScheme {
private scheme: GestureOrigin = "watch";
originFor(gesture: KeyGesture): GestureOrigin {
if (gesture.startsWith("swipe-")) this.scheme = "watch";
else if (gesture.startsWith("scroll-")) this.scheme = "ring";
return this.scheme;
}
}
+566
View File
@@ -0,0 +1,566 @@
/**
* The phone side of the CFW keyboard bridge for one glasses session: the
* client, the remembered keyboard, reconnecting it, and what the UI shows.
* The session hands it field-131 records and a way to send controls; the
* rest stays in here. Modelled on the Glassly app's G2KeyboardBridge (MIT).
*
* The firmware keeps nothing across a reboot (it comes back with the bridge
* off), and holds the keyboard's connection only while told to. So the
* remembered keyboard is brought back whenever the glasses link comes up and
* whenever the keyboard drops, backing off while it stays away. The
* discovered handle mapping is good for one connection and is never stored.
* No platform imports: this runs under node in the tests.
*/
import { KeyboardClient, KeyboardClientError, type KeyboardStatus } from "./client";
import type { HidKeyEvent, HidPointerSample } from "./hid";
import {
KeyboardDm,
KeyboardError,
mergeKeyboardAdverts,
parseKeyboardAddress,
type KeyboardAdvert,
} from "./protocol";
export type SavedKeyboard = { address: string; addressType: number; name: string };
export type KeyboardBridgeState =
/** No glasses link that can host a keyboard. */
| "unavailable"
| "idle"
| "scanning"
| "connecting"
| "pairing"
| "configuring"
| "connected"
| "error";
/** A pairing step waiting on the user. */
export type KeyboardPairingPrompt =
/** Type this on the keyboard, then Enter. */
| { kind: "display"; pin: number }
/** Confirm both sides show this value (answerCompare). */
| { kind: "compare"; value: number }
/** The keyboard shows a code to enter here (answerPasskey). */
| { kind: "entry" };
export type KeyboardBridgeSnapshot = {
state: KeyboardBridgeState;
saved: SavedKeyboard | null;
/** Name of the keyboard being connected or in use. */
activeName: string;
error: string;
pairing: KeyboardPairingPrompt | null;
/** Advertisers from the last scan, HID devices first, strongest first. */
scanResults: KeyboardAdvert[];
/** A user-started scan or connect is running. */
busy: boolean;
/** The remembered keyboard is away and the bridge keeps trying to bring it back. */
waiting: boolean;
/** The connected keyboard has a touchpad or mouse buttons. */
pointer: boolean;
};
export type KeyboardBridgeHost = {
/** Queue a control record on the live session; false when there is none. */
send(record: Uint8Array): boolean;
loadSaved(): SavedKeyboard | null;
storeSaved(saved: SavedKeyboard | null): void;
/** Key transitions from the keyboard (including forced releases). */
onKeys(events: HidKeyEvent[]): void;
/** A report from the keyboard's touchpad or mouse (including a forced release of its buttons). */
onPointer(sample: HidPointerSample): void;
/** The keyboard stopped being usable; drop held-key state without releases. */
onKeyboardGone(): void;
/** Something in snapshot() changed. */
onChanged(): void;
/**
* A line for the log. `routine` marks the ones that repeat for as long as
* a remembered keyboard stays away, which only a verbose log wants.
*/
log(line: string, routine?: boolean): void;
/** A fresh six-digit pairing passkey from a good random source. */
randomPasskey?(): number;
/**
* Whether looking for an absent keyboard is worthwhile now. Each attempt
* has the glasses' radio listening for the keyboard for up to half a
* minute, so the owner limits it to when the glasses are in use, and calls
* retryNow() when that starts. Default: always.
*/
shouldReconnect?(): boolean;
};
/** Delay before each automatic attempt after a failure; the last one repeats. */
const RECONNECT_BACKOFF_MS = [2_000, 5_000, 10_000, 20_000, 30_000];
/** After the glasses link comes up, before the first attempt. */
const RESUME_DELAY_MS = 1_000;
/** After the keyboard drops (it may just be going to sleep). */
const DROP_RETRY_DELAY_MS = 2_000;
const RELEASE_POLLS = 20;
const RELEASE_POLL_MS = 150;
export class KeyboardBridge {
private client: KeyboardClient | null = null;
private available = false;
private state: KeyboardBridgeState = "unavailable";
private saved: SavedKeyboard | null;
private activeName = "";
private error = "";
private pairing: KeyboardPairingPrompt | null = null;
private scanResults: KeyboardAdvert[] = [];
/** A user-started scan/connect holds the bridge. */
private busy = false;
/** The keyboard is connected, encrypted and subscribed. */
private ready = false;
private userDisconnected = false;
/** The keyboard was let go while the glasses were out of reach; tell the firmware when they are back. */
private releaseOnLink = false;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private reconnectAttempt = 0;
/** Bumped whenever the client is replaced; work started for an older one stops quietly. */
private generation = 0;
constructor(private readonly host: KeyboardBridgeHost) {
this.saved = host.loadSaved();
}
snapshot(): KeyboardBridgeSnapshot {
return {
state: this.state,
saved: this.saved,
activeName: this.activeName,
error: this.error,
pairing: this.pairing,
scanResults: sortAdverts(this.scanResults),
busy: this.busy,
waiting: this.available && !!this.saved && !this.ready && !this.userDisconnected && !this.busy,
pointer: this.ready && this.client?.configuration?.pointer === true,
};
}
/** True while key events can arrive. */
isConnected(): boolean {
return this.ready;
}
// ---------- Link lifecycle (driven by the session owner) ----------
/** The glasses link is up on firmware with the bridge: bring the remembered keyboard back. */
linkReady(): void {
if (this.available) return;
this.available = true;
this.userDisconnected = false;
this.reconnectAttempt = 0;
this.error = "";
const client = this.replaceClient();
this.setState("idle");
if (this.saved && !this.releaseOnLink) {
this.scheduleReconnect(RESUME_DELAY_MS);
} else if (this.releaseOnLink && client) {
this.userDisconnected = !!this.saved;
client.query().then((status) => {
if (status.enabled || status.conn) return client.disable();
}).catch(() => {});
}
this.releaseOnLink = false;
}
/** The glasses link is gone. Only the remembered keyboard survives. */
linkLost(): void {
if (!this.available && !this.client) return;
this.available = false;
this.cancelReconnect();
this.generation++;
this.client?.close();
this.client = null;
this.busy = false;
this.pairing = null;
this.error = "";
this.dropKeyboard();
this.setState("unavailable");
}
/**
* The app is about to close the glasses link on purpose: tell the firmware
* to let the keyboard go, so it isn't left attached to glasses nobody is
* listening to. Best effort; nothing waits for the answer.
*/
linkClosing(): void {
const client = this.client;
if (!client || !this.available || !client.status.enabled) return;
client.disable().catch(() => {});
}
/** One field-131 payload from the session. */
accept(payload: Uint8Array): void {
this.client?.accept(payload);
}
/**
* The keyboard may be reachable now, or is wanted now (the display woke):
* if it is away and the bridge is waiting out a backoff, try at once.
*/
retryNow(): void {
if (!this.available || !this.saved || this.ready || this.userDisconnected || this.busy) return;
// No timer means an attempt is already in flight.
if (this.reconnectTimer === null) return;
this.reconnectAttempt = 0;
this.scheduleReconnect(0);
}
// ---------- Commands (the settings UI) ----------
/** Scan for about five seconds; results accumulate in the snapshot as they arrive. */
async scan(): Promise<KeyboardAdvert[]> {
const client = this.beginUserOperation();
const generation = this.generation;
this.scanResults = [];
this.error = "";
this.setState("scanning");
try {
await this.releaseLink(client);
await client.enable();
await client.scan();
if (generation === this.generation) this.setState("idle");
} catch (e) {
if (generation === this.generation) this.fail(e);
throw friendly(e);
} finally {
if (generation === this.generation) this.endUserOperation();
}
return sortAdverts(this.scanResults);
}
/** Connect, pair and subscribe a keyboard, then remember it. */
async connect(keyboard: SavedKeyboard): Promise<void> {
if (!parseKeyboardAddress(keyboard.address) || keyboard.addressType < 0 || keyboard.addressType > 3) {
throw new KeyboardClientError("Invalid keyboard address");
}
const client = this.beginUserOperation();
const generation = this.generation;
this.userDisconnected = false;
try {
await this.connectWith(client, generation, keyboard, false);
} finally {
if (generation === this.generation) this.endUserOperation();
}
}
/** Try the remembered keyboard again now. */
async reconnect(): Promise<void> {
if (!this.saved) throw new KeyboardClientError("No remembered keyboard");
await this.connect(this.saved);
}
/** Drop the keyboard but keep it remembered. No automatic reconnect until the next explicit connect or glasses session. */
async disconnect(): Promise<void> {
this.userDisconnected = true;
this.cancelReconnect();
const client = this.replaceClient();
// A scan or connect this interrupts no longer owns the bridge.
this.busy = false;
this.dropKeyboard();
this.pairing = null;
this.error = "";
if (!client) {
this.releaseOnLink = true;
this.setState("unavailable");
return;
}
this.setState("idle");
try {
await client.query();
await client.disable();
} catch (e) {
this.host.log(`keyboard disconnect: ${messageOf(e)}`);
}
}
/** Forget the remembered keyboard. The glasses keep their bond with it. */
async forget(): Promise<void> {
this.saved = null;
this.host.storeSaved(null);
await this.disconnect();
this.scanResults = [];
this.activeName = "";
this.host.onChanged();
}
/** Answer a "compare" pairing prompt. */
async answerCompare(accept: boolean): Promise<void> {
if (!this.client || this.pairing?.kind !== "compare") throw new KeyboardClientError("No pairing comparison is pending");
this.pairing = null;
this.host.onChanged();
await this.client.confirm(accept);
}
/** Answer an "entry" pairing prompt with the code the keyboard shows. */
async answerPasskey(pin: number): Promise<void> {
if (!this.client || this.pairing?.kind !== "entry") throw new KeyboardClientError("No pairing code is pending");
this.pairing = null;
this.host.onChanged();
await this.client.authenticate(pin);
}
// ---------- Internals ----------
private beginUserOperation(): KeyboardClient {
if (!this.available) throw new KeyboardClientError("Connect to G2 glasses running Faceclaw firmware first");
if (this.busy) throw new KeyboardClientError("Another keyboard operation is in progress");
// An automatic attempt may be mid-flight (a sleeping keyboard holds one
// for up to 30 s): a fresh client fails it at once and takes over.
this.cancelReconnect();
const client = this.replaceClient()!;
this.dropKeyboard();
this.pairing = null;
this.busy = true;
return client;
}
private endUserOperation(): void {
this.busy = false;
this.host.onChanged();
}
/** Close the current client and, while the link is up, start a fresh one. */
private replaceClient(): KeyboardClient | null {
this.generation++;
this.client?.close();
this.client = null;
if (!this.available) return null;
const generation = this.generation;
const live = () => generation === this.generation;
const client = new KeyboardClient(
(record) => live() && this.host.send(record),
{
keys: (events) => { if (live() && this.ready) this.host.onKeys(events); },
pointer: (sample) => { if (live() && this.ready) this.host.onPointer(sample); },
advert: (advert) => { if (live()) this.addAdvert(advert); },
status: (status) => { if (live()) this.onStatus(status); },
dm: (event, status) => { if (live()) this.onDm(event, status); },
passkeyDisplay: (pin) => { if (live()) this.prompt({ kind: "display", pin }); },
passkeyRequest: () => { if (live()) this.prompt({ kind: "entry" }); },
compareRequest: (value) => { if (live()) this.prompt({ kind: "compare", value }); },
servicesChanged: () => { if (live()) this.onServicesChanged(); },
error: (message) => { if (live()) this.host.log(`keyboard: ${message}`); },
log: (line) => { if (live()) this.host.log(`keyboard setup: ${line}`); },
},
{ randomPasskey: this.host.randomPasskey ? () => this.host.randomPasskey!() : undefined },
);
this.client = client;
return client;
}
private async connectWith(
client: KeyboardClient,
generation: number,
keyboard: SavedKeyboard,
resume: boolean,
): Promise<void> {
const live = () => generation === this.generation;
this.error = "";
this.activeName = keyboard.name;
this.setState("connecting");
try {
const status = await client.query();
if (resume && status.enabled && status.connected && status.encrypted) {
// The glasses kept the keyboard through a phone-side drop (or an app
// restart): only the handle mapping is missing.
this.host.log("keyboard still connected to the glasses; rediscovering");
} else {
await this.releaseLink(client);
await client.enable();
await client.connect(keyboard.addressType, parseKeyboardAddress(keyboard.address)!);
}
if (!live()) return;
this.pairing = null;
this.setState("configuring");
const config = await client.configure();
if (!live()) return;
this.ready = true;
this.reconnectAttempt = 0;
// The list was for choosing; it is stale from here on.
this.scanResults = [];
if (!sameKeyboard(this.saved, keyboard)) {
this.saved = { ...keyboard };
this.host.storeSaved(this.saved);
}
this.host.log(
`keyboard ready: ${keyboard.name || keyboard.address}, ${config.watched.length} of ` +
`${config.reports.filter((r) => r.type === 1).length} input reports subscribed` +
`${config.pointer ? ", with a pointer" : ""}, epoch ${config.epoch}`);
this.setState("connected");
} catch (e) {
if (!live()) return;
this.ready = false;
this.pairing = null;
if (resume) {
// An automatic attempt: a keyboard that is asleep or out of range is
// the ordinary case, not an error to put in front of the user.
this.host.log(`keyboard reconnect: ${messageOf(friendly(e))}`, true);
this.setState("idle");
} else {
this.fail(e);
}
throw friendly(e);
}
}
/**
* The firmware refuses SCAN and CONNECT while it still owns a keyboard
* slot, and an interrupted connect keeps initiating until the close lands.
* DISABLE is epoch-checked and a fresh client hasn't seen the epoch yet,
* so QUERY first (it isn't, and its answer carries the epoch), then wait
* for the slot to clear.
*/
private async releaseLink(client: KeyboardClient): Promise<void> {
const status = await client.query();
if (!status.enabled && status.conn === 0) return;
await client.disable();
for (let i = 0; i < RELEASE_POLLS; i++) {
if ((await client.query()).conn === 0) return;
await delay(RELEASE_POLL_MS);
}
throw new KeyboardClientError("The glasses are still closing the previous keyboard link; try again in a moment");
}
private cancelReconnect(): void {
if (this.reconnectTimer !== null) clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
private scheduleReconnect(delayMs: number): void {
this.cancelReconnect();
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
void this.attemptReconnect();
}, delayMs);
}
private async attemptReconnect(): Promise<void> {
const saved = this.saved;
const client = this.client;
if (!this.available || !client || !saved || this.userDisconnected || this.ready || this.busy) return;
if (this.host.shouldReconnect && !this.host.shouldReconnect()) {
this.parkFirmware(client);
this.scheduleReconnect(RECONNECT_BACKOFF_MS[RECONNECT_BACKOFF_MS.length - 1]!);
return;
}
const generation = this.generation;
try {
await this.connectWith(client, generation, saved, true);
} catch {
if (generation !== this.generation || !this.available || this.userDisconnected) return;
// Keyboards sleep, and the ring's own scan can hold the controller
// (BUSY): keep trying, less often, while the glasses stay up.
const backoff = RECONNECT_BACKOFF_MS[Math.min(this.reconnectAttempt, RECONNECT_BACKOFF_MS.length - 1)]!;
this.reconnectAttempt++;
this.scheduleReconnect(backoff);
}
}
/**
* While it is enabled the firmware bridge runs a 20 ms timer and reports
* its status every second, keyboard or no keyboard. When nothing will be
* tried for a while, switch it off; the next attempt enables it again.
*/
private parkFirmware(client: KeyboardClient): void {
if (!client.status.enabled) return;
client.disable().catch(() => {});
}
private onStatus(status: KeyboardStatus): void {
if (this.ready && (!status.connected || !status.encrypted)) this.keyboardDropped("status");
else if (this.state === "connecting" && status.connected && !status.encrypted) this.setState("pairing");
}
private onDm(event: number, status: number): void {
if (event === KeyboardDm.connClose || event === KeyboardDm.pairFailed || event === KeyboardDm.encryptFailed) {
this.host.log(`keyboard DM event 0x${event.toString(16)} status ${status}`);
if (this.ready) this.keyboardDropped("dm");
}
}
private onServicesChanged(): void {
this.host.log("keyboard services changed; rediscovering");
this.dropKeyboard();
this.setState("idle");
if (this.saved && !this.userDisconnected && !this.busy) this.scheduleReconnect(0);
}
private keyboardDropped(reason: string): void {
this.host.log(`keyboard disconnected (${reason})`);
this.dropKeyboard();
this.setState("idle");
if (this.saved && !this.userDisconnected && !this.busy) this.scheduleReconnect(DROP_RETRY_DELAY_MS);
}
/** The keyboard is no longer usable: tell the host once. */
private dropKeyboard(): void {
if (!this.ready) return;
this.ready = false;
this.host.onKeyboardGone();
}
private addAdvert(advert: KeyboardAdvert): void {
const index = this.scanResults.findIndex(
(a) => a.address === advert.address && a.addressType === advert.addressType);
if (index < 0) {
this.scanResults.push(advert);
} else {
const merged = mergeKeyboardAdverts(this.scanResults[index]!, advert);
const old = this.scanResults[index]!;
if (merged.name === old.name && merged.hid === old.hid && merged.rssi === old.rssi) return;
this.scanResults[index] = merged;
}
this.host.onChanged();
}
private prompt(pairing: KeyboardPairingPrompt): void {
this.pairing = pairing;
this.setState("pairing");
}
private setState(next: KeyboardBridgeState): void {
this.state = next;
if (next !== "pairing" && next !== "connecting") this.pairing = null;
this.host.onChanged();
}
private fail(error: unknown): void {
this.error = messageOf(friendly(error));
this.host.log(`keyboard error: ${this.error}`);
this.setState("error");
}
}
function sameKeyboard(a: SavedKeyboard | null, b: SavedKeyboard): boolean {
return !!a && a.address === b.address && a.addressType === b.addressType && a.name === b.name;
}
function sortAdverts(adverts: readonly KeyboardAdvert[]): KeyboardAdvert[] {
return [...adverts].sort((a, b) => Number(b.hid) - Number(a.hid) || b.rssi - a.rssi);
}
/**
* Firmware BUSY nearly always means the ring's own scan or reconnect holds
* the radio; say that rather than the raw control error.
*/
function friendly(error: unknown): Error {
if (error instanceof KeyboardClientError && error.code === KeyboardError.busy) {
return new KeyboardClientError(
"The glasses' radio is busy (usually the ring reconnecting). Wait a few seconds and try again.", error.code);
}
if (error instanceof KeyboardClientError && error.code === KeyboardError.noSpace) {
return new KeyboardClientError(
"The glasses have no room for another connection or pairing.", error.code);
}
return error instanceof Error ? error : new Error(String(error));
}
function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function delay(ms: number): Promise<void> {
return new Promise<void>((resolve) => setTimeout(resolve, ms));
}
+754
View File
@@ -0,0 +1,754 @@
/**
* Transport-independent client for the CFW keyboard bridge: enable, scan,
* connect and pair a keyboard through the glasses, discover its HID service
* over the relayed ATT operations, subscribe its input reports, and decode
* them into key transitions. From the reference client in g2flash demos/
* (keyboard-client.ts), with the fixes its use in the Glassly app turned up.
*
* The firmware keeps almost no policy: it owns the connection and the bond,
* and forwards one ATT request at a time. Everything HID lives here. Handle
* numbers found by configure() are only good for one connection epoch.
* No platform imports: this runs under node in the tests.
*/
import {
HidKeys,
parseReportMap,
reportHasKeys,
reportHasPointer,
reportPointer,
type HidKeyEvent,
type HidPointerSample,
type HidReport,
} from "./hid";
import {
KeyboardDm,
KeyboardError,
KeyboardKind,
KeyboardOp,
KeyboardRecords,
keyboardControl,
keyboardErrorName,
le16,
parseKeyboardAdvert,
u16,
u32,
type KeyboardAdvert,
type KeyboardRecord,
} from "./protocol";
/** Queue one field-130 control record; false when the glasses link can't take it now. */
export type KeyboardTransport = (record: Uint8Array) => boolean;
export type KeyboardStatus = {
enabled: boolean;
/** The firmware's connection id for the keyboard, 0 when it owns none. */
conn: number;
connected: boolean;
encrypted: boolean;
scanning: boolean;
};
export type KeyboardClientEvents = {
/** Key transitions from one input report (or the releases a gap forces). */
keys?(events: HidKeyEvent[]): void;
/**
* One report from the keyboard's touchpad or mouse: the buttons now held
* (across all its pointer reports) and the movement and wheel since the
* last one. Also sent with everything zero when a gap forces a release.
*/
pointer?(sample: HidPointerSample): void;
advert?(advert: KeyboardAdvert): void;
status?(status: KeyboardStatus): void;
/** The stream may have lost records; held keys were released. */
gap?(): void;
/** Device-manager event from the glasses' BLE host (KeyboardDm). */
dm?(event: number, status: number): void;
/** The keyboard shows a six-digit passkey; answer with authenticate(). */
passkeyRequest?(): void;
/** We chose a passkey and answered with it; the user types it on the keyboard, then Enter. */
passkeyDisplay?(pin: number): void;
/** Numeric comparison: answer with confirm() once the user has compared. */
compareRequest?(value: number): void;
/** The keyboard's GATT table changed; the mapping is void until configure() runs again. */
servicesChanged?(): void;
error?(message: string): void;
/** Setup diagnostics: the keyboard's attribute table and what was made of it. Never key contents. */
log?(line: string): void;
};
export type KeyboardReport = {
/** Handle of the HID service declaration the report belongs to. */
service: number;
handle: number;
id: number;
/** 1 input, 2 output, 3 feature. */
type: number;
ccc?: number;
/** From the characteristic's declaration; undefined when the keyboard wouldn't say (see declaredProperties). */
properties?: number;
map: Uint8Array;
};
export type KeyboardConfiguration = {
epoch: number;
/** Every report found, input or not. */
reports: KeyboardReport[];
/** Handles of the input reports subscribed: those that carry keys, or a pointer's buttons and movement. */
watched: number[];
/** Whether any of them is a pointer's (a touchpad, mouse buttons, a wheel). */
pointer: boolean;
serviceChanged?: number;
};
export type KeyboardClientOptions = {
/** A fresh six-digit passkey for display pairing. Default: Math.random (pass a stronger source). */
randomPasskey?: () => number;
};
export class KeyboardClientError extends Error {
/** The firmware's control error code (KeyboardError), when that is what failed. */
readonly code: number | undefined;
/** The keyboard's ATT error code, when it refused a read or write. */
readonly attStatus: number | undefined;
constructor(message: string, code?: number, attStatus?: number) {
super(message);
this.name = "KeyboardClientError";
this.code = code;
this.attStatus = attStatus;
}
}
type Attribute = { handle: number; uuid: number | string };
type Pending = {
op: number;
resolve: (r: KeyboardRecord) => void;
reject: (e: Error) => void;
timer: ReturnType<typeof setTimeout>;
};
type Waiter = { test: () => boolean; resolve: () => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> };
type Input = { report: HidReport; id: number; service: number; keys: boolean; pointer: boolean };
const COMMAND_TIMEOUT_MS = 32_000;
const MAX_PENDING = 8;
const MAX_ATTRIBUTES = 512;
const MAX_VALUE = 512;
const MAX_WATCHES = 16;
const UUID_PRIMARY_SERVICE = 0x2800;
const UUID_SECONDARY_SERVICE = 0x2801;
const UUID_CHARACTERISTIC = 0x2803;
const UUID_CCC = 0x2902;
const UUID_EXTERNAL_REPORT_REFERENCE = 0x2907;
const UUID_REPORT_REFERENCE = 0x2908;
const UUID_HID_SERVICE = 0x1812;
const UUID_SERVICE_CHANGED = 0x2a05;
const UUID_REPORT_MAP = 0x2a4b;
const UUID_REPORT = 0x2a4d;
const UUID_PROTOCOL_MODE = 0x2a4e;
/** ATT error codes that end a paged read rather than failing it. */
const ATT_INVALID_OFFSET = 7;
const ATT_NOT_LONG = 11;
const ATT_NOT_FOUND = 10;
/**
* ATT UUIDs are little endian. A 128-bit UUID on the Bluetooth base collapses
* to its 16-bit form; vendor UUIDs stay as hex strings.
*/
export function attributeUuid(bytes: Uint8Array): number | string {
if (bytes.length === 2) return u16(bytes);
if (bytes.length !== 16) throw new KeyboardClientError("Invalid ATT UUID length");
const base = [0xfb, 0x34, 0x9b, 0x5f, 0x80, 0, 0, 0x80, 0, 0x10, 0, 0];
if (base.every((v, i) => bytes[i] === v) && bytes[14] === 0 && bytes[15] === 0) return u16(bytes, 12);
return Array.from(bytes, (x) => x.toString(16).padStart(2, "0")).join("");
}
export class KeyboardClient {
/** The firmware's connection generation; every control but QUERY/ENABLE must carry the live one. */
epoch = 0;
status: KeyboardStatus = { enabled: false, conn: 0, connected: false, encrypted: false, scanning: false };
configuration: KeyboardConfiguration | undefined;
private serial = 0;
private closed = false;
/** The link under connect() went away (close, failed open, failed pairing). */
private linkLost = false;
private readonly pending = new Map<number, Pending>();
private readonly waiters = new Set<Waiter>();
private readonly keys = new HidKeys();
/** Pointer buttons held, by the report that says so (as HidKeys does for keys). */
private readonly pointerButtons = new Map<number, number>();
private inputs = new Map<number, Input>();
private readonly records = new KeyboardRecords(() => this.gap());
private readonly randomPasskey: () => number;
constructor(
private readonly transport: KeyboardTransport,
private readonly events: KeyboardClientEvents = {},
options: KeyboardClientOptions = {},
) {
this.randomPasskey = options.randomPasskey ?? (() => Math.floor(Math.random() * 1_000_000));
}
// ---------- Inbound ----------
/** Feed one field-131 payload, in arrival order. */
accept(data: Uint8Array): void {
if (this.closed) return;
const r = this.records.push(data);
if (!r) return;
this.epoch = r.epoch;
if (r.kind === KeyboardKind.status && r.data.length === 10) {
this.status = {
enabled: !!r.data[0], conn: r.data[1]!, connected: !!r.data[2], encrypted: !!r.data[3], scanning: !!r.data[4],
};
if (!this.status.enabled || !this.status.connected || !this.status.encrypted) this.release();
this.events.status?.(this.status);
}
const pending = this.pending.get(r.request);
if (pending && (r.kind === KeyboardKind.status || r.kind === KeyboardKind.att)) {
this.pending.delete(r.request);
clearTimeout(pending.timer);
if (r.kind === KeyboardKind.status && r.code) {
pending.reject(new KeyboardClientError(
`Keyboard command ${opName(pending.op)}: firmware ${keyboardErrorName(r.code)}`, r.code));
} else {
pending.resolve(r);
}
}
if (r.kind === KeyboardKind.dm) this.acceptDm(r);
else if (r.kind === KeyboardKind.input) this.acceptInput(r);
else if (r.kind === KeyboardKind.advert) {
const advert = parseKeyboardAdvert(r.data);
if (advert) this.events.advert?.(advert);
}
this.checkWaiters();
}
private acceptDm(r: KeyboardRecord): void {
const event = r.code & 255;
const status = r.code >>> 8;
if (event === KeyboardDm.connClose || event === KeyboardDm.pairFailed || event === KeyboardDm.encryptFailed ||
(event === KeyboardDm.connOpen && status !== 0)) {
this.linkLost = true;
this.gap();
this.inputs.clear();
this.configuration = undefined;
}
if (event === KeyboardDm.authRequest && r.data.length === 2) {
if (r.data[0]) {
this.command(KeyboardOp.disconnect).catch(() => {});
this.events.error?.("This keyboard only pairs out of band, which isn't supported.");
} else if (r.data[1]) {
// KeyboardDisplay on our side: choose the passkey, show it, and
// answer at once; the user types it on the keyboard.
const pin = Math.min(999_999, Math.max(0, Math.floor(this.randomPasskey())));
this.events.passkeyDisplay?.(pin);
this.authenticate(pin).catch((e) => this.events.error?.(messageOf(e)));
} else {
this.events.passkeyRequest?.();
}
}
if (event === KeyboardDm.compare && r.data.length === 4) this.events.compareRequest?.(u32(r.data));
this.events.dm?.(event, status);
}
private acceptInput(r: KeyboardRecord): void {
const changed = this.configuration?.serviceChanged;
if (changed !== undefined && r.handle === changed) {
this.gap();
this.inputs.clear();
this.configuration = undefined;
this.events.servicesChanged?.();
return;
}
const input = this.inputs.get(r.handle);
if (!input || this.configuration?.epoch !== r.epoch) {
this.gap();
return;
}
if (input.keys) {
try {
const events = this.keys.update(r.handle, input.report, r.data);
if (events.length) this.events.keys?.(events);
} catch (e) {
this.release();
this.events.error?.(messageOf(e));
}
}
if (input.pointer) {
const sample = reportPointer(input.report, r.data);
this.pointerButtons.set(r.handle, sample.buttons);
this.events.pointer?.({ ...sample, buttons: this.heldButtons() });
}
}
private heldButtons(): number {
let buttons = 0;
for (const held of this.pointerButtons.values()) buttons |= held;
return buttons;
}
/** Nothing is held any more, as far as anyone can tell: say so. */
private release(): void {
const events = this.keys.reset();
if (events.length) this.events.keys?.(events);
const buttons = this.heldButtons();
this.pointerButtons.clear();
if (buttons) this.events.pointer?.({ buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 });
}
private gap(): void {
this.release();
this.events.gap?.();
for (const [id, p] of this.pending) {
if (p.op < KeyboardOp.find) continue;
clearTimeout(p.timer);
this.pending.delete(id);
p.reject(new KeyboardClientError("Keyboard stream gap during ATT transaction"));
}
}
/** The phone's link to the glasses dropped: release held keys and fail everything in flight. */
lostPhoneLink(): void {
this.records.reset();
this.cancel("Glasses link lost");
}
/** Stop for good; later accept() calls are ignored. */
close(): void {
if (this.closed) return;
this.lostPhoneLink();
this.closed = true;
}
private cancel(reason: string): void {
for (const p of this.pending.values()) {
clearTimeout(p.timer);
p.reject(new KeyboardClientError(reason));
}
this.pending.clear();
for (const w of this.waiters) {
clearTimeout(w.timer);
w.reject(new KeyboardClientError(reason));
}
this.waiters.clear();
}
private checkWaiters(): void {
for (const w of [...this.waiters]) {
let done = false;
try {
done = w.test();
} catch (e) {
this.waiters.delete(w);
clearTimeout(w.timer);
w.reject(e instanceof Error ? e : new KeyboardClientError(String(e)));
continue;
}
if (!done) continue;
this.waiters.delete(w);
clearTimeout(w.timer);
w.resolve();
}
}
/** Resolve once `test` holds; it is re-evaluated after every accepted record. */
private waitFor(test: () => boolean, timeoutMs: number, what: string): Promise<void> {
if (this.closed) return Promise.reject(new KeyboardClientError("Keyboard client closed"));
if (test()) return Promise.resolve();
return new Promise<void>((resolve, reject) => {
const waiter: Waiter = {
test, resolve, reject,
timer: setTimeout(() => {
this.waiters.delete(waiter);
reject(new KeyboardClientError(`Timed out waiting for ${what}`));
}, timeoutMs),
};
this.waiters.add(waiter);
});
}
// ---------- Commands ----------
/** Send one control and await the STATUS or ATT record that answers it. */
command(op: number, body: readonly number[] = []): Promise<KeyboardRecord> {
if (this.closed) return Promise.reject(new KeyboardClientError("Keyboard client closed"));
if (this.pending.size >= MAX_PENDING) return Promise.reject(new KeyboardClientError("Too many keyboard commands"));
const request = this.serial = this.serial % 65535 + 1;
return new Promise<KeyboardRecord>((resolve, reject) => {
const timer = setTimeout(() => {
this.pending.delete(request);
reject(new KeyboardClientError(`Keyboard command ${opName(op)} timed out`));
}, COMMAND_TIMEOUT_MS);
this.pending.set(request, { op, resolve, reject, timer });
let queued = false;
try {
queued = this.transport(keyboardControl(op, request, this.epoch, body));
} catch {
queued = false;
}
if (!queued) {
clearTimeout(timer);
this.pending.delete(request);
reject(new KeyboardClientError("The glasses aren't ready for keyboard commands"));
}
});
}
async enable(): Promise<void> {
await this.command(KeyboardOp.enable);
}
/** Disconnect the keyboard and stop the bridge; the bond stays. */
async disable(): Promise<void> {
await this.command(KeyboardOp.disable);
this.release();
}
/** Ask for the firmware's state. Also the way to learn the live epoch: QUERY isn't epoch-checked. */
async query(): Promise<KeyboardStatus> {
await this.command(KeyboardOp.query);
return this.status;
}
/**
* Active scan for about five seconds; adverts arrive through events.advert.
* The firmware only clears `scanning` once the stock scanner is idle again,
* and a ring looking for its lens can hold that well past the window, so a
* lingering flag is not an error: the adverts already delivered stand.
*/
async scan(): Promise<void> {
await this.command(KeyboardOp.scan);
try {
await this.waitFor(() => !this.status.scanning, 8000, "the scan to finish");
} catch (e) {
if (this.closed) throw e;
}
}
/** Connect, then pair unless the link comes up encrypted. `address` is LSB first. */
async connect(addressType: number, address: Uint8Array): Promise<void> {
if (address.length !== 6 || !Number.isInteger(addressType) || addressType < 0 || addressType > 3) {
throw new KeyboardClientError("Invalid BLE address");
}
this.release();
this.inputs.clear();
this.configuration = undefined;
this.linkLost = false;
await this.command(KeyboardOp.connect, [addressType, ...address]);
await this.waitFor(() => this.status.connected || this.linkLost, 32_000, "the keyboard to connect");
if (!this.status.connected) throw new KeyboardClientError("The keyboard didn't answer");
if (!this.status.encrypted) {
// The keyboard's own security request may have started pairing already.
// BUSY is not success: the encrypted status below still decides.
try {
await this.command(KeyboardOp.pair);
} catch (e) {
if (!(e instanceof KeyboardClientError) || e.code !== KeyboardError.busy) throw e;
}
await this.waitFor(() => this.status.encrypted || this.linkLost, 65_000, "pairing to finish");
if (!this.status.encrypted) throw new KeyboardClientError("Pairing with the keyboard failed");
}
}
async disconnect(): Promise<void> {
await this.command(KeyboardOp.disconnect);
this.release();
}
/** Answer a pending passkey prompt. */
async authenticate(pin: number): Promise<void> {
if (!Number.isInteger(pin) || pin < 0 || pin > 999_999) throw new KeyboardClientError("Invalid pairing passkey");
await this.command(KeyboardOp.auth, [pin & 255, (pin >>> 8) & 255, pin >>> 16]);
}
/** Answer a pending numeric comparison. */
async confirm(accept: boolean): Promise<void> {
await this.command(KeyboardOp.compare, [accept ? 1 : 0]);
}
private async att(op: number, body: readonly number[]): Promise<KeyboardRecord> {
const r = await this.command(op, body);
if (r.kind !== KeyboardKind.att) throw new KeyboardClientError("Expected keyboard ATT response");
return r;
}
/** Read a whole attribute: Read, then Read Blob until a short or empty reply. Never assumes the MTU. */
async read(handle: number): Promise<Uint8Array> {
const parts: number[] = [];
for (;;) {
const r = await this.att(KeyboardOp.read, [...le16(handle), ...le16(parts.length)]);
const status = r.code >>> 8;
if (parts.length && (status === ATT_INVALID_OFFSET || status === ATT_NOT_LONG)) break;
if (status) throw new KeyboardClientError(`ATT read ${handle}: ${status}`, undefined, status);
if (!r.data.length) break;
for (const b of r.data) parts.push(b);
if (parts.length > MAX_VALUE) throw new KeyboardClientError("Attribute exceeds 512 bytes");
if (parts.length === MAX_VALUE) break;
}
return Uint8Array.from(parts);
}
/** Read a short attribute once (a Report Reference, a declaration, a service UUID). */
private async shortRead(handle: number): Promise<Uint8Array> {
const r = await this.att(KeyboardOp.read, [...le16(handle), 0, 0]);
if (r.code >>> 8) throw new KeyboardClientError(`ATT read ${handle}: ${r.code >>> 8}`, undefined, r.code >>> 8);
return r.data;
}
/** Write 1..18 bytes (an output report such as the LEDs, a CCC, the protocol mode). */
async write(handle: number, value: Uint8Array, withoutResponse = false): Promise<void> {
if (!value.length || value.length > 18) throw new KeyboardClientError("Keyboard write supports 1..18 bytes");
const r = await this.att(KeyboardOp.write, [...le16(handle), withoutResponse ? 1 : 0, value.length, ...value]);
if (r.code >>> 8) throw new KeyboardClientError(`ATT write ${handle}: ${r.code >>> 8}`, undefined, r.code >>> 8);
}
private async attributes(): Promise<Attribute[]> {
const attrs: Attribute[] = [];
let start = 1;
while (start <= 65535) {
const r = await this.att(KeyboardOp.find, [...le16(start), 255, 255]);
const status = r.code >>> 8;
if (status === ATT_NOT_FOUND) break; // the normal end of the table
if (status) throw new KeyboardClientError(`ATT Find Information: ${status}`);
const width = r.data[0] === 1 ? 4 : r.data[0] === 2 ? 18 : 0;
if (!width || r.data.length <= 1 || (r.data.length - 1) % width) {
throw new KeyboardClientError("Malformed Find Information response");
}
for (let i = 1; i < r.data.length; i += width) {
const handle = u16(r.data, i);
if (handle < start || attrs.length >= MAX_ATTRIBUTES) throw new KeyboardClientError("Invalid or excessive GATT handles");
attrs.push({ handle, uuid: attributeUuid(r.data.subarray(i + 2, i + width)) });
start = handle + 1;
}
}
return attrs;
}
/**
* The properties byte of a characteristic, from its declaration, or
* undefined when the keyboard won't say. A GATT client normally learns
* declarations through Read By Type, which the bridge doesn't relay, so
* this reads the declaration attribute itself. Most stacks answer that
* with the full declaration. Some keep only the properties byte there and
* build the rest for Read By Type alone (older Telink SDKs, common in
* small keyboards), and some refuse the read. None of that is a reason to
* turn a keyboard away: the report's handle and UUID are already known.
*/
private async declaredProperties(attrs: readonly Attribute[], a: Attribute): Promise<number | undefined> {
const decl = attrs.filter((d) => d.handle < a.handle && d.uuid === UUID_CHARACTERISTIC).pop();
if (!decl) return undefined;
let declaration: Uint8Array;
try {
declaration = await this.shortRead(decl.handle);
} catch (e) {
const status = attRefusal(e);
this.events.log?.(`declaration ${decl.handle} of ${a.handle}: read refused (ATT ${status})`);
return undefined;
}
if (declaration.length === 5 || declaration.length === 19) {
if (u16(declaration, 1) === a.handle && attributeUuid(declaration.subarray(3)) === a.uuid) return declaration[0]!;
this.events.log?.(`declaration ${decl.handle} of ${a.handle}: describes something else (${hex(declaration)})`);
return undefined;
}
if (declaration.length === 1) return declaration[0]!;
this.events.log?.(`declaration ${decl.handle} of ${a.handle}: unexpected value (${hex(declaration)})`);
return undefined;
}
/**
* Discover every HID service, its Report Map and each report's Report
* Reference; select Report Protocol and subscribe the input reports that
* carry keys or a pointer's buttons and movement. Needs an encrypted link.
* No handle is assumed.
*
* Only what input needs is required of the device: a Report Map that
* parses, and at least one such input report with a Report Reference and
* a CCC. A report that can't be placed is skipped and logged, as is
* anything wrong with an external report (the battery level, usually).
*/
async configure(): Promise<KeyboardConfiguration> {
const epoch = this.epoch;
const log = (line: string): void => this.events.log?.(line);
// A new discovery replaces any earlier mapping, whether or not it succeeds.
this.release();
this.inputs.clear();
this.configuration = undefined;
const attrs = await this.attributes();
log(`gatt ${attrs.map((a) => `${a.handle}=${typeof a.uuid === "number" ? a.uuid.toString(16) : a.uuid}`).join(" ")}`);
const reports: KeyboardReport[] = [];
const watched: KeyboardReport[] = [];
let pointerFound = false;
const skipped: string[] = [];
const descriptorsFor = (handle: number): Attribute[] => {
const end = attrs.find((a) => a.handle > handle &&
(a.uuid === UUID_PRIMARY_SERVICE || a.uuid === UUID_SECONDARY_SERVICE || a.uuid === UUID_CHARACTERISTIC))?.handle ?? 65536;
return attrs.filter((a) => a.handle > handle && a.handle < end);
};
/** Why a report was left out; the link going away is not a reason to carry on. */
const skip = (a: Attribute, reason: string): void => {
skipped.push(`${a.handle}: ${reason}`);
log(`report ${a.handle} skipped: ${reason}`);
};
const addReport = async (service: number, a: Attribute, map: Uint8Array, parsed: Map<number, HidReport>) => {
if (reports.some((r) => r.handle === a.handle)) return skip(a, "already listed by another HID service");
const descriptors = descriptorsFor(a.handle);
const ref = descriptors.find((d) => d.uuid === UUID_REPORT_REFERENCE);
if (!ref) return skip(a, "no Report Reference descriptor");
let reference: Uint8Array;
try {
reference = await this.shortRead(ref.handle);
} catch (e) {
return skip(a, `Report Reference ${ref.handle} unreadable (ATT ${attRefusal(e)})`);
}
if (reference.length !== 2) return skip(a, `Report Reference ${ref.handle} is ${hex(reference) || "empty"}`);
const id = reference[0]!;
const type = reference[1]!;
if (type < 1 || type > 3) return skip(a, `Report Reference type ${type}`);
const ccc = descriptors.find((d) => d.uuid === UUID_CCC)?.handle;
const properties = await this.declaredProperties(attrs, a);
const report: KeyboardReport = { service, handle: a.handle, id, type, ccc, properties, map };
if (type === 1) {
const layout = parsed.get(id);
if (!layout) return skip(a, `input report id ${id} is not in the Report Map`);
if (ccc === undefined || (properties !== undefined && !(properties & 0x10))) return skip(a, "input report can't notify");
const keys = reportHasKeys(layout);
const pointer = reportHasPointer(layout);
if (keys || pointer) watched.push(report);
else log(`report ${a.handle} (id ${id}) carries no keys or pointer; not subscribed`);
if (pointer) {
pointerFound = true;
log(`report ${a.handle} (id ${id}) is a pointer's${keys ? ", with keys" : ""}`);
}
}
reports.push(report);
log(`report ${a.handle}: id ${id} type ${type} ccc ${ccc ?? "-"} properties ${properties ?? "?"}`);
};
for (let i = 0; i < attrs.length; i++) {
const service = attrs[i]!;
if (service.uuid !== UUID_PRIMARY_SERVICE) continue;
if (attributeUuid(await this.shortRead(service.handle)) !== UUID_HID_SERVICE) continue;
const end = attrs.slice(i + 1).find((a) => a.uuid === UUID_PRIMARY_SERVICE || a.uuid === UUID_SECONDARY_SERVICE)?.handle ?? 65536;
const members = attrs.filter((a) => a.handle > service.handle && a.handle < end);
const mapAttr = members.find((a) => a.uuid === UUID_REPORT_MAP);
if (!mapAttr) throw new KeyboardClientError(`HID service ${service.handle} has no Report Map`);
const map = await this.read(mapAttr.handle);
log(`report map ${mapAttr.handle}: ${hex(map)}`);
const parsed = parseReportMap(map);
for (const mode of members.filter((a) => a.uuid === UUID_PROTOCOL_MODE)) {
try {
await this.write(mode.handle, Uint8Array.of(1), true);
} catch (e) {
// Report Protocol is the default after a connection; a keyboard
// that won't take the write is almost certainly in it already.
log(`protocol mode ${mode.handle}: write refused (ATT ${attRefusal(e)})`);
}
}
for (const a of members.filter((m) => m.uuid === UUID_REPORT)) await addReport(service.handle, a, map, parsed);
// HOGP lets a Report Map refer to reports in other services (commonly Battery).
for (const ext of descriptorsFor(mapAttr.handle).filter((a) => a.uuid === UUID_EXTERNAL_REPORT_REFERENCE)) {
let value: Uint8Array;
try {
value = await this.shortRead(ext.handle);
} catch (e) {
log(`external report reference ${ext.handle} skipped: unreadable (ATT ${attRefusal(e)})`);
continue;
}
if (value.length !== 2 && value.length !== 16) {
log(`external report reference ${ext.handle} skipped: value ${hex(value) || "empty"}`);
continue;
}
const uuid = attributeUuid(value);
for (const a of attrs.filter((m) => m.uuid === uuid && (m.handle < service.handle || m.handle >= end))) {
await addReport(service.handle, a, map, parsed);
}
}
}
if (this.closed || this.epoch !== epoch) throw new KeyboardClientError("The keyboard link changed during setup");
if (!watched.length) {
throw new KeyboardClientError(reports.length || skipped.length
? `No usable keyboard input report (${skipped.join("; ") || "none carries keys or a pointer"})`
: "This device has no HID service");
}
if (watched.length > MAX_WATCHES) throw new KeyboardClientError("Keyboard has more than 16 input reports");
const changed = attrs.find((a) => a.uuid === UUID_SERVICE_CHANGED);
const changedCcc = changed && descriptorsFor(changed.handle).find((a) => a.uuid === UUID_CCC);
const watches = watched.map((r) => r.handle);
if (changed && changedCcc && watches.length < MAX_WATCHES) watches.push(changed.handle);
const serviceChanged = changed && changedCcc && watches.includes(changed.handle) ? changed.handle : undefined;
const config: KeyboardConfiguration = {
epoch: this.epoch, reports, watched: watched.map((r) => r.handle), pointer: pointerFound, serviceChanged,
};
this.restore(config);
try {
await this.command(KeyboardOp.watch, [watches.length, ...watches.flatMap(le16)]);
if (serviceChanged !== undefined) {
try {
await this.write(changedCcc!.handle, Uint8Array.of(2, 0));
} catch (e) {
log(`service changed ${changedCcc!.handle}: subscription refused (ATT ${attRefusal(e)})`);
}
}
for (const r of watched) await this.write(r.ccc!, Uint8Array.of(1, 0));
} catch (e) {
this.inputs.clear();
this.configuration = undefined;
this.release();
throw e;
}
return config;
}
/**
* Install a mapping found by configure(). It is only good while the glasses
* keep the same connection (the same epoch): after a keyboard reconnect or
* a firmware boot, discover again.
*/
restore(config: KeyboardConfiguration): void {
if (!config.reports.length || config.reports.length > 64) throw new KeyboardClientError("Invalid keyboard configuration");
const inputs = new Map<number, Input>();
for (const r of config.reports) {
if (r.type !== 1 || !config.watched.includes(r.handle)) continue;
if (!Number.isInteger(r.handle) || r.handle < 1 || r.handle > 65535 || inputs.has(r.handle)) {
throw new KeyboardClientError("Invalid restored Report handle");
}
if (!r.map.length || r.map.length > MAX_VALUE) throw new KeyboardClientError("Invalid restored Report Map");
const report = parseReportMap(r.map).get(r.id);
if (!report) throw new KeyboardClientError("Restored Report ID missing");
inputs.set(r.handle, { report, id: r.id, service: r.service, keys: reportHasKeys(report), pointer: reportHasPointer(report) });
}
if (!inputs.size || inputs.size > MAX_WATCHES) throw new KeyboardClientError("Invalid restored input count");
this.release();
this.inputs = inputs;
this.configuration = config;
}
/** Output reports of the configured keyboard (the LED report is the usual one). */
outputReports(): KeyboardReport[] {
return this.configuration?.reports.filter((r) => r.type === 2) ?? [];
}
}
/**
* The ATT error code when `error` is the keyboard refusing a read or write.
* Anything else (the link dropping, a timeout, a stream gap) is rethrown:
* those end the setup, where a refusal only costs the one attribute.
*/
function attRefusal(error: unknown): number {
if (error instanceof KeyboardClientError && error.attStatus !== undefined) return error.attStatus;
throw error;
}
function hex(bytes: Uint8Array): string {
return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
}
function opName(op: number): string {
for (const [name, value] of Object.entries(KeyboardOp)) if (value === op) return name;
return String(op);
}
function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
+268
View File
@@ -0,0 +1,268 @@
/**
* Bounded HID report-descriptor decoder for the keyboard bridge: turns a
* keyboard's Report Map into the layout of each input report, and input
* reports into the set of keys held. From the reference client in g2flash
* demos/ (keyboard-hid.ts).
*
* HID over GATT carries the Report ID out of band (in each characteristic's
* Report Reference descriptor), so byte zero of a report value is data: it is
* never stripped as USB HID would. The keyboard/keypad page (7) and the
* consumer page (12) produce keys. A mouse's or touchpad's report (buttons,
* relative X/Y, wheel) is read by reportPointer.
* No platform imports: this runs under node in the tests.
*/
export type HidUsage = { page: number; usage: number };
export type HidKeyEvent = HidUsage & { down: boolean };
type Global = { page: number; min: number; max: number; size: number; count: number; id: number };
type Field = {
bit: number;
size: number;
count: number;
variable: boolean;
/** Reports a change since the last report (a mouse's axes and wheel), not a state. */
relative: boolean;
min: number;
max: number;
usages: HidUsage[];
range?: [HidUsage, HidUsage];
};
export type HidReport = { id: number; bits: number; fields: Field[] };
const HID_PAGE_KEYBOARD = 7;
const HID_PAGE_CONSUMER = 12;
const REPORT_MAP_MAX = 512;
function fail(reason: string): never {
throw new Error(`Unsupported or malformed HID descriptor: ${reason}`);
}
const usageKey = (u: HidUsage): string => `${u.page}:${u.usage}`;
/** Input reports by Report ID (0 when the descriptor uses none). Throws on anything out of bounds. */
export function parseReportMap(bytes: Uint8Array): Map<number, HidReport> {
if (!bytes.length || bytes.length > REPORT_MAP_MAX) fail("report map must contain 1..512 bytes");
const reports = new Map<number, HidReport>();
let g: Global = { page: 0, min: 0, max: 0, size: 0, count: 0, id: 0 };
const stack: Global[] = [];
let usages: HidUsage[] = [];
let lo: HidUsage | undefined;
let hi: HidUsage | undefined;
let depth = 0;
const usage = (v: number, n: number): HidUsage =>
n === 4 ? { page: v >>> 16, usage: v & 65535 } : { page: g.page, usage: v };
for (let p = 0; p < bytes.length;) {
const prefix = bytes[p++]!;
if (prefix === 0xfe) fail("long items");
const n = [0, 1, 2, 4][prefix & 3]!;
if (p + n > bytes.length) fail("truncated item");
let v = 0;
for (let j = 0; j < n; j++) v += bytes[p++]! * 2 ** (8 * j);
const signed = n && v >= 2 ** (n * 8 - 1) ? v - 2 ** (n * 8) : v;
const type = (prefix >> 2) & 3;
const tag = prefix >> 4;
if (type === 1) {
switch (tag) {
case 0: g.page = v; break;
case 1: g.min = signed; break;
case 2: g.max = g.min < 0 ? signed : v; break;
case 7: g.size = v; break;
case 8: if (!v || v > 255) fail("Report ID"); g.id = v; break;
case 9: g.count = v; break;
case 10: if (stack.length >= 8) fail("global stack overflow"); stack.push({ ...g }); break;
case 11: if (!stack.length) fail("global stack underflow"); g = stack.pop()!; break;
case 3: case 4: case 5: case 6: break; // physical range, unit and exponent
default: fail("unknown global item");
}
} else if (type === 2) {
if (tag === 0) { if (usages.length >= 512) fail("too many usages"); usages.push(usage(v, n)); }
else if (tag === 1) lo = usage(v, n);
else if (tag === 2) hi = usage(v, n);
else if (tag === 10) fail("usage delimiters");
else if (tag > 9) fail("unknown local item");
} else if (type === 0) {
if (tag === 10) { if (++depth > 32) fail("collection depth"); }
else if (tag === 12) { if (--depth < 0) fail("collection underflow"); }
else if (tag === 8) {
if (!g.size || g.size > 32 || !g.count || g.count > 512) fail("field dimensions");
let r = reports.get(g.id);
if (!r) {
if (reports.size >= 16) fail("too many input reports");
r = { id: g.id, bits: 0, fields: [] };
reports.set(g.id, r);
}
const field: Field = {
bit: r.bits, size: g.size, count: g.count, variable: !!(v & 2), relative: !!(v & 4),
min: g.min, max: g.max, usages: [...usages],
};
if (lo || hi) {
if (!lo || !hi || lo.page !== hi.page || hi.usage < lo.usage) fail("usage range");
field.range = [lo!, hi!];
}
r.bits += g.size * g.count;
if (r.bits > 4096) fail("report exceeds 512 bytes");
// Constant fields are padding.
if (!(v & 1)) r.fields.push(field);
} else if (tag !== 9 && tag !== 11) fail("unknown main item");
usages = [];
lo = hi = undefined;
} else fail("reserved item type");
}
if (depth || stack.length) fail("unbalanced collection/global stack");
if (reports.has(0) && reports.size > 1) fail("mixed numbered and unnumbered input reports");
if (!reports.size) fail("no input reports");
return reports;
}
function usageAt(f: Field, index: number): HidUsage | undefined {
if (index < 0) return undefined;
if (index < f.usages.length) return f.usages[index];
const rangeIndex = index - f.usages.length;
if (f.range && rangeIndex <= f.range[1].usage - f.range[0].usage) {
return { page: f.range[0].page, usage: f.range[0].usage + rangeIndex };
}
// Variable items reuse their last usage when Report Count exceeds the usages.
if (f.variable) return f.range?.[1] ?? f.usages[f.usages.length - 1];
return undefined;
}
/**
* Whether an input report can ever say a key is held. False for a pointer's
* report (a trackpad or mouse on a combined device), whose movement would
* otherwise be relayed, and thrown away, a hundred times a second.
*/
export function reportHasKeys(report: HidReport): boolean {
const isKeyPage = (u: HidUsage): boolean => u.page === HID_PAGE_KEYBOARD || u.page === HID_PAGE_CONSUMER;
return report.fields.some((f) => !f.relative && (f.usages.some(isKeyPage) || (f.range !== undefined && isKeyPage(f.range[0]))));
}
/** One field's `index`th value, sign-extended when the field is signed. Bits past the data read as zero. */
function fieldValue(f: Field, index: number, data: Uint8Array): number {
let value = 0;
for (let j = 0; j < f.size; j++) {
const bit = f.bit + index * f.size + j;
if ((data[bit >>> 3] ?? 0) & (1 << (bit & 7))) value += 2 ** j;
}
if (f.min < 0 && value >= 2 ** (f.size - 1)) value -= 2 ** f.size;
return value;
}
/** What one report from a mouse or touchpad says. */
export type HidPointerSample = {
/** Buttons held, bit 0 for button 1 (left), bit 1 for button 2 (right), bit 2 for button 3 (middle). */
buttons: number;
/** Movement since the last report, in the device's counts; +x right, +y down. */
dx: number;
dy: number;
/** Wheel detents since the last report; positive is away from the user ("up"). */
wheel: number;
/** Horizontal wheel (AC Pan); positive is right. */
pan: number;
};
const HID_PAGE_GENERIC_DESKTOP = 1;
const HID_PAGE_BUTTON = 9;
const USAGE_X = 0x30;
const USAGE_Y = 0x31;
const USAGE_WHEEL = 0x38;
const USAGE_AC_PAN = 0x238;
/** Buttons past this are not tracked; a mouse with more is rare and has nothing to bind them to anyway. */
const MAX_POINTER_BUTTONS = 8;
/** Which part of a pointer sample a field's usage feeds, if any. */
function pointerRole(f: Field, u: HidUsage | undefined): "button" | "dx" | "dy" | "wheel" | "pan" | null {
if (!u || !f.variable) return null;
if (u.page === HID_PAGE_BUTTON) return !f.relative && u.usage >= 1 && u.usage <= MAX_POINTER_BUTTONS ? "button" : null;
if (!f.relative) return null; // an absolute X/Y is a touchscreen or a digitizer, not a mouse
if (u.page === HID_PAGE_GENERIC_DESKTOP) {
if (u.usage === USAGE_X) return "dx";
if (u.usage === USAGE_Y) return "dy";
if (u.usage === USAGE_WHEEL) return "wheel";
}
if (u.page === HID_PAGE_CONSUMER && u.usage === USAGE_AC_PAN) return "pan";
return null;
}
/** Whether an input report carries a mouse's buttons, movement or wheel. */
export function reportHasPointer(report: HidReport): boolean {
return report.fields.some((f) => {
for (let i = 0; i < f.count; i++) if (pointerRole(f, usageAt(f, i))) return true;
return false;
});
}
/**
* Read a mouse or touchpad report. Lenient about length, unlike reportKeys:
* some devices leave the wheel bytes off when there is nothing to say, and a
* missing byte is zero either way.
*/
export function reportPointer(report: HidReport, data: Uint8Array): HidPointerSample {
const sample: HidPointerSample = { buttons: 0, dx: 0, dy: 0, wheel: 0, pan: 0 };
for (const f of report.fields) {
for (let i = 0; i < f.count; i++) {
const u = usageAt(f, i);
const role = pointerRole(f, u);
if (!role) continue;
const value = fieldValue(f, i, data);
if (value < f.min || value > f.max) continue; // out of range: no reading
if (role === "button") {
if (value) sample.buttons |= 1 << (u!.usage - 1);
} else {
sample[role] += value;
}
}
}
return sample;
}
/** The keys one input report says are held. Throws on a length mismatch or a rollover report. */
export function reportKeys(report: HidReport, data: Uint8Array): HidUsage[] {
if (data.length !== Math.ceil(report.bits / 8)) throw new Error("HID input length does not match Report Map");
const keys = new Map<string, HidUsage>();
for (const f of report.fields) {
// A relative field is a movement, not a key that is down.
if (f.relative) continue;
for (let i = 0; i < f.count; i++) {
const value = fieldValue(f, i, data);
if (value < f.min || value > f.max) continue; // null state
const u = usageAt(f, f.variable ? i : value - f.min);
if (!u || (f.variable && value === 0) || !u.usage) continue;
if (u.page === HID_PAGE_KEYBOARD && u.usage <= 3) throw new Error("HID keyboard rollover/error report");
if (u.page === HID_PAGE_KEYBOARD || u.page === HID_PAGE_CONSUMER) keys.set(usageKey(u), u);
}
}
return [...keys.values()];
}
/**
* Held keys across a keyboard's input reports. Ownership is per report
* characteristic, then unioned: releasing a key in one report cannot release
* a modifier another report still holds.
*/
export class HidKeys {
private reports = new Map<number, HidUsage[]>();
private held = new Map<string, HidUsage>();
/** Apply one report from characteristic `handle`; returns the transitions it caused. */
update(handle: number, report: HidReport, data: Uint8Array): HidKeyEvent[] {
this.reports.set(handle, reportKeys(report, data));
return this.diff();
}
/** Release everything (a stream gap, a dropped link). */
reset(): HidKeyEvent[] {
this.reports.clear();
return this.diff();
}
private diff(): HidKeyEvent[] {
const next = new Map<string, HidUsage>();
for (const values of this.reports.values()) for (const u of values) next.set(usageKey(u), u);
const events: HidKeyEvent[] = [];
for (const [k, u] of this.held) if (!next.has(k)) events.push({ ...u, down: false });
for (const [k, u] of next) if (!this.held.has(k)) events.push({ ...u, down: true });
this.held = next;
return events;
}
}
+146
View File
@@ -0,0 +1,146 @@
/**
* Where a key from the hardware keyboard goes: to a keybinding being set up,
* to a gesture it is bound to, to the window with input focus, or to the
* built-in navigation. The meanings are in bindings.ts; this is the order of
* asking, and the bookkeeping around it. Shared by the Android and iOS
* controllers through KeyRouterHost.
* No platform imports: this runs under node in the tests.
*/
import {
GestureScheme,
chordOf,
isModifierCode,
resolveKey,
type GestureOrigin,
type KeyAction,
type KeyChord,
type KeyGesture,
} from "./bindings";
import type { KeyEvent } from "./keys";
export type KeyRouterHost = {
/** Connected to glasses that are in use. */
ready(): boolean;
locked(): boolean;
screenOn(): boolean;
/** Turn a dark display on, and nothing else. */
wake(): void;
/** Offer a key to the window with input focus; true when that window takes keys. */
offerToWindow(event: KeyEvent): boolean;
/** Perform a gesture as if it came from the ring or the watch. */
gesture(gesture: KeyGesture, origin: GestureOrigin): Promise<void>;
/** The custom keybindings, by chord id. */
bindings(): ReadonlyMap<string, KeyAction>;
/** A key was used for something other than a gesture or typing (count it against the screen timeout). */
noteActivity(): void;
log(line: string): void;
now?(): number;
};
/**
* A held key repeats twenty times a second, and a wheel or a two-finger
* scroll reports faster still: more than a list can repaint over the
* glasses' link. Those are dropped while a gesture is still being handled,
* and closer together than this.
*/
const REPEAT_MIN_INTERVAL_MS = 100;
export class KeyRouter {
private capture: ((chord: KeyChord) => void) | null = null;
/** Keys whose press was spoken for, so their release is too. */
private readonly consumed = new Set<string>();
private readonly scheme = new GestureScheme();
private gesturesInFlight = 0;
private lastGestureAtMs = 0;
constructor(private readonly host: KeyRouterHost) {}
/** One key transition from the keyboard. */
handleKey(event: KeyEvent): void {
const host = this.host;
if (!host.ready()) return;
if (host.locked() || !host.screenOn()) {
// A dark display takes no typing: the first press only wakes it, as
// the ring's double tap would, and a locked one stays locked.
if (event.phase === "down" && !event.repeat && !host.screenOn()) host.wake();
return;
}
if (this.capture) {
this.captureKey(event);
return;
}
if (event.phase === "up") {
if (!this.consumed.delete(event.code)) host.offerToWindow(event);
return;
}
const resolution = resolveKey(event, host.bindings());
const droppable = event.repeat || event.burst === true;
if (resolution.consume) {
this.consumed.add(event.code);
if (resolution.gesture) this.perform(resolution.gesture, droppable);
return;
}
if (host.offerToWindow(event)) return;
if (resolution.gesture) this.perform(resolution.gesture, droppable);
}
/**
* A key the focused window was offered and had no use for (a terminal
* showing its session list rather than a session): it gets the built-in
* navigation meaning after all.
*/
handleUnhandledKey(event: KeyEvent): void {
const host = this.host;
if (event.phase !== "down" || this.capture || !host.ready() || host.locked() || !host.screenOn()) return;
const resolution = resolveKey(event, host.bindings());
if (!resolution.consume && resolution.gesture) this.perform(resolution.gesture, event.repeat || event.burst === true);
}
/**
* Take the next key press instead of acting on it (setting up a
* keybinding). Modifiers held with it are part of the chord; pressed alone
* they are waited through. Returns a function that calls the capture off.
*/
captureNextKey(onChord: (chord: KeyChord) => void): () => void {
this.capture = onChord;
return () => {
if (this.capture === onChord) this.capture = null;
};
}
/** The keyboard went away: nothing is held any more. */
reset(): void {
this.consumed.clear();
}
private captureKey(event: KeyEvent): void {
this.host.noteActivity();
if (event.phase === "up") {
this.consumed.delete(event.code);
return;
}
if (event.repeat || isModifierCode(event.code)) return;
const capture = this.capture!;
this.capture = null;
this.consumed.add(event.code);
capture(chordOf(event));
}
private perform(gesture: KeyGesture, droppable: boolean): void {
const now = this.host.now ? this.host.now() : Date.now();
if (droppable && (this.gesturesInFlight > 0 || now - this.lastGestureAtMs < REPEAT_MIN_INTERVAL_MS)) return;
this.lastGestureAtMs = now;
void this.run(gesture, this.scheme.originFor(gesture));
}
private async run(gesture: KeyGesture, origin: GestureOrigin): Promise<void> {
this.gesturesInFlight++;
try {
await this.host.gesture(gesture, origin);
} catch (error) {
this.host.log(`keyboard gesture ${gesture} failed: ${error instanceof Error ? error.message : String(error)}`);
} finally {
this.gesturesInFlight--;
}
}
}
+306
View File
@@ -0,0 +1,306 @@
/**
* Key events as the rest of Faceclaw sees them, and the translation from the
* keyboard bridge's raw HID transitions: layout (US only for now), modifier
* and Caps Lock state, and typematic repeat, which a BLE keyboard never sends
* (its report just stays asserted). The translation tables follow the Glassly
* app's (GlassesKeyboardInput.ts, MIT).
* No platform imports: this runs under node in the tests, and in app workers.
*/
import type { HidKeyEvent } from "./hid";
export type KeyModifiers = { shift: boolean; ctrl: boolean; alt: boolean; meta: boolean };
/**
* One key transition, shaped like a DOM KeyboardEvent: `key` is the logical
* key with the layout applied ("a", "A", "Enter", "ArrowUp"), `code` the
* physical key ("KeyA"), `text` what a press types (absent for releases, for
* keys that type nothing, and for Ctrl/Alt/Meta chords, which are commands).
*/
export type KeyEvent = {
phase: "down" | "up";
key: string;
code: string;
text?: string;
modifiers: KeyModifiers;
/** A synthesized auto-repeat of a held key. */
repeat: boolean;
/**
* One of a stream that can arrive faster than it can be acted on (a wheel
* turning): whoever acts on it may drop some rather than fall behind.
*/
burst?: boolean;
/** Date.now() when the phone saw the transition. */
timestampMs: number;
};
const HID_PAGE_KEYBOARD = 7;
const HID_PAGE_CONSUMER = 12;
/** A mouse's buttons, which arrive here as keys: usage 1 is the left button, 2 the right, 3 the middle. */
export const HID_PAGE_BUTTON = 9;
const USAGE_CAPS_LOCK = 57;
/** [unshifted, shifted, code] for US-layout punctuation on the keyboard page. */
const PUNCTUATION: Record<number, [string, string, string]> = {
45: ["-", "_", "Minus"],
46: ["=", "+", "Equal"],
47: ["[", "{", "BracketLeft"],
48: ["]", "}", "BracketRight"],
49: ["\\", "|", "Backslash"],
50: ["#", "~", "IntlHash"],
51: [";", ":", "Semicolon"],
52: ["'", '"', "Quote"],
53: ["`", "~", "Backquote"],
54: [",", "<", "Comma"],
55: [".", ">", "Period"],
56: ["/", "?", "Slash"],
100: ["\\", "|", "IntlBackslash"],
};
const DIGITS = "1234567890";
const SHIFTED_DIGITS = "!@#$%^&*()";
/** [key, code] for keys that type nothing. */
const NAMED: Record<number, [string, string]> = {
40: ["Enter", "Enter"],
41: ["Escape", "Escape"],
42: ["Backspace", "Backspace"],
43: ["Tab", "Tab"],
57: ["CapsLock", "CapsLock"],
70: ["PrintScreen", "PrintScreen"],
71: ["ScrollLock", "ScrollLock"],
72: ["Pause", "Pause"],
73: ["Insert", "Insert"],
74: ["Home", "Home"],
75: ["PageUp", "PageUp"],
76: ["Delete", "Delete"],
77: ["End", "End"],
78: ["PageDown", "PageDown"],
79: ["ArrowRight", "ArrowRight"],
80: ["ArrowLeft", "ArrowLeft"],
81: ["ArrowDown", "ArrowDown"],
82: ["ArrowUp", "ArrowUp"],
83: ["NumLock", "NumLock"],
88: ["Enter", "NumpadEnter"],
101: ["ContextMenu", "ContextMenu"],
102: ["Power", "Power"],
224: ["Control", "ControlLeft"],
225: ["Shift", "ShiftLeft"],
226: ["Alt", "AltLeft"],
227: ["Meta", "MetaLeft"],
228: ["Control", "ControlRight"],
229: ["Shift", "ShiftRight"],
230: ["Alt", "AltRight"],
231: ["Meta", "MetaRight"],
};
/** [key and text, code] for the numeric keypad's operators. */
const NUMPAD: Record<number, [string, string]> = {
84: ["/", "NumpadDivide"],
85: ["*", "NumpadMultiply"],
86: ["-", "NumpadSubtract"],
87: ["+", "NumpadAdd"],
99: [".", "NumpadDecimal"],
103: ["=", "NumpadEqual"],
};
const CONSUMER: Record<number, string> = {
0xe9: "AudioVolumeUp",
0xea: "AudioVolumeDown",
0xe2: "AudioVolumeMute",
0xcd: "MediaPlayPause",
0xb0: "MediaPlay",
0xb1: "MediaPause",
0xb5: "MediaTrackNext",
0xb6: "MediaTrackPrevious",
0xb7: "MediaStop",
0x6f: "BrightnessUp",
0x70: "BrightnessDown",
0x221: "BrowserSearch",
0x223: "BrowserHome",
0x224: "BrowserBack",
0x225: "BrowserForward",
0x18a: "LaunchMail",
0x192: "LaunchApplication2",
0x194: "LaunchApplication1",
};
const MODIFIER_USAGES: Record<number, keyof KeyModifiers> = {
224: "ctrl", 228: "ctrl", 225: "shift", 229: "shift", 226: "alt", 230: "alt", 227: "meta", 231: "meta",
};
type Translated = { key: string; code: string; text?: string };
/** One HID usage under the US layout. Caps Lock only affects letters. */
export function translateUsage(page: number, usage: number, shift: boolean, capsLock: boolean): Translated {
if (page === HID_PAGE_KEYBOARD) {
if (usage >= 4 && usage <= 29) {
const lower = String.fromCharCode(97 + usage - 4);
const key = shift !== capsLock ? lower.toUpperCase() : lower;
return { key, code: `Key${lower.toUpperCase()}`, text: key };
}
if (usage >= 30 && usage <= 39) {
const index = usage - 30;
const key = shift ? SHIFTED_DIGITS[index]! : DIGITS[index]!;
return { key, code: `Digit${DIGITS[index]}`, text: key };
}
if (usage === 44) return { key: " ", code: "Space", text: " " };
const punctuation = PUNCTUATION[usage];
if (punctuation) {
const key = shift ? punctuation[1] : punctuation[0];
return { key, code: punctuation[2], text: key };
}
if (usage >= 58 && usage <= 69) return { key: `F${usage - 57}`, code: `F${usage - 57}` };
if (usage >= 104 && usage <= 115) return { key: `F${usage - 91}`, code: `F${usage - 91}` };
if (usage >= 89 && usage <= 97) {
const digit = String(usage - 88);
return { key: digit, code: `Numpad${digit}`, text: digit };
}
if (usage === 98) return { key: "0", code: "Numpad0", text: "0" };
const numpad = NUMPAD[usage];
if (numpad) return { key: numpad[0], code: numpad[1], text: numpad[0] };
const named = NAMED[usage];
if (named) return { key: named[0], code: named[1] };
return { key: "Unidentified", code: `Usage${usage.toString(16).padStart(2, "0")}` };
}
if (page === HID_PAGE_CONSUMER) {
const name = CONSUMER[usage];
return name ? { key: name, code: name } : { key: "Unidentified", code: `Consumer${usage.toString(16)}` };
}
if (page === HID_PAGE_BUTTON) return { key: `MouseButton${usage}`, code: `MouseButton${usage}` };
return { key: "Unidentified", code: `Page${page.toString(16)}Usage${usage.toString(16)}` };
}
export type KeyTranslatorOptions = {
emit: (event: KeyEvent) => void;
now?: () => number;
setTimer?: (callback: () => void, delayMs: number) => unknown;
clearTimer?: (handle: unknown) => void;
/** Hold time before the first repeat, and the gap between repeats. */
repeatDelayMs?: number;
repeatIntervalMs?: number;
};
const REPEAT_DELAY_MS = 400;
const REPEAT_INTERVAL_MS = 50;
/**
* Raw HID transitions in, KeyEvents out. Modifier state comes from the same
* stream, and the bridge reports a release for every key when the stream may
* have lost something, so a missed key-up cannot leave a modifier stuck or a
* key repeating forever.
*/
export class KeyTranslator {
private readonly emitEvent: (event: KeyEvent) => void;
private readonly now: () => number;
private readonly setTimer: (callback: () => void, delayMs: number) => unknown;
private readonly clearTimer: (handle: unknown) => void;
private readonly repeatDelayMs: number;
private readonly repeatIntervalMs: number;
private readonly held = new Set<string>();
private modifiers: KeyModifiers = { shift: false, ctrl: false, alt: false, meta: false };
private capsLock = false;
private repeatTimer: unknown = null;
private repeating: { page: number; usage: number } | null = null;
constructor(options: KeyTranslatorOptions) {
this.emitEvent = options.emit;
this.now = options.now ?? (() => Date.now());
this.setTimer = options.setTimer ?? ((callback, delayMs) => setTimeout(callback, delayMs));
this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as ReturnType<typeof setTimeout>));
this.repeatDelayMs = options.repeatDelayMs ?? REPEAT_DELAY_MS;
this.repeatIntervalMs = options.repeatIntervalMs ?? REPEAT_INTERVAL_MS;
}
/** Apply the transitions one input report produced, in order. */
handle(events: readonly HidKeyEvent[]): void {
for (const event of events) this.handleOne(event);
}
private handleOne(event: HidKeyEvent): void {
const id = `${event.page}:${event.usage}`;
const modifier = event.page === HID_PAGE_KEYBOARD ? MODIFIER_USAGES[event.usage] : undefined;
if (event.down) {
if (this.held.has(id)) return;
this.held.add(id);
if (modifier) this.modifiers[modifier] = true;
if (event.page === HID_PAGE_KEYBOARD && event.usage === USAGE_CAPS_LOCK) this.capsLock = !this.capsLock;
this.emit("down", event.page, event.usage, false);
// The newest non-modifier key is the one that repeats, as on a PC. A
// mouse button neither repeats nor takes the repeat from a held key.
if (!modifier && event.page !== HID_PAGE_BUTTON) this.startRepeat(event.page, event.usage);
} else {
if (!this.held.delete(id)) return;
if (modifier) this.modifiers[modifier] = this.otherHeld(event.usage, modifier);
if (this.repeating?.page === event.page && this.repeating.usage === event.usage) this.stopRepeat();
this.emit("up", event.page, event.usage, false);
}
}
/**
* A press and release in one, for input that has no "held" (a wheel's
* detent): `name` is both the key and the code, e.g. "WheelUp". Carries
* the modifiers held on the keyboard, like any key.
*/
pulse(name: string): void {
const timestampMs = this.now();
for (const phase of ["down", "up"] as const) {
this.emitEvent({ phase, key: name, code: name, modifiers: { ...this.modifiers }, repeat: false, burst: true, timestampMs });
}
}
/** The keyboard went away: forget everything without emitting releases. */
reset(): void {
this.stopRepeat();
this.held.clear();
this.modifiers = { shift: false, ctrl: false, alt: false, meta: false };
this.capsLock = false;
}
/** Whether the same modifier's key on the other side is still down. */
private otherHeld(released: number, modifier: keyof KeyModifiers): boolean {
for (const [usage, name] of Object.entries(MODIFIER_USAGES)) {
if (name === modifier && Number(usage) !== released && this.held.has(`${HID_PAGE_KEYBOARD}:${usage}`)) return true;
}
return false;
}
private emit(phase: "down" | "up", page: number, usage: number, repeat: boolean): void {
const translated = translateUsage(page, usage, this.modifiers.shift, this.capsLock);
const event: KeyEvent = {
phase,
key: translated.key,
code: translated.code,
modifiers: { ...this.modifiers },
repeat,
timestampMs: this.now(),
};
if (phase === "down" && translated.text !== undefined &&
!this.modifiers.ctrl && !this.modifiers.alt && !this.modifiers.meta) {
event.text = translated.text;
}
this.emitEvent(event);
}
private startRepeat(page: number, usage: number): void {
this.stopRepeat();
this.repeating = { page, usage };
this.repeatTimer = this.setTimer(() => this.repeatTick(), this.repeatDelayMs);
}
private repeatTick(): void {
const key = this.repeating;
this.repeatTimer = null;
if (!key || !this.held.has(`${key.page}:${key.usage}`)) {
this.repeating = null;
return;
}
this.emit("down", key.page, key.usage, true);
this.repeatTimer = this.setTimer(() => this.repeatTick(), this.repeatIntervalMs);
}
private stopRepeat(): void {
if (this.repeatTimer !== null) this.clearTimer(this.repeatTimer);
this.repeatTimer = null;
this.repeating = null;
}
}
+83
View File
@@ -0,0 +1,83 @@
/**
* A keyboard's touchpad or mouse, as the rest of the keyboard support sees
* it. Its buttons and wheel become keys ("MouseButton1", "WheelUp"), so
* they get default meanings and custom keybindings like any key
* (bindings.ts). Its movement goes nowhere unless something has taken the
* pointer (the cursor test page): there is no cursor in the main UI.
* No platform imports: this runs under node in the tests.
*/
import type { HidKeyEvent, HidPointerSample } from "./hid";
import { HID_PAGE_BUTTON } from "./keys";
/** The names a wheel's detents arrive under, as keys. */
export type WheelKey = "WheelUp" | "WheelDown" | "WheelLeft" | "WheelRight";
export type PointerInputHost = {
/** Button transitions, as HID usages on the Button page. */
buttons(events: HidKeyEvent[]): void;
/** One wheel report in a direction, however many detents it claimed. */
wheel(key: WheelKey): void;
};
export class PointerInput {
/** The buttons the host has been told are down. */
private reported = 0;
/** The buttons actually down, as of the last sample. */
private held = 0;
/** Buttons that went down while the pointer was taken: not keys until they have come up. */
private ignored = 0;
private grab: ((sample: HidPointerSample) => void) | null = null;
constructor(private readonly host: PointerInputHost) {}
/** One report from the pointer (see KeyboardClientEvents.pointer). */
handle(sample: HidPointerSample): void {
this.held = sample.buttons;
if (this.grab) {
this.grab(sample);
return;
}
this.ignored &= sample.buttons;
this.report(sample.buttons & ~this.ignored);
// One step per report: a touchpad's two-finger scroll sends many small
// ones, and the gesture they turn into is rate-limited downstream.
if (sample.wheel) this.host.wheel(sample.wheel > 0 ? "WheelUp" : "WheelDown");
if (sample.pan) this.host.wheel(sample.pan > 0 ? "WheelRight" : "WheelLeft");
}
/**
* Take the pointer: every sample goes to `listener`, movement included,
* and its buttons and wheel stop acting as keys. Buttons held at that
* moment are released first, so nothing is left down. Returns a function
* that gives the pointer back.
*/
take(listener: (sample: HidPointerSample) => void): () => void {
this.report(0);
this.grab = listener;
return () => {
if (this.grab !== listener) return;
this.grab = null;
// A button still down was pressed for the listener, not as a key: its
// release is nobody's business either.
this.ignored = this.held;
};
}
/** The keyboard went away: nothing is held, and nobody is told. */
reset(): void {
this.reported = 0;
this.held = 0;
this.ignored = 0;
}
private report(buttons: number): void {
const changed = buttons ^ this.reported;
if (!changed) return;
const events: HidKeyEvent[] = [];
for (let bit = 0; bit < 8; bit++) {
if (changed & (1 << bit)) events.push({ page: HID_PAGE_BUTTON, usage: bit + 1, down: !!(buttons & (1 << bit)) });
}
this.reported = buttons;
this.host.buttons(events);
}
}
+297
View File
@@ -0,0 +1,297 @@
/**
* Wire format of the CFW keyboard bridge (Faceclaw/39). The master temple acts
* as a BLE central for one HID-over-GATT keyboard; the phone drives that link
* over private settings fields and gets the keyboard's raw input reports back.
* The firmware side is g2flash patches/keyboard_core.h, documented in its
* docs/keyboard.md. This follows the reference client in g2flash demos/
* (keyboard-protocol.ts), which came with the firmware from tetramo-labs.
*
* Phone -> glasses, settings field 130 (wrapped by the native session):
* ['K','B',1,op, request LE16, epoch LE16, body...] at most 80 bytes
* Glasses -> phone, settings field 131:
* ['K','B',1,kind, sequence LE32, epoch LE16, request LE16, code LE16,
* handle LE16, total LE16, offset LE16, timestamp LE32, fragment...]
*
* All integers are little endian. No platform imports: this runs under node
* in the tests.
*/
export const KEYBOARD_PROTOCOL_VERSION = 1;
/** Fixed part of a field-131 record, before its fragment. */
export const KEYBOARD_RECORD_HEADER = 24;
/** Most body bytes one record carries; longer bodies are fragmented. */
export const KEYBOARD_RECORD_CHUNK = 96;
/** Largest reassembled body (an ATT value or a HID report). */
export const KEYBOARD_BODY_MAX = 512;
/** Largest control body after the 8-byte prefix. */
export const KEYBOARD_CONTROL_BODY_MAX = 72;
export const KeyboardOp = {
query: 0,
enable: 1,
disable: 2,
scan: 3,
connect: 4,
disconnect: 5,
pair: 6,
auth: 7,
compare: 8,
find: 9,
read: 10,
write: 11,
watch: 12,
} as const;
export type KeyboardOpCode = (typeof KeyboardOp)[keyof typeof KeyboardOp];
export const KeyboardKind = {
status: 1,
att: 2,
input: 3,
dm: 4,
advert: 5,
} as const;
/** Control error codes, carried in a STATUS record's `code`. */
export const KeyboardError = {
ok: 0,
badRequest: 1,
disabled: 2,
busy: 3,
staleEpoch: 4,
noSpace: 5,
notConnected: 6,
notEncrypted: 7,
timeout: 8,
noMemory: 9,
} as const;
export function keyboardErrorName(code: number): string {
switch (code) {
case KeyboardError.ok: return "ok";
case KeyboardError.badRequest: return "bad request";
case KeyboardError.disabled: return "keyboard bridge disabled";
case KeyboardError.busy: return "busy";
case KeyboardError.staleEpoch: return "stale epoch";
case KeyboardError.noSpace: return "no connection or bond space";
case KeyboardError.notConnected: return "not connected";
case KeyboardError.notEncrypted: return "not encrypted";
case KeyboardError.timeout: return "timeout";
case KeyboardError.noMemory: return "allocation failure";
default: return `error ${code}`;
}
}
/** Cordio device-manager events the bridge relays (a DM record's `code & 0xff`). */
export const KeyboardDm = {
connOpen: 0x27,
connClose: 0x28,
pairComplete: 0x2a,
pairFailed: 0x2b,
encrypted: 0x2c,
encryptFailed: 0x2d,
authRequest: 0x2e,
compare: 0x35,
} as const;
/** ATT client events (an ATT record's `code & 0xff`; the status is `code >> 8`). */
export const KeyboardAtt = {
notification: 13,
indication: 14,
} as const;
export type KeyboardRecord = {
kind: number;
sequence: number;
epoch: number;
request: number;
code: number;
handle: number;
/** Glasses uptime in ms when the record was queued. */
timestamp: number;
data: Uint8Array;
};
export const u16 = (p: Uint8Array, off = 0): number => p[off]! | (p[off + 1]! << 8);
export const u32 = (p: Uint8Array, off = 0): number =>
(p[off]! | (p[off + 1]! << 8) | (p[off + 2]! << 16) | (p[off + 3]! << 24)) >>> 0;
export const le16 = (v: number): number[] => [v & 255, (v >>> 8) & 255];
/** A field-130 control record. The session wraps it in the settings package. */
export function keyboardControl(op: number, request: number, epoch: number, body: readonly number[] = []): Uint8Array {
if (!Number.isInteger(op) || op < 0 || op > KeyboardOp.watch || body.length > KEYBOARD_CONTROL_BODY_MAX) {
throw new Error("Keyboard control bounds");
}
return Uint8Array.from([75, 66, KEYBOARD_PROTOCOL_VERSION, op, ...le16(request), ...le16(epoch), ...body]);
}
/**
* Bounded, ordered reassembly of field-131 records. `onGap` fires whenever
* the stream may have lost something (a skipped sequence, an epoch change, a
* malformed or orphaned fragment): the caller then releases every held key
* and fails any setup transaction in flight. The firmware's once-a-second
* status record makes a lost final release visible even when no further keys
* arrive.
*/
export class KeyboardRecords {
private epoch: number | undefined;
private last: number | undefined;
private fragment: { record: KeyboardRecord; total: number; used: number } | undefined;
constructor(private readonly onGap: () => void) {}
/** Forget the stream position (the phone link dropped); counts as a gap. */
reset(): void {
this.epoch = undefined;
this.last = undefined;
this.fragment = undefined;
this.onGap();
}
/** Feed one field-131 payload; returns a complete record, or undefined. */
push(p: Uint8Array): KeyboardRecord | undefined {
const invalid = (): undefined => {
this.fragment = undefined;
this.onGap();
return undefined;
};
if (
p.length < KEYBOARD_RECORD_HEADER || p.length > KEYBOARD_RECORD_HEADER + KEYBOARD_RECORD_CHUNK ||
p[0] !== 75 || p[1] !== 66 || p[2] !== KEYBOARD_PROTOCOL_VERSION
) {
return invalid();
}
const r: KeyboardRecord = {
kind: p[3]!,
sequence: u32(p, 4),
epoch: u16(p, 8),
request: u16(p, 10),
code: u16(p, 12),
handle: u16(p, 14),
timestamp: u32(p, 20),
data: new Uint8Array(0),
};
const total = u16(p, 16);
const offset = u16(p, 18);
const n = p.length - KEYBOARD_RECORD_HEADER;
if (r.kind < KeyboardKind.status || r.kind > KeyboardKind.advert || total > KEYBOARD_BODY_MAX ||
offset + n > total || (total > 0 && n === 0)) {
return invalid();
}
if (this.epoch !== r.epoch) {
this.onGap();
this.fragment = undefined;
this.last = undefined;
this.epoch = r.epoch;
}
if (offset === 0) {
if (this.last !== undefined) {
const distance = (r.sequence - this.last) >>> 0;
if (distance === 0 || distance >= 0x80000000) return undefined; // duplicate or old delivery
}
if (this.fragment || (this.last !== undefined && r.sequence !== ((this.last + 1) >>> 0))) this.onGap();
this.last = r.sequence;
this.fragment = { record: { ...r, data: new Uint8Array(total) }, total, used: 0 };
}
const f = this.fragment;
if (!f || f.used !== offset || f.total !== total || f.record.sequence !== r.sequence ||
f.record.kind !== r.kind || f.record.request !== r.request || f.record.code !== r.code ||
f.record.handle !== r.handle || f.record.timestamp !== r.timestamp) {
return invalid();
}
f.record.data.set(p.subarray(KEYBOARD_RECORD_HEADER), offset);
f.used += n;
if (f.used !== total) return undefined;
this.fragment = undefined;
return f.record;
}
}
/** BLE address types as the controller reports them. */
export const KEYBOARD_ADDRESS_TYPES = ["public", "random", "public-identity", "random-identity"] as const;
/** One advertiser seen during a scan (an ADVERT record's body, parsed). */
export type KeyboardAdvert = {
addressType: number;
/** "aa:bb:cc:dd:ee:ff", most significant byte first. */
address: string;
rssi: number;
name: string;
/** Advertises the HID service, or a HID appearance. */
hid: boolean;
};
/** ADVERT body: addrType, addr[6] (LSB first), RSSI, event type, AD length, AD structures. */
export function parseKeyboardAdvert(body: Uint8Array): KeyboardAdvert | null {
if (body.length < 10 || body[0]! > 3) return null;
let name = "";
let hid = false;
for (let i = 10; i < body.length;) {
const n = body[i++]!;
if (n === 0 || i + n > body.length) break;
const type = body[i]!;
if (type === 8 || type === 9) name = decodeUtf8(body.subarray(i + 1, i + n));
// 16-bit service lists (incomplete / complete): 0x1812 is HID.
if (type === 2 || type === 3) {
for (let j = i + 1; j + 1 < i + n; j += 2) if (body[j] === 0x12 && body[j + 1] === 0x18) hid = true;
}
// Appearance 0x03C0..0x03FF is the HID category (0x03C1 keyboard); some
// keyboards advertise that without the service list.
if (type === 0x19 && n >= 3 && body[i + 2] === 0x03 && body[i + 1]! >= 0xc0) hid = true;
i += n;
}
return {
addressType: body[0]!,
address: formatKeyboardAddress(body.subarray(1, 7)),
rssi: (body[7]! << 24) >> 24,
name,
hid,
};
}
/**
* Fold a later record for the same advertiser into an earlier one. An active
* scan reports the advertisement and its scan response separately and the
* name usually only rides in the latter: keep a name once seen, HID once
* seen, and the best signal.
*/
export function mergeKeyboardAdverts(a: KeyboardAdvert, b: KeyboardAdvert): KeyboardAdvert {
return { ...a, rssi: Math.max(a.rssi, b.rssi), name: a.name || b.name, hid: a.hid || b.hid };
}
/** Controller byte order (LSB first) to the usual "aa:bb:cc:dd:ee:ff". */
export function formatKeyboardAddress(lsbFirst: Uint8Array): string {
return Array.from(lsbFirst).reverse().map((b) => b.toString(16).padStart(2, "0")).join(":");
}
/** "aa:bb:cc:dd:ee:ff" to the controller's LSB-first bytes, or null if malformed. */
export function parseKeyboardAddress(text: string): Uint8Array | null {
if (!/^(?:[0-9a-f]{2}:){5}[0-9a-f]{2}$/i.test(text)) return null;
return Uint8Array.from(text.split(":").reverse().map((part) => parseInt(part, 16)));
}
/** Lenient UTF-8 decode (advertised names); no TextDecoder in every runtime this loads in. */
function decodeUtf8(bytes: Uint8Array): string {
let out = "";
for (let i = 0; i < bytes.length;) {
const b = bytes[i++]!;
let code = b;
let extra = 0;
if (b >= 0xf0 && b < 0xf8) { code = b & 0x07; extra = 3; }
else if (b >= 0xe0) { code = b & 0x0f; extra = 2; }
else if (b >= 0xc0) { code = b & 0x1f; extra = 1; }
else if (b >= 0x80) { out += "�"; continue; }
if (i + extra > bytes.length) { out += "�"; break; }
let valid = true;
for (let k = 0; k < extra; k++) {
const c = bytes[i + k]!;
if ((c & 0xc0) !== 0x80) { valid = false; break; }
code = (code << 6) | (c & 0x3f);
}
if (!valid || code > 0x10ffff) { out += "�"; continue; }
i += extra;
if (code === 0) continue;
out += String.fromCodePoint(code);
}
return out;
}
+235
View File
@@ -0,0 +1,235 @@
/**
* The keyboard bridge as the app uses it: one KeyboardBridge for the life of
* the app, fed by whichever glasses session is live, its remembered keyboard
* and the custom keybindings kept in the settings store, and its key
* transitions turned into KeyEvents and routed. The platform's controller
* reports the link coming and going and performs the gestures keys stand
* for; the Settings app drives pairing through `bridge` and edits the
* keybindings.
*/
import { addKeyboardEventListener, keyboardLinkSession, sendKeyboardControlRecord } from "../../native/keyboard-link";
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import { shell } from "../../ui/shell/shell";
import {
MAX_KEY_BINDINGS,
chordId,
keyBindingMap,
parseKeyBindings,
serializeKeyBindings,
type GestureOrigin,
type KeyAction,
type KeyBinding,
type KeyChord,
type KeyGesture,
} from "./bindings";
import { KeyboardBridge, type KeyboardBridgeSnapshot, type SavedKeyboard } from "./bridge";
import type { HidPointerSample } from "./hid";
import { KeyRouter } from "./key-router";
import { KeyTranslator, type KeyEvent } from "./keys";
import { PointerInput } from "./pointer";
declare const java: any;
declare const arc4random_uniform: ((bound: number) => number) | undefined;
/** JSON of the remembered keyboard ({ address, addressType, name }), or "" for none. */
const SAVED_KEYBOARD_KEY = "keyboard.saved";
/** JSON list of the custom keybindings ({ key, action }); see bindings.ts. */
const KEY_BINDINGS_KEY = "keyboard.bindings";
/** What the platform's controller does for the keyboard. */
export type KeyboardServiceHost = {
/** Connected to glasses that are in use (not charging in their case). */
ready(): boolean;
locked(): boolean;
/** Turn a dark display on, and nothing else (a locked display stays locked). */
wake(): void;
/** Perform a gesture as if it came from the ring or the watch. */
gesture(gesture: KeyGesture, origin: GestureOrigin): Promise<void>;
/** Routine lines only (failed background reconnects); the rest go to the device log directly. */
log(line: string): void;
/** Whether an absent keyboard is worth looking for now (see KeyboardBridgeHost). */
shouldReconnect(): boolean;
};
function loadSavedKeyboard(): SavedKeyboard | null {
try {
const raw = getStringSetting(SAVED_KEYBOARD_KEY, "");
if (!raw) return null;
const value = JSON.parse(raw);
if (typeof value?.address !== "string" || typeof value?.addressType !== "number") return null;
return { address: value.address, addressType: value.addressType, name: typeof value.name === "string" ? value.name : "" };
} catch {
return null;
}
}
/** Passkey entry only protects the pairing if the passkey can't be guessed. */
function randomPasskey(): number {
try {
if (global.isAndroid) return new java.security.SecureRandom().nextInt(1_000_000);
if (typeof arc4random_uniform === "function") return arc4random_uniform(1_000_000);
} catch {
// Fall through to the weaker source rather than failing the pairing.
}
return Math.floor(Math.random() * 1_000_000);
}
class KeyboardService {
private host: KeyboardServiceHost | null = null;
private session: unknown = null;
private offEvents: (() => void) | null = null;
private readonly listeners = new Set<() => void>();
private bindings: KeyBinding[] = parseKeyBindings(getStringSetting(KEY_BINDINGS_KEY, ""));
private bindingMap = keyBindingMap(this.bindings);
private readonly router = new KeyRouter({
ready: () => this.host?.ready() ?? false,
locked: () => this.host?.locked() ?? false,
screenOn: () => shell.isScreenOn(),
wake: () => this.host?.wake(),
offerToWindow: (event) => shell.receiveKeyEvent(event),
gesture: (gesture, origin) => this.host?.gesture(gesture, origin) ?? Promise.resolve(),
bindings: () => this.bindingMap,
noteActivity: () => shell.noteUserActivity(),
log: (line) => console.log(`[keyboard] ${line}`),
});
private readonly translator = new KeyTranslator({ emit: (event) => this.router.handleKey(event) });
/** The touchpad's buttons and wheel go in as keys; see pointer.ts. */
private readonly pointer = new PointerInput({
buttons: (events) => this.translator.handle(events),
wheel: (key) => this.translator.pulse(key),
});
readonly bridge = new KeyboardBridge({
send: (record) => {
const session = this.session;
if (!session) return false;
try {
return sendKeyboardControlRecord(session, record, `op=${record[3]}`);
} catch {
return false;
}
},
loadSaved: loadSavedKeyboard,
storeSaved: (saved) => setStringSetting(SAVED_KEYBOARD_KEY, saved ? JSON.stringify(saved) : ""),
onKeys: (events) => this.translator.handle(events),
onPointer: (sample) => this.pointer.handle(sample),
onKeyboardGone: () => {
this.translator.reset();
this.pointer.reset();
this.router.reset();
},
onChanged: () => {
for (const listener of [...this.listeners]) listener();
},
// Pairing and setup are rare and are what a failure report needs, so
// they always reach the device log. Nothing here ever includes a key.
log: (line, routine) => {
if (routine) this.host?.log(line);
else console.log(`[keyboard] ${line}`);
},
randomPasskey,
shouldReconnect: () => this.host?.shouldReconnect() ?? true,
});
setHost(host: KeyboardServiceHost): void {
this.host = host;
}
snapshot(): KeyboardBridgeSnapshot {
return this.bridge.snapshot();
}
/** Called on every bridge state change. Returns an unsubscribe function. */
subscribe(listener: () => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
/** The glasses came into use (the display woke): look for an absent keyboard now. */
retryNow(): void {
this.bridge.retryNow();
}
/** A G2 session on firmware with the bridge is up. Idempotent. */
linkReady(): void {
const session = keyboardLinkSession();
if (!session) return;
if (session !== this.session) {
// The native session object is rebuilt across reconnects.
this.offEvents?.();
this.session = session;
this.offEvents = addKeyboardEventListener(session, (body) => this.bridge.accept(body));
}
this.bridge.linkReady();
}
/**
* The session can't host the keyboard any more. `release` asks the firmware
* to let the keyboard go first, for when the link itself stays up (the
* glasses went into their case, or the app is disconnecting on purpose).
* Idempotent.
*/
linkLost(release = false): void {
if (release) this.bridge.linkClosing();
this.bridge.linkLost();
this.offEvents?.();
this.offEvents = null;
this.session = null;
}
/** A key a worker's window was sent and had no use for (the "key-unhandled" reply). */
handleUnhandledKey(event: KeyEvent): void {
this.router.handleUnhandledKey(event);
}
// ---------- Custom keybindings ----------
/** The custom keybindings, in the order they were added. */
keyBindings(): readonly KeyBinding[] {
return this.bindings;
}
/** Bind a chord, replacing what it was bound to. False when the list is full. */
setKeyBinding(chord: KeyChord, action: KeyAction): boolean {
const id = chordId(chord);
const index = this.bindings.findIndex((binding) => chordId(binding.chord) === id);
if (index < 0 && this.bindings.length >= MAX_KEY_BINDINGS) return false;
const next = [...this.bindings];
if (index < 0) next.push({ chord, action });
else next[index] = { chord, action };
this.storeBindings(next);
return true;
}
removeKeyBinding(chord: KeyChord): void {
const id = chordId(chord);
this.storeBindings(this.bindings.filter((binding) => chordId(binding.chord) !== id));
}
/**
* Take the next key pressed on the keyboard instead of acting on it. Returns
* a function that calls the capture off.
*/
captureNextKey(onChord: (chord: KeyChord) => void): () => void {
return this.router.captureNextKey(onChord);
}
/**
* Take the keyboard's touchpad or mouse: every report goes to `listener`,
* movement included, and its buttons and wheel stop acting as keys until
* the returned function is called. For the cursor test page; the main UI
* has no cursor.
*/
takePointer(listener: (sample: HidPointerSample) => void): () => void {
return this.pointer.take(listener);
}
private storeBindings(bindings: KeyBinding[]): void {
this.bindings = bindings;
this.bindingMap = keyBindingMap(bindings);
setStringSetting(KEY_BINDINGS_KEY, serializeKeyBindings(bindings));
}
}
export const keyboardService = new KeyboardService();
+20 -2
View File
@@ -3,11 +3,18 @@ import type { DiscoveryEvents } from './device-discovery'
import type { RawAdvertisement } from '../g2/even-advertisement'
export { buildAddressSet } from './device-discovery-common'
/**
* How often a scan re-reads which arms the system is connected to. Each read refreshes
* their rows, which age out like advertisements once the link drops and they go quiet.
*/
const CONNECTED_ARMS_POLL_MS = 2000
export class DeviceDiscoveryBridge {
private ble = iosBluetooth()
private off: (() => void) | null = null
private generation = 0
private cancelCollection: (() => void) | null = null
private connectedArmsTimer: ReturnType<typeof setInterval> | null = null
get bluetoothEnabled(): boolean { return this.ble.state === 5 }
get scanning(): boolean { return !!this.off && this.ble.scanning }
startScan(events: DiscoveryEvents): boolean {
@@ -21,16 +28,27 @@ export class DeviceDiscoveryBridge {
void this.ble.startScan(24 * 60 * 60 * 1000).catch(error => {
if (generation === this.generation) events.onScanFailed?.(-1, String(error.message ?? error))
})
// A connected arm doesn't advertise (iOS holds the right arm's ANCS link open after a
// session ends), so the scan alone would show those glasses one-armed.
const reportConnectedArms = () => { for (const device of this.ble.systemConnectedDevices()) events.onAdvertisement(device) }
reportConnectedArms()
this.connectedArmsTimer = setInterval(reportConnectedArms, CONNECTED_ARMS_POLL_MS)
return true
}
stopScan(): void {
++this.generation; this.cancelCollection?.()
if (this.connectedArmsTimer) { clearInterval(this.connectedArmsTimer); this.connectedArmsTimer = null }
if (this.off) { this.off(); this.off = null; this.ble.stopScan() }
}
// CoreBluetooth does not expose the phone's bond list. Fresh advertisements
// supply both identity and the UUID mapping that iOS needs for connections.
// supply both identity and the UUID mapping that iOS needs for connections;
// startScan() reports the arms the system is connected to.
emitBondedDevices(): void {}
async getBondedCandidates(): Promise<RawAdvertisement[]> { return [...this.ble.devices.values()] }
async getBondedCandidates(): Promise<RawAdvertisement[]> {
const devices = new Map(this.ble.devices)
for (const device of this.ble.systemConnectedDevices()) if (!devices.has(device.address)) devices.set(device.address, device)
return [...devices.values()]
}
async scanCandidates(timeoutMs = 6000): Promise<RawAdvertisement[]> {
const generation = this.generation
await this.ble.ensureReady()
+21
View File
@@ -19,6 +19,7 @@ declare const FaceclawKitFaceclawCompassListener: any;
declare const FaceclawKitFaceclawImuListener: any;
declare const FaceclawKitIosAmbientLightListener: any;
declare const FaceclawKitIosMicStatusListener: any;
declare const FaceclawKitIosKeyboardEventListener: any;
export type CommunicatorPhase =
@@ -198,6 +199,13 @@ class MicStatusListener extends NSObject {
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
}
@NativeClass()
class KeyboardEventListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosKeyboardEventListener];
handler!: (body: Uint8Array) => void;
onNsDataKeyboardEventBody(body: NSData): void { this.handler(fromData(body)); }
}
/**
* The iOS twin of the Android communicator bridge: the same public API over
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
@@ -803,6 +811,19 @@ export class FaceclawCommunicatorBridge {
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
}
/** Raw field-131 keyboard-bridge records, in arrival order, on the main queue. */
addKeyboardEventListener(listener: (body: Uint8Array) => void): () => void {
const proxy = KeyboardEventListener.new() as KeyboardEventListener;
proxy.handler = listener;
const handle = this.communicator.addKeyboardEventListenerNsDataListener(proxy);
return () => { this.communicator.removeKeyboardEventListenerListener(handle); };
}
/** Queue a CFW keyboard-bridge control for the right temple; false when nothing was queued. */
sendFaceclawKeyboardControl(record: Uint8Array, label: string): boolean {
return Boolean(this.communicator.sendFaceclawKeyboardControlRecordLabel(toData(new Uint8Array(record)), label));
}
/**
* Every audio-characteristic packet, whatever enabled the mics, on the
* main queue. Passive: sends nothing and doesn't affect capture.
+12 -51
View File
@@ -1,5 +1,6 @@
import { openSocket } from "./socket";
import { openSocket, type SocketConnection, type SocketListener } from "./socket";
import { toUint8Array } from "../util/array-util";
import type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState } from "./terminal-session-types";
declare const android: any;
declare const java: any;
@@ -19,54 +20,7 @@ const LAUNCH_REPLY_TIMEOUT_MS = 10_000;
// otherwise treat the "\r" as a pasted newline instead of the Enter key.
const SUBMIT_DELAY_MS = 150;
export type G2MirrorSession = {
socket: string;
pid: number;
cwdHint: string;
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
lastBellAt: number | null;
/**
* Unix epoch ms (host clock) of the app's last output, or null if none
* observed. Fresh to within the wrapper's 2s activity reporting interval
* while we're connected; compare against other host timestamps, not the
* phone clock (see the worker's local-receive-time activity tracking).
*/
lastOutputAt: number | null;
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
title: string | null;
};
export type G2MirrorPhase =
| "idle"
| "connecting"
| "connected" // handshake accepted; can list/attach
| "attached" // relaying to one session
| "failed";
export type G2MirrorState = {
phase: G2MirrorPhase;
status: string;
sessions: G2MirrorSession[];
attachedCommand: string;
/**
* server_name from the init success reply: a human-readable name for the
* machine the server runs on. "" until the handshake succeeds (or when an
* older server omits the field).
*/
serverName: string;
};
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
export type G2MirrorHistoryReply = {
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
start: number;
/** Oldest index still retained by the wrapper (older requests return nothing). */
oldest: number;
/** Current splice point (next line to be archived). */
next: number;
/** Plain text, ANSI-stripped, oldest-to-newest. */
lines: string[];
};
export type { G2MirrorHistoryReply, G2MirrorPhase, G2MirrorSession, G2MirrorState };
export type G2MirrorClientOptions = {
/** TLS (g2mirrors:// → wss) vs plain (g2mirror:// → ws). */
@@ -77,6 +31,12 @@ export type G2MirrorClientOptions = {
deviceName: string;
cols: number;
rows: number;
/**
* Opens the websocket some other way than dialing host:port directly
* (e.g. through an SSH tunnel); `label` then names the endpoint in status
* lines instead of host:port.
*/
transport?: { open: (listener: SocketListener) => SocketConnection; label: string };
};
type TerminalDataKind = "snapshot" | "output";
@@ -180,7 +140,8 @@ export class G2MirrorClient {
this.stopped = false;
const generation = ++this.connectionGeneration;
const url = `${this.options.secure ? "wss" : "ws"}://${this.options.host}:${this.options.port}`;
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
const transport = this.options.transport;
this.setState("connecting", `Connecting to ${transport ? transport.label : `${this.options.host}:${this.options.port}`}...`);
this.listenerProxy = {
onOpen: () => {
if (this.stopped || generation !== this.connectionGeneration) return;
@@ -208,7 +169,7 @@ export class G2MirrorClient {
},
};
try {
this.ws = openSocket(url, this.listenerProxy);
this.ws = transport ? transport.open(this.listenerProxy!) : openSocket(url, this.listenerProxy);
} catch (error) {
this.ws = null;
this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`);
+65
View File
@@ -0,0 +1,65 @@
declare const com: any;
/**
* Head-up (look-up wake) trigger on the active glasses session
* (GlassesSessionCore.setHeadUpTrigger / getHeadUpStatus). Android only for
* now (headUpControlsAvailable): on iOS, or with no session, the setter does
* nothing and the getter returns null.
*/
/** Whether this platform can change the glasses' head-up setting at all. */
export function headUpControlsAvailable(): boolean {
return Boolean(global.isAndroid);
}
/** GlassesSessionCore.getHeadUpStatus. switch -1 = not read on this connection. */
export type HeadUpStatus = {
switch: number;
angle: number;
calibration: number;
/** The trigger Faceclaw is maintaining, or -1 when it leaves the glasses' own setting. */
desired: number;
readAgeMs: number;
/** The glasses' own switch and angle from before Faceclaw changed them (switch -1 = none held). */
originalSwitch: number;
originalAngle: number;
/** Those originals are being written back. */
restorePending: boolean;
};
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/**
* Ask the session to keep the head-up trigger at `degrees` (1..90); 0 stops
* managing it and writes back `original` (or what the session saved itself)
* if Faceclaw changed the glasses' values.
*/
export function setHeadUpTrigger(degrees: number, original: { switch: number; angle: number } | null = null): boolean {
const active = activeCommunicator();
if (!active) return false;
try {
active.setHeadUpTrigger(Math.round(degrees), original ? original.switch : -1, original ? Math.round(original.angle) : 0);
return true;
} catch (error) {
console.warn(`setHeadUpTrigger failed: ${error}`);
return false;
}
}
export function getHeadUpStatus(): HeadUpStatus | null {
const active = activeCommunicator();
if (!active) return null;
try {
return JSON.parse(String(active.getHeadUpStatus())) as HeadUpStatus;
} catch (error) {
console.warn(`getHeadUpStatus failed: ${error}`);
return null;
}
}
+16
View File
@@ -1,7 +1,23 @@
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
/**
* An accelerometer sample in g, in Faceclaw's frame for every glasses model
* (the G2's own; the session rotates other models' samples into it, see
* ImuOrientation.kt): +x forward, +y to the wearer's left, +z up through the
* top of the head. At rest it is the "up" direction: head level reads
* (0, 0, 1), looking 45° up (0.71, 0, 0.71), the head tilted 45° to the right
* (0, 0.71, 0.71).
*/
export type ImuReading = { x: number; y: number; z: number; source: number };
/**
* How far the head is tilted up from level, in degrees (negative: down), from
* a reading taken at rest. Rolling the head to the side does not change it.
*/
export function imuPitchDegrees(reading: { x: number; y: number; z: number }): number {
return (Math.atan2(reading.x, Math.hypot(reading.y, reading.z)) * 180) / Math.PI;
}
/** Map an IMU sample's EventSource to a short label. */
export function imuSourceLabel(source: number): string {
switch (source) {
+74 -4
View File
@@ -1,4 +1,7 @@
import { ApplicationSettings } from '@nativescript/core'
import { loadPairedGlassesIdentity } from '../g2/device-addresses'
import { g2ManufacturerData } from '../g2/even-advertisement'
import { serialKey } from '../g2/glasses-hardware-identity'
import { identifyIosPeripheral, type IosAdvertisement, type IosDevice } from '../g2/ios-peripheral-identity'
import { hexToBytes } from '../util/hex-util'
import { kotlinListener } from './kotlin-listener.ios'
@@ -10,6 +13,13 @@ export const BLE_STATE_UNSUPPORTED = 2, BLE_STATE_UNAUTHORIZED = 3, BLE_STATE_PO
/** CBManagerAuthorization.allowedAlways. */
const BLE_AUTHORIZATION_ALLOWED = 3
/**
* Per address, the peripheral identifier and side last heard there and (since 2026-11-16)
* the advertisement itself, from which an arm that has stopped advertising is rebuilt.
*/
const PERIPHERAL_KEY_PREFIX = 'ios.ble.peripheral.'
type RememberedPeripheral = { identifier: string; role: string; name?: string; manufacturerData?: string }
export type BluetoothEvent =
| { kind: 'state'; state: number; authorization: number }
| { kind: 'advertisement'; identifier: string; name: string; manufacturerData: string; rssi: number; connectable: boolean }
@@ -28,6 +38,8 @@ export class IosBluetooth {
private readonly native = FaceclawKitIosBleCentral.alloc().initWithPlatform(FaceclawKitIosProtocolPlatform.shared)
private readonly listeners = new Set<(event: BluetoothEvent) => void>()
readonly devices = new Map<string, IosDevice>()
/** The record last written per settings key, so repeated advertisements don't rewrite it. */
private readonly rememberedValues = new Map<string, string>()
private scanTimer: ReturnType<typeof setTimeout> | null = null
private scanGeneration = 0
private readonly scanListener: object
@@ -47,12 +59,68 @@ export class IosBluetooth {
const device = identifyIosPeripheral(raw)
if (device) {
this.devices.set(device.address, device)
ApplicationSettings.setString(`ios.ble.peripheral.${device.address}`, JSON.stringify({ identifier: device.identifier, role: device.role }))
this.remember(device)
this.emit({ kind: 'device', device })
}
},
})
}
/** Persists what `device` advertised; skips the write when nothing changed (scans report duplicates). */
private remember(device: IosDevice): void {
const key = `${PERIPHERAL_KEY_PREFIX}${device.address}`
const record: RememberedPeripheral = { identifier: device.identifier, role: device.role, name: device.name, manufacturerData: device.manufacturerData }
const value = JSON.stringify(record)
if (this.rememberedValues.get(key) === value) return
this.rememberedValues.set(key, value)
ApplicationSettings.setString(key, value)
}
/**
* Even arms the system is connected to, rebuilt from what they last advertised here. A
* connected arm doesn't advertise, and iOS keeps the right arm's ANCS link open after a
* session ends (even after pairing other glasses), so a scan alone hears only the left
* arm. An arm this phone has never heard advertise can't be rebuilt: its MAC and the
* pair's serial come only from the advertisement. The device has no signal strength.
*/
systemConnectedDevices(): IosDevice[] {
if (this.state !== BLE_STATE_POWERED_ON) return []
let connected: any
try {
connected = this.native.systemConnectedG2Arms()
} catch (error) {
// Scanning still works without this; don't let it take the pairing page down.
console.log(`[IosBluetooth] systemConnectedG2Arms failed: ${error}`)
return []
}
if (!connected?.count) return []
const records = new Map<string, { address: string; record: RememberedPeripheral }>()
for (const key of ApplicationSettings.getAllKeys()) {
if (!key.startsWith(PERIPHERAL_KEY_PREFIX)) continue
try {
const record = JSON.parse(ApplicationSettings.getString(key, '{}')) as RememberedPeripheral
if (typeof record.identifier === 'string') records.set(record.identifier.toUpperCase(), { address: key.slice(PERIPHERAL_KEY_PREFIX.length), record })
} catch { /* Malformed: a fresh advertisement rewrites it. */ }
}
const identity = loadPairedGlassesIdentity()
const devices: IosDevice[] = []
for (let i = 0; i < connected.count; i++) {
const peripheral = connected.objectAtIndex(i)
const identifier = String(peripheral.identifier)
const remembered = records.get(identifier)
if (!remembered) continue
const { address, record } = remembered
let name = record.name ?? ''
let manufacturerData = record.manufacturerData ?? ''
if (!manufacturerData && identity && (address === identity.leftAddress || address === identity.rightAddress)) {
// Remembered before advertisements were: the saved pairing supplies the serial.
const side = address === identity.leftAddress ? 'left' : 'right'
name ||= side === 'left' ? identity.leftName : identity.rightName
manufacturerData = g2ManufacturerData(serialKey(identity.serial) ?? '', address) ?? ''
}
const device = identifyIosPeripheral({ identifier, name: name || String(peripheral.name ?? ''), manufacturerData, rssi: 0, connectable: true })
if (device?.address === address && device.role === record.role) devices.push({ ...device, rssi: null, source: 'paired' })
}
return devices
}
onEvent(listener: (event: BluetoothEvent) => void): () => void {
this.listeners.add(listener); return () => { this.listeners.delete(listener) }
}
@@ -111,7 +179,7 @@ export class IosBluetooth {
const live = this.devices.get(address)
if (live && live.role !== role) throw new Error(`${address} advertises as ${live.role}, not ${role}. Check Configure devices.`)
try {
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
if (saved.role === role && typeof saved.identifier === 'string') result[role] = saved.identifier
} catch { /* A fresh scan can recover malformed settings. */ }
}
@@ -138,13 +206,15 @@ export class IosBluetooth {
/** The peripheral last heard at this address, if it was on this side. */
peripheralIdentifier(address: string, role: string): string | null {
try {
const saved = JSON.parse(ApplicationSettings.getString(`ios.ble.peripheral.${address}`, '{}'))
const saved = JSON.parse(ApplicationSettings.getString(`${PERIPHERAL_KEY_PREFIX}${address}`, '{}'))
return saved.role === role && typeof saved.identifier === 'string' ? saved.identifier : null
} catch {
return null
}
}
forget(address: string): void { ApplicationSettings.remove(`ios.ble.peripheral.${address}`); this.devices.delete(address) }
forget(address: string): void {
ApplicationSettings.remove(`${PERIPHERAL_KEY_PREFIX}${address}`); this.rememberedValues.delete(`${PERIPHERAL_KEY_PREFIX}${address}`); this.devices.delete(address)
}
}
let instance: IosBluetooth | null = null
export function iosBluetooth(): IosBluetooth { return instance ??= new IosBluetooth() }
+24
View File
@@ -0,0 +1,24 @@
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
/** The iOS twin of keyboard-link.ts, over the running shared-Kotlin session. */
export function keyboardLinkSession(): unknown {
return FaceclawCommunicatorBridge.getActive();
}
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
return (session as FaceclawCommunicatorBridge).sendFaceclawKeyboardControl(record, label);
}
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
const unsubscribe = (session as FaceclawCommunicatorBridge).addKeyboardEventListener(listener);
return () => {
try {
unsubscribe();
} catch {
// The session may have been closed during a reconnect.
}
};
}
+40
View File
@@ -0,0 +1,40 @@
import { toJavaBytes } from "./cloud-stt";
import { toUint8Array } from "../util/array-util";
declare const com: any;
/**
* Platform plumbing for the CFW keyboard bridge (app/g2/keyboard/): the running
* session to send field-130 controls on and to hear field-131 records from.
* The session is opaque and is a different object after a reconnect.
*/
export function keyboardLinkSession(): unknown {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** Queue a control record for the right temple; false when the session wasn't ready and nothing was queued. */
export function sendKeyboardControlRecord(session: unknown, record: Uint8Array, label: string): boolean {
return (session as any).sendFaceclawKeyboardControl(toJavaBytes(record), label) !== false;
}
/** Raw keyboard-bridge records (undecoded, possibly fragments), in arrival order. Returns an unsubscribe function. */
export function addKeyboardEventListener(session: unknown, listener: (body: Uint8Array) => void): () => void {
const communicator = session as any;
const proxy = new com.faceclaw.app.FaceclawKeyboardEventListener({
onKeyboardEvent: (body: any) => listener(toUint8Array(body)),
});
communicator.addKeyboardEventListener(proxy);
return () => {
try {
communicator.removeKeyboardEventListener(proxy);
} catch {
// The old communicator instance may be gone after a reconnect.
}
};
}
+50
View File
@@ -0,0 +1,50 @@
import { knownFolders, path as paths } from '@nativescript/core'
// Kotlin/Native facade exported by FaceclawKit (native/kotlin/shared/src/iosMain/.../IosMicRecorder.kt).
declare const FaceclawKitIosMicRecordings: any
/**
* Records 16 kHz mono PCM16 to a Matroska audio file (.mka) in
* Documents/Downloads while it is captured, through the shared Kotlin writer
* that Android's captures are recorded with. The file is written about a
* second at a time, so a crash leaves everything up to the last second.
*/
export class MicRecorder {
private constructor(private recorder: any, readonly path: string) {}
/**
* A recorder for Downloads/<name>.mka (a -2, -3... suffix if taken), or null.
* `info` describes the capture and is stored in the file.
*/
static open(name: string, info: Record<string, unknown> = {}): MicRecorder | null {
const fm = NSFileManager.defaultManager
const dir = paths.join(knownFolders.documents().path, 'Downloads')
try {
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
let path = paths.join(dir, `${name}.mka`)
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.mka`)
const recorder = FaceclawKitIosMicRecordings.shared.openPathConfigJson(path, JSON.stringify(info))
return recorder ? new MicRecorder(recorder, path) : null
} catch {
return null
}
}
append(pcm: Uint8Array): void {
if (!this.recorder || !pcm.length) return
// The handle is the buffer's start: a view into a larger buffer is copied out first.
const bytes = pcm.byteOffset === 0 && pcm.byteLength === pcm.buffer.byteLength ? pcm : pcm.slice()
try {
// Copied by the recorder before this returns.
this.recorder.appendPcm16le(NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength))
} catch {
// Stop recording rather than retry.
this.close()
}
}
close(): void {
try { this.recorder?.finish() } catch { /* already closed */ }
this.recorder = null
}
}
+29
View File
@@ -0,0 +1,29 @@
declare const android: any;
declare const java: any;
/** Extension of the recordings Save Recording writes, and Transcribe replays (Matroska audio). */
export const RECORDING_EXTENSION = ".mka";
export function isMicRecordingFile(name: string): boolean {
return name.toLowerCase().endsWith(RECORDING_EXTENSION);
}
/**
* The path for a new recording in the public Downloads directory:
* Downloads/<name>.mka, with a -2, -3... suffix if that is taken. Null when
* Downloads isn't available. The file itself is created by whoever records.
*/
export function newRecordingPath(name: string): string | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
let file = new java.io.File(downloads, `${name}${RECORDING_EXTENSION}`);
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}${RECORDING_EXTENSION}`);
return String(file.getAbsolutePath());
} catch (error) {
console.warn(`Voice: no Downloads directory for ${name}${RECORDING_EXTENSION}: ${error}`);
return null;
}
}
+153
View File
@@ -0,0 +1,153 @@
/**
* Platform-independent half of the SSH transport bindings (ssh.ts /
* ssh.ios.ts): the types the app uses and the decoding of the Go
* transport's events (native/ssh/faceclawssh.go, Listener).
*/
/** Where and as whom to connect. */
export type SshEndpoint = {
host: string;
port: number;
user: string;
/** Pinned host keys (SSH wire format, base64). Empty: first connection reports hostkey-unknown. */
hostKeys: string[];
/** Saved password, or "" to try only the device key. */
password: string;
/** Offer the device's client key. */
useKey: boolean;
};
export type SshChannelSpec =
| {
/** Run a command on the host; text is exchanged a line at a time. */
type: "exec";
command: string;
}
| {
/** WebSocket to a URL resolved on the host's side (e.g. ws://127.0.0.1:8737/). */
type: "websocket";
url: string;
};
export type SshHostKey = {
/** SSH key type, e.g. "ssh-ed25519". */
type: string;
/** OpenSSH-style "SHA256:..." fingerprint. */
fingerprint: string;
/** SSH wire format, base64: the value to pin in SshEndpoint.hostKeys. */
key: string;
};
export type SshFailureKind =
| "hostkey-unknown"
| "hostkey-mismatch"
| "auth"
| "network"
| "channel"
| "config"
| "internal";
export type SshFailure = {
kind: SshFailureKind;
message: string;
/** For hostkey-*: the key the host presented. */
hostKey?: SshHostKey;
/** For auth: the host takes passwords / we sent one / we offered our key. */
passwordAllowed?: boolean;
passwordTried?: boolean;
keyTried?: boolean;
};
export type SshClosed = {
/** Exec channels: the command's exit status (and signal, if killed). */
exitStatus?: number;
signal?: string;
/** WebSocket channels: the close frame's code and reason. */
code?: number;
reason?: string;
};
export type SshChannelListener = {
onOpen(): void;
onText(text: string): void;
onStderr?(line: string): void;
/** Terminal: the channel ended normally. */
onClosed(info: SshClosed): void;
/** Terminal: the channel couldn't open or was lost. */
onFailure(failure: SshFailure): void;
};
export type SshChannel = {
/** Queue a websocket text frame / an exec stdin line. False once closed. */
send(text: string): boolean;
/** Flush queued sends, then end the channel. No events follow. */
close(): void;
};
export type SshClientKey = {
/** "ecdsa-sha2-nistp256 AAAA... comment", for ~/.ssh/authorized_keys. */
authorizedKey: string;
fingerprint: string;
};
/** Build the Go transport's openSpec JSON. */
export function encodeSshSpec(endpoint: SshEndpoint, channel: SshChannelSpec): string {
return JSON.stringify({
host: endpoint.host,
port: endpoint.port,
user: endpoint.user,
password: endpoint.password,
hostKeys: endpoint.hostKeys,
useKey: endpoint.useKey,
channel,
});
}
/** Route one transport event to the listener. Unknown kinds are ignored. */
export function dispatchSshEvent(listener: SshChannelListener, kind: string, payload: string): void {
switch (kind) {
case "open":
listener.onOpen();
return;
case "text":
listener.onText(payload);
return;
case "stderr":
listener.onStderr?.(payload);
return;
case "closed":
listener.onClosed(parseJson(payload) ?? {});
return;
case "failure": {
const failure = parseJson(payload);
listener.onFailure(
failure && typeof failure.kind === "string"
? failure
: { kind: "internal", message: `bad failure payload: ${payload}` },
);
return;
}
}
}
function parseJson(text: string): any {
try {
return JSON.parse(text);
} catch {
return null;
}
}
/** Human summary of a failure for status lines. */
export function describeSshFailure(failure: SshFailure): string {
switch (failure.kind) {
case "hostkey-unknown":
return "Host key not yet trusted.";
case "hostkey-mismatch":
return "HOST KEY CHANGED; refusing to connect.";
case "auth":
return failure.message || "Authentication failed.";
default:
return failure.message || failure.kind;
}
}
+69
View File
@@ -0,0 +1,69 @@
import {
dispatchSshEvent,
encodeSshSpec,
type SshChannel,
type SshChannelListener,
type SshChannelSpec,
type SshClientKey,
type SshEndpoint,
} from "./ssh-common";
export * from "./ssh-common";
declare const FaceclawSshChannel: any;
declare const FaceclawSshKeys: any;
// Like socket.ios.ts: Go delivers events on its own threads, so the native
// side queues them and this isolate drains the queue on a timer.
const POLL_INTERVAL_MS = 25;
/** iOS SSH transport: FaceclawSshChannel (ObjC) over the gomobile-bound Go package in native/ssh. */
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
const native = FaceclawSshChannel.alloc().initWithSpecUseKey(encodeSshSpec(endpoint, channel), endpoint.useKey);
let ended = false;
const stop = () => {
if (ended) return;
ended = true;
clearInterval(timer);
native.close();
};
const timer = setInterval(() => {
const events = JSON.parse(String(native.takeEvents())) as Array<{ kind: string; payload: string }>;
for (const event of events) {
if (ended) break;
if (event.kind === "closed" || event.kind === "failure") {
// Terminal events: stop polling, then deliver.
ended = true;
clearInterval(timer);
}
dispatchSshEvent(listener, event.kind, event.payload);
}
}, POLL_INTERVAL_MS);
return {
send: (text) => !ended && native.send(text) === true,
close: stop,
};
}
export function sshClientKey(comment: string): SshClientKey {
const authorizedKey = FaceclawSshKeys.authorizedKeyWithComment(comment);
const fingerprint = FaceclawSshKeys.fingerprint();
if (!authorizedKey || !fingerprint) throw new Error("Could not create the SSH client key.");
return { authorizedKey: String(authorizedKey), fingerprint: String(fingerprint) };
}
export function getSshSecret(id: string): string {
return String(FaceclawSshKeys.secretForId(id) ?? "");
}
export function setSshSecret(id: string, value: string): void {
FaceclawSshKeys.setSecretForId(value, id);
}
export function removeSshSecret(id: string): void {
FaceclawSshKeys.removeSecretForId(id);
}
export function copyToPhoneClipboard(_label: string, text: string): void {
FaceclawSshKeys.copyToClipboard(text);
}
+66
View File
@@ -0,0 +1,66 @@
import { Utils } from "@nativescript/core";
import {
dispatchSshEvent,
encodeSshSpec,
type SshChannel,
type SshChannelListener,
type SshChannelSpec,
type SshClientKey,
type SshEndpoint,
} from "./ssh-common";
export * from "./ssh-common";
declare const com: any;
/**
* Android SSH transport: FaceclawSshChannel (Kotlin) over the gomobile-bound
* Go package in native/ssh. Callable from the main isolate or a worker;
* events arrive on the calling isolate's thread.
*/
export function openSshChannel(endpoint: SshEndpoint, channel: SshChannelSpec, listener: SshChannelListener): SshChannel {
let closed = false;
const proxy = new com.faceclaw.app.FaceclawSshListener({
onEvent: (kind: string, payload: string) => {
if (closed) return;
dispatchSshEvent(listener, String(kind), payload == null ? "" : String(payload));
},
});
const native = new com.faceclaw.app.FaceclawSshChannel(encodeSshSpec(endpoint, channel), endpoint.useKey, proxy);
return {
send: (text) => !closed && native.send(text) === true,
close: () => {
if (closed) return;
closed = true;
native.close();
},
};
}
function context(): any {
return Utils.android.getApplicationContext();
}
/** The device's SSH client key (generated on first use). */
export function sshClientKey(comment: string): SshClientKey {
return {
authorizedKey: String(com.faceclaw.app.FaceclawSshKeys.authorizedKey(comment)),
fingerprint: String(com.faceclaw.app.FaceclawSshKeys.fingerprint()),
};
}
export function getSshSecret(id: string): string {
return String(com.faceclaw.app.FaceclawSshKeys.getSecret(context(), id) ?? "");
}
export function setSshSecret(id: string, value: string): void {
com.faceclaw.app.FaceclawSshKeys.setSecret(context(), id, value);
}
export function removeSshSecret(id: string): void {
com.faceclaw.app.FaceclawSshKeys.removeSecret(context(), id);
}
export function copyToPhoneClipboard(label: string, text: string): void {
com.faceclaw.app.FaceclawSshKeys.copyToClipboard(context(), label, text);
}
+59
View File
@@ -0,0 +1,59 @@
/**
* Session-list and connection-state types shared by the Terminal app's
* backends (g2mirror-client.ts, apps/terminal/tmux-client.ts). Kept free of
* platform imports so the tmux client can be unit-tested under node.
*/
export type G2MirrorSession = {
socket: string;
pid: number;
cwdHint: string;
/** Unix epoch ms of the terminal's last bell, or null if none observed. */
lastBellAt: number | null;
/**
* Unix epoch ms (host clock) of the app's last output, or null if none
* observed. Fresh to within the wrapper's 2s activity reporting interval
* while we're connected; compare against other host timestamps, not the
* phone clock (see the worker's local-receive-time activity tracking).
*/
lastOutputAt: number | null;
/** Window title the app last set (xterm OSC 0/2), or null if none observed. */
title: string | null;
};
export type G2MirrorPhase =
| "idle"
| "connecting"
| "connected" // handshake accepted; can list/attach
| "attached" // relaying to one session
| "failed";
export type G2MirrorState = {
phase: G2MirrorPhase;
status: string;
sessions: G2MirrorSession[];
attachedCommand: string;
/**
* server_name from the init success reply: a human-readable name for the
* machine the server runs on. "" until the handshake succeeds (or when an
* older server omits the field).
*/
serverName: string;
/**
* False when reconnecting can't help until the user changes something on
* the host (e.g. tmux missing or too old). Absent means retryable.
*/
retryable?: boolean;
};
/** A page of archived scrollback lines (see PROTOCOL.md "Scrollback history"). */
export type G2MirrorHistoryReply = {
/** Absolute index of lines[0]; the page covers [start, start + lines.length). */
start: number;
/** Oldest index still retained by the wrapper (older requests return nothing). */
oldest: number;
/** Current splice point (next line to be archived). */
next: number;
/** Plain text, ANSI-stripped, oldest-to-newest. */
lines: string[];
};
+8 -5
View File
@@ -3,7 +3,7 @@ import type { CloudSttTranscriptEvent } from './cloud-stt'
import type { ReconnectingSttClient } from './reconnecting-stt'
import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider'
import { createSpeechAnalyzerClient, speechAnalyzerSupported } from './speech-analyzer-stt.ios'
import { WavRecorder } from './wav-recorder.ios'
import { MicRecorder } from './mic-recorder.ios'
import { pcmLevel } from '../util/audio-level'
declare const FaceclawSpeech: any
@@ -57,11 +57,11 @@ export class IosVoiceControlBridge {
private continuous = false
// Shown instead of the native "ended" status once a drain the bridge started completes.
private drainStatus: string | null = null
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = ''
private recorder: WavRecorder | null = null
private recorder: MicRecorder | null = null
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false
private readonly recordingListeners = new Set<() => void>()
@@ -209,7 +209,7 @@ export class IosVoiceControlBridge {
/** Where a listening continuous capture gets its audio; null when none is listening. */
get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null }
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
* until called with another name or ''. Only cloud captures supply PCM.
*/
recordTo(name: string): void {
@@ -232,7 +232,10 @@ export class IosVoiceControlBridge {
private record(pcm: Uint8Array): void {
if (!this.recording) return
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName)
this.recorder = MicRecorder.open(this.recordingName, {
recording: this.recordingName, startedAt: new Date().toISOString(), platform: 'ios',
source: this.source === 'phone' ? 'phone' : 'glasses', mics: 1,
})
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
}
+316 -24
View File
@@ -1,9 +1,9 @@
import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { type CloudSttTranscriptEvent, CloudSttClient, toJavaBytes } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { newRecordingPath } from "./mic-recording";
import { SpeechPauseDetector } from "./speech-pause";
import { WavRecorder } from "./wav-recorder";
import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level";
@@ -84,6 +84,49 @@ export type PushToTalkOptions = {
* frame). Null/absent leaves the mic omnidirectional.
*/
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
/**
* Capture from the temples' own microphones (the custom firmware's
* mic_control stream) instead of the glasses' mixed stream. Each temple
* streams its microphones on its own link and clock; the phone mixes them
* into the mono feed that is transcribed. Null/absent keeps the mixed
* stream; ignored with usePhoneMic.
*/
micArray?: MicArrayCapture | null;
/**
* Play this recording (a .mka file from Save Recording) in real time in
* place of any microphone. The capture needs no glasses, and ends by
* itself when the recording does.
*/
replay?: { path: string; startMs: number } | null;
/** What the caller knows of the capture, for the description stored in a recording of it. */
captureInfo?: Record<string, unknown>;
};
export type MicArrayCapture = {
/** The mic_control CONFIGURE record both temples are armed with (see mic-protocol.ts). */
configure: Uint8Array;
};
/** Progress of a capture that replays a recording; positions are ms into it. */
export type VoiceReplayEvent = {
/**
* `start` once the file is open, `position` about once per second played,
* `seek` when a jump took effect, `end` when the recording ran out, and
* `error` when it could not be opened (the status says why).
*/
kind: "start" | "position" | "seek" | "end" | "error";
positionMs: number;
durationMs: number;
};
/** What a listening view (Transcribe, Translate) takes from a capture: the microphone's, or a replay's. */
export type VoiceEventSource = {
onStatus(listener: (state: VoiceControlState) => void): () => void;
onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void;
onSpeechPause(listener: () => void): () => void;
onMicLevel(listener: (level: number) => void): () => void;
readonly recording: boolean;
readonly recordingFailed: boolean;
};
/** Who currently wants the mic running. */
@@ -130,9 +173,13 @@ export class FaceclawVoiceControlBridge {
private captureEndpointing = false;
// Non-null while a cloud provider owns the transcript; Java only decodes PCM.
private cloudClient: CloudSttClient | null = null;
// What the running STT stream transcribes with (transcriptionKey); a holder
// asking for something else restarts the stream rather than sharing it.
// What the running STT stream captures from and transcribes with
// (streamKey); a holder asking for something else restarts the stream
// rather than sharing it.
private streamProvider: string | null = null;
// The options of the running stream, for the description of a recording of it.
private streamOptions: PushToTalkOptions | null = null;
private readonly replayListeners = new Set<(event: VoiceReplayEvent) => void>();
// Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the
// controller runs in the decode-only "cloud" mode and every decoded PCM
// frame is broadcast to these listeners. STT capture preempts it — a live
@@ -150,11 +197,14 @@ export class FaceclawVoiceControlBridge {
// the bridge swallow it.
private verificationActive = false;
private verificationRejected = false;
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
// Continuous captures' audio goes to Downloads/<recordingName>.mka while it is
// set: every source's chunks as they arrive, before any processing, written
// by the controller. The file outlives a capture, so a restarted one keeps
// adding to it, and it is created by the first audio, so a failed start
// leaves no file.
private recordingName = "";
private recorder: WavRecorder | null = null;
// The file is open in the controller.
private recordingOpen = false;
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false;
private readonly recordingListeners = new Set<() => void>();
@@ -251,16 +301,55 @@ export class FaceclawVoiceControlBridge {
}
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* End the continuous capture as an utterance: stop it and have the
* transcript of its last audio come as a final. For a capture whose audio
* ran out by itself (a replay).
*/
finishContinuousCapture(): void {
this.releaseCapture("continuous", true);
}
/** Progress of a capture that replays a recording (PushToTalkOptions.replay). */
onReplay(listener: (event: VoiceReplayEvent) => void): () => void {
this.replayListeners.add(listener);
return () => this.replayListeners.delete(listener);
}
/** Continue the running replay from `positionMs` into its recording. */
seekReplay(positionMs: number): void {
this.controller?.seekReplay(Math.max(0, Math.round(positionMs)));
}
/** Hold the running replay where it is, or let it go on. */
setReplayPaused(paused: boolean): void {
this.controller?.setReplayPaused(paused);
}
/** Whether a capture is running and still has audio coming (a replay stops being so at its end). */
isCaptureLive(): boolean {
return this.micIsLive();
}
/**
* Record continuous captures' audio to Downloads/<name>.mka, across restarts,
* until called with another name or "".
*/
recordTo(name: string): void {
if (name === this.recordingName) return;
this.recorder?.close();
this.recorder = null;
this.closeRecording();
this.setRecording(name, false);
}
private closeRecording(): void {
if (!this.recordingOpen) return;
this.recordingOpen = false;
try {
this.controller?.stopRecording();
} catch (error) {
console.warn(`Voice: closing the recording failed: ${error}`);
}
}
/** Whether continuous captures' audio is being saved (see recordTo). */
get recording(): boolean {
return this.recordingName !== "" && !this.recordingFailure;
@@ -286,18 +375,48 @@ export class FaceclawVoiceControlBridge {
for (const listener of Array.from(this.recordingListeners)) listener();
}
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
private record(pcm: any): void {
if (!this.recording) return;
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName);
if (!this.recorder) {
/**
* Audio arrived for a continuous capture: open the recording asked for, if
* it isn't open yet. The controller writes it from then on, as the audio
* arrives on its own thread.
*/
private ensureRecording(): void {
if (!this.recording || this.recordingOpen) return;
const path = newRecordingPath(this.recordingName);
let opened = false;
try {
opened = path !== null && Boolean(this.controller?.startRecording(path, JSON.stringify(this.describeCapture())));
} catch (error) {
console.warn(`Voice: could not create the recording: ${error}`);
}
if (!opened) {
this.setRecording(this.recordingName, true);
return;
}
console.log(`Voice: recording audio to ${this.recorder.path}`);
this.recordingOpen = true;
console.log(`Voice: recording audio to ${path}`);
}
this.recorder.append(pcm);
/** Only captures a continuous holder has a part in are recorded: push-to-talk on its own is not. */
private syncRecordingEnabled(): void {
this.controller?.setRecordingEnabled(this.captureHolders.has("continuous"));
}
/** The running capture, as the JSON stored in a recording of it. */
private describeCapture(): Record<string, unknown> {
const options = this.streamOptions;
return {
recording: this.recordingName,
startedAt: new Date().toISOString(),
source: options?.usePhoneMic ? "phone" : "glasses",
micArray: Boolean(options?.micArray) && !options?.usePhoneMic,
transcription: { provider: options?.provider ?? "", language: options?.language ?? "" },
processing: {
noiseSuppression: Boolean(options?.noiseSuppression),
beamFilter: options?.beamFilter ?? null,
},
...options?.captureInfo,
};
}
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
@@ -364,6 +483,8 @@ export class FaceclawVoiceControlBridge {
// preview-mode capture may have left on the shared controller.
this.controller?.setUsePhoneMic(false);
this.controller?.setEndpointing(false);
this.controller?.setMicArray(null);
this.controller?.setReplay(null, 0);
// Raw means raw: the Microphones session (and EvenHub apps) get the
// unprocessed stream and run their own beam/suppression processing.
this.controller?.setNoiseSuppression(false);
@@ -399,18 +520,33 @@ export class FaceclawVoiceControlBridge {
const micLive = this.micIsLive();
this.suspendedHolders.delete(holder);
this.captureHolders.add(holder);
if (micLive && this.streamProvider === transcriptionKey(options)) {
if (micLive && this.streamProvider === streamKey(options)) {
// Mic already running; the new holder just shares the existing stream
// (transcripts are already broadcast to its listeners).
this.syncRecordingEnabled();
return;
}
// Any leftover capture is dead, or transcribes with another provider than
// Any leftover capture is dead, or captures or transcribes otherwise than
// this holder needs; drop it before starting the new one.
if (this.started) this.teardownCapture();
this.streamProvider = transcriptionKey(options);
this.streamProvider = streamKey(options);
this.streamOptions = options;
this.ensureController();
this.syncRecordingEnabled();
// A recording under way carries on into this capture: note what changed.
if (this.recordingOpen) {
try {
this.controller?.noteRecordingEvent(JSON.stringify({ event: "capture", ...this.describeCapture() }));
} catch (error) {
console.warn(`Voice: recording event failed: ${error}`);
}
}
this.controller?.setCommunicator(options.communicator);
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
this.controller?.setMicArray(
options.micArray && !options.usePhoneMic && !options.replay ? toJavaBytes(options.micArray.configure) : null,
);
this.controller?.setReplay(options.replay?.path ?? null, Math.max(0, Math.round(options.replay?.startMs ?? 0)));
this.controller?.setEndpointing(Boolean(options.endpointing));
this.captureEndpointing = Boolean(options.endpointing);
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
@@ -435,7 +571,7 @@ export class FaceclawVoiceControlBridge {
this.speechPause.reset();
// Replaces whatever the last capture left behind ("Listening...", an
// error, the permission prompt) until the controller reports progress.
this.setStatus("Starting microphone...");
this.setStatus(options.replay ? "Opening the recording..." : "Starting microphone...");
// A previous push-to-talk commit may still be awaiting its final result.
this.cloudClient?.stop();
const cloudClient = this.createCloudClient(options);
@@ -480,6 +616,7 @@ export class FaceclawVoiceControlBridge {
if (commit) this.cloudClient?.finish();
return;
}
this.syncRecordingEnabled();
if (this.captureHolders.size > 0) {
// Another holder still wants the mic; keep it running.
return;
@@ -504,6 +641,7 @@ export class FaceclawVoiceControlBridge {
this.captureHolders.clear();
this.suspendedHolders.clear();
this.suspendedRaw = false;
this.syncRecordingEnabled();
this.teardownCapture();
this.setStatus("Voice control stopped.");
}
@@ -521,6 +659,7 @@ export class FaceclawVoiceControlBridge {
this.suspendedHolders.add(holder);
}
this.captureHolders.clear();
this.syncRecordingEnabled();
this.suspendedRaw = this.rawActive;
this.teardownCapture();
if (this.suspendedHolders.size > 0) {
@@ -578,6 +717,7 @@ export class FaceclawVoiceControlBridge {
private teardownCapture(): void {
this.rawActive = false;
this.streamProvider = null;
this.streamOptions = null;
if (global.isAndroid) {
this.controller?.stop();
}
@@ -611,7 +751,7 @@ export class FaceclawVoiceControlBridge {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
// Push-to-talk on a capture of its own is not recorded.
if (this.captureHolders.has("continuous")) this.record(pcm);
if (this.captureHolders.has("continuous")) this.ensureRecording();
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener();
}
@@ -642,6 +782,10 @@ export class FaceclawVoiceControlBridge {
);
}
},
onReplay: (kind: string, positionMs: number, durationMs: number) => {
const event = { kind: String(kind) as VoiceReplayEvent["kind"], positionMs: Number(positionMs), durationMs: Number(durationMs) };
for (const listener of Array.from(this.replayListeners)) listener(event);
},
});
this.controller.setListener(this.listenerProxy);
}
@@ -678,6 +822,21 @@ export class FaceclawVoiceControlBridge {
}
}
/**
* What a capture reads and transcribes with. A running capture is shared only
* by holders asking for the same: the provider and language, the microphones
* (the mixed stream, or the temples' own under a given configuration), or the
* recording it replays.
*/
function streamKey(options: PushToTalkOptions): string {
const source = options.replay
? `replay:${options.replay.path}`
: options.micArray && !options.usePhoneMic
? `array:${Array.from(options.micArray.configure).join(",")}`
: "mixed";
return `${transcriptionKey(options)}|${source}`;
}
function toJavaFloats(values: Float32Array): any {
const javaFloats = Array.create("float", values.length);
for (let i = 0; i < values.length; i++) {
@@ -687,3 +846,136 @@ function toJavaFloats(values: Float32Array): any {
}
export const voiceControlBridge = new FaceclawVoiceControlBridge();
export type VoiceReplayState = "opening" | "playing" | "paused" | "ended" | "failed";
/**
* A recording played back as a capture of its own: transcribed by the same
* pipeline as the microphone, with the same settings, on a bridge and native
* controller separate from the microphone's. Nothing here touches the mic,
* so voice input and a live Transcribe window carry on beside a replay.
*/
export class VoiceReplay {
private readonly bridge = new FaceclawVoiceControlBridge();
private readonly listeners = new Set<(jumpedToMs?: number) => void>();
private readonly offReplay: () => void;
private position = 0;
private duration = 0;
private current: VoiceReplayState = "opening";
private closed = false;
/**
* `options` gives the capture options (provider, keys, processing) each
* time the replay starts, so a restart picks up changed settings.
*/
constructor(readonly path: string, private readonly options: () => PushToTalkOptions) {
this.offReplay = this.bridge.onReplay((event) => this.handle(event));
}
/** The replay's status, transcript and level events. */
get events(): VoiceEventSource {
return this.bridge;
}
/** Milliseconds into the recording. */
get positionMs(): number {
return this.position;
}
/** The recording's length in ms; 0 until it has been opened. */
get durationMs(): number {
return this.duration;
}
get state(): VoiceReplayState {
return this.current;
}
/**
* The position, duration or state changed. `jumpedToMs` is given when
* playback moved there other than by playing on: a jump, or a start
* part-way in. It comes after the transcript of what played before.
*/
onChange(listener: (jumpedToMs?: number) => void): () => void {
this.listeners.add(listener);
return () => this.listeners.delete(listener);
}
/** Play from `positionMs` into the recording. */
start(positionMs = 0): void {
if (this.closed) return;
this.position = Math.max(0, positionMs);
this.set("opening");
this.bridge.startContinuousCapture({
...this.options(),
communicator: null,
usePhoneMic: false,
micArray: null,
replay: { path: this.path, startMs: this.position },
});
}
/** Continue from `positionMs`: a jump while playing, a fresh start once the replay has ended. */
jumpTo(positionMs: number): void {
if (this.closed) return;
const target = Math.max(0, this.duration > 0 ? Math.min(positionMs, this.duration) : positionMs);
if ((this.current === "playing" || this.current === "paused") && this.bridge.isCaptureLive()) {
this.bridge.seekReplay(target);
this.position = target;
this.notify();
return;
}
this.start(target);
}
setPaused(paused: boolean): void {
if (this.closed || (this.current !== "playing" && this.current !== "paused")) return;
this.bridge.setReplayPaused(paused);
this.set(paused ? "paused" : "playing");
}
/** Stop for good and release the capture. */
close(): void {
if (this.closed) return;
this.closed = true;
this.offReplay();
this.listeners.clear();
this.bridge.stop();
}
private handle(event: VoiceReplayEvent): void {
this.duration = event.durationMs;
switch (event.kind) {
case "start":
this.position = event.positionMs;
this.current = "playing";
this.notify(event.positionMs > 0 ? event.positionMs : undefined);
return;
case "seek":
this.position = event.positionMs;
this.notify(event.positionMs);
return;
case "end":
this.position = event.durationMs;
// Its last audio is still to be transcribed (a cloud provider's final).
this.bridge.finishContinuousCapture();
this.set("ended");
return;
case "error":
this.set("failed");
return;
default:
this.position = event.positionMs;
this.notify();
}
}
private set(state: VoiceReplayState): void {
this.current = state;
this.notify();
}
private notify(jumpedToMs?: number): void {
for (const listener of Array.from(this.listeners)) listener(jumpedToMs);
}
}
-22
View File
@@ -1,22 +0,0 @@
/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */
export function wavHeader(dataBytes: number): Uint8Array {
const header = new Uint8Array(44);
const view = new DataView(header.buffer);
const ascii = (offset: number, text: string) => {
for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i);
};
ascii(0, "RIFF");
view.setUint32(4, 36 + dataBytes, true);
ascii(8, "WAVE");
ascii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true); // PCM
view.setUint16(22, 1, true); // mono
view.setUint32(24, 16000, true);
view.setUint32(28, 32000, true); // byte rate
view.setUint16(32, 2, true); // block align
view.setUint16(34, 16, true); // bits per sample
ascii(36, "data");
view.setUint32(40, dataBytes, true);
return header;
}
-75
View File
@@ -1,75 +0,0 @@
import { knownFolders, path as paths } from '@nativescript/core'
import { wavHeader } from './wav-header'
// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet.
const FLUSH_BYTES = 32000
/**
* Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is
* captured. The header's sizes are rewritten on every flush, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private pending: Uint8Array[] = []
private pendingBytes = 0
private dataBytes = 0
private constructor(private handle: NSFileHandle | null, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
const fm = NSFileManager.defaultManager
const dir = paths.join(knownFolders.documents().path, 'Downloads')
try {
fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null)
let path = paths.join(dir, `${name}.wav`)
for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`)
if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null
const handle = NSFileHandle.fileHandleForWritingAtPath(path)
if (!handle || !handle.seekToEndReturningOffsetError(null)) return null
return new WavRecorder(handle, path)
} catch {
return null
}
}
append(pcm: Uint8Array): void {
if (!this.handle || !pcm.length) return
this.pending.push(pcm.slice())
this.pendingBytes += pcm.length
if (this.pendingBytes >= FLUSH_BYTES) this.flush()
}
close(): void {
this.flush()
try { this.handle?.closeAndReturnError() } catch { /* already closed */ }
this.handle = null
}
private flush(): void {
const handle = this.handle
if (!handle || !this.pendingBytes) return
const chunk = new Uint8Array(this.pendingBytes)
let offset = 0
for (const part of this.pending) { chunk.set(part, offset); offset += part.length }
this.pending = []
this.pendingBytes = 0
try {
// NativeScript throws a failed call's NSError (storage full, file removed).
handle.writeDataError(toData(chunk))
handle.seekToOffsetError(0)
handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length)))
handle.seekToEndReturningOffsetError(null)
this.dataBytes += chunk.length
return
} catch {
// Stop recording rather than retry.
}
try { handle.closeAndReturnError() } catch { /* ignore */ }
this.handle = null
}
}
function toData(bytes: Uint8Array): NSData {
return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength)
}
-42
View File
@@ -1,42 +0,0 @@
declare const android: any;
declare const java: any;
declare const com: any;
const SAMPLE_RATE = 16000;
/**
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
* native recorder keeps the header's sizes current, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private constructor(private recorder: any, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
let file = new java.io.File(downloads, `${name}.wav`);
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
const path = String(file.getAbsolutePath());
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
} catch (error) {
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
return null;
}
}
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
append(pcm: any): void {
this.recorder?.append(pcm);
}
close(): void {
this.recorder?.finish();
this.recorder = null;
}
}
+37 -5
View File
@@ -947,7 +947,7 @@ export const useMicControlSetting = new ConfigSettingBoolean({
storageKey: "developer.useMicControl",
defaultValue: true,
description:
"Use the custom firmware's per-temple mic-control channel for the Microphones app's array capture. When off, use the standard single mixed stream.",
"Use the custom firmware's per-temple mic-control channel for capturing the microphones separately (the Microphones app's array capture, and Transcribe's Microphones option). When off, use the standard single mixed stream.",
});
export const showBleBandwidthSetting = new ConfigSettingBoolean({
@@ -1054,7 +1054,9 @@ export const saveRecordingSetting = new ConfigSettingBoolean({
label: "Save Recording",
storageKey: "voice.saveRecording",
defaultValue: false,
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
description: global.isIOS
? "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them."
: "Record the audio of Transcribe and Translate sessions as Matroska audio (.mka) files in Downloads: about 2 MB per minute, or 10 MB with all four microphones. Named after the app and the session's start time, and kept until you delete them. Transcribe can replay one: pick it in the Files app.",
});
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
@@ -1338,11 +1340,28 @@ export const terminalNewConnectionSetting = new ConfigSettingString({
label: "New connection",
storageKey: "terminal.newConnectionDraft",
defaultValue: "",
editorTitle: "g2mirror connection string (g2mirror://token@host)",
editorTitle: "Connection (ssh://user@host or g2mirror://token@host)",
glassesEditTitle: "Add connection",
normalize: (value) => (value ?? "").replace(/[\x00-\x1f]+/g, "").trim(),
});
/**
* Staging buffer for the Terminal app's SSH password prompt: the worker opens
* the phone's (masked) text editor on it, then moves what was typed into the
* platform keystore (app/native/ssh) and clears it as soon as the user
* confirms or cancels on the glasses. Not listed in the Settings app.
*/
export const terminalSshPasswordSetting = new ConfigSettingString({
id: "terminal-ssh-password",
label: "SSH password",
storageKey: "terminal.sshPasswordDraft",
defaultValue: "",
inputKind: "password",
editorTitle: "SSH password",
glassesEditTitle: "SSH password",
normalize: (value) => (value ?? "").replace(/[\r\n]+/g, ""),
});
/**
* Staging buffer for the T3 Code app's Pair-environment screen: the phone
* text editor types the `t3 pair` link into it, and the app reads it back
@@ -1722,9 +1741,18 @@ type SettingsMenuOptions<T> = {
label?: string
}
type EnumSettingMenuOptions<T> = SettingsMenuOptions<T> & {
/**
* A live line under the picker's values, such as a sensor reading to pick
* by. open() runs as the picker opens, with a function that repaints it,
* and returns what to run when the picker closes.
*/
pickerFooter?: { text: () => string; open: (repaint: () => void) => () => void }
}
export function enumSettingMenuItem<TValue extends string, TId extends string = string>(
setting: ConfigSettingEnum<TValue, TId>,
opts?: SettingsMenuOptions<TValue>
opts?: EnumSettingMenuOptions<TValue>
): MenuItem {
const label = opts?.label ?? setting.label;
return {
@@ -1752,7 +1780,11 @@ export function enumSettingMenuItem<TValue extends string, TId extends string =
);
},
}));
openModalMenu(ctx, label, items, Math.max(0, setting.values.indexOf(current)));
const footer = opts?.pickerFooter;
openModalMenu(ctx, label, items, Math.max(0, setting.values.indexOf(current)), {
footer: footer?.text,
onClosed: footer?.open(() => ctx.actions.requestRender()),
});
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, setting.displayValue(setting.get()));
+10
View File
@@ -103,6 +103,8 @@ import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from ".
import { uiFontPickerMenuItem } from "../font-picker";
import { NotificationFilterLayer } from "../notification-filter";
import { compatibleModelDiscoveryMenuItem } from "../openai-compatible-model-picker";
import { keyboardSettingsMenuItem } from "../keyboard-settings";
import { keybindingsMenuItem } from "../keybinding-settings";
import { isVeniceApiBaseUrl } from "../../native/openai-chat";
/** The Settings section the Notifications app's "Settings" menu item opens. */
@@ -242,6 +244,14 @@ function settingsSections(): SettingsSection[] {
],
renderDetail: renderWatchStatus,
},
// A Bluetooth keyboard paired with the glasses themselves (the G2's
// custom firmware hosts it; app/g2/keyboard/). Hidden rather than left
// out: the Settings window can be open across a change of glasses.
{
label: "Keyboard",
items: [keyboardSettingsMenuItem(), keybindingsMenuItem()],
hidden: () => glassesTypeSetting.get() !== "g2",
},
{
label: "Developer",
items: [
+30 -2
View File
@@ -28,6 +28,13 @@ export type SettingsSection = {
height: number;
ctx: LayerContext;
}) => number | void;
/**
* Leave the section out of the list while this returns true. Re-read on
* every paint and input, for a section that only applies with some other
* setting (the paired glasses' type): the Settings window outlives changes
* to those.
*/
hidden?: () => boolean;
};
const LEFT_W = 150;
@@ -66,7 +73,7 @@ export class SettingsPanelLayer implements Layer {
constructor(private readonly sections: SettingsSection[]) {
this.leftMenu = new Menu<SettingsSection>({
items: sections,
items: sections.filter((section) => !section.hidden?.()),
wrap: true,
rowGap: ROW_GAP,
highlight: { radius: 6 },
@@ -108,9 +115,28 @@ export class SettingsPanelLayer implements Layer {
return this.leftMenu.selectedItem ?? this.sections[0]!;
}
/**
* Bring the left column in line with which sections are hidden now. The
* highlight stays on its section; if that one just went away it moves to
* the list's start, with focus back in the left column.
*/
private syncSections(): void {
const visible = this.sections.filter((section) => !section.hidden?.());
const shown = this.leftMenu.items;
if (visible.length === shown.length && visible.every((section, index) => section === shown[index])) return;
const selected = this.leftMenu.selectedItem;
const index = selected ? visible.indexOf(selected) : -1;
this.leftMenu.setItems(visible, Math.max(0, index));
if (index < 0) {
this.focus = "left";
this.resetRight();
}
}
/** Select a left-column section by label (deep link, e.g. from an app's menu). */
focusSection(label: string): void {
const index = this.sections.findIndex((section) => section.label === label);
this.syncSections();
const index = this.leftMenu.items.findIndex((section) => section.label === label);
if (index < 0) return;
this.leftMenu.select(index);
this.focus = "left";
@@ -123,6 +149,7 @@ export class SettingsPanelLayer implements Layer {
const { width, height, padding } = viewport;
const image = new GrayImage(width, height, 0);
const appFocused = ctx.stack.isFocused();
this.syncSections();
const section = this.section();
const rightItems = section.items;
this.rightMenu.setItems(rightItems);
@@ -228,6 +255,7 @@ export class SettingsPanelLayer implements Layer {
}
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
this.syncSections();
const scroll = scrollEventFor(event);
if (this.focus === "left") {
if (scroll) {
+147
View File
@@ -0,0 +1,147 @@
import { getDefaultSmallFont } from "../graphics/ui-fonts";
import {
DEFAULT_KEYS_HELP,
KEY_ACTIONS,
chordId,
describeChord,
keyActionLabel,
type KeyChord,
} from "../g2/keyboard/bindings";
import { keyboardService } from "../g2/keyboard/service";
import { TextViewerLayer } from "../apps/files/text-viewer";
import type { LayerContext } from "./layers";
import { drawRightValueMenuItem, MenuLayer, openModalMenu, submenuItem, TextPageLayer, type MenuItem } from "./menu";
/**
* Settings > Keyboard > Keybindings: the custom keybindings (a key, with
* any modifiers held, to a gesture), which come before the built-in
* meanings in app/g2/keyboard/bindings.ts. Adding one asks for the key on
* the keyboard itself, so it works for a remote whose buttons send who
* knows what.
*/
/** Waits for the key to bind. Double tap backs out, as from any page. */
class KeyCaptureLayer extends TextPageLayer {
private cancel: (() => void) | null = null;
constructor(private readonly onChord: (chord: KeyChord, ctx: LayerContext) => void) {
super(
"Add keybinding",
"Press the key to bind, on the keyboard: a key, a mouse button, or a turn of the wheel. Hold Shift, Ctrl or Alt with it to bind the combination.",
);
}
open(ctx: LayerContext): void {
ctx.stack.push(this);
this.cancel = keyboardService.captureNextKey((chord) => {
this.cancel = null;
ctx.stack.popThrough(this);
this.onChord(chord, ctx);
ctx.actions.requestRender();
});
}
onRemoved(): void {
this.cancel?.();
this.cancel = null;
}
}
class KeybindingsLayer extends MenuLayer {
constructor(viewport: { width: number; height: number }) {
super("Keybindings", [], {
x: "center",
y: 8,
width: Math.min(320, viewport.width - 40),
minHeight: Math.min(160, viewport.height - 40),
maxHeight: Math.min(240, viewport.height - 40),
depth: 4,
});
}
open(ctx: LayerContext): void {
this.updateRows();
ctx.stack.push(this);
}
/** Rebuild the rows, with the highlight on `selectId`'s keybinding when given. */
private updateRows(selectId?: string): void {
const bindings = keyboardService.keyBindings();
const rows: MenuItem[] = bindings.map((binding) => {
const label = describeChord(binding.chord);
return {
label,
onSelect: (ctx: LayerContext) => this.pickAction(ctx, binding.chord),
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, keyActionLabel(binding.action));
},
};
});
rows.push(
{ label: "Add keybinding", onSelect: (ctx) => this.add(ctx) },
// Scrolls, where a page would cut the text off in a short window.
submenuItem("Default keys", (ctx) => ctx.stack.push(new TextViewerLayer(DEFAULT_KEYS_HELP, "Default keys"))),
);
const selected = selectId === undefined ? -1 : bindings.findIndex((binding) => chordId(binding.chord) === selectId);
if (selected >= 0) this.setItems(rows, selected);
else this.setItems(rows);
}
private add(ctx: LayerContext): void {
if (!keyboardService.bridge.isConnected()) {
ctx.stack.push(new TextPageLayer(
"Add keybinding",
"No keyboard is connected. Pair or reconnect one under Bluetooth keyboard, then come back: the key to bind is read from the keyboard.",
));
return;
}
new KeyCaptureLayer((chord, captureCtx) => this.pickAction(captureCtx, chord)).open(ctx);
}
/** Choose what `chord` does: for a new keybinding, or to change or remove one. */
private pickAction(ctx: LayerContext, chord: KeyChord): void {
const id = chordId(chord);
const current = keyboardService.keyBindings().find((binding) => chordId(binding.chord) === id)?.action;
const done = (inner: LayerContext): void => {
inner.stack.pop();
this.updateRows(id);
inner.actions.requestRender();
};
const rows: MenuItem[] = KEY_ACTIONS.map((entry) => ({
label: entry.action === current ? `${entry.label} *` : entry.label,
onSelect: (inner: LayerContext) => {
if (!keyboardService.setKeyBinding(chord, entry.action)) {
inner.stack.pop();
inner.stack.push(new TextPageLayer("Keybindings", "There is no room for another keybinding. Remove one first."));
return;
}
done(inner);
},
}));
if (current !== undefined) {
rows.push({
label: "Remove keybinding",
onSelect: (inner) => {
keyboardService.removeKeyBinding(chord);
done(inner);
},
});
}
const selected = Math.max(0, KEY_ACTIONS.findIndex((entry) => entry.action === current));
openModalMenu(ctx, describeChord(chord), rows, selected);
}
}
/** The Settings row that opens the keybindings menu. */
export function keybindingsMenuItem(): MenuItem {
return {
label: "Keybindings",
description:
"Choose what keys do: arrows, Enter, Esc and a mouse's buttons and wheel navigate by default, and any key (or a remote's button) can be bound to select, back, a swipe, a scroll or a menu.",
onSelect: (ctx) => new KeybindingsLayer(ctx.stack.getBaseSize()).open(ctx),
render: ({ image, x, y, width }) => {
const count = keyboardService.keyBindings().length;
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "Keybindings", count ? `${count} custom` : "Default");
},
};
}
+232
View File
@@ -0,0 +1,232 @@
import { getDefaultSmallFont } from "../graphics/ui-fonts";
import type { KeyboardBridgeSnapshot } from "../g2/keyboard/bridge";
import type { KeyboardAdvert } from "../g2/keyboard/protocol";
import { keyboardService } from "../g2/keyboard/service";
import { ConfigSettingString, textSettingMenuItem } from "./dashboard-settings";
import type { LayerContext } from "./layers";
import { drawRightValueMenuItem, MenuLayer, type MenuItem } from "./menu";
/**
* Settings > Keyboard: pairing a Bluetooth keyboard with the glasses (the
* CFW keyboard bridge, app/g2/keyboard/). The keyboard pairs with the
* glasses, not the phone, so all of it happens here, on the glasses: scan,
* pick, and for a keyboard that wants one, the passkey to type on it.
*/
/** The code a keyboard with a display shows, typed on the phone. Scratch, not a preference. */
const pairingCodeSetting = new ConfigSettingString({
id: "keyboardPairingCode",
storageKey: "keyboard.pairingCode",
label: "Code on keyboard",
defaultValue: "",
normalize: (value) => (value ?? "").replace(/\D/g, "").slice(0, 6),
});
const sixDigits = (value: number): string => {
const text = String(value).padStart(6, "0");
return `${text.slice(0, 3)} ${text.slice(3)}`;
};
function advertLabel(advert: KeyboardAdvert): string {
const name = advert.name.trim() || advert.address;
return name.length > 30 ? `${name.slice(0, 27)}...` : name;
}
/** One line saying where the keyboard stands, for the Settings row and the menu's first row. */
export function keyboardStatusText(snapshot: KeyboardBridgeSnapshot): string {
const name = snapshot.activeName || snapshot.saved?.name || "keyboard";
switch (snapshot.state) {
case "unavailable":
return snapshot.saved ? "Glasses not connected" : "Not set up";
case "scanning":
return "Scanning...";
case "connecting":
return `Connecting to ${name}...`;
case "pairing":
return `Pairing with ${name}...`;
case "configuring":
return `Setting up ${name}...`;
case "connected":
return `Connected: ${name}`;
case "error":
return snapshot.error || "Failed";
default:
if (snapshot.waiting) return `Waiting for ${name}`;
return snapshot.saved ? `${name} (disconnected)` : "Not set up";
}
}
/** The short form for the right edge of the Settings row. */
function keyboardStatusBrief(snapshot: KeyboardBridgeSnapshot): string {
switch (snapshot.state) {
case "connected":
return "Connected";
case "scanning":
return "Scanning";
case "connecting":
case "pairing":
case "configuring":
return "Connecting";
case "error":
return "Failed";
case "unavailable":
return snapshot.saved ? "Offline" : "Off";
default:
return snapshot.waiting ? "Waiting" : snapshot.saved ? "Disconnected" : "Off";
}
}
const info = (label: string): MenuItem => ({ label, disabled: true, onSelect: () => {} });
class KeyboardSettingsLayer extends MenuLayer {
private unsubscribe: (() => void) | null = null;
private showOthers = false;
/** What kind of rows are showing; a change moves the highlight to the first row that does something. */
private shape = "";
constructor(viewport: { width: number; height: number }) {
super("Bluetooth keyboard", [], {
x: "center",
y: 8,
width: Math.min(320, viewport.width - 40),
minHeight: Math.min(160, viewport.height - 40),
maxHeight: Math.min(240, viewport.height - 40),
depth: 4,
});
}
open(ctx: LayerContext): void {
ctx.stack.push(this);
this.updateRows();
this.unsubscribe = keyboardService.subscribe(() => {
this.updateRows();
ctx.actions.requestRender();
});
}
onRemoved(): void {
this.unsubscribe?.();
this.unsubscribe = null;
}
/** Run a bridge command; its failure shows up in the snapshot, which repaints the rows. */
private run(action: () => Promise<unknown>): void {
void action().catch(() => {});
}
private updateRows(): void {
const snapshot = keyboardService.snapshot();
const bridge = keyboardService.bridge;
const rows: MenuItem[] = [info(keyboardStatusText(snapshot))];
let shape: string = snapshot.state;
const scanRow = (label: string): MenuItem => ({
label,
disabled: snapshot.busy,
onSelect: () => {
this.showOthers = false;
this.run(() => bridge.scan());
},
});
if (snapshot.state === "unavailable") {
rows.push(info("Connect G2 glasses to pair one."));
if (snapshot.saved) rows.push({ label: `Forget ${snapshot.saved.name || "keyboard"}`, onSelect: () => this.run(() => bridge.forget()) });
} else if (snapshot.pairing) {
const pairing = snapshot.pairing;
shape = `pairing:${pairing.kind}`;
if (pairing.kind === "display") {
rows.push(info(`Type ${sixDigits(pairing.pin)} on the keyboard,`), info("then press Enter."));
} else if (pairing.kind === "compare") {
rows.push(
info(`Does the keyboard show ${sixDigits(pairing.value)}?`),
{ label: "Yes, pair", onSelect: () => this.run(() => bridge.answerCompare(true)) },
{ label: "No", onSelect: () => this.run(() => bridge.answerCompare(false)) },
);
} else {
rows.push(
info("Enter the code the keyboard shows."),
textSettingMenuItem(pairingCodeSetting),
{
label: "Pair with this code",
disabled: () => pairingCodeSetting.get().length !== 6,
onSelect: () => {
const code = pairingCodeSetting.get();
pairingCodeSetting.set("");
this.run(() => bridge.answerPasskey(parseInt(code, 10)));
},
},
);
}
rows.push({ label: "Cancel pairing", onSelect: () => this.run(() => bridge.disconnect()) });
} else if (snapshot.state === "connecting" || snapshot.state === "pairing" || snapshot.state === "configuring") {
rows.push({ label: "Cancel", onSelect: () => this.run(() => bridge.disconnect()) });
} else if (snapshot.state === "scanning" || (snapshot.scanResults.length > 0 && snapshot.state !== "connected")) {
shape = "results";
const hid = snapshot.scanResults.filter((advert) => advert.hid);
const others = snapshot.scanResults.filter((advert) => !advert.hid);
const pick = (advert: KeyboardAdvert): MenuItem => ({
label: advertLabel(advert),
disabled: snapshot.busy,
onSelect: () => {
this.run(() => bridge.connect({ address: advert.address, addressType: advert.addressType, name: advert.name.trim() || advert.address }));
},
});
rows.push(...hid.map(pick));
if (!hid.length && snapshot.state !== "scanning") rows.push(info("No keyboards found."), info("Put it in pairing mode."));
if (others.length && this.showOthers) rows.push(...others.map(pick));
else if (others.length) {
rows.push({
label: `Show ${others.length} other device${others.length === 1 ? "" : "s"}`,
onSelect: (ctx) => {
this.showOthers = true;
this.updateRows();
ctx.actions.requestRender();
},
});
}
if (snapshot.state !== "scanning") rows.push(scanRow("Scan again"));
if (snapshot.saved && snapshot.state !== "scanning") {
rows.push({ label: `Reconnect ${snapshot.saved.name || "keyboard"}`, disabled: snapshot.busy, onSelect: () => this.run(() => bridge.reconnect()) });
}
} else if (snapshot.state === "connected") {
rows.push(
{ label: "Disconnect", onSelect: () => this.run(() => bridge.disconnect()) },
{ label: "Forget this keyboard", onSelect: () => this.run(() => bridge.forget()) },
scanRow("Pair a different keyboard"),
);
} else if (snapshot.saved) {
shape = snapshot.waiting ? "waiting" : "saved";
rows.push({ label: "Reconnect now", disabled: snapshot.busy, onSelect: () => this.run(() => bridge.reconnect()) });
if (snapshot.waiting) rows.push({ label: "Stop reconnecting", onSelect: () => this.run(() => bridge.disconnect()) });
rows.push(
{ label: "Forget this keyboard", onSelect: () => this.run(() => bridge.forget()) },
scanRow("Pair a different keyboard"),
);
} else {
shape = "empty";
rows.push(info("Put the keyboard in pairing mode,"), info("then scan."), scanRow("Scan for keyboards"));
}
const firstAction = rows.findIndex((row) => row.disabled !== true);
if (shape !== this.shape) {
this.shape = shape;
this.setItems(rows, Math.max(0, firstAction));
} else {
this.setItems(rows);
}
}
}
/** The Settings row that opens the keyboard menu. */
export function keyboardSettingsMenuItem(): MenuItem {
return {
label: "Bluetooth keyboard",
description:
"Pair a Bluetooth keyboard with the glasses. It types into Terminal sessions, and its arrow keys, Enter and Escape work everywhere as scroll, select and back.",
onSelect: (ctx) => new KeyboardSettingsLayer(ctx.stack.getBaseSize()).open(ctx),
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "Bluetooth keyboard", keyboardStatusBrief(keyboardService.snapshot()));
},
};
}
+19 -1
View File
@@ -3,6 +3,7 @@ import { screenHeight, screenWidth } from "../graphics/screen-size";
import { dimPlanes, type Plane } from "../graphics/plane";
import { spanCurrent } from "../native/frame-timings";
import { type ConfigSettingBoolean, type ConfigSettingString, type VoiceProvider } from "./dashboard-settings";
import { type VoiceReplay } from "../native/voice-control";
import { directionalFallback, isDirectionalInput, type InputEvent } from "./gestures";
import { toViewport, type Viewport, type ViewportSize } from "./viewport";
@@ -13,8 +14,17 @@ export type TextSettingsEditToggle = {
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
export type ContinuousCaptureOptions = {
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
/** Names a recording (.mka) of the captured audio in Downloads, written while Save Recording is on. */
recordAs?: string;
/**
* How many of the glasses' microphones to capture: 1 is their own mixed
* stream; 2 (one per temple) and 4 (both of each temple) stream the
* microphones separately through the custom firmware, for recording and
* array processing. While any client asks for more than one, every capture
* uses the most asked for. Glasses without the microphone channel, and the
* phone's microphone, give 1 whatever is asked.
*/
mics?: MicCount;
/**
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
* or "" for the provider to detect it), instead of the Transcription
@@ -26,6 +36,8 @@ export type ContinuousCaptureOptions = {
export type TranscriptionChoice = { provider: VoiceProvider; language: string };
export type MicCount = 1 | 2 | 4;
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
@@ -55,6 +67,12 @@ export type LayerActions = {
*/
startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => Promise<void> | void;
stopContinuousVoiceCapture: (client: string) => Promise<void> | void;
/**
* Open a recording (.mka) for playing back through transcription in place
* of the microphone; the caller starts and closes it. Absent, or null,
* where recordings can't be replayed.
*/
openVoiceReplay?: (path: string) => VoiceReplay | null;
/** Play a CFW tone-sequencer payload (see sound-effects.ts). */
playBuzzerSequence: (payload: Uint8Array) => Promise<void> | void;
};
+4 -3
View File
@@ -1,6 +1,6 @@
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
import { truncateText } from "../graphics/textwrap";
import { voiceControlBridge } from "../native/voice-control";
import { voiceControlBridge, type VoiceEventSource } from "../native/voice-control";
import { lineStep } from "./metrics";
import type { Edges } from "./viewport";
@@ -68,8 +68,9 @@ export class MicLevelMeter {
private timer: ReturnType<typeof setInterval> | null = null;
private unsubscribe: (() => void) | null = null;
start(requestRender: () => void, isVisible: () => boolean): void {
this.unsubscribe = voiceControlBridge.onMicLevel((level) => {
/** `source` is the capture whose level to show: the microphone's unless given. */
start(requestRender: () => void, isVisible: () => boolean, source: VoiceEventSource = voiceControlBridge): void {
this.unsubscribe = source.onMicLevel((level) => {
this.peak = Math.max(this.peak, level);
});
this.timer = setInterval(() => {
+58 -6
View File
@@ -39,6 +39,8 @@ const PAGE_TITLE_OFFSET_Y = 2;
const PAGE_TITLE_GAP = 4;
/** Room a page menu keeps right of its rows for the scrollbar. */
const PAGE_SCROLLBAR_GUTTER = 6;
/** A heading row's text: dimmer than a row's, brighter than a disabled row's. */
const MENU_HEADING_VALUE = 90;
export type MenuLayout = {
/** Stereo depth of the menu surface, including the selected row. */
@@ -57,7 +59,7 @@ export type MenuLayout = {
* Dim hint line pinned to the bottom edge inside the box, below the items
* (gesture help such as "●— system menu"). Takes a line off the item area.
*/
footer?: string;
footer?: MenuFooter;
/** Brightness factor for everything beneath the menu (see Layer.dimUnderneath). Default: none. */
dimUnderneath?: number;
/**
@@ -83,9 +85,16 @@ export type MenuPageLayout = {
/** Width of the rows. Default (and at most): the window's, inside its side padding. */
width?: number;
/** Dim hint line pinned above the window's bottom padding, below the rows. */
footer?: string;
footer?: MenuFooter;
};
/** A menu's footer line: fixed text, or read on every paint (a live value). */
export type MenuFooter = string | (() => string);
function footerText(footer: MenuFooter): string {
return typeof footer === "function" ? footer() : footer;
}
export type MenuItemRenderArgs = {
image: GrayImage;
x: number;
@@ -104,6 +113,12 @@ export type MenuItem = {
description?: string;
/** Disabled rows stay visible but render dimly and ignore clicks. */
disabled?: boolean | (() => boolean);
/**
* A label for the group of rows after it (see menuHeadingItem). MenuLayer's
* highlight skips it; a list that doesn't know headings shows it as a
* disabled row.
*/
heading?: boolean;
onSelect: (ctx: LayerContext, menu: MenuLayer) => Promise<void> | void;
render?: (args: MenuItemRenderArgs) => void;
};
@@ -207,6 +222,22 @@ export function submenuItem(
};
}
/**
* A heading for the rows that follow it, splitting a long menu into sections:
* dim capitals in the rows' text column, never selected.
*/
export function menuHeadingItem(label: string): MenuItem {
return {
label,
heading: true,
disabled: true,
onSelect: () => {},
render: ({ image, x, y, text }) => {
image.drawText(getDefaultSmallFont(), x, y + LIST_ROW_TEXT_INSET, text.toUpperCase(), MENU_HEADING_VALUE);
},
};
}
export function drawToggleMenuItem(
image: GrayImage,
font: UiFont,
@@ -274,6 +305,7 @@ export class MenuLayer implements Layer {
wrap: true,
rowGap: 1,
getHeight: () => listRowHeight(getDefaultSmallFont()),
isSelectable: (item) => !item.heading,
draw: ({ image, item, x, y, width, height, selected }) => {
const font = getDefaultSmallFont();
const disabled = isMenuItemDisabled(item);
@@ -360,7 +392,7 @@ export class MenuLayer implements Layer {
image.drawText(font, x + 12, y + 8, this.title, 220);
}
if (layout.footer) {
image.drawText(font, x + 22, y + height - MENU_BODY_PADDING - font.lineHeight, layout.footer, 110);
image.drawText(font, x + 22, y + height - MENU_BODY_PADDING - font.lineHeight, footerText(layout.footer), 110);
}
const bodyY = y + chromeTop;
@@ -397,7 +429,7 @@ export class MenuLayer implements Layer {
let fixedBottom = height - padding.bottom;
if (layout.footer) {
const footerY = height - padding.bottom - font.lineHeight;
image.drawText(font, x + MENU_ROW_TEXT_INSET, footerY, layout.footer, 110);
image.drawText(font, x + MENU_ROW_TEXT_INSET, footerY, footerText(layout.footer), 110);
bottom = fixedBottom = footerY - MENU_FOOTER_GAP;
}
if (bottom <= top) return image;
@@ -431,17 +463,26 @@ export class MenuLayer implements Layer {
}
}
export type ModalMenuOptions = {
/** A dim line under the items (see MenuLayout.footer). */
footer?: MenuFooter;
/** Runs when the menu leaves the stack, however it closes. */
onClosed?: () => void;
};
/** Open a centered, bordered menu over the current layer. */
export function openModalMenu(
ctx: LayerContext,
title: string,
items: MenuItem[],
initialSelectedIndex = 0,
options: ModalMenuOptions = {},
): void {
const { width, height } = ctx.stack.getBaseSize();
const font = getDefaultSmallFont();
const menuWidth = Math.min(320, width - 40);
const naturalHeight = menuTitleHeight(font) + 2 * MENU_BODY_PADDING + items.length * listRowHeight(font);
const footerHeight = options.footer ? MENU_FOOTER_GAP + lineStep(font) : 0;
const naturalHeight = menuTitleHeight(font) + 2 * MENU_BODY_PADDING + items.length * listRowHeight(font) + footerHeight;
// 20px clear above and below, or 8 in a short window (the RayNeo iO's),
// where those rows are worth more than the margin.
const marginY = height < SHORT_WINDOW_HEIGHT ? 8 : 20;
@@ -454,8 +495,19 @@ export function openModalMenu(
maxHeight: menuHeight,
// Floats nearer than the content it opens over, like the window and system menus.
depth: 4,
footer: options.footer,
};
ctx.stack.push(new MenuLayer(title, items, layout).selectItem(initialSelectedIndex));
ctx.stack.push(new ModalMenuLayer(title, items, layout, options.onClosed).selectItem(initialSelectedIndex));
}
class ModalMenuLayer extends MenuLayer {
constructor(title: string, items: MenuItem[], layout: MenuLayout, private readonly onClosed?: () => void) {
super(title, items, layout);
}
onRemoved(): void {
this.onClosed?.();
}
}
export function isMenuItemDisabled(item: MenuItem): boolean {
+3
View File
@@ -48,6 +48,8 @@ export type InProcessWindowOptions = {
setScreenOn?: ShellWindow["setScreenOn"];
/** See ShellWindow.keepsScreenOn. */
keepsScreenOn?: ShellWindow["keepsScreenOn"];
/** See ShellWindow.restoresApp. */
restoresApp?: boolean;
/** The window came to or left the foreground (after its surface visibility follows). */
onForegroundChanged?: (foreground: boolean) => void;
/** Shared actions; requestRender is rebound to this window's render. */
@@ -177,6 +179,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces
holdToTalk: options.holdToTalk,
isVoiceCapturing: options.isVoiceCapturing,
keepsScreenOn: options.keepsScreenOn,
restoresApp: options.restoresApp,
setScreenOn: options.setScreenOn,
heightMode,
hasAppMenu: () => appMenuItems().length > 0,
+32
View File
@@ -17,6 +17,7 @@ import {
isWatchInput,
makeInputEvent,
} from "../gestures";
import type { KeyEvent } from "../../g2/keyboard/keys";
import { Layer, LayerActions, LayerContext, LayerStack, noopLayerActions } from "../layers";
import { CONTEXT_MENU_DIM, MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "./voice-input";
@@ -151,6 +152,13 @@ export type ShellWindow = {
* draws this window's top bar.
*/
heightMode: WindowHeightMode;
/**
* False for a window that must not bring its app back when Faceclaw
* restarts (the controller's open-apps restore relaunches apps, not
* windows): one that shows something a plain launch of its app would not,
* where the relaunch would do something else in its place.
*/
restoresApp?: boolean;
/** App-side cleanup when the shell closes the window (worker notification, surface removal). */
close?: () => void;
drawIcon: ShellChromeWindow["drawIcon"];
@@ -185,6 +193,15 @@ export type ShellWindow = {
* only windows that consume typed text (the terminal) implement it.
*/
receiveTextInput?: (text: string, options?: { submit?: boolean }) => void;
/**
* Take a key from a hardware keyboard (app/g2/keyboard) as a key. Optional:
* only windows that give keys a meaning of their own (the terminal)
* implement it. They get the keys no keybinding claimed while they have
* input focus, presses and releases alike; the rest of the time, and for
* every other window, the navigation keys act as gestures (see
* app/g2/keyboard/bindings.ts).
*/
handleKeyEvent?: (event: KeyEvent) => void;
/** Foreground state changed: this window's surface is (not) the visible one. */
setForeground?: (foreground: boolean) => void;
/**
@@ -1599,6 +1616,21 @@ class Shell {
this.foregroundWindow()?.receiveTextInput?.(text, options);
}
/**
* Offer a hardware-keyboard key to the window that has input focus. True
* when that window takes keys (ShellWindow.handleKeyEvent) and got this
* one; false when the key is the caller's to interpret: the screen is off,
* the sidebar or a shell overlay has focus, or the window has no use for
* keys. Typing counts as activity for the screen timeout.
*/
receiveKeyEvent(event: KeyEvent): boolean {
const window = this.inputTargetWindow();
if (!window?.handleKeyEvent) return false;
if (event.phase === "down") this.lastInputAtMs = Date.now();
window.handleKeyEvent(event);
return true;
}
/**
* Open the voice dialog aimed at the foreground window. Called when the
* user picks Voice input from the system menu (or an app asks via a
+35
View File
@@ -28,6 +28,8 @@ export type WorkerAppMessage =
| { type: "close-window"; windowId: string }
| { type: "input"; windowId: string; event: unknown; frameId: number; focused: boolean }
| { type: "text-input"; windowId: string; text: string; submit?: boolean }
/** A hardware-keyboard key (a KeyEvent), for windows opened with acceptsKeys. */
| { type: "key"; windowId: string; event: unknown; focused: boolean }
| { type: "render"; windowId: string; focused: boolean }
| { type: "foreground"; windowId: string; foreground: boolean; focused: boolean }
/**
@@ -56,6 +58,16 @@ export type WorkerAppReply =
type: "worker-ready";
}
| { type: "yield-focus"; windowId: string }
| {
/**
* A "key" message the window had no use for as a key (a terminal
* showing its session list rather than a session). The host gives it
* the keyboard's navigation meaning instead.
*/
type: "key-unhandled";
windowId: string;
event: unknown;
}
| {
/** Foreground and focus one of the app's existing windows. */
type: "focus-window";
@@ -81,6 +93,8 @@ export type WorkerAppReply =
focus?: boolean;
/** Window height: the standard 288px band ("min", default) or full screen ("max"). */
heightMode?: WindowHeightMode;
/** Send the window hardware-keyboard keys (see WorkerWindowSpec). */
acceptsKeys?: boolean;
}
| {
/**
@@ -196,6 +210,13 @@ export type WorkerWindowSpec = {
* of all four directions in every window state (see ShellWindow).
*/
acceptsDirectional?: boolean;
/**
* Deliver hardware-keyboard keys to the worker as "key" messages while the
* window has input focus (see ShellWindow.handleKeyEvent). The worker
* answers "key-unhandled" for a key it doesn't want as a key just then,
* which gets the keyboard's navigation meaning instead.
*/
acceptsKeys?: boolean;
};
export type WorkerAppHostOptions = {
@@ -223,6 +244,8 @@ export type WorkerAppHostOptions = {
startTextSettingEdit: (settingId: string) => void;
/** Close the phone text editor opened by startTextSettingEdit. */
endTextSettingEdit: () => void;
/** A key a window was sent and handed back (the "key-unhandled" reply); a KeyEvent. */
unhandledKey?: (event: unknown) => void;
};
/**
@@ -312,6 +335,10 @@ export class WorkerAppHost {
shell.yieldFocusToSidebar();
}
break;
case "key-unhandled":
// Stale once focus has moved on: the key was meant for that window.
if (shell.isWindowFocused(message.windowId)) this.options.unhandledKey?.(message.event);
break;
case "focus-window":
if (this.openWindows.has(message.windowId)) {
shell.focusWindow(message.windowId);
@@ -335,6 +362,7 @@ export class WorkerAppHost {
iconGlyph: message.iconGlyph,
focus: message.focus,
heightMode: message.heightMode,
acceptsKeys: message.acceptsKeys,
});
break;
case "set-height-mode":
@@ -522,6 +550,13 @@ export class WorkerAppHost {
receiveTextInput: (text, options) => {
this.post({ type: "text-input", windowId: spec.windowId, text, ...(options?.submit === undefined ? {} : { submit: options.submit }) });
},
...(spec.acceptsKeys
? {
handleKeyEvent: (event) => {
this.post({ type: "key", windowId: spec.windowId, event, focused: shell.isWindowFocused(spec.windowId) });
},
}
: {}),
setForeground: (foreground) => {
this.options.setSurfaceVisible(surfaceId, foreground);
if (foreground) {
+4 -1
View File
@@ -1,8 +1,11 @@
const { buildKotlin, checkAndroidRuntime } = require('../../scripts/kotlin-build.cjs');
const { buildGoSsh } = require('../../scripts/go-ssh-build.cjs');
module.exports = function (hookArgs) {
// NativeScript parses hooks with an older JavaScript parser; avoid optional chaining.
const { platform, release } = hookArgs.prepareData;
if (platform === 'android') checkAndroidRuntime();
return buildKotlin(platform, release ? 'release' : 'debug');
return buildKotlin(platform, release ? 'release' : 'debug').then(function () {
return buildGoSsh(platform);
});
};
+2 -1
View File
@@ -11,7 +11,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader, the Matroska
writer/reader and the MicRecording capture format built on it, MicArrayProtocol),
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
ColorGlyphArt, GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
@@ -69,3 +69,41 @@ actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exis
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
actual fun openFileOutput(path: String): SeekableOutput = object : SeekableOutput {
private val file = java.io.RandomAccessFile(path, "rw").also { it.setLength(0) }
override var size = 0L
private set
override fun write(bytes: ByteArray, offset: Int, length: Int) {
file.write(bytes, offset, length)
size += length
}
override fun writeAt(position: Long, bytes: ByteArray) {
require(position >= 0 && position + bytes.size <= size)
file.seek(position)
file.write(bytes)
file.seek(size)
}
override fun close() = file.close()
}
actual fun openFileInput(path: String): SeekableInput = object : SeekableInput {
private val file = java.io.RandomAccessFile(path, "r")
override val length = file.length()
override fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int {
file.seek(position)
var total = 0
while (total < length) {
val read = file.read(buffer, offset + total, length - total)
if (read <= 0) break
total += read
}
return total
}
override fun close() = file.close()
}
@@ -20,3 +20,33 @@ expect fun fileLength(path: String): Long
expect fun deleteFile(path: String): Boolean
expect fun renameFile(from: String, to: String): Boolean
/** A file being written front to back whose earlier bytes can still be patched (container headers). */
interface SeekableOutput {
/** Bytes appended so far. */
val size: Long
/** Appends [length] bytes of [bytes] from [offset]. */
fun write(bytes: ByteArray, offset: Int, length: Int)
/** Overwrites bytes already written, at [position]; later appends still go to the end. */
fun writeAt(position: Long, bytes: ByteArray)
fun close()
}
/** A file read at arbitrary positions without loading it whole. */
interface SeekableInput {
val length: Long
/** Reads up to [length] bytes at [position] into [buffer]; returns the count read (short only at the end). */
fun readAt(position: Long, buffer: ByteArray, offset: Int, length: Int): Int
fun close()
}
/** Creates (or truncates) [path] for writing; parent directories must exist. Throws when it cannot. */
expect fun openFileOutput(path: String): SeekableOutput
/** Opens [path] for reading. Throws when it cannot. */
expect fun openFileInput(path: String): SeekableInput
@@ -0,0 +1,764 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/**
* A small Matroska (EBML) muxer and demuxer for audio captures (.mka): PCM16 audio tracks that
* each have their own sample rate and channel count, UTF-8 text tracks for timed metadata, and
* file attachments. Every block carries its own timestamp, so tracks on independent clocks and
* gaps in a track survive in the file. See notes/mic-recording-format.md for how captures use it.
*
* The writer produces files standard players open (VLC, mpv, ffmpeg): known-size clusters of
* about a second each, cues and the segment size written when the recording finishes. A
* recording cut short by a crash is still a valid live-style stream of everything up to its last
* complete cluster, and the reader takes it as such.
*/
internal object MatroskaIds {
const val EBML = 0x1A45DFA3
const val EBML_VERSION = 0x4286
const val EBML_READ_VERSION = 0x42F7
const val EBML_MAX_ID_LENGTH = 0x42F2
const val EBML_MAX_SIZE_LENGTH = 0x42F3
const val DOC_TYPE = 0x4282
const val DOC_TYPE_VERSION = 0x4287
const val DOC_TYPE_READ_VERSION = 0x4285
const val VOID = 0xEC
const val SEGMENT = 0x18538067
const val SEEK_HEAD = 0x114D9B74
const val SEEK = 0x4DBB
const val SEEK_ID = 0x53AB
const val SEEK_POSITION = 0x53AC
const val INFO = 0x1549A966
const val TIMESTAMP_SCALE = 0x2AD7B1
const val DURATION = 0x4489
const val DATE_UTC = 0x4461
const val TITLE = 0x7BA9
const val MUXING_APP = 0x4D80
const val WRITING_APP = 0x5741
const val TRACKS = 0x1654AE6B
const val TRACK_ENTRY = 0xAE
const val TRACK_NUMBER = 0xD7
const val TRACK_UID = 0x73C5
const val TRACK_TYPE = 0x83
const val FLAG_DEFAULT = 0x88
const val FLAG_LACING = 0x9C
const val NAME = 0x536E
const val LANGUAGE = 0x22B59C
const val CODEC_ID = 0x86
const val AUDIO = 0xE1
const val SAMPLING_FREQUENCY = 0xB5
const val CHANNELS = 0x9F
const val BIT_DEPTH = 0x6264
const val ATTACHMENTS = 0x1941A469
const val ATTACHED_FILE = 0x61A7
const val FILE_NAME = 0x466E
const val FILE_MIME_TYPE = 0x4660
const val FILE_DATA = 0x465C
const val FILE_UID = 0x46AE
const val CLUSTER = 0x1F43B675
const val TIMESTAMP = 0xE7
const val SIMPLE_BLOCK = 0xA3
const val BLOCK_GROUP = 0xA0
const val BLOCK = 0xA1
const val BLOCK_DURATION = 0x9B
const val CUES = 0x1C53BB6B
const val CUE_POINT = 0xBB
const val CUE_TIME = 0xB3
const val CUE_TRACK_POSITIONS = 0xB7
const val CUE_TRACK = 0xF7
const val CUE_CLUSTER_POSITION = 0xF1
}
/** Builds EBML elements into a byte buffer. */
internal class EbmlWriter(capacity: Int = 256) {
private val sink = ByteSink(capacity)
fun size(): Int = sink.size()
fun bytes(): ByteArray = sink.toByteArray()
/** An element ID, written as given: its length marker is part of the value. */
fun id(id: Int) {
if (id > 0xFFFFFF) sink.write(id ushr 24)
if (id > 0xFFFF) sink.write(id ushr 16)
if (id > 0xFF) sink.write(id ushr 8)
sink.write(id)
}
/** A data size in [width] bytes, or in the fewest that hold it when [width] is 0. */
fun dataSize(size: Long, width: Int = 0) {
var bytes = width
if (bytes == 0) {
bytes = 1
// The all-ones value of each width means "unknown size".
while (bytes < 8 && size >= (1L shl (7 * bytes)) - 1) bytes++
}
val marked = size or (1L shl (7 * bytes))
for (index in bytes - 1 downTo 0) sink.write((marked ushr (8 * index)).toInt())
}
fun uint(id: Int, value: Long) {
var bytes = 1
while (bytes < 8 && (value ushr (8 * bytes)) != 0L) bytes++
id(id)
dataSize(bytes.toLong())
for (index in bytes - 1 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
/** A fixed eight-byte integer (dates). */
fun int64(id: Int, value: Long) {
id(id)
dataSize(8)
for (index in 7 downTo 0) sink.write((value ushr (8 * index)).toInt())
}
fun float64(id: Int, value: Double) = int64(id, value.toRawBits())
fun string(id: Int, text: String) {
val bytes = text.encodeToByteArray()
binary(id, bytes, 0, bytes.size)
}
fun binary(id: Int, bytes: ByteArray, offset: Int, length: Int) {
id(id)
dataSize(length.toLong())
sink.write(bytes, offset, length)
}
fun raw(bytes: ByteArray) = sink.write(bytes)
fun raw(value: Int) = sink.write(value)
fun master(id: Int, build: EbmlWriter.() -> Unit) {
val child = EbmlWriter()
child.build()
id(id)
dataSize(child.size().toLong())
sink.write(child.bytes())
}
companion object {
/** [element] followed by a Void element filling the rest of [reserved] bytes, or null when it doesn't fit. */
fun padded(element: ByteArray, reserved: Int): ByteArray? {
val filler = reserved - element.size
// A Void element is at least two bytes.
if (filler < 0 || filler == 1) return null
val out = element.copyOf(reserved)
if (filler == 0) return out
val at = element.size
out[at] = MatroskaIds.VOID.toByte()
if (filler < 9) {
out[at + 1] = (0x80 or (filler - 2)).toByte()
} else {
val size = (filler - 9).toLong()
out[at + 1] = 0x01
for (index in 0 until 7) out[at + 2 + index] = (size ushr (8 * (6 - index))).toByte()
}
return out
}
}
}
class MkaAttachment(@JvmField val name: String, @JvmField val mimeType: String, @JvmField val data: ByteArray)
/**
* Writes one Matroska audio file. Tracks can be added at any time before the first block that
* uses them: the track list sits in a reserved region that is rewritten in place. Timestamps are
* milliseconds from the start of the file. Not thread-safe.
*/
class MkaWriter(
private val output: SeekableOutput,
title: String?,
/** When the recording began (Unix epoch ms), for the file's date. */
dateUtcMs: Long,
attachments: List<MkaAttachment>,
) {
companion object {
const val CODEC_PCM = "A_PCM/INT/LIT"
const val CODEC_TEXT = "S_TEXT/UTF8"
const val TRACK_AUDIO = 2
const val TRACK_SUBTITLE = 0x11
const val APP_NAME = "Faceclaw"
private const val SEEK_RESERVED = 128
private const val TRACKS_RESERVED = 4096
private const val CLUSTER_SPAN_MS = 1000L
private const val CLUSTER_MAX_BYTES = 1 shl 20
// Block timestamps are signed 16-bit offsets from their cluster's.
private const val CLUSTER_MAX_BACKSTEP_MS = 30_000L
private const val DURATION_PATCH_CLUSTERS = 5
// Matroska dates count nanoseconds from 2001-01-01T00:00:00 UTC.
private const val DATE_EPOCH_MS = 978_307_200_000L
}
private class Track(
val number: Int,
val type: Int,
val name: String,
val sampleRate: Int,
val channels: Int,
val preferred: Boolean,
)
private class Cue(val clusterPosition: Long, val timeMs: Long)
private val tracks = ArrayList<Track>()
private val cues = ArrayList<Cue>()
private val segmentSizePosition: Long
private val segmentStart: Long
private val seekPosition: Long
private val infoPosition: Long
private val durationPosition: Long
private val tracksPosition: Long
private val attachmentsPosition: Long
private var cluster: EbmlWriter? = null
private var clusterBaseMs = 0L
private var clustersSincePatch = 0
private var finished = false
private val hasAttachments = attachments.isNotEmpty()
/** The end of the latest audio or text written, in ms from the start of the file. */
var durationMs = 0L
private set
init {
val head = EbmlWriter()
head.master(MatroskaIds.EBML) {
uint(MatroskaIds.EBML_VERSION, 1)
uint(MatroskaIds.EBML_READ_VERSION, 1)
uint(MatroskaIds.EBML_MAX_ID_LENGTH, 4)
uint(MatroskaIds.EBML_MAX_SIZE_LENGTH, 8)
string(MatroskaIds.DOC_TYPE, "matroska")
uint(MatroskaIds.DOC_TYPE_VERSION, 4)
uint(MatroskaIds.DOC_TYPE_READ_VERSION, 2)
}
head.id(MatroskaIds.SEGMENT)
segmentSizePosition = head.size().toLong()
// Unknown size (a live stream) until finish() knows it.
head.raw(0x01)
repeat(7) { head.raw(0xFF) }
segmentStart = head.size().toLong()
append(head.bytes())
seekPosition = output.size
infoPosition = seekPosition + SEEK_RESERVED
val info = EbmlWriter()
info.master(MatroskaIds.INFO) {
uint(MatroskaIds.TIMESTAMP_SCALE, 1_000_000)
string(MatroskaIds.MUXING_APP, APP_NAME)
string(MatroskaIds.WRITING_APP, APP_NAME)
int64(MatroskaIds.DATE_UTC, (dateUtcMs - DATE_EPOCH_MS) * 1_000_000)
if (!title.isNullOrEmpty()) string(MatroskaIds.TITLE, title)
// Last, so its eight bytes are the element's final ones.
float64(MatroskaIds.DURATION, 0.0)
}
durationPosition = infoPosition + info.size() - 8
tracksPosition = infoPosition + info.size()
attachmentsPosition = tracksPosition + TRACKS_RESERVED
append(seekHead(-1))
append(info.bytes())
append(EbmlWriter.padded(ByteArray(0), TRACKS_RESERVED)!!)
if (attachments.isNotEmpty()) {
val files = EbmlWriter()
files.master(MatroskaIds.ATTACHMENTS) {
attachments.forEachIndexed { index, file ->
master(MatroskaIds.ATTACHED_FILE) {
string(MatroskaIds.FILE_NAME, file.name)
string(MatroskaIds.FILE_MIME_TYPE, file.mimeType)
binary(MatroskaIds.FILE_DATA, file.data, 0, file.data.size)
uint(MatroskaIds.FILE_UID, index + 1L)
}
}
}
append(files.bytes())
}
}
private fun append(bytes: ByteArray) = output.write(bytes, 0, bytes.size)
private fun seekHead(cuesPosition: Long): ByteArray {
val head = EbmlWriter()
head.master(MatroskaIds.SEEK_HEAD) {
fun entry(id: Int, position: Long) = master(MatroskaIds.SEEK) {
val idBytes = EbmlWriter()
idBytes.id(id)
binary(MatroskaIds.SEEK_ID, idBytes.bytes(), 0, idBytes.size())
uint(MatroskaIds.SEEK_POSITION, position - segmentStart)
}
entry(MatroskaIds.INFO, infoPosition)
entry(MatroskaIds.TRACKS, tracksPosition)
if (hasAttachments) entry(MatroskaIds.ATTACHMENTS, attachmentsPosition)
if (cuesPosition >= 0) entry(MatroskaIds.CUES, cuesPosition)
}
return EbmlWriter.padded(head.bytes(), SEEK_RESERVED) ?: error("seek head exceeds its reserved space")
}
/**
* Adds a PCM16 little-endian audio track and returns its number, or -1 when the track list
* is full. A [preferred] track is listed ahead of the others, and the first listed track is
* the one players pick by default.
*/
fun addAudioTrack(name: String, sampleRate: Int, channels: Int, preferred: Boolean): Int =
addTrack(Track(tracks.size + 1, TRACK_AUDIO, name, sampleRate, channels, preferred))
/** Adds a UTF-8 text track (one string per block) and returns its number, or -1 when the track list is full. */
fun addTextTrack(name: String): Int = addTrack(Track(tracks.size + 1, TRACK_SUBTITLE, name, 0, 0, false))
private fun addTrack(track: Track): Int {
check(!finished) { "recording already finished" }
// A block's track number is written in one byte.
if (track.number > 126) return -1
tracks.add(track)
val region = EbmlWriter.padded(tracksElement(), TRACKS_RESERVED)
if (region == null) {
tracks.remove(track)
return -1
}
output.writeAt(tracksPosition, region)
return track.number
}
private fun listedTracks(): List<Track> =
tracks.filter { it.type == TRACK_AUDIO && it.preferred } +
tracks.filter { it.type == TRACK_AUDIO && !it.preferred } +
tracks.filter { it.type != TRACK_AUDIO }
private fun tracksElement(): ByteArray {
val listed = listedTracks()
val defaultTrack = listed.firstOrNull { it.type == TRACK_AUDIO }
val out = EbmlWriter(1024)
out.master(MatroskaIds.TRACKS) {
for (track in listed) {
master(MatroskaIds.TRACK_ENTRY) {
uint(MatroskaIds.TRACK_NUMBER, track.number.toLong())
uint(MatroskaIds.TRACK_UID, track.number.toLong())
uint(MatroskaIds.TRACK_TYPE, track.type.toLong())
uint(MatroskaIds.FLAG_DEFAULT, if (track === defaultTrack) 1 else 0)
uint(MatroskaIds.FLAG_LACING, 0)
string(MatroskaIds.NAME, track.name)
string(MatroskaIds.LANGUAGE, "und")
if (track.type == TRACK_AUDIO) {
string(MatroskaIds.CODEC_ID, CODEC_PCM)
master(MatroskaIds.AUDIO) {
float64(MatroskaIds.SAMPLING_FREQUENCY, track.sampleRate.toDouble())
uint(MatroskaIds.CHANNELS, track.channels.toLong())
uint(MatroskaIds.BIT_DEPTH, 16)
}
} else {
string(MatroskaIds.CODEC_ID, CODEC_TEXT)
}
}
}
}
return out.bytes()
}
private fun track(number: Int): Track = tracks.getOrNull(number - 1) ?: error("no track $number")
/** Appends interleaved PCM16 little-endian samples to an audio track, starting at [timeMs]. */
fun writeAudio(trackNumber: Int, timeMs: Long, pcm16le: ByteArray, offset: Int, length: Int) {
val track = track(trackNumber)
require(track.type == TRACK_AUDIO) { "track $trackNumber is not audio" }
if (length <= 0) return
val block = openCluster(timeMs)
block.id(MatroskaIds.SIMPLE_BLOCK)
block.dataSize(4L + length)
blockHeader(block, trackNumber, timeMs, 0x80)
block.raw(if (offset == 0 && length == pcm16le.size) pcm16le else pcm16le.copyOfRange(offset, offset + length))
val frames = length / (2 * track.channels)
noteEnd(timeMs + frames * 1000L / track.sampleRate)
}
/** Appends one string to a text track, shown from [timeMs] for [durationMs]. */
fun writeText(trackNumber: Int, timeMs: Long, durationMs: Long, text: String) {
require(track(trackNumber).type == TRACK_SUBTITLE) { "track $trackNumber is not text" }
val bytes = text.encodeToByteArray()
val block = openCluster(timeMs)
block.master(MatroskaIds.BLOCK_GROUP) {
id(MatroskaIds.BLOCK)
dataSize(4L + bytes.size)
blockHeader(this, trackNumber, timeMs, 0)
raw(bytes)
uint(MatroskaIds.BLOCK_DURATION, maxOf(1L, durationMs))
}
noteEnd(timeMs + maxOf(1L, durationMs))
}
private fun blockHeader(block: EbmlWriter, trackNumber: Int, timeMs: Long, flags: Int) {
val relative = (maxOf(0L, timeMs) - clusterBaseMs).toInt()
block.raw(0x80 or trackNumber)
block.raw(relative shr 8)
block.raw(relative)
block.raw(flags)
}
private fun noteEnd(endMs: Long) {
if (endMs > durationMs) durationMs = endMs
}
/** The cluster a block at [timeMs] goes in: the open one, or a new one once that is full or too far away in time. */
private fun openCluster(timeMs: Long): EbmlWriter {
check(!finished) { "recording already finished" }
val time = maxOf(0L, timeMs)
val open = cluster
if (open != null) {
val relative = time - clusterBaseMs
if (relative < CLUSTER_SPAN_MS && relative > -CLUSTER_MAX_BACKSTEP_MS && open.size() < CLUSTER_MAX_BYTES) return open
flushCluster()
}
clusterBaseMs = time
return EbmlWriter(64 * 1024).also { cluster = it }
}
private fun flushCluster() {
val body = cluster ?: return
cluster = null
if (body.size() == 0) return
val timestamp = EbmlWriter(16)
timestamp.uint(MatroskaIds.TIMESTAMP, clusterBaseMs)
val header = EbmlWriter(32)
header.id(MatroskaIds.CLUSTER)
header.dataSize(timestamp.size().toLong() + body.size())
header.raw(timestamp.bytes())
cues.add(Cue(output.size - segmentStart, clusterBaseMs))
append(header.bytes())
append(body.bytes())
if (++clustersSincePatch >= DURATION_PATCH_CLUSTERS) patchDuration()
}
private fun patchDuration() {
clustersSincePatch = 0
val value = durationMs.toDouble().toRawBits()
output.writeAt(durationPosition, ByteArray(8) { (value ushr (8 * (7 - it))).toByte() })
}
/** Writes what is buffered, then the cues, duration and segment size, and closes the file. */
fun finish() {
if (finished) return
try {
flushCluster()
finished = true
val cuesPosition = output.size
val cueTrack = listedTracks().firstOrNull()?.number ?: 1
if (cues.isNotEmpty()) {
val index = EbmlWriter(cues.size * 16 + 16)
index.master(MatroskaIds.CUES) {
for (cue in cues) {
master(MatroskaIds.CUE_POINT) {
uint(MatroskaIds.CUE_TIME, cue.timeMs)
master(MatroskaIds.CUE_TRACK_POSITIONS) {
uint(MatroskaIds.CUE_TRACK, cueTrack.toLong())
uint(MatroskaIds.CUE_CLUSTER_POSITION, cue.clusterPosition)
}
}
}
}
append(index.bytes())
output.writeAt(seekPosition, seekHead(cuesPosition))
}
patchDuration()
val size = EbmlWriter(8)
size.dataSize(output.size - segmentStart, 8)
output.writeAt(segmentSizePosition, size.bytes())
} finally {
finished = true
output.close()
}
}
}
class MkaTrackInfo(
@JvmField val number: Int,
/** [MkaWriter.TRACK_AUDIO], [MkaWriter.TRACK_SUBTITLE] or another Matroska track type. */
@JvmField val type: Int,
@JvmField val codecId: String,
@JvmField val name: String,
@JvmField val sampleRate: Int,
@JvmField val channels: Int,
@JvmField val bitDepth: Int,
) {
val isPcm16: Boolean get() = type == MkaWriter.TRACK_AUDIO && codecId == MkaWriter.CODEC_PCM && bitDepth == 16
}
class MkaBlock(
@JvmField val track: Int,
@JvmField val timeMs: Long,
@JvmField val data: ByteArray,
/** From the block's group, or -1 when the file doesn't say (audio blocks). */
@JvmField val durationMs: Long,
)
/**
* Reads the blocks of a Matroska file in file order, a cluster at a time. Handles what
* [MkaWriter] and ffmpeg write for PCM audio: known-size clusters, unlaced blocks. Throws
* IllegalArgumentException for anything that is not such a file.
*/
class MkaReader(private val input: SeekableInput) {
private class ClusterRef(val position: Long, val size: Int, val rawTime: Long)
private val clusters = ArrayList<ClusterRef>()
private val trackList = ArrayList<MkaTrackInfo>()
private val attachmentList = ArrayList<MkaAttachment>()
private var timestampScale = 1_000_000L
private var infoDurationMs = 0L
private var nextCluster = 0
private var blocks: List<MkaBlock> = emptyList()
private var blockIndex = 0
private var skipBeforeMs = 0L
val tracks: List<MkaTrackInfo> get() = trackList
val attachments: List<MkaAttachment> get() = attachmentList
/** The end of the last audio or text in the file, in ms. */
val durationMs: Long
init {
scan()
var end = infoDurationMs
if (clusters.isNotEmpty()) {
for (block in readCluster(clusters.last())) end = maxOf(end, block.timeMs + blockSpanMs(block))
}
durationMs = end
}
fun track(number: Int): MkaTrackInfo? = trackList.firstOrNull { it.number == number }
/** Continue from the first block at or after [timeMs]. */
fun seekTo(timeMs: Long) {
val index = clusters.indexOfLast { toMs(it.rawTime) <= timeMs }
nextCluster = maxOf(0, index)
blocks = emptyList()
blockIndex = 0
skipBeforeMs = timeMs
}
/** The next block, or null at the end of the file. */
fun next(): MkaBlock? {
while (true) {
while (blockIndex >= blocks.size) {
if (nextCluster >= clusters.size) return null
blocks = readCluster(clusters[nextCluster++])
blockIndex = 0
}
val block = blocks[blockIndex++]
if (block.timeMs >= skipBeforeMs) return block
}
}
fun close() = input.close()
private fun toMs(raw: Long): Long = raw * timestampScale / 1_000_000L
private fun blockSpanMs(block: MkaBlock): Long {
if (block.durationMs >= 0) return block.durationMs
val track = track(block.track) ?: return 0
if (!track.isPcm16 || track.channels <= 0 || track.sampleRate <= 0) return 0
return block.data.size / (2L * track.channels) * 1000L / track.sampleRate
}
/** An element header at [at] in [bytes]: its ID, payload size (-1 unknown) and header length; null when cut off. */
private class Header(val id: Int, val size: Long, val length: Int)
private fun header(bytes: ByteArray, at: Int, end: Int): Header? {
if (at >= end) return null
val first = bytes[at].toInt() and 0xff
val idLength = first.countLeadingZeroBits() - 24 + 1
if (idLength > 4 || at + idLength >= end) return null
var id = 0
for (index in 0 until idLength) id = (id shl 8) or (bytes[at + index].toInt() and 0xff)
val sizeAt = at + idLength
val sizeFirst = bytes[sizeAt].toInt() and 0xff
val sizeLength = sizeFirst.countLeadingZeroBits() - 24 + 1
if (sizeLength > 8 || sizeAt + sizeLength > end) return null
var size = (sizeFirst and (0xff shr sizeLength)).toLong()
var allOnes = size == (0xff shr sizeLength).toLong()
for (index in 1 until sizeLength) {
val byte = bytes[sizeAt + index].toInt() and 0xff
if (byte != 0xff) allOnes = false
size = (size shl 8) or byte.toLong()
}
return Header(id, if (allOnes) -1 else size, idLength + sizeLength)
}
private fun readAt(position: Long, length: Int): ByteArray {
val buffer = ByteArray(length)
val read = input.readAt(position, buffer, 0, length)
return if (read == length) buffer else buffer.copyOf(maxOf(0, read))
}
private fun uint(bytes: ByteArray, at: Int, size: Int): Long {
var value = 0L
for (index in 0 until size) value = (value shl 8) or (bytes[at + index].toLong() and 0xff)
return value
}
private fun float(bytes: ByteArray, at: Int, size: Int): Double = when (size) {
4 -> Float.fromBits(uint(bytes, at, 4).toInt()).toDouble()
8 -> Double.fromBits(uint(bytes, at, 8))
else -> 0.0
}
/** Calls [visit] with the ID, payload offset and payload size of each complete child element. */
private inline fun children(bytes: ByteArray, start: Int, end: Int, visit: (id: Int, at: Int, size: Int) -> Unit) {
var at = start
while (true) {
val child = header(bytes, at, end) ?: return
val payload = at + child.length
if (child.size < 0 || child.size > end - payload) return
visit(child.id, payload, child.size.toInt())
at = payload + child.size.toInt()
}
}
private fun scan() {
val length = input.length
val start = readAt(0, 64)
val ebml = header(start, 0, start.size)
require(ebml != null && ebml.id == MatroskaIds.EBML && ebml.size >= 0) { "not a Matroska file" }
var position = ebml.length + ebml.size
val segment = readAt(position, 12).let { header(it, 0, it.size) }
require(segment != null && segment.id == MatroskaIds.SEGMENT) { "not a Matroska file" }
position += segment.length
val end = if (segment.size < 0) length else minOf(length, position + segment.size)
while (position < end) {
val head = readAt(position, 12)
val element = header(head, 0, head.size) ?: break
val payload = position + element.length
if (element.id == MatroskaIds.CLUSTER) {
require(element.size >= 0) { "unsupported Matroska file (clusters of unknown size)" }
// A recording cut short ends in a partial cluster; its complete blocks still count.
val available = minOf(element.size, end - payload)
require(available <= Int.MAX_VALUE) { "unsupported Matroska file (oversized cluster)" }
clusters.add(ClusterRef(payload, available.toInt(), clusterTime(payload, available.toInt())))
} else if (element.size < 0) {
break
} else if (element.size <= end - payload && element.size <= 64L * 1024 * 1024) {
when (element.id) {
MatroskaIds.INFO -> parseInfo(readAt(payload, element.size.toInt()))
MatroskaIds.TRACKS -> parseTracks(readAt(payload, element.size.toInt()))
MatroskaIds.ATTACHMENTS -> parseAttachments(readAt(payload, element.size.toInt()))
}
}
if (element.size < 0) break
position = payload + element.size
}
}
private fun clusterTime(position: Long, size: Int): Long {
val head = readAt(position, minOf(size, 16))
val first = header(head, 0, head.size)
if (first != null && first.id == MatroskaIds.TIMESTAMP && first.size in 0..8 && first.length + first.size <= head.size) {
return uint(head, first.length, first.size.toInt())
}
val whole = readAt(position, size)
var time = 0L
children(whole, 0, whole.size) { id, at, length -> if (id == MatroskaIds.TIMESTAMP) time = uint(whole, at, length) }
return time
}
private fun parseInfo(bytes: ByteArray) {
var duration = 0.0
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP_SCALE -> timestampScale = maxOf(1L, uint(bytes, at, size))
MatroskaIds.DURATION -> duration = float(bytes, at, size)
}
}
infoDurationMs = (duration * timestampScale / 1_000_000.0).toLong()
}
private fun parseTracks(bytes: ByteArray) {
trackList.clear()
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.TRACK_ENTRY) {
var number = 0
var type = 0
var codec = ""
var name = ""
var rate = 8000.0
var channels = 1
var depth = 0
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.TRACK_NUMBER -> number = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.TRACK_TYPE -> type = uint(bytes, fieldAt, fieldSize).toInt()
MatroskaIds.CODEC_ID -> codec = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize).trimEnd('\u0000')
MatroskaIds.AUDIO -> children(bytes, fieldAt, fieldAt + fieldSize) { audio, audioAt, audioSize ->
when (audio) {
MatroskaIds.SAMPLING_FREQUENCY -> rate = float(bytes, audioAt, audioSize)
MatroskaIds.CHANNELS -> channels = uint(bytes, audioAt, audioSize).toInt()
MatroskaIds.BIT_DEPTH -> depth = uint(bytes, audioAt, audioSize).toInt()
}
}
}
}
val audio = type == MkaWriter.TRACK_AUDIO
trackList.add(
MkaTrackInfo(number, type, codec, name, if (audio) rate.toInt() else 0, if (audio) channels else 0, if (audio) depth else 0),
)
}
}
}
private fun parseAttachments(bytes: ByteArray) {
children(bytes, 0, bytes.size) { id, at, size ->
if (id == MatroskaIds.ATTACHED_FILE) {
var name = ""
var mime = ""
var data = ByteArray(0)
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.FILE_NAME -> name = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_MIME_TYPE -> mime = bytes.decodeToString(fieldAt, fieldAt + fieldSize)
MatroskaIds.FILE_DATA -> data = bytes.copyOfRange(fieldAt, fieldAt + fieldSize)
}
}
attachmentList.add(MkaAttachment(name, mime, data))
}
}
}
private fun readCluster(cluster: ClusterRef): List<MkaBlock> {
val bytes = readAt(cluster.position, cluster.size)
val out = ArrayList<MkaBlock>()
var clusterRaw = cluster.rawTime
children(bytes, 0, bytes.size) { id, at, size ->
when (id) {
MatroskaIds.TIMESTAMP -> clusterRaw = uint(bytes, at, size)
MatroskaIds.SIMPLE_BLOCK -> block(bytes, at, size, clusterRaw, -1)?.let { out.add(it) }
MatroskaIds.BLOCK_GROUP -> {
var blockAt = -1
var blockSize = 0
var duration = -1L
children(bytes, at, at + size) { field, fieldAt, fieldSize ->
when (field) {
MatroskaIds.BLOCK -> {
blockAt = fieldAt
blockSize = fieldSize
}
MatroskaIds.BLOCK_DURATION -> duration = toMs(uint(bytes, fieldAt, fieldSize))
}
}
if (blockAt >= 0) block(bytes, blockAt, blockSize, clusterRaw, duration)?.let { out.add(it) }
}
}
}
return out
}
/** A block's payload: track number, 16-bit timestamp offset, flags, then the frame. Laced blocks are skipped. */
private fun block(bytes: ByteArray, at: Int, size: Int, clusterRaw: Long, durationMs: Long): MkaBlock? {
if (size < 4) return null
val first = bytes[at].toInt() and 0xff
val trackLength = first.countLeadingZeroBits() - 24 + 1
if (trackLength > 2 || size < trackLength + 3) return null
var track = first and (0xff shr trackLength)
if (trackLength == 2) track = (track shl 8) or (bytes[at + 1].toInt() and 0xff)
val head = at + trackLength
val relative = ((bytes[head].toInt() shl 8) or (bytes[head + 1].toInt() and 0xff)).toShort().toLong()
val flags = bytes[head + 2].toInt() and 0xff
if (flags and 0x06 != 0) return null
return MkaBlock(track, toMs(clusterRaw + relative), bytes.copyOfRange(head + 3, at + size), durationMs)
}
}
@@ -0,0 +1,93 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/**
* The parts of the CFW mic_control wire format the capture session speaks itself: the 'SM'
* multi-channel stream frame each temple sends, and the fixed STOP/RENEW control records. The
* CONFIGURE record is built by the caller (app/apps/microphones/mic-protocol.ts encodes it; the
* contract is the header of g2flash patches/mic_control.c).
*/
object MicArrayProtocol {
const val VERSION = 1
const val HEADER_BYTES = 21
const val OP_CONFIGURE = 1
const val OP_STOP = 3
const val OP_RENEW = 4
/** How long an armed temple streams without a CONFIGURE or RENEW. */
const val LEASE_MS = 90_000L
const val FLAG_TRUNCATED = 0x80
const val CODEC_RAW = 1
const val FORMAT_PCM16 = 0
/** One decoded 'SM' frame header; the samples are [payloadLength] bytes at [payloadOffset] of its packet. */
class Frame(
/** Low 16 bits of the temple's frame counter. */
@JvmField val sequence: Int,
/** The temple's own 1 ms tick when it packed the frame (unsigned 32-bit). */
@JvmField val deviceTickMs: Long,
@JvmField val channels: Int,
@JvmField val sampleRate: Int,
@JvmField val format: Int,
@JvmField val codec: Int,
/** The temple's own bearing estimate, in signed degrees (0 unless it was asked for one). */
@JvmField val angleDegrees: Int,
@JvmField val ssr: Int,
/** The firmware cut the payload short of the samples it captured. */
@JvmField val truncated: Boolean,
@JvmField val payloadOffset: Int,
@JvmField val payloadLength: Int,
)
/** A control record with no payload: ['M','C', version, op]. */
@JvmStatic
fun control(op: Int): ByteArray = byteArrayOf('M'.code.toByte(), 'C'.code.toByte(), VERSION.toByte(), op.toByte())
@JvmStatic
fun isConfigure(record: ByteArray?): Boolean =
record != null && record.size >= 13 && record[0] == 'M'.code.toByte() && record[1] == 'C'.code.toByte() &&
record[2] == VERSION.toByte() && record[3] == OP_CONFIGURE.toByte()
/** Decodes a stream frame's header, or returns null when [data] isn't one. */
@JvmStatic
fun decodeFrame(data: ByteArray): Frame? {
if (data.size < HEADER_BYTES || data[0] != 'S'.code.toByte() || data[1] != 'M'.code.toByte() || data[2] != VERSION.toByte()) {
return null
}
val length = minOf(u16(data, 19), data.size - HEADER_BYTES)
return Frame(
sequence = u16(data, 4),
deviceTickMs = u16(data, 6).toLong() or (u16(data, 8).toLong() shl 16),
channels = data[10].toInt() and 0xff,
sampleRate = u16(data, 11) * 100,
format = data[13].toInt() and 0xff,
codec = data[14].toInt() and 0xff,
angleDegrees = u16(data, 15).toShort().toInt(),
ssr = u16(data, 17).toShort().toInt(),
truncated = (data[3].toInt() and FLAG_TRUNCATED) != 0,
payloadOffset = HEADER_BYTES,
payloadLength = length,
)
}
/**
* The frame's samples as interleaved PCM16 (whole sample frames only), or null when the
* payload isn't raw 16-bit PCM or holds none.
*/
@JvmStatic
fun pcm16(data: ByteArray, frame: Frame): ShortArray? {
if (frame.codec != CODEC_RAW || frame.format != FORMAT_PCM16 || frame.channels < 1) return null
val frames = frame.payloadLength / (2 * frame.channels)
if (frames <= 0) return null
val samples = ShortArray(frames * frame.channels)
var at = frame.payloadOffset
for (index in samples.indices) {
samples[index] = AudioSegmentation.pcm16le(data[at], data[at + 1])
at += 2
}
return samples
}
private fun u16(data: ByteArray, at: Int): Int = (data[at].toInt() and 0xff) or ((data[at + 1].toInt() and 0xff) shl 8)
}
Loaded 100 of 167 files, more files were not shown because too many files have changed in this diff. Show more