diff --git a/.ackrc b/.ackrc index 8121069e..d1d22fb3 100644 --- a/.ackrc +++ b/.ackrc @@ -1,4 +1,6 @@ --ignore-file=match:.mjs +--ignore-file=match:frame-timings* --ignore-dir=tempPlugin --ignore-dir=build ---ignore-dir=platforms/android/build-tools +--ignore-dir=old +--ignore-dir=platforms diff --git a/.gitignore b/.gitignore index f780c2fc..b7c078e4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ hooks/ node_modules/ platforms/ +# Release APKs from scripts/release.sh +dist/ + # Logs logs *.log @@ -22,6 +25,7 @@ typings/ .gradle-cursor App_Resources/Android/src/main/jniLibs/**/*.so App_Resources/Android/src/main/assets/faceclaw-voice-asr/ +old # Visual Studio Code .vscode/* @@ -32,3 +36,7 @@ App_Resources/Android/src/main/assets/faceclaw-voice-asr/ # pull_config.sh output: the phone's live settings, including API keys faceclaw_settings.xml + +# Developer configs +build.sh +build_and_run.sh diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 00b6d984..3b554502 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -19,10 +19,8 @@ dependencies { def faceclawSherpaOnnxVersion = "1.13.0" def faceclawLibLc3Version = "1.1.3" -def faceclawAsrModelName = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27" def faceclawAbi = "arm64-v8a" def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi") -def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr") def faceclawNativeBuildDir = file("$buildDir/faceclaw-native") def faceclawDownload = { String url, File destination -> @@ -70,15 +68,11 @@ tasks.register("prepareFaceclawNativeLibs") { outputs.files( file("$faceclawJniLibsDir/libonnxruntime.so"), file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"), - file("$faceclawJniLibsDir/libfaceclaw_lc3.so"), - file("$faceclawAssetsDir/$faceclawAsrModelName/encoder_model.ort"), - file("$faceclawAssetsDir/$faceclawAsrModelName/decoder_model_merged.ort"), - file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt") + file("$faceclawJniLibsDir/libfaceclaw_lc3.so") ) doLast { faceclawJniLibsDir.mkdirs() - faceclawAssetsDir.mkdirs() def downloadsDir = file("$faceclawNativeBuildDir/downloads") def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2") @@ -95,20 +89,6 @@ tasks.register("prepareFaceclawNativeLibs") { into faceclawJniLibsDir } - def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2") - faceclawDownload( - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2", - asrArchive - ) - copy { - from tarTree(resources.bzip2(asrArchive)) - include "$faceclawAsrModelName/encoder_model.ort" - include "$faceclawAsrModelName/decoder_model_merged.ort" - include "$faceclawAsrModelName/tokens.txt" - includeEmptyDirs = false - into faceclawAssetsDir - } - def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz") faceclawDownload( "https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz", @@ -255,6 +235,22 @@ tasks.register("prepareFaceclawLlama") { } } +// versionName tracks FACECLAW_VERSION in app/version.ts, the single source of +// truth for the Faceclaw version. The gradle root project is platforms/android, +// so the repo root is two levels up from rootDir. +def faceclawVersionName = "0.0" +def faceclawVersionFile = file("$rootDir/../../app/version.ts") +if (faceclawVersionFile.exists()) { + def faceclawVersionMatch = (faceclawVersionFile.text =~ /FACECLAW_VERSION = "([^"]+)"/) + if (faceclawVersionMatch.find()) { + faceclawVersionName = faceclawVersionMatch.group(1) + } +} +def faceclawVersionCode = { + def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger() + return parts[0] * 10000 + parts[1] * 100 + parts[2] +}() + android { compileSdkVersion 35 buildToolsVersion "35" @@ -268,12 +264,17 @@ android { minSdkVersion 24 targetSdkVersion 35 ndk { + // The NativeScript platform build.gradle has already added all four + // ABIs by the time app.gradle is applied, and abiFilters is additive, + // so the set must be cleared for this filter to take effect. Only + // arm64 devices are supported (all our own JNI libs are arm64-only). + abiFilters.clear() abiFilters "arm64-v8a" } - // Version Information - versionCode 1 - versionName "1.0.0" + // Version Information (derived from app/version.ts above) + versionCode faceclawVersionCode + versionName faceclawVersionName generatedDensities = [] } diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/README.md b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/README.md deleted file mode 100644 index acdb1042..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -frameworks: -- 其他 -license: Apache License 2.0 -tasks: -- keyword-spotting ---- - -sherpa 中自定义唤醒词模型,训练数据为 gigaspeech XL (10000 小时)数据,模型大小约为 3.3 M, 建模单元为 BPE 。 使用 icefall 训练,已转换为 onnx 格式,此仓库主要是给 sherpa-onnx 这个 inference 引擎使用。 - -模型结构为 zipformer 模型, 本质是一个非常小的语音识别模型,为了实现唤醒词的功能,我们在解码端做了一些修改和约束。 支持自定义唤醒词,数量不限,效果需要单个调整参数。 - -#### Clone with HTTP -```bash - git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.git -``` \ No newline at end of file diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/bpe.model b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/bpe.model deleted file mode 100644 index 05c2ab6f..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/bpe.model and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx deleted file mode 100644 index 1900ac41..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx deleted file mode 100644 index fbf89f92..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx deleted file mode 100644 index a36c3fcf..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords.txt deleted file mode 100644 index abd2cb99..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords.txt +++ /dev/null @@ -1,10 +0,0 @@ -▁HE LL O ▁WORLD -▁HI ▁GO O G LE -▁HE Y ▁S I RI -▁A LE X A -▁LOVE ▁AND ▁P E A CE -▁PLAY ▁MU S IC -▁GO ▁HOME -▁HA PP Y ▁NEW ▁YEAR -▁ME R RY ▁ CH R IST MA S -▁S C RE EN ▁ON :1.5 #0.35 diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords_raw.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords_raw.txt deleted file mode 100644 index c3d6f6d4..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords_raw.txt +++ /dev/null @@ -1,9 +0,0 @@ -HELLO WORLD -HI GOOGLE -HEY SIRI -ALEXA -LOVE AND PEACE -PLAY MUSIC -GO HOME -HAPPY NEW YEAR -MERRY CHRISTMAS diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/screen-on-keywords.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/screen-on-keywords.txt deleted file mode 100644 index dd978326..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/screen-on-keywords.txt +++ /dev/null @@ -1 +0,0 @@ -▁S C RE EN ▁ON :1.5 #0.35 diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/tokens.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/tokens.txt deleted file mode 100644 index 61bb1828..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/tokens.txt +++ /dev/null @@ -1,500 +0,0 @@ - 0 - 1 - 2 -S 3 -T 4 -▁THE 5 -▁A 6 -▁TO 7 -▁AND 8 -N 9 -E 10 -D 11 -ING 12 -' 13 -▁OF 14 -▁I 15 -▁IN 16 -Y 17 -ED 18 -M 19 -▁ 20 -▁THAT 21 -O 22 -C 23 -R 24 -A 25 -P 26 -▁YOU 27 -U 28 -RE 29 -▁IT 30 -▁S 31 -G 32 -L 33 -B 34 -▁IS 35 -I 36 -AL 37 -ER 38 -▁RE 39 -▁C 40 -▁WE 41 -▁SO 42 -▁BE 43 -OR 44 -AR 45 -IN 46 -▁FOR 47 -F 48 -▁HE 49 -LE 50 -LY 51 -▁E 52 -▁WAS 53 -K 54 -▁F 55 -W 56 -VE 57 -RI 58 -CH 59 -LL 60 -▁THIS 61 -▁T 62 -EN 63 -▁ON 64 -IT 65 -CE 66 -▁WITH 67 -▁B 68 -▁W 69 -▁ME 70 -▁DO 71 -▁DE 72 -▁HAVE 73 -▁BUT 74 -ENT 75 -▁P 76 -UR 77 -ON 78 -▁AS 79 -IC 80 -TER 81 -TE 82 -▁THEY 83 -ES 84 -RA 85 -▁NOT 86 -TH 87 -▁ARE 88 -RO 89 -IL 90 -ATION 91 -AN 92 -▁ST 93 -▁WHAT 94 -SE 95 -LI 96 -V 97 -▁CAN 98 -▁G 99 -▁OR 100 -▁ALL 101 -NE 102 -▁AT 103 -▁CON 104 -H 105 -▁LIKE 106 -▁MO 107 -▁HIS 108 -▁AN 109 -▁MA 110 -▁THERE 111 -▁SU 112 -▁MY 113 -NG 114 -- 115 -▁ABOUT 116 -▁SE 117 -LA 118 -▁ONE 119 -LO 120 -▁SHE 121 -CK 122 -▁HAD 123 -AD 124 -MENT 125 -VER 126 -▁CA 127 -IR 128 -▁IF 129 -▁NO 130 -EL 131 -▁O 132 -▁FROM 133 -▁PA 134 -▁YOUR 135 -▁KNOW 136 -RY 137 -ION 138 -UN 139 -▁JUST 140 -▁UP 141 -▁US 142 -▁WHO 143 -▁GO 144 -ME 145 -▁EX 146 -▁OUT 147 -ND 148 -CI 149 -▁PO 150 -▁HER 151 -▁FA 152 -▁PRO 153 -▁WHEN 154 -ATE 155 -AT 156 -▁BY 157 -PE 158 -AS 159 -▁LA 160 -GE 161 -▁CO 162 -▁PEOPLE 163 -▁K 164 -▁SOME 165 -IGHT 166 -▁THINK 167 -VI 168 -DE 169 -▁LE 170 -▁WERE 171 -▁LI 172 -OM 173 -▁WOULD 174 -▁SEE 175 -CO 176 -▁BO 177 -▁OUR 178 -▁HOW 179 -ET 180 -Z 181 -UL 182 -▁HO 183 -IG 184 -▁GET 185 -▁DON 186 -IS 187 -URE 188 -▁WHICH 189 -▁MI 190 -IES 191 -US 192 -X 193 -▁MORE 194 -▁NOW 195 -▁TIME 196 -▁UN 197 -▁FI 198 -TY 199 -▁WILL 200 -RU 201 -IVE 202 -▁THEIR 203 -▁SAY 204 -ITY 205 -▁HA 206 -LU 207 -▁RO 208 -HO 209 -OL 210 -IST 211 -AGE 212 -MA 213 -KE 214 -▁RA 215 -ID 216 -ERS 217 -▁LO 218 -▁SP 219 -▁DA 220 -▁SAID 221 -TION 222 -THER 223 -▁EN 224 -▁BECAUSE 225 -▁HAS 226 -EST 227 -TI 228 -ACT 229 -▁BA 230 -▁VERY 231 -▁THEM 232 -▁BEEN 233 -▁WA 234 -ABLE 235 -▁SA 236 -▁DID 237 -▁HIM 238 -▁DIS 239 -▁WANT 240 -▁OTHER 241 -IM 242 -▁PRE 243 -▁LOOK 244 -▁GOING 245 -▁VI 246 -▁WORK 247 -▁DR 248 -ENCE 249 -▁DI 250 -FF 251 -TA 252 -▁REALLY 253 -▁RIGHT 254 -MO 255 -ALLY 256 -▁INTO 257 -OW 258 -▁WAY 259 -▁FE 260 -▁COULD 261 -▁THEN 262 -TED 263 -▁HERE 264 -▁THESE 265 -▁OVER 266 -▁WELL 267 -▁NA 268 -▁ANY 269 -HE 270 -▁COMP 271 -▁COM 272 -▁TH 273 -▁EVEN 274 -▁TWO 275 -▁JO 276 -▁MAKE 277 -▁BACK 278 -▁WHERE 279 -SH 280 -▁QU 281 -MAN 282 -▁YEAR 283 -▁TRA 284 -▁CHA 285 -QU 286 -CA 287 -▁ALSO 288 -CTION 289 -ANCE 290 -UM 291 -PER 292 -DI 293 -▁HU 294 -▁MU 295 -FUL 296 -▁NEW 297 -ARD 298 -J 299 -ICAL 300 -ANT 301 -END 302 -▁MAR 303 -▁NE 304 -IF 305 -DER 306 -ONE 307 -▁THOSE 308 -PP 309 -▁MAN 310 -▁GOOD 311 -▁TAKE 312 -SIDE 313 -PO 314 -HA 315 -▁LITTLE 316 -▁APP 317 -▁BI 318 -UGH 319 -▁COME 320 -ISH 321 -TIC 322 -▁PER 323 -▁THINGS 324 -▁MUCH 325 -▁FIRST 326 -▁THAN 327 -IA 328 -▁AGAIN 329 -▁STA 330 -▁PART 331 -▁GA 332 -▁MEAN 333 -▁LET 334 -▁DOWN 335 -▁NEED 336 -▁GU 337 -LES 338 -▁RU 339 -ATED 340 -▁SHA 341 -VO 342 -▁TALK 343 -▁DAY 344 -▁CHI 345 -▁SOMETHING 346 -▁AFTER 347 -▁THOUGH 348 -▁CALL 349 -▁GRA 350 -▁EVERY 351 -▁KIND 352 -ROW 353 -AIN 354 -▁BU 355 -▁THROUGH 356 -▁SHOULD 357 -▁ONLY 358 -▁TA 359 -▁MIN 360 -▁GOT 361 -IAN 362 -▁GREAT 363 -▁RI 364 -▁DOES 365 -▁THING 366 -DAY 367 -OUS 368 -▁START 369 -▁MOST 370 -▁COL 371 -PH 372 -▁LOT 373 -IZE 374 -BER 375 -IOUS 376 -▁LIFE 377 -▁MANY 378 -▁FEEL 379 -ITION 380 -▁VA 381 -NESS 382 -▁BEFORE 383 -▁THREE 384 -▁WORLD 385 -▁GIVE 386 -▁WHY 387 -▁PI 388 -▁LONG 389 -PORT 390 -▁SHOW 391 -▁ACTUALLY 392 -▁NEVER 393 -IV 394 -FORM 395 -▁THANK 396 -▁HI 397 -▁PUT 398 -▁LOVE 399 -▁QUESTION 400 -▁HELP 401 -▁HAPPEN 402 -▁MR 403 -▁OWN 404 -WARD 405 -▁MADE 406 -▁HAND 407 -▁STATE 408 -IBLE 409 -▁TELL 410 -▁PH 411 -▁CAME 412 -▁TEA 413 -▁STILL 414 -▁DIFFERENT 415 -▁PLACE 416 -▁MIGHT 417 -▁FIND 418 -▁DU 419 -▁IMP 420 -▁OLD 421 -▁PERSON 422 -▁COUNT 423 -▁JA 424 -▁POINT 425 -▁HIGH 426 -SHIP 427 -▁LAST 428 -▁SAME 429 -▁FOUR 430 -▁AROUND 431 -▁OH 432 -▁CHANGE 433 -▁UNDER 434 -▁HOME 435 -▁ANOTHER 436 -▁FOUND 437 -ATIVE 438 -▁TURN 439 -▁YES 440 -LAND 441 -▁ALWAYS 442 -▁YEAH 443 -▁SCHOOL 444 -▁JE 445 -▁NEXT 446 -▁WORD 447 -▁JU 448 -LINE 449 -▁SIX 450 -▁LEARN 451 -▁BOOK 452 -▁PLAY 453 -▁AWAY 454 -▁HOUSE 455 -▁PLAN 456 -▁WHILE 457 -▁COMMUNI 458 -▁FRIEND 459 -▁AMERICA 460 -▁INTEREST 461 -▁KEEP 462 -▁BETTER 463 -▁IDEA 464 -▁HARD 465 -▁TWENTY 466 -▁THOUSAND 467 -▁HUNDRED 468 -▁WEEK 469 -▁NUMBER 470 -▁COURSE 471 -▁OKAY 472 -PRESS 473 -▁SYSTEM 474 -▁NINE 475 -▁UNDERSTAND 476 -▁WITHOUT 477 -▁BELIEVE 478 -▁POWER 479 -▁MAYBE 480 -▁BUILD 481 -▁MONEY 482 -▁BETWEEN 483 -▁GONNA 484 -▁IMPORTANT 485 -▁ANYTHING 486 -▁LEFT 487 -▁HAVING 488 -▁WHITE 489 -▁PROBLEM 490 -▁SECOND 491 -▁SPEAK 492 -▁TOGETHER 493 -▁BLACK 494 -▁PUBLIC 495 -▁HEALTH 496 -▁NOTHING 497 -▁EVERYTHING 498 -Q 499 diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java index d57c4d3a..0452bbda 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java @@ -6,151 +6,392 @@ import android.bluetooth.BluetoothDevice; import android.bluetooth.BluetoothManager; import android.bluetooth.le.BluetoothLeScanner; import android.bluetooth.le.ScanCallback; +import android.bluetooth.le.ScanRecord; import android.bluetooth.le.ScanResult; import android.bluetooth.le.ScanSettings; import android.content.Context; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.util.SparseArray; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; import java.util.Set; -import java.util.regex.Pattern; +/** + * Finds Even Realities hardware over BLE and hands every advertisement up to + * TypeScript with the raw fields the pairing screen needs: the LIVE local name + * from the scan record (BluetoothDevice.getName() is a cache that can hold a + * pre-reset value indefinitely), the full manufacturer-specific payload with + * its company identifier restored so the "ER"+serial+MAC layout survives + * intact, RSSI, advertised TX power, and connectability. + * + * Admission is deliberately loose — a name containing G2 or R1, or the Even + * "ER" manufacturer signature — because renamed custom firmware may drop the + * stock name. Side, serial, and pair matching are decided in TypeScript + * (app/g2/even-advertisement.ts, app/g2/pairing-candidates.ts) where they can + * be unit-tested. + * + * Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover + * and R1's advertisement snapshot), reduced to what Android exposes. + */ @SuppressLint("MissingPermission") public class FaceclawDeviceDiscovery { - private static final Pattern RING_NAME = Pattern.compile("^EVEN R1_[0-9A-F]{6}$"); - private static final Pattern RIGHT_NAME = Pattern.compile("^Even G2_32_R_[0-9A-F]{6}$"); - private static final Pattern LEFT_NAME = Pattern.compile("^Even G2_32_L_[0-9A-F]{6}$"); + /** Even Realities' company identifier: the ASCII bytes "ER" read little-endian. */ + public static final int EVEN_COMPANY_ID = 0x5245; - private final Context appContext; private final BluetoothAdapter bluetoothAdapter; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + + private final Object lock = new Object(); + private ScanCallback activeCallback; + private volatile FaceclawDeviceDiscoveryListener listener; public FaceclawDeviceDiscovery(Context context) { - this.appContext = context.getApplicationContext(); + Context appContext = context.getApplicationContext(); BluetoothManager bluetoothManager = - (BluetoothManager) this.appContext.getSystemService(Context.BLUETOOTH_SERVICE); + (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE); if (bluetoothManager == null || bluetoothManager.getAdapter() == null) { throw new IllegalStateException("Bluetooth adapter unavailable"); } this.bluetoothAdapter = bluetoothManager.getAdapter(); } - public String getBondedCandidatesJson() { - LinkedHashMap matches = new LinkedHashMap<>(); - Set bonded = bluetoothAdapter.getBondedDevices(); - if (bonded != null) { - for (BluetoothDevice device : bonded) { - maybeAddCandidate(matches, device, "paired"); - } - } - return encodeCandidates(matches); + public boolean isBluetoothEnabled() { + return bluetoothAdapter.isEnabled(); } - public String scanCandidatesJson(int timeoutMs) throws InterruptedException { - LinkedHashMap matches = new LinkedHashMap<>(); - BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); - if (scanner == null) { - return encodeCandidates(matches); - } - final ScanCallback callback = new ScanCallback() { - @Override - public void onScanResult(int callbackType, ScanResult result) { - if (result == null) { - return; - } - maybeAddCandidate(matches, result.getDevice(), "scan"); - } + public void setListener(FaceclawDeviceDiscoveryListener newListener) { + this.listener = newListener; + } - @Override - public void onBatchScanResults(java.util.List results) { - if (results == null) { - return; - } - for (ScanResult result : results) { + // ------------------------------------------------------------------ + // Live scan + // ------------------------------------------------------------------ + + /** + * Start streaming advertisements to the listener. Returns false when the + * radio is off or the scanner is unavailable; a later platform refusal + * arrives through onScanFailed. Calling while a scan is running is a no-op. + */ + public boolean startScan() { + synchronized (lock) { + if (activeCallback != null) { + return true; + } + if (!bluetoothAdapter.isEnabled()) { + emitLog("scan not started: Bluetooth is off"); + return false; + } + BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); + if (scanner == null) { + emitLog("scan not started: no BLE scanner"); + return false; + } + final ScanCallback callback = new ScanCallback() { + @Override + public void onScanResult(int callbackType, ScanResult result) { if (result != null) { - maybeAddCandidate(matches, result.getDevice(), "scan"); + emitAdvertisement(result); } } + + @Override + public void onBatchScanResults(List results) { + if (results == null) { + return; + } + for (ScanResult result : results) { + if (result != null) { + emitAdvertisement(result); + } + } + } + + @Override + public void onScanFailed(int errorCode) { + synchronized (lock) { + if (activeCallback == this) { + activeCallback = null; + } + } + final FaceclawDeviceDiscoveryListener l = listener; + if (l != null) { + final String message = describeScanError(errorCode); + mainHandler.post(() -> l.onScanFailed(errorCode, message)); + } + } + }; + activeCallback = callback; + try { + scanner.startScan(null, buildScanSettings(), callback); + } catch (Throwable t) { + activeCallback = null; + emitLog("scan not started: " + t); + return false; } - }; - scanner.startScan( - null, - new ScanSettings.Builder().setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY).build(), - callback - ); - try { - Thread.sleep(Math.max(500, timeoutMs)); - } finally { + emitLog("scan started (unfiltered, low latency)"); + return true; + } + } + + public void stopScan() { + ScanCallback callback; + synchronized (lock) { + callback = activeCallback; + activeCallback = null; + } + if (callback == null) { + return; + } + BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); + if (scanner != null) { try { scanner.stopScan(callback); } catch (Throwable ignored) { } } - return encodeCandidates(matches); + emitLog("scan stopped"); } - private void maybeAddCandidate(Map matches, BluetoothDevice device, String source) { - if (device == null) { - return; + public boolean isScanning() { + synchronized (lock) { + return activeCallback != null; } - String name = device.getName(); - String role = classifyRole(name); - if (role == null) { - return; - } - String address = device.getAddress(); - if (address == null || address.isEmpty()) { - return; - } - matches.put(address, new CandidateDevice(address, name == null ? "" : name, role, source)); } - private String classifyRole(String name) { - if (name == null) { - return null; + /** + * Replay bonded Even devices to the listener. A bonded listing has no + * signal sample and no manufacturer data, so its serial stays unknown until + * the device is also heard advertising. + */ + public void emitBondedDevices() { + final FaceclawDeviceDiscoveryListener l = listener; + if (l == null) { + return; } - if (RIGHT_NAME.matcher(name).matches()) { - return "right"; + for (String json : bondedCandidateJsonList()) { + l.onAdvertisement(json); } - if (LEFT_NAME.matcher(name).matches()) { - return "left"; - } - if (RING_NAME.matcher(name).matches()) { - return "ring"; - } - return null; } - private String encodeCandidates(Map matches) { + /** Bonded Even devices as one JSON array, for callers without a listener. */ + public String getBondedCandidatesJson() { JSONArray array = new JSONArray(); - for (CandidateDevice candidate : matches.values()) { - JSONObject object = new JSONObject(); + for (String json : bondedCandidateJsonList()) { try { - object.put("address", candidate.address); - object.put("name", candidate.name); - object.put("role", candidate.role); - object.put("source", candidate.source); - array.put(object); + array.put(new JSONObject(json)); } catch (JSONException ignored) { } } return array.toString(); } - private static final class CandidateDevice { - final String address; - final String name; - final String role; - final String source; + // ------------------------------------------------------------------ + // Internals + // ------------------------------------------------------------------ - CandidateDevice(String address, String name, String role, String source) { - this.address = address; - this.name = name; - this.role = role; - this.source = source; + private static ScanSettings buildScanSettings() { + ScanSettings.Builder builder = new ScanSettings.Builder() + .setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY) + .setReportDelay(0); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + builder.setCallbackType(ScanSettings.CALLBACK_TYPE_ALL_MATCHES); + builder.setMatchMode(ScanSettings.MATCH_MODE_AGGRESSIVE); + builder.setNumOfMatches(ScanSettings.MATCH_NUM_MAX_ADVERTISEMENT); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + // Accept extended advertising too; setLegacy(true) would hide it. + builder.setLegacy(false); + } + return builder.build(); + } + + private void emitAdvertisement(ScanResult result) { + final JSONObject object = toJson(result); + if (object == null) { + return; + } + final FaceclawDeviceDiscoveryListener l = listener; + if (l == null) { + return; + } + final String json = object.toString(); + if (Looper.myLooper() == Looper.getMainLooper()) { + l.onAdvertisement(json); + } else { + mainHandler.post(() -> { + FaceclawDeviceDiscoveryListener current = listener; + if (current != null) { + current.onAdvertisement(json); + } + }); + } + } + + private List bondedCandidateJsonList() { + List out = new ArrayList<>(); + Set bonded = bluetoothAdapter.getBondedDevices(); + if (bonded == null) { + return out; + } + long now = System.currentTimeMillis(); + for (BluetoothDevice device : bonded) { + if (device == null) { + continue; + } + String name = device.getName(); + if (!isAdmissible(name, null)) { + continue; + } + JSONObject object = buildJson( + device.getAddress(), name, "", null, null, null, true, "paired", now); + if (object != null) { + out.add(object.toString()); + } + } + return out; + } + + /** Null when the result is not something we pair with. */ + private static JSONObject toJson(ScanResult result) { + BluetoothDevice device = result.getDevice(); + if (device == null) { + return null; + } + ScanRecord record = result.getScanRecord(); + String liveName = record == null ? null : record.getDeviceName(); + String name = liveName != null && !liveName.isEmpty() ? liveName : device.getName(); + byte[] manufacturerData = extractEvenManufacturerData(record); + if (!isAdmissible(name, manufacturerData)) { + return null; + } + Integer txPower = null; + if (record != null && record.getTxPowerLevel() != Integer.MIN_VALUE) { + txPower = record.getTxPowerLevel(); + } + if (txPower == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O + && result.getTxPower() != ScanResult.TX_POWER_NOT_PRESENT) { + txPower = result.getTxPower(); + } + Boolean connectable = null; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + connectable = result.isConnectable(); + } + int rssi = result.getRssi(); + // 127 is the "unavailable" sentinel; 0 dBm never happens for a real sample. + Integer rssiValue = (rssi == 127 || rssi == 0) ? null : rssi; + boolean bonded = device.getBondState() == BluetoothDevice.BOND_BONDED; + return buildJson( + device.getAddress(), name, toHex(manufacturerData), rssiValue, txPower, + connectable, bonded, "scan", System.currentTimeMillis()); + } + + /** + * Android strips the 2-byte company identifier from each manufacturer + * record and keys the SparseArray by it. Put the bytes back (little-endian) + * so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures + * document. Prefers Even's own id; otherwise returns the first record so an + * unexpected company id still reaches the diagnostics log. + */ + private static byte[] extractEvenManufacturerData(ScanRecord record) { + if (record == null) { + return new byte[0]; + } + SparseArray all = record.getManufacturerSpecificData(); + if (all == null || all.size() == 0) { + return new byte[0]; + } + byte[] payload = all.get(EVEN_COMPANY_ID); + int companyId = EVEN_COMPANY_ID; + if (payload == null) { + companyId = all.keyAt(0); + payload = all.valueAt(0); + } + if (payload == null) { + return new byte[0]; + } + byte[] out = new byte[payload.length + 2]; + out[0] = (byte) (companyId & 0xff); + out[1] = (byte) ((companyId >> 8) & 0xff); + System.arraycopy(payload, 0, out, 2, payload.length); + return out; + } + + private static boolean isAdmissible(String name, byte[] manufacturerData) { + if (manufacturerData != null && manufacturerData.length >= 2 + && manufacturerData[0] == 0x45 && manufacturerData[1] == 0x52) { + return true; + } + if (name == null) { + return false; + } + String upper = name.toUpperCase(); + return upper.contains("G2") || upper.contains("R1"); + } + + private static JSONObject buildJson( + String address, String name, String manufacturerHex, Integer rssi, Integer txPower, + Boolean connectable, boolean bonded, String source, long seenAtMs) { + if (address == null || address.isEmpty()) { + return null; + } + try { + JSONObject object = new JSONObject(); + object.put("address", address); + object.put("name", name == null ? "" : name); + object.put("manufacturerData", manufacturerHex == null ? "" : manufacturerHex); + object.put("rssi", rssi == null ? JSONObject.NULL : rssi); + object.put("txPower", txPower == null ? JSONObject.NULL : txPower); + object.put("connectable", connectable == null ? JSONObject.NULL : connectable); + object.put("bonded", bonded); + object.put("source", source); + object.put("seenAtMs", seenAtMs); + return object; + } catch (JSONException e) { + return null; + } + } + + private static String toHex(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return ""; + } + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02X", b & 0xff)); + } + return sb.toString(); + } + + private static String describeScanError(int errorCode) { + switch (errorCode) { + case ScanCallback.SCAN_FAILED_ALREADY_STARTED: + return "a scan is already running"; + case ScanCallback.SCAN_FAILED_APPLICATION_REGISTRATION_FAILED: + return "the Bluetooth stack refused to register the scan (try toggling Bluetooth)"; + case ScanCallback.SCAN_FAILED_FEATURE_UNSUPPORTED: + return "BLE scanning is not supported on this device"; + case ScanCallback.SCAN_FAILED_INTERNAL_ERROR: + return "internal Bluetooth error"; + case 5: // SCAN_FAILED_OUT_OF_HARDWARE_RESOURCES (API 31) + return "out of Bluetooth hardware resources"; + case 6: // SCAN_FAILED_SCANNING_TOO_FREQUENTLY (API 31) + return "scanning too frequently; wait a moment and try again"; + default: + return "scan failed (code " + errorCode + ")"; + } + } + + private void emitLog(String line) { + FaceclawDeviceDiscoveryListener l = listener; + if (l != null) { + l.onLog(line); } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscoveryListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscoveryListener.java new file mode 100644 index 00000000..922e7adb --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscoveryListener.java @@ -0,0 +1,18 @@ +package com.faceclaw.app; + +/** Callbacks from FaceclawDeviceDiscovery's live scan to the TypeScript layer. */ +public interface FaceclawDeviceDiscoveryListener { + /** + * One advertisement (or one bonded device) as a JSON object: + * {address, name, manufacturerData (hex, company id included), rssi|null, + * txPower|null, connectable|null, bonded, source, seenAtMs}. + * Protocol decoding happens in TypeScript (see app/g2/even-advertisement.ts). + */ + void onAdvertisement(String json); + + /** The platform refused to start or continue the scan. */ + void onScanFailed(int errorCode, String message); + + /** Diagnostic line for the pairing log. */ + void onLog(String line); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 401808ca..e0a75ab3 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -1,30 +1,20 @@ package com.faceclaw.app; import android.content.Context; -import android.content.res.AssetManager; import android.os.Handler; import android.os.Looper; import android.os.SystemClock; import android.util.Log; import com.k2fsa.sherpa.onnx.FeatureConfig; -import com.k2fsa.sherpa.onnx.KeywordSpotter; -import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; -import com.k2fsa.sherpa.onnx.KeywordSpotterResult; import com.k2fsa.sherpa.onnx.OfflineModelConfig; import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizer; import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; import com.k2fsa.sherpa.onnx.OfflineStream; -import com.k2fsa.sherpa.onnx.OnlineModelConfig; -import com.k2fsa.sherpa.onnx.OnlineStream; -import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.Arrays; @@ -57,17 +47,15 @@ public class FaceclawVoiceController { private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f; private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f; private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; - private static final String ASSET_ROOT = "faceclaw-voice"; - private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; - private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr"; + // Model directory shared with the TS-side download flow (asr-model.ts), + // which fetches the Moonshine files here on demand; they are no longer + // bundled in the APK. + private static final String ASR_ROOT = "faceclaw-voice-asr"; private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"; - private static final String[] MODEL_FILES = { - "encoder-epoch-12-avg-2-chunk-16-left-64.onnx", - "decoder-epoch-12-avg-2-chunk-16-left-64.onnx", - "joiner-epoch-12-avg-2-chunk-16-left-64.onnx", - "tokens.txt", - "screen-on-keywords.txt" - }; + // Model files for the retired on-phone wake-word spotter, copied to + // filesDir by earlier releases; deleted on sight to reclaim the space. + // (The wakeword is now detected by the glasses firmware itself.) + private static final String LEGACY_KWS_ROOT = "faceclaw-voice"; private static final String[] ASR_MODEL_FILES = { "encoder_model.ort", "decoder_model_merged.ort", @@ -75,7 +63,6 @@ public class FaceclawVoiceController { }; private enum VoiceInputMode { - WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired) ONBOARD, // on-phone Moonshine transcription CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side } @@ -89,10 +76,8 @@ public class FaceclawVoiceController { private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; - private VoiceInputMode mode = VoiceInputMode.WAKEWORD; - private KeywordSpotter keywordSpotter; + private VoiceInputMode mode = VoiceInputMode.CLOUD; private OfflineRecognizer recognizer; - private OnlineStream stream; private FaceclawLc3Decoder lc3Decoder; private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES]; private int transcriptSampleCount; @@ -140,10 +125,6 @@ public class FaceclawVoiceController { this.endpointing = endpointing; } - public void start() { - start("wakeword"); - } - public void start(String requestedMode) { synchronized (lock) { if (started) { @@ -194,26 +175,23 @@ public class FaceclawVoiceController { if ("cloud".equals(requestedMode)) { return VoiceInputMode.CLOUD; } - if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) { - return VoiceInputMode.ONBOARD; - } - return VoiceInputMode.WAKEWORD; + return VoiceInputMode.ONBOARD; } private void runLoop() { try { + deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; if (currentMode == VoiceInputMode.ONBOARD) { + File modelDir = findAsrModelDir(); + if (modelDir == null) { + emitStatus("Voice model not downloaded (see Settings > Voice)."); + return; + } emitStatus("Loading transcription model..."); - File modelDir = installAsrModelFiles(); recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); resetTranscriptState(); lastTranscript = ""; - } else if (currentMode == VoiceInputMode.WAKEWORD) { - emitStatus("Loading wake-word model..."); - File modelDir = installModelFiles(); - keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); - stream = keywordSpotter.createStream(); } lc3Decoder = new FaceclawLc3Decoder(); endpointDetector.reset(); @@ -225,9 +203,7 @@ public class FaceclawVoiceController { emitStatus(currentMode == VoiceInputMode.CLOUD ? "Listening (cloud)..." - : currentMode == VoiceInputMode.ONBOARD - ? "Listening..." - : "Listening for \"screen on\"..."); + : "Listening..."); processG2Audio(); // Button released / stop requested: emit one final full-utterance // transcript so the UI can freeze it. @@ -312,29 +288,6 @@ public class FaceclawVoiceController { return b.array(); } - private KeywordSpotterConfig buildConfig(File modelDir) { - return KeywordSpotterConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setOnlineModelConfig(OnlineModelConfig.builder() - .setTransducer(OnlineTransducerModelConfig.builder() - .setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setModelType("zipformer2") - .setModelingUnit("") - .setNumThreads(1) - .build()) - .setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath()) - .setKeywordsScore(1.5f) - .setKeywordsThreshold(0.35f) - .build(); - } - private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) { return OfflineRecognizerConfig.builder() .setFeatureConfig(FeatureConfig.builder() @@ -352,50 +305,42 @@ public class FaceclawVoiceController { .build(); } - private File installModelFiles() throws IOException { - File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR); - if (!modelDir.exists() && !modelDir.mkdirs()) { - throw new IOException("Could not create " + modelDir.getAbsolutePath()); - } - AssetManager assets = appContext.getAssets(); - for (String fileName : MODEL_FILES) { - copyAssetIfNeeded( - assets, - ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName, - new File(modelDir, fileName) - ); - } - return modelDir; - } - - private File installAsrModelFiles() throws IOException { - File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR); - if (!modelDir.exists() && !modelDir.mkdirs()) { - throw new IOException("Could not create " + modelDir.getAbsolutePath()); - } - AssetManager assets = appContext.getAssets(); + /** + * The Moonshine model directory, populated by the download flow in + * asr-model.ts (releases before 0.5.0 copied the same files out of the + * APK, so upgraded installs are already complete). Null when any file is + * missing, i.e. the model still needs to be downloaded. + */ + private File findAsrModelDir() { + File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR); for (String fileName : ASR_MODEL_FILES) { - copyAssetIfNeeded( - assets, - ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName, - new File(modelDir, fileName) - ); - } - return modelDir; - } - - private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException { - if (destination.exists() && destination.length() > 0) { - return; - } - try (InputStream input = assets.open(assetPath); - FileOutputStream output = new FileOutputStream(destination)) { - byte[] buffer = new byte[64 * 1024]; - int read; - while ((read = input.read(buffer)) >= 0) { - output.write(buffer, 0, read); + File file = new File(modelDir, fileName); + if (!file.exists() || file.length() == 0) { + return null; } } + return modelDir; + } + + private void deleteLegacyKwsFiles() { + try { + deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT)); + } catch (Throwable t) { + Log.w(TAG, "failed to delete legacy wake-word files", t); + } + } + + private static void deleteRecursively(File file) { + if (!file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); } private boolean startG2Audio() { @@ -413,9 +358,8 @@ public class FaceclawVoiceController { private void processG2Audio() { short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET]; while (started && !Thread.currentThread().isInterrupted()) { - OnlineStream currentStream = stream; FaceclawLc3Decoder currentDecoder = lc3Decoder; - if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) { + if (currentDecoder == null) { return; } @@ -438,40 +382,17 @@ public class FaceclawVoiceController { } if (mode == VoiceInputMode.CLOUD) { emitPcm(pcm, count); - } else if (mode == VoiceInputMode.ONBOARD) { - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - processRecognizer(samples); } else { float[] samples = new float[count]; for (int i = 0; i < count; i++) { samples[i] = pcm[i] / 32768.0f; } - currentStream.acceptWaveform(samples, SAMPLE_RATE); - processKeywordSpotter(currentStream); + processRecognizer(samples); } maybeEmitAudioStats(false); } } - private void processKeywordSpotter(OnlineStream currentStream) { - KeywordSpotter currentSpotter = keywordSpotter; - if (currentSpotter == null) { - return; - } - while (currentSpotter.isReady(currentStream)) { - currentSpotter.decode(currentStream); - KeywordSpotterResult result = currentSpotter.getResult(currentStream); - String keyword = result == null ? "" : result.getKeyword(); - if (keyword != null && keyword.trim().length() > 0) { - currentSpotter.reset(currentStream); - emitWakeWord(keyword); - } - } - } - private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); long now = SystemClock.elapsedRealtime(); @@ -783,14 +704,6 @@ public class FaceclawVoiceController { } private void releaseSherpa() { - if (stream != null) { - stream.release(); - stream = null; - } - if (keywordSpotter != null) { - keywordSpotter.release(); - keywordSpotter = null; - } if (recognizer != null) { recognizer.release(); recognizer = null; @@ -893,14 +806,6 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onStatus(status)); } - private void emitWakeWord(String keyword) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onWakeWord(keyword)); - } - private void emitTranscript(String text, boolean isFinal) { FaceclawVoiceControllerListener currentListener = listener; if (currentListener == null) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index bce73665..504a5df3 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -3,8 +3,6 @@ package com.faceclaw.app; public interface FaceclawVoiceControllerListener { void onStatus(String status); - void onWakeWord(String keyword); - /** * Best transcript of the current utterance so far. REPLACE semantics: text * is the complete transcript, not a delta — display it verbatim, replacing diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java deleted file mode 100644 index 836a07c2..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotter { - private long ptr; - - public KeywordSpotter(KeywordSpotterConfig config) { - System.loadLibrary("onnxruntime"); - System.loadLibrary("sherpa-onnx-jni"); - ptr = newFromFile(config); - if (ptr == 0) { - throw new IllegalArgumentException("Invalid KeywordSpotterConfig"); - } - } - - public OnlineStream createStream() { - return createStream(""); - } - - public OnlineStream createStream(String keywords) { - return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords)); - } - - public boolean isReady(OnlineStream stream) { - return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr()); - } - - public void decode(OnlineStream stream) { - decode(ptr, stream.getPtr()); - } - - public void reset(OnlineStream stream) { - reset(ptr, stream.getPtr()); - } - - public KeywordSpotterResult getResult(OnlineStream stream) { - return getResult(ptr, stream.getPtr()); - } - - public void release() { - if (ptr == 0) { - return; - } - delete(ptr); - ptr = 0; - } - - @Override - protected void finalize() throws Throwable { - release(); - super.finalize(); - } - - private native long newFromFile(KeywordSpotterConfig config); - - private native void delete(long ptr); - - private native long createStream(long ptr, String keywords); - - private native boolean isReady(long ptr, long streamPtr); - - private native void decode(long ptr, long streamPtr); - - private native void reset(long ptr, long streamPtr); - - private native KeywordSpotterResult getResult(long ptr, long streamPtr); -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java deleted file mode 100644 index 3c4ce6f3..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotterConfig { - private final FeatureConfig featConfig; - private final OnlineModelConfig modelConfig; - private final int maxActivePaths; - private final String keywordsFile; - private final float keywordsScore; - private final float keywordsThreshold; - private final int numTrailingBlanks; - - private KeywordSpotterConfig(Builder builder) { - this.featConfig = builder.featConfig; - this.modelConfig = builder.modelConfig; - this.maxActivePaths = builder.maxActivePaths; - this.keywordsFile = builder.keywordsFile; - this.keywordsScore = builder.keywordsScore; - this.keywordsThreshold = builder.keywordsThreshold; - this.numTrailingBlanks = builder.numTrailingBlanks; - } - - public static Builder builder() { - return new Builder(); - } - - public FeatureConfig getFeatConfig() { - return featConfig; - } - - public OnlineModelConfig getModelConfig() { - return modelConfig; - } - - public int getMaxActivePaths() { - return maxActivePaths; - } - - public String getKeywordsFile() { - return keywordsFile; - } - - public float getKeywordsScore() { - return keywordsScore; - } - - public float getKeywordsThreshold() { - return keywordsThreshold; - } - - public int getNumTrailingBlanks() { - return numTrailingBlanks; - } - - public static class Builder { - private FeatureConfig featConfig = FeatureConfig.builder().build(); - private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build(); - private int maxActivePaths = 4; - private String keywordsFile = "keywords.txt"; - private float keywordsScore = 1.5f; - private float keywordsThreshold = 0.25f; - private int numTrailingBlanks = 2; - - public Builder setFeatureConfig(FeatureConfig featConfig) { - this.featConfig = featConfig; - return this; - } - - public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) { - this.modelConfig = modelConfig; - return this; - } - - public Builder setKeywordsFile(String keywordsFile) { - this.keywordsFile = keywordsFile; - return this; - } - - public Builder setKeywordsScore(float keywordsScore) { - this.keywordsScore = keywordsScore; - return this; - } - - public Builder setKeywordsThreshold(float keywordsThreshold) { - this.keywordsThreshold = keywordsThreshold; - return this; - } - - public KeywordSpotterConfig build() { - return new KeywordSpotterConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java deleted file mode 100644 index 77e3a72e..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotterResult { - private final String keyword; - private final String[] tokens; - private final float[] timestamps; - - public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) { - this.keyword = keyword == null ? "" : keyword; - this.tokens = tokens == null ? new String[0] : tokens; - this.timestamps = timestamps == null ? new float[0] : timestamps; - } - - public String getKeyword() { - return keyword; - } - - public String[] getTokens() { - return tokens; - } - - public float[] getTimestamps() { - return timestamps; - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java deleted file mode 100644 index d3f0db66..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineModelConfig { - private final OnlineTransducerModelConfig transducer; - private final OnlineParaformerModelConfig paraformer; - private final OnlineZipformer2CtcModelConfig zipformer2Ctc; - private final OnlineNeMoCtcModelConfig neMoCtc; - private final OnlineToneCtcModelConfig toneCtc; - private final String tokens; - private final int numThreads; - private final boolean debug; - private final String provider; - private final String modelType; - private final String modelingUnit; - private final String bpeVocab; - - private OnlineModelConfig(Builder builder) { - this.transducer = builder.transducer; - this.paraformer = builder.paraformer; - this.zipformer2Ctc = builder.zipformer2Ctc; - this.neMoCtc = builder.neMoCtc; - this.toneCtc = builder.toneCtc; - this.tokens = builder.tokens; - this.numThreads = builder.numThreads; - this.debug = builder.debug; - this.provider = builder.provider; - this.modelType = builder.modelType; - this.modelingUnit = builder.modelingUnit; - this.bpeVocab = builder.bpeVocab; - } - - public static Builder builder() { - return new Builder(); - } - - public OnlineTransducerModelConfig getTransducer() { - return transducer; - } - - public OnlineParaformerModelConfig getParaformer() { - return paraformer; - } - - public OnlineZipformer2CtcModelConfig getZipformer2Ctc() { - return zipformer2Ctc; - } - - public OnlineNeMoCtcModelConfig getNeMoCtc() { - return neMoCtc; - } - - public OnlineToneCtcModelConfig getToneCtc() { - return toneCtc; - } - - public String getTokens() { - return tokens; - } - - public int getNumThreads() { - return numThreads; - } - - public boolean getDebug() { - return debug; - } - - public String getProvider() { - return provider; - } - - public String getModelType() { - return modelType; - } - - public String getModelingUnit() { - return modelingUnit; - } - - public String getBpeVocab() { - return bpeVocab; - } - - public static class Builder { - private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build(); - private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build(); - private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build(); - private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build(); - private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build(); - private String tokens = ""; - private int numThreads = 1; - private boolean debug = false; - private String provider = "cpu"; - private String modelType = ""; - private String modelingUnit = "cjkchar"; - private String bpeVocab = ""; - - public Builder setTransducer(OnlineTransducerModelConfig transducer) { - this.transducer = transducer; - return this; - } - - public Builder setTokens(String tokens) { - this.tokens = tokens; - return this; - } - - public Builder setModelType(String modelType) { - this.modelType = modelType; - return this; - } - - public Builder setNumThreads(int numThreads) { - this.numThreads = numThreads; - return this; - } - - public Builder setProvider(String provider) { - this.provider = provider; - return this; - } - - public Builder setModelingUnit(String modelingUnit) { - this.modelingUnit = modelingUnit; - return this; - } - - public Builder setBpeVocab(String bpeVocab) { - this.bpeVocab = bpeVocab; - return this; - } - - public OnlineModelConfig build() { - return new OnlineModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java deleted file mode 100644 index 1982804c..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineNeMoCtcModelConfig { - private final String model; - - private OnlineNeMoCtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineNeMoCtcModelConfig build() { - return new OnlineNeMoCtcModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java deleted file mode 100644 index 3e44e2c5..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineParaformerModelConfig { - private final String encoder; - private final String decoder; - - private OnlineParaformerModelConfig(Builder builder) { - this.encoder = builder.encoder; - this.decoder = builder.decoder; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEncoder() { - return encoder; - } - - public String getDecoder() { - return decoder; - } - - public static class Builder { - private String encoder = ""; - private String decoder = ""; - - public Builder setEncoder(String encoder) { - this.encoder = encoder; - return this; - } - - public Builder setDecoder(String decoder) { - this.decoder = decoder; - return this; - } - - public OnlineParaformerModelConfig build() { - return new OnlineParaformerModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java deleted file mode 100644 index 90a249cc..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineStream { - private long ptr; - - public OnlineStream(long ptr) { - this.ptr = ptr; - } - - public long getPtr() { - return ptr; - } - - public void acceptWaveform(float[] samples, int sampleRate) { - acceptWaveform(ptr, samples, sampleRate); - } - - public void release() { - if (ptr == 0) { - return; - } - delete(ptr); - ptr = 0; - } - - @Override - protected void finalize() throws Throwable { - release(); - super.finalize(); - } - - private native void acceptWaveform(long ptr, float[] samples, int sampleRate); - - private native void delete(long ptr); -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java deleted file mode 100644 index a847e79f..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineToneCtcModelConfig { - private final String model; - - private OnlineToneCtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineToneCtcModelConfig build() { - return new OnlineToneCtcModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java deleted file mode 100644 index ca003533..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineTransducerModelConfig { - private final String encoder; - private final String decoder; - private final String joiner; - - private OnlineTransducerModelConfig(Builder builder) { - this.encoder = builder.encoder; - this.decoder = builder.decoder; - this.joiner = builder.joiner; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEncoder() { - return encoder; - } - - public String getDecoder() { - return decoder; - } - - public String getJoiner() { - return joiner; - } - - public static class Builder { - private String encoder = ""; - private String decoder = ""; - private String joiner = ""; - - public Builder setEncoder(String encoder) { - this.encoder = encoder; - return this; - } - - public Builder setDecoder(String decoder) { - this.decoder = decoder; - return this; - } - - public Builder setJoiner(String joiner) { - this.joiner = joiner; - return this; - } - - public OnlineTransducerModelConfig build() { - return new OnlineTransducerModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java deleted file mode 100644 index d73f73dd..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineZipformer2CtcModelConfig { - private final String model; - - private OnlineZipformer2CtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineZipformer2CtcModelConfig build() { - return new OnlineZipformer2CtcModelConfig(this); - } - } -} diff --git a/README.md b/README.md index d102e885..92f73153 100644 --- a/README.md +++ b/README.md @@ -65,17 +65,22 @@ To install, run battery life similar to the stock Even app. * Connection management with auto-reconnect, and autodetection of conflict with the official Even Realities app + * A Wear OS watch app that replaces (and outdoes) the R1 ring: tap, swipe, + hold and crown gestures, side buttons, app launching and window switching, + voice or keyboard queries to the assistant with the reply on your wrist, + typing into apps, and glasses status/lock/display control (see "Watch + remote" below) * On-phone screen mirroring with touch control: tap what you see on the mirror (sidebar icons, launcher cells), or use the phone's own touchpad, d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a compact ring simulator. A display-mode picker (576×288 band, 576×480 tall, or the full 640×480 panel with an auto-hiding sidebar) and a brightness slider with an Auto toggle sit beside the mirror. - * A Wear OS watch app that replaces (and outdoes) the R1 ring: tap, swipe, - hold and crown gestures, side buttons, app launching and window switching, - voice or keyboard queries to the assistant with the reply on your wrist, - typing into apps, and glasses status/lock/display control (see "Watch - remote" below) + * Bluetooth pairing that scans for nearby glasses and identifies each pair + before connecting: model, frame shape, and colour decoded from the + advertised serial (with product photos), left and right arms matched to + each other by that serial, an estimated distance so the pair in your hand + sorts first, and the optional R1 ring * Dual-language NativeScript architecture with Java for the multithreaded Android API and bluetooth stack bits, Typescript for the bits you want to hack on diff --git a/app/app.css b/app/app.css index a3665e0d..fffdac2a 100644 --- a/app/app.css +++ b/app/app.css @@ -130,3 +130,129 @@ Button.-primary { padding: 0; font-size: 13; } +/* + * Pairing scan list (phone-ui/pairing-page). Rows are Repeater items: product + * art (or a colourway swatch) on the left, decoded SKU + proximity + serial in + * the middle, a chevron/check on the right. + */ +.pairing-instructions { + color: #555555; + font-size: 14; +} + +.pairing-status { + font-size: 14; + margin-left: 8; +} + +.pairing-section { + font-size: 16; + font-weight: bold; + margin-bottom: 4; +} + +.pairing-empty { + color: #777777; + font-size: 14; + padding: 8 0; +} + +.pairing-row { + background-color: #f4f4f4; + border-color: #dddddd; + border-width: 1; + border-radius: 10; + padding: 10; + margin-bottom: 8; +} + +.pairing-row-selected { + background-color: #e6f0ff; + border-color: #3b6fd6; +} + +.pairing-row-incomplete { + background-color: #fafafa; + border-color: #e6e6e6; +} + +.pairing-art { + vertical-align: center; +} + +.pairing-title { + font-size: 16; + font-weight: bold; +} + +.pairing-badge { + font-size: 11; + font-weight: bold; + color: #2a5bc4; + background-color: #dbe6fb; + border-radius: 9; + padding: 1 7; +} + +.pairing-variant { + font-size: 13; + font-weight: bold; + color: #333333; +} + +.pairing-proximity { + font-size: 12; + color: #666666; +} + +.pairing-proximity-immediate { + color: #2a5bc4; + font-weight: bold; +} + +.pairing-serial { + font-size: 12; + color: #666666; + font-family: monospace; +} + +.pairing-arms { + font-size: 12; + color: #444444; +} + +.pairing-warning { + font-size: 12; + color: #a33a1f; + margin-top: 4; +} + +.pairing-swatch-label { + font-size: 11; + color: #555555; +} + +.pairing-chevron { + font-size: 22; + color: #444444; + margin-left: 6; +} + +.pairing-selection { + background-color: #eef4ff; + border-radius: 10; + padding: 10; +} + +.pairing-diagnostics { + font-size: 11; + color: #888888; + font-family: monospace; +} + +Button.pairing-link { + background-color: transparent; + color: #2a5bc4; + font-size: 14; + border-width: 0; +} diff --git a/app/g2/ble-proximity.ts b/app/g2/ble-proximity.ts new file mode 100644 index 00000000..48e5d3dd --- /dev/null +++ b/app/g2/ble-proximity.ts @@ -0,0 +1,203 @@ +/** + * How far away an advertising device probably is, from its signal strength. + * + * Ported from SybilSight's `BLEProximity.swift` (itself from ProLink's + * `PeripheralInfo.basicEstimatedDistance`), with the calibration table + * narrowed to the two products this app pairs. + * + * ## The model + * + * The log-distance path loss model: + * + * RSSI = txPower - 10 · n · log₁₀(d) ⇒ d = 10 ^ ((txPower - RSSI) / (10 · n)) + * + * `txPower` is the expected signal at one metre and `n` the path loss + * exponent — 2.0 in free space, higher indoors where walls and bodies absorb. + * A device that advertises its own TX power gets used as measured; everything + * else falls back to a per-product calibration. + * + * ## What this is for, and what it is not + * + * It exists so a pairing list can be ordered by "which of these is in my + * hand", and so a row can say roughly how far away its device is. That is a + * ranking problem, and RSSI ordering is far more trustworthy than RSSI + * magnitude — the ordering survives calibration error that would put the + * absolute numbers metres out. + * + * It is **not** a measurement. A hand over the temple, a pocket, or a body + * between phone and glasses moves the estimate by metres. Every user-facing + * number is therefore rounded hard, prefixed with "~" when confidence is low, + * and paired with a coarse zone rather than being presented as a distance the + * wearer could act on precisely. + * + * This module is pure (no NativeScript imports) so it can run under node tests. + */ + +/** Expected RSSI at one metre, and the path loss exponent, per product. */ +export type ProximityCalibration = { + readonly txPowerAtOneMeter: number; + readonly pathLossExponent: number; +}; + +/** + * Even Realities G2. Two temple radios on a head, usually held or worn + * indoors, so the exponent sits above free space. Verified only as a ranking + * input — see the module note. + */ +export const GLASSES_CALIBRATION: ProximityCalibration = { txPowerAtOneMeter: -62, pathLossExponent: 2.2 }; + +/** + * Even Realities R1. A ring antenna is smaller and more easily shadowed by a + * hand, so it reads weaker at the same distance than the glasses do. + */ +export const RING_CALIBRATION: ProximityCalibration = { txPowerAtOneMeter: -68, pathLossExponent: 2.3 }; + +/** + * Estimates below this are indistinguishable from "touching the phone", and + * above it from "somewhere else entirely". Clamping keeps a noisy sample from + * rendering as 0.001 m. + */ +export const MINIMUM_METERS = 0.1; +export const MAXIMUM_METERS = 100; + +/** + * Coarse buckets. The wearer is being asked "is this the pair in your hand?", + * and a bucket answers that better than a number they might over-trust. + */ +export type ProximityZone = "immediate" | "near" | "far" | "distant"; + +const ZONE_ORDER: Record = { immediate: 0, near: 1, far: 2, distant: 3 }; + +export function zoneFromMeters(meters: number): ProximityZone { + if (meters < 0.5) return "immediate"; + if (meters < 3) return "near"; + if (meters < 10) return "far"; + return "distant"; +} + +export function zoneRank(zone: ProximityZone): number { + return ZONE_ORDER[zone]; +} + +export function zoneLabel(zone: ProximityZone): string { + switch (zone) { + case "immediate": + return "In your hand"; + case "near": + return "Nearby"; + case "far": + return "Across the room"; + case "distant": + return "Far away"; + } +} + +/** A glyph for the zone; SF Symbols have no Android equivalent, so plain text. */ +export function zoneGlyph(zone: ProximityZone): string { + switch (zone) { + case "immediate": + return "●●●●"; + case "near": + return "●●●○"; + case "far": + return "●●○○"; + case "distant": + return "●○○○"; + } +} + +/** A distance estimate and how much to trust it. */ +export type ProximityEstimate = { + readonly meters: number; + /** 0…1. Drops with weak signal and with having to guess the TX power. */ + readonly confidence: number; + readonly zone: ProximityZone; +}; + +/** "0.4 m", "~3 m". The tilde marks an estimate the reader should not lean on. */ +export function formatDistance(estimate: ProximityEstimate): string { + const prefix = estimate.confidence < 0.6 ? "~" : ""; + const meters = estimate.meters; + if (meters < 1) { + return `${prefix}${Math.round(meters * 100)} cm`; + } + if (meters < 10) { + return `${prefix}${meters.toFixed(1)} m`; + } + if (meters < MAXIMUM_METERS) { + return `${prefix}${Math.round(meters)} m`; + } + return `${prefix}>${MAXIMUM_METERS} m`; +} + +/** "In your hand · 0.3 m" — zone first, because the zone is the trustworthy half. */ +export function proximitySummary(estimate: ProximityEstimate): string { + return `${zoneLabel(estimate.zone)} · ${formatDistance(estimate)}`; +} + +/** + * Estimate distance from a signal sample. Null when there is no RSSI to work + * from, so callers can say "signal unknown" rather than render a fabricated + * distance. 127 is the Bluetooth "RSSI unavailable" sentinel. + */ +export function estimateProximity( + rssi: number | null | undefined, + txPower: number | null | undefined, + calibration: ProximityCalibration, +): ProximityEstimate | null { + if (rssi == null || !Number.isFinite(rssi) || rssi === 127) return null; + const hasTxPower = txPower != null && Number.isFinite(txPower); + const referencePower = hasTxPower ? txPower! : calibration.txPowerAtOneMeter; + const ratio = (referencePower - rssi) / (10 * calibration.pathLossExponent); + const meters = Math.min(Math.max(Math.pow(10, ratio), MINIMUM_METERS), MAXIMUM_METERS); + return { + meters, + confidence: proximityConfidence(rssi, hasTxPower), + zone: zoneFromMeters(meters), + }; +} + +/** + * Confidence falls with signal strength, because a weak sample is a noisy one, + * and with having to assume the TX power instead of being told it. + */ +export function proximityConfidence(rssi: number, hasAdvertisedTxPower: boolean): number { + let confidence = 1.0; + if (rssi < -85) confidence *= 0.4; + else if (rssi < -75) confidence *= 0.7; + else if (rssi < -65) confidence *= 0.85; + confidence *= hasAdvertisedTxPower ? 1.0 : 0.85; + return confidence; +} + +/** + * Order a list closest-first. + * + * Items whose signal never arrived sort last rather than being dropped: a pair + * that cannot be ranked is still a pair the wearer may need to choose, and + * hiding it would be worse than showing it at the bottom. Ties break on the + * stable id so the list does not reshuffle between scan passes — a list that + * reorders under the finger is how the wrong device gets tapped, which is the + * whole failure this ordering exists to prevent. + */ +export function sortedByProximity( + items: readonly T[], + estimate: (item: T) => ProximityEstimate | null, + id: (item: T) => string, +): T[] { + return items.slice().sort((lhs, rhs) => { + const left = estimate(lhs); + const right = estimate(rhs); + if (left && right) { + if (left.meters !== right.meters) return left.meters < right.meters ? -1 : 1; + return compareIds(id(lhs), id(rhs)); + } + if (left && !right) return -1; + if (!left && right) return 1; + return compareIds(id(lhs), id(rhs)); + }); +} + +function compareIds(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index f7e917e5..8b3d18e6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -230,7 +230,6 @@ class DashboardController { private offFrameMetrics: (() => void) | null = null; private offFirmwareInfo: (() => void) | null = null; private offVoiceStatus: (() => void) | null = null; - private offVoiceWakeWord: (() => void) | null = null; private offAndroidNotification: (() => void) | null = null; private lastInput = "waiting..."; private lastSys = "none yet"; @@ -1118,11 +1117,6 @@ class DashboardController { this.offVoiceStatus = voiceControlBridge.onStatus((state) => { this.appendLog(state.status); }); - this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => { - void this.handleWakeWord(keyword).catch((error) => { - this.appendLog(`wake-word handler failed: ${this.formatError(error)}`); - }); - }); await mediaControllerBridge.start(); await nightscoutBridge.start(); @@ -1189,8 +1183,6 @@ class DashboardController { this.offFirmwareInfo = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; - this.offVoiceWakeWord?.(); - this.offVoiceWakeWord = null; await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); @@ -1289,8 +1281,6 @@ class DashboardController { this.offFirmwareInfo = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; - this.offVoiceWakeWord?.(); - this.offVoiceWakeWord = null; const communicator = this.communicator; this.communicator = null; @@ -2004,20 +1994,6 @@ class DashboardController { this.schedulePreviewUpdate(); } - private async handleWakeWord(keyword: string): Promise { - const normalized = keyword.trim(); - if (normalized.length === 0) return; - this.appendLog(`wake-word detected: ${normalized}`); - const woke = shell.wake("sidebar"); - if (woke) { - const ready = await this.ensureEvenHubSessionActive(); - if (!ready) { - this.appendLog("EvenHub wake barrier timed out for phone wakeword"); - } - this.requestShellRender(); - } - } - private async handleAndroidNotificationPosted(notificationKey: string): Promise { if (!notificationKey) { this.requestShellRender(); diff --git a/app/g2/device-addresses.ts b/app/g2/device-addresses.ts index 6e58ef40..90277ddb 100644 --- a/app/g2/device-addresses.ts +++ b/app/g2/device-addresses.ts @@ -50,3 +50,81 @@ export function isValidMacAddress(value: string, allowEmpty = false): boolean { } return /^[0-9A-F]{2}(?::[0-9A-F]{2}){5}$/.test(normalizeMacAddress(trimmed)); } + +// --------------------------------------------------------------------------- +// Paired-glasses identity +// --------------------------------------------------------------------------- + +/** + * What the pairing scan learned about the pair whose addresses were saved, so + * later screens can name and picture the exact glasses without a connection. + * Kept separate from the addresses so hand-entered MACs keep working. + */ +export type PairedGlassesIdentity = { + /** The 14-character serial both arms advertised (pair identity). */ + serial: string; + leftName: string; + rightName: string; + leftAddress: string; + rightAddress: string; + /** Ring local name ("EVEN R1_B56EE2"), when a ring was chosen from the scan. */ + ringName: string; + ringAddress: string; + pairedAtMs: number; +}; + +const IDENTITY_KEYS = { + serial: "deviceIdentity.serial", + leftName: "deviceIdentity.leftName", + rightName: "deviceIdentity.rightName", + leftAddress: "deviceIdentity.leftAddress", + rightAddress: "deviceIdentity.rightAddress", + ringName: "deviceIdentity.ringName", + ringAddress: "deviceIdentity.ringAddress", + pairedAtMs: "deviceIdentity.pairedAtMs", +} as const; + +export function loadPairedGlassesIdentity(): PairedGlassesIdentity | null { + const serial = ApplicationSettings.getString(IDENTITY_KEYS.serial, "").trim(); + if (!serial) return null; + return { + serial, + leftName: ApplicationSettings.getString(IDENTITY_KEYS.leftName, ""), + rightName: ApplicationSettings.getString(IDENTITY_KEYS.rightName, ""), + leftAddress: normalizeMacAddress(ApplicationSettings.getString(IDENTITY_KEYS.leftAddress, "")), + rightAddress: normalizeMacAddress(ApplicationSettings.getString(IDENTITY_KEYS.rightAddress, "")), + ringName: ApplicationSettings.getString(IDENTITY_KEYS.ringName, ""), + ringAddress: normalizeMacAddress(ApplicationSettings.getString(IDENTITY_KEYS.ringAddress, "")), + pairedAtMs: ApplicationSettings.getNumber(IDENTITY_KEYS.pairedAtMs, 0), + }; +} + +export function savePairedGlassesIdentity(identity: PairedGlassesIdentity): void { + ApplicationSettings.setString(IDENTITY_KEYS.serial, identity.serial.trim().toUpperCase()); + ApplicationSettings.setString(IDENTITY_KEYS.leftName, identity.leftName); + ApplicationSettings.setString(IDENTITY_KEYS.rightName, identity.rightName); + ApplicationSettings.setString(IDENTITY_KEYS.leftAddress, normalizeMacAddress(identity.leftAddress)); + ApplicationSettings.setString(IDENTITY_KEYS.rightAddress, normalizeMacAddress(identity.rightAddress)); + ApplicationSettings.setString(IDENTITY_KEYS.ringName, identity.ringName); + ApplicationSettings.setString(IDENTITY_KEYS.ringAddress, normalizeMacAddress(identity.ringAddress)); + ApplicationSettings.setNumber(IDENTITY_KEYS.pairedAtMs, identity.pairedAtMs); +} + +export function clearPairedGlassesIdentity(): void { + for (const key of Object.values(IDENTITY_KEYS)) { + ApplicationSettings.remove(key); + } +} + +/** + * The saved identity only describes the saved addresses. If the user later + * hand-edits an arm's MAC to something else, the serial no longer applies — + * report null rather than a stale name for a different pair. + */ +export function loadPairedGlassesIdentityForAddresses(addresses: DeviceAddresses): PairedGlassesIdentity | null { + const identity = loadPairedGlassesIdentity(); + if (!identity) return null; + if (identity.leftAddress !== normalizeMacAddress(addresses.left)) return null; + if (identity.rightAddress !== normalizeMacAddress(addresses.right)) return null; + return identity; +} diff --git a/app/g2/even-advertisement.ts b/app/g2/even-advertisement.ts new file mode 100644 index 00000000..0e9b1360 --- /dev/null +++ b/app/g2/even-advertisement.ts @@ -0,0 +1,302 @@ +/** + * Parsers for what Even Realities hardware puts in a BLE advertisement. + * + * Ported from SybilSight's SDK (`extractSN` / `extractMac` / + * `extractG2AdvertisementFlag` in `G2.swift`, and `ringMacWireBytes` / + * `ringNameAddressOctets` in `R1.swift`). + * + * ## G2 temple (confirmed by HCI capture) + * + * Manufacturer-specific AD record: `45 52` + SN(14, ASCII) + MAC(6, little-endian) + flag(1). + * The `45 52` ("ER") bytes sit in the company-identifier field, which Android + * parses as little-endian uint16 `0x5245`. Both arms of a pair advertise the + * SAME serial; that is the pair identity. The local name is + * `Even G2___` where the trailing hex is the last three + * bytes of THAT ARM's address — per-arm, NOT a pair id. One arm's name + * predicts nothing about the other's. + * + * ## R1 ring + * + * `45 52` + MAC(6, wire order) and, on ring firmware 2.2.7.x, further bytes + * after the MAC (observed live: the ASCII device-serial suffix, e.g. "140137"). + * The local name is `EVEN R1_` where the hex is the last three octets + * of the human-order MAC. Anchor on the marker and, when the name carries the + * conventional suffix, require the candidate MAC to reproduce it — a naive + * `suffix(6)` slice once captured serial digits as the ring identity. + * + * This module is pure (no NativeScript imports) so it can run under node tests. + */ + +/** Even Realities' manufacturer-data company identifier (`"ER"` read little-endian). */ +export const EVEN_COMPANY_IDENTIFIER = 0x5245; + +export type EvenDeviceRole = "left" | "right" | "ring"; + +export function hexToBytes(hex: string | null | undefined): Uint8Array { + const clean = (hex ?? "").replace(/[^0-9a-fA-F]/g, ""); + const out = new Uint8Array(Math.floor(clean.length / 2)); + for (let i = 0; i < out.length; i++) { + out[i] = parseInt(clean.substr(i * 2, 2), 16); + } + return out; +} + +export function bytesToHex(bytes: ArrayLike): string { + let out = ""; + for (let i = 0; i < bytes.length; i++) { + out += (bytes[i]! & 0xff).toString(16).padStart(2, "0").toUpperCase(); + } + return out; +} + +/** True when the manufacturer data opens with Even's `"ER"` marker. */ +export function hasEvenManufacturerSignature(mfg: Uint8Array): boolean { + return mfg.length >= 2 && mfg[0] === 0x45 && mfg[1] === 0x52; +} + +/** + * Extract the serial number from G2 manufacturer data. + * Layout: "ER"(2) + SN(14) + MAC(6, LE) + flag(1). Null when too short or when + * the 14 bytes are not printable ASCII (control bytes are stripped, as the + * SDK does, but a result that then fails to look like a serial is rejected). + */ +export function extractG2Serial(mfg: Uint8Array): string | null { + if (mfg.length < 16) return null; + let serial = ""; + for (let i = 2; i < 16; i++) { + const byte = mfg[i]!; + if (byte <= 0x1f || byte === 0x7f) continue; + if (byte > 0x7e) return null; + serial += String.fromCharCode(byte); + } + return serial.length ? serial : null; +} + +/** + * Extract the BLE MAC from G2 manufacturer data. + * Layout: "ER"(2) + SN(14) + MAC(6, little-endian) + flag(1). + * Returns "AA:BB:CC:DD:EE:FF" (big-endian, colon-separated). + */ +export function extractG2Mac(mfg: Uint8Array): string | null { + if (mfg.length < 22) return null; + return humanReadableAddressFromWireBytes(mfg.subarray(16, 22)); +} + +/** Preserve the final G2 manufacturer-data byte without guessing its meaning. */ +export function extractG2AdvertisementFlag(mfg: Uint8Array): number | null { + if (mfg.length < 23 || !hasEvenManufacturerSignature(mfg)) return null; + return mfg[22]!; +} + +/** + * Which side a G2 temple advertisement belongs to, from the local name. + * Null when the name carries neither or both markers. + */ +export function g2SideFromName(name: string | null | undefined): "left" | "right" | null { + if (!name) return null; + const upper = name.toUpperCase(); + const isLeft = upper.includes("_L_"); + const isRight = upper.includes("_R_"); + if (isLeft === isRight) return null; + return isLeft ? "left" : "right"; +} + +/** "Even G2_32_L_ACD458" → "ACD458"; null when the name does not end in 6 hex digits. */ +export function g2NameAddressTail(name: string | null | undefined): string | null { + if (!name) return null; + const match = /_([0-9A-F]{6})\s*$/i.exec(name); + return match ? match[1]!.toUpperCase() : null; +} + +/** + * Even's wire order is reversed human order: "AA:BB:CC:DD:EE:FF" ⇄ [FF EE DD CC BB AA]. + */ +export function wireBytesFromHumanReadableAddress(address: string): Uint8Array | null { + const cleaned = address.replace(/[:\-]/g, "").trim(); + if (cleaned.length !== 12 || !/^[0-9a-fA-F]{12}$/.test(cleaned)) return null; + const human = hexToBytes(cleaned); + return new Uint8Array(Array.from(human).reverse()); +} + +export function humanReadableAddressFromWireBytes(bytes: Uint8Array): string | null { + if (bytes.length !== 6) return null; + return Array.from(bytes) + .reverse() + .map((b) => b.toString(16).padStart(2, "0").toUpperCase()) + .join(":"); +} + +/** + * The conventional R1 local name ("EVEN R1_B56EE2") ends in the human-order + * MAC's last three octets. Returns those octets when the name carries a + * parseable 6-hex-digit suffix. A bare ring id ("B56EE2") is accepted too. + */ +export function ringNameAddressOctets(name: string | null | undefined): number[] | null { + if (!name) return null; + const marker = name.indexOf("R1_"); + const suffix = (marker >= 0 ? name.slice(marker + 3) : name).trim(); + if (suffix.length !== 6 || !/^[0-9a-fA-F]{6}$/.test(suffix)) return null; + return [0, 2, 4].map((i) => parseInt(suffix.substr(i, 2), 16)); +} + +/** + * A wire-order candidate is reversed(human), so its bytes [2],[1],[0] must + * equal the name suffix octets in order for the candidate to belong to the + * named ring. + */ +export function wireBytesMatchNameOctets(wire: ArrayLike, nameOctets: number[]): boolean { + return wire.length === 6 && nameOctets.length === 3 && wire[2] === nameOctets[0] && wire[1] === nameOctets[1] && wire[0] === nameOctets[2]; +} + +/** + * Extract the ring's wire-order MAC from its manufacturer data, validated + * against the advertised name when the name is verifiable. Fails closed + * (null) when the name is verifiable and no 6-byte window reproduces it. + */ +export function ringMacWireBytes(mfg: Uint8Array, localName: string | null | undefined): Uint8Array | null { + if (mfg.length < 6) return null; + const markerAnchored = mfg.length >= 8 && hasEvenManufacturerSignature(mfg) ? mfg.subarray(2, 8) : null; + const nameOctets = ringNameAddressOctets(localName); + if (!nameOctets) { + // No verifiable name: prefer the documented marker layout, else legacy suffix. + return new Uint8Array(markerAnchored ?? mfg.subarray(mfg.length - 6)); + } + if (markerAnchored && wireBytesMatchNameOctets(markerAnchored, nameOctets)) { + return new Uint8Array(markerAnchored); + } + for (let start = 0; start + 6 <= mfg.length; start++) { + const window = mfg.subarray(start, start + 6); + if (wireBytesMatchNameOctets(window, nameOctets)) return new Uint8Array(window); + } + return null; +} + +/** + * Ring firmware 2.2.7.x appends the ASCII serial suffix after the MAC. Return + * it when present (digits/uppercase letters only) so the row can show it. + */ +export function extractRingSerialSuffix(mfg: Uint8Array): string | null { + if (!hasEvenManufacturerSignature(mfg) || mfg.length <= 8) return null; + let text = ""; + for (let i = 8; i < mfg.length; i++) { + const byte = mfg[i]!; + if (byte < 0x30 || byte > 0x5a) return null; + text += String.fromCharCode(byte); + } + return text.length ? text : null; +} + +// --------------------------------------------------------------------------- +// Classification of a raw advertisement +// --------------------------------------------------------------------------- + +/** What the Java scanner hands up: one advertisement (or one bonded device). */ +export type RawAdvertisement = { + address: string; + /** Live local name from the scan record, falling back to the cached device name. */ + name: string; + /** Full manufacturer-specific data including the 2-byte company id, as hex. Empty when absent. */ + manufacturerData: string; + rssi: number | null; + txPower: number | null; + connectable: boolean | null; + bonded: boolean; + source: "scan" | "paired"; + seenAtMs: number; +}; + +/** One advertisement after the protocol knowledge has been applied. */ +export type EvenAdvertisement = { + address: string; + name: string; + role: EvenDeviceRole; + /** G2: the 14-char pair serial. Ring: null (the ring's serial suffix is `ringSerialSuffix`). */ + serial: string | null; + /** The address embedded in the manufacturer data, when present. Should equal `address`. */ + embeddedMac: string | null; + /** True when the embedded MAC disagrees with the address the radio reported. */ + embeddedMacMismatch: boolean; + flag: number | null; + ringSerialSuffix: string | null; + rssi: number | null; + txPower: number | null; + connectable: boolean | null; + bonded: boolean; + source: "scan" | "paired"; + seenAtMs: number; + manufacturerData: string; + /** Why an otherwise Even-looking advertisement could not be fully parsed. */ + note: string | null; +}; + +/** + * Classify a raw advertisement. Returns null for anything that is not an Even + * Realities G2 temple or R1 ring. Admission mirrors the SDK: stock names + * contain "G2"/"R1", but renamed firmware may not, so the "ER" manufacturer + * signature is an equally strong admission path. + */ +export function classifyAdvertisement(raw: RawAdvertisement): EvenAdvertisement | null { + const mfg = hexToBytes(raw.manufacturerData); + const name = raw.name ?? ""; + const upper = name.toUpperCase(); + const hasSignature = hasEvenManufacturerSignature(mfg); + const looksLikeRing = /\bR1\b|EVEN R1|R1_/.test(upper); + const looksLikeG2 = upper.includes("G2"); + + if (!looksLikeRing && !looksLikeG2 && !hasSignature) return null; + + const base = { + address: normalizeAddress(raw.address), + name, + rssi: raw.rssi, + txPower: raw.txPower, + connectable: raw.connectable, + bonded: raw.bonded, + source: raw.source, + seenAtMs: raw.seenAtMs, + manufacturerData: raw.manufacturerData ?? "", + }; + + if (looksLikeRing) { + const wire = mfg.length >= 6 ? ringMacWireBytes(mfg, name) : null; + const embeddedMac = wire ? humanReadableAddressFromWireBytes(wire) : null; + return { + ...base, + role: "ring", + serial: null, + embeddedMac, + embeddedMacMismatch: !!embeddedMac && embeddedMac !== base.address, + flag: null, + ringSerialSuffix: extractRingSerialSuffix(mfg), + note: mfg.length >= 6 && !wire ? "advertisement carried no name-consistent MAC window" : null, + }; + } + + const side = g2SideFromName(name); + if (!side) { + // A G2-signature advertisement with no side marker cannot be placed in a pair. + return null; + } + const serial = mfg.length >= 16 ? extractG2Serial(mfg) : null; + const embeddedMac = extractG2Mac(mfg); + let note: string | null = null; + if (mfg.length === 0) note = "no manufacturer data (serial unknown)"; + else if (mfg.length < 16) note = `manufacturer data too short (${mfg.length} bytes); cannot extract a serial`; + else if (!serial) note = "could not extract a serial from the manufacturer data"; + return { + ...base, + role: side, + serial, + embeddedMac, + embeddedMacMismatch: !!embeddedMac && embeddedMac !== base.address, + flag: extractG2AdvertisementFlag(mfg), + ringSerialSuffix: null, + note, + }; +} + +export function normalizeAddress(value: string | null | undefined): string { + const compact = (value ?? "").replace(/[^0-9a-fA-F]/g, "").toUpperCase(); + if (compact.length !== 12) return (value ?? "").trim().toUpperCase(); + return compact.match(/.{2}/g)!.join(":"); +} diff --git a/app/g2/glasses-artwork.ts b/app/g2/glasses-artwork.ts new file mode 100644 index 00000000..7c9757e5 --- /dev/null +++ b/app/g2/glasses-artwork.ts @@ -0,0 +1,42 @@ +/** + * Product artwork for Even Realities hardware, keyed by the decoded serial. + * + * Ported from SybilSight's `SybilSightAssets.glassesAssetName(for:identity:)`. + * Prefer the photo of the exact SKU the serial decoded to, and fall back to + * the generic product shot. Every surface that draws "the user's glasses" + * should route through this — once the pair is known, showing brown Frame B + * for a grey Frame A pair is a small lie the wearer will notice immediately. + * + * The PNGs live under `app/images/glasses/` (copied from SybilSight's asset + * catalog; six SKU crops at 600×196 on a shared bounding box so they render at + * the same scale, plus the generic G2 and R1 product shots). NativeScript + * resolves `~/` to the app folder for ``. + * + * This module is pure (no NativeScript imports) so it can run under node tests. + */ + +import { BUNDLED_VARIANT_ASSET_NAMES, type GlassesHardwareIdentity } from "./glasses-hardware-identity"; + +export const GLASSES_IMAGE_DIR = "~/images/glasses"; + +/** Generic Even G2 product shot, used before the serial is known or for SKUs without art. */ +export const GENERIC_G2_IMAGE_PATH = `${GLASSES_IMAGE_DIR}/even_realities_g2.png`; + +/** Even R1 ring product shot. */ +export const R1_IMAGE_PATH = `${GLASSES_IMAGE_DIR}/even_realities_r1.png`; + +/** The SKU-specific photo, or null when none is bundled for this identity. */ +export function variantImagePath(identity: GlassesHardwareIdentity | null | undefined): string | null { + const name = identity?.imageAssetName; + return name ? `${GLASSES_IMAGE_DIR}/${name}.png` : null; +} + +/** The best available photo for these glasses: the SKU's own, else the generic shot. */ +export function glassesImagePath(identity: GlassesHardwareIdentity | null | undefined): string { + return variantImagePath(identity) ?? GENERIC_G2_IMAGE_PATH; +} + +/** Relative file names (under `app/images/glasses/`) that must exist for the resolver's claims to hold. */ +export function requiredArtworkFileNames(): string[] { + return [...Array.from(BUNDLED_VARIANT_ASSET_NAMES).map((name) => `${name}.png`), "even_realities_g2.png", "even_realities_r1.png"].sort(); +} diff --git a/app/g2/glasses-hardware-identity.ts b/app/g2/glasses-hardware-identity.ts new file mode 100644 index 00000000..291963cd --- /dev/null +++ b/app/g2/glasses-hardware-identity.ts @@ -0,0 +1,317 @@ +/** + * Decodes an Even Realities serial number into the product family, frame shape, + * and colourway the wearer can actually see. + * + * Ported from SybilSight's `GlassesHardwareIdentity.swift`. + * + * ## Why this exists + * + * A G2 arm advertises its 14-character serial inside the manufacturer data + * (`"ER"` + SN(14) + MAC(6) + flag(1), see `even-advertisement.ts`). Two people + * in the same room with glasses in pairing mode therefore produce two rows of + * near-identical alphanumerics, and the only way to tell which one you are + * holding is to guess. The serial already encodes the answer — the shape of + * the frame and its colour — so decoding it turns "S211GBBC180304" into + * "Even G2 B · Brown", which the wearer can check against the thing in their + * hand without connecting to a stranger's hardware first. + * + * ## Where the mapping comes from + * + * Recovered from the Even Realities app 2.2.0 + * (`even/common/utils/device_image_resolver` `_parseGlassesSku`, and + * `even/common/extension/ble/ble_mtach_device_ext` `evenSNName` / `matchModel`). + * That app does exactly this decode to pick its product artwork: + * + * | Serial slice | Meaning | + * | --- | --- | + * | `[0..<2]` | Product family — `S1` = G1 glasses, `S2` = G2 glasses, `B2` = R1 ring | + * | `[0..<3]` | Frame shape — `S20`/`S28` → A, `S21`/`S29` → B, `S22` → C (G1: `S11` → B, else A) | + * | `[0..<4]` | Model code (`S211`, `S110`, `B210`, `B290`) | + * | `[5]` | Colourway — `A` grey, `B` brown, `C` green | + * + * Two deliberate departures from the Even app: it silently defaults an + * unrecognised prefix to frame A and an unrecognised colour byte to grey. For + * identification that is the wrong trade — mislabelling a future SKU as + * "Frame A · Grey" is worse than saying nothing — so unknown values decode to + * `null` here and the UI falls back to the serial. + * + * This module is pure (no NativeScript imports) so it can run under node tests. + */ + +export type GlassesFamily = "g1" | "g2" | "r1"; + +/** Frame shape. Even ships these as bare letters — frame A is round, B is square. */ +export type GlassesFrame = "a" | "b" | "c"; + +/** Frame finish. The three colourways Even sells across G1 and G2. */ +export type GlassesColorway = "grey" | "brown" | "green"; + +const FAMILY_PRODUCT_NAME: Record = { + g1: "Even G1", + g2: "Even G2", + r1: "Even R1", +}; + +const COLORWAY_DISPLAY_NAME: Record = { + grey: "Grey", + brown: "Brown", + green: "Green", +}; + +/** + * sRGB hex for the swatch shown beside a scan row. Approximations of the + * shipping finishes, chosen to stay distinguishable at small sizes. + */ +const COLORWAY_SWATCH_HEX: Record = { + grey: "#8A8D91", + brown: "#6B4F3A", + green: "#3E5B45", +}; + +export function familyProductName(family: GlassesFamily): string { + return FAMILY_PRODUCT_NAME[family]; +} + +export function familyIsGlasses(family: GlassesFamily): boolean { + return family === "g1" || family === "g2"; +} + +export function frameLetter(frame: GlassesFrame): string { + return frame.toUpperCase(); +} + +export function frameDisplayName(frame: GlassesFrame): string { + return `Frame ${frameLetter(frame)}`; +} + +export function colorwayDisplayName(colorway: GlassesColorway): string { + return COLORWAY_DISPLAY_NAME[colorway]; +} + +export function colorwaySwatchHex(colorway: GlassesColorway): string { + return COLORWAY_SWATCH_HEX[colorway]; +} + +export class GlassesHardwareIdentity { + /** The normalised (uppercased, trimmed, suffix-stripped) serial this identity was decoded from. */ + readonly serial: string; + readonly family: GlassesFamily; + /** Null when the prefix is a family we recognise but a shape we do not, and always null for the ring. */ + readonly frame: GlassesFrame | null; + /** Null when the colour byte is not one of A/B/C, and always null for the ring. */ + readonly colorway: GlassesColorway | null; + + private constructor(serial: string, family: GlassesFamily, frame: GlassesFrame | null, colorway: GlassesColorway | null) { + this.serial = serial; + this.family = family; + this.frame = frame; + this.colorway = colorway; + } + + /** + * Decode a serial, or null when the string is not an Even serial at all. + * + * Accepts the advertised name verbatim: the scan list carries the bare + * serial, but saved pair identities and log lines carry arm suffixes + * (`S211GBBC180304_L_1`), so the leading serial run is taken and the rest + * ignored. + */ + static decode(rawValue: string | null | undefined): GlassesHardwareIdentity | null { + if (!rawValue) return null; + const normalized = normalizeSerial(rawValue); + // Both the family prefix (3) and the colour byte (index 5) must be present + // for the string to carry any variant information at all. + if (normalized.length < 6) return null; + + const prefix2 = normalized.slice(0, 2); + const prefix3 = normalized.slice(0, 3); + + let family: GlassesFamily; + switch (prefix2) { + case "S1": + family = "g1"; + break; + case "S2": + family = "g2"; + break; + case "B2": + family = "r1"; + break; + default: + return null; + } + + const isGlasses = familyIsGlasses(family); + return new GlassesHardwareIdentity( + normalized, + family, + isGlasses ? decodeFrame(prefix3, family) : null, + isGlasses ? decodeColorway(normalized) : null, + ); + } + + /** The four-character model code (`S211`), matching the Even app's `matchModel`. */ + get modelCode(): string | null { + return this.serial.length >= 4 ? this.serial.slice(0, 4) : null; + } + + /** + * Last four characters of the serial. Two people with the same frame and + * finish are the case this exists for: it is the shortest string that still + * separates them. + */ + get shortSerial(): string { + return this.serial.slice(-4); + } + + /** "Even G2 B" — product plus frame letter, the way Even names the SKU. */ + get productName(): string { + const base = familyProductName(this.family); + return this.frame ? `${base} ${frameLetter(this.frame)}` : base; + } + + /** "Frame B · Brown", or just the half that decoded. Null when neither did. */ + get variantSummary(): string | null { + const parts: string[] = []; + if (this.frame) parts.push(frameDisplayName(this.frame)); + if (this.colorway) parts.push(colorwayDisplayName(this.colorway)); + return parts.length ? parts.join(" · ") : null; + } + + /** "Even G2 B · Brown" — the one-line label for a device row or status line. */ + get displayName(): string { + return this.colorway ? `${this.productName} · ${colorwayDisplayName(this.colorway)}` : this.productName; + } + + /** + * Bundled artwork key for the product photo of this exact SKU, or null when + * the shape or finish did not decode — or when the SKU is real but no + * artwork is bundled for it. Membership is explicit so a missing image fails + * to a generic product shot rather than to a blank ``. + */ + get imageAssetName(): string | null { + if (!this.frame || !this.colorway) return null; + const candidate = `glasses_${this.family}_${this.frame}_${this.colorway}`; + return BUNDLED_VARIANT_ASSET_NAMES.has(candidate) ? candidate : null; + } + + equals(other: GlassesHardwareIdentity | null | undefined): boolean { + return !!other && other.serial === this.serial && other.family === this.family && other.frame === this.frame && other.colorway === this.colorway; + } +} + +/** + * Frame C artwork is not bundled yet; add the PNG under `app/images/glasses/` + * and the name here together (see `glasses-artwork.ts`). + */ +export const BUNDLED_VARIANT_ASSET_NAMES: ReadonlySet = new Set([ + "glasses_g2_a_grey", + "glasses_g2_a_brown", + "glasses_g2_a_green", + "glasses_g2_b_grey", + "glasses_g2_b_brown", + "glasses_g2_b_green", +]); + +/** + * Uppercase, drop surrounding whitespace, and cut at the first character that + * cannot appear in a serial — which is how the `_L_1` / `_R_1` arm suffix gets + * removed. + */ +export function normalizeSerial(rawValue: string): string { + const trimmed = rawValue.trim().toUpperCase(); + const match = /^[0-9A-Z]*/.exec(trimmed); + return match ? match[0] : ""; +} + +function decodeFrame(prefix3: string, family: GlassesFamily): GlassesFrame | null { + switch (family) { + case "g1": + // G1 ships two shapes; the Even app tests only for `S11` and treats every + // other `S1` prefix as A. + return prefix3 === "S11" ? "b" : "a"; + case "g2": + switch (prefix3) { + case "S20": + case "S28": + return "a"; + case "S21": + case "S29": + return "b"; + case "S22": + return "c"; + default: + return null; + } + case "r1": + return null; + } +} + +function decodeColorway(serial: string): GlassesColorway | null { + switch (serial.charAt(5)) { + case "A": + return "grey"; + case "B": + return "brown"; + case "C": + return "green"; + default: + return null; + } +} + +// --------------------------------------------------------------------------- +// Left/right pair check +// --------------------------------------------------------------------------- + +/** + * Do the two temples belong to the same pair? + * + * A G2 is two independent BLE peripherals, each advertising its own serial. + * They should read identically. They will not when two pairs share a case, + * when an arm from one pair is charged alongside another, or when a + * replacement temple is fitted — and the result is a link that starts, + * authenticates on one side, and then behaves inexplicably. This names that + * failure before a connection is attempted. + */ +export type GlassesPairSerialVerdict = + | { kind: "matched"; serial: string } + | { kind: "unknown" } + | { kind: "mismatched"; left: string; right: string }; + +export function evaluatePairSerials(left: string | null | undefined, right: string | null | undefined): GlassesPairSerialVerdict { + const l = nonEmpty(left); + const r = nonEmpty(right); + if (!l || !r) return { kind: "unknown" }; + // Compare on the decoded serial when both decode, so an arm suffix or casing + // difference in the reported string is not read as different hardware. + const leftKey = GlassesHardwareIdentity.decode(l)?.serial ?? l.toUpperCase(); + const rightKey = GlassesHardwareIdentity.decode(r)?.serial ?? r.toUpperCase(); + if (leftKey !== rightKey) return { kind: "mismatched", left: l, right: r }; + return { kind: "matched", serial: leftKey }; +} + +/** + * One sentence naming what is wrong and what to do, or null when nothing is + * wrong. Deliberately concrete about the serials: the wearer has to physically + * sort two piles of near-identical temples, and the printed serial is what + * they can sort them by. + */ +export function pairSerialWarning(verdict: GlassesPairSerialVerdict): string | null { + if (verdict.kind !== "mismatched") return null; + const leftVariant = GlassesHardwareIdentity.decode(verdict.left)?.displayName; + const rightVariant = GlassesHardwareIdentity.decode(verdict.right)?.displayName; + let message = `The left and right temples report different serial numbers (left ${verdict.left}, right ${verdict.right}).`; + if (leftVariant && rightVariant && leftVariant !== rightVariant) { + message += ` They are not even the same model: ${leftVariant} and ${rightVariant}.`; + } + message += " These are arms from two different pairs — check the serial printed inside each temple and pair a matched set."; + return message; +} + +function nonEmpty(value: string | null | undefined): string | null { + if (value == null) return null; + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} diff --git a/app/g2/pairing-candidates.ts b/app/g2/pairing-candidates.ts new file mode 100644 index 00000000..0c261f19 --- /dev/null +++ b/app/g2/pairing-candidates.ts @@ -0,0 +1,383 @@ +/** + * Turns a stream of Even advertisements into the things a pairing list shows: + * G2 pairs (a left and a right temple that advertise the same serial) and R1 + * rings, each ranked by how close it probably is. + * + * Ported from SybilSight's `PairingScanPresentation` / `PairingFlowState` + * (nearest-device margin, previously-paired match, closest-first ordering) + * and extended with the left↔right pair grouping that Faceclaw needs because + * it connects to each arm by address rather than through a vendor SDK. + * + * This module is pure (no NativeScript imports) so it can run under node tests. + */ + +import { + estimateProximity, + GLASSES_CALIBRATION, + proximitySummary, + RING_CALIBRATION, + sortedByProximity, + zoneGlyph, + type ProximityEstimate, + type ProximityZone, +} from "./ble-proximity"; +import { classifyAdvertisement, type EvenAdvertisement, type RawAdvertisement } from "./even-advertisement"; +import { glassesImagePath, R1_IMAGE_PATH } from "./glasses-artwork"; +import { + colorwayDisplayName, + colorwaySwatchHex, + evaluatePairSerials, + GlassesHardwareIdentity, + pairSerialWarning, + type GlassesColorway, +} from "./glasses-hardware-identity"; + +export type PairCompleteness = "complete" | "left-only" | "right-only"; + +/** A G2 pair as seen over the air: up to one advertisement per side, joined on serial. */ +export type GlassesPairCandidate = { + /** Stable id: the serial when known, otherwise the lone arm's address. */ + id: string; + serial: string | null; + identity: GlassesHardwareIdentity | null; + left: EvenAdvertisement | null; + right: EvenAdvertisement | null; + completeness: PairCompleteness; + /** Strongest smoothed signal across the arms that have one. */ + rssi: number | null; + txPower: number | null; + proximity: ProximityEstimate | null; + lastSeenMs: number; +}; + +export type RingCandidate = { + id: string; + advertisement: EvenAdvertisement; + rssi: number | null; + proximity: ProximityEstimate | null; + lastSeenMs: number; +}; + +/** + * Android delivers a callback per advertisement (several per second per arm), + * far more often than CoreBluetooth's coalesced reports, so the raw RSSI + * jitters visibly. A light exponential average keeps the list calm without + * hiding a real move. Ordering still uses the smoothed value; the margin for + * the "Closest" badge is applied on top. + */ +const RSSI_SMOOTHING = 0.35; + +/** + * The device whose signal is strong enough to call "the one you are holding" + * needs a clear margin over the runner-up: RSSI wanders by a few dB between + * advertisements, so a badge awarded on a 1 dB lead would hop between rows + * every scan pass and mean nothing. + */ +export const NEAREST_SIGNAL_MARGIN_DBM = 8; + +/** Advertisements older than this are dropped — the pair went back in its case or walked off. */ +export const DEFAULT_STALE_AFTER_MS = 12_000; + +export class DiscoveryAggregator { + private readonly byAddress = new Map(); + private readonly rejectedNames = new Set(); + + /** Feed one raw advertisement. Returns the classified record, or null when ignored. */ + ingest(raw: RawAdvertisement): EvenAdvertisement | null { + const classified = classifyAdvertisement(raw); + if (!classified) { + if (raw.name) this.rejectedNames.add(raw.name); + return null; + } + const previous = this.byAddress.get(classified.address); + let smoothedRssi: number | null = classified.rssi; + if (classified.rssi != null && previous?.smoothedRssi != null) { + smoothedRssi = previous.smoothedRssi + RSSI_SMOOTHING * (classified.rssi - previous.smoothedRssi); + } else if (classified.rssi == null && previous) { + // A bonded-device entry carries no signal; keep what the scan told us. + smoothedRssi = previous.smoothedRssi; + } + const merged: EvenAdvertisement & { smoothedRssi: number | null } = { + ...classified, + // A bonded listing never has a serial; do not let it erase a scanned one. + serial: classified.serial ?? previous?.serial ?? null, + embeddedMac: classified.embeddedMac ?? previous?.embeddedMac ?? null, + manufacturerData: classified.manufacturerData || previous?.manufacturerData || "", + bonded: classified.bonded || (previous?.bonded ?? false), + rssi: classified.rssi ?? previous?.rssi ?? null, + txPower: classified.txPower ?? previous?.txPower ?? null, + seenAtMs: Math.max(classified.seenAtMs, previous?.seenAtMs ?? 0), + smoothedRssi, + }; + this.byAddress.set(merged.address, merged); + return merged; + } + + /** Forget advertisements not refreshed within `maxAgeMs`. Bonded entries are kept. */ + prune(nowMs: number, maxAgeMs: number = DEFAULT_STALE_AFTER_MS): void { + for (const [address, record] of this.byAddress) { + if (record.bonded) continue; + if (nowMs - record.seenAtMs > maxAgeMs) this.byAddress.delete(address); + } + } + + clear(): void { + this.byAddress.clear(); + this.rejectedNames.clear(); + } + + get size(): number { + return this.byAddress.size; + } + + /** Names of advertisements that looked like nothing we pair with — for the diagnostics log. */ + get rejectedNameList(): string[] { + return Array.from(this.rejectedNames).sort(); + } + + advertisements(): EvenAdvertisement[] { + return Array.from(this.byAddress.values()); + } + + /** Every G2 pair seen, joined on serial, closest first. */ + pairs(): GlassesPairCandidate[] { + const groups = new Map(); + for (const record of this.byAddress.values()) { + if (record.role === "ring") continue; + const key = record.serial ? `serial:${record.serial}` : `arm:${record.address}`; + const group = groups.get(key) ?? { left: null, right: null }; + const slot = record.role === "left" ? "left" : "right"; + const incumbent = group[slot]; + // Two arms of one side claiming the same serial is a replaced temple or a + // stale row; keep the one heard most recently. + if (!incumbent || incumbent.seenAtMs <= record.seenAtMs) group[slot] = record; + groups.set(key, group); + } + const pairs: GlassesPairCandidate[] = []; + for (const group of groups.values()) { + const left = group.left; + const right = group.right; + const any = (left ?? right)!; + const serial = any.serial; + const samples = [left, right].filter((arm): arm is NonNullable => !!arm && arm.smoothedRssi != null); + const strongest = samples.length ? samples.reduce((a, b) => (a.smoothedRssi! >= b.smoothedRssi! ? a : b)) : null; + const rssi = strongest ? Math.round(strongest.smoothedRssi!) : null; + const txPower = strongest?.txPower ?? left?.txPower ?? right?.txPower ?? null; + pairs.push({ + id: serial ?? `arm:${any.address}`, + serial, + identity: GlassesHardwareIdentity.decode(serial), + left: left ? stripSmoothing(left) : null, + right: right ? stripSmoothing(right) : null, + completeness: left && right ? "complete" : left ? "left-only" : "right-only", + rssi, + txPower, + proximity: estimateProximity(rssi, txPower, GLASSES_CALIBRATION), + lastSeenMs: Math.max(left?.seenAtMs ?? 0, right?.seenAtMs ?? 0), + }); + } + return sortedByProximity(pairs, (pair) => pair.proximity, (pair) => pair.id); + } + + /** Every R1 ring seen, closest first. */ + rings(): RingCandidate[] { + const rings: RingCandidate[] = []; + for (const record of this.byAddress.values()) { + if (record.role !== "ring") continue; + const rssi = record.smoothedRssi != null ? Math.round(record.smoothedRssi) : null; + rings.push({ + id: record.address, + advertisement: stripSmoothing(record), + rssi, + proximity: estimateProximity(rssi, record.txPower, RING_CALIBRATION), + lastSeenMs: record.seenAtMs, + }); + } + return sortedByProximity(rings, (ring) => ring.proximity, (ring) => ring.id); + } +} + +function stripSmoothing(record: EvenAdvertisement & { smoothedRssi: number | null }): EvenAdvertisement { + const { smoothedRssi: _ignored, ...rest } = record; + return rest; +} + +/** + * The candidate whose signal is strong enough to call "the one you are + * holding". `null` when nothing leads clearly, which is the honest answer for + * two pairs sitting on the same table. + */ +export function nearestCandidateId(candidates: ReadonlyArray<{ id: string; rssi: number | null }>): string | null { + const ranked = candidates + .filter((candidate): candidate is { id: string; rssi: number } => candidate.rssi != null) + .slice() + // Stable order for equal signal so repeated polls nominate the same row. + .sort((lhs, rhs) => (lhs.rssi === rhs.rssi ? (lhs.id < rhs.id ? -1 : 1) : rhs.rssi - lhs.rssi)); + const best = ranked[0]; + if (!best) return null; + if (ranked.length === 1) return best.id; + return best.rssi - ranked[1]!.rssi >= NEAREST_SIGNAL_MARGIN_DBM ? best.id : null; +} + +/** + * The saved pair identity is stored as the serial, but historical values may + * carry the `_L_1` arm suffix, so compare on the decoded serial when both + * sides decode and fall back to a containment check when they do not. + */ +export function isPreviouslyPaired(name: string | null | undefined, pairedSerial: string | null | undefined): boolean { + if (!name || !pairedSerial) return false; + const saved = pairedSerial.trim(); + if (!saved) return false; + const left = GlassesHardwareIdentity.decode(name); + const right = GlassesHardwareIdentity.decode(saved); + if (left && right) return left.serial === right.serial; + const a = name.toLowerCase(); + const b = saved.toLowerCase(); + return a.includes(b) || b.includes(a); +} + +// --------------------------------------------------------------------------- +// Row presentation +// --------------------------------------------------------------------------- + +export type PairingRowPresentation = { + id: string; + kind: "glasses" | "ring"; + /** "Even G2 B" / "Even R1" / fallback product name. */ + modelTitle: string; + /** The serial (glasses) or local name (ring). */ + deviceName: string; + /** "Frame B · Brown", or null when the serial does not carry a variant we recognise. */ + variantSummary: string | null; + colorway: GlassesColorway | null; + /** Hex colour for the swatch. Null suppresses it rather than guessing a finish. */ + swatchHex: string | null; + /** Product photo path for ``. Variant art when bundled, else the generic shot. */ + imagePath: string; + /** True only when the variant-specific photo is available (the swatch is redundant then). */ + hasVariantImage: boolean; + isNearest: boolean; + isPreviouslyPaired: boolean; + /** "Yours" / "Closest" / "". */ + badge: string; + proximitySummary: string; + proximityGlyph: string; + zone: ProximityZone | null; + /** "Left + right arms found", "Only the left arm has been seen", … */ + armsSummary: string; + /** A problem the wearer should read before tapping, or "" when none. */ + warning: string; + /** Both arms present (or a ring), so tapping can save addresses. */ + canSelect: boolean; + leftAddress: string; + rightAddress: string; + ringAddress: string; +}; + +export type RowContext = { + /** Serial of the pair this phone paired with before, if any. */ + pairedSerial?: string | null; + /** From `nearestCandidateId` over the same list. */ + nearestId?: string | null; + /** Ring address saved before, if any. */ + pairedRingAddress?: string | null; +}; + +export function glassesRow(pair: GlassesPairCandidate, context: RowContext = {}): PairingRowPresentation { + const identity = pair.identity; + const imagePath = glassesImagePath(identity); + const isPreviouslyPairedRow = isPreviouslyPaired(pair.serial, context.pairedSerial); + const isNearest = !!context.nearestId && context.nearestId === pair.id; + const warnings: string[] = []; + + const leftNote = pair.left?.note; + const rightNote = pair.right?.note; + if (leftNote) warnings.push(`Left arm: ${leftNote}.`); + if (rightNote) warnings.push(`Right arm: ${rightNote}.`); + if (pair.left?.embeddedMacMismatch) { + warnings.push(`Left arm's advertised address ${pair.left.embeddedMac} differs from its radio address ${pair.left.address}.`); + } + if (pair.right?.embeddedMacMismatch) { + warnings.push(`Right arm's advertised address ${pair.right.embeddedMac} differs from its radio address ${pair.right.address}.`); + } + const verdict = evaluatePairSerials(pair.left?.serial, pair.right?.serial); + const mismatch = pairSerialWarning(verdict); + if (mismatch) warnings.push(mismatch); + + let armsSummary: string; + switch (pair.completeness) { + case "complete": + armsSummary = verdict.kind === "matched" ? "Left + right arms found · serials match" : "Left + right arms found"; + break; + case "left-only": + armsSummary = "Only the left arm has been seen — waiting for the right"; + break; + case "right-only": + armsSummary = "Only the right arm has been seen — waiting for the left"; + break; + } + if (!pair.serial) armsSummary += " · serial unknown, cannot match arms"; + + return { + id: pair.id, + kind: "glasses", + modelTitle: identity?.productName ?? "Even Realities G2", + deviceName: pair.serial ?? (pair.left ?? pair.right)!.name, + variantSummary: identity?.variantSummary ?? null, + colorway: identity?.colorway ?? null, + swatchHex: identity?.colorway ? colorwaySwatchHex(identity.colorway) : null, + imagePath, + hasVariantImage: !!identity?.imageAssetName, + isNearest, + isPreviouslyPaired: isPreviouslyPairedRow, + badge: isPreviouslyPairedRow ? "Yours" : isNearest ? "Closest" : "", + proximitySummary: pair.proximity ? proximitySummary(pair.proximity) : "Signal unknown", + proximityGlyph: pair.proximity ? zoneGlyph(pair.proximity.zone) : "○○○○", + zone: pair.proximity?.zone ?? null, + armsSummary, + warning: warnings.join(" "), + canSelect: pair.completeness === "complete" && verdict.kind !== "mismatched", + leftAddress: pair.left?.address ?? "", + rightAddress: pair.right?.address ?? "", + ringAddress: "", + }; +} + +export function ringRow(ring: RingCandidate, context: RowContext = {}): PairingRowPresentation { + const ad = ring.advertisement; + const isPaired = !!context.pairedRingAddress && context.pairedRingAddress.toUpperCase() === ad.address; + const isNearest = !!context.nearestId && context.nearestId === ring.id; + const warnings: string[] = []; + if (ad.note) warnings.push(`${ad.note}.`); + if (ad.embeddedMacMismatch) { + warnings.push(`The ring's advertised address ${ad.embeddedMac} differs from its radio address ${ad.address}.`); + } + return { + id: ring.id, + kind: "ring", + modelTitle: "Even R1", + deviceName: ad.ringSerialSuffix ? `${ad.name} · serial …${ad.ringSerialSuffix}` : ad.name, + variantSummary: null, + colorway: null, + swatchHex: null, + imagePath: R1_IMAGE_PATH, + hasVariantImage: true, + isNearest, + isPreviouslyPaired: isPaired, + badge: isPaired ? "Yours" : isNearest ? "Closest" : "", + proximitySummary: ring.proximity ? proximitySummary(ring.proximity) : "Signal unknown", + proximityGlyph: ring.proximity ? zoneGlyph(ring.proximity.zone) : "○○○○", + zone: ring.proximity?.zone ?? null, + armsSummary: ad.bonded ? "Paired with this phone" : "Advertising", + warning: warnings.join(" "), + canSelect: true, + leftAddress: "", + rightAddress: "", + ringAddress: ad.address, + }; +} + +/** Convenience: a colourway's display text, for rows that show the swatch alone. */ +export function colorwayLabel(colorway: GlassesColorway | null): string { + return colorway ? colorwayDisplayName(colorway) : ""; +} diff --git a/app/images/glasses/even_realities_g2.png b/app/images/glasses/even_realities_g2.png new file mode 100644 index 00000000..468bd968 Binary files /dev/null and b/app/images/glasses/even_realities_g2.png differ diff --git a/app/images/glasses/even_realities_r1.png b/app/images/glasses/even_realities_r1.png new file mode 100644 index 00000000..97db655e Binary files /dev/null and b/app/images/glasses/even_realities_r1.png differ diff --git a/app/images/glasses/glasses_g2_a_brown.png b/app/images/glasses/glasses_g2_a_brown.png new file mode 100644 index 00000000..0456ffb6 Binary files /dev/null and b/app/images/glasses/glasses_g2_a_brown.png differ diff --git a/app/images/glasses/glasses_g2_a_green.png b/app/images/glasses/glasses_g2_a_green.png new file mode 100644 index 00000000..63579732 Binary files /dev/null and b/app/images/glasses/glasses_g2_a_green.png differ diff --git a/app/images/glasses/glasses_g2_a_grey.png b/app/images/glasses/glasses_g2_a_grey.png new file mode 100644 index 00000000..8dd8445e Binary files /dev/null and b/app/images/glasses/glasses_g2_a_grey.png differ diff --git a/app/images/glasses/glasses_g2_b_brown.png b/app/images/glasses/glasses_g2_b_brown.png new file mode 100644 index 00000000..f380302b Binary files /dev/null and b/app/images/glasses/glasses_g2_b_brown.png differ diff --git a/app/images/glasses/glasses_g2_b_green.png b/app/images/glasses/glasses_g2_b_green.png new file mode 100644 index 00000000..89745eea Binary files /dev/null and b/app/images/glasses/glasses_g2_b_green.png differ diff --git a/app/images/glasses/glasses_g2_b_grey.png b/app/images/glasses/glasses_g2_b_grey.png new file mode 100644 index 00000000..cce0a06b Binary files /dev/null and b/app/images/glasses/glasses_g2_b_grey.png differ diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts new file mode 100644 index 00000000..3c71e31a --- /dev/null +++ b/app/native/asr-model.ts @@ -0,0 +1,176 @@ +import { Utils } from "@nativescript/core"; + +declare const com: any; +declare const java: any; + +/** + * Download management for the on-device transcription model (Moonshine, via + * sherpa-onnx). The model is no longer bundled in the APK; it is fetched on + * demand into the same filesDir location that earlier releases copied the + * bundled files to, so upgraded installs that already used on-device + * transcription need no download. FaceclawVoiceController reads the files + * from this directory. + * + * Mirrors the on-phone assistant model flow in llama.ts, except the model is + * three files rather than one; they download sequentially through + * FaceclawModelDownloader (resume + pinned sha256 per file). + */ + +type AsrModelFile = { + name: string; + sha256: string; + sizeBytes: number; +}; + +export const ASR_MODEL = { + label: "Moonshine (English)", + dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27", + // Hugging Face mirror of the sherpa-onnx release asset of the same name. + // The GitHub release only offers a tar.bz2, which the phone can't unpack; + // this mirror serves the same files (sha256-verified) individually. + baseUrl: + "https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/", + files: [ + { + name: "decoder_model_merged.ort", + sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d", + sizeBytes: 109424400, + }, + { + name: "encoder_model.ort", + sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf", + sizeBytes: 31326816, + }, + { + name: "tokens.txt", + sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049", + sizeBytes: 549350, + }, + ] as AsrModelFile[], + totalBytes: 141300566, +}; + +export type AsrModelState = { + status: "absent" | "downloading" | "ready"; + bytesDownloaded: number; + totalBytes: number; +}; + +let downloader: any = null; +// Bytes of files already fully downloaded in this run, plus progress within +// the file currently downloading; drives the aggregate percentage. +let completedBytes = 0; +let currentFileBytes = 0; +const stateListeners = new Set<(state: AsrModelState) => void>(); + +function modelDirPath(): string { + const context = Utils.android.getApplicationContext(); + return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`; +} + +function filePath(file: AsrModelFile): string { + return `${modelDirPath()}/${file.name}`; +} + +function isFilePresent(file: AsrModelFile): boolean { + try { + const javaFile = new java.io.File(filePath(file)); + return javaFile.exists() && javaFile.length() > 0; + } catch { + return false; + } +} + +export function isAsrModelReady(): boolean { + if (!global.isAndroid) return false; + return ASR_MODEL.files.every(isFilePresent); +} + +export function asrModelState(): AsrModelState { + if (downloader) { + return { + status: "downloading", + bytesDownloaded: completedBytes + currentFileBytes, + totalBytes: ASR_MODEL.totalBytes, + }; + } + return { + status: isAsrModelReady() ? "ready" : "absent", + bytesDownloaded: 0, + totalBytes: ASR_MODEL.totalBytes, + }; +} + +export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void { + stateListeners.add(listener); + return () => stateListeners.delete(listener); +} + +function notifyStateChanged(): void { + const state = asrModelState(); + stateListeners.forEach((listener) => listener(state)); +} + +export function startAsrModelDownload(): void { + if (!global.isAndroid || downloader || isAsrModelReady()) return; + completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0); + downloadNextFile(); + notifyStateChanged(); +} + +function downloadNextFile(): void { + const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file)); + if (!nextFile) { + downloader = null; + notifyStateChanged(); + return; + } + currentFileBytes = 0; + const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({ + onProgress: (bytes: number, _total: number) => { + currentFileBytes = Number(bytes); + notifyStateChanged(); + }, + onDone: () => { + completedBytes += nextFile.sizeBytes; + currentFileBytes = 0; + downloadNextFile(); + }, + onError: (message: string) => { + console.error(`Voice model download failed (${nextFile.name}): ${message}`); + downloader = null; + notifyStateChanged(); + }, + }); + downloader = new com.faceclaw.app.FaceclawModelDownloader( + `${ASR_MODEL.baseUrl}${nextFile.name}`, + filePath(nextFile), + nextFile.sha256, + nextFile.sizeBytes, + listener, + ); + downloader.start(); +} + +/** Stops the download; already-fetched bytes are kept and resumed next time. */ +export function cancelAsrModelDownload(): void { + if (!downloader) return; + downloader.cancel(); + downloader = null; + notifyStateChanged(); +} + +export function deleteAsrModel(): void { + if (!global.isAndroid) return; + cancelAsrModelDownload(); + try { + for (const file of ASR_MODEL.files) { + new java.io.File(filePath(file)).delete(); + new java.io.File(`${filePath(file)}.part`).delete(); + } + new java.io.File(modelDirPath()).delete(); + } catch (error) { + console.error(`Voice model delete failed: ${String(error)}`); + } + notifyStateChanged(); +} diff --git a/app/native/device-discovery.ts b/app/native/device-discovery.ts index 3dc6ff63..fc5063e6 100644 --- a/app/native/device-discovery.ts +++ b/app/native/device-discovery.ts @@ -1,12 +1,13 @@ import { Utils } from "@nativescript/core"; +import { classifyAdvertisement, type RawAdvertisement } from "../g2/even-advertisement"; + declare const com: any; -export type DiscoveredDevice = { - address: string; - name: string; - role: "left" | "right" | "ring"; - source: "paired" | "scan"; +export type DiscoveryEvents = { + onAdvertisement: (raw: RawAdvertisement) => void; + onScanFailed?: (code: number, message: string) => void; + onLog?: (line: string) => void; }; export type DiscoveredAddressSet = { @@ -16,8 +17,16 @@ export type DiscoveredAddressSet = { summary: string; }; +/** + * TS wrapper around the native FaceclawDeviceDiscovery. The live scan streams + * raw advertisements (see `RawAdvertisement`); protocol decoding happens in + * `app/g2/even-advertisement.ts` and pair grouping in + * `app/g2/pairing-candidates.ts`. + */ export class DeviceDiscoveryBridge { private readonly discovery: any; + private listenerProxy: any = null; + private events: DiscoveryEvents | null = null; constructor() { const context = Utils.android.getApplicationContext(); @@ -25,40 +34,133 @@ export class DeviceDiscoveryBridge { this.discovery = new com.faceclaw.app.FaceclawDeviceDiscovery(context); } - async getBondedCandidates(): Promise { - return parseCandidates(String(this.discovery.getBondedCandidatesJson())); + get bluetoothEnabled(): boolean { + return !!this.discovery.isBluetoothEnabled(); } - async scanCandidates(timeoutMs = 6000): Promise { - return parseCandidates(String(this.discovery.scanCandidatesJson(timeoutMs))); + get scanning(): boolean { + return !!this.discovery.isScanning(); + } + + /** Start (or re-target) the live scan. Returns false when the radio is off. */ + startScan(events: DiscoveryEvents): boolean { + this.events = events; + if (!this.listenerProxy) { + this.listenerProxy = new com.faceclaw.app.FaceclawDeviceDiscoveryListener({ + onAdvertisement: (json: string) => { + const raw = parseRaw(String(json)); + if (raw) this.events?.onAdvertisement(raw); + }, + onScanFailed: (code: number, message: string) => this.events?.onScanFailed?.(Number(code), String(message)), + onLog: (line: string) => this.events?.onLog?.(String(line)), + }); + this.discovery.setListener(this.listenerProxy); + } + return !!this.discovery.startScan(); + } + + stopScan(): void { + try { + this.discovery.stopScan(); + } catch { + // ignore + } + } + + /** Push bonded Even devices through the listener as `source: "paired"` entries. */ + emitBondedDevices(): void { + if (!this.listenerProxy) return; + this.discovery.emitBondedDevices(); + } + + async getBondedCandidates(): Promise { + return parseRawArray(String(this.discovery.getBondedCandidatesJson())); + } + + /** + * Collect advertisements for `timeoutMs` without blocking the UI thread + * (scan callbacks are delivered on the main looper, so a blocking sleep + * there would starve them). Latest sample per address wins. + */ + scanCandidates(timeoutMs = 6000): Promise { + return new Promise((resolve, reject) => { + const latest = new Map(); + const started = this.startScan({ + onAdvertisement: (raw) => latest.set(raw.address, raw), + onScanFailed: (code, message) => { + this.stopScan(); + reject(new Error(`Bluetooth scan failed: ${message} (${code})`)); + }, + }); + if (!started) { + reject(new Error("Bluetooth is off or BLE scanning is unavailable.")); + return; + } + setTimeout(() => { + this.stopScan(); + resolve(Array.from(latest.values())); + }, Math.max(500, timeoutMs)); + }); } } -export function buildAddressSet(candidates: DiscoveredDevice[]): DiscoveredAddressSet { - const latestForRole = new Map(); - for (const candidate of candidates) { - latestForRole.set(candidate.role, candidate); +/** + * Pick one address per role from a batch of raw advertisements — the manual + * address page's "load" helper. The latest-seen device of each role wins; + * the summary names the serial where one was advertised. + */ +export function buildAddressSet(candidates: RawAdvertisement[]): DiscoveredAddressSet { + const classified = candidates.map(classifyAdvertisement).filter((item): item is NonNullable => !!item); + const latestForRole = new Map<"left" | "right" | "ring", (typeof classified)[number]>(); + for (const candidate of classified) { + const incumbent = latestForRole.get(candidate.role); + if (!incumbent || incumbent.seenAtMs <= candidate.seenAtMs) latestForRole.set(candidate.role, candidate); } - const selected = { + const lines = classified.map((candidate) => { + const extras = [candidate.serial ? `serial ${candidate.serial}` : null, candidate.rssi != null ? `${candidate.rssi} dBm` : null].filter(Boolean); + return `${candidate.role}: ${candidate.name} ${candidate.address}${extras.length ? ` (${extras.join(", ")})` : ""}`; + }); + return { left: latestForRole.get("left")?.address ?? "", right: latestForRole.get("right")?.address ?? "", ring: latestForRole.get("ring")?.address ?? "", - }; - const lines = candidates.map((candidate) => `${candidate.role}: ${candidate.name} ${candidate.address}`); - return { - ...selected, summary: lines.length ? lines.join("\n") : "No matching devices found.", }; } -function parseCandidates(json: string): DiscoveredDevice[] { - const raw = JSON.parse(json) as Array>; - return raw - .map((item) => ({ - address: String(item.address ?? ""), - name: String(item.name ?? ""), - role: String(item.role ?? "") as DiscoveredDevice["role"], - source: String(item.source ?? "") as DiscoveredDevice["source"], - })) - .filter((item) => !!item.address && (item.role === "left" || item.role === "right" || item.role === "ring")); +function parseRawArray(json: string): RawAdvertisement[] { + const items = JSON.parse(json) as unknown[]; + return items.map((item) => parseRawObject(item)).filter((item): item is RawAdvertisement => !!item); +} + +function parseRaw(json: string): RawAdvertisement | null { + try { + return parseRawObject(JSON.parse(json)); + } catch { + return null; + } +} + +function parseRawObject(item: unknown): RawAdvertisement | null { + if (!item || typeof item !== "object") return null; + const record = item as Record; + const address = String(record.address ?? ""); + if (!address) return null; + return { + address, + name: String(record.name ?? ""), + manufacturerData: String(record.manufacturerData ?? ""), + rssi: numberOrNull(record.rssi), + txPower: numberOrNull(record.txPower), + connectable: typeof record.connectable === "boolean" ? record.connectable : null, + bonded: record.bonded === true, + source: record.source === "paired" ? "paired" : "scan", + seenAtMs: numberOrNull(record.seenAtMs) ?? Date.now(), + }; +} + +function numberOrNull(value: unknown): number | null { + if (value === null || value === undefined) return null; + const n = Number(value); + return Number.isFinite(n) ? n : null; } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index a5686680..121acf09 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -45,7 +45,6 @@ export type RawPcmListener = (pcm: Uint8Array) => void; export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); - private readonly wakeWordListeners = new Set<(keyword: string) => void>(); private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>(); private readonly speechEndListeners = new Set<() => void>(); private controller: any | null = null; @@ -72,11 +71,6 @@ export class FaceclawVoiceControlBridge { return () => this.statusListeners.delete(listener); } - onWakeWord(listener: (keyword: string) => void): () => void { - this.wakeWordListeners.add(listener); - return () => this.wakeWordListeners.delete(listener); - } - onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void { this.transcriptListeners.add(listener); return () => this.transcriptListeners.delete(listener); @@ -266,11 +260,6 @@ export class FaceclawVoiceControlBridge { if (this.rawActive) return; this.setStatus(String(status)); }, - onWakeWord: (keyword: string) => { - for (const listener of this.wakeWordListeners) { - listener(String(keyword)); - } - }, onTranscript: (text: string, isFinal: boolean) => { this.emitTranscript(String(text), Boolean(isFinal)); }, diff --git a/app/phone-ui/config-page.xml b/app/phone-ui/config-page.xml index 5048e69c..410fda3d 100644 --- a/app/phone-ui/config-page.xml +++ b/app/phone-ui/config-page.xml @@ -1,37 +1,47 @@ - +