diff --git a/.ackrc b/.ackrc index 8121069e..d1d22fb3 100644 --- a/.ackrc +++ b/.ackrc @@ -1,4 +1,6 @@ --ignore-file=match:.mjs +--ignore-file=match:frame-timings* --ignore-dir=tempPlugin --ignore-dir=build ---ignore-dir=platforms/android/build-tools +--ignore-dir=old +--ignore-dir=platforms diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..cffc8d6a --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,40 @@ +name: Deploy website + +# Publishes website/ to GitHub Pages. Requires the repository's +# Settings > Pages > Source to be set to "GitHub Actions". + +on: + push: + branches: [main] + paths: + - 'website/**' + - 'PRIVACY' + - 'scripts/sync-site.mjs' + - '.github/workflows/pages.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v4 + - name: Check generated blocks are up to date + run: node scripts/sync-site.mjs --check + - uses: actions/configure-pages@v5 + - uses: actions/upload-pages-artifact@v3 + with: + path: website + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index f780c2fc..5f98a77a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ hooks/ node_modules/ platforms/ +# Release APKs from scripts/release.sh +dist/ + # Logs logs *.log @@ -10,6 +13,9 @@ npm-debug.log* yarn-debug.log* yarn-error.log* +# npm test compile output (tests/tsconfig.json outDir) +.test-build/ + # General .DS_Store .AppleDouble @@ -22,6 +28,7 @@ typings/ .gradle-cursor App_Resources/Android/src/main/jniLibs/**/*.so App_Resources/Android/src/main/assets/faceclaw-voice-asr/ +old # Visual Studio Code .vscode/* @@ -32,3 +39,7 @@ App_Resources/Android/src/main/assets/faceclaw-voice-asr/ # pull_config.sh output: the phone's live settings, including API keys faceclaw_settings.xml + +# Developer configs +build_paths.sh + diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index f6e02ef6..a5584773 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -1,11 +1,16 @@ +Acknowledgements +================ + +Faceclaw is an open source project, at https://github.com/jimrandomh/faceclaw. +Contributors include: + James Babcock + Kalani Helekunihi + + Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficial/, evenRealities-openCFW https://github.com/kalanihelekunihi/evenRealities-openCFW/, and others who contributed to documenting the G2's bluetooth protocol and firmware. -Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/ -and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa -community. - G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3 The Terminus font is by Dimitar Zhekov's Terminus: https://framagit.org/ohnonot/terv-terc @@ -15,8 +20,8 @@ The TerminusV proportional font is by ohnonot, derived from Terminus: https://framagit.org/ohnonot/terv-terc. It is also distributed under the SIL Open Font License; see app/fonts/terminusv/LICENSE. -The CJK font used for EvenHub compatibility is Source Han Sans SC Light by -Adobe: https://github.com/adobe-fonts/source-han-sans. It is distributed under -the SIL Open Font License, Version 1.1; see app/fonts/source-han-sans/LICENSE.txt. -Faceclaw includes the G2's serialized 20 px LVGL build of the font so phone-side -rendering matches the glasses. +The CJK font is Source Han Sans SC Light by Adobe: +https://github.com/adobe-fonts/source-han-sans. It is distributed under the SIL +Open Font License, Version 1.1; see app/fonts/source-han-sans/LICENSE.txt. +Faceclaw includes the G2's serialized 20 px LVGL build of the font so +phone-side rendering matches the glasses. diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 7569cf33..0e24319a 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -12,14 +12,19 @@ dependencies { // code" entry. Scanner UI and camera live in Play Services, so Faceclaw // needs no camera permission and no third-party scanner app. implementation 'com.google.android.gms:play-services-code-scanner:16.1.0' + // Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for + // the Wear OS watch remote; see FaceclawWearBridge and wear/. + implementation 'com.google.android.gms:play-services-wearable:19.0.0' + // On-device language identification and translation for the Microphones + // app's captions (translation models download on demand, per language). + implementation 'com.google.mlkit:language-id:17.0.6' + implementation 'com.google.mlkit:translate:17.0.3' } def faceclawSherpaOnnxVersion = "1.13.0" def faceclawLibLc3Version = "1.1.3" -def faceclawAsrModelName = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27" def faceclawAbi = "arm64-v8a" def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi") -def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr") def faceclawNativeBuildDir = file("$buildDir/faceclaw-native") def faceclawDownload = { String url, File destination -> @@ -67,15 +72,11 @@ tasks.register("prepareFaceclawNativeLibs") { outputs.files( file("$faceclawJniLibsDir/libonnxruntime.so"), file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"), - file("$faceclawJniLibsDir/libfaceclaw_lc3.so"), - file("$faceclawAssetsDir/$faceclawAsrModelName/encoder_model.ort"), - file("$faceclawAssetsDir/$faceclawAsrModelName/decoder_model_merged.ort"), - file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt") + file("$faceclawJniLibsDir/libfaceclaw_lc3.so") ) doLast { faceclawJniLibsDir.mkdirs() - faceclawAssetsDir.mkdirs() def downloadsDir = file("$faceclawNativeBuildDir/downloads") def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2") @@ -92,20 +93,6 @@ tasks.register("prepareFaceclawNativeLibs") { into faceclawJniLibsDir } - def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2") - faceclawDownload( - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2", - asrArchive - ) - copy { - from tarTree(resources.bzip2(asrArchive)) - include "$faceclawAsrModelName/encoder_model.ort" - include "$faceclawAsrModelName/decoder_model_merged.ort" - include "$faceclawAsrModelName/tokens.txt" - includeEmptyDirs = false - into faceclawAssetsDir - } - def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz") faceclawDownload( "https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz", @@ -252,6 +239,22 @@ tasks.register("prepareFaceclawLlama") { } } +// versionName tracks FACECLAW_VERSION in app/version.ts, the single source of +// truth for the Faceclaw version. The gradle root project is platforms/android, +// so the repo root is two levels up from rootDir. +def faceclawVersionName = "0.0" +def faceclawVersionFile = file("$rootDir/../../app/version.ts") +if (faceclawVersionFile.exists()) { + def faceclawVersionMatch = (faceclawVersionFile.text =~ /FACECLAW_VERSION = "([^"]+)"/) + if (faceclawVersionMatch.find()) { + faceclawVersionName = faceclawVersionMatch.group(1) + } +} +def faceclawVersionCode = { + def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger() + return parts[0] * 10000 + parts[1] * 100 + parts[2] +}() + android { compileSdkVersion 35 buildToolsVersion "35" @@ -265,12 +268,17 @@ android { minSdkVersion 24 targetSdkVersion 35 ndk { + // The NativeScript platform build.gradle has already added all four + // ABIs by the time app.gradle is applied, and abiFilters is additive, + // so the set must be cleared for this filter to take effect. Only + // arm64 devices are supported (all our own JNI libs are arm64-only). + abiFilters.clear() abiFilters "arm64-v8a" } - // Version Information - versionCode 1 - versionName "1.0.0" + // Version Information (derived from app/version.ts above) + versionCode faceclawVersionCode + versionName faceclawVersionName generatedDensities = [] } diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml index 48dda00b..98e2715e 100644 --- a/App_Resources/Android/src/main/AndroidManifest.xml +++ b/App_Resources/Android/src/main/AndroidManifest.xml @@ -77,6 +77,7 @@ android:name="com.tns.NativeScriptActivity" android:label="@string/title_activity_kimera" android:configChanges="keyboard|keyboardHidden|orientation|screenSize|smallestScreenSize|screenLayout|locale|uiMode" + android:windowSoftInputMode="adjustResize" android:theme="@style/LaunchScreenTheme" android:hardwareAccelerated="true" android:launchMode="singleTask" @@ -110,6 +111,22 @@ + + + + + + + + + + + 0 - 1 - 2 -S 3 -T 4 -▁THE 5 -▁A 6 -▁TO 7 -▁AND 8 -N 9 -E 10 -D 11 -ING 12 -' 13 -▁OF 14 -▁I 15 -▁IN 16 -Y 17 -ED 18 -M 19 -▁ 20 -▁THAT 21 -O 22 -C 23 -R 24 -A 25 -P 26 -▁YOU 27 -U 28 -RE 29 -▁IT 30 -▁S 31 -G 32 -L 33 -B 34 -▁IS 35 -I 36 -AL 37 -ER 38 -▁RE 39 -▁C 40 -▁WE 41 -▁SO 42 -▁BE 43 -OR 44 -AR 45 -IN 46 -▁FOR 47 -F 48 -▁HE 49 -LE 50 -LY 51 -▁E 52 -▁WAS 53 -K 54 -▁F 55 -W 56 -VE 57 -RI 58 -CH 59 -LL 60 -▁THIS 61 -▁T 62 -EN 63 -▁ON 64 -IT 65 -CE 66 -▁WITH 67 -▁B 68 -▁W 69 -▁ME 70 -▁DO 71 -▁DE 72 -▁HAVE 73 -▁BUT 74 -ENT 75 -▁P 76 -UR 77 -ON 78 -▁AS 79 -IC 80 -TER 81 -TE 82 -▁THEY 83 -ES 84 -RA 85 -▁NOT 86 -TH 87 -▁ARE 88 -RO 89 -IL 90 -ATION 91 -AN 92 -▁ST 93 -▁WHAT 94 -SE 95 -LI 96 -V 97 -▁CAN 98 -▁G 99 -▁OR 100 -▁ALL 101 -NE 102 -▁AT 103 -▁CON 104 -H 105 -▁LIKE 106 -▁MO 107 -▁HIS 108 -▁AN 109 -▁MA 110 -▁THERE 111 -▁SU 112 -▁MY 113 -NG 114 -- 115 -▁ABOUT 116 -▁SE 117 -LA 118 -▁ONE 119 -LO 120 -▁SHE 121 -CK 122 -▁HAD 123 -AD 124 -MENT 125 -VER 126 -▁CA 127 -IR 128 -▁IF 129 -▁NO 130 -EL 131 -▁O 132 -▁FROM 133 -▁PA 134 -▁YOUR 135 -▁KNOW 136 -RY 137 -ION 138 -UN 139 -▁JUST 140 -▁UP 141 -▁US 142 -▁WHO 143 -▁GO 144 -ME 145 -▁EX 146 -▁OUT 147 -ND 148 -CI 149 -▁PO 150 -▁HER 151 -▁FA 152 -▁PRO 153 -▁WHEN 154 -ATE 155 -AT 156 -▁BY 157 -PE 158 -AS 159 -▁LA 160 -GE 161 -▁CO 162 -▁PEOPLE 163 -▁K 164 -▁SOME 165 -IGHT 166 -▁THINK 167 -VI 168 -DE 169 -▁LE 170 -▁WERE 171 -▁LI 172 -OM 173 -▁WOULD 174 -▁SEE 175 -CO 176 -▁BO 177 -▁OUR 178 -▁HOW 179 -ET 180 -Z 181 -UL 182 -▁HO 183 -IG 184 -▁GET 185 -▁DON 186 -IS 187 -URE 188 -▁WHICH 189 -▁MI 190 -IES 191 -US 192 -X 193 -▁MORE 194 -▁NOW 195 -▁TIME 196 -▁UN 197 -▁FI 198 -TY 199 -▁WILL 200 -RU 201 -IVE 202 -▁THEIR 203 -▁SAY 204 -ITY 205 -▁HA 206 -LU 207 -▁RO 208 -HO 209 -OL 210 -IST 211 -AGE 212 -MA 213 -KE 214 -▁RA 215 -ID 216 -ERS 217 -▁LO 218 -▁SP 219 -▁DA 220 -▁SAID 221 -TION 222 -THER 223 -▁EN 224 -▁BECAUSE 225 -▁HAS 226 -EST 227 -TI 228 -ACT 229 -▁BA 230 -▁VERY 231 -▁THEM 232 -▁BEEN 233 -▁WA 234 -ABLE 235 -▁SA 236 -▁DID 237 -▁HIM 238 -▁DIS 239 -▁WANT 240 -▁OTHER 241 -IM 242 -▁PRE 243 -▁LOOK 244 -▁GOING 245 -▁VI 246 -▁WORK 247 -▁DR 248 -ENCE 249 -▁DI 250 -FF 251 -TA 252 -▁REALLY 253 -▁RIGHT 254 -MO 255 -ALLY 256 -▁INTO 257 -OW 258 -▁WAY 259 -▁FE 260 -▁COULD 261 -▁THEN 262 -TED 263 -▁HERE 264 -▁THESE 265 -▁OVER 266 -▁WELL 267 -▁NA 268 -▁ANY 269 -HE 270 -▁COMP 271 -▁COM 272 -▁TH 273 -▁EVEN 274 -▁TWO 275 -▁JO 276 -▁MAKE 277 -▁BACK 278 -▁WHERE 279 -SH 280 -▁QU 281 -MAN 282 -▁YEAR 283 -▁TRA 284 -▁CHA 285 -QU 286 -CA 287 -▁ALSO 288 -CTION 289 -ANCE 290 -UM 291 -PER 292 -DI 293 -▁HU 294 -▁MU 295 -FUL 296 -▁NEW 297 -ARD 298 -J 299 -ICAL 300 -ANT 301 -END 302 -▁MAR 303 -▁NE 304 -IF 305 -DER 306 -ONE 307 -▁THOSE 308 -PP 309 -▁MAN 310 -▁GOOD 311 -▁TAKE 312 -SIDE 313 -PO 314 -HA 315 -▁LITTLE 316 -▁APP 317 -▁BI 318 -UGH 319 -▁COME 320 -ISH 321 -TIC 322 -▁PER 323 -▁THINGS 324 -▁MUCH 325 -▁FIRST 326 -▁THAN 327 -IA 328 -▁AGAIN 329 -▁STA 330 -▁PART 331 -▁GA 332 -▁MEAN 333 -▁LET 334 -▁DOWN 335 -▁NEED 336 -▁GU 337 -LES 338 -▁RU 339 -ATED 340 -▁SHA 341 -VO 342 -▁TALK 343 -▁DAY 344 -▁CHI 345 -▁SOMETHING 346 -▁AFTER 347 -▁THOUGH 348 -▁CALL 349 -▁GRA 350 -▁EVERY 351 -▁KIND 352 -ROW 353 -AIN 354 -▁BU 355 -▁THROUGH 356 -▁SHOULD 357 -▁ONLY 358 -▁TA 359 -▁MIN 360 -▁GOT 361 -IAN 362 -▁GREAT 363 -▁RI 364 -▁DOES 365 -▁THING 366 -DAY 367 -OUS 368 -▁START 369 -▁MOST 370 -▁COL 371 -PH 372 -▁LOT 373 -IZE 374 -BER 375 -IOUS 376 -▁LIFE 377 -▁MANY 378 -▁FEEL 379 -ITION 380 -▁VA 381 -NESS 382 -▁BEFORE 383 -▁THREE 384 -▁WORLD 385 -▁GIVE 386 -▁WHY 387 -▁PI 388 -▁LONG 389 -PORT 390 -▁SHOW 391 -▁ACTUALLY 392 -▁NEVER 393 -IV 394 -FORM 395 -▁THANK 396 -▁HI 397 -▁PUT 398 -▁LOVE 399 -▁QUESTION 400 -▁HELP 401 -▁HAPPEN 402 -▁MR 403 -▁OWN 404 -WARD 405 -▁MADE 406 -▁HAND 407 -▁STATE 408 -IBLE 409 -▁TELL 410 -▁PH 411 -▁CAME 412 -▁TEA 413 -▁STILL 414 -▁DIFFERENT 415 -▁PLACE 416 -▁MIGHT 417 -▁FIND 418 -▁DU 419 -▁IMP 420 -▁OLD 421 -▁PERSON 422 -▁COUNT 423 -▁JA 424 -▁POINT 425 -▁HIGH 426 -SHIP 427 -▁LAST 428 -▁SAME 429 -▁FOUR 430 -▁AROUND 431 -▁OH 432 -▁CHANGE 433 -▁UNDER 434 -▁HOME 435 -▁ANOTHER 436 -▁FOUND 437 -ATIVE 438 -▁TURN 439 -▁YES 440 -LAND 441 -▁ALWAYS 442 -▁YEAH 443 -▁SCHOOL 444 -▁JE 445 -▁NEXT 446 -▁WORD 447 -▁JU 448 -LINE 449 -▁SIX 450 -▁LEARN 451 -▁BOOK 452 -▁PLAY 453 -▁AWAY 454 -▁HOUSE 455 -▁PLAN 456 -▁WHILE 457 -▁COMMUNI 458 -▁FRIEND 459 -▁AMERICA 460 -▁INTEREST 461 -▁KEEP 462 -▁BETTER 463 -▁IDEA 464 -▁HARD 465 -▁TWENTY 466 -▁THOUSAND 467 -▁HUNDRED 468 -▁WEEK 469 -▁NUMBER 470 -▁COURSE 471 -▁OKAY 472 -PRESS 473 -▁SYSTEM 474 -▁NINE 475 -▁UNDERSTAND 476 -▁WITHOUT 477 -▁BELIEVE 478 -▁POWER 479 -▁MAYBE 480 -▁BUILD 481 -▁MONEY 482 -▁BETWEEN 483 -▁GONNA 484 -▁IMPORTANT 485 -▁ANYTHING 486 -▁LEFT 487 -▁HAVING 488 -▁WHITE 489 -▁PROBLEM 490 -▁SECOND 491 -▁SPEAK 492 -▁TOGETHER 493 -▁BLACK 494 -▁PUBLIC 495 -▁HEALTH 496 -▁NOTHING 497 -▁EVERYTHING 498 -Q 499 diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java new file mode 100644 index 00000000..0aab2ff6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.java @@ -0,0 +1,242 @@ +package com.faceclaw.app; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.media.MediaMuxer; +import android.util.Log; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A + * container, cutting speech recordings to roughly a tenth of their WAV size. + * Runs on its own thread; the source WAV is deleted on success when requested. + */ +public class FaceclawAudioTranscoder { + private static final String TAG = "FaceclawTranscode"; + private static final long CODEC_TIMEOUT_US = 10_000; + + public interface Listener { + void onDone(String outputPath, long outputBytes); + + void onError(String message); + } + + private final String wavPath; + private final String outputPath; + private final int bitrateBps; + private final boolean deleteSourceOnSuccess; + private final Listener listener; + + public FaceclawAudioTranscoder( + String wavPath, + String outputPath, + int bitrateBps, + boolean deleteSourceOnSuccess, + Listener listener) { + this.wavPath = wavPath; + this.outputPath = outputPath; + this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000; + this.deleteSourceOnSuccess = deleteSourceOnSuccess; + this.listener = listener; + } + + public void start() { + Thread thread = new Thread(this::run, "FaceclawTranscode"); + thread.setPriority(Thread.MIN_PRIORITY); + thread.start(); + } + + private void run() { + try { + transcode(); + long size = new File(outputPath).length(); + if (deleteSourceOnSuccess) { + new File(wavPath).delete(); + } + if (listener != null) { + listener.onDone(outputPath, size); + } + } catch (Throwable t) { + Log.w(TAG, "transcode failed for " + wavPath, t); + new File(outputPath).delete(); + if (listener != null) { + listener.onError(t.getMessage() == null ? t.toString() : t.getMessage()); + } + } + } + + private void transcode() throws IOException { + WavInfo wav = readWavHeader(wavPath); + MediaFormat format = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels); + format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps); + format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024); + + MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC); + MediaMuxer muxer = null; + FileInputStream input = null; + try { + codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + codec.start(); + muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); + input = new FileInputStream(wavPath); + long skipped = 0; + while (skipped < wav.dataOffset) { + long step = input.skip(wav.dataOffset - skipped); + if (step <= 0) { + throw new IOException("could not seek past WAV header"); + } + skipped += step; + } + + MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + byte[] chunk = new byte[8192]; + int track = -1; + boolean muxerStarted = false; + boolean inputDone = false; + boolean outputDone = false; + long remaining = wav.dataBytes; + long presentationUs = 0; + long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2; + + while (!outputDone) { + if (!inputDone) { + int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US); + if (inIndex >= 0) { + ByteBuffer inBuf = codec.getInputBuffer(inIndex); + int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity())); + int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1; + if (read <= 0) { + codec.queueInputBuffer(inIndex, 0, 0, presentationUs, + MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + } else { + inBuf.clear(); + inBuf.put(chunk, 0, read); + codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0); + remaining -= read; + presentationUs += read * 1_000_000L / bytesPerSecond; + } + } + } + int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US); + if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + track = muxer.addTrack(codec.getOutputFormat()); + muxer.start(); + muxerStarted = true; + } else if (outIndex >= 0) { + if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) { + if (!muxerStarted) { + throw new IOException("encoder produced data before format"); + } + ByteBuffer outBuf = codec.getOutputBuffer(outIndex); + outBuf.position(info.offset); + outBuf.limit(info.offset + info.size); + muxer.writeSampleData(track, outBuf, info); + } + codec.releaseOutputBuffer(outIndex, false); + if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + } + } + } finally { + if (input != null) { + try { + input.close(); + } catch (IOException ignored) { + } + } + try { + codec.stop(); + } catch (Throwable ignored) { + } + codec.release(); + if (muxer != null) { + try { + muxer.stop(); + } catch (Throwable ignored) { + } + muxer.release(); + } + } + } + + private static final class WavInfo { + int sampleRate; + int channels; + long dataOffset; + long dataBytes; + } + + /** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */ + private static WavInfo readWavHeader(String path) throws IOException { + try (FileInputStream in = new FileInputStream(path)) { + byte[] head = new byte[12]; + if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') { + throw new IOException("not a WAV file"); + } + WavInfo info = new WavInfo(); + long offset = 12; + byte[] chunkHead = new byte[8]; + while (in.read(chunkHead) == 8) { + ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN); + int id = b.getInt(); + long size = b.getInt() & 0xffffffffL; + offset += 8; + if (id == 0x20746d66) { // "fmt " + byte[] fmt = new byte[(int) Math.min(size, 16)]; + if (in.read(fmt) != fmt.length) { + throw new IOException("truncated fmt chunk"); + } + ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN); + int audioFormat = f.getShort(); + info.channels = f.getShort(); + info.sampleRate = f.getInt(); + f.getInt(); + f.getShort(); + int bits = f.getShort(); + if (audioFormat != 1 || bits != 16) { + throw new IOException("only 16-bit PCM WAV is supported"); + } + long skip = size - fmt.length; + while (skip > 0) { + long step = in.skip(skip); + if (step <= 0) { + break; + } + skip -= step; + } + offset += size; + } else if (id == 0x61746164) { // "data" + info.dataOffset = offset; + // A zero size means the header was never patched (crash + // mid-recording); recover the length from the file size. + info.dataBytes = size > 0 ? size : new File(path).length() - offset; + break; + } else { + long skip = size; + while (skip > 0) { + long step = in.skip(skip); + if (step <= 0) { + break; + } + skip -= step; + } + offset += size; + } + } + if (info.sampleRate == 0 || info.dataOffset == 0) { + throw new IOException("missing fmt or data chunk"); + } + return info; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index f1e2002c..9d9daf4b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -60,9 +60,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { new java.util.concurrent.CopyOnWriteArrayList<>(); private final java.util.List compassListeners = new java.util.concurrent.CopyOnWriteArrayList<>(); + private final java.util.List micStatusListeners = + new java.util.concurrent.CopyOnWriteArrayList<>(); + private volatile String lastFirmwareCapabilities = ""; private volatile Thread workerThread; private volatile boolean running; private volatile boolean userDisconnectRequested; + // Set when a connect attempt failed while an arm's Android bond is gone: + // retrying is pointless until the user re-pairs, so the worker loop parks + // instead of redialing. Cleared by start() (a fresh explicit connect). + private volatile boolean reconnectHalted; private String phase = "disconnected"; private String status = "Disconnected."; @@ -239,6 +246,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } running = true; userDisconnectRequested = false; + reconnectHalted = false; shutdownRequested = false; activeInstance = this; workerThread = new Thread(this, "FaceclawBleCommunicator"); @@ -355,6 +363,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + /** + * Whether the glasses mic is enabled right now. The enable lives in the + * current EvenHub session, so it dies with a transport drop, the charging + * case, or a suspend — silently, from the phone's point of view. Callers + * that track a capture across those events must check this rather than + * assume their earlier enable still holds. + */ + public boolean isAudioCaptureActive() { + synchronized (lock) { + return running && sessionReady && !shutdownRequested && audioCaptureActive; + } + } + /** * Acquire/renew or release CFW's volatile wake-takeover lease on both * arms. Delivery (not a protocol ACK) is awaited so a caller can ensure @@ -530,6 +551,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + /** The CFW capability token string from the last firmware-info read ("" before one arrives). */ + public String getFirmwareCapabilities() { + return lastFirmwareCapabilities; + } + + public void addMicStatusListener(FaceclawMicStatusListener listener) { + if (listener != null) { + micStatusListeners.add(listener); + } + } + + public void removeMicStatusListener(FaceclawMicStatusListener listener) { + if (listener != null) { + micStatusListeners.remove(listener); + } + } + + private void emitMicStatus(byte[] body, String address) { + if (micStatusListeners.isEmpty()) { + return; + } + String arm = address.equalsIgnoreCase(leftAddress) ? "L" + : address.equalsIgnoreCase(rightAddress) ? "R" : "?"; + byte[] copy = java.util.Arrays.copyOf(body, body.length); + mainHandler.post(() -> { + for (FaceclawMicStatusListener micListener : micStatusListeners) { + try { + micListener.onMicStatus(copy, arm); + } catch (Throwable t) { + Log.w(TAG, "mic status listener failed", t); + } + } + }); + } + + /** + * Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings + * field-103 write to both temples, or to a single one. Fire-and-forget: + * the firmware answers with a field-104 status notify per temple, which + * arrives through addMicStatusListener. + */ + public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) { + if (record == null || record.length < 4) { + return; + } + synchronized (lock) { + if (!running || !sessionReady) { + logLine("skip mic control (" + label + "); session not ready"); + return; + } + if (rightTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false)); + } + if (leftTemple) { + pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true)); + } + logLine("queue mic control " + label); + } + interruptibleSleep.interrupt(); + } + + /** + * Forward render-characteristic audio packets to the listener WITHOUT + * sending the stock EvenHub audio-control enable. Used for the CFW + * mic_control streaming path, where capture is armed through settings + * field 103 and the temples emit 'SM' frames on the same characteristic + * that stock mono LC3 uses. Returns false when no session is up. + */ + public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) { + if (listener == null) { + throw new IllegalArgumentException("listener is required"); + } + synchronized (lock) { + if (!running || !sessionReady || shutdownRequested) { + logLine("skip G2 audio forwarding; session not ready"); + return false; + } + audioPacketListener = listener; + audioCaptureActive = true; + logLine("G2 audio forwarding enabled"); + } + return true; + } + + public void stopG2AudioForwarding() { + synchronized (lock) { + audioPacketListener = null; + audioCaptureActive = false; + logLine("G2 audio forwarding disabled"); + } + } + public void addCompassListener(FaceclawCompassListener listener) { if (listener != null) { compassListeners.add(listener); @@ -564,12 +677,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { * foreground), including worker-app frames the TS side never sees. */ public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) { + return getCompositePreviewBitmap(brightenGamma, false); + } + + /** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */ + public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) { SurfaceCompositor.Composite composite = compositor.previewComposite(); if (composite == null) { return null; } return PreviewBitmapUtil.fromGray( - java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma); + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green); } /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @@ -1118,6 +1236,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } emitPhoneLockStateIfChanged(false); if (!sessionReady) { + if (reconnectHalted) { + interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS); + continue; + } long now = SystemClock.elapsedRealtime(); if (now < reconnectAfterMs) { interruptibleSleep.sleep(Math.min(ConnectionOptions.IDLE_SLEEP_MS, reconnectAfterMs - now)); @@ -1216,6 +1338,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { 0 ); } + if (!faceclawWakeNotification + && frame.ok + && frame.sid == BleProtocol.SID_UI_SETTING) { + // CFW mic status (field 104) rides both standalone pushes and + // settings read acks, from each temple on its own link. + byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb); + if (micStatus != null) { + emitMicStatus(micStatus, address); + } + } if (!faceclawWakeNotification && decodedWearState < 0 && frame.ok @@ -1365,13 +1497,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { audioPacketListener = null; clearAllMessagesLocked("connection lost"); displayedFingerprint = ""; - reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS; + if (!reconnectHalted) { + reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS; + } } } interruptibleSleep.interrupt(); if (connected) { setStateDisplay("connected", "Connected."); - } else { + } else if (!reconnectHalted) { + // While parked on a missing bond, keep the "unpaired" display: this + // callback is just the teardown of the arm that did connect. setStateDisplay("connecting", "Connecting to the glasses..."); } } @@ -1384,6 +1520,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { if (!sleepDuringConnectSettling(800)) { return; } + authenticateArms(); sendPrelude(); synchronized (lock) { @@ -1440,7 +1577,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { tryConnectRing("initial"); } catch (Throwable t) { logLine("connect failed: " + safeMessage(t)); - handleTransportFailure("connect failed"); + String unpairedArm = firstUnpairedArm(); + if (unpairedArm != null) { + handleUnpairedFailure(unpairedArm); + } else { + handleTransportFailure("connect failed"); + } } } @@ -1559,6 +1701,55 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } } + /** + * Complete the sid-0x80 security-auth exchange on both freshly opened arm + * connections. Firmware 2.2.9 answers no queries until it completes over an + * encrypted link and closes unauthenticated links after ~30 s (see + * ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is + * also what triggers SMP pairing. Deliberately soft: on timeout we log and + * continue rather than fail the connect — the custom firmware's response + * behavior is not yet hardware-verified, and on stock firmware an + * unanswered auth just means the prelude fails exactly as it did before. + * A pairing prompt accepted after our window still bonds at the OS level, + * so the next reconnect attempt authenticates promptly. + */ + private void authenticateArms() throws InterruptedException { + OutboundMessage right = messageBuilder.securityAuth(false); + OutboundMessage left = messageBuilder.securityAuth(true); + long now = SystemClock.elapsedRealtime(); + for (OutboundMessage message : new OutboundMessage[] {right, left}) { + message.onAck = () -> { + }; + message.onTimeout = () -> { + }; + message.sentAtMs = now; + writeMessage(message); + } + long deadline = SystemClock.elapsedRealtime() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS; + while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) { + synchronized (lock) { + if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) { + break; + } + } + long remaining = deadline - SystemClock.elapsedRealtime(); + if (remaining <= 0) { + break; + } + interruptibleSleep.sleep(Math.min(remaining, 100)); + } + synchronized (lock) { + if (!inFlightMessages.isEmpty()) { + clearInFlightMessagesLocked("security auth timeout"); + logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)"); + return; + } + } + boolean rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic); + boolean leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic); + logLine("security auth R=" + (rightOk ? "ok" : "unconfirmed") + " L=" + (leftOk ? "ok" : "unconfirmed")); + } + private void sendPrelude() throws InterruptedException { sendPrelude(false); } @@ -2429,6 +2620,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { } BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload); if (firmwareInfo != null) { + lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities; cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11"); textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12") && hasCapability(firmwareInfo.capabilities, "texstr14"); @@ -2928,6 +3120,49 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { + " release=" + previous.reason); } + /** The address of a configured arm Android no longer holds a bond for, or null. */ + private String firstUnpairedArm() { + if (!bleManager.isBonded(rightAddress)) return rightAddress; + if (!bleManager.isBonded(leftAddress)) return leftAddress; + return null; + } + + /** + * A connect failure while an arm's bond is missing (the pairing was + * forgotten in Android settings, or the address was typed in by hand and + * never paired) will repeat forever, so instead of scheduling a retry, + * park the worker loop and tell the user to re-pair. Only an explicit + * connect (which builds a fresh communicator) starts a new attempt. + */ + private void handleUnpairedFailure(String address) { + Log.e(TAG, "Connect failed and " + address + " is not paired; suspending reconnect"); + synchronized (lock) { + reconnectHalted = true; + sessionReady = false; + fixedLayoutCreated = false; + startupProbePending = false; + shutdownRequested = false; + chargingMode = false; + imageRetryAfterMs = 0; + displayedFingerprint = ""; + faceclawWakePendingNonce = -1; + lastFaceclawWakeLeaseQueuedAtMs = 0; + faceclawWakeControlSentCount = 0; + clearAllMessagesLocked("arm not paired: " + address); + reconnectAfterMs = Long.MAX_VALUE; + bleManager.disconnect(rightAddress); + bleManager.disconnect(leftAddress); + } + if (!userDisconnectRequested) { + setStateDisplay( + "unpaired", + "The glasses (" + address + ") are not paired with this phone." + + " Use \"Pair glasses\" to pair them again, then connect." + ); + } + interruptibleSleep.interrupt(); + } + private void handleTransportFailure(String reason) { Log.e(TAG, "Transport failure: "+reason); synchronized (lock) { @@ -2963,6 +3198,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { ringConnected = false; ringNotificationsReady = false; reconnectAfterMs = 0; + reconnectHalted = false; ringReconnectAfterMs = 0; lastAckAtMs = 0; lastIncomingAtMs = 0; diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java index 69e54a61..179f6e04 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.java @@ -64,6 +64,23 @@ public class FaceclawBleManager { this.listener = listener; } + /** + * Whether Android still holds a bond (pairing) for this address. Returns + * true when the state cannot be determined, so callers only act on a + * definite BOND_NONE. + */ + public boolean isBonded(String address) { + if (address == null || address.trim().isEmpty()) { + return true; + } + try { + BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address); + return device == null || device.getBondState() != BluetoothDevice.BOND_NONE; + } catch (Throwable t) { + return true; + } + } + public boolean connect(String address, int timeoutMs) { if (address == null || address.trim().isEmpty()) { throw new IllegalArgumentException("address is required"); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java new file mode 100644 index 00000000..488d2bca --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.java @@ -0,0 +1,446 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import com.k2fsa.sherpa.onnx.FeatureConfig; +import com.k2fsa.sherpa.onnx.OfflineModelConfig; +import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; +import com.k2fsa.sherpa.onnx.OfflineRecognizer; +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; +import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; +import com.k2fsa.sherpa.onnx.OfflineStream; + +import java.io.File; +import java.util.ArrayDeque; +import java.util.Arrays; + +/** + * Continuous captioning over the decoded mic PCM: segments speech into + * utterances with an adaptive energy gate, transcribes each utterance with + * the on-device Moonshine model, and attaches a speaker voice-print per + * utterance. PCM is pushed in from the TS side (which owns mic arbitration + * and any beam-direction gating), so this engine has no BLE dependencies. + * + * Utterance boundaries are measured on the sample clock, not the wall clock: + * BLE delivers mic packets in bursts, so elapsed real time overestimates the + * audio heard. The TS side converts startMs/endMs to wall-clock times using + * the engine start timestamp. + */ +public class FaceclawCaptionEngine { + private static final String TAG = "FaceclawCaptions"; + private static final int SAMPLE_RATE = 16000; + private static final int FEATURE_DIM = 80; + private static final int MAX_QUEUE_PACKETS = 200; + + // Segmentation: RMS thresholds relative to a rolling noise floor, in the + // same spirit as FaceclawVoiceController.EndpointDetector but recurring. + private static final double ONSET_FACTOR = 3.0; + private static final double RELEASE_FACTOR = 1.8; + private static final double MIN_RMS = 220.0; + private static final double NOISE_EMA_ALPHA = 0.05; + private static final int PRE_ROLL_MS = 400; + private static final int DEFAULT_SILENCE_MS = 800; + private static final int MIN_UTTERANCE_MS = 350; + private static final int MAX_UTTERANCE_MS = 15000; + // Moonshine v2 fails past ~9.1 s of input; decode long utterances in + // segments cut at the quietest window (mirrors FaceclawVoiceController). + private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8; + private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2; + private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000; + private static final float NORMALIZE_TARGET_PEAK = 0.9f; + private static final float NORMALIZE_MAX_GAIN = 30f; + // Voice-prints degrade on very long inputs; embed at most the first 10 s. + private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10; + + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private final Object lock = new Object(); + private final Object queueLock = new Object(); + private final ArrayDeque queue = new ArrayDeque<>(); + + private volatile FaceclawCaptionEngineListener listener; + private volatile String asrModelDir; + private volatile String speakerModelPath; + private volatile int silenceMs = DEFAULT_SILENCE_MS; + private Thread workerThread; + private volatile boolean started; + + private OfflineRecognizer recognizer; + private FaceclawSpeakerId speakerId; + + // Segmentation state (worker thread only). + private long totalSamples; + private double noiseFloor; + private boolean inUtterance; + private long utteranceStartSample; + private long silenceRunSamples; + private double utterancePeakRms; + private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)]; + private int utteranceLength; + private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000]; + private int preRollLength; + + public void setListener(FaceclawCaptionEngineListener listener) { + this.listener = listener; + } + + /** Directory holding the Moonshine model files, or null to disable ASR. */ + public void setAsrModelDir(String dir) { + this.asrModelDir = dir; + } + + /** Speaker-embedding ONNX model path, or null to disable voice-prints. */ + public void setSpeakerModelPath(String path) { + this.speakerModelPath = path; + } + + public void setSilenceMs(int ms) { + this.silenceMs = Math.max(200, Math.min(3000, ms)); + } + + public void start() { + synchronized (lock) { + if (started) { + return; + } + started = true; + workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine"); + workerThread.start(); + } + } + + public void stop() { + Thread threadToJoin; + synchronized (lock) { + if (!started) { + return; + } + started = false; + threadToJoin = workerThread; + workerThread = null; + } + synchronized (queueLock) { + queueLock.notifyAll(); + } + if (threadToJoin != null) { + threadToJoin.interrupt(); + if (Thread.currentThread() != threadToJoin) { + try { + threadToJoin.join(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + + /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ + public void acceptPcm(byte[] pcm16le) { + if (!started || pcm16le == null || pcm16le.length < 2) { + return; + } + synchronized (queueLock) { + if (queue.size() >= MAX_QUEUE_PACKETS) { + queue.removeFirst(); + } + queue.addLast(pcm16le); + queueLock.notifyAll(); + } + } + + private void runLoop() { + try { + loadModels(); + resetSegmentation(); + while (started && !Thread.currentThread().isInterrupted()) { + byte[] chunk = takeChunk(); + if (chunk == null) { + continue; + } + processChunk(chunk); + } + // Flush a trailing utterance so its text isn't lost on stop. + if (inUtterance && utteranceLength > 0) { + finalizeUtterance(); + } + } catch (Throwable t) { + Log.e(TAG, "caption engine failed", t); + emitStatus("Captions failed: " + t.getMessage()); + } finally { + releaseModels(); + } + } + + private void loadModels() { + String modelDir = asrModelDir; + if (modelDir != null && new File(modelDir, "tokens.txt").exists()) { + emitStatus("Loading caption model..."); + recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setModelConfig(OfflineModelConfig.builder() + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) + .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) + .build()) + .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) + .setNumThreads(1) + .build()) + .build()); + } else { + recognizer = null; + } + String embedModel = speakerModelPath; + if (embedModel != null && new File(embedModel).exists()) { + speakerId = new FaceclawSpeakerId(embedModel); + speakerId.ensureLoaded(); + } else { + speakerId = null; + } + emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)..."); + } + + private void releaseModels() { + if (recognizer != null) { + recognizer.release(); + recognizer = null; + } + if (speakerId != null) { + speakerId.close(); + speakerId = null; + } + } + + private byte[] takeChunk() { + synchronized (queueLock) { + while (started && queue.isEmpty()) { + try { + queueLock.wait(250); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return null; + } + } + return queue.pollFirst(); + } + } + + private void resetSegmentation() { + totalSamples = 0; + noiseFloor = 0; + inUtterance = false; + utteranceLength = 0; + preRollLength = 0; + silenceRunSamples = 0; + utterancePeakRms = 0; + } + + private void processChunk(byte[] chunk) { + int count = chunk.length / 2; + short[] pcm = new short[count]; + double sumSquares = 0; + for (int i = 0; i < count; i++) { + short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8)); + pcm[i] = s; + sumSquares += (double) s * s; + } + double rms = Math.sqrt(sumSquares / Math.max(1, count)); + totalSamples += count; + + if (!inUtterance) { + // Track the noise floor only while idle so speech doesn't raise it. + noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA; + double threshold = Math.max(noiseFloor, MIN_RMS); + if (rms >= threshold * ONSET_FACTOR) { + inUtterance = true; + utteranceLength = 0; + silenceRunSamples = 0; + utterancePeakRms = rms; + appendUtterance(preRoll, preRollLength); + utteranceStartSample = Math.max(0, totalSamples - count - preRollLength); + appendUtterance(pcm, count); + emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE); + } else { + appendPreRoll(pcm, count); + } + return; + } + + appendUtterance(pcm, count); + utterancePeakRms = Math.max(utterancePeakRms, rms); + double threshold = Math.max(noiseFloor, MIN_RMS); + if (rms < threshold * RELEASE_FACTOR) { + silenceRunSamples += count; + } else { + silenceRunSamples = 0; + } + long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE; + boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs; + if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) { + finalizeUtterance(); + inUtterance = false; + preRollLength = 0; + } + } + + private void appendPreRoll(short[] pcm, int count) { + // Keep the last PRE_ROLL_MS of idle audio so onset consonants survive. + if (count >= preRoll.length) { + System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length); + preRollLength = preRoll.length; + return; + } + int keep = Math.min(preRollLength, preRoll.length - count); + System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep); + System.arraycopy(pcm, 0, preRoll, keep, count); + preRollLength = keep + count; + } + + private void appendUtterance(short[] pcm, int count) { + int room = utterance.length - utteranceLength; + int copied = Math.min(room, count); + if (copied > 0) { + System.arraycopy(pcm, 0, utterance, utteranceLength, copied); + utteranceLength += copied; + } + } + + private void finalizeUtterance() { + int length = utteranceLength; + long startMs = utteranceStartSample * 1000 / SAMPLE_RATE; + long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE; + if (endMs - startMs < MIN_UTTERANCE_MS) { + return; + } + String text = recognizeUtterance(length); + float[] embedding = embedUtterance(length); + emitUtterance(text, embedding, startMs, endMs, utterancePeakRms); + } + + private String recognizeUtterance(int length) { + if (recognizer == null || length <= 0) { + return ""; + } + StringBuilder joined = new StringBuilder(); + int offset = 0; + while (offset < length) { + int remaining = length - offset; + int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES); + if (remaining > DECODE_SEGMENT_MAX_SAMPLES) { + segment = findQuietCut(offset, segment); + } + String part = recognizeRange(offset, segment); + if (part.length() > 0) { + if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) { + joined.append(' '); + } + joined.append(part); + } + offset += segment; + } + return joined.toString().trim(); + } + + /** + * End a decode segment at the center of the quietest window near its end + * so the cut lands between words (same approach as the PTT controller). + */ + private int findQuietCut(int offset, int segment) { + int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES); + int win = CUT_WINDOW_SAMPLES; + if (segment - searchStart <= win) { + return segment; + } + double sum = 0; + for (int i = searchStart; i < searchStart + win; i++) { + double s = utterance[offset + i]; + sum += s * s; + } + double best = sum; + int bestStart = searchStart; + for (int start = searchStart + 1; start + win <= segment; start++) { + double dropped = utterance[offset + start - 1]; + double added = utterance[offset + start + win - 1]; + sum += added * added - dropped * dropped; + if (sum < best) { + best = sum; + bestStart = start; + } + } + return bestStart + win / 2; + } + + private String recognizeRange(int offset, int count) { + float[] samples = new float[count]; + float peak = 0f; + for (int i = 0; i < count; i++) { + float v = utterance[offset + i] / 32768.0f; + samples[i] = v; + float a = Math.abs(v); + if (a > peak) { + peak = a; + } + } + if (peak > 0f) { + float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN); + if (gain > 1f) { + for (int i = 0; i < count; i++) { + samples[i] *= gain; + } + } + } + OfflineStream stream = recognizer.createStream(); + try { + stream.acceptWaveform(samples, SAMPLE_RATE); + recognizer.decode(stream); + OfflineRecognizerResult result = recognizer.getResult(stream); + String raw = result == null ? "" : result.getText(); + return raw == null ? "" : raw.trim(); + } finally { + stream.release(); + } + } + + private float[] embedUtterance(int length) { + FaceclawSpeakerId currentSpeakerId = speakerId; + if (currentSpeakerId == null || length <= 0) { + return null; + } + int count = Math.min(length, EMBED_MAX_SAMPLES); + byte[] le = new byte[count * 2]; + for (int i = 0; i < count; i++) { + short s = utterance[i]; + le[i * 2] = (byte) (s & 0xff); + le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); + } + return currentSpeakerId.embed(le, SAMPLE_RATE); + } + + private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) { + FaceclawCaptionEngineListener currentListener = listener; + if (currentListener == null) { + return; + } + float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length); + mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms)); + } + + private void emitSpeechStart(long startMs) { + FaceclawCaptionEngineListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onSpeechStart(startMs)); + } + + private void emitStatus(String status) { + FaceclawCaptionEngineListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onStatus(status)); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngineListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngineListener.java new file mode 100644 index 00000000..24b58a85 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngineListener.java @@ -0,0 +1,17 @@ +package com.faceclaw.app; + +public interface FaceclawCaptionEngineListener { + /** + * One finished utterance. text may be empty when no ASR model is loaded + * or recognition produced nothing; embedding is the L2-normalized + * voice-print or null when the speaker model is unavailable or the + * utterance was too short. startMs/endMs are on the engine's sample + * clock (milliseconds of audio since start()). + */ + void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms); + + /** A speech onset was detected; the utterance is now being collected. */ + void onSpeechStart(long startMs); + + void onStatus(String status); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java new file mode 100644 index 00000000..38f723fe --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.java @@ -0,0 +1,778 @@ +package com.faceclaw.app; + +import android.content.ContentValues; +import android.content.Context; +import android.database.Cursor; +import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteOpenHelper; +import android.util.Base64; +import android.util.Log; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +/** + * Persistent store for caption sessions, transcript segments, and speaker + * voice profiles, backing the Microphones app and its phone-side review UI. + * All complex values cross the JS bridge as JSON strings; voice-print + * embeddings are float32 LE blobs, exposed as base64. + */ +public class FaceclawConversationStore extends SQLiteOpenHelper { + private static final String TAG = "FaceclawConvStore"; + private static final String DB_NAME = "faceclaw-conversations.db"; + private static final int DB_VERSION = 2; + + private static FaceclawConversationStore instance; + + public static synchronized FaceclawConversationStore getInstance(Context context) { + if (instance == null) { + instance = new FaceclawConversationStore(context.getApplicationContext()); + } + return instance; + } + + private FaceclawConversationStore(Context context) { + super(context, DB_NAME, null, DB_VERSION); + } + + @Override + public void onCreate(SQLiteDatabase db) { + db.execSQL("CREATE TABLE speakers (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "name TEXT NOT NULL," + + "color TEXT NOT NULL DEFAULT '#4FC3F7'," + + "is_wearer INTEGER NOT NULL DEFAULT 0," + + "embedding BLOB," + + "embedding_count INTEGER NOT NULL DEFAULT 0," + + "created_at INTEGER NOT NULL," + + "last_heard_at INTEGER," + + "tag TEXT," + + "last_recap TEXT," + + "action_items TEXT," + + "facts TEXT," + + "insights_updated_at INTEGER," + + "insights_session_id INTEGER)"); + db.execSQL("CREATE TABLE sessions (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "started_at INTEGER NOT NULL," + + "ended_at INTEGER," + + "title TEXT," + + "audio_path TEXT," + + "audio_codec TEXT," + + "avg_sentiment REAL," + + "segment_count INTEGER NOT NULL DEFAULT 0)"); + db.execSQL("CREATE TABLE segments (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "session_id INTEGER NOT NULL," + + "speaker_id INTEGER," + + "started_at INTEGER NOT NULL," + + "ended_at INTEGER NOT NULL," + + "audio_offset_ms INTEGER," + + "text TEXT NOT NULL," + + "lang TEXT," + + "translation TEXT," + + "translation_lang TEXT," + + "sentiment REAL," + + "emotion TEXT," + + "search_meta TEXT," + + "angle INTEGER," + + "embedding BLOB)"); + db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)"); + db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)"); + db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)"); + db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)"); + } + + @Override + public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { + if (oldVersion < 2) { + // v2: per-speaker conversation insights (recap of the most recent + // conversation, its action items, and accumulated inferred facts). + db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT"); + db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT"); + db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT"); + db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER"); + db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER"); + } + } + + // ---- sessions ---- + + public long startSession(long startedAtMs, String title) { + ContentValues values = new ContentValues(); + values.put("started_at", startedAtMs); + values.put("title", title == null || title.isEmpty() ? null : title); + return getWritableDatabase().insert("sessions", null, values); + } + + public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) { + ContentValues values = new ContentValues(); + values.put("ended_at", endedAtMs); + if (audioPath != null && !audioPath.isEmpty()) { + values.put("audio_path", audioPath); + values.put("audio_codec", audioCodec); + } + values.put("avg_sentiment", avgSentiment); + getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)}); + } + + public void setSessionAudio(long sessionId, String audioPath, String audioCodec) { + ContentValues values = new ContentValues(); + values.put("audio_path", audioPath); + values.put("audio_codec", audioCodec); + getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)}); + } + + public String getSessionAudioPath(long sessionId) { + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) { + if (c.moveToFirst()) { + return c.isNull(0) ? "" : c.getString(0); + } + } + return ""; + } + + /** + * Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs, + * speakerId, emotion (matches any segment emotion in the session), query + * (LIKE over title and segment text/search metadata), limit. + */ + public String querySessions(String filterJson) { + try { + JSONObject filter = filterJson == null || filterJson.isEmpty() + ? new JSONObject() : new JSONObject(filterJson); + StringBuilder sql = new StringBuilder( + "SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec," + + " s.avg_sentiment, s.segment_count FROM sessions s"); + List args = new ArrayList<>(); + List where = new ArrayList<>(); + boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query"); + if (joinSegments) { + sql.append(" LEFT JOIN segments g ON g.session_id = s.id"); + } + if (filter.has("sinceMs")) { + where.add("s.started_at >= ?"); + args.add(String.valueOf(filter.getLong("sinceMs"))); + } + if (filter.has("untilMs")) { + where.add("s.started_at <= ?"); + args.add(String.valueOf(filter.getLong("untilMs"))); + } + if (filter.has("speakerId")) { + where.add("g.speaker_id = ?"); + args.add(String.valueOf(filter.getLong("speakerId"))); + } + if (filter.has("emotion")) { + where.add("g.emotion = ?"); + args.add(filter.getString("emotion")); + } + if (filter.has("query")) { + String like = "%" + filter.getString("query") + "%"; + where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)"); + args.add(like); + args.add(like); + args.add(like); + } + if (!where.isEmpty()) { + sql.append(" WHERE ").append(String.join(" AND ", where)); + } + sql.append(" ORDER BY s.started_at DESC"); + int limit = filter.optInt("limit", 200); + sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000))); + + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + long id = c.getLong(0); + row.put("id", id); + row.put("startedAt", c.getLong(1)); + row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2)); + row.put("title", c.isNull(3) ? "" : c.getString(3)); + row.put("audioPath", c.isNull(4) ? "" : c.getString(4)); + row.put("audioCodec", c.isNull(5) ? "" : c.getString(5)); + row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6)); + row.put("segmentCount", c.getLong(7)); + row.put("speakers", sessionSpeakersJson(id)); + out.put(row); + } + } + return out.toString(); + } catch (JSONException e) { + Log.w(TAG, "querySessions failed", e); + return "[]"; + } + } + + private JSONArray sessionSpeakersJson(long sessionId) throws JSONException { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT DISTINCT p.id, p.name, p.color FROM segments g" + + " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?", + new String[]{String.valueOf(sessionId)})) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("name", c.getString(1)); + row.put("color", c.getString(2)); + out.put(row); + } + } + return out; + } + + // ---- segments ---- + + /** + * Insert one transcript segment. json fields: sessionId, speakerId?, + * startedAt, endedAt, audioOffsetMs?, text, lang?, translation?, + * translationLang?, sentiment?, emotion?, searchMeta?, angle?, + * embeddingBase64?. Returns the new row id, or -1. + */ + public long insertSegment(String json) { + try { + JSONObject s = new JSONObject(json); + ContentValues values = new ContentValues(); + long sessionId = s.getLong("sessionId"); + values.put("session_id", sessionId); + if (s.has("speakerId")) { + values.put("speaker_id", s.getLong("speakerId")); + } + values.put("started_at", s.getLong("startedAt")); + values.put("ended_at", s.getLong("endedAt")); + if (s.has("audioOffsetMs")) { + values.put("audio_offset_ms", s.getLong("audioOffsetMs")); + } + values.put("text", s.getString("text")); + putOptString(values, s, "lang", "lang"); + putOptString(values, s, "translation", "translation"); + putOptString(values, s, "translationLang", "translation_lang"); + if (s.has("sentiment")) { + values.put("sentiment", s.getDouble("sentiment")); + } + putOptString(values, s, "emotion", "emotion"); + putOptString(values, s, "searchMeta", "search_meta"); + if (s.has("angle")) { + values.put("angle", s.getInt("angle")); + } + if (s.has("embeddingBase64")) { + values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP)); + } + SQLiteDatabase db = getWritableDatabase(); + long id = db.insert("segments", null, values); + db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?", + new Object[]{sessionId}); + return id; + } catch (Throwable t) { + Log.w(TAG, "insertSegment failed", t); + return -1; + } + } + + private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column) + throws JSONException { + if (s.has(jsonKey) && !s.isNull(jsonKey)) { + values.put(column, s.getString(jsonKey)); + } + } + + /** Update mutable fields of a segment (same keys as insertSegment). */ + public void updateSegment(long segmentId, String json) { + try { + JSONObject s = new JSONObject(json); + ContentValues values = new ContentValues(); + if (s.has("speakerId")) { + if (s.isNull("speakerId")) { + values.putNull("speaker_id"); + } else { + values.put("speaker_id", s.getLong("speakerId")); + } + } + if (s.has("text")) { + values.put("text", s.getString("text")); + } + putOptString(values, s, "lang", "lang"); + putOptString(values, s, "translation", "translation"); + putOptString(values, s, "translationLang", "translation_lang"); + if (s.has("sentiment")) { + values.put("sentiment", s.getDouble("sentiment")); + } + putOptString(values, s, "emotion", "emotion"); + putOptString(values, s, "searchMeta", "search_meta"); + if (values.size() > 0) { + getWritableDatabase().update("segments", values, "id=?", + new String[]{String.valueOf(segmentId)}); + } + } catch (Throwable t) { + Log.w(TAG, "updateSegment failed", t); + } + } + + public String querySegments(long sessionId) { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," + + " translation, translation_lang, sentiment, emotion, angle" + + " FROM segments WHERE session_id=? ORDER BY started_at ASC", + new String[]{String.valueOf(sessionId)})) { + while (c.moveToNext()) { + out.put(segmentRowJson(c)); + } + } catch (JSONException e) { + Log.w(TAG, "querySegments failed", e); + } + return out.toString(); + } + + private JSONObject segmentRowJson(Cursor c) throws JSONException { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1)); + row.put("startedAt", c.getLong(2)); + row.put("endedAt", c.getLong(3)); + row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4)); + row.put("text", c.getString(5)); + row.put("lang", c.isNull(6) ? "" : c.getString(6)); + row.put("translation", c.isNull(7) ? "" : c.getString(7)); + row.put("translationLang", c.isNull(8) ? "" : c.getString(8)); + row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9)); + row.put("emotion", c.isNull(10) ? "" : c.getString(10)); + row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11)); + return row; + } + + /** + * Search segments across sessions. filterJson: query?, emotion?, + * speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and + * the sentiment search metadata. + */ + public String searchSegments(String filterJson) { + try { + JSONObject filter = filterJson == null || filterJson.isEmpty() + ? new JSONObject() : new JSONObject(filterJson); + StringBuilder sql = new StringBuilder( + "SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang," + + " translation, translation_lang, sentiment, emotion, angle, session_id" + + " FROM segments"); + List args = new ArrayList<>(); + List where = new ArrayList<>(); + if (filter.has("query") && !filter.getString("query").isEmpty()) { + String like = "%" + filter.getString("query") + "%"; + where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)"); + args.add(like); + args.add(like); + args.add(like); + } + if (filter.has("emotion")) { + where.add("emotion = ?"); + args.add(filter.getString("emotion")); + } + if (filter.has("speakerId")) { + where.add("speaker_id = ?"); + args.add(String.valueOf(filter.getLong("speakerId"))); + } + if (filter.has("sinceMs")) { + where.add("started_at >= ?"); + args.add(String.valueOf(filter.getLong("sinceMs"))); + } + if (filter.has("untilMs")) { + where.add("started_at <= ?"); + args.add(String.valueOf(filter.getLong("untilMs"))); + } + if (!where.isEmpty()) { + sql.append(" WHERE ").append(String.join(" AND ", where)); + } + sql.append(" ORDER BY started_at DESC LIMIT ") + .append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000))); + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) { + while (c.moveToNext()) { + JSONObject row = segmentRowJson(c); + row.put("sessionId", c.getLong(12)); + out.put(row); + } + } + return out.toString(); + } catch (JSONException e) { + Log.w(TAG, "searchSegments failed", e); + return "[]"; + } + } + + /** Per-segment voice-print embeddings of a session (for re-diarization). */ + public String querySegmentEmbeddings(long sessionId) { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL", + new String[]{String.valueOf(sessionId)})) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP)); + out.put(row); + } + } catch (JSONException e) { + Log.w(TAG, "querySegmentEmbeddings failed", e); + } + return out.toString(); + } + + // ---- speakers ---- + + public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) { + ContentValues values = new ContentValues(); + values.put("name", name); + values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color); + values.put("is_wearer", isWearer ? 1 : 0); + values.put("created_at", System.currentTimeMillis()); + values.put("last_heard_at", System.currentTimeMillis()); + if (embeddingBase64 != null && !embeddingBase64.isEmpty()) { + values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP)); + values.put("embedding_count", 1); + } + return getWritableDatabase().insert("speakers", null, values); + } + + public String querySpeakers() { + JSONArray out = new JSONArray(); + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count," + + " p.last_heard_at, p.tag," + + " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id," + + " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments" + + " FROM speakers p ORDER BY p.last_heard_at DESC", null)) { + while (c.moveToNext()) { + JSONObject row = new JSONObject(); + row.put("id", c.getLong(0)); + row.put("name", c.getString(1)); + row.put("color", c.getString(2)); + row.put("isWearer", c.getInt(3) != 0); + byte[] blob = c.isNull(4) ? null : c.getBlob(4); + row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP)); + row.put("embeddingCount", c.getInt(5)); + row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6)); + row.put("tag", c.isNull(7) ? "" : c.getString(7)); + row.put("lastRecap", c.isNull(8) ? "" : c.getString(8)); + row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9)); + row.put("factsJson", c.isNull(10) ? "" : c.getString(10)); + row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11)); + row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12)); + row.put("segmentCount", c.getLong(13)); + out.put(row); + } + } catch (JSONException e) { + Log.w(TAG, "querySpeakers failed", e); + } + return out.toString(); + } + + public void renameSpeaker(long speakerId, String name) { + ContentValues values = new ContentValues(); + values.put("name", name); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + public void setSpeakerColor(long speakerId, String color) { + ContentValues values = new ContentValues(); + values.put("color", color); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + public void setSpeakerTag(long speakerId, String tag) { + ContentValues values = new ContentValues(); + values.put("tag", tag); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + /** + * Store LLM-derived conversation insights for a speaker: a recap of their + * most recent conversation, its open action items (JSON string array), and + * the accumulated inferred facts about them (JSON string array). + */ + public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson, + String factsJson, long sessionId) { + ContentValues values = new ContentValues(); + values.put("last_recap", recap == null ? "" : recap); + values.put("action_items", actionItemsJson == null ? "" : actionItemsJson); + values.put("facts", factsJson == null ? "" : factsJson); + values.put("insights_updated_at", System.currentTimeMillis()); + values.put("insights_session_id", sessionId); + getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + public void setSpeakerWearer(long speakerId, boolean isWearer) { + SQLiteDatabase db = getWritableDatabase(); + if (isWearer) { + // Exactly one wearer profile at a time. + db.execSQL("UPDATE speakers SET is_wearer = 0"); + } + ContentValues values = new ContentValues(); + values.put("is_wearer", isWearer ? 1 : 0); + db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)}); + } + + /** + * Running-mean centroid update for a speaker's voice-print, capped so a + * long session cannot drown the enrolled voice. + */ + public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) { + try { + byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP); + float[] update = blobToFloats(incoming); + if (update == null) { + return; + } + float[] current = null; + int count = 0; + try (Cursor c = getReadableDatabase().rawQuery( + "SELECT embedding, embedding_count FROM speakers WHERE id=?", + new String[]{String.valueOf(speakerId)})) { + if (c.moveToFirst()) { + current = c.isNull(0) ? null : blobToFloats(c.getBlob(0)); + count = c.getInt(1); + } + } + float[] merged; + if (current == null || current.length != update.length) { + merged = update; + count = 1; + } else { + int effective = Math.min(count, Math.max(1, maxCount)); + merged = new float[current.length]; + for (int i = 0; i < merged.length; i++) { + merged[i] = (current[i] * effective + update[i]) / (effective + 1); + } + l2NormalizeInPlace(merged); + count = count + 1; + } + ContentValues values = new ContentValues(); + values.put("embedding", floatsToBlob(merged)); + values.put("embedding_count", count); + values.put("last_heard_at", System.currentTimeMillis()); + getWritableDatabase().update("speakers", values, "id=?", + new String[]{String.valueOf(speakerId)}); + } catch (Throwable t) { + Log.w(TAG, "updateSpeakerEmbedding failed", t); + } + } + + /** Move every segment from one speaker onto another and drop the source. */ + public void mergeSpeakers(long fromId, long intoId) { + if (fromId == intoId) { + return; + } + SQLiteDatabase db = getWritableDatabase(); + db.beginTransaction(); + try { + db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?", + new Object[]{intoId, fromId}); + // Blend the two centroids weighted by their sample counts. + float[] from = null; + float[] into = null; + int fromCount = 0; + int intoCount = 0; + boolean fromWearer = false; + try (Cursor c = db.rawQuery( + "SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)", + new String[]{String.valueOf(fromId), String.valueOf(intoId)})) { + while (c.moveToNext()) { + long id = c.getLong(0); + float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1)); + int count = c.getInt(2); + if (id == fromId) { + from = embedding; + fromCount = count; + fromWearer = c.getInt(3) != 0; + } else { + into = embedding; + intoCount = count; + } + } + } + float[] merged = into; + if (from != null && into != null && from.length == into.length) { + merged = new float[into.length]; + int total = Math.max(1, fromCount + intoCount); + for (int i = 0; i < merged.length; i++) { + merged[i] = (into[i] * intoCount + from[i] * fromCount) / total; + } + l2NormalizeInPlace(merged); + } else if (into == null) { + merged = from; + } + ContentValues values = new ContentValues(); + if (merged != null) { + values.put("embedding", floatsToBlob(merged)); + values.put("embedding_count", fromCount + intoCount); + } + if (fromWearer) { + values.put("is_wearer", 1); + } + values.put("last_heard_at", System.currentTimeMillis()); + // Keep the target's conversation insights; adopt the source's only + // where the target has none (e.g. merging a rich profile into a + // freshly auto-created one). + try (Cursor c = db.rawQuery( + "SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id" + + " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END", + new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) { + boolean targetHasInsights = false; + while (c.moveToNext()) { + boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty(); + if (c.getPosition() == 0) { + targetHasInsights = hasInsights; + } else if (!targetHasInsights && hasInsights) { + values.put("last_recap", c.getString(0)); + values.put("action_items", c.isNull(1) ? "" : c.getString(1)); + values.put("facts", c.isNull(2) ? "" : c.getString(2)); + values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3)); + values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4)); + } + } + } + db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)}); + db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)}); + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + } + } + + public void reassignSegmentSpeaker(long segmentId, long speakerId) { + ContentValues values = new ContentValues(); + values.put("speaker_id", speakerId); + getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)}); + } + + /** Reassign every segment of a speaker within one session (split support). */ + public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) { + getWritableDatabase().execSQL( + "UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?", + new Object[]{toSpeakerId, sessionId, fromSpeakerId}); + } + + public void deleteSpeaker(long speakerId) { + SQLiteDatabase db = getWritableDatabase(); + db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId}); + db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)}); + } + + // ---- retention ---- + + /** + * Delete caption sessions older than the cutoff (0 disables), and strip + * recordings older than their own cutoff while keeping the transcript. + * Returns a JSON summary of what was removed. + */ + public String applyRetention(long captionCutoffMs, long recordingCutoffMs) { + SQLiteDatabase db = getWritableDatabase(); + int sessionsDeleted = 0; + int recordingsDeleted = 0; + if (recordingCutoffMs > 0) { + List paths = new ArrayList<>(); + try (Cursor c = db.rawQuery( + "SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", + new String[]{String.valueOf(recordingCutoffMs)})) { + while (c.moveToNext()) { + if (!c.isNull(1)) { + paths.add(c.getString(1)); + } + } + } + for (String path : paths) { + if (new File(path).delete()) { + recordingsDeleted++; + } + } + db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?", + new Object[]{recordingCutoffMs}); + } + if (captionCutoffMs > 0) { + List paths = new ArrayList<>(); + try (Cursor c = db.rawQuery( + "SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?", + new String[]{String.valueOf(captionCutoffMs)})) { + while (c.moveToNext()) { + paths.add(c.getString(0)); + } + } + for (String path : paths) { + if (path != null && new File(path).delete()) { + recordingsDeleted++; + } + } + db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)", + new Object[]{captionCutoffMs}); + Cursor count = db.rawQuery("SELECT changes()", null); + count.close(); + db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs}); + try (Cursor c = db.rawQuery("SELECT changes()", null)) { + if (c.moveToFirst()) { + sessionsDeleted = c.getInt(0); + } + } + } + try { + JSONObject summary = new JSONObject(); + summary.put("sessionsDeleted", sessionsDeleted); + summary.put("recordingsDeleted", recordingsDeleted); + return summary.toString(); + } catch (JSONException e) { + return "{}"; + } + } + + public void deleteSession(long sessionId) { + SQLiteDatabase db = getWritableDatabase(); + String path = getSessionAudioPath(sessionId); + if (!path.isEmpty()) { + new File(path).delete(); + } + db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)}); + db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)}); + } + + // ---- blob helpers ---- + + private static float[] blobToFloats(byte[] blob) { + if (blob == null || blob.length < 4 || blob.length % 4 != 0) { + return null; + } + ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN); + float[] out = new float[blob.length / 4]; + for (int i = 0; i < out.length; i++) { + out[i] = buffer.getFloat(); + } + return out; + } + + private static byte[] floatsToBlob(float[] values) { + ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (float v : values) { + buffer.putFloat(v); + } + return buffer.array(); + } + + private static void l2NormalizeInPlace(float[] v) { + double sum = 0; + for (float x : v) { + sum += (double) x * x; + } + double norm = Math.sqrt(sum); + if (norm <= 0) { + return; + } + for (int i = 0; i < v.length; i++) { + v[i] = (float) (v[i] / norm); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java index d57c4d3a..53645f27 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscovery.java @@ -6,151 +6,381 @@ import android.bluetooth.BluetoothDevice; import android.bluetooth.BluetoothManager; import android.bluetooth.le.BluetoothLeScanner; import android.bluetooth.le.ScanCallback; +import android.bluetooth.le.ScanRecord; import android.bluetooth.le.ScanResult; import android.bluetooth.le.ScanSettings; import android.content.Context; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.util.SparseArray; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.ArrayList; +import java.util.List; import java.util.Set; -import java.util.regex.Pattern; +/** + * Finds Even Realities hardware over BLE and hands every advertisement up to + * TypeScript with the raw fields the pairing screen needs: the LIVE local name + * from the scan record (BluetoothDevice.getName() is a cache that can hold a + * pre-reset value indefinitely), the full manufacturer-specific payload with + * its company identifier restored so the "ER"+serial+MAC layout survives + * intact, RSSI, advertised TX power, and connectability. + * + * Admission is deliberately loose — a name containing G2, the stock + * "EVEN R1…" ring prefix, or the Even "ER" manufacturer signature — because + * renamed custom firmware may drop the stock name. Side, serial, and pair + * matching are decided in TypeScript (app/g2/even-advertisement.ts, + * app/g2/pairing-candidates.ts) where they can be unit-tested; this filter + * must stay a superset of the TS classifier's admission. + */ @SuppressLint("MissingPermission") public class FaceclawDeviceDiscovery { - private static final Pattern RING_NAME = Pattern.compile("^EVEN R1_[0-9A-F]{6}$"); - private static final Pattern RIGHT_NAME = Pattern.compile("^Even G2_32_R_[0-9A-F]{6}$"); - private static final Pattern LEFT_NAME = Pattern.compile("^Even G2_32_L_[0-9A-F]{6}$"); + /** Even Realities' company identifier: the ASCII bytes "ER" read little-endian. */ + public static final int EVEN_COMPANY_ID = 0x5245; - private final Context appContext; private final BluetoothAdapter bluetoothAdapter; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + + private final Object lock = new Object(); + private ScanCallback activeCallback; + private volatile FaceclawDeviceDiscoveryListener listener; public FaceclawDeviceDiscovery(Context context) { - this.appContext = context.getApplicationContext(); + Context appContext = context.getApplicationContext(); BluetoothManager bluetoothManager = - (BluetoothManager) this.appContext.getSystemService(Context.BLUETOOTH_SERVICE); + (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE); if (bluetoothManager == null || bluetoothManager.getAdapter() == null) { throw new IllegalStateException("Bluetooth adapter unavailable"); } this.bluetoothAdapter = bluetoothManager.getAdapter(); } - public String getBondedCandidatesJson() { - LinkedHashMap matches = new LinkedHashMap<>(); - Set bonded = bluetoothAdapter.getBondedDevices(); - if (bonded != null) { - for (BluetoothDevice device : bonded) { - maybeAddCandidate(matches, device, "paired"); - } - } - return encodeCandidates(matches); + public boolean isBluetoothEnabled() { + return bluetoothAdapter.isEnabled(); } - public String scanCandidatesJson(int timeoutMs) throws InterruptedException { - LinkedHashMap matches = new LinkedHashMap<>(); - BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); - if (scanner == null) { - return encodeCandidates(matches); - } - final ScanCallback callback = new ScanCallback() { - @Override - public void onScanResult(int callbackType, ScanResult result) { - if (result == null) { - return; - } - maybeAddCandidate(matches, result.getDevice(), "scan"); - } + public void setListener(FaceclawDeviceDiscoveryListener newListener) { + this.listener = newListener; + } - @Override - public void onBatchScanResults(java.util.List results) { - if (results == null) { - return; - } - for (ScanResult result : results) { + // ------------------------------------------------------------------ + // Live scan + // ------------------------------------------------------------------ + + /** + * Start streaming advertisements to the listener. Returns false when the + * radio is off or the scanner is unavailable; a later platform refusal + * arrives through onScanFailed. Calling while a scan is running is a no-op. + */ + public boolean startScan() { + synchronized (lock) { + if (activeCallback != null) { + return true; + } + if (!bluetoothAdapter.isEnabled()) { + emitLog("scan not started: Bluetooth is off"); + return false; + } + BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); + if (scanner == null) { + emitLog("scan not started: no BLE scanner"); + return false; + } + final ScanCallback callback = new ScanCallback() { + @Override + public void onScanResult(int callbackType, ScanResult result) { if (result != null) { - maybeAddCandidate(matches, result.getDevice(), "scan"); + emitAdvertisement(result); } } + + @Override + public void onBatchScanResults(List results) { + if (results == null) { + return; + } + for (ScanResult result : results) { + if (result != null) { + emitAdvertisement(result); + } + } + } + + @Override + public void onScanFailed(int errorCode) { + synchronized (lock) { + if (activeCallback == this) { + activeCallback = null; + } + } + final FaceclawDeviceDiscoveryListener l = listener; + if (l != null) { + final String message = describeScanError(errorCode); + mainHandler.post(() -> l.onScanFailed(errorCode, message)); + } + } + }; + activeCallback = callback; + try { + scanner.startScan(null, buildScanSettings(), callback); + } catch (Throwable t) { + activeCallback = null; + emitLog("scan not started: " + t); + return false; } - }; - scanner.startScan( - null, - new ScanSettings.Builder().setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY).build(), - callback - ); - try { - Thread.sleep(Math.max(500, timeoutMs)); - } finally { + emitLog("scan started (unfiltered, low latency)"); + return true; + } + } + + public void stopScan() { + ScanCallback callback; + synchronized (lock) { + callback = activeCallback; + activeCallback = null; + } + if (callback == null) { + return; + } + BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner(); + if (scanner != null) { try { scanner.stopScan(callback); } catch (Throwable ignored) { } } - return encodeCandidates(matches); + emitLog("scan stopped"); } - private void maybeAddCandidate(Map matches, BluetoothDevice device, String source) { - if (device == null) { - return; + public boolean isScanning() { + synchronized (lock) { + return activeCallback != null; } - String name = device.getName(); - String role = classifyRole(name); - if (role == null) { - return; - } - String address = device.getAddress(); - if (address == null || address.isEmpty()) { - return; - } - matches.put(address, new CandidateDevice(address, name == null ? "" : name, role, source)); } - private String classifyRole(String name) { - if (name == null) { - return null; + /** + * Replay bonded Even devices to the listener. A bonded listing has no + * signal sample and no manufacturer data, so its serial stays unknown until + * the device is also heard advertising. + */ + public void emitBondedDevices() { + final FaceclawDeviceDiscoveryListener l = listener; + if (l == null) { + return; } - if (RIGHT_NAME.matcher(name).matches()) { - return "right"; + for (String json : bondedCandidateJsonList()) { + l.onAdvertisement(json); } - if (LEFT_NAME.matcher(name).matches()) { - return "left"; - } - if (RING_NAME.matcher(name).matches()) { - return "ring"; - } - return null; } - private String encodeCandidates(Map matches) { + /** Bonded Even devices as one JSON array, for callers without a listener. */ + public String getBondedCandidatesJson() { JSONArray array = new JSONArray(); - for (CandidateDevice candidate : matches.values()) { - JSONObject object = new JSONObject(); + for (String json : bondedCandidateJsonList()) { try { - object.put("address", candidate.address); - object.put("name", candidate.name); - object.put("role", candidate.role); - object.put("source", candidate.source); - array.put(object); + array.put(new JSONObject(json)); } catch (JSONException ignored) { } } return array.toString(); } - private static final class CandidateDevice { - final String address; - final String name; - final String role; - final String source; + // ------------------------------------------------------------------ + // Internals + // ------------------------------------------------------------------ - CandidateDevice(String address, String name, String role, String source) { - this.address = address; - this.name = name; - this.role = role; - this.source = source; + private static ScanSettings buildScanSettings() { + ScanSettings.Builder builder = new ScanSettings.Builder() + .setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY) + .setReportDelay(0); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + builder.setCallbackType(ScanSettings.CALLBACK_TYPE_ALL_MATCHES); + builder.setMatchMode(ScanSettings.MATCH_MODE_AGGRESSIVE); + builder.setNumOfMatches(ScanSettings.MATCH_NUM_MAX_ADVERTISEMENT); + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + // Accept extended advertising too; setLegacy(true) would hide it. + builder.setLegacy(false); + } + return builder.build(); + } + + private void emitAdvertisement(ScanResult result) { + final JSONObject object = toJson(result); + if (object == null) { + return; + } + final FaceclawDeviceDiscoveryListener l = listener; + if (l == null) { + return; + } + final String json = object.toString(); + if (Looper.myLooper() == Looper.getMainLooper()) { + l.onAdvertisement(json); + } else { + mainHandler.post(() -> { + FaceclawDeviceDiscoveryListener current = listener; + if (current != null) { + current.onAdvertisement(json); + } + }); + } + } + + private List bondedCandidateJsonList() { + List out = new ArrayList<>(); + Set bonded = bluetoothAdapter.getBondedDevices(); + if (bonded == null) { + return out; + } + long now = System.currentTimeMillis(); + for (BluetoothDevice device : bonded) { + if (device == null) { + continue; + } + String name = device.getName(); + if (!isAdmissible(name, null)) { + continue; + } + JSONObject object = buildJson( + device.getAddress(), name, "", null, null, null, true, "paired", now); + if (object != null) { + out.add(object.toString()); + } + } + return out; + } + + /** Null when the result is not something we pair with. */ + private static JSONObject toJson(ScanResult result) { + BluetoothDevice device = result.getDevice(); + if (device == null) { + return null; + } + ScanRecord record = result.getScanRecord(); + String liveName = record == null ? null : record.getDeviceName(); + String name = liveName != null && !liveName.isEmpty() ? liveName : device.getName(); + byte[] manufacturerData = extractEvenManufacturerData(record); + if (!isAdmissible(name, manufacturerData)) { + return null; + } + Integer txPower = null; + if (record != null && record.getTxPowerLevel() != Integer.MIN_VALUE) { + txPower = record.getTxPowerLevel(); + } + if (txPower == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O + && result.getTxPower() != ScanResult.TX_POWER_NOT_PRESENT) { + txPower = result.getTxPower(); + } + Boolean connectable = null; + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + connectable = result.isConnectable(); + } + int rssi = result.getRssi(); + // 127 is the "unavailable" sentinel; 0 dBm never happens for a real sample. + Integer rssiValue = (rssi == 127 || rssi == 0) ? null : rssi; + boolean bonded = device.getBondState() == BluetoothDevice.BOND_BONDED; + return buildJson( + device.getAddress(), name, FaceclawFirmwareUtil.bytesToHex(manufacturerData), rssiValue, txPower, + connectable, bonded, "scan", System.currentTimeMillis()); + } + + /** + * Android strips the 2-byte company identifier from each manufacturer + * record and keys the SparseArray by it. Put the bytes back (little-endian) + * so TypeScript sees the same "ER"+… layout the HCI captures + * document. Prefers Even's own id; otherwise returns the first record so an + * unexpected company id still reaches the diagnostics log. + */ + private static byte[] extractEvenManufacturerData(ScanRecord record) { + if (record == null) { + return new byte[0]; + } + SparseArray all = record.getManufacturerSpecificData(); + if (all == null || all.size() == 0) { + return new byte[0]; + } + byte[] payload = all.get(EVEN_COMPANY_ID); + int companyId = EVEN_COMPANY_ID; + if (payload == null) { + companyId = all.keyAt(0); + payload = all.valueAt(0); + } + if (payload == null) { + return new byte[0]; + } + byte[] out = new byte[payload.length + 2]; + out[0] = (byte) (companyId & 0xff); + out[1] = (byte) ((companyId >> 8) & 0xff); + System.arraycopy(payload, 0, out, 2, payload.length); + return out; + } + + private static boolean isAdmissible(String name, byte[] manufacturerData) { + if (manufacturerData != null && manufacturerData.length >= 2 + && manufacturerData[0] == 0x45 && manufacturerData[1] == 0x52) { + return true; + } + if (name == null) { + return false; + } + String upper = name.toUpperCase(); + // A bare "R1" substring also matches unrelated hardware ("Oppo Enco + // R1"), and TS only classifies rings from the stock name prefix. + return upper.contains("G2") || upper.startsWith("EVEN R1"); + } + + private static JSONObject buildJson( + String address, String name, String manufacturerHex, Integer rssi, Integer txPower, + Boolean connectable, boolean bonded, String source, long seenAtMs) { + if (address == null || address.isEmpty()) { + return null; + } + try { + JSONObject object = new JSONObject(); + object.put("address", address); + object.put("name", name == null ? "" : name); + object.put("manufacturerData", manufacturerHex == null ? "" : manufacturerHex); + object.put("rssi", rssi == null ? JSONObject.NULL : rssi); + object.put("txPower", txPower == null ? JSONObject.NULL : txPower); + object.put("connectable", connectable == null ? JSONObject.NULL : connectable); + object.put("bonded", bonded); + object.put("source", source); + object.put("seenAtMs", seenAtMs); + return object; + } catch (JSONException e) { + return null; + } + } + + private static String describeScanError(int errorCode) { + switch (errorCode) { + case ScanCallback.SCAN_FAILED_ALREADY_STARTED: + return "a scan is already running"; + case ScanCallback.SCAN_FAILED_APPLICATION_REGISTRATION_FAILED: + return "the Bluetooth stack refused to register the scan (try toggling Bluetooth)"; + case ScanCallback.SCAN_FAILED_FEATURE_UNSUPPORTED: + return "BLE scanning is not supported on this device"; + case ScanCallback.SCAN_FAILED_INTERNAL_ERROR: + return "internal Bluetooth error"; + case 5: // SCAN_FAILED_OUT_OF_HARDWARE_RESOURCES (API 31) + return "out of Bluetooth hardware resources"; + case 6: // SCAN_FAILED_SCANNING_TOO_FREQUENTLY (API 31) + return "scanning too frequently; wait a moment and try again"; + default: + return "scan failed (code " + errorCode + ")"; + } + } + + private void emitLog(String line) { + FaceclawDeviceDiscoveryListener l = listener; + if (l != null) { + l.onLog(line); } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscoveryListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscoveryListener.java new file mode 100644 index 00000000..922e7adb --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceDiscoveryListener.java @@ -0,0 +1,18 @@ +package com.faceclaw.app; + +/** Callbacks from FaceclawDeviceDiscovery's live scan to the TypeScript layer. */ +public interface FaceclawDeviceDiscoveryListener { + /** + * One advertisement (or one bonded device) as a JSON object: + * {address, name, manufacturerData (hex, company id included), rssi|null, + * txPower|null, connectable|null, bonded, source, seenAtMs}. + * Protocol decoding happens in TypeScript (see app/g2/even-advertisement.ts). + */ + void onAdvertisement(String json); + + /** The platform refused to start or continue the scan. */ + void onScanFailed(int errorCode, String message); + + /** Diagnostic line for the pairing log. */ + void onLog(String line); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java index f89fa8b0..f5c0f51c 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDeviceInfoProbe.java @@ -23,6 +23,7 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener { private final Context context; private final String rightAddress; + private final String leftAddress; private final FaceclawBleManager bleManager; private final Handler mainHandler = new Handler(Looper.getMainLooper()); @@ -35,13 +36,21 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener { private byte[] awaitPb = null; private CountDownLatch awaitLatch = null; + /** + * A sid-0x09 frame that carried firmware versions without matching the + * awaited magic — some firmware pushes the settings snapshot on a magic it + * picked itself instead of (or in addition to) acking the read. + */ + private volatile byte[] unsolicitedSettingsPb; + private volatile FaceclawDeviceInfoProbeListener listener; private volatile Thread worker; private volatile boolean cancelled = false; - public FaceclawDeviceInfoProbe(Context context, String rightAddress) { + public FaceclawDeviceInfoProbe(Context context, String rightAddress, String leftAddress) { this.context = context.getApplicationContext(); this.rightAddress = rightAddress == null ? "" : rightAddress; + this.leftAddress = leftAddress == null ? "" : leftAddress; this.bleManager = new FaceclawBleManager(this.context); this.bleManager.setListener(this); } @@ -83,27 +92,28 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener { return; } - emitState("connecting", ""); - connectArm(rightAddress); + byte[] ack = probeArm(rightAddress, "right"); + // The right lens is the documented control endpoint, but a silent + // right lens has been observed on stock 2.2.9 even after a + // successful security auth — run the same probe against the left + // lens rather than giving up, and log which lens answered. + if (ack == null && !cancelled && !leftAddress.trim().isEmpty() + && !leftAddress.equalsIgnoreCase(rightAddress)) { + emitLog("right lens did not answer the settings query; probing the left lens"); + try { + ack = probeArm(leftAddress, "left"); + } catch (Exception e) { + emitLog("left-lens probe failed: " + (e.getMessage() == null ? e.toString() : e.getMessage())); + } + } if (cancelled) { emitError("Cancelled."); return; } - - emitState("querying", ""); - // Session prelude, then a settings/device-info read (both arms' - // versions and the CFW capability string ride back in one response). - if (writeAndAwaitAck(rightAddress, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST, - BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD, - ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { - throw new IllegalStateException("session prelude not acked"); - } - - int magic = allocMagic(); - byte[] ack = writeAndAwaitAck(rightAddress, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, - magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS); if (ack == null) { - throw new IllegalStateException("no response to the device-info query"); + throw new IllegalStateException( + "no response to the device-info query on either lens — check `adb logcat -s " + + TAG + "` for the frame trace"); } BleProtocol.FirmwareInfo info = BleProtocol.parseSettingsFirmwareInfo(ack); @@ -123,6 +133,73 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener { } } + /** + * Connect + authenticate + prelude + settings read on one lens. Returns the + * settings ack protobuf, an unsolicited settings push that carried firmware + * versions, or null when the lens never answered the read. Throws on + * connect or prelude failure; the caller wraps the fallback lens's attempt + * so its failure cannot mask the primary lens's outcome. + */ + private byte[] probeArm(String address, String label) throws InterruptedException { + emitState("connecting", label); + connectArm(address); + if (cancelled) { + return null; + } + + // Firmware 2.2.9 answers no queries until the security-auth exchange + // completes over an encrypted link; on a phone with no existing bond + // this is also what triggers SMP pairing (and its OS prompt), so it + // must come before the prelude and query. Soft: this probe also runs + // against the custom firmware, whose response to the exchange is not + // yet hardware-verified, so an unconfirmed auth falls through. + emitState("authenticating", label); + boolean authenticated = authenticate(address); + if (cancelled) { + return null; + } + + emitState("querying", label); + // Session prelude, then a settings/device-info read (both arms' + // versions and the CFW capability string ride back in one response). + if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST, + BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD, + ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { + throw new IllegalStateException("session prelude not acked (" + label + " lens)" + + (authenticated ? "" : "; authentication did not complete —" + + " if Android shows a Bluetooth pairing request, accept it and try again")); + } + + // Two attempts: the first read straight after a fresh pairing has been + // seen to go unanswered while a later one succeeds. + for (int attempt = 0; attempt < 2 && !cancelled; attempt++) { + int magic = allocMagic(); + byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, + magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS); + if (ack != null) { + return ack; + } + // A push with the firmware versions on the device's own magic is + // as good as the ack we asked for. + byte[] pushed = unsolicitedSettingsPb; + if (pushed != null) { + emitLog("using unsolicited settings push instead of the read ack (" + label + " lens)"); + return pushed; + } + emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)"); + } + return null; + } + + private boolean authenticate(String address) throws InterruptedException { + int magic = allocMagic(); + byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, + magic, BleProtocol.buildAuthenticationRequest(magic), ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS); + boolean success = ack != null && BleProtocol.isAuthenticationSuccess(ack, magic); + emitLog(success ? "security auth complete: " + address : "security auth unconfirmed: " + address); + return success; + } + private void connectArm(String address) { if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { throw new IllegalStateException("connect failed"); @@ -150,10 +227,13 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener { synchronized (lock) { seq = nextSeq++ & 0xff; } + emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d", + address, sid, flag, magic, seq, payload.length)); List frames = BleProtocol.framePb(payload, sid, flag, seq); boolean written = bleManager.writeFrames( address, BleProtocol.WRITE_CHAR_UUID, frames, ConnectionOptions.WRITE_TYPE, ConnectionOptions.WRITE_TIMEOUT_MS); if (!written) { + emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid)); synchronized (lock) { awaitLatch = null; } @@ -171,10 +251,38 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener { if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) { return; } - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data); + // One notification value can carry several envelope frames back to + // back; reading only the first would silently drop the rest. + List frames = BleProtocol.splitFrames(data); + if (frames.size() > 1) { + emitLog("rx " + address + " value carries " + frames.size() + " frames (raw " + data.length + " bytes)"); + } + for (byte[] buf : frames) { + handleFrame(address, buf, data.length); + } + } + + private void handleFrame(String address, byte[] buf, int rawValueLength) { + BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf); if (!frame.ok) { + emitLog("rx " + address + " unparseable frame len=" + buf.length + " (raw value " + rawValueLength + ")" + + " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.length)))); return; } + // Log every control frame while diagnosing 2.2.9: sid/flag/type/magic + // plus a payload prefix is enough to reconstruct what the lens said. + int declared = buf.length > 3 ? buf[3] & 0xff : 0; + String truncated = buf.length < 8 + declared ? " TRUNCATED(declared=" + declared + ")" : ""; + emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s", + address, frame.sid, frame.flag, frame.msgType, frame.msgSeq, + buf.length > 5 ? buf[5] & 0xff : 0, buf.length > 4 ? buf[4] & 0xff : 0, frame.pb.length, truncated, + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.length))))); + if (frame.sid == BleProtocol.SID_UI_SETTING + && BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) { + // Any settings frame carrying firmware versions answers the probe's + // question, whether or not it matches the magic we asked with. + unsolicitedSettingsPb = frame.pb; + } if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { return; // async event, not an ack } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java new file mode 100644 index 00000000..ea9352aa --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawDiarizer.java @@ -0,0 +1,262 @@ +package com.faceclaw.app; + +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import com.k2fsa.sherpa.onnx.FastClusteringConfig; +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization; +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig; +import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment; +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig; +import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig; +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Offline re-diarization of a stored conversation recording (16-bit mono + * WAV): pyannote segmentation plus the shared speaker-embedding model with + * clustering, via sherpa-onnx. Runs on its own thread; results are handed to + * TS as JSON turns for reconciliation against the transcript segments. + */ +public class FaceclawDiarizer { + private static final String TAG = "FaceclawDiarizer"; + + public interface Listener { + /** progress in [0, 1]. */ + void onProgress(double progress); + + /** turns: [{startMs, endMs, cluster}], sorted by start time. */ + void onDone(String turnsJson); + + void onError(String message); + } + + private final String segmentationModelPath; + private final String embeddingModelPath; + private final String wavPath; + private final int numClusters; + private final float threshold; + private final Listener listener; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private volatile boolean cancelled; + + public FaceclawDiarizer( + String segmentationModelPath, + String embeddingModelPath, + String wavPath, + int numClusters, + float threshold, + Listener listener) { + this.segmentationModelPath = segmentationModelPath; + this.embeddingModelPath = embeddingModelPath; + this.wavPath = wavPath; + this.numClusters = numClusters; + this.threshold = threshold <= 0 ? 0.5f : threshold; + this.listener = listener; + } + + public void start() { + Thread thread = new Thread(this::run, "FaceclawDiarizer"); + thread.setPriority(Thread.MIN_PRIORITY); + thread.start(); + } + + public void cancel() { + cancelled = true; + } + + private void run() { + OfflineSpeakerDiarization diarization = null; + try { + float[] samples = wavPath.endsWith(".wav") + ? readWavMono16(wavPath) + : decodeCompressedMono(wavPath); + OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder() + .setSegmentation(OfflineSpeakerSegmentationModelConfig.builder() + .setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder() + .setModel(segmentationModelPath) + .build()) + .setNumThreads(1) + .build()) + .setEmbedding(SpeakerEmbeddingExtractorConfig.builder() + .setModel(embeddingModelPath) + .setNumThreads(1) + .build()) + .setClustering(FastClusteringConfig.builder() + .setNumClusters(numClusters > 0 ? numClusters : -1) + .setThreshold(threshold) + .build()) + .setMinDurationOn(0.3f) + .setMinDurationOff(0.5f) + .build(); + diarization = new OfflineSpeakerDiarization(config); + OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback( + samples, + (numProcessedChunks, numTotalChunks, arg) -> { + emitProgress(numTotalChunks <= 0 ? 0 + : (double) numProcessedChunks / numTotalChunks); + return cancelled ? 1 : 0; + }); + JSONArray turns = new JSONArray(); + if (segments != null) { + for (OfflineSpeakerDiarizationSegment segment : segments) { + JSONObject turn = new JSONObject(); + turn.put("startMs", Math.round(segment.getStart() * 1000)); + turn.put("endMs", Math.round(segment.getEnd() * 1000)); + turn.put("cluster", segment.getSpeaker()); + turns.put(turn); + } + } + String result = turns.toString(); + mainHandler.post(() -> listener.onDone(result)); + } catch (Throwable t) { + Log.w(TAG, "diarization failed for " + wavPath, t); + String message = t.getMessage() == null ? t.toString() : t.getMessage(); + mainHandler.post(() -> listener.onError(message)); + } finally { + if (diarization != null) { + diarization.release(); + } + } + } + + private void emitProgress(double progress) { + mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress)))); + } + + /** + * Decode a compressed recording (M4A/AAC from the transcode sweep) into + * [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written + * as 16 kHz mono, so no resampling is needed. + */ + private static float[] decodeCompressedMono(String path) throws IOException { + android.media.MediaExtractor extractor = new android.media.MediaExtractor(); + android.media.MediaCodec codec = null; + java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream(); + try { + extractor.setDataSource(path); + int trackIndex = -1; + android.media.MediaFormat format = null; + for (int i = 0; i < extractor.getTrackCount(); i++) { + android.media.MediaFormat candidate = extractor.getTrackFormat(i); + String mime = candidate.getString(android.media.MediaFormat.KEY_MIME); + if (mime != null && mime.startsWith("audio/")) { + trackIndex = i; + format = candidate; + break; + } + } + if (trackIndex < 0 || format == null) { + throw new IOException("no audio track in " + path); + } + extractor.selectTrack(trackIndex); + codec = android.media.MediaCodec.createDecoderByType( + format.getString(android.media.MediaFormat.KEY_MIME)); + codec.configure(format, null, null, 0); + codec.start(); + android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo(); + boolean inputDone = false; + boolean outputDone = false; + while (!outputDone) { + if (!inputDone) { + int inIndex = codec.dequeueInputBuffer(10_000); + if (inIndex >= 0) { + ByteBuffer inBuf = codec.getInputBuffer(inIndex); + int size = extractor.readSampleData(inBuf, 0); + if (size < 0) { + codec.queueInputBuffer(inIndex, 0, 0, 0, + android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM); + inputDone = true; + } else { + codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0); + extractor.advance(); + } + } + } + int outIndex = codec.dequeueOutputBuffer(info, 10_000); + if (outIndex >= 0) { + ByteBuffer outBuf = codec.getOutputBuffer(outIndex); + byte[] chunk = new byte[info.size]; + outBuf.position(info.offset); + outBuf.get(chunk); + pcm.write(chunk); + codec.releaseOutputBuffer(outIndex, false); + if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + outputDone = true; + } + } + } + } finally { + extractor.release(); + if (codec != null) { + try { + codec.stop(); + } catch (Throwable ignored) { + } + codec.release(); + } + } + byte[] bytes = pcm.toByteArray(); + int count = bytes.length / 2; + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f; + } + return samples; + } + + /** Load a 16-bit mono PCM WAV into [-1, 1] floats. */ + private static float[] readWavMono16(String path) throws IOException { + File file = new File(path); + long fileLength = file.length(); + try (DataInputStream in = new DataInputStream(new FileInputStream(file))) { + byte[] header = new byte[44]; + in.readFully(header); + if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') { + throw new IOException("not a WAV file"); + } + int channels = (header[22] & 0xff) | (header[23] << 8); + int bits = (header[34] & 0xff) | (header[35] << 8); + if (channels != 1 || bits != 16) { + throw new IOException("only 16-bit mono WAV is supported"); + } + long dataBytes = fileLength - 44; + int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE); + float[] samples = new float[count]; + byte[] chunk = new byte[65536]; + int sampleIndex = 0; + int carried = 0; + byte carriedByte = 0; + int read; + while (sampleIndex < count && (read = in.read(chunk)) > 0) { + int offset = 0; + if (carried == 1) { + samples[sampleIndex++] = + ((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f; + offset = 1; + carried = 0; + } + int pairs = (read - offset) / 2; + for (int i = 0; i < pairs && sampleIndex < count; i++) { + int base = offset + i * 2; + samples[sampleIndex++] = + ((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f; + } + if (((read - offset) & 1) == 1) { + carried = 1; + carriedByte = chunk[read - 1]; + } + } + return samples; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java index 455d5bff..84d66a4f 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasher.java @@ -14,17 +14,23 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Executors; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; /** * Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA * firmware container to each lens over the firmware-data service using the - * aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol, with - * a heartbeat on the EvenHub control char to keep the session alive. + * aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol. + * + * Each lens connection completes the sid-0x80 security-auth exchange before + * OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing + * the transfer mid-component. No control-channel traffic is injected during + * the transfer itself: official OTA captures show none between BEGIN and the + * final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a + * concurrent control writer would interleave with the marker/data + * transactions. (See ../notes/ble-connections-2.2.9.md.) * * Lenses are flashed one at a time (left, then right). Finishing either lens * reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect @@ -40,7 +46,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { // OTA message types (envelope sid byte) and control opcodes. private static final int SID_CTRL = 0xc0; private static final int SID_DATA = 0xc1; - private static final int SID_HEARTBEAT = 0x80; private static final int FLAG_OTA = 0x00; private static final int OP_BEGIN = 0x00; private static final int OP_FILE_CHECK = 0x01; @@ -52,9 +57,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { private static final int CTRL_ACK_TIMEOUT_MS = 8_000; private static final int BLOCK_NAK_RETRIES = 3; private static final int COMPONENT_RETRIES = 3; - private static final int HEARTBEAT_INTERVAL_MS = 12_000; - private static final byte[] HEARTBEAT_PAYLOAD = - new byte[] {0x08, 0x0e, 0x10, 0x26, 0x6a, 0x00}; // Reconnect windows. The first lens connects from an idle device; the second // must ride out the post-first-lens reboot of both lenses. @@ -89,10 +91,12 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { private volatile FaceclawFirmwareFlasherListener listener; private volatile Thread worker; private volatile boolean cancelled = false; - private volatile ScheduledExecutorService heartbeatExecutor; - private volatile String heartbeatAddress; private int nextSeq = 0; + private int nextAuthMagic = 0x60; + private int pendingAuthMagic = -1; + private CountDownLatch authLatch; + private byte[] authAckPb; public FaceclawFirmwareFlasher(Context context, String rightAddress, String leftAddress, String firmwarePath) { this.context = context.getApplicationContext(); @@ -127,7 +131,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { public void close() { cancel(); - stopHeartbeat(); try { bleManager.close(); } catch (Exception ignored) { @@ -172,27 +175,34 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { private void flashLens(String lens, String address, byte[] img, List segs, int connectWindowMs) throws TimeoutException { - resetSeq(); emitState("connecting", lens); connectLensResilient(lens, address, connectWindowMs); sleepInterruptibly(NOTIFY_SETTLE_MS); drainAcks(); + // Fresh envelope counter for the OTA stream (the auth exchange during + // bring-up used its own); mirrors the stock sequence. + resetSeq(); emitState("flashing", lens); - startHeartbeat(address); - try { - int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS); - if (!isEndOk(beginStatus)) { - emitLog("warning: unexpected begin status " + beginStatus + "; continuing"); + int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS); + if (!isEndOk(beginStatus)) { + emitLog("warning: unexpected begin status " + beginStatus + "; continuing"); + } + // Progress is reported in bytes across the whole image: the segments + // are one large firmware blob plus several small ones, so a bar + // stepped per segment would sit still through the big one and then + // sprint through the rest. + long totalBytes = 0; + for (Segment seg : segs) { + totalBytes += seg.ps; + } + long bytesBefore = 0; + for (int i = 0; i < segs.size(); i++) { + if (cancelled) { + throw new IllegalStateException("Cancelled."); } - for (int i = 0; i < segs.size(); i++) { - if (cancelled) { - throw new IllegalStateException("Cancelled."); - } - flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img); - } - } finally { - stopHeartbeat(); + flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img, bytesBefore, totalBytes); + bytesBefore += segs.get(i).ps; } emitLog(lens + " lens: all components verified"); try { @@ -201,14 +211,15 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { } } - private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img) { + private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img, + long bytesBefore, long totalBytes) { for (int attempt = 0; attempt < COMPONENT_RETRIES; attempt++) { if (attempt > 0) { emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES); } int endStatus; try { - endStatus = flashComponent(lens, address, index, count, seg, img); + endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes); } catch (TimeoutException | RuntimeException e) { emitLog(seg.name + ": block phase failed: " + e.getMessage()); endStatus = -1; @@ -226,8 +237,8 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { throw new IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts"); } - private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img) - throws TimeoutException { + private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img, + long bytesBefore, long totalBytes) throws TimeoutException { byte[] sub = Arrays.copyOfRange(img, seg.off, seg.off + 128); int ps = seg.ps; int payloadStart = seg.off + 128; @@ -260,7 +271,8 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { throw new IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times"); } if (b % 20 == 0 || b == blockCount - 1) { - emitProgress(lens, index + 1, count, b + 1, blockCount); + long bytesSent = bytesBefore + Math.min((long) (b + 1) * BLOCK_SIZE, (long) ps); + emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes); } } emitLog(seg.name + ": data phase done; sending END"); @@ -356,50 +368,64 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) { return false; } + // Control-channel notify carries the auth response; mandatory. + if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { + return false; + } if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { return false; } - // Control-channel notify (heartbeat responses); best-effort. - bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS); + if (!authenticate(address)) { + throw new IllegalStateException( + "security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it"); + } return true; } - // ---- heartbeat ----------------------------------------------------------- - - private void startHeartbeat(String address) { - stopHeartbeat(); - heartbeatAddress = address; - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(runnable -> { - Thread t = new Thread(runnable, "faceclaw-flash-hb"); - t.setDaemon(true); - return t; - }); - heartbeatExecutor = executor; - executor.scheduleWithFixedDelay( - this::sendHeartbeat, HEARTBEAT_INTERVAL_MS, HEARTBEAT_INTERVAL_MS, TimeUnit.MILLISECONDS); - } - - private void sendHeartbeat() { - if (cancelled) { - return; - } - String address = heartbeatAddress; - if (address == null) { - return; + /** + * Complete the sid-0x80 security-auth exchange on this connection. On an + * unbonded phone this triggers SMP pairing (possibly with an OS prompt), + * so the wait is generous. Must succeed before any OTA traffic. + */ + private boolean authenticate(String address) { + int magic; + CountDownLatch latch = new CountDownLatch(1); + synchronized (lock) { + magic = nextAuthMagic; + nextAuthMagic = nextAuthMagic >= 0x7f ? 0x60 : nextAuthMagic + 1; + pendingAuthMagic = magic; + authAckPb = null; + authLatch = latch; } try { - writeOta(address, BleProtocol.WRITE_CHAR_UUID, SID_HEARTBEAT, HEARTBEAT_PAYLOAD, nextSeq()); - } catch (Exception e) { - Log.w(TAG, "heartbeat failed: " + e.getMessage()); - } - } - - private void stopHeartbeat() { - ScheduledExecutorService executor = heartbeatExecutor; - heartbeatExecutor = null; - heartbeatAddress = null; - if (executor != null) { - executor.shutdownNow(); + if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH, + BleProtocol.buildAuthenticationRequest(magic), nextSeq())) { + emitLog("auth write failed: " + address); + return false; + } + boolean signalled; + try { + signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + byte[] pb; + synchronized (lock) { + pb = authAckPb; + } + if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) { + emitLog("security auth not acknowledged (" + address + + ") — if Android shows a Bluetooth pairing request, accept it"); + return false; + } + emitLog("security auth complete: " + address); + return true; + } finally { + synchronized (lock) { + pendingAuthMagic = -1; + authLatch = null; + } } } @@ -497,8 +523,25 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { @Override public void onNotification(String address, String characteristicUuid, byte[] data) { + if (BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) { + // Control channel: only the security-auth response is awaited here. + // A value can carry several envelope frames back to back (e.g. the + // response packed with one of the lens's own sid-0x80 notifies). + for (byte[] buf : BleProtocol.splitFrames(data)) { + BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf); + if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) { + synchronized (lock) { + if (authLatch != null && frame.msgSeq == pendingAuthMagic) { + authAckPb = frame.pb; + authLatch.countDown(); + } + } + } + } + return; + } if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equalsIgnoreCase(characteristicUuid)) { - return; // acks arrive on the data-notify char; ignore heartbeat responses + return; // OTA acks arrive on the data-notify char } BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data); if (!frame.ok || frame.pb.length < 2) { @@ -517,7 +560,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { // ---- helpers ------------------------------------------------------------- private void teardown() { - stopHeartbeat(); try { bleManager.close(); } catch (Exception ignored) { @@ -621,11 +663,12 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener { }); } - private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount) { + private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount, + long bytesSent, long bytesTotal) { mainHandler.post(() -> { FaceclawFirmwareFlasherListener current = listener; if (current != null) { - current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount); + current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal); } }); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasherListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasherListener.java index a111c174..2cd24e20 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasherListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareFlasherListener.java @@ -6,9 +6,12 @@ public interface FaceclawFirmwareFlasherListener { /** * Fine-grained progress for the UI bar. `lens` is "left"/"right"; - * component/block indices are 1-based against their counts. + * component/block indices are 1-based against their counts. `bytesSent` / + * `bytesTotal` cover the whole lens image, so a bar driven by them moves + * evenly even though the components differ wildly in size. */ - void onProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount); + void onProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount, + long bytesSent, long bytesTotal); /** * Coarse lifecycle. `state` is one of: validating, connecting, flashing, diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareUtil.java index 31a43576..32ce5dc1 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareUtil.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFirmwareUtil.java @@ -11,24 +11,32 @@ import java.security.NoSuchAlgorithmException; * Native helpers for the on-device custom-firmware build: SHA-256 hashing and * binary file writing. Both take a java.nio.ByteBuffer so NativeScript can * marshal a JS ArrayBuffer straight across without a per-element copy of the - * ~4 MB firmware image (and with no overload ambiguity). + * ~4 MB firmware image (and with no overload ambiguity). Also home of the one + * shared byte→hex encoder for com.faceclaw.app. */ public final class FaceclawFirmwareUtil { private FaceclawFirmwareUtil() {} + /** Lowercase hex of the bytes. The TS-side decoder is case-insensitive. */ + public static String bytesToHex(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return ""; + } + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + int v = b & 0xff; + sb.append(Character.forDigit(v >>> 4, 16)); + sb.append(Character.forDigit(v & 0x0f, 16)); + } + return sb.toString(); + } + /** Lowercase hex SHA-256 of the buffer's remaining bytes. */ public static String sha256Hex(ByteBuffer data) { try { MessageDigest md = MessageDigest.getInstance("SHA-256"); md.update(data.duplicate()); - byte[] digest = md.digest(); - StringBuilder sb = new StringBuilder(digest.length * 2); - for (byte b : digest) { - int v = b & 0xff; - sb.append(Character.forDigit(v >>> 4, 16)); - sb.append(Character.forDigit(v & 0x0f, 16)); - } - return sb.toString(); + return bytesToHex(md.digest()); } catch (NoSuchAlgorithmException e) { throw new RuntimeException("SHA-256 unavailable", e); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java index 16454990..e7bce6ca 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFlashPromptCommunicator.java @@ -26,6 +26,10 @@ import java.util.concurrent.TimeUnit; * protobuf helpers (BleProtocol), but owns its own GATT connection and runs its * whole lifecycle on a single worker thread. It expects the main app to be * disconnected while it runs (onboarding, before flashing). + * + * Only the right arm is used: the lenses relay messages to each other, so + * anything sent to the right arm reaches both, and acks/events only ever come + * from the right arm anyway. */ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private static final String TAG = "FaceclawFlashPrompt"; @@ -43,13 +47,13 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private final Context context; private final String rightAddress; - private final String leftAddress; private final String warningText; private final FaceclawBleManager bleManager; private final Handler mainHandler = new Handler(Looper.getMainLooper()); private final Object lock = new Object(); private final ConcurrentHashMap pendingAcks = new ConcurrentHashMap<>(); + private final ConcurrentHashMap ackPayloads = new ConcurrentHashMap<>(); private final CountDownLatch selectionLatch = new CountDownLatch(1); private volatile FaceclawFlashPromptListener listener; @@ -57,7 +61,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private volatile boolean cancelled = false; private volatile boolean finished = false; private volatile boolean rightConnected = false; - private volatile boolean leftConnected = false; private volatile boolean rightLost = false; private volatile Boolean approved = null; private volatile ScheduledExecutorService heartbeatExecutor; @@ -65,10 +68,9 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { private int nextMagic = 100; private int nextSeq = 0x40; - public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String leftAddress, String warningText) { + public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String warningText) { this.context = context.getApplicationContext(); this.rightAddress = rightAddress == null ? "" : rightAddress; - this.leftAddress = leftAddress == null ? "" : leftAddress; this.warningText = warningText == null ? "" : warningText; this.bleManager = new FaceclawBleManager(this.context); this.bleManager.setListener(this); @@ -115,18 +117,8 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } emitState("connecting", ""); - connectArm(rightAddress, "right"); + connectArm(rightAddress); rightConnected = true; - - boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress); - if (haveLeft) { - try { - connectArm(leftAddress, "left"); - leftConnected = true; - } catch (Exception e) { - emitLog("left arm connect failed (continuing on right only): " + e.getMessage()); - } - } if (cancelled) { teardown(); return; @@ -134,14 +126,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { emitState("connected", ""); bringUpArm(rightAddress); - if (leftConnected) { - try { - bringUpArm(leftAddress); - } catch (Exception e) { - emitLog("left arm prompt failed (continuing on right only): " + e.getMessage()); - leftConnected = false; - } - } startHeartbeat(); emitState("prompting", ""); @@ -157,7 +141,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (approved != null) { boolean result = Boolean.TRUE.equals(approved); try { - sendShutdownBoth(); + sendShutdown(); } catch (Exception ignored) { } emitResult(result); @@ -181,7 +165,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } } - private void connectArm(String address, String labelForLog) { + private void connectArm(String address) { if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) { throw new IllegalStateException("connect failed: " + address); } @@ -193,42 +177,75 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) { throw new IllegalStateException("enableNotifications failed: " + address); } - emitLog("connected " + labelForLog + " arm"); + emitLog("connected right arm"); } private void bringUpArm(String address) throws InterruptedException { - // 1. Mandatory session prelude on sid=0x01 (app-launch). - if (!writeAndAwaitAck( + // 1. Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a + // session (and closes the link after ~30 s) without it, and on an + // unbonded phone this is what triggers SMP pairing, so it may sit + // waiting on an OS pairing prompt. + int authMagic = allocMagic(); + byte[] authAck = writeAndAwaitAck( + address, + BleProtocol.SID_SECURITY_AUTH, + BleProtocol.FLAG_SECURITY_AUTH, + authMagic, + BleProtocol.buildAuthenticationRequest(authMagic), + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS); + if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) { + throw new IllegalStateException( + "could not authenticate with the glasses (" + address + + ") — if Android shows a Bluetooth pairing request, accept it and try again"); + } + // 2. Mandatory session prelude on sid=0x01 (app-launch). + if (writeAndAwaitAck( address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST, BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD, - ConnectionOptions.PRELUDE_TIMEOUT_MS)) { + ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) { throw new IllegalStateException("session prelude not acked: " + address); } - // 2. Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. - int magic = allocMagic(); - byte[] page = BleProtocol.buildCreatePromptPage( - magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS); - if (!writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS)) { + // 3. Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. + // Two attempts: on 2.2.9 the first request sent right after the + // prelude can be dropped while the lens emits its own sid-0x80 + // notifications (observed with the device-info settings read). + byte[] pageAck = null; + for (int attempt = 0; attempt < 2 && pageAck == null && !cancelled; attempt++) { + int magic = allocMagic(); + byte[] page = BleProtocol.buildCreatePromptPage( + magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS); + pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS); + if (pageAck == null) { + emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address); + } + } + if (pageAck == null) { throw new IllegalStateException("prompt page not acked: " + address); } emitLog("prompt page shown on " + address); } - private boolean writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs) + /** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */ + private byte[] writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); String key = ackKey(sid, magic); pendingAcks.put(key, latch); try { if (!writeFrame(address, sid, flag, payload)) { - return false; + return null; } - return latch.await(timeoutMs, TimeUnit.MILLISECONDS); + if (!latch.await(timeoutMs, TimeUnit.MILLISECONDS)) { + return null; + } + byte[] pb = ackPayloads.get(key); + return pb == null ? new byte[0] : pb; } finally { pendingAcks.remove(key, latch); + ackPayloads.remove(key); } } @@ -267,9 +284,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (rightConnected) { writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat); } - if (leftConnected) { - writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat); - } } catch (Exception e) { Log.w(TAG, "heartbeat failed: " + e.getMessage()); } @@ -283,14 +297,11 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } } - private void sendShutdownBoth() { + private void sendShutdown() { byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0); if (rightConnected) { writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown); } - if (leftConnected) { - writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown); - } } private void teardown() { @@ -301,7 +312,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { } catch (Exception ignored) { } rightConnected = false; - leftConnected = false; } @Override @@ -309,20 +319,26 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) { return; } - BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data); + // A value can carry several envelope frames back to back. + for (byte[] buf : BleProtocol.splitFrames(data)) { + handleFrame(address, buf); + } + } + + private void handleFrame(String address, byte[] buf) { + BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf); if (!frame.ok) { return; } if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) { - // Async events (taps/selections) only ever come from the right arm. - if (address.equalsIgnoreCase(rightAddress)) { - handleEvent(frame); - } + handleEvent(frame); return; } if (frame.msgSeq >= 0) { - CountDownLatch latch = pendingAcks.get(ackKey(frame.sid, frame.msgSeq)); + String key = ackKey(frame.sid, frame.msgSeq); + CountDownLatch latch = pendingAcks.get(key); if (latch != null) { + ackPayloads.put(key, frame.pb); latch.countDown(); } } @@ -364,8 +380,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener { selectionLatch.countDown(); } } - } else if (address.equalsIgnoreCase(leftAddress)) { - leftConnected = false; } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java index ce9f0557..1ed6b950 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawLc3Decoder.java @@ -12,6 +12,12 @@ public class FaceclawLc3Decoder { public static final int COUNTER_OFFSET = 204; public static final int SAMPLES_PER_FRAME = 160; public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME; + // Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the + // glasses DSP appends two signed 16-bit LE values computed on the raw + // stereo capture before the mono downmix — a signal-strength ratio and the + // asin-derived direction-of-arrival in degrees — then the packet counter. + public static final int SSR_OFFSET = 200; + public static final int ANGLE_OFFSET = 202; static { System.loadLibrary("faceclaw_lc3"); @@ -23,6 +29,8 @@ public class FaceclawLc3Decoder { private long duplicatePackets; private long missingPackets; private long decodeErrors; + private int lastSsr; + private int lastAngleDegrees; public FaceclawLc3Decoder() { nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE); @@ -64,9 +72,21 @@ public class FaceclawLc3Decoder { } realPackets++; lastCounter = counter; + lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8)); + lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8)); return decoded; } + /** Firmware signal-strength ratio from the most recent decoded packet. */ + public synchronized int getLastSsr() { + return lastSsr; + } + + /** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */ + public synchronized int getLastAngleDegrees() { + return lastAngleDegrees; + } + public synchronized long getRealPackets() { return realPackets; } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMicStatusListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMicStatusListener.java new file mode 100644 index 00000000..e739ea34 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMicStatusListener.java @@ -0,0 +1,10 @@ +package com.faceclaw.app; + +public interface FaceclawMicStatusListener { + /** + * A CFW mic_control field-104 status record from one temple. body is the + * raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is + * "L" or "R" for the temple's own BLE link, "?" if unmatched. + */ + void onMicStatus(byte[] body, String arm); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java index 02f50bb5..c1764d6e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawModelDownloader.java @@ -144,7 +144,7 @@ public class FaceclawModelDownloader { + expectedTotalBytes + " bytes); try again to resume"); return; } - String actualSha = toHex(digest.digest()); + String actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest()); if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) { partFile.delete(); postError("Model download was corrupted (checksum mismatch); download it again"); @@ -177,12 +177,6 @@ public class FaceclawModelDownloader { return hashed; } - private static String toHex(byte[] bytes) { - StringBuilder sb = new StringBuilder(bytes.length * 2); - for (byte b : bytes) sb.append(String.format("%02x", b)); - return sb.toString(); - } - private void post(Runnable runnable) { callbackHandler.post(runnable); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawNoiseSuppressor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawNoiseSuppressor.java new file mode 100644 index 00000000..5d9514dc --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawNoiseSuppressor.java @@ -0,0 +1,266 @@ +package com.faceclaw.app; + +import android.media.audiofx.AcousticEchoCanceler; +import android.media.audiofx.AudioEffect; +import android.media.audiofx.AutomaticGainControl; +import android.media.audiofx.NoiseSuppressor; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Phone-side noise suppression for the microphone stream. + * + * Android's built-in noise cancellation (the {@link NoiseSuppressor} capture + * effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio + * arrives as raw PCM over BLE from the glasses, which that effect chain never + * sees, so this class picks the best engine that can actually reach the + * samples: + * + * - Audio captured by the phone itself: {@link #attachPlatformEffects(int)} + * enables the device's built-in NS/AEC/AGC chain (hardware or OEM software, + * whatever the Android build ships) on the AudioRecord session. + * - Audio from the glasses (the normal case): {@link #process(byte[])} runs a + * WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann + * frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an + * over-subtracted Wiener gain with a residual floor so speech never pumps + * in and out of silence. + */ +public final class FaceclawNoiseSuppressor { + + private static final int FFT_SIZE = 256; + private static final int HOP = FFT_SIZE / 2; + private static final float OVERSUBTRACTION = 1.6f; + /** About -16 dB of residual: keeps the noise bed natural instead of gated. */ + private static final float GAIN_FLOOR = 0.15f; + private static final float GAIN_SMOOTHING = 0.6f; + /** Frames of pure noise-floor learning after start/reset (~100 ms). */ + private static final int NOISE_WARMUP_FRAMES = 12; + + private final int sampleRate; + private final float[] window = new float[FFT_SIZE]; + private final float[] history = new float[FFT_SIZE]; + private final float[] overlap = new float[HOP]; + private final float[] noise = new float[FFT_SIZE / 2 + 1]; + private final float[] gain = new float[FFT_SIZE / 2 + 1]; + private final float[] re = new float[FFT_SIZE]; + private final float[] im = new float[FFT_SIZE]; + private final short[] pending = new short[HOP]; + private int pendingCount = 0; + private int warmupFrames = 0; + + public FaceclawNoiseSuppressor(int sampleRate) { + this.sampleRate = sampleRate; + // sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap. + for (int i = 0; i < FFT_SIZE; i++) { + window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE); + } + reset(); + } + + public int getSampleRate() { + return sampleRate; + } + + /** Forget the learned noise floor (e.g. after the suppressor was toggled off). */ + public void reset() { + Arrays.fill(history, 0f); + Arrays.fill(overlap, 0f); + Arrays.fill(noise, 0f); + Arrays.fill(gain, 1f); + pendingCount = 0; + warmupFrames = 0; + } + + /** + * Suppress noise in 16-bit little-endian mono PCM. Streaming: input not + * filling a whole 8 ms hop is carried into the next call, so the output + * may be up to one hop shorter than the input (and longer on a later + * call); total latency is one hop. + */ + public byte[] process(byte[] pcm16le) { + if (pcm16le == null || pcm16le.length < 2) { + return pcm16le; + } + int sampleCount = pcm16le.length / 2; + int hops = (pendingCount + sampleCount) / HOP; + byte[] out = new byte[hops * HOP * 2]; + int outPos = 0; + int inPos = 0; + while (inPos < sampleCount) { + int take = Math.min(HOP - pendingCount, sampleCount - inPos); + for (int i = 0; i < take; i++) { + int lo = pcm16le[(inPos + i) * 2] & 0xff; + int hi = pcm16le[(inPos + i) * 2 + 1]; + pending[pendingCount + i] = (short) ((hi << 8) | lo); + } + pendingCount += take; + inPos += take; + if (pendingCount == HOP) { + outPos = processHop(out, outPos); + pendingCount = 0; + } + } + return out; + } + + private int processHop(byte[] out, int outPos) { + System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP); + for (int i = 0; i < HOP; i++) { + history[FFT_SIZE - HOP + i] = pending[i] / 32768f; + } + for (int i = 0; i < FFT_SIZE; i++) { + re[i] = history[i] * window[i]; + im[i] = 0f; + } + fft(re, im, false); + + boolean warming = warmupFrames < NOISE_WARMUP_FRAMES; + if (warming) { + warmupFrames++; + } + for (int b = 0; b <= FFT_SIZE / 2; b++) { + float power = re[b] * re[b] + im[b] * im[b]; + if (warming) { + noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames; + } else if (power < noise[b]) { + // Track the floor down quickly; drift up slowly so speech + // energy never gets learned as noise. + noise[b] += (power - noise[b]) * 0.2f; + } else { + noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f); + } + float clean = power - OVERSUBTRACTION * noise[b]; + float g = clean > 0f ? clean / power : 0f; + if (g < GAIN_FLOOR) { + g = GAIN_FLOOR; + } + g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g; + gain[b] = g; + re[b] *= g; + im[b] *= g; + if (b > 0 && b < FFT_SIZE / 2) { + re[FFT_SIZE - b] *= g; + im[FFT_SIZE - b] *= g; + } + } + + fft(re, im, true); + for (int i = 0; i < HOP; i++) { + float value = (overlap[i] + re[i] * window[i]) * 32767f; + int sample = Math.round(value); + if (sample > 32767) { + sample = 32767; + } + if (sample < -32768) { + sample = -32768; + } + out[outPos + i * 2] = (byte) (sample & 0xff); + out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff); + } + for (int i = 0; i < HOP; i++) { + overlap[i] = re[HOP + i] * window[HOP + i]; + } + return outPos + HOP * 2; + } + + /** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */ + private static void fft(float[] re, float[] im, boolean inverse) { + int n = re.length; + for (int i = 1, j = 0; i < n; i++) { + int bit = n >> 1; + for (; (j & bit) != 0; bit >>= 1) { + j ^= bit; + } + j ^= bit; + if (i < j) { + float t = re[i]; + re[i] = re[j]; + re[j] = t; + t = im[i]; + im[i] = im[j]; + im[j] = t; + } + } + for (int len = 2; len <= n; len <<= 1) { + double ang = 2 * Math.PI / len * (inverse ? 1 : -1); + float wRe = (float) Math.cos(ang); + float wIm = (float) Math.sin(ang); + for (int i = 0; i < n; i += len) { + float curRe = 1f; + float curIm = 0f; + for (int k = 0; k < len / 2; k++) { + int a = i + k; + int b = a + len / 2; + float xr = re[b] * curRe - im[b] * curIm; + float xi = re[b] * curIm + im[b] * curRe; + re[b] = re[a] - xr; + im[b] = im[a] - xi; + re[a] += xr; + im[a] += xi; + float nextRe = curRe * wRe - curIm * wIm; + curIm = curRe * wIm + curIm * wRe; + curRe = nextRe; + } + } + } + if (inverse) { + for (int i = 0; i < n; i++) { + re[i] /= n; + im[i] /= n; + } + } + } + + /** Whether this Android build ships the built-in capture noise suppressor. */ + public static boolean platformSuppressorAvailable() { + try { + return NoiseSuppressor.isAvailable(); + } catch (Throwable t) { + return false; + } + } + + /** Short engine name for the UI. */ + public static String engineDescription() { + return "spectral (phone DSP)"; + } + + /** + * Attach the platform's built-in capture-effect chain (NoiseSuppressor, + * AcousticEchoCanceler, AutomaticGainControl — whichever this Android + * build provides) to a phone AudioRecord session. Only applicable when the + * phone itself captures the audio; the BLE glasses stream must go through + * {@link #process(byte[])} instead. The caller keeps the returned effects + * for the life of the recording and release()s them with it. + */ + public static AudioEffect[] attachPlatformEffects(int audioSessionId) { + ArrayList effects = new ArrayList<>(); + try { + if (NoiseSuppressor.isAvailable()) { + NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId); + if (ns != null) { + ns.setEnabled(true); + effects.add(ns); + } + } + if (AcousticEchoCanceler.isAvailable()) { + AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId); + if (aec != null) { + aec.setEnabled(true); + effects.add(aec); + } + } + if (AutomaticGainControl.isAvailable()) { + AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId); + if (agc != null) { + agc.setEnabled(true); + effects.add(agc); + } + } + } catch (Throwable t) { + // A missing effect just means this tier is unavailable. + } + return effects.toArray(new AudioEffect[0]); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java new file mode 100644 index 00000000..c258ad91 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSpeakerId.java @@ -0,0 +1,111 @@ +package com.faceclaw.app; + +import android.util.Log; + +import com.k2fsa.sherpa.onnx.OnlineStream; +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor; +import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig; + +/** + * Speaker voice-print extraction over sherpa-onnx. One embedding per + * utterance; matching/enrollment happens on the TS side (cosine similarity + * against the stored speaker profiles). + */ +public class FaceclawSpeakerId { + private static final String TAG = "FaceclawSpeakerId"; + + private SpeakerEmbeddingExtractor extractor; + private final String modelPath; + + public FaceclawSpeakerId(String modelPath) { + this.modelPath = modelPath; + } + + /** Lazily loads the model; returns false when it cannot be loaded. */ + public synchronized boolean ensureLoaded() { + if (extractor != null) { + return true; + } + try { + SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder() + .setModel(modelPath) + .setNumThreads(1) + .setDebug(false) + .setProvider("cpu") + .build(); + extractor = new SpeakerEmbeddingExtractor(config); + return true; + } catch (Throwable t) { + Log.w(TAG, "could not load speaker model at " + modelPath, t); + extractor = null; + return false; + } + } + + public synchronized int embeddingDim() { + return extractor == null ? 0 : extractor.getDim(); + } + + /** + * Compute an L2-normalized voice-print for one utterance of 16 kHz mono + * S16LE PCM. Returns null when the model is unavailable or the utterance + * is too short to embed (under ~0.5 s). + */ + public synchronized float[] embed(byte[] pcm16le, int sampleRate) { + if (pcm16le == null || pcm16le.length < sampleRate) { + return null; + } + if (!ensureLoaded()) { + return null; + } + int count = pcm16le.length / 2; + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8)); + samples[i] = s / 32768.0f; + } + OnlineStream stream = null; + try { + stream = extractor.createStream(); + stream.acceptWaveform(samples, sampleRate); + stream.inputFinished(); + if (!extractor.isReady(stream)) { + return null; + } + float[] embedding = extractor.compute(stream); + return l2Normalize(embedding); + } catch (Throwable t) { + Log.w(TAG, "embedding failed", t); + return null; + } finally { + if (stream != null) { + stream.release(); + } + } + } + + private static float[] l2Normalize(float[] v) { + if (v == null) { + return null; + } + double sum = 0; + for (float x : v) { + sum += (double) x * x; + } + double norm = Math.sqrt(sum); + if (norm <= 0) { + return v; + } + for (int i = 0; i < v.length; i++) { + v[i] = (float) (v[i] / norm); + } + return v; + } + + public synchronized void close() { + if (extractor != null) { + extractor.release(); + extractor = null; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 401808ca..1de06ca9 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -1,30 +1,20 @@ package com.faceclaw.app; import android.content.Context; -import android.content.res.AssetManager; import android.os.Handler; import android.os.Looper; import android.os.SystemClock; import android.util.Log; import com.k2fsa.sherpa.onnx.FeatureConfig; -import com.k2fsa.sherpa.onnx.KeywordSpotter; -import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; -import com.k2fsa.sherpa.onnx.KeywordSpotterResult; import com.k2fsa.sherpa.onnx.OfflineModelConfig; import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizer; import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; import com.k2fsa.sherpa.onnx.OfflineStream; -import com.k2fsa.sherpa.onnx.OnlineModelConfig; -import com.k2fsa.sherpa.onnx.OnlineStream; -import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.Arrays; @@ -57,17 +47,15 @@ public class FaceclawVoiceController { private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f; private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f; private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; - private static final String ASSET_ROOT = "faceclaw-voice"; - private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; - private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr"; + // Model directory shared with the TS-side download flow (asr-model.ts), + // which fetches the Moonshine files here on demand; they are no longer + // bundled in the APK. + private static final String ASR_ROOT = "faceclaw-voice-asr"; private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"; - private static final String[] MODEL_FILES = { - "encoder-epoch-12-avg-2-chunk-16-left-64.onnx", - "decoder-epoch-12-avg-2-chunk-16-left-64.onnx", - "joiner-epoch-12-avg-2-chunk-16-left-64.onnx", - "tokens.txt", - "screen-on-keywords.txt" - }; + // Model files for the retired on-phone wake-word spotter, copied to + // filesDir by earlier releases; deleted on sight to reclaim the space. + // (The wakeword is now detected by the glasses firmware itself.) + private static final String LEGACY_KWS_ROOT = "faceclaw-voice"; private static final String[] ASR_MODEL_FILES = { "encoder_model.ort", "decoder_model_merged.ort", @@ -75,7 +63,6 @@ public class FaceclawVoiceController { }; private enum VoiceInputMode { - WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired) ONBOARD, // on-phone Moonshine transcription CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side } @@ -89,10 +76,11 @@ public class FaceclawVoiceController { private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; - private VoiceInputMode mode = VoiceInputMode.WAKEWORD; - private KeywordSpotter keywordSpotter; + // Set once the worker has the glasses mic enabled for this session. + // Read and written under `lock`, so it flips with `started` atomically. + private boolean audioStarted; + private VoiceInputMode mode = VoiceInputMode.CLOUD; private OfflineRecognizer recognizer; - private OnlineStream stream; private FaceclawLc3Decoder lc3Decoder; private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES]; private int transcriptSampleCount; @@ -105,6 +93,26 @@ public class FaceclawVoiceController { private volatile boolean endpointing; private final EndpointDetector endpointDetector = new EndpointDetector(); private java.io.ByteArrayOutputStream recordingPcm; + // Speaker verification against the enrolled wearer voice-print ("my voice + // only" command gating). Configured before start(); the utterance PCM is + // buffered (capped) and verified once at session end. + private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10; + private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE; + private volatile String verifySpeakerModelPath; + private volatile float[] verifyWearerEmbedding; + private volatile float verifyThreshold = 0.8f; + private short[] verifyBuffer; + private int verifyCount; + // Global mic processing (Microphones app config): spectral noise + // suppression and firmware-DoA beam gating, applied to every capture + // session that opts in (assistant push-to-talk, Transcribe, hands-free). + // The raw tap opts out — raw means raw, and the Microphones session does + // its own beam-compensated processing on that path. + private volatile boolean suppressionEnabled; + private volatile boolean beamFilterEnabled; + private volatile int beamCenterDeg; + private volatile int beamHalfWidthDeg = 180; + private FaceclawNoiseSuppressor suppressor; private long queuedPackets; private long queueDroppedPackets; private long decodedSamples; @@ -140,8 +148,66 @@ public class FaceclawVoiceController { this.endpointing = endpointing; } - public void start() { - start("wakeword"); + /** + * Verify this session's speaker against the enrolled wearer voice-print + * and report the result via onSpeakerVerified just before the final + * transcript. Must be set before {@link #start}; pass a null model path + * to disable. + */ + public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) { + this.verifySpeakerModelPath = speakerModelPath; + this.verifyWearerEmbedding = wearerEmbedding; + this.verifyThreshold = threshold > 0 ? threshold : 0.8f; + } + + public void clearSpeakerVerification() { + this.verifySpeakerModelPath = null; + this.verifyWearerEmbedding = null; + } + + /** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */ + public void setNoiseSuppression(boolean enabled) { + this.suppressionEnabled = enabled; + } + + /** + * Direction gating from the Sonic Radar beam: packets whose firmware + * direction-of-arrival falls outside centerDeg ± halfWidthDeg (device + * frame, 0 = straight ahead, positive right) are dropped before any + * consumer sees them. Safe to update mid-run. + */ + public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) { + this.beamFilterEnabled = enabled; + this.beamCenterDeg = centerDeg; + this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg)); + } + + private boolean withinBeam(int angleDegrees) { + int delta = angleDegrees - beamCenterDeg; + while (delta > 180) delta -= 360; + while (delta < -180) delta += 360; + return Math.abs(delta) <= beamHalfWidthDeg; + } + + private void applySuppression(short[] pcm, int count) { + try { + if (suppressor == null) { + suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE); + } + byte[] le = new byte[count * 2]; + for (int i = 0; i < count; i++) { + le[i * 2] = (byte) (pcm[i] & 0xff); + le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff); + } + byte[] cleaned = suppressor.process(le); + int cleanedCount = Math.min(count, cleaned.length / 2); + for (int i = 0; i < cleanedCount; i++) { + pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8)); + } + } catch (Throwable t) { + Log.w(TAG, "noise suppression failed; passing audio through", t); + suppressionEnabled = false; + } } public void start(String requestedMode) { @@ -156,11 +222,36 @@ public class FaceclawVoiceController { } mode = parseMode(requestedMode); started = true; + audioStarted = false; workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); workerThread.start(); } } + /** + * Whether mic audio is actually flowing. {@link #start} only records + * intent: the enable lives in the glasses' EvenHub session, so a transport + * drop or a session suspend can leave this controller started with a + * worker that will never see another packet. Anything deciding whether to + * (re)start capture must ask this rather than assume its own bookkeeping. + */ + public boolean isCapturing() { + boolean audioUp; + synchronized (lock) { + if (!started) { + return false; + } + audioUp = audioStarted; + } + if (!audioUp) { + // The worker is still bringing the mic up; report it as running so + // a concurrent request shares it instead of restarting it. + return true; + } + FaceclawBleCommunicator currentCommunicator = communicator; + return currentCommunicator != null && currentCommunicator.isAudioCaptureActive(); + } + public void stop() { Thread threadToJoin; synchronized (lock) { @@ -194,41 +285,49 @@ public class FaceclawVoiceController { if ("cloud".equals(requestedMode)) { return VoiceInputMode.CLOUD; } - if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) { - return VoiceInputMode.ONBOARD; - } - return VoiceInputMode.WAKEWORD; + return VoiceInputMode.ONBOARD; } private void runLoop() { try { + deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; if (currentMode == VoiceInputMode.ONBOARD) { + File modelDir = findAsrModelDir(); + if (modelDir == null) { + emitStatus("Voice model not downloaded (see Settings > Voice)."); + return; + } emitStatus("Loading transcription model..."); - File modelDir = installAsrModelFiles(); recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); resetTranscriptState(); lastTranscript = ""; - } else if (currentMode == VoiceInputMode.WAKEWORD) { - emitStatus("Loading wake-word model..."); - File modelDir = installModelFiles(); - keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); - stream = keywordSpotter.createStream(); } lc3Decoder = new FaceclawLc3Decoder(); endpointDetector.reset(); + if (suppressor != null) { + suppressor.reset(); + } recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null; + boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null; + verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null; + verifyCount = 0; if (!startG2Audio()) { emitStatus("Could not start G2 microphone input."); return; } + synchronized (lock) { + audioStarted = true; + } emitStatus(currentMode == VoiceInputMode.CLOUD ? "Listening (cloud)..." - : currentMode == VoiceInputMode.ONBOARD - ? "Listening..." - : "Listening for \"screen on\"..."); + : "Listening..."); processG2Audio(); + // Verification result must precede the final transcript so the + // TS bridge can suppress a non-wearer command before it is acted + // on (the callbacks are posted in order to the main handler). + runSpeakerVerification(); // Button released / stop requested: emit one final full-utterance // transcript so the UI can freeze it. if (currentMode == VoiceInputMode.ONBOARD) { @@ -244,6 +343,7 @@ public class FaceclawVoiceController { releaseLc3(); synchronized (lock) { started = false; + audioStarted = false; workerThread = null; } } @@ -312,29 +412,6 @@ public class FaceclawVoiceController { return b.array(); } - private KeywordSpotterConfig buildConfig(File modelDir) { - return KeywordSpotterConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setOnlineModelConfig(OnlineModelConfig.builder() - .setTransducer(OnlineTransducerModelConfig.builder() - .setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setModelType("zipformer2") - .setModelingUnit("") - .setNumThreads(1) - .build()) - .setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath()) - .setKeywordsScore(1.5f) - .setKeywordsThreshold(0.35f) - .build(); - } - private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) { return OfflineRecognizerConfig.builder() .setFeatureConfig(FeatureConfig.builder() @@ -352,50 +429,42 @@ public class FaceclawVoiceController { .build(); } - private File installModelFiles() throws IOException { - File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR); - if (!modelDir.exists() && !modelDir.mkdirs()) { - throw new IOException("Could not create " + modelDir.getAbsolutePath()); - } - AssetManager assets = appContext.getAssets(); - for (String fileName : MODEL_FILES) { - copyAssetIfNeeded( - assets, - ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName, - new File(modelDir, fileName) - ); - } - return modelDir; - } - - private File installAsrModelFiles() throws IOException { - File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR); - if (!modelDir.exists() && !modelDir.mkdirs()) { - throw new IOException("Could not create " + modelDir.getAbsolutePath()); - } - AssetManager assets = appContext.getAssets(); + /** + * The Moonshine model directory, populated by the download flow in + * asr-model.ts (releases before 0.5.0 copied the same files out of the + * APK, so upgraded installs are already complete). Null when any file is + * missing, i.e. the model still needs to be downloaded. + */ + private File findAsrModelDir() { + File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR); for (String fileName : ASR_MODEL_FILES) { - copyAssetIfNeeded( - assets, - ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName, - new File(modelDir, fileName) - ); - } - return modelDir; - } - - private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException { - if (destination.exists() && destination.length() > 0) { - return; - } - try (InputStream input = assets.open(assetPath); - FileOutputStream output = new FileOutputStream(destination)) { - byte[] buffer = new byte[64 * 1024]; - int read; - while ((read = input.read(buffer)) >= 0) { - output.write(buffer, 0, read); + File file = new File(modelDir, fileName); + if (!file.exists() || file.length() == 0) { + return null; } } + return modelDir; + } + + private void deleteLegacyKwsFiles() { + try { + deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT)); + } catch (Throwable t) { + Log.w(TAG, "failed to delete legacy wake-word files", t); + } + } + + private static void deleteRecursively(File file) { + if (!file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); } private boolean startG2Audio() { @@ -413,9 +482,8 @@ public class FaceclawVoiceController { private void processG2Audio() { short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET]; while (started && !Thread.currentThread().isInterrupted()) { - OnlineStream currentStream = stream; FaceclawLc3Decoder currentDecoder = lc3Decoder; - if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) { + if (currentDecoder == null) { return; } @@ -430,48 +498,48 @@ public class FaceclawVoiceController { continue; } decodedSamples += count; + // Global mic processing from the Microphones app config: the + // beam filter drops packets whose firmware direction-of-arrival + // falls outside the listening wedge (isolating the aimed talker + // for every consumer, recognition included), and the spectral + // noise suppressor cleans what remains before it reaches the + // recognizer, cloud PCM, endpointing, or speaker verification. + int angleDegrees = currentDecoder.getLastAngleDegrees(); + int ssr = currentDecoder.getLastSsr(); + if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) { + emitFrameMeta(angleDegrees, ssr); + maybeEmitAudioStats(false); + continue; + } + if (suppressionEnabled) { + applySuppression(pcm, count); + } if (recordingPcm != null) { appendRecording(pcm, count); } + if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { + int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount); + System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied); + verifyCount += copied; + } if (endpointing && endpointDetector.accept(pcm, count)) { emitSpeechEnd(); } - if (mode == VoiceInputMode.CLOUD) { - emitPcm(pcm, count); - } else if (mode == VoiceInputMode.ONBOARD) { + // PCM and frame metadata flow in every mode so levels, recording, + // and the Microphones radar keep working alongside onboard ASR. + emitPcm(pcm, count); + emitFrameMeta(angleDegrees, ssr); + if (mode != VoiceInputMode.CLOUD) { float[] samples = new float[count]; for (int i = 0; i < count; i++) { samples[i] = pcm[i] / 32768.0f; } processRecognizer(samples); - } else { - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - currentStream.acceptWaveform(samples, SAMPLE_RATE); - processKeywordSpotter(currentStream); } maybeEmitAudioStats(false); } } - private void processKeywordSpotter(OnlineStream currentStream) { - KeywordSpotter currentSpotter = keywordSpotter; - if (currentSpotter == null) { - return; - } - while (currentSpotter.isReady(currentStream)) { - currentSpotter.decode(currentStream); - KeywordSpotterResult result = currentSpotter.getResult(currentStream); - String keyword = result == null ? "" : result.getKeyword(); - if (keyword != null && keyword.trim().length() > 0) { - currentSpotter.reset(currentStream); - emitWakeWord(keyword); - } - } - } - private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); long now = SystemClock.elapsedRealtime(); @@ -661,6 +729,83 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onPcm(le)); } + /** + * Embed the session's buffered utterance and compare it to the enrolled + * wearer voice-print. Fails open: a session too short to verify, or a + * model that will not load, counts as the wearer rather than silencing + * every command. + */ + private void runSpeakerVerification() { + short[] buffer = verifyBuffer; + float[] wearer = verifyWearerEmbedding; + String modelPath = verifySpeakerModelPath; + verifyBuffer = null; + if (buffer == null || wearer == null || modelPath == null) { + return; + } + if (verifyCount < VERIFY_MIN_SAMPLES) { + emitSpeakerVerified(true, 0f); + return; + } + FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath); + try { + byte[] le = new byte[verifyCount * 2]; + for (int i = 0; i < verifyCount; i++) { + short s = buffer[i]; + le[i * 2] = (byte) (s & 0xff); + le[i * 2 + 1] = (byte) ((s >> 8) & 0xff); + } + float[] embedding = speakerId.embed(le, SAMPLE_RATE); + if (embedding == null || embedding.length != wearer.length) { + emitSpeakerVerified(true, 0f); + return; + } + double dot = 0; + for (int i = 0; i < embedding.length; i++) { + dot += (double) embedding[i] * wearer[i]; + } + boolean isWearer = dot >= verifyThreshold; + Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot) + + " threshold=" + verifyThreshold + " isWearer=" + isWearer); + emitSpeakerVerified(isWearer, (float) dot); + } catch (Throwable t) { + Log.w(TAG, "speaker verification failed", t); + emitSpeakerVerified(true, 0f); + } + } + + // The 28 MB embedding model takes seconds to load; keep one instance + // across capture sessions so verification adds only the embed time. + private static FaceclawSpeakerId sharedSpeakerId; + private static String sharedSpeakerIdPath; + + private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) { + if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) { + if (sharedSpeakerId != null) { + sharedSpeakerId.close(); + } + sharedSpeakerId = new FaceclawSpeakerId(modelPath); + sharedSpeakerIdPath = modelPath; + } + return sharedSpeakerId; + } + + private void emitSpeakerVerified(boolean isWearer, float similarity) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity)); + } + + private void emitFrameMeta(int angleDegrees, int ssr) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr)); + } + private void emitSpeechEnd() { FaceclawVoiceControllerListener currentListener = listener; if (currentListener == null) { @@ -783,14 +928,6 @@ public class FaceclawVoiceController { } private void releaseSherpa() { - if (stream != null) { - stream.release(); - stream = null; - } - if (keywordSpotter != null) { - keywordSpotter.release(); - keywordSpotter = null; - } if (recognizer != null) { recognizer.release(); recognizer = null; @@ -893,14 +1030,6 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onStatus(status)); } - private void emitWakeWord(String keyword) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onWakeWord(keyword)); - } - private void emitTranscript(String text, boolean isFinal) { FaceclawVoiceControllerListener currentListener = listener; if (currentListener == null) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index bce73665..c60ec655 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -3,8 +3,6 @@ package com.faceclaw.app; public interface FaceclawVoiceControllerListener { void onStatus(String status); - void onWakeWord(String keyword); - /** * Best transcript of the current utterance so far. REPLACE semantics: text * is the complete transcript, not a delta — display it verbatim, replacing @@ -13,15 +11,34 @@ public interface FaceclawVoiceControllerListener { void onTranscript(String text, boolean isFinal); /** - * Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit - * little-endian PCM. Empty/absent in onboard mode. + * Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM. + * Emitted in every mode so raw-PCM consumers (levels, recording, captions) + * keep working while the onboard recognizer owns the transcript. */ void onPcm(byte[] pcm16le); + /** + * Per-packet DSP metadata the glasses firmware appends to each 50 ms audio + * packet, computed on the raw stereo capture before its mono downmix: the + * direction-of-arrival in signed degrees (0 = straight ahead, positive to + * the wearer's right) and a signal-strength ratio that serves as a + * confidence/energy proxy for the angle. + */ + void onFrameMeta(int angleDegrees, int ssr); + /** * The speaker stopped, in a hands-free session that has no button release * to end it. Only fires when endpointing was enabled for the session; see * FaceclawVoiceController.setEndpointing. */ void onSpeechEnd(); + + /** + * Result of speaker verification against the enrolled wearer voice-print, + * emitted once per capture session just before the final transcript. + * Only fires when verification was configured for the session; fails + * open (isWearer=true) when the utterance was too short to verify or the + * speaker model could not be loaded. + */ + void onSpeakerVerified(boolean isWearer, float similarity); } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWavRecorder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWavRecorder.java new file mode 100644 index 00000000..ed08d388 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWavRecorder.java @@ -0,0 +1,100 @@ +package com.faceclaw.app; + +import android.util.Log; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +/** + * Streaming WAV writer for conversation recordings: the header is written up + * front with zero sizes and patched on finish, so a crash mid-session leaves + * a recoverable file (data length can be derived from the file size). + */ +public class FaceclawWavRecorder { + private static final String TAG = "FaceclawWavRecorder"; + + private final String path; + private final int sampleRate; + private final int channels; + private RandomAccessFile file; + private long dataBytes; + + public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException { + this.path = path; + this.sampleRate = sampleRate; + this.channels = channels; + File parent = new File(path).getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IOException("could not create " + parent); + } + file = new RandomAccessFile(path, "rw"); + file.setLength(0); + file.write(buildHeader(0)); + } + + public synchronized void append(byte[] pcm16le) { + if (file == null || pcm16le == null || pcm16le.length == 0) { + return; + } + try { + file.write(pcm16le); + dataBytes += pcm16le.length; + } catch (IOException e) { + Log.w(TAG, "append failed", e); + } + } + + public synchronized long getDurationMs() { + return dataBytes * 1000L / (sampleRate * channels * 2L); + } + + public String getPath() { + return path; + } + + /** Patch the header and close. Returns the PCM byte count (0 = empty). */ + public synchronized long finish() { + RandomAccessFile current = file; + file = null; + if (current == null) { + return dataBytes; + } + try { + current.seek(0); + current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE))); + current.close(); + } catch (IOException e) { + Log.w(TAG, "finish failed", e); + } + return dataBytes; + } + + /** Delete the (finished or abandoned) recording. */ + public synchronized void discard() { + finish(); + new File(path).delete(); + } + + private byte[] buildHeader(int pcmBytes) { + int byteRate = sampleRate * channels * 2; + ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN); + b.put("RIFF".getBytes(StandardCharsets.US_ASCII)); + b.putInt(36 + pcmBytes); + b.put("WAVE".getBytes(StandardCharsets.US_ASCII)); + b.put("fmt ".getBytes(StandardCharsets.US_ASCII)); + b.putInt(16); + b.putShort((short) 1); + b.putShort((short) channels); + b.putInt(sampleRate); + b.putInt(byteRate); + b.putShort((short) (channels * 2)); + b.putShort((short) 16); + b.put("data".getBytes(StandardCharsets.US_ASCII)); + b.putInt(pcmBytes); + return b.array(); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java new file mode 100644 index 00000000..8e8f96ef --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java @@ -0,0 +1,383 @@ +package com.faceclaw.app; + +import android.content.Context; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import com.google.android.gms.common.ConnectionResult; +import com.google.android.gms.common.GoogleApiAvailability; +import com.google.android.gms.wearable.CapabilityClient; +import com.google.android.gms.wearable.CapabilityInfo; +import com.google.android.gms.wearable.DataMap; +import com.google.android.gms.wearable.MessageEvent; +import com.google.android.gms.wearable.Node; +import com.google.android.gms.wearable.PutDataMapRequest; +import com.google.android.gms.wearable.PutDataRequest; +import com.google.android.gms.wearable.Wearable; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * Phone side of the Wear OS integration: the single place that talks to the + * Wearable Data Layer (Google Play services), so the JS side only ever sees + * JSON strings and never touches Play services types. + * + * Inbound: FaceclawWearListenerService hands every watch message to + * {@link #handleMessage}; it is forwarded to the JS listener on the thread + * that registered it (the main isolate), mirroring FaceclawSettings. Messages + * that arrive before a listener exists (the watch poked a phone whose JS + * dashboard is not up yet) are acked with jsReady=false so the watch can say + * "open Faceclaw on your phone"; only idempotent state requests are kept for + * replay on registration — the watch already reported gestures and commands + * as failed, and executing them tens of seconds later would act on whatever + * the glasses happen to show by then. + * + * Outbound: {@link #publishState} mirrors the dashboard state into a Data + * Layer item (delivered even if the watch is out of range right now, and + * available to the watch app the moment it opens), while {@link #sendToWatch} + * sends fire-and-forget messages (acks, assistant text) to every reachable + * watch that advertises the Faceclaw watch-app capability. + * + * Everything is a no-op on phones without Google Play services. + */ +public final class FaceclawWearBridge { + private static final String TAG = "FaceclawWear"; + + /** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */ + public static final String CAPABILITY_WATCH = "faceclaw_watch"; + /** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */ + public static final String CAPABILITY_PHONE = "faceclaw_phone"; + + public static final String PATH_PREFIX = "/faceclaw"; + public static final String PATH_STATE = "/faceclaw/state"; + public static final String PATH_STATE_REQUEST = "/faceclaw/state/request"; + public static final String PATH_ACK = "/faceclaw/ack"; + public static final String PATH_EVENT = "/faceclaw/event"; + + private static final String STATE_KEY_JSON = "json"; + private static final String STATE_KEY_UPDATED_AT = "updatedAt"; + + /** Messages kept for a listener that is not registered yet. */ + private static final int PENDING_LIMIT = 16; + private static final long PENDING_MAX_AGE_MS = 30_000; + + private static volatile FaceclawWearBridge instance; + + private final Context context; + private final boolean available; + private final Object lock = new Object(); + private FaceclawWearListener listener; + private Handler listenerHandler; + private final ArrayDeque pending = new ArrayDeque<>(); + private final List watchNodes = new ArrayList<>(); + private String lastPublishedState = null; + + private static final class PendingMessage { + final String path; + final String json; + final String nodeId; + final long receivedAt; + + PendingMessage(String path, String json, String nodeId, long receivedAt) { + this.path = path; + this.json = json; + this.nodeId = nodeId; + this.receivedAt = receivedAt; + } + } + + private FaceclawWearBridge(Context context) { + this.context = context.getApplicationContext(); + boolean playServices; + try { + playServices = GoogleApiAvailability.getInstance() + .isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS; + } catch (Throwable error) { + Log.w(TAG, "Play services availability check failed", error); + playServices = false; + } + this.available = playServices; + if (!available) { + Log.i(TAG, "Google Play services unavailable; watch integration disabled"); + } + } + + /** Initialize (idempotent) and return the singleton. */ + public static FaceclawWearBridge getInstance(Context context) { + if (instance == null) { + synchronized (FaceclawWearBridge.class) { + if (instance == null) { + instance = new FaceclawWearBridge(context); + } + } + } + return instance; + } + + /** Whether the Wearable Data Layer can be used on this phone at all. */ + public boolean isAvailable() { + return available; + } + + /** + * Register the JS listener. Must be called from the thread whose isolate + * owns it; that thread's Looper is captured for dispatch. Replays any + * recently queued watch messages and refreshes the reachable-watch set. + */ + public void setListener(FaceclawWearListener newListener) { + Looper looper = Looper.myLooper(); + if (looper == null) { + Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified"); + } + List replay; + synchronized (lock) { + listener = newListener; + listenerHandler = looper != null ? new Handler(looper) : null; + replay = new ArrayList<>(pending); + pending.clear(); + } + long now = System.currentTimeMillis(); + for (PendingMessage message : replay) { + if (now - message.receivedAt <= PENDING_MAX_AGE_MS) { + dispatchMessage(message.path, message.json, message.nodeId); + } + } + refreshWatchNodes(); + } + + public void clearListener() { + synchronized (lock) { + listener = null; + listenerHandler = null; + } + } + + public boolean isWatchReachable() { + synchronized (lock) { + return !watchNodes.isEmpty(); + } + } + + /** Display name of a reachable watch ("" if none). */ + public String getWatchName() { + synchronized (lock) { + return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); + } + } + + /** Re-query which watches advertise the Faceclaw watch-app capability. */ + public void refreshWatchNodes() { + if (!available) return; + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener(this::updateWatchNodes) + .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); + } catch (Throwable error) { + Log.w(TAG, "capability query failed", error); + } + } + + /** + * Mirror the dashboard state to the watch. Identical consecutive states + * are not re-sent (the Data Layer would drop them anyway). + */ + public void publishState(String json) { + publishState(json, false); + } + + /** As {@link #publishState(String)}; `force` re-sends an unchanged state. */ + public void publishState(String json, boolean force) { + if (!available || json == null) return; + synchronized (lock) { + if (!force && json.equals(lastPublishedState)) return; + lastPublishedState = json; + } + try { + PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE); + DataMap map = request.getDataMap(); + map.putString(STATE_KEY_JSON, json); + map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis()); + PutDataRequest put = request.asPutDataRequest().setUrgent(); + Wearable.getDataClient(context) + .putDataItem(put) + .addOnFailureListener(error -> Log.w(TAG, "state publish failed", error)); + } catch (Throwable error) { + Log.w(TAG, "state publish failed", error); + } + } + + /** Send a message to every reachable watch running the Faceclaw watch app. */ + public void sendToWatch(String path, String json) { + if (!available) return; + List targets; + synchronized (lock) { + targets = new ArrayList<>(watchNodes); + } + if (targets.isEmpty()) { + // Nothing known yet (first message after boot): look the watch up, then send. + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener(info -> { + updateWatchNodes(info); + for (Node node : info.getNodes()) { + sendToNode(node.getId(), path, json); + } + }) + .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); + } catch (Throwable error) { + Log.w(TAG, "capability query failed", error); + } + return; + } + for (Node node : targets) { + sendToNode(node.getId(), path, json); + } + } + + /** Send a message to one specific node (e.g. an ack back to the sender). */ + public void sendToNode(String nodeId, String path, String json) { + if (!available || nodeId == null || nodeId.isEmpty()) return; + try { + byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8); + Wearable.getMessageClient(context) + .sendMessage(nodeId, path, payload) + .addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error)); + } catch (Throwable error) { + Log.w(TAG, "send " + path + " failed", error); + } + } + + /** Reply to a watch message; `seq` echoes the watch's sequence number. */ + public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) { + JSONObject ack = new JSONObject(); + try { + ack.put("seq", seq); + ack.put("ok", ok); + ack.put("jsReady", jsReady); + ack.put("message", message != null ? message : ""); + } catch (JSONException ignored) { + // The keys are constants; this cannot happen. + } + sendToNode(nodeId, PATH_ACK, ack.toString()); + } + + /** Entry point for FaceclawWearListenerService; may run on any thread. */ + void handleMessage(MessageEvent event) { + if (event == null) return; + String path = event.getPath(); + if (path == null || !path.startsWith(PATH_PREFIX)) return; + byte[] data = event.getData(); + String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}"; + String nodeId = event.getSourceNodeId(); + Log.d(TAG, "recv " + path + " seq=" + readSeq(json)); + + boolean delivered = dispatchMessage(path, json, nodeId); + if (delivered) return; + + // No JS listener yet: tell the watch so it can explain why nothing is + // happening. Only state requests are kept for replay (see class doc); + // everything else is dropped after the failure ack. + if (PATH_STATE_REQUEST.equals(path)) { + synchronized (lock) { + long now = System.currentTimeMillis(); + while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) { + pending.pollFirst(); + } + if (pending.size() >= PENDING_LIMIT) { + pending.pollFirst(); + } + pending.addLast(new PendingMessage(path, json, nodeId, now)); + } + } + sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses."); + } + + /** Entry point for FaceclawWearListenerService's capability callback. */ + void handleCapabilityChanged(CapabilityInfo info) { + if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return; + updateWatchNodes(info); + } + + private void updateWatchNodes(CapabilityInfo info) { + Set nodes = info != null ? info.getNodes() : Collections.emptySet(); + // Every reachable watch gets messages; a nearby (Bluetooth-linked) + // one is listed first so it is the name the phone UI shows. + List reachable = new ArrayList<>(); + for (Node node : nodes) { + if (node.isNearby()) reachable.add(0, node); else reachable.add(node); + } + boolean changed; + String name; + synchronized (lock) { + changed = !sameNodeIds(watchNodes, reachable); + watchNodes.clear(); + watchNodes.addAll(reachable); + name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); + } + if (changed) { + Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name); + boolean reachableNow = !reachable.isEmpty(); + FaceclawWearListener target; + Handler handler; + synchronized (lock) { + target = listener; + handler = listenerHandler; + } + if (target != null && handler != null) { + handler.post(() -> { + try { + target.onWatchConnection(reachableNow, name); + } catch (Exception error) { + Log.w(TAG, "watch connection listener failed", error); + } + }); + } + } + } + + private static boolean sameNodeIds(List a, List b) { + if (a.size() != b.size()) return false; + for (int i = 0; i < a.size(); i++) { + if (!a.get(i).getId().equals(b.get(i).getId())) return false; + } + return true; + } + + /** Post to the JS listener; false when there is none to post to. */ + private boolean dispatchMessage(String path, String json, String nodeId) { + FaceclawWearListener target; + Handler handler; + synchronized (lock) { + target = listener; + handler = listenerHandler; + } + if (target == null || handler == null) return false; + handler.post(() -> { + try { + target.onMessage(path, json, nodeId); + } catch (Exception error) { + Log.w(TAG, "wear message listener failed for " + path, error); + } + }); + return true; + } + + private static long readSeq(String json) { + try { + return new JSONObject(json).optLong("seq", 0); + } catch (JSONException error) { + return 0; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java new file mode 100644 index 00000000..590c87d6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java @@ -0,0 +1,19 @@ +package com.faceclaw.app; + +/** + * Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every + * call is delivered on the thread that registered the listener (see + * FaceclawWearBridge.setListener). + */ +public interface FaceclawWearListener { + /** + * A message from the watch app. `path` is the Data Layer message path + * (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON + * object, "{}" when the watch sent none); `nodeId` identifies the watch + * so a reply can be addressed to it. + */ + void onMessage(String path, String json, String nodeId); + + /** The set of reachable watches running the Faceclaw watch app changed. */ + void onWatchConnection(boolean reachable, String watchName); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java new file mode 100644 index 00000000..7520d780 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java @@ -0,0 +1,26 @@ +package com.faceclaw.app; + +import android.util.Log; + +import com.google.android.gms.wearable.CapabilityInfo; +import com.google.android.gms.wearable.MessageEvent; +import com.google.android.gms.wearable.WearableListenerService; + +/** + * Play services starts this service (in the app process) for every Data + * Layer message under /faceclaw and for capability changes, whether or not + * the app is running. It only forwards to FaceclawWearBridge, which owns the + * hand-off to the JS side. + */ +public class FaceclawWearListenerService extends WearableListenerService { + @Override + public void onMessageReceived(MessageEvent event) { + Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId()); + FaceclawWearBridge.getInstance(this).handleMessage(event); + } + + @Override + public void onCapabilityChanged(CapabilityInfo info) { + FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java index fd1de064..4b599d67 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.java @@ -15,13 +15,18 @@ public final class PreviewBitmapUtil { private PreviewBitmapUtil() {} public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) { + return fromGray(gray, width, height, brightenGamma, false); + } + + /** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */ + public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) { if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { throw new IllegalArgumentException("invalid gray preview buffer"); } int[] lut = new int[256]; for (int g = 0; g < 256; g++) { int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))); - lut[g] = 0xff000000 | (v << 16) | (v << 8) | v; + lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v); } byte[] bytes = new byte[width * height]; gray.get(bytes); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java index 91c89e73..b5660a5b 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/BleProtocol.java @@ -28,6 +28,16 @@ public class BleProtocol { public static final int PRELUDE_ACK_SID = 0x01; public static final int PRELUDE_ACK_MAGIC = 156; + /** + * DEVICE_SETTINGS security-auth channel. Firmware 2.2.9 enforces completing + * this exchange shortly after a GATT connection opens (a ~30 s deadline + * closes unauthenticated links) and gates query responses on it; the + * message itself exists back to the 2.2.4 era, so sending it is safe on all + * stock versions. See ../notes/ble-connections-2.2.9.md. + */ + public static final int SID_SECURITY_AUTH = 0x80; + public static final int FLAG_SECURITY_AUTH = 0x00; + private static final int SECURITY_AUTH_CMD = 4; public static final int SID_APP_LAUNCH = 0x01; public static final int SID_EVENHUB = 0xe0; public static final int SID_UI_SETTING = 0x09; @@ -39,6 +49,14 @@ public class BleProtocol { // protobuf fields to stock implementations, so they are safely ignored. public static final int FACECLAW_WAKE_CONTROL_FIELD = 101; public static final int FACECLAW_WAKE_EVENT_FIELD = 102; + /** + * CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host + * writes an ['M','C',ver,op,...] record as settings field 103; each temple + * reports a 21-byte ['M','C',ver,...] status as field 104, both appended + * to settings read responses and as standalone commandId=3 pushes. + */ + public static final int FACECLAW_MIC_CONTROL_FIELD = 103; + public static final int FACECLAW_MIC_STATUS_FIELD = 104; public static final int FACECLAW_WAKE_OP_ACQUIRE = 1; public static final int FACECLAW_WAKE_OP_RELEASE = 2; public static final int FACECLAW_WAKE_OP_CLAIM = 3; @@ -323,6 +341,48 @@ public class BleProtocol { )); } + /** + * The application authentication request: DEVICE_SETTINGS/AUTHENTICATION(4) + * on sid 0x80, flag 0x00. Carries no key material — it arms the firmware's + * "security auth" flag; the firmware sends the success notification once + * the flag is set AND the BLE link is encrypted (SMP pairing/bond + * restoration, handled by the OS below GATT, possibly after a pairing + * prompt). Protobuf: f1=4 (command), f2=magic, f3={f1=1, f2=4}. + */ + public static byte[] buildAuthenticationRequest(int magic) { + byte[] auth = concat(CollectionUtils.listOf( + encodeVarintField(1, 1), + encodeVarintField(2, SECURITY_AUTH_CMD) + )); + return concat(CollectionUtils.listOf( + encodeVarintField(1, SECURITY_AUTH_CMD), + encodeVarintField(2, magic), + encodeMessageField(3, auth) + )); + } + + /** + * True when `pb` (trailing CRC still attached) is the success notification + * for the authentication request sent with `magic`: command 4, the same + * magic, and an EMPTY field-3 result message (`1a 00` — the omitted result + * code defaults to zero). A GATT write completing is NOT success; only this + * notification is. + */ + public static boolean isAuthenticationSuccess(byte[] pb, int magic) { + if (pb == null) { + return false; + } + byte[] root = stripTrailingCrc(pb); + if (readVarintFieldValue(root, 1, -1) != SECURITY_AUTH_CMD) { + return false; + } + if (readVarintFieldValue(root, 2, -1) != magic) { + return false; + } + byte[] result = readFieldBytes(root, 3); + return result != null && result.length == 0; + } + public static byte[] buildSettingsQuery(int magic) { byte[] request = encodeVarintField(1, 1); return concat(CollectionUtils.listOf( @@ -353,6 +413,40 @@ public class BleProtocol { )); } + /** + * Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a + * settings write carrying CFW field 103. Magic zero makes it + * fire-and-forget like the wake lease: the firmware consumes the field + * before its stock nanopb decoder discards the unknown tag, and the + * "ack" is the field-104 status notify that follows. + */ + public static byte[] buildFaceclawMicControl(byte[] record) { + return concat(CollectionUtils.listOf( + encodeVarintField(1, 1), + encodeVarintField(2, 0), + encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record) + )); + } + + /** + * Extract the CFW mic status record (field 104) from a settings frame, or + * null when absent. The 21-byte body layout is decoded on the TS side; + * here we only validate the 'M','C',version prelude. + */ + public static byte[] parseFaceclawMicStatus(byte[] pb) { + if (pb == null) { + return null; + } + byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD); + if (body == null + || body.length < 3 + || body[0] != 'M' + || body[1] != 'C') { + return null; + } + return body; + } + /** * Return the uint16 wake nonce from a CFW field-102 notification, or -1 * when this is an ordinary settings frame. @@ -552,6 +646,35 @@ public class BleProtocol { } + /** + * Split one notification value into individual `aa21`/`aa12` envelope + * frames. A value normally holds exactly one frame, but the firmware can + * pack several (each self-describing via the length byte at offset 3), and + * a frame that only reads the first would silently drop the rest. A value + * that does not start with an envelope, or a trailing partial frame + * (truncated by the stack — observed on a Samsung tablet for values over + * 64 bytes), is returned as-is so the caller can log it. + */ + public static List splitFrames(byte[] value) { + List out = new ArrayList<>(); + if (value == null || value.length == 0) { + return out; + } + int offset = 0; + while (offset + 8 <= value.length + && value[offset] == (byte) 0xaa + && (value[offset + 1] == 0x21 || value[offset + 1] == 0x12)) { + int len = value[offset + 3] & 0xff; + int end = Math.min(value.length, offset + 8 + len); + out.add(Arrays.copyOfRange(value, offset, end)); + offset = end; + } + if (offset < value.length) { + out.add(Arrays.copyOfRange(value, offset, value.length)); + } + return out; + } + public static ParsedFrame parseFrame(byte[] buf) { if (buf == null || buf.length < 10 || buf[0] != (byte) 0xaa || (buf[1] != 0x21 && buf[1] != 0x12)) { return new ParsedFrame(false, 0, 0, new byte[0], -1, -1); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java index 6656f9a6..a358dba9 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/ConnectionOptions.java @@ -12,6 +12,14 @@ public class ConnectionOptions { public static final int RING_RECONNECT_DELAY_MS = 2_000; public static final int WRITE_TIMEOUT_MS = 2_000; public static final int PRELUDE_TIMEOUT_MS = 2_000; + // Application security-auth exchange (sid 0x80). The success notification + // only arrives once the BLE link is encrypted, which on a first-time + // connection means SMP pairing — possibly with an OS dialog the user has + // to accept — so the stock-firmware paths wait generously. + public static final int SECURITY_AUTH_TIMEOUT_MS = 30_000; + // The main communicator's wait is soft (it proceeds on timeout so firmware + // that never answers can't stall every reconnect), so it is kept short. + public static final int SECURITY_AUTH_SOFT_TIMEOUT_MS = 6_000; public static final int ACK_TIMEOUT_MS = 3_500; public static final int HEARTBEAT_FAILURE_DEADLINE_MS = 10_000; public static final int HEARTBEAT_READY_MS = 4_000; diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java index 14ed0271..82a7ca44 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/g2protocol/MessageBuilder.java @@ -23,6 +23,22 @@ public class MessageBuilder { ); } + /** The per-connection sid-0x80 security-auth request (one per arm). */ + public OutboundMessage securityAuth(boolean leftArm) { + int magic = magicPool.allocate(); + return new OutboundMessage( + "security-auth", + "security auth " + (leftArm ? "L" : "R"), + BleProtocol.SID_SECURITY_AUTH, + BleProtocol.FLAG_SECURITY_AUTH, + magic, + BleProtocol.buildAuthenticationRequest(magic), + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS, + -1, + leftArm + ); + } + public OutboundMessage shutdown(int exitMode) { int magic = magicPool.allocate(); return new OutboundMessage( @@ -251,6 +267,21 @@ public class MessageBuilder { ); } + /** CFW mic_control write (settings field 103) targeted at one temple. */ + public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) { + return new OutboundMessage( + "mic-control", + "mic control " + label + (leftArm ? " L" : " R"), + BleProtocol.SID_UI_SETTING, + BleProtocol.FLAG_REQUEST, + 0, + BleProtocol.buildFaceclawMicControl(record), + ACK_TIMEOUT_MS, + -1, + leftArm + ); + } + public OutboundMessage faceclawWearQuery(boolean leftArm) { return new OutboundMessage( "wear-query-control", diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FastClusteringConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FastClusteringConfig.java new file mode 100644 index 00000000..f2e95725 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FastClusteringConfig.java @@ -0,0 +1,44 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class FastClusteringConfig { + private final int numClusters; + private final float threshold; + + private FastClusteringConfig(Builder builder) { + this.numClusters = builder.numClusters; + this.threshold = builder.threshold; + } + + public static Builder builder() { + return new Builder(); + } + + public int getNumClusters() { + return numClusters; + } + + public float getThreshold() { + return threshold; + } + + public static class Builder { + private int numClusters = -1; + private float threshold = 0.5f; + + public FastClusteringConfig build() { + return new FastClusteringConfig(this); + } + + public Builder setNumClusters(int numClusters) { + this.numClusters = numClusters; + return this; + } + + public Builder setThreshold(float threshold) { + this.threshold = threshold; + return this; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java deleted file mode 100644 index 836a07c2..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotter { - private long ptr; - - public KeywordSpotter(KeywordSpotterConfig config) { - System.loadLibrary("onnxruntime"); - System.loadLibrary("sherpa-onnx-jni"); - ptr = newFromFile(config); - if (ptr == 0) { - throw new IllegalArgumentException("Invalid KeywordSpotterConfig"); - } - } - - public OnlineStream createStream() { - return createStream(""); - } - - public OnlineStream createStream(String keywords) { - return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords)); - } - - public boolean isReady(OnlineStream stream) { - return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr()); - } - - public void decode(OnlineStream stream) { - decode(ptr, stream.getPtr()); - } - - public void reset(OnlineStream stream) { - reset(ptr, stream.getPtr()); - } - - public KeywordSpotterResult getResult(OnlineStream stream) { - return getResult(ptr, stream.getPtr()); - } - - public void release() { - if (ptr == 0) { - return; - } - delete(ptr); - ptr = 0; - } - - @Override - protected void finalize() throws Throwable { - release(); - super.finalize(); - } - - private native long newFromFile(KeywordSpotterConfig config); - - private native void delete(long ptr); - - private native long createStream(long ptr, String keywords); - - private native boolean isReady(long ptr, long streamPtr); - - private native void decode(long ptr, long streamPtr); - - private native void reset(long ptr, long streamPtr); - - private native KeywordSpotterResult getResult(long ptr, long streamPtr); -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java deleted file mode 100644 index 3c4ce6f3..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotterConfig { - private final FeatureConfig featConfig; - private final OnlineModelConfig modelConfig; - private final int maxActivePaths; - private final String keywordsFile; - private final float keywordsScore; - private final float keywordsThreshold; - private final int numTrailingBlanks; - - private KeywordSpotterConfig(Builder builder) { - this.featConfig = builder.featConfig; - this.modelConfig = builder.modelConfig; - this.maxActivePaths = builder.maxActivePaths; - this.keywordsFile = builder.keywordsFile; - this.keywordsScore = builder.keywordsScore; - this.keywordsThreshold = builder.keywordsThreshold; - this.numTrailingBlanks = builder.numTrailingBlanks; - } - - public static Builder builder() { - return new Builder(); - } - - public FeatureConfig getFeatConfig() { - return featConfig; - } - - public OnlineModelConfig getModelConfig() { - return modelConfig; - } - - public int getMaxActivePaths() { - return maxActivePaths; - } - - public String getKeywordsFile() { - return keywordsFile; - } - - public float getKeywordsScore() { - return keywordsScore; - } - - public float getKeywordsThreshold() { - return keywordsThreshold; - } - - public int getNumTrailingBlanks() { - return numTrailingBlanks; - } - - public static class Builder { - private FeatureConfig featConfig = FeatureConfig.builder().build(); - private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build(); - private int maxActivePaths = 4; - private String keywordsFile = "keywords.txt"; - private float keywordsScore = 1.5f; - private float keywordsThreshold = 0.25f; - private int numTrailingBlanks = 2; - - public Builder setFeatureConfig(FeatureConfig featConfig) { - this.featConfig = featConfig; - return this; - } - - public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) { - this.modelConfig = modelConfig; - return this; - } - - public Builder setKeywordsFile(String keywordsFile) { - this.keywordsFile = keywordsFile; - return this; - } - - public Builder setKeywordsScore(float keywordsScore) { - this.keywordsScore = keywordsScore; - return this; - } - - public Builder setKeywordsThreshold(float keywordsThreshold) { - this.keywordsThreshold = keywordsThreshold; - return this; - } - - public KeywordSpotterConfig build() { - return new KeywordSpotterConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java deleted file mode 100644 index 77e3a72e..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotterResult { - private final String keyword; - private final String[] tokens; - private final float[] timestamps; - - public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) { - this.keyword = keyword == null ? "" : keyword; - this.tokens = tokens == null ? new String[0] : tokens; - this.timestamps = timestamps == null ? new float[0] : timestamps; - } - - public String getKeyword() { - return keyword; - } - - public String[] getTokens() { - return tokens; - } - - public float[] getTimestamps() { - return timestamps; - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java new file mode 100644 index 00000000..20c9ae7e --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarization.java @@ -0,0 +1,62 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarization { + private long ptr = 0; + + public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization"); + } + } + + public int getSampleRate() { + return getSampleRate(ptr); + } + + // Only config.clustering is used. All other fields are ignored + public void setConfig(OfflineSpeakerDiarizationConfig config) { + setConfig(ptr, config); + } + + public OfflineSpeakerDiarizationSegment[] process(float[] samples) { + return process(ptr, samples); + } + + public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) { + return processWithCallback(ptr, samples, callback, 0); + } + + public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) { + return processWithCallback(ptr, samples, callback, arg); + } + + protected void finalize() throws Throwable { + release(); + } + + // You'd better call it manually if it is not used anymore + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + private native int getSampleRate(long ptr); + + private native void delete(long ptr); + + private native long newFromFile(OfflineSpeakerDiarizationConfig config); + + private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config); + + private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples); + + private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java new file mode 100644 index 00000000..7787386d --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationCallback.java @@ -0,0 +1,8 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +@FunctionalInterface +public interface OfflineSpeakerDiarizationCallback { + Integer invoke(int numProcessedChunks, int numTotalCunks, long arg); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java new file mode 100644 index 00000000..9965c574 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationConfig.java @@ -0,0 +1,79 @@ +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarizationConfig { + private final OfflineSpeakerSegmentationModelConfig segmentation; + private final SpeakerEmbeddingExtractorConfig embedding; + private final FastClusteringConfig clustering; + private final float minDurationOn; + private final float minDurationOff; + + private OfflineSpeakerDiarizationConfig(Builder builder) { + this.segmentation = builder.segmentation; + this.embedding = builder.embedding; + this.clustering = builder.clustering; + this.minDurationOff = builder.minDurationOff; + this.minDurationOn = builder.minDurationOn; + } + + public static Builder builder() { + return new Builder(); + } + + public OfflineSpeakerSegmentationModelConfig getSegmentation() { + return segmentation; + } + + public SpeakerEmbeddingExtractorConfig getEmbedding() { + return embedding; + } + + public FastClusteringConfig getClustering() { + return clustering; + } + + public float getMinDurationOff() { + return minDurationOff; + } + + public float getMinDurationOn() { + return minDurationOn; + } + + public static class Builder { + private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build(); + private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build(); + private FastClusteringConfig clustering = FastClusteringConfig.builder().build(); + private float minDurationOn = 0.2f; + private float minDurationOff = 0.5f; + + public OfflineSpeakerDiarizationConfig build() { + return new OfflineSpeakerDiarizationConfig(this); + } + + public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) { + this.segmentation = segmentation; + return this; + } + + public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) { + this.embedding = embedding; + return this; + } + + public Builder setClustering(FastClusteringConfig clustering) { + this.clustering = clustering; + return this; + } + + public Builder setMinDurationOff(float minDurationOff) { + this.minDurationOff = minDurationOff; + return this; + } + + public Builder setMinDurationOn(float minDurationOn) { + this.minDurationOn = minDurationOn; + return this; + } + } + +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java new file mode 100644 index 00000000..1bb1a763 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerDiarizationSegment.java @@ -0,0 +1,27 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerDiarizationSegment { + private final float start; + private final float end; + private final int speaker; + + public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) { + this.start = start; + this.end = end; + this.speaker = speaker; + } + + public float getStart() { + return start; + } + + public float getEnd() { + return end; + } + + public int getSpeaker() { + return speaker; + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java new file mode 100644 index 00000000..55df6c29 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationModelConfig.java @@ -0,0 +1,52 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineSpeakerSegmentationModelConfig { + private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote; + private final int numThreads; + private final boolean debug; + private final String provider; + + private OfflineSpeakerSegmentationModelConfig(Builder builder) { + this.pyannote = builder.pyannote; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build(); + private int numThreads = 1; + private boolean debug = true; + private String provider = "cpu"; + + public OfflineSpeakerSegmentationModelConfig build() { + return new OfflineSpeakerSegmentationModelConfig(this); + } + + public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) { + this.pyannote = pyannote; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + } +} \ No newline at end of file diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java similarity index 58% rename from App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java rename to App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java index a847e79f..51fd9987 100644 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineSpeakerSegmentationPyannoteModelConfig.java @@ -1,9 +1,11 @@ +// Copyright 2024 Xiaomi Corporation + package com.k2fsa.sherpa.onnx; -public class OnlineToneCtcModelConfig { +public class OfflineSpeakerSegmentationPyannoteModelConfig { private final String model; - private OnlineToneCtcModelConfig(Builder builder) { + private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) { this.model = builder.model; } @@ -18,13 +20,13 @@ public class OnlineToneCtcModelConfig { public static class Builder { private String model = ""; + public OfflineSpeakerSegmentationPyannoteModelConfig build() { + return new OfflineSpeakerSegmentationPyannoteModelConfig(this); + } + public Builder setModel(String model) { this.model = model; return this; } - - public OnlineToneCtcModelConfig build() { - return new OnlineToneCtcModelConfig(this); - } } } diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java deleted file mode 100644 index d3f0db66..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineModelConfig { - private final OnlineTransducerModelConfig transducer; - private final OnlineParaformerModelConfig paraformer; - private final OnlineZipformer2CtcModelConfig zipformer2Ctc; - private final OnlineNeMoCtcModelConfig neMoCtc; - private final OnlineToneCtcModelConfig toneCtc; - private final String tokens; - private final int numThreads; - private final boolean debug; - private final String provider; - private final String modelType; - private final String modelingUnit; - private final String bpeVocab; - - private OnlineModelConfig(Builder builder) { - this.transducer = builder.transducer; - this.paraformer = builder.paraformer; - this.zipformer2Ctc = builder.zipformer2Ctc; - this.neMoCtc = builder.neMoCtc; - this.toneCtc = builder.toneCtc; - this.tokens = builder.tokens; - this.numThreads = builder.numThreads; - this.debug = builder.debug; - this.provider = builder.provider; - this.modelType = builder.modelType; - this.modelingUnit = builder.modelingUnit; - this.bpeVocab = builder.bpeVocab; - } - - public static Builder builder() { - return new Builder(); - } - - public OnlineTransducerModelConfig getTransducer() { - return transducer; - } - - public OnlineParaformerModelConfig getParaformer() { - return paraformer; - } - - public OnlineZipformer2CtcModelConfig getZipformer2Ctc() { - return zipformer2Ctc; - } - - public OnlineNeMoCtcModelConfig getNeMoCtc() { - return neMoCtc; - } - - public OnlineToneCtcModelConfig getToneCtc() { - return toneCtc; - } - - public String getTokens() { - return tokens; - } - - public int getNumThreads() { - return numThreads; - } - - public boolean getDebug() { - return debug; - } - - public String getProvider() { - return provider; - } - - public String getModelType() { - return modelType; - } - - public String getModelingUnit() { - return modelingUnit; - } - - public String getBpeVocab() { - return bpeVocab; - } - - public static class Builder { - private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build(); - private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build(); - private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build(); - private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build(); - private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build(); - private String tokens = ""; - private int numThreads = 1; - private boolean debug = false; - private String provider = "cpu"; - private String modelType = ""; - private String modelingUnit = "cjkchar"; - private String bpeVocab = ""; - - public Builder setTransducer(OnlineTransducerModelConfig transducer) { - this.transducer = transducer; - return this; - } - - public Builder setTokens(String tokens) { - this.tokens = tokens; - return this; - } - - public Builder setModelType(String modelType) { - this.modelType = modelType; - return this; - } - - public Builder setNumThreads(int numThreads) { - this.numThreads = numThreads; - return this; - } - - public Builder setProvider(String provider) { - this.provider = provider; - return this; - } - - public Builder setModelingUnit(String modelingUnit) { - this.modelingUnit = modelingUnit; - return this; - } - - public Builder setBpeVocab(String bpeVocab) { - this.bpeVocab = bpeVocab; - return this; - } - - public OnlineModelConfig build() { - return new OnlineModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java deleted file mode 100644 index 1982804c..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineNeMoCtcModelConfig { - private final String model; - - private OnlineNeMoCtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineNeMoCtcModelConfig build() { - return new OnlineNeMoCtcModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java deleted file mode 100644 index 3e44e2c5..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineParaformerModelConfig { - private final String encoder; - private final String decoder; - - private OnlineParaformerModelConfig(Builder builder) { - this.encoder = builder.encoder; - this.decoder = builder.decoder; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEncoder() { - return encoder; - } - - public String getDecoder() { - return decoder; - } - - public static class Builder { - private String encoder = ""; - private String decoder = ""; - - public Builder setEncoder(String encoder) { - this.encoder = encoder; - return this; - } - - public Builder setDecoder(String decoder) { - this.decoder = decoder; - return this; - } - - public OnlineParaformerModelConfig build() { - return new OnlineParaformerModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java index 90a249cc..985ce8de 100644 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java @@ -1,7 +1,16 @@ +// Copyright 2022-2023 by zhaoming +// Copyright 2024 Xiaomi Corporation + package com.k2fsa.sherpa.onnx; public class OnlineStream { - private long ptr; + private long ptr = 0; + + public OnlineStream() { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + this.ptr = 0; + } public OnlineStream(long ptr) { this.ptr = ptr; @@ -11,25 +20,58 @@ public class OnlineStream { return ptr; } + public void setPtr(long ptr) { + this.ptr = ptr; + } + public void acceptWaveform(float[] samples, int sampleRate) { - acceptWaveform(ptr, samples, sampleRate); + acceptWaveform(this.ptr, samples, sampleRate); + } + + public void inputFinished() { + inputFinished(this.ptr); + } + + public void setOption(String key, String value) { + setOption(this.ptr, key, value); + } + + public String getOption(String key) { + return getOption(this.ptr, key); + } + + public boolean hasOption(String key) { + return hasOption(this.ptr, key); } public void release() { - if (ptr == 0) { - return; - } - delete(ptr); - ptr = 0; + close(); } - + + public void close() { + // stream object must be release after used + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + @Override protected void finalize() throws Throwable { - release(); + close(); super.finalize(); } private native void acceptWaveform(long ptr, float[] samples, int sampleRate); + private native void inputFinished(long ptr); + + private native void setOption(long ptr, String key, String value); + + private native String getOption(long ptr, String key); + + private native boolean hasOption(long ptr, String key); + private native void delete(long ptr); -} +} \ No newline at end of file diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java deleted file mode 100644 index ca003533..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineTransducerModelConfig { - private final String encoder; - private final String decoder; - private final String joiner; - - private OnlineTransducerModelConfig(Builder builder) { - this.encoder = builder.encoder; - this.decoder = builder.decoder; - this.joiner = builder.joiner; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEncoder() { - return encoder; - } - - public String getDecoder() { - return decoder; - } - - public String getJoiner() { - return joiner; - } - - public static class Builder { - private String encoder = ""; - private String decoder = ""; - private String joiner = ""; - - public Builder setEncoder(String encoder) { - this.encoder = encoder; - return this; - } - - public Builder setDecoder(String decoder) { - this.decoder = decoder; - return this; - } - - public Builder setJoiner(String joiner) { - this.joiner = joiner; - return this; - } - - public OnlineTransducerModelConfig build() { - return new OnlineTransducerModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java deleted file mode 100644 index d73f73dd..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineZipformer2CtcModelConfig { - private final String model; - - private OnlineZipformer2CtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineZipformer2CtcModelConfig build() { - return new OnlineZipformer2CtcModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractor.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractor.java new file mode 100644 index 00000000..026b2ed6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractor.java @@ -0,0 +1,58 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class SpeakerEmbeddingExtractor { + private long ptr = 0; + + public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor"); + } + } + + @Override + protected void finalize() throws Throwable { + release(); + } + + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + public OnlineStream createStream() { + long p = createStream(ptr); + return new OnlineStream(p); + } + + public boolean isReady(OnlineStream s) { + return isReady(ptr, s.getPtr()); + } + + public float[] compute(OnlineStream s) { + return compute(ptr, s.getPtr()); + } + + public int getDim() { + return dim(ptr); + } + + private native void delete(long ptr); + + private native long newFromFile(SpeakerEmbeddingExtractorConfig config); + + private native long createStream(long ptr); + + private native boolean isReady(long ptr, long streamPtr); + + private native float[] compute(long ptr, long streamPtr); + + private native int dim(long ptr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java new file mode 100644 index 00000000..80f800cd --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingExtractorConfig.java @@ -0,0 +1,53 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class SpeakerEmbeddingExtractorConfig { + private final String model; + private final int numThreads; + private final boolean debug; + private final String provider; + + private SpeakerEmbeddingExtractorConfig(Builder builder) { + this.model = builder.model; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String model = ""; + private int numThreads = 1; + private boolean debug = true; + private String provider = "cpu"; + + public SpeakerEmbeddingExtractorConfig build() { + return new SpeakerEmbeddingExtractorConfig(this); + } + + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setDebug(boolean debug) { + this.debug = debug; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingManager.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingManager.java new file mode 100644 index 00000000..29bc7cdc --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/SpeakerEmbeddingManager.java @@ -0,0 +1,78 @@ +// Copyright 2024 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class SpeakerEmbeddingManager { + private long ptr = 0; + + public SpeakerEmbeddingManager(int dim) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = create(dim); + } + + @Override + protected void finalize() throws Throwable { + release(); + } + + public void release() { + if (this.ptr == 0) { + return; + } + delete(this.ptr); + this.ptr = 0; + } + + public boolean add(String name, float[] embedding) { + return add(ptr, name, embedding); + } + + public boolean add(String name, float[][] embedding) { + return addList(ptr, name, embedding); + } + + public boolean remove(String name) { + return remove(ptr, name); + } + + public String search(float[] embedding, float threshold) { + return search(ptr, embedding, threshold); + } + + public boolean verify(String name, float[] embedding, float threshold) { + return verify(ptr, name, embedding, threshold); + } + + public boolean contains(String name) { + return contains(ptr, name); + } + + public int getNumSpeakers() { + return numSpeakers(ptr); + } + + public String[] getAllSpeakerNames() { + return allSpeakerNames(ptr); + } + + private native long create(int dim); + + private native void delete(long ptr); + + private native boolean add(long ptr, String name, float[] embedding); + + private native boolean addList(long ptr, String name, float[][] embedding); + + private native boolean remove(long ptr, String name); + + private native String search(long ptr, float[] embedding, float threshold); + + private native boolean verify(long ptr, String name, float[] embedding, float threshold); + + private native boolean contains(long ptr, String name); + + private native int numSpeakers(long ptr); + + private native String[] allSpeakerNames(long ptr); +} diff --git a/App_Resources/Android/src/main/res/drawable-hdpi/background.png b/App_Resources/Android/src/main/res/drawable-hdpi/background.png deleted file mode 100644 index bbefbf42..00000000 Binary files a/App_Resources/Android/src/main/res/drawable-hdpi/background.png and /dev/null differ diff --git a/App_Resources/Android/src/main/res/drawable-hdpi/logo.png b/App_Resources/Android/src/main/res/drawable-hdpi/logo.png index e788deb3..9a212517 100644 Binary files a/App_Resources/Android/src/main/res/drawable-hdpi/logo.png and b/App_Resources/Android/src/main/res/drawable-hdpi/logo.png differ diff --git a/App_Resources/Android/src/main/res/drawable-ldpi/background.png b/App_Resources/Android/src/main/res/drawable-ldpi/background.png deleted file mode 100644 index f6a08eea..00000000 Binary files a/App_Resources/Android/src/main/res/drawable-ldpi/background.png and /dev/null differ diff --git a/App_Resources/Android/src/main/res/drawable-ldpi/logo.png b/App_Resources/Android/src/main/res/drawable-ldpi/logo.png index e4cac1ad..10758abe 100644 Binary files a/App_Resources/Android/src/main/res/drawable-ldpi/logo.png and b/App_Resources/Android/src/main/res/drawable-ldpi/logo.png differ diff --git a/App_Resources/Android/src/main/res/drawable-mdpi/background.png b/App_Resources/Android/src/main/res/drawable-mdpi/background.png deleted file mode 100644 index 0c90f0f7..00000000 Binary files a/App_Resources/Android/src/main/res/drawable-mdpi/background.png and /dev/null differ diff --git a/App_Resources/Android/src/main/res/drawable-mdpi/logo.png b/App_Resources/Android/src/main/res/drawable-mdpi/logo.png index ce3c3a4b..0f4c7204 100644 Binary files a/App_Resources/Android/src/main/res/drawable-mdpi/logo.png and b/App_Resources/Android/src/main/res/drawable-mdpi/logo.png differ diff --git a/App_Resources/Android/src/main/res/drawable-nodpi/splash_screen.xml b/App_Resources/Android/src/main/res/drawable-nodpi/splash_screen.xml index ada77f92..a0acfca1 100644 --- a/App_Resources/Android/src/main/res/drawable-nodpi/splash_screen.xml +++ b/App_Resources/Android/src/main/res/drawable-nodpi/splash_screen.xml @@ -1,8 +1,8 @@ - + - \ No newline at end of file + diff --git a/App_Resources/Android/src/main/res/drawable-xhdpi/background.png b/App_Resources/Android/src/main/res/drawable-xhdpi/background.png deleted file mode 100644 index 3541570c..00000000 Binary files a/App_Resources/Android/src/main/res/drawable-xhdpi/background.png and /dev/null differ diff --git a/App_Resources/Android/src/main/res/drawable-xhdpi/logo.png b/App_Resources/Android/src/main/res/drawable-xhdpi/logo.png index 88267df0..a44364f4 100644 Binary files a/App_Resources/Android/src/main/res/drawable-xhdpi/logo.png and b/App_Resources/Android/src/main/res/drawable-xhdpi/logo.png differ diff --git a/App_Resources/Android/src/main/res/drawable-xxhdpi/background.png b/App_Resources/Android/src/main/res/drawable-xxhdpi/background.png deleted file mode 100644 index abb0fc70..00000000 Binary files a/App_Resources/Android/src/main/res/drawable-xxhdpi/background.png and /dev/null differ diff --git a/App_Resources/Android/src/main/res/drawable-xxhdpi/logo.png b/App_Resources/Android/src/main/res/drawable-xxhdpi/logo.png index 55800c9a..32c4f53f 100644 Binary files a/App_Resources/Android/src/main/res/drawable-xxhdpi/logo.png and b/App_Resources/Android/src/main/res/drawable-xxhdpi/logo.png differ diff --git a/App_Resources/Android/src/main/res/drawable-xxxhdpi/background.png b/App_Resources/Android/src/main/res/drawable-xxxhdpi/background.png deleted file mode 100644 index 10897752..00000000 Binary files a/App_Resources/Android/src/main/res/drawable-xxxhdpi/background.png and /dev/null differ diff --git a/App_Resources/Android/src/main/res/drawable-xxxhdpi/logo.png b/App_Resources/Android/src/main/res/drawable-xxxhdpi/logo.png index 0703f90f..bdf0bb04 100644 Binary files a/App_Resources/Android/src/main/res/drawable-xxxhdpi/logo.png and b/App_Resources/Android/src/main/res/drawable-xxxhdpi/logo.png differ diff --git a/App_Resources/Android/src/main/res/drawable/ic_tab_mic.xml b/App_Resources/Android/src/main/res/drawable/ic_tab_mic.xml new file mode 100644 index 00000000..3132c4ea --- /dev/null +++ b/App_Resources/Android/src/main/res/drawable/ic_tab_mic.xml @@ -0,0 +1,18 @@ + + + + + + diff --git a/App_Resources/Android/src/main/res/values/styles.xml b/App_Resources/Android/src/main/res/values/styles.xml index 5cb0a093..0eef3110 100644 --- a/App_Resources/Android/src/main/res/values/styles.xml +++ b/App_Resources/Android/src/main/res/values/styles.xml @@ -25,6 +25,17 @@ @color/ns_primary @color/ns_primaryDark @color/ns_accent + + + @style/FaceclawOverflowButtonStyle + + + + diff --git a/wear/app/src/main/res/values/wear.xml b/wear/app/src/main/res/values/wear.xml new file mode 100644 index 00000000..ebb4878f --- /dev/null +++ b/wear/app/src/main/res/values/wear.xml @@ -0,0 +1,8 @@ + + + + + faceclaw_watch + + diff --git a/wear/build.gradle.kts b/wear/build.gradle.kts new file mode 100644 index 00000000..0fdc2bca --- /dev/null +++ b/wear/build.gradle.kts @@ -0,0 +1,9 @@ +// Faceclaw watch app (Wear OS companion). A standalone Gradle project: it +// shares nothing with the NativeScript build except the applicationId and +// (for release builds) the signing key, both of which the Wearable Data +// Layer requires to match the phone app. +plugins { + id("com.android.application") version "8.7.3" apply false + id("org.jetbrains.kotlin.android") version "2.1.10" apply false + id("org.jetbrains.kotlin.plugin.compose") version "2.1.10" apply false +} diff --git a/wear/gradle.properties b/wear/gradle.properties new file mode 100644 index 00000000..5f4cff37 --- /dev/null +++ b/wear/gradle.properties @@ -0,0 +1,4 @@ +org.gradle.jvmargs=-Xmx2g -Dfile.encoding=UTF-8 +android.useAndroidX=true +android.nonTransitiveRClass=true +kotlin.code.style=official diff --git a/wear/gradle/wrapper/gradle-wrapper.jar b/wear/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..249e5832 Binary files /dev/null and b/wear/gradle/wrapper/gradle-wrapper.jar differ diff --git a/wear/gradle/wrapper/gradle-wrapper.properties b/wear/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..d4081da4 --- /dev/null +++ b/wear/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/wear/gradlew b/wear/gradlew new file mode 100755 index 00000000..a69d9cb6 --- /dev/null +++ b/wear/gradlew @@ -0,0 +1,240 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/wear/gradlew.bat b/wear/gradlew.bat new file mode 100644 index 00000000..f127cfd4 --- /dev/null +++ b/wear/gradlew.bat @@ -0,0 +1,91 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/wear/settings.gradle.kts b/wear/settings.gradle.kts new file mode 100644 index 00000000..b273625d --- /dev/null +++ b/wear/settings.gradle.kts @@ -0,0 +1,16 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} +rootProject.name = "faceclaw-wear" +include(":app") diff --git a/website/CNAME b/website/CNAME new file mode 100644 index 00000000..c248600c --- /dev/null +++ b/website/CNAME @@ -0,0 +1 @@ +faceclaw.org diff --git a/website/README.md b/website/README.md new file mode 100644 index 00000000..bc65a419 --- /dev/null +++ b/website/README.md @@ -0,0 +1,59 @@ +# faceclaw.org + +The Faceclaw website: plain HTML and one stylesheet, no build step and no +external dependencies. GitHub Actions publishes this directory to GitHub Pages +on every push to `main` that touches it (`.github/workflows/pages.yml`). + +## Layout + + index.html landing page + install.html install instructions + privacy.html privacy policy (body generated, see below) + style.css the whole stylesheet + content/ canonical copies of content shared with the repo root + images/ logo and favicon, derived from ../images/FaceclawAppIcon.png + screenshots/ glasses captures, also used by the root README + CNAME custom domain + +## Shared content + +Two things appear both here and in the repository root, and are generated +rather than copied by hand: + +| Source | Destinations | +| --------------------------- | ------------------------------------- | +| `content/features.md` | `README.md`, `index.html` | +| `../PRIVACY` | `privacy.html` | + +Edit the source, then run: + + node scripts/sync-site.mjs + +Each destination has a `` / `` +pair; the script rewrites what is between them and leaves the rest alone. CI +runs `node scripts/sync-site.mjs --check` and fails the deploy if a +destination is stale. + +## Previewing + +Any static file server will do, since there is nothing to compile: + + python3 -m http.server -d website 8000 + +## Setting up the custom domain + +Publishing needs Settings > Pages > Source set to "GitHub Actions". For +`faceclaw.org`, point the apex at GitHub's Pages addresses: + + A faceclaw.org 185.199.108.153 + A faceclaw.org 185.199.109.153 + A faceclaw.org 185.199.110.153 + A faceclaw.org 185.199.111.153 + AAAA faceclaw.org 2606:50c0:8000::153 + AAAA faceclaw.org 2606:50c0:8001::153 + AAAA faceclaw.org 2606:50c0:8002::153 + AAAA faceclaw.org 2606:50c0:8003::153 + CNAME www jimrandomh.github.io. + +Then set the custom domain in Settings > Pages and enable "Enforce HTTPS" +once the certificate has been issued. diff --git a/website/content/features.md b/website/content/features.md new file mode 100644 index 00000000..b59baa8d --- /dev/null +++ b/website/content/features.md @@ -0,0 +1,24 @@ + + +- **A voice assistant** that wakes up when you say "Hey Even", transcribes text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds to queries and commands using an onboard model (Qwen3 4B, slow) or with an Anthropic or OpenAI model (requires an API key), or using your own long-running OpenClaw agent. +- **Multitasking**, with an app-switcher sidebar and app launcher. +- **Mostly-compatible with EvenHub apps.** +- **A lock screen**; glasses lock automatically when you take them off and unlock when you unlock your phone. +- **The full display.** Full-screen apps can use the full 640x480 display area, rather than the 576x288 that EvenHub apps can use. +- **Integration with Android notifications**: a top bar that shows the same icons your phone does, popups when notifications arrive, and menu items to dismiss or use Android-app-provided custom actions like mark as read or quick reply. +- **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on the glasses, and send them inputs with the voice assistant. +- **Media player controls** including playlist and media library navigation, compatible with most Android media players. +- **Turn-by-turn directions** (requires a Mapbox API token). +- **Nightscout**, an app for viewing blood-glucose data (requires a cloud server and API token). +- **Power management**: the glasses go to sleep properly when the screen is off, and wake when you double-tap the ring or speak the wakeword, allowing battery life similar to the stock Even app. +- **Connection management** with auto-reconnect, and autodetection of conflict with the official Even Realities app. +- **A Wear OS watch app** that replaces (and outdoes) the R1 ring: tap, swipe, hold and crown gestures, side buttons, app launching and window switching, voice or keyboard queries to the assistant with the reply on your wrist, typing into apps, and glasses status/lock/display control. +- **On-phone screen mirroring with touch control**: tap what you see on the mirror (sidebar icons, launcher cells), or use the phone's own touchpad, d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a compact ring simulator. A display-mode picker (576×288 band, 576×480 tall, or the full 640×480 panel with an auto-hiding sidebar) and a brightness slider with an Auto toggle sit beside the mirror. +- **Bluetooth pairing** that scans for nearby glasses and identifies each pair before connecting: model, frame shape, and colour decoded from the advertised serial (with product photos), left and right arms matched to each other by that serial, an estimated distance so the pair in your hand sorts first, and the optional R1 ring. +- **Dual-language NativeScript architecture**, with Java for the multithreaded Android API and bluetooth stack bits, Typescript for the bits you want to hack on. diff --git a/website/images/faceclaw-logo.png b/website/images/faceclaw-logo.png new file mode 100644 index 00000000..36053dff Binary files /dev/null and b/website/images/faceclaw-logo.png differ diff --git a/website/images/favicon.png b/website/images/favicon.png new file mode 100644 index 00000000..805cc0c0 Binary files /dev/null and b/website/images/favicon.png differ diff --git a/website/index.html b/website/index.html new file mode 100644 index 00000000..755c4dfb --- /dev/null +++ b/website/index.html @@ -0,0 +1,147 @@ + + + + + +Faceclaw — an unofficial interface for the Even Realities G2 + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ +

Faceclaw

+

An unofficial user interface for the Even Realities G2 smart glasses. Apps, a voice assistant, notifications and a watch remote — running from your Android phone.

+ +
+
+ +
+
+
+

Unofficial

+

Faceclaw is not made by, endorsed by, or supported by Even Realities. It comes with no warranty from them or from anyone. It runs on Android, and it requires installing custom firmware on the glasses — the app can install and uninstall that firmware for you, downloading the stock firmware from Even and applying patches generated from g2flash.

+
+
+
+ +
+
+

What it looks like

+
+
+ The Faceclaw app launcher on the glasses display +
The app launcher, with the app-switcher sidebar on the left.
+
+
+ The music player on the glasses display +
Media controls, working with most Android media players.
+
+
+ Assistant settings on the glasses display +
Assistant settings, edited on the glasses themselves.
+
+
+ Display settings on the glasses display +
Display settings, including height and vertical position.
+
+
+
+
+ +
+
+

Features

+ +
    +
  • A voice assistant that wakes up when you say "Hey Even", transcribes text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and responds to queries and commands using an onboard model (Qwen3 4B, slow) or with an Anthropic or OpenAI model (requires an API key), or using your own long-running OpenClaw agent.
  • +
  • Multitasking, with an app-switcher sidebar and app launcher.
  • +
  • Mostly-compatible with EvenHub apps.
  • +
  • A lock screen; glasses lock automatically when you take them off and unlock when you unlock your phone.
  • +
  • The full display. Full-screen apps can use the full 640x480 display area, rather than the 576x288 that EvenHub apps can use.
  • +
  • Integration with Android notifications: a top bar that shows the same icons your phone does, popups when notifications arrive, and menu items to dismiss or use Android-app-provided custom actions like mark as read or quick reply.
  • +
  • Terminal mirroring. Mirror terminal apps such as Claude Code or Codex CLI with g2mirror, view them on the glasses, and send them inputs with the voice assistant.
  • +
  • Media player controls including playlist and media library navigation, compatible with most Android media players.
  • +
  • Turn-by-turn directions (requires a Mapbox API token).
  • +
  • Nightscout, an app for viewing blood-glucose data (requires a cloud server and API token).
  • +
  • Power management: the glasses go to sleep properly when the screen is off, and wake when you double-tap the ring or speak the wakeword, allowing battery life similar to the stock Even app.
  • +
  • Connection management with auto-reconnect, and autodetection of conflict with the official Even Realities app.
  • +
  • A Wear OS watch app that replaces (and outdoes) the R1 ring: tap, swipe, hold and crown gestures, side buttons, app launching and window switching, voice or keyboard queries to the assistant with the reply on your wrist, typing into apps, and glasses status/lock/display control.
  • +
  • On-phone screen mirroring with touch control: tap what you see on the mirror (sidebar icons, launcher cells), or use the phone's own touchpad, d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a compact ring simulator. A display-mode picker (576×288 band, 576×480 tall, or the full 640×480 panel with an auto-hiding sidebar) and a brightness slider with an Auto toggle sit beside the mirror.
  • +
  • Bluetooth pairing that scans for nearby glasses and identifies each pair before connecting: model, frame shape, and colour decoded from the advertised serial (with product photos), left and right arms matched to each other by that serial, an estimated distance so the pair in your hand sorts first, and the optional R1 ring.
  • +
  • Dual-language NativeScript architecture, with Java for the multithreaded Android API and bluetooth stack bits, Typescript for the bits you want to hack on.
  • +
+ +
+
+ +
+
+

Beyond the glasses

+
+
+

Watch remote

+

A Wear OS companion app turns the whole watch face into a touchpad that outdoes the R1 ring: swipes, crown scrolling, side buttons, app launching, and spoken or typed queries with the reply on your wrist.

+
+
+

EvenHub apps

+

Faceclaw runs EvenHub apps through an emulation layer. Install them from the Hub, or sideload your own by opening an .ehpk file on your phone.

+
+
+

Your own agent

+

Instead of calling an LLM API from the phone, the assistant can route queries to your own long-running OpenClaw agent, which also gets access to the glasses' tools.

+
+
+
+
+ +
+
+

Contributing

+

Be bold. Modify Faceclaw into the app that you want it to be for yourself, without worrying about whether other people will like your version — then, if you think your changes might be useful to others, open a pull request. Faceclaw is Free Software under the GPLv3. See the contributing notes for what to test before you send a change.

+
+
+ +
+ + + + + diff --git a/website/install.html b/website/install.html new file mode 100644 index 00000000..66323def --- /dev/null +++ b/website/install.html @@ -0,0 +1,80 @@ + + + + + +Install — Faceclaw + + + + + + +
+ +
+ +
+
+
+

Installing Faceclaw

+ +
+

Read this first

+

Faceclaw is unofficial and unsupported, and it replaces the firmware on your glasses. Installing it means running code that Even Realities did not write and will not help you with. The app can put the stock firmware back, but you are doing this at your own risk.

+
+ +

What you need

+
    +
  • A pair of Even Realities G2 glasses.
  • +
  • An Android phone. Faceclaw runs as a background app and talks to the glasses over Bluetooth LE.
  • +
  • Optionally, a Wear OS 3+ watch, if you want the watch remote instead of the R1 ring.
  • +
+

There is no iOS version, and there will not be one — the app depends on Android APIs for background Bluetooth, notification access and media control.

+ +

Install the app

+

Faceclaw is not on the Play Store. Download the most recent APK from the releases page and open it on your phone; Android will ask you to allow installing apps from that source.

+ +

Install the firmware

+

Faceclaw needs custom firmware on the glasses, and installs it for you. On first run, the onboarding flow will offer to download the stock firmware from Even Realities, apply the g2flash patches locally, and flash the result. You confirm the flash on the glasses themselves.

+

Onboarding also offers a preview-only path, which connects to unmodified glasses so you can look around before committing. Faceclaw can restore the stock firmware later from the same screens.

+ +

Grant permissions

+

Android does not reliably prompt for everything Faceclaw uses. After installing, go to Settings → Apps → Faceclaw and check that permissions are granted — notification access and Bluetooth in particular — or parts of the app will silently do nothing.

+

Also exempt Faceclaw from battery optimization. Being a background app on a locked Android phone is fraught, and manufacturer-specific battery software may pause it or throttle it to low CPU usage.

+ +

Optional extras

+
    +
  • Watch remote. The Wear OS app is a separate Gradle build in wear/, sideloaded over adb; it needs no Play Store, but the phone side requires Google Play services.
  • +
  • API keys. The voice assistant needs a model provider key (Anthropic or OpenAI), cloud transcription needs a key for OpenAI Whisper, ElevenLabs or Soniox, and turn-by-turn directions need a Mapbox token. On-device transcription needs no key.
  • +
  • Terminal mirroring. Install g2mirror on the machine whose terminal you want to see.
  • +
  • External agent. To route assistant queries to your own long-running agent, install the faceclaw-agent-bridge OpenClaw plugin; setup lives in that repository's README.
  • +
+ +

Building from source

+

Faceclaw is a NativeScript app written in TypeScript and Java, and builds with an Android SDK 35 environment and a JDK 21. The README has the current build and adb instructions; they are kept there rather than duplicated here, so they stay in step with the code.

+
+
+
+ +
+
+ +

Faceclaw is Free Software, licensed under the GPLv3. It is unofficial, and is not affiliated with Even Realities.

+
+
+ + + diff --git a/website/privacy.html b/website/privacy.html new file mode 100644 index 00000000..3cd84806 --- /dev/null +++ b/website/privacy.html @@ -0,0 +1,80 @@ + + + + + +Privacy policy — Faceclaw + + + + + + +
+ +
+ +
+
+
+ +

Faceclaw Privacy Policy

+

Faceclaw is a privacy-conscious app. The developers of Faceclaw don't collect or store user data. Faceclaw does integrate with some third-party services, which have their own privacy policies; this document describes what those services are, what they're used for and what data they collect, and links to their privacy policies.

+

Audio

+

Faceclaw allows users to transcribe audio (for use with the voice assistant, the Transcribe app, typing into Terminal, and other apps). The default setting is that this is done using an on-device model running on your phone. If an on-device model is selected, audio never leaves your device, and is not stored after it is processed.

+

Users can also provide their own API key for a third-party transcription service such as OpenAI Whisper, ElevenLabs, or Soniox, which may offer better accuracy. Faceclaw never transmits audio to any party other than a user-selected transcription service. If a third-party transcription service is selected, audio will be sent to that third party, is linked to the account with which that API key was created, and is subject to that party's privacy and retention policies:

+ +

Third party data retention and privacy policies may differ based on the API key provided. API keys are part of a relationship between the user and the API provider; Faceclaw does not manage accounts or process billing information.

+

AI Assistant Transcripts

+

Faceclaw has a builtin AI voice assistant. Using the voice assistant requires providing an API key for a model provider such as Anthropic or OpenAI. Prompts and responses are subject to that model provider's privacy policy:

+ +

AI conversation histories and memory files may be stored on your phone. Faceclaw does not transmit voice-assistant prompts or responses to any party other than a user-selected AI model provider. The voice assistant is given tools with which it is able to access most things on the user's phone, including all content displayed on the glasses, location, notifications, and content accessible through the Terminal app.

+

The AI assistant can take actions on the user's behalf, including sharing any information it has access to. The user is responsible for any instructions they give it.

+

Location

+

Faceclaw has a builtin Navigate app, which provides turn-by-turn directions. Using the Navigate app requires providing an API key for Mapbox. While the Navigate app is active, the user's exact location and selected destination are transmitted to Mapbox, and are subject to Mapbox's privacy policy.

+ +

The Weather app will retrieve the user's coarse location, and will use it to query a third-party weather information provider.

+

EvenHub

+

The EvenHub app store is subject to Even Realities' privacy policy: https://support.evenrealities.com/hc/en-us/articles/14127464826511-Privacy-Policy. Apps downloaded from the EvenHub app store come with their own privacy policies.

+

Wear OS Watch

+

The optional Faceclaw watch app (wear/) talks to the phone over Google Play services' Wearable Data Layer, the same channel the watch uses for notifications and other companion apps. Gestures, commands, the watch's own speech-recognition results, and the mirrored glasses status and assistant replies travel between your watch and your phone over their Bluetooth (or, when paired that way, Wi-Fi) link; Faceclaw does not send any of it to a server. Speech recognized on the watch uses the watch's system speech recognizer, which may send audio to Google under Google's privacy policy:

+ +

The phone does nothing watch-related on phones without Google Play services.

+ +

This page is generated from the PRIVACY file in the Faceclaw repository, which is the authoritative copy.

+
+
+
+ + + + + diff --git a/screenshots/launcher.png b/website/screenshots/launcher.png similarity index 100% rename from screenshots/launcher.png rename to website/screenshots/launcher.png diff --git a/screenshots/music-player.png b/website/screenshots/music-player.png similarity index 100% rename from screenshots/music-player.png rename to website/screenshots/music-player.png diff --git a/screenshots/settings-assistant.png b/website/screenshots/settings-assistant.png similarity index 100% rename from screenshots/settings-assistant.png rename to website/screenshots/settings-assistant.png diff --git a/screenshots/settings-display.png b/website/screenshots/settings-display.png similarity index 100% rename from screenshots/settings-display.png rename to website/screenshots/settings-display.png diff --git a/website/style.css b/website/style.css new file mode 100644 index 00000000..e067c5e5 --- /dev/null +++ b/website/style.css @@ -0,0 +1,178 @@ +/* Faceclaw website. Hand-written, no build step, no external assets. */ + +:root { + --bg: #0d1226; + --bg-raised: #141b38; + --line: #26305a; + --fg: #e9ecf7; + --fg-dim: #98a1c4; + --accent: #f5c531; + --accent-warm: #e8503a; + --lens: #000; + --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + --sans: system-ui, -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--bg); + color: var(--fg); + font: 16px/1.65 var(--sans); + -webkit-font-smoothing: antialiased; +} + +.wrap { width: 100%; max-width: 960px; margin: 0 auto; padding: 0 24px; } + +a { color: var(--accent); text-decoration-color: rgba(245, 197, 49, 0.4); text-underline-offset: 2px; } +a:hover { text-decoration-color: var(--accent); } + +h1, h2, h3 { line-height: 1.2; letter-spacing: -0.015em; } +h2 { font-size: 1.5rem; margin: 0 0 16px; } +h3 { font-size: 1.05rem; margin: 0 0 8px; } +p { margin: 0 0 16px; } + +code, kbd { font-family: var(--mono); font-size: 0.9em; } +code { + background: var(--bg-raised); + border: 1px solid var(--line); + border-radius: 4px; + padding: 1px 5px; +} +pre { + background: var(--bg-raised); + border: 1px solid var(--line); + border-radius: 8px; + padding: 14px 16px; + overflow-x: auto; + margin: 0 0 16px; +} +pre code { background: none; border: 0; padding: 0; } + +/* ---- site chrome ---- */ + +.topbar { + border-bottom: 1px solid var(--line); + position: sticky; + top: 0; + background: rgba(13, 18, 38, 0.9); + backdrop-filter: blur(8px); + z-index: 10; +} +.topbar .wrap { display: flex; align-items: center; gap: 20px; height: 56px; } +.brand { display: flex; align-items: center; gap: 9px; font-weight: 650; color: var(--fg); text-decoration: none; } +.brand img { width: 22px; height: 22px; } +.topbar nav { margin-left: auto; display: flex; gap: 20px; font-size: 0.94rem; } +.topbar nav a { color: var(--fg-dim); text-decoration: none; } +.topbar nav a:hover, .topbar nav a[aria-current="page"] { color: var(--fg); } + +section { padding: 56px 0; border-top: 1px solid var(--line); } +section:first-of-type { border-top: 0; } + +footer { + border-top: 1px solid var(--line); + padding: 32px 0 56px; + color: var(--fg-dim); + font-size: 0.9rem; +} +footer a { color: var(--fg-dim); } +footer .links { display: flex; flex-wrap: wrap; gap: 18px; margin-bottom: 12px; } + +/* ---- hero ---- */ + +.hero { text-align: center; padding: 72px 0 56px; } +.hero img.logo { width: 104px; height: 104px; } +.hero h1 { font-size: clamp(2.2rem, 6vw, 3.1rem); margin: 18px 0 10px; } +.hero .tagline { font-size: 1.2rem; color: var(--fg-dim); max-width: 34em; margin: 0 auto 28px; } + +.buttons { display: flex; gap: 12px; justify-content: center; flex-wrap: wrap; } +.btn { + display: inline-block; + padding: 11px 22px; + border-radius: 8px; + border: 1px solid var(--line); + font-weight: 600; + text-decoration: none; + color: var(--fg); +} +.btn:hover { border-color: var(--fg-dim); } +.btn.primary { background: var(--accent); border-color: var(--accent); color: #1a1400; } +.btn.primary:hover { background: #ffd44f; border-color: #ffd44f; } + +.eyebrow { + font-family: var(--mono); + font-size: 0.78rem; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--accent-warm); +} + +/* ---- callout ---- */ + +.callout { + border: 1px solid var(--line); + border-left: 3px solid var(--accent-warm); + border-radius: 8px; + background: var(--bg-raised); + padding: 16px 20px; + margin: 0 0 24px; +} +.callout p:last-child { margin-bottom: 0; } + +/* ---- screenshots ---- */ + +.shots { display: grid; grid-template-columns: repeat(2, 1fr); gap: 24px; } +.shot { margin: 0; } +.shot img { + display: block; + width: 100%; + background: var(--lens); + border: 1px solid var(--line); + border-radius: 6px; + image-rendering: pixelated; +} +.shot figcaption { color: var(--fg-dim); font-size: 0.88rem; margin-top: 8px; } + +/* ---- features ---- */ + +.features { list-style: none; padding: 0; margin: 0; columns: 2; column-gap: 40px; } +.features li { + break-inside: avoid; + margin: 0 0 18px; + padding-left: 20px; + position: relative; + color: var(--fg-dim); +} +.features li::before { content: "▸"; position: absolute; left: 0; color: var(--accent); } +.features li strong { color: var(--fg); font-weight: 600; } + +/* ---- cards ---- */ + +.cards { display: grid; grid-template-columns: repeat(3, 1fr); gap: 20px; } +.card { + border: 1px solid var(--line); + border-radius: 10px; + background: var(--bg-raised); + padding: 20px; +} +.card p { color: var(--fg-dim); font-size: 0.94rem; margin: 0; } + +/* ---- long-form prose (privacy, install) ---- */ + +.prose { max-width: 44em; } +.prose h2 { margin-top: 40px; padding-top: 8px; } +.prose h2:first-child { margin-top: 0; } +.prose ul { padding-left: 22px; } +.prose li { margin-bottom: 6px; } +.prose ul.links { list-style: none; padding-left: 0; font-family: var(--mono); font-size: 0.88rem; } +.prose ul.links li { margin-bottom: 4px; } +.prose .meta { color: var(--fg-dim); font-size: 0.9rem; } + +@media (max-width: 700px) { + .shots { grid-template-columns: 1fr; } + .features { columns: 1; } + .cards { grid-template-columns: 1fr; } + .topbar nav { gap: 14px; } + section { padding: 40px 0; } +}