diff --git a/.ackrc b/.ackrc index 8121069e..d1d22fb3 100644 --- a/.ackrc +++ b/.ackrc @@ -1,4 +1,6 @@ --ignore-file=match:.mjs +--ignore-file=match:frame-timings* --ignore-dir=tempPlugin --ignore-dir=build ---ignore-dir=platforms/android/build-tools +--ignore-dir=old +--ignore-dir=platforms diff --git a/.gitignore b/.gitignore index f780c2fc..b7c078e4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ hooks/ node_modules/ platforms/ +# Release APKs from scripts/release.sh +dist/ + # Logs logs *.log @@ -22,6 +25,7 @@ typings/ .gradle-cursor App_Resources/Android/src/main/jniLibs/**/*.so App_Resources/Android/src/main/assets/faceclaw-voice-asr/ +old # Visual Studio Code .vscode/* @@ -32,3 +36,7 @@ App_Resources/Android/src/main/assets/faceclaw-voice-asr/ # pull_config.sh output: the phone's live settings, including API keys faceclaw_settings.xml + +# Developer configs +build.sh +build_and_run.sh diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 7569cf33..cda991f6 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -16,10 +16,8 @@ dependencies { def faceclawSherpaOnnxVersion = "1.13.0" def faceclawLibLc3Version = "1.1.3" -def faceclawAsrModelName = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27" def faceclawAbi = "arm64-v8a" def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi") -def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr") def faceclawNativeBuildDir = file("$buildDir/faceclaw-native") def faceclawDownload = { String url, File destination -> @@ -67,15 +65,11 @@ tasks.register("prepareFaceclawNativeLibs") { outputs.files( file("$faceclawJniLibsDir/libonnxruntime.so"), file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"), - file("$faceclawJniLibsDir/libfaceclaw_lc3.so"), - file("$faceclawAssetsDir/$faceclawAsrModelName/encoder_model.ort"), - file("$faceclawAssetsDir/$faceclawAsrModelName/decoder_model_merged.ort"), - file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt") + file("$faceclawJniLibsDir/libfaceclaw_lc3.so") ) doLast { faceclawJniLibsDir.mkdirs() - faceclawAssetsDir.mkdirs() def downloadsDir = file("$faceclawNativeBuildDir/downloads") def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2") @@ -92,20 +86,6 @@ tasks.register("prepareFaceclawNativeLibs") { into faceclawJniLibsDir } - def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2") - faceclawDownload( - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2", - asrArchive - ) - copy { - from tarTree(resources.bzip2(asrArchive)) - include "$faceclawAsrModelName/encoder_model.ort" - include "$faceclawAsrModelName/decoder_model_merged.ort" - include "$faceclawAsrModelName/tokens.txt" - includeEmptyDirs = false - into faceclawAssetsDir - } - def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz") faceclawDownload( "https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz", @@ -252,6 +232,22 @@ tasks.register("prepareFaceclawLlama") { } } +// versionName tracks FACECLAW_VERSION in app/version.ts, the single source of +// truth for the Faceclaw version. The gradle root project is platforms/android, +// so the repo root is two levels up from rootDir. +def faceclawVersionName = "0.0" +def faceclawVersionFile = file("$rootDir/../../app/version.ts") +if (faceclawVersionFile.exists()) { + def faceclawVersionMatch = (faceclawVersionFile.text =~ /FACECLAW_VERSION = "([^"]+)"/) + if (faceclawVersionMatch.find()) { + faceclawVersionName = faceclawVersionMatch.group(1) + } +} +def faceclawVersionCode = { + def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger() + return parts[0] * 10000 + parts[1] * 100 + parts[2] +}() + android { compileSdkVersion 35 buildToolsVersion "35" @@ -265,12 +261,17 @@ android { minSdkVersion 24 targetSdkVersion 35 ndk { + // The NativeScript platform build.gradle has already added all four + // ABIs by the time app.gradle is applied, and abiFilters is additive, + // so the set must be cleared for this filter to take effect. Only + // arm64 devices are supported (all our own JNI libs are arm64-only). + abiFilters.clear() abiFilters "arm64-v8a" } - // Version Information - versionCode 1 - versionName "1.0.0" + // Version Information (derived from app/version.ts above) + versionCode faceclawVersionCode + versionName faceclawVersionName generatedDensities = [] } diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/README.md b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/README.md deleted file mode 100644 index acdb1042..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -frameworks: -- 其他 -license: Apache License 2.0 -tasks: -- keyword-spotting ---- - -sherpa 中自定义唤醒词模型,训练数据为 gigaspeech XL (10000 小时)数据,模型大小约为 3.3 M, 建模单元为 BPE 。 使用 icefall 训练,已转换为 onnx 格式,此仓库主要是给 sherpa-onnx 这个 inference 引擎使用。 - -模型结构为 zipformer 模型, 本质是一个非常小的语音识别模型,为了实现唤醒词的功能,我们在解码端做了一些修改和约束。 支持自定义唤醒词,数量不限,效果需要单个调整参数。 - -#### Clone with HTTP -```bash - git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.git -``` \ No newline at end of file diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/bpe.model b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/bpe.model deleted file mode 100644 index 05c2ab6f..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/bpe.model and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx deleted file mode 100644 index 1900ac41..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx deleted file mode 100644 index fbf89f92..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx deleted file mode 100644 index a36c3fcf..00000000 Binary files a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx and /dev/null differ diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords.txt deleted file mode 100644 index abd2cb99..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords.txt +++ /dev/null @@ -1,10 +0,0 @@ -▁HE LL O ▁WORLD -▁HI ▁GO O G LE -▁HE Y ▁S I RI -▁A LE X A -▁LOVE ▁AND ▁P E A CE -▁PLAY ▁MU S IC -▁GO ▁HOME -▁HA PP Y ▁NEW ▁YEAR -▁ME R RY ▁ CH R IST MA S -▁S C RE EN ▁ON :1.5 #0.35 diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords_raw.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords_raw.txt deleted file mode 100644 index c3d6f6d4..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/keywords_raw.txt +++ /dev/null @@ -1,9 +0,0 @@ -HELLO WORLD -HI GOOGLE -HEY SIRI -ALEXA -LOVE AND PEACE -PLAY MUSIC -GO HOME -HAPPY NEW YEAR -MERRY CHRISTMAS diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/screen-on-keywords.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/screen-on-keywords.txt deleted file mode 100644 index dd978326..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/screen-on-keywords.txt +++ /dev/null @@ -1 +0,0 @@ -▁S C RE EN ▁ON :1.5 #0.35 diff --git a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/tokens.txt b/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/tokens.txt deleted file mode 100644 index 61bb1828..00000000 --- a/App_Resources/Android/src/main/assets/faceclaw-voice/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/tokens.txt +++ /dev/null @@ -1,500 +0,0 @@ - 0 - 1 - 2 -S 3 -T 4 -▁THE 5 -▁A 6 -▁TO 7 -▁AND 8 -N 9 -E 10 -D 11 -ING 12 -' 13 -▁OF 14 -▁I 15 -▁IN 16 -Y 17 -ED 18 -M 19 -▁ 20 -▁THAT 21 -O 22 -C 23 -R 24 -A 25 -P 26 -▁YOU 27 -U 28 -RE 29 -▁IT 30 -▁S 31 -G 32 -L 33 -B 34 -▁IS 35 -I 36 -AL 37 -ER 38 -▁RE 39 -▁C 40 -▁WE 41 -▁SO 42 -▁BE 43 -OR 44 -AR 45 -IN 46 -▁FOR 47 -F 48 -▁HE 49 -LE 50 -LY 51 -▁E 52 -▁WAS 53 -K 54 -▁F 55 -W 56 -VE 57 -RI 58 -CH 59 -LL 60 -▁THIS 61 -▁T 62 -EN 63 -▁ON 64 -IT 65 -CE 66 -▁WITH 67 -▁B 68 -▁W 69 -▁ME 70 -▁DO 71 -▁DE 72 -▁HAVE 73 -▁BUT 74 -ENT 75 -▁P 76 -UR 77 -ON 78 -▁AS 79 -IC 80 -TER 81 -TE 82 -▁THEY 83 -ES 84 -RA 85 -▁NOT 86 -TH 87 -▁ARE 88 -RO 89 -IL 90 -ATION 91 -AN 92 -▁ST 93 -▁WHAT 94 -SE 95 -LI 96 -V 97 -▁CAN 98 -▁G 99 -▁OR 100 -▁ALL 101 -NE 102 -▁AT 103 -▁CON 104 -H 105 -▁LIKE 106 -▁MO 107 -▁HIS 108 -▁AN 109 -▁MA 110 -▁THERE 111 -▁SU 112 -▁MY 113 -NG 114 -- 115 -▁ABOUT 116 -▁SE 117 -LA 118 -▁ONE 119 -LO 120 -▁SHE 121 -CK 122 -▁HAD 123 -AD 124 -MENT 125 -VER 126 -▁CA 127 -IR 128 -▁IF 129 -▁NO 130 -EL 131 -▁O 132 -▁FROM 133 -▁PA 134 -▁YOUR 135 -▁KNOW 136 -RY 137 -ION 138 -UN 139 -▁JUST 140 -▁UP 141 -▁US 142 -▁WHO 143 -▁GO 144 -ME 145 -▁EX 146 -▁OUT 147 -ND 148 -CI 149 -▁PO 150 -▁HER 151 -▁FA 152 -▁PRO 153 -▁WHEN 154 -ATE 155 -AT 156 -▁BY 157 -PE 158 -AS 159 -▁LA 160 -GE 161 -▁CO 162 -▁PEOPLE 163 -▁K 164 -▁SOME 165 -IGHT 166 -▁THINK 167 -VI 168 -DE 169 -▁LE 170 -▁WERE 171 -▁LI 172 -OM 173 -▁WOULD 174 -▁SEE 175 -CO 176 -▁BO 177 -▁OUR 178 -▁HOW 179 -ET 180 -Z 181 -UL 182 -▁HO 183 -IG 184 -▁GET 185 -▁DON 186 -IS 187 -URE 188 -▁WHICH 189 -▁MI 190 -IES 191 -US 192 -X 193 -▁MORE 194 -▁NOW 195 -▁TIME 196 -▁UN 197 -▁FI 198 -TY 199 -▁WILL 200 -RU 201 -IVE 202 -▁THEIR 203 -▁SAY 204 -ITY 205 -▁HA 206 -LU 207 -▁RO 208 -HO 209 -OL 210 -IST 211 -AGE 212 -MA 213 -KE 214 -▁RA 215 -ID 216 -ERS 217 -▁LO 218 -▁SP 219 -▁DA 220 -▁SAID 221 -TION 222 -THER 223 -▁EN 224 -▁BECAUSE 225 -▁HAS 226 -EST 227 -TI 228 -ACT 229 -▁BA 230 -▁VERY 231 -▁THEM 232 -▁BEEN 233 -▁WA 234 -ABLE 235 -▁SA 236 -▁DID 237 -▁HIM 238 -▁DIS 239 -▁WANT 240 -▁OTHER 241 -IM 242 -▁PRE 243 -▁LOOK 244 -▁GOING 245 -▁VI 246 -▁WORK 247 -▁DR 248 -ENCE 249 -▁DI 250 -FF 251 -TA 252 -▁REALLY 253 -▁RIGHT 254 -MO 255 -ALLY 256 -▁INTO 257 -OW 258 -▁WAY 259 -▁FE 260 -▁COULD 261 -▁THEN 262 -TED 263 -▁HERE 264 -▁THESE 265 -▁OVER 266 -▁WELL 267 -▁NA 268 -▁ANY 269 -HE 270 -▁COMP 271 -▁COM 272 -▁TH 273 -▁EVEN 274 -▁TWO 275 -▁JO 276 -▁MAKE 277 -▁BACK 278 -▁WHERE 279 -SH 280 -▁QU 281 -MAN 282 -▁YEAR 283 -▁TRA 284 -▁CHA 285 -QU 286 -CA 287 -▁ALSO 288 -CTION 289 -ANCE 290 -UM 291 -PER 292 -DI 293 -▁HU 294 -▁MU 295 -FUL 296 -▁NEW 297 -ARD 298 -J 299 -ICAL 300 -ANT 301 -END 302 -▁MAR 303 -▁NE 304 -IF 305 -DER 306 -ONE 307 -▁THOSE 308 -PP 309 -▁MAN 310 -▁GOOD 311 -▁TAKE 312 -SIDE 313 -PO 314 -HA 315 -▁LITTLE 316 -▁APP 317 -▁BI 318 -UGH 319 -▁COME 320 -ISH 321 -TIC 322 -▁PER 323 -▁THINGS 324 -▁MUCH 325 -▁FIRST 326 -▁THAN 327 -IA 328 -▁AGAIN 329 -▁STA 330 -▁PART 331 -▁GA 332 -▁MEAN 333 -▁LET 334 -▁DOWN 335 -▁NEED 336 -▁GU 337 -LES 338 -▁RU 339 -ATED 340 -▁SHA 341 -VO 342 -▁TALK 343 -▁DAY 344 -▁CHI 345 -▁SOMETHING 346 -▁AFTER 347 -▁THOUGH 348 -▁CALL 349 -▁GRA 350 -▁EVERY 351 -▁KIND 352 -ROW 353 -AIN 354 -▁BU 355 -▁THROUGH 356 -▁SHOULD 357 -▁ONLY 358 -▁TA 359 -▁MIN 360 -▁GOT 361 -IAN 362 -▁GREAT 363 -▁RI 364 -▁DOES 365 -▁THING 366 -DAY 367 -OUS 368 -▁START 369 -▁MOST 370 -▁COL 371 -PH 372 -▁LOT 373 -IZE 374 -BER 375 -IOUS 376 -▁LIFE 377 -▁MANY 378 -▁FEEL 379 -ITION 380 -▁VA 381 -NESS 382 -▁BEFORE 383 -▁THREE 384 -▁WORLD 385 -▁GIVE 386 -▁WHY 387 -▁PI 388 -▁LONG 389 -PORT 390 -▁SHOW 391 -▁ACTUALLY 392 -▁NEVER 393 -IV 394 -FORM 395 -▁THANK 396 -▁HI 397 -▁PUT 398 -▁LOVE 399 -▁QUESTION 400 -▁HELP 401 -▁HAPPEN 402 -▁MR 403 -▁OWN 404 -WARD 405 -▁MADE 406 -▁HAND 407 -▁STATE 408 -IBLE 409 -▁TELL 410 -▁PH 411 -▁CAME 412 -▁TEA 413 -▁STILL 414 -▁DIFFERENT 415 -▁PLACE 416 -▁MIGHT 417 -▁FIND 418 -▁DU 419 -▁IMP 420 -▁OLD 421 -▁PERSON 422 -▁COUNT 423 -▁JA 424 -▁POINT 425 -▁HIGH 426 -SHIP 427 -▁LAST 428 -▁SAME 429 -▁FOUR 430 -▁AROUND 431 -▁OH 432 -▁CHANGE 433 -▁UNDER 434 -▁HOME 435 -▁ANOTHER 436 -▁FOUND 437 -ATIVE 438 -▁TURN 439 -▁YES 440 -LAND 441 -▁ALWAYS 442 -▁YEAH 443 -▁SCHOOL 444 -▁JE 445 -▁NEXT 446 -▁WORD 447 -▁JU 448 -LINE 449 -▁SIX 450 -▁LEARN 451 -▁BOOK 452 -▁PLAY 453 -▁AWAY 454 -▁HOUSE 455 -▁PLAN 456 -▁WHILE 457 -▁COMMUNI 458 -▁FRIEND 459 -▁AMERICA 460 -▁INTEREST 461 -▁KEEP 462 -▁BETTER 463 -▁IDEA 464 -▁HARD 465 -▁TWENTY 466 -▁THOUSAND 467 -▁HUNDRED 468 -▁WEEK 469 -▁NUMBER 470 -▁COURSE 471 -▁OKAY 472 -PRESS 473 -▁SYSTEM 474 -▁NINE 475 -▁UNDERSTAND 476 -▁WITHOUT 477 -▁BELIEVE 478 -▁POWER 479 -▁MAYBE 480 -▁BUILD 481 -▁MONEY 482 -▁BETWEEN 483 -▁GONNA 484 -▁IMPORTANT 485 -▁ANYTHING 486 -▁LEFT 487 -▁HAVING 488 -▁WHITE 489 -▁PROBLEM 490 -▁SECOND 491 -▁SPEAK 492 -▁TOGETHER 493 -▁BLACK 494 -▁PUBLIC 495 -▁HEALTH 496 -▁NOTHING 497 -▁EVERYTHING 498 -Q 499 diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 401808ca..e0a75ab3 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -1,30 +1,20 @@ package com.faceclaw.app; import android.content.Context; -import android.content.res.AssetManager; import android.os.Handler; import android.os.Looper; import android.os.SystemClock; import android.util.Log; import com.k2fsa.sherpa.onnx.FeatureConfig; -import com.k2fsa.sherpa.onnx.KeywordSpotter; -import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; -import com.k2fsa.sherpa.onnx.KeywordSpotterResult; import com.k2fsa.sherpa.onnx.OfflineModelConfig; import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizer; import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; import com.k2fsa.sherpa.onnx.OfflineStream; -import com.k2fsa.sherpa.onnx.OnlineModelConfig; -import com.k2fsa.sherpa.onnx.OnlineStream; -import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.Arrays; @@ -57,17 +47,15 @@ public class FaceclawVoiceController { private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f; private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f; private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80; - private static final String ASSET_ROOT = "faceclaw-voice"; - private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; - private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr"; + // Model directory shared with the TS-side download flow (asr-model.ts), + // which fetches the Moonshine files here on demand; they are no longer + // bundled in the APK. + private static final String ASR_ROOT = "faceclaw-voice-asr"; private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"; - private static final String[] MODEL_FILES = { - "encoder-epoch-12-avg-2-chunk-16-left-64.onnx", - "decoder-epoch-12-avg-2-chunk-16-left-64.onnx", - "joiner-epoch-12-avg-2-chunk-16-left-64.onnx", - "tokens.txt", - "screen-on-keywords.txt" - }; + // Model files for the retired on-phone wake-word spotter, copied to + // filesDir by earlier releases; deleted on sight to reclaim the space. + // (The wakeword is now detected by the glasses firmware itself.) + private static final String LEGACY_KWS_ROOT = "faceclaw-voice"; private static final String[] ASR_MODEL_FILES = { "encoder_model.ort", "decoder_model_merged.ort", @@ -75,7 +63,6 @@ public class FaceclawVoiceController { }; private enum VoiceInputMode { - WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired) ONBOARD, // on-phone Moonshine transcription CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side } @@ -89,10 +76,8 @@ public class FaceclawVoiceController { private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; - private VoiceInputMode mode = VoiceInputMode.WAKEWORD; - private KeywordSpotter keywordSpotter; + private VoiceInputMode mode = VoiceInputMode.CLOUD; private OfflineRecognizer recognizer; - private OnlineStream stream; private FaceclawLc3Decoder lc3Decoder; private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES]; private int transcriptSampleCount; @@ -140,10 +125,6 @@ public class FaceclawVoiceController { this.endpointing = endpointing; } - public void start() { - start("wakeword"); - } - public void start(String requestedMode) { synchronized (lock) { if (started) { @@ -194,26 +175,23 @@ public class FaceclawVoiceController { if ("cloud".equals(requestedMode)) { return VoiceInputMode.CLOUD; } - if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) { - return VoiceInputMode.ONBOARD; - } - return VoiceInputMode.WAKEWORD; + return VoiceInputMode.ONBOARD; } private void runLoop() { try { + deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; if (currentMode == VoiceInputMode.ONBOARD) { + File modelDir = findAsrModelDir(); + if (modelDir == null) { + emitStatus("Voice model not downloaded (see Settings > Voice)."); + return; + } emitStatus("Loading transcription model..."); - File modelDir = installAsrModelFiles(); recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); resetTranscriptState(); lastTranscript = ""; - } else if (currentMode == VoiceInputMode.WAKEWORD) { - emitStatus("Loading wake-word model..."); - File modelDir = installModelFiles(); - keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); - stream = keywordSpotter.createStream(); } lc3Decoder = new FaceclawLc3Decoder(); endpointDetector.reset(); @@ -225,9 +203,7 @@ public class FaceclawVoiceController { emitStatus(currentMode == VoiceInputMode.CLOUD ? "Listening (cloud)..." - : currentMode == VoiceInputMode.ONBOARD - ? "Listening..." - : "Listening for \"screen on\"..."); + : "Listening..."); processG2Audio(); // Button released / stop requested: emit one final full-utterance // transcript so the UI can freeze it. @@ -312,29 +288,6 @@ public class FaceclawVoiceController { return b.array(); } - private KeywordSpotterConfig buildConfig(File modelDir) { - return KeywordSpotterConfig.builder() - .setFeatureConfig(FeatureConfig.builder() - .setSampleRate(SAMPLE_RATE) - .setFeatureDim(FEATURE_DIM) - .build()) - .setOnlineModelConfig(OnlineModelConfig.builder() - .setTransducer(OnlineTransducerModelConfig.builder() - .setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setModelType("zipformer2") - .setModelingUnit("") - .setNumThreads(1) - .build()) - .setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath()) - .setKeywordsScore(1.5f) - .setKeywordsThreshold(0.35f) - .build(); - } - private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) { return OfflineRecognizerConfig.builder() .setFeatureConfig(FeatureConfig.builder() @@ -352,50 +305,42 @@ public class FaceclawVoiceController { .build(); } - private File installModelFiles() throws IOException { - File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR); - if (!modelDir.exists() && !modelDir.mkdirs()) { - throw new IOException("Could not create " + modelDir.getAbsolutePath()); - } - AssetManager assets = appContext.getAssets(); - for (String fileName : MODEL_FILES) { - copyAssetIfNeeded( - assets, - ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName, - new File(modelDir, fileName) - ); - } - return modelDir; - } - - private File installAsrModelFiles() throws IOException { - File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR); - if (!modelDir.exists() && !modelDir.mkdirs()) { - throw new IOException("Could not create " + modelDir.getAbsolutePath()); - } - AssetManager assets = appContext.getAssets(); + /** + * The Moonshine model directory, populated by the download flow in + * asr-model.ts (releases before 0.5.0 copied the same files out of the + * APK, so upgraded installs are already complete). Null when any file is + * missing, i.e. the model still needs to be downloaded. + */ + private File findAsrModelDir() { + File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR); for (String fileName : ASR_MODEL_FILES) { - copyAssetIfNeeded( - assets, - ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName, - new File(modelDir, fileName) - ); - } - return modelDir; - } - - private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException { - if (destination.exists() && destination.length() > 0) { - return; - } - try (InputStream input = assets.open(assetPath); - FileOutputStream output = new FileOutputStream(destination)) { - byte[] buffer = new byte[64 * 1024]; - int read; - while ((read = input.read(buffer)) >= 0) { - output.write(buffer, 0, read); + File file = new File(modelDir, fileName); + if (!file.exists() || file.length() == 0) { + return null; } } + return modelDir; + } + + private void deleteLegacyKwsFiles() { + try { + deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT)); + } catch (Throwable t) { + Log.w(TAG, "failed to delete legacy wake-word files", t); + } + } + + private static void deleteRecursively(File file) { + if (!file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteRecursively(child); + } + } + file.delete(); } private boolean startG2Audio() { @@ -413,9 +358,8 @@ public class FaceclawVoiceController { private void processG2Audio() { short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET]; while (started && !Thread.currentThread().isInterrupted()) { - OnlineStream currentStream = stream; FaceclawLc3Decoder currentDecoder = lc3Decoder; - if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) { + if (currentDecoder == null) { return; } @@ -438,40 +382,17 @@ public class FaceclawVoiceController { } if (mode == VoiceInputMode.CLOUD) { emitPcm(pcm, count); - } else if (mode == VoiceInputMode.ONBOARD) { - float[] samples = new float[count]; - for (int i = 0; i < count; i++) { - samples[i] = pcm[i] / 32768.0f; - } - processRecognizer(samples); } else { float[] samples = new float[count]; for (int i = 0; i < count; i++) { samples[i] = pcm[i] / 32768.0f; } - currentStream.acceptWaveform(samples, SAMPLE_RATE); - processKeywordSpotter(currentStream); + processRecognizer(samples); } maybeEmitAudioStats(false); } } - private void processKeywordSpotter(OnlineStream currentStream) { - KeywordSpotter currentSpotter = keywordSpotter; - if (currentSpotter == null) { - return; - } - while (currentSpotter.isReady(currentStream)) { - currentSpotter.decode(currentStream); - KeywordSpotterResult result = currentSpotter.getResult(currentStream); - String keyword = result == null ? "" : result.getKeyword(); - if (keyword != null && keyword.trim().length() > 0) { - currentSpotter.reset(currentStream); - emitWakeWord(keyword); - } - } - } - private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); long now = SystemClock.elapsedRealtime(); @@ -783,14 +704,6 @@ public class FaceclawVoiceController { } private void releaseSherpa() { - if (stream != null) { - stream.release(); - stream = null; - } - if (keywordSpotter != null) { - keywordSpotter.release(); - keywordSpotter = null; - } if (recognizer != null) { recognizer.release(); recognizer = null; @@ -893,14 +806,6 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onStatus(status)); } - private void emitWakeWord(String keyword) { - FaceclawVoiceControllerListener currentListener = listener; - if (currentListener == null) { - return; - } - mainHandler.post(() -> currentListener.onWakeWord(keyword)); - } - private void emitTranscript(String text, boolean isFinal) { FaceclawVoiceControllerListener currentListener = listener; if (currentListener == null) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index bce73665..504a5df3 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -3,8 +3,6 @@ package com.faceclaw.app; public interface FaceclawVoiceControllerListener { void onStatus(String status); - void onWakeWord(String keyword); - /** * Best transcript of the current utterance so far. REPLACE semantics: text * is the complete transcript, not a delta — display it verbatim, replacing diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java deleted file mode 100644 index 836a07c2..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotter { - private long ptr; - - public KeywordSpotter(KeywordSpotterConfig config) { - System.loadLibrary("onnxruntime"); - System.loadLibrary("sherpa-onnx-jni"); - ptr = newFromFile(config); - if (ptr == 0) { - throw new IllegalArgumentException("Invalid KeywordSpotterConfig"); - } - } - - public OnlineStream createStream() { - return createStream(""); - } - - public OnlineStream createStream(String keywords) { - return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords)); - } - - public boolean isReady(OnlineStream stream) { - return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr()); - } - - public void decode(OnlineStream stream) { - decode(ptr, stream.getPtr()); - } - - public void reset(OnlineStream stream) { - reset(ptr, stream.getPtr()); - } - - public KeywordSpotterResult getResult(OnlineStream stream) { - return getResult(ptr, stream.getPtr()); - } - - public void release() { - if (ptr == 0) { - return; - } - delete(ptr); - ptr = 0; - } - - @Override - protected void finalize() throws Throwable { - release(); - super.finalize(); - } - - private native long newFromFile(KeywordSpotterConfig config); - - private native void delete(long ptr); - - private native long createStream(long ptr, String keywords); - - private native boolean isReady(long ptr, long streamPtr); - - private native void decode(long ptr, long streamPtr); - - private native void reset(long ptr, long streamPtr); - - private native KeywordSpotterResult getResult(long ptr, long streamPtr); -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java deleted file mode 100644 index 3c4ce6f3..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotterConfig { - private final FeatureConfig featConfig; - private final OnlineModelConfig modelConfig; - private final int maxActivePaths; - private final String keywordsFile; - private final float keywordsScore; - private final float keywordsThreshold; - private final int numTrailingBlanks; - - private KeywordSpotterConfig(Builder builder) { - this.featConfig = builder.featConfig; - this.modelConfig = builder.modelConfig; - this.maxActivePaths = builder.maxActivePaths; - this.keywordsFile = builder.keywordsFile; - this.keywordsScore = builder.keywordsScore; - this.keywordsThreshold = builder.keywordsThreshold; - this.numTrailingBlanks = builder.numTrailingBlanks; - } - - public static Builder builder() { - return new Builder(); - } - - public FeatureConfig getFeatConfig() { - return featConfig; - } - - public OnlineModelConfig getModelConfig() { - return modelConfig; - } - - public int getMaxActivePaths() { - return maxActivePaths; - } - - public String getKeywordsFile() { - return keywordsFile; - } - - public float getKeywordsScore() { - return keywordsScore; - } - - public float getKeywordsThreshold() { - return keywordsThreshold; - } - - public int getNumTrailingBlanks() { - return numTrailingBlanks; - } - - public static class Builder { - private FeatureConfig featConfig = FeatureConfig.builder().build(); - private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build(); - private int maxActivePaths = 4; - private String keywordsFile = "keywords.txt"; - private float keywordsScore = 1.5f; - private float keywordsThreshold = 0.25f; - private int numTrailingBlanks = 2; - - public Builder setFeatureConfig(FeatureConfig featConfig) { - this.featConfig = featConfig; - return this; - } - - public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) { - this.modelConfig = modelConfig; - return this; - } - - public Builder setKeywordsFile(String keywordsFile) { - this.keywordsFile = keywordsFile; - return this; - } - - public Builder setKeywordsScore(float keywordsScore) { - this.keywordsScore = keywordsScore; - return this; - } - - public Builder setKeywordsThreshold(float keywordsThreshold) { - this.keywordsThreshold = keywordsThreshold; - return this; - } - - public KeywordSpotterConfig build() { - return new KeywordSpotterConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java deleted file mode 100644 index 77e3a72e..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class KeywordSpotterResult { - private final String keyword; - private final String[] tokens; - private final float[] timestamps; - - public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) { - this.keyword = keyword == null ? "" : keyword; - this.tokens = tokens == null ? new String[0] : tokens; - this.timestamps = timestamps == null ? new float[0] : timestamps; - } - - public String getKeyword() { - return keyword; - } - - public String[] getTokens() { - return tokens; - } - - public float[] getTimestamps() { - return timestamps; - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java deleted file mode 100644 index d3f0db66..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineModelConfig { - private final OnlineTransducerModelConfig transducer; - private final OnlineParaformerModelConfig paraformer; - private final OnlineZipformer2CtcModelConfig zipformer2Ctc; - private final OnlineNeMoCtcModelConfig neMoCtc; - private final OnlineToneCtcModelConfig toneCtc; - private final String tokens; - private final int numThreads; - private final boolean debug; - private final String provider; - private final String modelType; - private final String modelingUnit; - private final String bpeVocab; - - private OnlineModelConfig(Builder builder) { - this.transducer = builder.transducer; - this.paraformer = builder.paraformer; - this.zipformer2Ctc = builder.zipformer2Ctc; - this.neMoCtc = builder.neMoCtc; - this.toneCtc = builder.toneCtc; - this.tokens = builder.tokens; - this.numThreads = builder.numThreads; - this.debug = builder.debug; - this.provider = builder.provider; - this.modelType = builder.modelType; - this.modelingUnit = builder.modelingUnit; - this.bpeVocab = builder.bpeVocab; - } - - public static Builder builder() { - return new Builder(); - } - - public OnlineTransducerModelConfig getTransducer() { - return transducer; - } - - public OnlineParaformerModelConfig getParaformer() { - return paraformer; - } - - public OnlineZipformer2CtcModelConfig getZipformer2Ctc() { - return zipformer2Ctc; - } - - public OnlineNeMoCtcModelConfig getNeMoCtc() { - return neMoCtc; - } - - public OnlineToneCtcModelConfig getToneCtc() { - return toneCtc; - } - - public String getTokens() { - return tokens; - } - - public int getNumThreads() { - return numThreads; - } - - public boolean getDebug() { - return debug; - } - - public String getProvider() { - return provider; - } - - public String getModelType() { - return modelType; - } - - public String getModelingUnit() { - return modelingUnit; - } - - public String getBpeVocab() { - return bpeVocab; - } - - public static class Builder { - private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build(); - private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build(); - private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build(); - private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build(); - private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build(); - private String tokens = ""; - private int numThreads = 1; - private boolean debug = false; - private String provider = "cpu"; - private String modelType = ""; - private String modelingUnit = "cjkchar"; - private String bpeVocab = ""; - - public Builder setTransducer(OnlineTransducerModelConfig transducer) { - this.transducer = transducer; - return this; - } - - public Builder setTokens(String tokens) { - this.tokens = tokens; - return this; - } - - public Builder setModelType(String modelType) { - this.modelType = modelType; - return this; - } - - public Builder setNumThreads(int numThreads) { - this.numThreads = numThreads; - return this; - } - - public Builder setProvider(String provider) { - this.provider = provider; - return this; - } - - public Builder setModelingUnit(String modelingUnit) { - this.modelingUnit = modelingUnit; - return this; - } - - public Builder setBpeVocab(String bpeVocab) { - this.bpeVocab = bpeVocab; - return this; - } - - public OnlineModelConfig build() { - return new OnlineModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java deleted file mode 100644 index 1982804c..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineNeMoCtcModelConfig { - private final String model; - - private OnlineNeMoCtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineNeMoCtcModelConfig build() { - return new OnlineNeMoCtcModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java deleted file mode 100644 index 3e44e2c5..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineParaformerModelConfig { - private final String encoder; - private final String decoder; - - private OnlineParaformerModelConfig(Builder builder) { - this.encoder = builder.encoder; - this.decoder = builder.decoder; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEncoder() { - return encoder; - } - - public String getDecoder() { - return decoder; - } - - public static class Builder { - private String encoder = ""; - private String decoder = ""; - - public Builder setEncoder(String encoder) { - this.encoder = encoder; - return this; - } - - public Builder setDecoder(String decoder) { - this.decoder = decoder; - return this; - } - - public OnlineParaformerModelConfig build() { - return new OnlineParaformerModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java deleted file mode 100644 index 90a249cc..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineStream { - private long ptr; - - public OnlineStream(long ptr) { - this.ptr = ptr; - } - - public long getPtr() { - return ptr; - } - - public void acceptWaveform(float[] samples, int sampleRate) { - acceptWaveform(ptr, samples, sampleRate); - } - - public void release() { - if (ptr == 0) { - return; - } - delete(ptr); - ptr = 0; - } - - @Override - protected void finalize() throws Throwable { - release(); - super.finalize(); - } - - private native void acceptWaveform(long ptr, float[] samples, int sampleRate); - - private native void delete(long ptr); -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java deleted file mode 100644 index a847e79f..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineToneCtcModelConfig { - private final String model; - - private OnlineToneCtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineToneCtcModelConfig build() { - return new OnlineToneCtcModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java deleted file mode 100644 index ca003533..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineTransducerModelConfig { - private final String encoder; - private final String decoder; - private final String joiner; - - private OnlineTransducerModelConfig(Builder builder) { - this.encoder = builder.encoder; - this.decoder = builder.decoder; - this.joiner = builder.joiner; - } - - public static Builder builder() { - return new Builder(); - } - - public String getEncoder() { - return encoder; - } - - public String getDecoder() { - return decoder; - } - - public String getJoiner() { - return joiner; - } - - public static class Builder { - private String encoder = ""; - private String decoder = ""; - private String joiner = ""; - - public Builder setEncoder(String encoder) { - this.encoder = encoder; - return this; - } - - public Builder setDecoder(String decoder) { - this.decoder = decoder; - return this; - } - - public Builder setJoiner(String joiner) { - this.joiner = joiner; - return this; - } - - public OnlineTransducerModelConfig build() { - return new OnlineTransducerModelConfig(this); - } - } -} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java deleted file mode 100644 index d73f73dd..00000000 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.k2fsa.sherpa.onnx; - -public class OnlineZipformer2CtcModelConfig { - private final String model; - - private OnlineZipformer2CtcModelConfig(Builder builder) { - this.model = builder.model; - } - - public static Builder builder() { - return new Builder(); - } - - public String getModel() { - return model; - } - - public static class Builder { - private String model = ""; - - public Builder setModel(String model) { - this.model = model; - return this; - } - - public OnlineZipformer2CtcModelConfig build() { - return new OnlineZipformer2CtcModelConfig(this); - } - } -} diff --git a/app/apps/developer/accelerometer-demo.ts b/app/apps/developer/accelerometer-demo.ts index 56d14700..2c1d11c4 100644 --- a/app/apps/developer/accelerometer-demo.ts +++ b/app/apps/developer/accelerometer-demo.ts @@ -116,7 +116,6 @@ export class AccelerometerDemoLayer implements Layer { image.drawText(small, 22, 82, "accelerometer at all — experimental.", 120); } - image.drawText(small, 20, height - 16, `${GESTURE_DOUBLE_CLICK} back`, 110); return image; } diff --git a/app/apps/developer/buzzer-demo.ts b/app/apps/developer/buzzer-demo.ts index 9f0d9309..08642ca7 100644 --- a/app/apps/developer/buzzer-demo.ts +++ b/app/apps/developer/buzzer-demo.ts @@ -16,7 +16,7 @@ import { const HEADER_HEIGHT = 30; const LIST_X = 20; -const FOOTER_HEIGHT = 22; +const FOOTER_HEIGHT = 0; function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -58,7 +58,6 @@ export class BuzzerDemoLayer implements Layer { image.drawText(font, LIST_X + 110, y + 1, effect.desc, selected ? 180 : 120); } - image.drawText(font, 20, height - 16, `${GESTURE_SCROLL} select ${GESTURE_CLICK} play ${GESTURE_DOUBLE_CLICK} back`, 110); return image; } diff --git a/app/apps/developer/developer-app.ts b/app/apps/developer/developer-app.ts index 44326911..aeac1b9c 100644 --- a/app/apps/developer/developer-app.ts +++ b/app/apps/developer/developer-app.ts @@ -25,6 +25,9 @@ const MENU_LAYOUT = { showBorder: false, minHeight: 0, maxHeight: appViewportSize("min").height - 16, + // These menus are pages, not popups: "Debug tests" replaces the root menu + // visually instead of letting the taller root show through beneath it. + opaque: true, }; /** A row that opens a nested page: the label plus a right-edge ">". */ diff --git a/app/apps/developer/resource-usage.ts b/app/apps/developer/resource-usage.ts index 27df5ea1..5e7cbad4 100644 --- a/app/apps/developer/resource-usage.ts +++ b/app/apps/developer/resource-usage.ts @@ -78,7 +78,7 @@ export class ResourceUsageLayer implements Layer { image.drawText( font, width - 16 - font.measureText(`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`), - 37, + 47 - font.lineHeight, `CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`, 155, ); @@ -105,7 +105,7 @@ export class ResourceUsageLayer implements Layer { ]); image.drawText(font, 16, height - 16, `-60s`, 80); - const nowLabel = `now ${GESTURE_DOUBLE_CLICK} back`; + const nowLabel = `now`; image.drawText(font, width - 16 - font.measureText(nowLabel), height - 16, nowLabel, 110); return image; } diff --git a/app/apps/music/music-app.ts b/app/apps/music/music-app.ts index 7a73b57f..1f48186b 100644 --- a/app/apps/music/music-app.ts +++ b/app/apps/music/music-app.ts @@ -5,7 +5,6 @@ import { clamp } from "../../util/numeric-util"; import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, - GESTURE_SCROLL, GESTURE_SCROLL_DOWN, GESTURE_SCROLL_UP, } from "../../ui/gestures"; @@ -29,16 +28,19 @@ import { export const MUSIC_WINDOW_ID = "music"; export const MUSIC_SURFACE_ID = "window:music"; -const ART_SIZE = 96; -const ART_X = 22; +const ART_SIZE = 112; +const ART_X = 8; // left margin matches the top margin const ART_Y = 8; const META_X = ART_X + ART_SIZE + 14; -const LIST_TOP = 120; +const PROGRESS_TIME_Y = ART_Y + 100; +const PROGRESS_BAR_Y = ART_Y + 116; +const LIST_TOP = 136; +/** Gap between the menus' last row band and the viewport bottom. */ +const LIST_BOTTOM_MARGIN = 4; const ACTION_X = 22; const ACTION_WIDTH = 154; const COLUMN_DIVIDER_X = 190; const QUEUE_X = 204; -const FOOTER_HEIGHT = 20; const PLAYLIST_ACTION_INDEX = 1; type MusicAction = @@ -83,7 +85,7 @@ class MusicAppLayer implements Layer { for (let index = 0; index < lines.length; index++) { image.drawText(font, 24, 16 + index * lineStep(font), lines[index]!, 180); } - image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings ${GESTURE_DOUBLE_CLICK} back`, 110); + image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings`, 110); return image; } @@ -92,10 +94,9 @@ class MusicAppLayer implements Layer { if (mediaBrowserBridge.listBrowsableApps().length) { image.drawText(font, 24, 22 + lineStep(font), "Click to browse a music app's library,", 150); image.drawText(font, 24, 22 + 2 * lineStep(font), "or start playback on the phone.", 150); - image.drawText(font, 20, height - 16, `${GESTURE_CLICK} browse ${GESTURE_DOUBLE_CLICK} back`, 110); + image.drawText(font, 20, height - 16, `${GESTURE_CLICK} browse`, 110); } else { image.drawText(font, 24, 22 + lineStep(font), "Start playback in another app on the phone.", 150); - image.drawText(font, 20, height - 16, `${GESTURE_DOUBLE_CLICK} back`, 110); } return image; } @@ -104,21 +105,36 @@ class MusicAppLayer implements Layer { const metaWidth = width - META_X - 24; const metaStep = lineStep(font) + 1; - const titleLines = wrapText(font, media.title || "Unknown title", metaWidth).slice(0, 2); + const metaTop = ART_Y + 2; + // Line budget above the progress-bar time line: the layout is five slots + // (title, second title line, artist, album, player app); when the font is + // too large for all five, the title gets one truncated line instead of + // wrapping to two, and the rest shift up a slot. + const maxLines = Math.floor((PROGRESS_TIME_Y - metaTop - 4 - font.lineHeight) / metaStep) + 1; + const titleSlots = maxLines >= 5 ? 2 : 1; + const title = media.title || "Unknown title"; + const titleLines = + titleSlots === 2 ? wrapText(font, title, metaWidth).slice(0, 2) : [truncateText(font, title, metaWidth)]; for (let index = 0; index < titleLines.length; index++) { - image.drawText(font, META_X, ART_Y + 2 + index * metaStep, titleLines[index]!, 230); + image.drawText(font, META_X, metaTop + index * metaStep, titleLines[index]!, 230); } - image.drawText(font, META_X, ART_Y + 6 + 2 * metaStep, media.artist || "Unknown artist", 180); + image.drawText(font, META_X, metaTop + 4 + titleSlots * metaStep, media.artist || "Unknown artist", 180); if (media.album) { - image.drawText(font, META_X, ART_Y + 6 + 3 * metaStep, truncateText(font, media.album, metaWidth), 150); + image.drawText(font, META_X, metaTop + 4 + (titleSlots + 1) * metaStep, truncateText(font, media.album, metaWidth), 150); } - image.drawText(font, META_X, ART_Y + 6 + 4 * metaStep, truncateText(font, media.appName || media.packageName, metaWidth), 110); + image.drawText( + font, + META_X, + metaTop + 4 + (titleSlots + 2) * metaStep, + truncateText(font, media.appName || media.packageName, metaWidth), + 110, + ); this.drawProgress(image, media, metaWidth); const queue = mediaControllerBridge.getQueue(); const actions = this.buildActions(media, queue); this.reconcileSelection(actions, queue); - const listHeight = height - LIST_TOP - FOOTER_HEIGHT; + const listHeight = height - LIST_TOP - LIST_BOTTOM_MARGIN; const rowH = tightRowHeight(font); const visibleRows = Math.max(1, (listHeight / rowH) | 0); if (this.selectedQueueIndex < this.queueScrollRow) { @@ -154,7 +170,7 @@ class MusicAppLayer implements Layer { } } - image.drawLine(COLUMN_DIVIDER_X, LIST_TOP - 3, COLUMN_DIVIDER_X, height - FOOTER_HEIGHT - 3, 45); + image.drawLine(COLUMN_DIVIDER_X, LIST_TOP - 3, COLUMN_DIVIDER_X, height - LIST_BOTTOM_MARGIN - 3, 45); if (!queue.length) { image.drawText(font, QUEUE_X, LIST_TOP + 1, "Playlist unavailable", 90); } else { @@ -189,14 +205,6 @@ class MusicAppLayer implements Layer { } } - const backTarget = this.focusColumn === "playlist" ? "actions" : "back"; - image.drawText( - font, - 20, - height - 16, - `${GESTURE_SCROLL} select ${GESTURE_CLICK} activate ${GESTURE_DOUBLE_CLICK} ${backTarget}`, - 110, - ); return image; } @@ -333,13 +341,12 @@ class MusicAppLayer implements Layer { private drawProgress(image: GrayImage, media: MediaControllerState, width: number): void { if (media.durationMs <= 0 || media.positionMs < 0) return; const font = getDefaultSmallFont(); - const y = ART_Y + 84; const elapsed = formatMediaTime(media.positionMs); const duration = formatMediaTime(media.durationMs); - image.drawText(font, META_X, y, elapsed, 140); - image.drawText(font, META_X + width - font.measureText(duration), y, duration, 140); + image.drawText(font, META_X, PROGRESS_TIME_Y, elapsed, 140); + image.drawText(font, META_X + width - font.measureText(duration), PROGRESS_TIME_Y, duration, 140); - const barY = ART_Y + 100; + const barY = PROGRESS_BAR_Y; image.drawRect(META_X, barY, width, 5, 55); const progress = clamp(media.positionMs / media.durationMs, 0, 1); image.fillRect(META_X + 1, barY + 1, Math.round((width - 2) * progress), 3, 170); @@ -347,7 +354,10 @@ class MusicAppLayer implements Layer { private drawArt(image: GrayImage, media: MediaControllerState): void { const key = `${media.packageName}|${media.title}|${media.album}`; - if (key !== this.artKey) { + // Retry while null: players (e.g. Spotify) often publish metadata before + // the art bitmap is downloaded, then re-emit the same strings with the + // bitmap attached. The no-art fetch path is a cheap near-no-op. + if (key !== this.artKey || this.art === null) { this.artKey = key; this.art = mediaControllerBridge.getAlbumArt(ART_SIZE); } @@ -360,7 +370,13 @@ class MusicAppLayer implements Layer { } else { image.drawRect(ART_X, ART_Y, ART_SIZE, ART_SIZE, 60); const font = getDefaultSmallFont(); - image.drawText(font, ART_X + 22, ART_Y + ART_SIZE / 2 - 7, "no art", 90); + image.drawText( + font, + ART_X + Math.round((ART_SIZE - font.measureText("no art")) / 2), + ART_Y + Math.round((ART_SIZE - font.lineHeight) / 2), + "no art", + 90, + ); } } } diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 26735a88..4f3f1712 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -195,7 +195,6 @@ class DashboardController { private offFrameMetrics: (() => void) | null = null; private offFirmwareInfo: (() => void) | null = null; private offVoiceStatus: (() => void) | null = null; - private offVoiceWakeWord: (() => void) | null = null; private offAndroidNotification: (() => void) | null = null; private lastInput = "waiting..."; private lastSys = "none yet"; @@ -1011,11 +1010,6 @@ class DashboardController { this.offVoiceStatus = voiceControlBridge.onStatus((state) => { this.appendLog(state.status); }); - this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => { - void this.handleWakeWord(keyword).catch((error) => { - this.appendLog(`wake-word handler failed: ${this.formatError(error)}`); - }); - }); await mediaControllerBridge.start(); await nightscoutBridge.start(); @@ -1082,8 +1076,6 @@ class DashboardController { this.offFirmwareInfo = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; - this.offVoiceWakeWord?.(); - this.offVoiceWakeWord = null; await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); @@ -1182,8 +1174,6 @@ class DashboardController { this.offFirmwareInfo = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; - this.offVoiceWakeWord?.(); - this.offVoiceWakeWord = null; const communicator = this.communicator; this.communicator = null; @@ -1816,20 +1806,6 @@ class DashboardController { this.updateCompositePreview(); } - private async handleWakeWord(keyword: string): Promise { - const normalized = keyword.trim(); - if (normalized.length === 0) return; - this.appendLog(`wake-word detected: ${normalized}`); - const woke = shell.wake("sidebar"); - if (woke) { - const ready = await this.ensureEvenHubSessionActive(); - if (!ready) { - this.appendLog("EvenHub wake barrier timed out for phone wakeword"); - } - this.requestShellRender(); - } - } - private async handleAndroidNotificationPosted(notificationKey: string): Promise { if (!notificationKey) { this.requestShellRender(); diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts new file mode 100644 index 00000000..3c71e31a --- /dev/null +++ b/app/native/asr-model.ts @@ -0,0 +1,176 @@ +import { Utils } from "@nativescript/core"; + +declare const com: any; +declare const java: any; + +/** + * Download management for the on-device transcription model (Moonshine, via + * sherpa-onnx). The model is no longer bundled in the APK; it is fetched on + * demand into the same filesDir location that earlier releases copied the + * bundled files to, so upgraded installs that already used on-device + * transcription need no download. FaceclawVoiceController reads the files + * from this directory. + * + * Mirrors the on-phone assistant model flow in llama.ts, except the model is + * three files rather than one; they download sequentially through + * FaceclawModelDownloader (resume + pinned sha256 per file). + */ + +type AsrModelFile = { + name: string; + sha256: string; + sizeBytes: number; +}; + +export const ASR_MODEL = { + label: "Moonshine (English)", + dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27", + // Hugging Face mirror of the sherpa-onnx release asset of the same name. + // The GitHub release only offers a tar.bz2, which the phone can't unpack; + // this mirror serves the same files (sha256-verified) individually. + baseUrl: + "https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/", + files: [ + { + name: "decoder_model_merged.ort", + sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d", + sizeBytes: 109424400, + }, + { + name: "encoder_model.ort", + sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf", + sizeBytes: 31326816, + }, + { + name: "tokens.txt", + sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049", + sizeBytes: 549350, + }, + ] as AsrModelFile[], + totalBytes: 141300566, +}; + +export type AsrModelState = { + status: "absent" | "downloading" | "ready"; + bytesDownloaded: number; + totalBytes: number; +}; + +let downloader: any = null; +// Bytes of files already fully downloaded in this run, plus progress within +// the file currently downloading; drives the aggregate percentage. +let completedBytes = 0; +let currentFileBytes = 0; +const stateListeners = new Set<(state: AsrModelState) => void>(); + +function modelDirPath(): string { + const context = Utils.android.getApplicationContext(); + return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`; +} + +function filePath(file: AsrModelFile): string { + return `${modelDirPath()}/${file.name}`; +} + +function isFilePresent(file: AsrModelFile): boolean { + try { + const javaFile = new java.io.File(filePath(file)); + return javaFile.exists() && javaFile.length() > 0; + } catch { + return false; + } +} + +export function isAsrModelReady(): boolean { + if (!global.isAndroid) return false; + return ASR_MODEL.files.every(isFilePresent); +} + +export function asrModelState(): AsrModelState { + if (downloader) { + return { + status: "downloading", + bytesDownloaded: completedBytes + currentFileBytes, + totalBytes: ASR_MODEL.totalBytes, + }; + } + return { + status: isAsrModelReady() ? "ready" : "absent", + bytesDownloaded: 0, + totalBytes: ASR_MODEL.totalBytes, + }; +} + +export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void { + stateListeners.add(listener); + return () => stateListeners.delete(listener); +} + +function notifyStateChanged(): void { + const state = asrModelState(); + stateListeners.forEach((listener) => listener(state)); +} + +export function startAsrModelDownload(): void { + if (!global.isAndroid || downloader || isAsrModelReady()) return; + completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0); + downloadNextFile(); + notifyStateChanged(); +} + +function downloadNextFile(): void { + const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file)); + if (!nextFile) { + downloader = null; + notifyStateChanged(); + return; + } + currentFileBytes = 0; + const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({ + onProgress: (bytes: number, _total: number) => { + currentFileBytes = Number(bytes); + notifyStateChanged(); + }, + onDone: () => { + completedBytes += nextFile.sizeBytes; + currentFileBytes = 0; + downloadNextFile(); + }, + onError: (message: string) => { + console.error(`Voice model download failed (${nextFile.name}): ${message}`); + downloader = null; + notifyStateChanged(); + }, + }); + downloader = new com.faceclaw.app.FaceclawModelDownloader( + `${ASR_MODEL.baseUrl}${nextFile.name}`, + filePath(nextFile), + nextFile.sha256, + nextFile.sizeBytes, + listener, + ); + downloader.start(); +} + +/** Stops the download; already-fetched bytes are kept and resumed next time. */ +export function cancelAsrModelDownload(): void { + if (!downloader) return; + downloader.cancel(); + downloader = null; + notifyStateChanged(); +} + +export function deleteAsrModel(): void { + if (!global.isAndroid) return; + cancelAsrModelDownload(); + try { + for (const file of ASR_MODEL.files) { + new java.io.File(filePath(file)).delete(); + new java.io.File(`${filePath(file)}.part`).delete(); + } + new java.io.File(modelDirPath()).delete(); + } catch (error) { + console.error(`Voice model delete failed: ${String(error)}`); + } + notifyStateChanged(); +} diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index a5686680..121acf09 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -45,7 +45,6 @@ export type RawPcmListener = (pcm: Uint8Array) => void; export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); - private readonly wakeWordListeners = new Set<(keyword: string) => void>(); private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>(); private readonly speechEndListeners = new Set<() => void>(); private controller: any | null = null; @@ -72,11 +71,6 @@ export class FaceclawVoiceControlBridge { return () => this.statusListeners.delete(listener); } - onWakeWord(listener: (keyword: string) => void): () => void { - this.wakeWordListeners.add(listener); - return () => this.wakeWordListeners.delete(listener); - } - onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void { this.transcriptListeners.add(listener); return () => this.transcriptListeners.delete(listener); @@ -266,11 +260,6 @@ export class FaceclawVoiceControlBridge { if (this.rawActive) return; this.setStatus(String(status)); }, - onWakeWord: (keyword: string) => { - for (const listener of this.wakeWordListeners) { - listener(String(keyword)); - } - }, onTranscript: (text: string, isFinal: boolean) => { this.emitTranscript(String(text), Boolean(isFinal)); }, diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 3f9694db..bae6c9f6 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -319,12 +319,12 @@ export const voiceProviderSetting = new ConfigSettingEnum({ if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0; return false; }, - description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription.", + description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).", }); const wakeWordActionLabels: Record = { "voice-input": "Voice Input", - off: "Off", + off: "Ignore", "turn-screen-on": "Turn Screen On", }; diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 447e2494..f55e269d 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -12,6 +12,14 @@ import { onLocalModelStateChanged, startLocalModelDownload, } from "../../native/llama"; +import { + ASR_MODEL, + asrModelState, + cancelAsrModelDownload, + deleteAsrModel, + onAsrModelStateChanged, + startAsrModelDownload, +} from "../../native/asr-model"; import { TextViewerLayer } from "../../apps/files/text-viewer"; import type { LayerContext } from "../layers"; import { drawRightValueMenuItem, openModalMenu, type MenuItem } from "../menu"; @@ -87,6 +95,7 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), + asrModelMenuItem(), ], }, { @@ -240,6 +249,75 @@ function localModelMenuItem(): MenuItem { }; } +const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`; + +let asrModelRenderUnsub: (() => void) | null = null; + +function watchAsrModelDownload(ctx: LayerContext): void { + asrModelRenderUnsub?.(); + asrModelRenderUnsub = onAsrModelStateChanged((state) => { + ctx.actions.requestRender(); + if (state.status !== "downloading") { + asrModelRenderUnsub?.(); + asrModelRenderUnsub = null; + } + }); +} + +function asrModelStatusText(): string { + const state = asrModelState(); + if (state.status === "ready") return "downloaded"; + if (state.status === "downloading") { + const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0; + return `${pct}% of ${ASR_MODEL_MB}`; + } + return "not downloaded"; +} + +/** Download/cancel/delete management for the on-device transcription model. */ +function asrModelMenuItem(): MenuItem { + return { + label: "On-device voice model", + description: + `${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` + + "Transcribes voice input on the phone itself, with no API key or cloud service. " + + "Required for the On-device transcription provider; the cloud providers work without it. " + + "An interrupted download resumes where it left off.", + onSelect: (ctx) => { + const state = asrModelState(); + const action: MenuItem = + state.status === "downloading" + ? { + label: "Cancel download", + onSelect: (innerCtx) => { + cancelAsrModelDownload(); + innerCtx.stack.pop(); + }, + } + : state.status === "ready" + ? { + label: "Delete model", + onSelect: (innerCtx) => { + deleteAsrModel(); + innerCtx.stack.pop(); + }, + } + : { + label: `Download (${ASR_MODEL_MB})`, + onSelect: (innerCtx) => { + startAsrModelDownload(); + watchAsrModelDownload(innerCtx); + innerCtx.stack.pop(); + }, + }; + openModalMenu(ctx, "On-device voice model", [action], 0); + }, + render: ({ image, x, y, width }) => { + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText()); + }, + }; +} + /** A row that opens one of the project docs (copied into the bundle under * about/ by webpack.config.js) in the paged text viewer. */ function bundledDocMenuItem(fileName: string, label: string): MenuItem { diff --git a/app/ui/font-picker.ts b/app/ui/font-picker.ts index 6f45ee86..c7388eed 100644 --- a/app/ui/font-picker.ts +++ b/app/ui/font-picker.ts @@ -142,7 +142,6 @@ export class FontPickerLayer implements Layer { const info = `Line height: ${preview.lineHeight}px`; image.drawText(font, rowX - 4, previewTop + 8 + preview.lineHeight + 6, info, 110); - image.drawText(font, rowX - 4, height - font.lineHeight - 8, `scroll select click change ${GESTURE_DOUBLE_CLICK} cancel`, 90); return image; } @@ -215,7 +214,7 @@ export class FontPickerLayer implements Layer { this.draft.kind === "bitmap" ? Math.max(0, this.options.bitmapFaces.findIndex((b) => b.face === (this.draft as { face: BitmapFace }).face)) : this.options.bitmapFaces.length + this.families.indexOf(this.draft.family); - openModalMenu(ctx, "Font", items, Math.max(0, currentIndex)); + openModalMenu(ctx, "Font Face", items, Math.max(0, currentIndex)); } /** Switch families, carrying the weight (nearest available) and size over. */ diff --git a/app/ui/menu.ts b/app/ui/menu.ts index 304c1b42..8dcc1f16 100644 --- a/app/ui/menu.ts +++ b/app/ui/menu.ts @@ -26,6 +26,12 @@ export type MenuLayout = { minHeight?: number; /** Height cap before the menu starts scrolling. Default: the full screen. */ maxHeight?: number; + /** + * Paint as a standalone page: the layers below stay in the stack for back + * navigation but are not composited underneath. Default: false — the menu + * renders over whatever is below it (modal/window menus). + */ + opaque?: boolean; }; export type MenuItemRenderArgs = { @@ -192,7 +198,8 @@ export class MenuLayer implements Layer { paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { const font = getDefaultSmallFont(); const rowHeight = listRowHeight(font); - const image = paintBelow(); + const base = ctx.stack.getBaseSize(); + const image = this.layout.opaque ? new GrayImage(base.width, base.height, 0) : paintBelow(); const { x, y, width } = this.layout; const chromeTop = (this.title ? menuTitleHeight(font) : 0) + MENU_BODY_PADDING; const minHeight = this.layout.minHeight ?? DEFAULT_MENU_MIN_HEIGHT; @@ -333,7 +340,7 @@ export class TextPageLayer implements Layer { private readonly body: string, ) {} - paint(ctx: LayerContext): GrayImage { + paint(_ctx: LayerContext): GrayImage { const font = getDefaultSmallFont(); const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); image.drawText(font, 18, 14, this.title, 220); diff --git a/app/ui/shell/assistant.ts b/app/ui/shell/assistant.ts index 837fce90..9accaacc 100644 --- a/app/ui/shell/assistant.ts +++ b/app/ui/shell/assistant.ts @@ -129,7 +129,7 @@ export class AssistantLayer implements Layer { image.drawText(font, left + 4, rowY + 2, rows[i]!, selected ? 255 : 200); } } else { - image.drawText(font, left, top + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110); + image.drawText(font, left, top + DIALOG_H - 14, `${GESTURE_DOUBLE_CLICK} cancel`, 110 - font.lineHeight); } return image; } diff --git a/app/ui/shell/voice-input.ts b/app/ui/shell/voice-input.ts index 62f1926d..6491f6c7 100644 --- a/app/ui/shell/voice-input.ts +++ b/app/ui/shell/voice-input.ts @@ -268,7 +268,7 @@ export class VoiceInputLayer implements Layer { image.drawText(font, left + 4, rowY + 2, row.label, row.dim ? 90 : selected ? 255 : 200); } } else { - image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110); + image.drawText(font, left, top + DIALOG_H - 14, this.hintText(), 110-font.lineHeight); } return image; } diff --git a/app/version.ts b/app/version.ts index 33cc77a9..d08823aa 100644 --- a/app/version.ts +++ b/app/version.ts @@ -1,2 +1,2 @@ -export const FACECLAW_VERSION = "0.4.1"; +export const FACECLAW_VERSION = "0.5"; export const USER_AGENT = "Faceclaw/"+FACECLAW_VERSION; diff --git a/package.json b/package.json index 0cb7c125..b82d42fc 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "scripts": { "build": "nativescript build android", "clean": "rm -rf platforms", + "release": "scripts/release.sh", "test": "tsc -p tests/tsconfig.json && node --test tests/*.test.cjs" }, "dependencies": { diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 00000000..dc68483f --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Build a signed release APK at dist/Faceclaw-.apk, where +# comes from FACECLAW_VERSION in app/version.ts. +# +# Prompts for the keystore passphrase. Override the keystore location with +# ANDROID_KEYSTORE=/path/to/store.jks. +# +# Note: the passphrase is passed to the nativescript CLI on its command line, +# so it is briefly visible in `ps` on this machine while the build runs. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +KEYSTORE="${ANDROID_KEYSTORE:-$HOME/repositories/AndroidKeystore/jimrandomh-android-keystore.jks}" +if [ ! -f "$KEYSTORE" ]; then + echo "Keystore not found: $KEYSTORE (set ANDROID_KEYSTORE to override)" >&2 + exit 1 +fi + +# build.sh is the machine-specific wrapper that exports JAVA_HOME/ANDROID_HOME. +# Pick up its exports so keytool and the build use the same JDK. +if [ -f build.sh ]; then + eval "$(grep '^export ' build.sh)" +fi +if ! command -v keytool >/dev/null; then + echo "keytool not found; set JAVA_HOME (usually via build.sh)" >&2 + exit 1 +fi + +VERSION="$(sed -n 's/^export const FACECLAW_VERSION = "\([^"]*\)".*/\1/p' app/version.ts)" +if [ -z "$VERSION" ]; then + echo "Could not read FACECLAW_VERSION from app/version.ts" >&2 + exit 1 +fi + +read -r -s -p "Keystore passphrase for $(basename "$KEYSTORE"): " STORE_PASS +echo + +# Validate the passphrase up front and discover the signing-key alias. +if ! LISTING="$(keytool -list -keystore "$KEYSTORE" -storepass "$STORE_PASS" 2>&1)"; then + printf '%s\n' "$LISTING" >&2 + exit 1 +fi +ALIAS="$(printf '%s\n' "$LISTING" | awk -F', ' '/PrivateKeyEntry/{print $1; exit}')" +if [ -z "$ALIAS" ]; then + echo "No private-key entry found in $KEYSTORE:" >&2 + printf '%s\n' "$LISTING" >&2 + exit 1 +fi +echo "Signing as '$ALIAS', version $VERSION" + +read -r -s -p "Key passphrase for '$ALIAS' (empty = same as keystore): " KEY_PASS +echo +KEY_PASS="${KEY_PASS:-$STORE_PASS}" + +OUT="dist/Faceclaw-$VERSION.apk" +mkdir -p dist +rm -f "$OUT" + +npx nativescript build android --release \ + --key-store-path "$KEYSTORE" \ + --key-store-password "$STORE_PASS" \ + --key-store-alias "$ALIAS" \ + --key-store-alias-password "$KEY_PASS" \ + --copy-to "$OUT" + +echo +echo "Built $OUT" +if [ -n "${ANDROID_HOME:-}" ] && [ -d "$ANDROID_HOME/build-tools" ]; then + APKSIGNER="$(printf '%s\n' "$ANDROID_HOME/build-tools"/*/apksigner | sort -V | tail -1)" + [ -x "$APKSIGNER" ] && "$APKSIGNER" verify --print-certs "$OUT" | head -5 +fi