diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index 3d9393c0..8d852af6 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -2,3 +2,6 @@ Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficia and others who contributed to documenting the G2's bluetooth protocol. It has enabled me to keep my room clean, so to speak. +Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/ +and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa +community. diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 9f60d1a5..abda62fd 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -8,9 +8,16 @@ android { buildToolsVersion "35" // ndkVersion "" + sourceSets.main { + jniLibs.srcDirs += ["$projectDir/src/main/jniLibs"] + } + defaultConfig { minSdkVersion 24 targetSdkVersion 35 + ndk { + abiFilters "arm64-v8a" + } // Version Information versionCode 1 diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml index 144a5326..5dfb457e 100644 --- a/App_Resources/Android/src/main/AndroidManifest.xml +++ b/App_Resources/Android/src/main/AndroidManifest.xml @@ -19,8 +19,11 @@ + + + + android:foregroundServiceType="connectedDevice|microphone" /> 0 + 1 + 2 +S 3 +T 4 +▁THE 5 +▁A 6 +▁TO 7 +▁AND 8 +N 9 +E 10 +D 11 +ING 12 +' 13 +▁OF 14 +▁I 15 +▁IN 16 +Y 17 +ED 18 +M 19 +▁ 20 +▁THAT 21 +O 22 +C 23 +R 24 +A 25 +P 26 +▁YOU 27 +U 28 +RE 29 +▁IT 30 +▁S 31 +G 32 +L 33 +B 34 +▁IS 35 +I 36 +AL 37 +ER 38 +▁RE 39 +▁C 40 +▁WE 41 +▁SO 42 +▁BE 43 +OR 44 +AR 45 +IN 46 +▁FOR 47 +F 48 +▁HE 49 +LE 50 +LY 51 +▁E 52 +▁WAS 53 +K 54 +▁F 55 +W 56 +VE 57 +RI 58 +CH 59 +LL 60 +▁THIS 61 +▁T 62 +EN 63 +▁ON 64 +IT 65 +CE 66 +▁WITH 67 +▁B 68 +▁W 69 +▁ME 70 +▁DO 71 +▁DE 72 +▁HAVE 73 +▁BUT 74 +ENT 75 +▁P 76 +UR 77 +ON 78 +▁AS 79 +IC 80 +TER 81 +TE 82 +▁THEY 83 +ES 84 +RA 85 +▁NOT 86 +TH 87 +▁ARE 88 +RO 89 +IL 90 +ATION 91 +AN 92 +▁ST 93 +▁WHAT 94 +SE 95 +LI 96 +V 97 +▁CAN 98 +▁G 99 +▁OR 100 +▁ALL 101 +NE 102 +▁AT 103 +▁CON 104 +H 105 +▁LIKE 106 +▁MO 107 +▁HIS 108 +▁AN 109 +▁MA 110 +▁THERE 111 +▁SU 112 +▁MY 113 +NG 114 +- 115 +▁ABOUT 116 +▁SE 117 +LA 118 +▁ONE 119 +LO 120 +▁SHE 121 +CK 122 +▁HAD 123 +AD 124 +MENT 125 +VER 126 +▁CA 127 +IR 128 +▁IF 129 +▁NO 130 +EL 131 +▁O 132 +▁FROM 133 +▁PA 134 +▁YOUR 135 +▁KNOW 136 +RY 137 +ION 138 +UN 139 +▁JUST 140 +▁UP 141 +▁US 142 +▁WHO 143 +▁GO 144 +ME 145 +▁EX 146 +▁OUT 147 +ND 148 +CI 149 +▁PO 150 +▁HER 151 +▁FA 152 +▁PRO 153 +▁WHEN 154 +ATE 155 +AT 156 +▁BY 157 +PE 158 +AS 159 +▁LA 160 +GE 161 +▁CO 162 +▁PEOPLE 163 +▁K 164 +▁SOME 165 +IGHT 166 +▁THINK 167 +VI 168 +DE 169 +▁LE 170 +▁WERE 171 +▁LI 172 +OM 173 +▁WOULD 174 +▁SEE 175 +CO 176 +▁BO 177 +▁OUR 178 +▁HOW 179 +ET 180 +Z 181 +UL 182 +▁HO 183 +IG 184 +▁GET 185 +▁DON 186 +IS 187 +URE 188 +▁WHICH 189 +▁MI 190 +IES 191 +US 192 +X 193 +▁MORE 194 +▁NOW 195 +▁TIME 196 +▁UN 197 +▁FI 198 +TY 199 +▁WILL 200 +RU 201 +IVE 202 +▁THEIR 203 +▁SAY 204 +ITY 205 +▁HA 206 +LU 207 +▁RO 208 +HO 209 +OL 210 +IST 211 +AGE 212 +MA 213 +KE 214 +▁RA 215 +ID 216 +ERS 217 +▁LO 218 +▁SP 219 +▁DA 220 +▁SAID 221 +TION 222 +THER 223 +▁EN 224 +▁BECAUSE 225 +▁HAS 226 +EST 227 +TI 228 +ACT 229 +▁BA 230 +▁VERY 231 +▁THEM 232 +▁BEEN 233 +▁WA 234 +ABLE 235 +▁SA 236 +▁DID 237 +▁HIM 238 +▁DIS 239 +▁WANT 240 +▁OTHER 241 +IM 242 +▁PRE 243 +▁LOOK 244 +▁GOING 245 +▁VI 246 +▁WORK 247 +▁DR 248 +ENCE 249 +▁DI 250 +FF 251 +TA 252 +▁REALLY 253 +▁RIGHT 254 +MO 255 +ALLY 256 +▁INTO 257 +OW 258 +▁WAY 259 +▁FE 260 +▁COULD 261 +▁THEN 262 +TED 263 +▁HERE 264 +▁THESE 265 +▁OVER 266 +▁WELL 267 +▁NA 268 +▁ANY 269 +HE 270 +▁COMP 271 +▁COM 272 +▁TH 273 +▁EVEN 274 +▁TWO 275 +▁JO 276 +▁MAKE 277 +▁BACK 278 +▁WHERE 279 +SH 280 +▁QU 281 +MAN 282 +▁YEAR 283 +▁TRA 284 +▁CHA 285 +QU 286 +CA 287 +▁ALSO 288 +CTION 289 +ANCE 290 +UM 291 +PER 292 +DI 293 +▁HU 294 +▁MU 295 +FUL 296 +▁NEW 297 +ARD 298 +J 299 +ICAL 300 +ANT 301 +END 302 +▁MAR 303 +▁NE 304 +IF 305 +DER 306 +ONE 307 +▁THOSE 308 +PP 309 +▁MAN 310 +▁GOOD 311 +▁TAKE 312 +SIDE 313 +PO 314 +HA 315 +▁LITTLE 316 +▁APP 317 +▁BI 318 +UGH 319 +▁COME 320 +ISH 321 +TIC 322 +▁PER 323 +▁THINGS 324 +▁MUCH 325 +▁FIRST 326 +▁THAN 327 +IA 328 +▁AGAIN 329 +▁STA 330 +▁PART 331 +▁GA 332 +▁MEAN 333 +▁LET 334 +▁DOWN 335 +▁NEED 336 +▁GU 337 +LES 338 +▁RU 339 +ATED 340 +▁SHA 341 +VO 342 +▁TALK 343 +▁DAY 344 +▁CHI 345 +▁SOMETHING 346 +▁AFTER 347 +▁THOUGH 348 +▁CALL 349 +▁GRA 350 +▁EVERY 351 +▁KIND 352 +ROW 353 +AIN 354 +▁BU 355 +▁THROUGH 356 +▁SHOULD 357 +▁ONLY 358 +▁TA 359 +▁MIN 360 +▁GOT 361 +IAN 362 +▁GREAT 363 +▁RI 364 +▁DOES 365 +▁THING 366 +DAY 367 +OUS 368 +▁START 369 +▁MOST 370 +▁COL 371 +PH 372 +▁LOT 373 +IZE 374 +BER 375 +IOUS 376 +▁LIFE 377 +▁MANY 378 +▁FEEL 379 +ITION 380 +▁VA 381 +NESS 382 +▁BEFORE 383 +▁THREE 384 +▁WORLD 385 +▁GIVE 386 +▁WHY 387 +▁PI 388 +▁LONG 389 +PORT 390 +▁SHOW 391 +▁ACTUALLY 392 +▁NEVER 393 +IV 394 +FORM 395 +▁THANK 396 +▁HI 397 +▁PUT 398 +▁LOVE 399 +▁QUESTION 400 +▁HELP 401 +▁HAPPEN 402 +▁MR 403 +▁OWN 404 +WARD 405 +▁MADE 406 +▁HAND 407 +▁STATE 408 +IBLE 409 +▁TELL 410 +▁PH 411 +▁CAME 412 +▁TEA 413 +▁STILL 414 +▁DIFFERENT 415 +▁PLACE 416 +▁MIGHT 417 +▁FIND 418 +▁DU 419 +▁IMP 420 +▁OLD 421 +▁PERSON 422 +▁COUNT 423 +▁JA 424 +▁POINT 425 +▁HIGH 426 +SHIP 427 +▁LAST 428 +▁SAME 429 +▁FOUR 430 +▁AROUND 431 +▁OH 432 +▁CHANGE 433 +▁UNDER 434 +▁HOME 435 +▁ANOTHER 436 +▁FOUND 437 +ATIVE 438 +▁TURN 439 +▁YES 440 +LAND 441 +▁ALWAYS 442 +▁YEAH 443 +▁SCHOOL 444 +▁JE 445 +▁NEXT 446 +▁WORD 447 +▁JU 448 +LINE 449 +▁SIX 450 +▁LEARN 451 +▁BOOK 452 +▁PLAY 453 +▁AWAY 454 +▁HOUSE 455 +▁PLAN 456 +▁WHILE 457 +▁COMMUNI 458 +▁FRIEND 459 +▁AMERICA 460 +▁INTEREST 461 +▁KEEP 462 +▁BETTER 463 +▁IDEA 464 +▁HARD 465 +▁TWENTY 466 +▁THOUSAND 467 +▁HUNDRED 468 +▁WEEK 469 +▁NUMBER 470 +▁COURSE 471 +▁OKAY 472 +PRESS 473 +▁SYSTEM 474 +▁NINE 475 +▁UNDERSTAND 476 +▁WITHOUT 477 +▁BELIEVE 478 +▁POWER 479 +▁MAYBE 480 +▁BUILD 481 +▁MONEY 482 +▁BETWEEN 483 +▁GONNA 484 +▁IMPORTANT 485 +▁ANYTHING 486 +▁LEFT 487 +▁HAVING 488 +▁WHITE 489 +▁PROBLEM 490 +▁SECOND 491 +▁SPEAK 492 +▁TOGETHER 493 +▁BLACK 494 +▁PUBLIC 495 +▁HEALTH 496 +▁NOTHING 497 +▁EVERYTHING 498 +Q 499 diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 7c877350..50175243 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -27,7 +27,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private static final int WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE; private static final int IMAGE_WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT; - private static final int DESIRED_MTU = 247; + private static final int DESIRED_MTU = 512; private static final int CONNECTION_PRIORITY_HIGH = BluetoothGatt.CONNECTION_PRIORITY_HIGH; private static final int CONNECT_TIMEOUT_MS = 5_000; @@ -40,7 +40,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private static final int BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000; private static final int BATTERY_INPUT_QUIET_MS = 5_000; private static final int WINDOW_SIZE = 3; - private static final int IMAGE_FRAGMENT_SIZE = 1000; + private static final int IMAGE_FRAGMENT_SIZE = 3800; private static final int IMAGE_RETRY_DELAY_MS = 10_000; private static final boolean IMAGE_FRAGMENT_NO_ACK = false; private static final int WARMUP_FRAGMENT_TIMEOUT_MS = 3_000; diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java index 8cf6c136..b96672c3 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java @@ -6,6 +6,7 @@ import android.app.NotificationManager; import android.app.PendingIntent; import android.app.Service; import android.content.Intent; +import android.content.pm.ServiceInfo; import android.os.Build; import android.os.IBinder; @@ -41,7 +42,11 @@ public class FaceclawForegroundService extends Service { text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected" ); - startForeground(NOTIFICATION_ID, notification); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + startForeground(NOTIFICATION_ID, notification, foregroundServiceType()); + } else { + startForeground(NOTIFICATION_ID, notification); + } if (ACTION_UPDATE.equals(action)) { NotificationManager manager = getSystemService(NotificationManager.class); @@ -95,4 +100,9 @@ public class FaceclawForegroundService extends Service { .setOnlyAlertOnce(true) .build(); } + + private int foregroundServiceType() { + return ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE + | ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE; + } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java new file mode 100644 index 00000000..9597818a --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -0,0 +1,341 @@ +package com.faceclaw.app; + +import android.Manifest; +import android.bluetooth.BluetoothAdapter; +import android.bluetooth.BluetoothDevice; +import android.bluetooth.BluetoothManager; +import android.bluetooth.BluetoothProfile; +import android.content.Context; +import android.content.pm.PackageManager; +import android.content.res.AssetManager; +import android.media.AudioDeviceInfo; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; +import android.util.Log; + +import androidx.core.content.ContextCompat; + +import com.k2fsa.sherpa.onnx.FeatureConfig; +import com.k2fsa.sherpa.onnx.KeywordSpotter; +import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; +import com.k2fsa.sherpa.onnx.KeywordSpotterResult; +import com.k2fsa.sherpa.onnx.OnlineModelConfig; +import com.k2fsa.sherpa.onnx.OnlineStream; +import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; + +public class FaceclawVoiceController { + private static final String TAG = "FaceclawVoice"; + private static final int SAMPLE_RATE = 16000; + private static final int FEATURE_DIM = 80; + private static final String ASSET_ROOT = "faceclaw-voice"; + private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; + private static final String[] MODEL_FILES = { + "encoder-epoch-12-avg-2-chunk-16-left-64.onnx", + "decoder-epoch-12-avg-2-chunk-16-left-64.onnx", + "joiner-epoch-12-avg-2-chunk-16-left-64.onnx", + "tokens.txt", + "screen-on-keywords.txt" + }; + + private final Context appContext; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private final Object lock = new Object(); + private volatile FaceclawVoiceControllerListener listener; + private Thread workerThread; + private volatile boolean started; + private AudioRecord audioRecord; + private KeywordSpotter keywordSpotter; + private OnlineStream stream; + + public FaceclawVoiceController(Context context) { + this.appContext = context.getApplicationContext(); + } + + public void setListener(FaceclawVoiceControllerListener listener) { + this.listener = listener; + } + + public void start() { + synchronized (lock) { + if (started) { + emitStatus("Voice control is already listening."); + return; + } + if (!hasRecordAudioPermission()) { + emitStatus("Voice control needs microphone permission."); + return; + } + if (!hasBluetoothHeadset()) { + emitStatus("Voice control needs a paired Bluetooth headset."); + return; + } + started = true; + workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); + workerThread.start(); + } + } + + public void stop() { + Thread threadToJoin; + synchronized (lock) { + if (!started) { + return; + } + started = false; + threadToJoin = workerThread; + } + stopAudio(); + if (threadToJoin != null) { + threadToJoin.interrupt(); + } + } + + public void close() { + stop(); + } + + private void runLoop() { + try { + emitStatus("Voice control loading wake-word model..."); + File modelDir = installModelFiles(); + keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); + stream = keywordSpotter.createStream(); + if (!startAudio()) { + emitStatus("Voice control could not start microphone input."); + return; + } + + emitStatus("Voice control listening for \"screen on\"."); + processAudio(); + } catch (Throwable error) { + Log.e(TAG, "Voice control failed", error); + emitStatus("Voice control failed: " + error.getMessage()); + } finally { + releaseSherpa(); + stopAudio(); + synchronized (lock) { + started = false; + workerThread = null; + } + } + } + + private KeywordSpotterConfig buildConfig(File modelDir) { + return KeywordSpotterConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setOnlineModelConfig(OnlineModelConfig.builder() + .setTransducer(OnlineTransducerModelConfig.builder() + .setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) + .setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) + .setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath()) + .build()) + .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) + .setModelType("zipformer2") + .setNumThreads(1) + .build()) + .setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath()) + .setKeywordsScore(1.5f) + .setKeywordsThreshold(0.35f) + .build(); + } + + private File installModelFiles() throws IOException { + File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR); + if (!modelDir.exists() && !modelDir.mkdirs()) { + throw new IOException("Could not create " + modelDir.getAbsolutePath()); + } + AssetManager assets = appContext.getAssets(); + for (String fileName : MODEL_FILES) { + copyAssetIfNeeded( + assets, + ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName, + new File(modelDir, fileName) + ); + } + return modelDir; + } + + private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException { + if (destination.exists() && destination.length() > 0) { + return; + } + try (InputStream input = assets.open(assetPath); + FileOutputStream output = new FileOutputStream(destination)) { + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = input.read(buffer)) >= 0) { + output.write(buffer, 0, read); + } + } + } + + private boolean startAudio() { + routeToBluetoothHeadset(); + int minBufferBytes = AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ); + if (minBufferBytes <= 0) { + return false; + } + if (!hasRecordAudioPermission()) { + return false; + } + audioRecord = new AudioRecord( + MediaRecorder.AudioSource.VOICE_RECOGNITION, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + minBufferBytes * 2 + ); + if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { + audioRecord.release(); + audioRecord = null; + return false; + } + audioRecord.startRecording(); + return true; + } + + private void processAudio() { + short[] buffer = new short[SAMPLE_RATE / 10]; + while (started && !Thread.currentThread().isInterrupted()) { + AudioRecord currentAudio = audioRecord; + OnlineStream currentStream = stream; + KeywordSpotter currentSpotter = keywordSpotter; + if (currentAudio == null || currentStream == null || currentSpotter == null) { + return; + } + + int count = currentAudio.read(buffer, 0, buffer.length); + if (count <= 0) { + continue; + } + + float[] samples = new float[count]; + for (int i = 0; i < count; i++) { + samples[i] = buffer[i] / 32768.0f; + } + currentStream.acceptWaveform(samples, SAMPLE_RATE); + + while (currentSpotter.isReady(currentStream)) { + currentSpotter.decode(currentStream); + KeywordSpotterResult result = currentSpotter.getResult(currentStream); + String keyword = result == null ? "" : result.getKeyword(); + if (keyword != null && keyword.trim().length() > 0) { + currentSpotter.reset(currentStream); + emitWakeWord(keyword); + } + } + } + } + + private void stopAudio() { + AudioRecord record = audioRecord; + audioRecord = null; + if (record != null) { + try { + record.stop(); + } catch (IllegalStateException ignored) { + } + record.release(); + } + AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE); + if (audioManager != null) { + if (Build.VERSION.SDK_INT >= 31) { + audioManager.clearCommunicationDevice(); + } + audioManager.stopBluetoothSco(); + audioManager.setBluetoothScoOn(false); + } + } + + private void releaseSherpa() { + if (stream != null) { + stream.release(); + stream = null; + } + if (keywordSpotter != null) { + keywordSpotter.release(); + keywordSpotter = null; + } + } + + private void routeToBluetoothHeadset() { + AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE); + if (audioManager == null) { + return; + } + audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION); + audioManager.startBluetoothSco(); + audioManager.setBluetoothScoOn(true); + if (Build.VERSION.SDK_INT >= 31) { + for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) { + if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + || device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) { + audioManager.setCommunicationDevice(device); + return; + } + } + } + } + + private boolean hasBluetoothHeadset() { + if (Build.VERSION.SDK_INT >= 31 + && ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT) + != PackageManager.PERMISSION_GRANTED) { + return false; + } + BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE); + BluetoothAdapter adapter = manager == null ? null : manager.getAdapter(); + if (adapter == null) { + return false; + } + try { + int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET); + if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) { + return true; + } + Set bondedDevices = adapter.getBondedDevices(); + return bondedDevices != null && !bondedDevices.isEmpty(); + } catch (SecurityException ignored) { + return false; + } + } + + private boolean hasRecordAudioPermission() { + return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO) + == PackageManager.PERMISSION_GRANTED; + } + + private void emitStatus(String status) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onStatus(status)); + } + + private void emitWakeWord(String keyword) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onWakeWord(keyword)); + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java new file mode 100644 index 00000000..60b306b5 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -0,0 +1,7 @@ +package com.faceclaw.app; + +public interface FaceclawVoiceControllerListener { + void onStatus(String status); + + void onWakeWord(String keyword); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.java new file mode 100644 index 00000000..1dd5c677 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.java @@ -0,0 +1,54 @@ +package com.k2fsa.sherpa.onnx; + +public class FeatureConfig { + private final int sampleRate; + private final int featureDim; + private final float dither; + + private FeatureConfig(Builder builder) { + this.sampleRate = builder.sampleRate; + this.featureDim = builder.featureDim; + this.dither = builder.dither; + } + + public static Builder builder() { + return new Builder(); + } + + public int getSampleRate() { + return sampleRate; + } + + public int getFeatureDim() { + return featureDim; + } + + public float getDither() { + return dither; + } + + public static class Builder { + private int sampleRate = 16000; + private int featureDim = 80; + private float dither = 0.0f; + + public Builder setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + return this; + } + + public Builder setFeatureDim(int featureDim) { + this.featureDim = featureDim; + return this; + } + + public Builder setDither(float dither) { + this.dither = dither; + return this; + } + + public FeatureConfig build() { + return new FeatureConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java new file mode 100644 index 00000000..836a07c2 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java @@ -0,0 +1,66 @@ +package com.k2fsa.sherpa.onnx; + +public class KeywordSpotter { + private long ptr; + + public KeywordSpotter(KeywordSpotterConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid KeywordSpotterConfig"); + } + } + + public OnlineStream createStream() { + return createStream(""); + } + + public OnlineStream createStream(String keywords) { + return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords)); + } + + public boolean isReady(OnlineStream stream) { + return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr()); + } + + public void decode(OnlineStream stream) { + decode(ptr, stream.getPtr()); + } + + public void reset(OnlineStream stream) { + reset(ptr, stream.getPtr()); + } + + public KeywordSpotterResult getResult(OnlineStream stream) { + return getResult(ptr, stream.getPtr()); + } + + public void release() { + if (ptr == 0) { + return; + } + delete(ptr); + ptr = 0; + } + + @Override + protected void finalize() throws Throwable { + release(); + super.finalize(); + } + + private native long newFromFile(KeywordSpotterConfig config); + + private native void delete(long ptr); + + private native long createStream(long ptr, String keywords); + + private native boolean isReady(long ptr, long streamPtr); + + private native void decode(long ptr, long streamPtr); + + private native void reset(long ptr, long streamPtr); + + private native KeywordSpotterResult getResult(long ptr, long streamPtr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java new file mode 100644 index 00000000..3c4ce6f3 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java @@ -0,0 +1,92 @@ +package com.k2fsa.sherpa.onnx; + +public class KeywordSpotterConfig { + private final FeatureConfig featConfig; + private final OnlineModelConfig modelConfig; + private final int maxActivePaths; + private final String keywordsFile; + private final float keywordsScore; + private final float keywordsThreshold; + private final int numTrailingBlanks; + + private KeywordSpotterConfig(Builder builder) { + this.featConfig = builder.featConfig; + this.modelConfig = builder.modelConfig; + this.maxActivePaths = builder.maxActivePaths; + this.keywordsFile = builder.keywordsFile; + this.keywordsScore = builder.keywordsScore; + this.keywordsThreshold = builder.keywordsThreshold; + this.numTrailingBlanks = builder.numTrailingBlanks; + } + + public static Builder builder() { + return new Builder(); + } + + public FeatureConfig getFeatConfig() { + return featConfig; + } + + public OnlineModelConfig getModelConfig() { + return modelConfig; + } + + public int getMaxActivePaths() { + return maxActivePaths; + } + + public String getKeywordsFile() { + return keywordsFile; + } + + public float getKeywordsScore() { + return keywordsScore; + } + + public float getKeywordsThreshold() { + return keywordsThreshold; + } + + public int getNumTrailingBlanks() { + return numTrailingBlanks; + } + + public static class Builder { + private FeatureConfig featConfig = FeatureConfig.builder().build(); + private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build(); + private int maxActivePaths = 4; + private String keywordsFile = "keywords.txt"; + private float keywordsScore = 1.5f; + private float keywordsThreshold = 0.25f; + private int numTrailingBlanks = 2; + + public Builder setFeatureConfig(FeatureConfig featConfig) { + this.featConfig = featConfig; + return this; + } + + public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) { + this.modelConfig = modelConfig; + return this; + } + + public Builder setKeywordsFile(String keywordsFile) { + this.keywordsFile = keywordsFile; + return this; + } + + public Builder setKeywordsScore(float keywordsScore) { + this.keywordsScore = keywordsScore; + return this; + } + + public Builder setKeywordsThreshold(float keywordsThreshold) { + this.keywordsThreshold = keywordsThreshold; + return this; + } + + public KeywordSpotterConfig build() { + return new KeywordSpotterConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java new file mode 100644 index 00000000..77e3a72e --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java @@ -0,0 +1,25 @@ +package com.k2fsa.sherpa.onnx; + +public class KeywordSpotterResult { + private final String keyword; + private final String[] tokens; + private final float[] timestamps; + + public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) { + this.keyword = keyword == null ? "" : keyword; + this.tokens = tokens == null ? new String[0] : tokens; + this.timestamps = timestamps == null ? new float[0] : timestamps; + } + + public String getKeyword() { + return keyword; + } + + public String[] getTokens() { + return tokens; + } + + public float[] getTimestamps() { + return timestamps; + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java new file mode 100644 index 00000000..d3f0db66 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java @@ -0,0 +1,137 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineModelConfig { + private final OnlineTransducerModelConfig transducer; + private final OnlineParaformerModelConfig paraformer; + private final OnlineZipformer2CtcModelConfig zipformer2Ctc; + private final OnlineNeMoCtcModelConfig neMoCtc; + private final OnlineToneCtcModelConfig toneCtc; + private final String tokens; + private final int numThreads; + private final boolean debug; + private final String provider; + private final String modelType; + private final String modelingUnit; + private final String bpeVocab; + + private OnlineModelConfig(Builder builder) { + this.transducer = builder.transducer; + this.paraformer = builder.paraformer; + this.zipformer2Ctc = builder.zipformer2Ctc; + this.neMoCtc = builder.neMoCtc; + this.toneCtc = builder.toneCtc; + this.tokens = builder.tokens; + this.numThreads = builder.numThreads; + this.debug = builder.debug; + this.provider = builder.provider; + this.modelType = builder.modelType; + this.modelingUnit = builder.modelingUnit; + this.bpeVocab = builder.bpeVocab; + } + + public static Builder builder() { + return new Builder(); + } + + public OnlineTransducerModelConfig getTransducer() { + return transducer; + } + + public OnlineParaformerModelConfig getParaformer() { + return paraformer; + } + + public OnlineZipformer2CtcModelConfig getZipformer2Ctc() { + return zipformer2Ctc; + } + + public OnlineNeMoCtcModelConfig getNeMoCtc() { + return neMoCtc; + } + + public OnlineToneCtcModelConfig getToneCtc() { + return toneCtc; + } + + public String getTokens() { + return tokens; + } + + public int getNumThreads() { + return numThreads; + } + + public boolean getDebug() { + return debug; + } + + public String getProvider() { + return provider; + } + + public String getModelType() { + return modelType; + } + + public String getModelingUnit() { + return modelingUnit; + } + + public String getBpeVocab() { + return bpeVocab; + } + + public static class Builder { + private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build(); + private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build(); + private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build(); + private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build(); + private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build(); + private String tokens = ""; + private int numThreads = 1; + private boolean debug = false; + private String provider = "cpu"; + private String modelType = ""; + private String modelingUnit = "cjkchar"; + private String bpeVocab = ""; + + public Builder setTransducer(OnlineTransducerModelConfig transducer) { + this.transducer = transducer; + return this; + } + + public Builder setTokens(String tokens) { + this.tokens = tokens; + return this; + } + + public Builder setModelType(String modelType) { + this.modelType = modelType; + return this; + } + + public Builder setNumThreads(int numThreads) { + this.numThreads = numThreads; + return this; + } + + public Builder setProvider(String provider) { + this.provider = provider; + return this; + } + + public Builder setModelingUnit(String modelingUnit) { + this.modelingUnit = modelingUnit; + return this; + } + + public Builder setBpeVocab(String bpeVocab) { + this.bpeVocab = bpeVocab; + return this; + } + + public OnlineModelConfig build() { + return new OnlineModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java new file mode 100644 index 00000000..1982804c --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java @@ -0,0 +1,30 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineNeMoCtcModelConfig { + private final String model; + + private OnlineNeMoCtcModelConfig(Builder builder) { + this.model = builder.model; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public static class Builder { + private String model = ""; + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public OnlineNeMoCtcModelConfig build() { + return new OnlineNeMoCtcModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java new file mode 100644 index 00000000..3e44e2c5 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java @@ -0,0 +1,42 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineParaformerModelConfig { + private final String encoder; + private final String decoder; + + private OnlineParaformerModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public OnlineParaformerModelConfig build() { + return new OnlineParaformerModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java new file mode 100644 index 00000000..90a249cc --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java @@ -0,0 +1,35 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineStream { + private long ptr; + + public OnlineStream(long ptr) { + this.ptr = ptr; + } + + public long getPtr() { + return ptr; + } + + public void acceptWaveform(float[] samples, int sampleRate) { + acceptWaveform(ptr, samples, sampleRate); + } + + public void release() { + if (ptr == 0) { + return; + } + delete(ptr); + ptr = 0; + } + + @Override + protected void finalize() throws Throwable { + release(); + super.finalize(); + } + + private native void acceptWaveform(long ptr, float[] samples, int sampleRate); + + private native void delete(long ptr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java new file mode 100644 index 00000000..a847e79f --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java @@ -0,0 +1,30 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineToneCtcModelConfig { + private final String model; + + private OnlineToneCtcModelConfig(Builder builder) { + this.model = builder.model; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public static class Builder { + private String model = ""; + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public OnlineToneCtcModelConfig build() { + return new OnlineToneCtcModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java new file mode 100644 index 00000000..ca003533 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java @@ -0,0 +1,54 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineTransducerModelConfig { + private final String encoder; + private final String decoder; + private final String joiner; + + private OnlineTransducerModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + this.joiner = builder.joiner; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public String getJoiner() { + return joiner; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + private String joiner = ""; + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public Builder setJoiner(String joiner) { + this.joiner = joiner; + return this; + } + + public OnlineTransducerModelConfig build() { + return new OnlineTransducerModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java new file mode 100644 index 00000000..d73f73dd --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java @@ -0,0 +1,30 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineZipformer2CtcModelConfig { + private final String model; + + private OnlineZipformer2CtcModelConfig(Builder builder) { + this.model = builder.model; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public static class Builder { + private String model = ""; + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public OnlineZipformer2CtcModelConfig build() { + return new OnlineZipformer2CtcModelConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/jniLibs/arm64-v8a/libonnxruntime.so b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libonnxruntime.so new file mode 100644 index 00000000..873e54e2 Binary files /dev/null and b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libonnxruntime.so differ diff --git a/App_Resources/Android/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so new file mode 100644 index 00000000..754aa66d Binary files /dev/null and b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so differ diff --git a/app/g2/android-permissions.ts b/app/g2/android-permissions.ts index 58a409b5..c58d2247 100644 --- a/app/g2/android-permissions.ts +++ b/app/g2/android-permissions.ts @@ -1,7 +1,9 @@ import { Application, Utils } from "@nativescript/core"; const PERMISSION_REQUEST_CODE = 4242; +const VOICE_PERMISSION_REQUEST_CODE = 4243; const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS"; +const RECORD_AUDIO_PERMISSION = "android.permission.RECORD_AUDIO"; function getActivity(): androidx.appcompat.app.AppCompatActivity { const activity = Application.android.foregroundActivity ?? Application.android.startActivity; @@ -43,17 +45,21 @@ function getRequiredPermissions(): string[] { return permissions; } -export async function ensureBlePermissions(): Promise { +function isPermissionGranted(permission: string): boolean { + const context = getContext(); + return androidx.core.content.ContextCompat.checkSelfPermission(context, permission) === + android.content.pm.PackageManager.PERMISSION_GRANTED; +} + +async function ensurePermissions( + permissions: string[], + requestCode: number, + denialLabel: string, +): Promise { if (!global.isAndroid) return; - const context = getContext(); const activity = getActivity(); - const required = getRequiredPermissions(); - const missing = required.filter( - (permission) => - androidx.core.content.ContextCompat.checkSelfPermission(context, permission) !== - android.content.pm.PackageManager.PERMISSION_GRANTED, - ); + const missing = permissions.filter((permission) => !isPermissionGranted(permission)); if (missing.length === 0) return; @@ -63,7 +69,7 @@ export async function ensureBlePermissions(): Promise { permissions: string[]; grantResults: number[]; }) => { - if (args.requestCode !== PERMISSION_REQUEST_CODE) return; + if (args.requestCode !== requestCode) return; Application.android.off(Application.android.activityRequestPermissionsEvent, callback); const denied = missing.filter( @@ -72,7 +78,7 @@ export async function ensureBlePermissions(): Promise { ); if (denied.length > 0) { - reject(new Error(`Bluetooth permissions denied: ${denied.join(", ")}`)); + reject(new Error(`${denialLabel} permissions denied: ${denied.join(", ")}`)); return; } resolve(); @@ -82,7 +88,15 @@ export async function ensureBlePermissions(): Promise { androidx.core.app.ActivityCompat.requestPermissions( activity, toJavaStringArray(missing), - PERMISSION_REQUEST_CODE, + requestCode, ); }); } + +export async function ensureBlePermissions(): Promise { + await ensurePermissions(getRequiredPermissions(), PERMISSION_REQUEST_CODE, "Bluetooth"); +} + +export async function ensureVoicePermissions(): Promise { + await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control"); +} diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index f509a1d3..d8287b81 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1,19 +1,21 @@ import { ImageSource } from "@nativescript/core"; import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events"; import { loadDeviceAddresses } from "./device-addresses"; -import { ensureBlePermissions } from "./android-permissions"; +import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions"; import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator"; import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service"; import { mediaControllerBridge } from "../native/media-controller"; import { nightscoutBridge } from "../native/nightscout"; import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict"; import { grayImageToPreviewSource } from "../native/gray-image-preview"; +import { voiceControlBridge } from "../native/voice-control"; import { applyDashboardScreenTimeout, consumeDashboardTiledWakePaint, drawDashboard, getDashboardNightscoutSettings, getDashboardSystemCardName, + isDashboardVoiceControlEnabled, receiveInput, resetDashboardSleepTimerAndWake, setDashboardBatteryLevels, @@ -21,6 +23,7 @@ import { setDashboardNightscoutApiToken, setDashboardNightscoutSiteUrl, setDashboardSystemCardName, + setDashboardVoiceControlEnabled, } from "../ui/dashboard"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting"; @@ -118,6 +121,8 @@ class DashboardController { private offFrameMetrics: (() => void) | null = null; private offMedia: (() => void) | null = null; private offNightscout: (() => void) | null = null; + private offVoiceStatus: (() => void) | null = null; + private offVoiceWakeWord: (() => void) | null = null; private lastInput = "waiting..."; private lastSys = "none yet"; private renderInProgress = false; @@ -132,6 +137,7 @@ class DashboardController { startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(), startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(), endTextSettingEdit: () => this.endTextSettingEdit(), + setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled), }); } @@ -322,11 +328,20 @@ class DashboardController { }); } }); + this.offVoiceStatus = voiceControlBridge.onStatus((state) => { + this.appendLog(state.status); + }); + this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => { + void this.handleWakeWord(keyword).catch((error) => { + this.appendLog(`wake-word handler failed: ${this.formatError(error)}`); + }); + }); await mediaControllerBridge.start(); await nightscoutBridge.start(); await communicator.start(); await this.requestRender("initial"); + this.startVoiceControlIfEnabled(); this.dashboardTimer = setInterval(() => { void this.requestRender("interval").catch((error) => { const message = this.formatError(error); @@ -360,8 +375,13 @@ class DashboardController { this.offMedia = null; this.offNightscout?.(); this.offNightscout = null; + this.offVoiceStatus?.(); + this.offVoiceStatus = null; + this.offVoiceWakeWord?.(); + this.offVoiceWakeWord = null; await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); + voiceControlBridge.stop(); if (communicator) { await communicator.close().catch(() => {}); } @@ -397,6 +417,10 @@ class DashboardController { this.offMedia = null; this.offNightscout?.(); this.offNightscout = null; + this.offVoiceStatus?.(); + this.offVoiceStatus = null; + this.offVoiceWakeWord?.(); + this.offVoiceWakeWord = null; const communicator = this.communicator; this.communicator = null; @@ -413,6 +437,7 @@ class DashboardController { } await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); + voiceControlBridge.stop(); await communicator?.close().catch(() => {}); } finally { stopForegroundNotification(); @@ -445,6 +470,30 @@ class DashboardController { this.emit(); } + private async setVoiceControlEnabled(enabled: boolean): Promise { + if (enabled) { + try { + await ensureVoicePermissions(); + } catch (error) { + this.appendLog(`voice control permission failed: ${this.formatError(error)}`); + return; + } + } + + setDashboardVoiceControlEnabled(enabled); + this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`); + if (enabled) { + this.startVoiceControlIfEnabled(); + } else { + voiceControlBridge.stop(); + } + if (this.phase === "connected" && this.communicator) { + await this.requestRender("interval").catch((error) => { + this.appendLog(`voice setting render failed: ${this.formatError(error)}`); + }); + } + } + private endTextSettingEdit(): void { const finishedKind = this.activeTextSettingEditorKind; this.editingSystemCardName = false; @@ -580,6 +629,29 @@ class DashboardController { } } + private async handleWakeWord(keyword: string): Promise { + const normalized = keyword.trim(); + if (normalized.length === 0) return; + this.appendLog(`wake-word detected: ${normalized}`); + const changed = resetDashboardSleepTimerAndWake(); + if (this.phase === "connected" && this.communicator) { + if (changed) { + await this.requestRender("initial"); + } + return; + } + if (changed) { + const image = drawDashboard(); + this.setDisplayPreview(grayImageToPreviewSource(image)); + } + } + + private startVoiceControlIfEnabled(): void { + if (!this.communicator) return; + if (!isDashboardVoiceControlEnabled()) return; + voiceControlBridge.start(); + } + private clearDashboardTimer(): void { if (this.dashboardTimer) { clearInterval(this.dashboardTimer); diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts new file mode 100644 index 00000000..91b190ae --- /dev/null +++ b/app/native/voice-control.ts @@ -0,0 +1,70 @@ +import { Utils } from "@nativescript/core"; + +declare const com: any; + +export type VoiceControlState = { + status: string; +}; + +export class FaceclawVoiceControlBridge { + private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); + private readonly wakeWordListeners = new Set<(keyword: string) => void>(); + private controller: any | null = null; + private listenerProxy: any | null = null; + private status = "Voice control stopped."; + private started = false; + + onStatus(listener: (state: VoiceControlState) => void): () => void { + this.statusListeners.add(listener); + listener({ status: this.status }); + return () => this.statusListeners.delete(listener); + } + + onWakeWord(listener: (keyword: string) => void): () => void { + this.wakeWordListeners.add(listener); + return () => this.wakeWordListeners.delete(listener); + } + + start(): void { + if (this.started || !global.isAndroid) return; + this.ensureController(); + this.started = true; + this.controller?.start(); + } + + stop(): void { + if (!this.started || !global.isAndroid) return; + this.started = false; + this.controller?.stop(); + this.setStatus("Voice control stopped."); + } + + private ensureController(): void { + if (!global.isAndroid || this.controller) return; + const context = Utils.android.getApplicationContext(); + if (!context) { + throw new Error("Android application context unavailable"); + } + this.controller = new com.faceclaw.app.FaceclawVoiceController(context); + this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({ + onStatus: (status: string) => { + this.setStatus(String(status)); + }, + onWakeWord: (keyword: string) => { + for (const listener of this.wakeWordListeners) { + listener(String(keyword)); + } + }, + }); + this.controller.setListener(this.listenerProxy); + } + + private setStatus(status: string): void { + this.status = status; + for (const listener of this.statusListeners) { + listener({ status }); + } + } +} + +export const voiceControlBridge = new FaceclawVoiceControlBridge(); diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index e462f8a1..78f8ed80 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -21,9 +21,11 @@ export type WakeModeSetting = "tiled" | "synchronized"; export const DEFAULT_SYSTEM_CARD_NAME = "Faceclaw"; export const DEFAULT_SCREEN_TIMEOUT: ScreenTimeoutSetting = "30s"; export const DEFAULT_WAKE_MODE: WakeModeSetting = "tiled"; +export const DEFAULT_VOICE_CONTROL_ENABLED = false; const SYSTEM_CARD_NAME_KEY = "dashboard.systemCardName"; const SCREEN_TIMEOUT_KEY = "display.screenTimeout"; const WAKE_MODE_KEY = "display.wakeMode"; +const VOICE_CONTROL_ENABLED_KEY = "voice.enabled"; const SYSTEM_CARD_SETTING_KEYS: Record = { showFaceclawLogo: "dashboard.systemCard.showFaceclawLogo", showBatteryIndicators: "dashboard.systemCard.showBatteryIndicators", @@ -138,6 +140,15 @@ export function saveWakeModeSetting(value: WakeModeSetting): WakeModeSetting { return normalized; } +export function loadVoiceControlEnabled(): boolean { + return ApplicationSettings.getBoolean(VOICE_CONTROL_ENABLED_KEY, DEFAULT_VOICE_CONTROL_ENABLED); +} + +export function saveVoiceControlEnabled(value: boolean): boolean { + ApplicationSettings.setBoolean(VOICE_CONTROL_ENABLED_KEY, value); + return value; +} + export function nextScreenTimeoutSetting(value: ScreenTimeoutSetting): ScreenTimeoutSetting { switch (value) { case "15s": diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts index da74a0db..0da8ab8b 100644 --- a/app/ui/dashboard.ts +++ b/app/ui/dashboard.ts @@ -25,6 +25,7 @@ import { loadScreenTimeoutSetting, loadSystemCardSettings, loadSystemCardName, + loadVoiceControlEnabled, loadWakeModeSetting, nextScreenTimeoutSetting, nextWakeModeSetting, @@ -37,6 +38,7 @@ import { screenTimeoutMs, saveSystemCardSetting, saveSystemCardName, + saveVoiceControlEnabled, wakeModeLabel, type DashboardSlotId, type NightscoutSettings, @@ -63,6 +65,7 @@ let dashboardState = { systemCardSettings: loadSystemCardSettings(), screenTimeout: loadScreenTimeoutSetting(), wakeMode: loadWakeModeSetting(), + voiceControlEnabled: loadVoiceControlEnabled(), tiledWakePaintPending: false, nightscoutSettings: loadNightscoutSettings(), battery: { @@ -80,6 +83,7 @@ const dashboardActions: LayerActions = { startNightscoutSiteUrlEdit: () => {}, startNightscoutApiTokenEdit: () => {}, endTextSettingEdit: () => {}, + setVoiceControlEnabled: () => {}, }; const dashboardFont = loadEmbeddedTerminus12(); const dashboardSystemFont = loadEmbeddedTerminus16(); @@ -210,6 +214,14 @@ export function setDashboardSystemCardSetting(key: SystemCardSettingKey, value: saveSystemCardSetting(key, value); } +export function isDashboardVoiceControlEnabled(): boolean { + return dashboardState.voiceControlEnabled; +} + +export function setDashboardVoiceControlEnabled(value: boolean): void { + dashboardState.voiceControlEnabled = saveVoiceControlEnabled(value); +} + export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean { const timeoutMs = screenTimeoutMs(dashboardState.screenTimeout); if (timeoutMs === null || !dashboardState.screenOn) return false; @@ -511,6 +523,12 @@ function createSettingsMenuLayer(): MenuLayer { ctx.stack.push(createDashboardSettingsMenuLayer()); }, }, + { + label: "Voice", + onSelect: (ctx) => { + ctx.stack.push(createVoiceSettingsMenuLayer()); + }, + }, { label: "Integrations", onSelect: (ctx) => { @@ -558,6 +576,24 @@ function setWakeMode(value: WakeModeSetting): void { dashboardState.wakeMode = saveWakeModeSetting(value); } +function createVoiceSettingsMenuLayer(): MenuLayer { + return new MenuLayer( + "Voice", + [ + { + label: "Enable", + onSelect: async (ctx) => { + await ctx.actions.setVoiceControlEnabled(!dashboardState.voiceControlEnabled); + }, + render: ({ image, x, y, width, selected }) => { + drawToggleMenuItem(image, dashboardFont, x, y, width, "Enable", dashboardState.voiceControlEnabled, selected); + }, + }, + ], + TOP_LEFT_MENU_LAYOUT, + ); +} + function createDashboardSettingsMenuLayer(): MenuLayer { return new MenuLayer( "Dashboard", diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 857e2402..f949fd5c 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -15,6 +15,7 @@ export type LayerActions = { startNightscoutSiteUrlEdit: () => Promise | void; startNightscoutApiTokenEdit: () => Promise | void; endTextSettingEdit: () => Promise | void; + setVoiceControlEnabled: (enabled: boolean) => Promise | void; }; export type PaintBelow = () => GrayImage;