diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS
index 3d9393c0..8d852af6 100644
--- a/ACKNOWLEDGEMENTS
+++ b/ACKNOWLEDGEMENTS
@@ -2,3 +2,6 @@ Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficia
and others who contributed to documenting the G2's bluetooth protocol. It has enabled me
to keep my room clean, so to speak.
+Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
+and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
+community.
diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle
index 9f60d1a5..abda62fd 100644
--- a/App_Resources/Android/app.gradle
+++ b/App_Resources/Android/app.gradle
@@ -8,9 +8,16 @@ android {
buildToolsVersion "35"
// ndkVersion ""
+ sourceSets.main {
+ jniLibs.srcDirs += ["$projectDir/src/main/jniLibs"]
+ }
+
defaultConfig {
minSdkVersion 24
targetSdkVersion 35
+ ndk {
+ abiFilters "arm64-v8a"
+ }
// Version Information
versionCode 1
diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml
index 144a5326..5dfb457e 100644
--- a/App_Resources/Android/src/main/AndroidManifest.xml
+++ b/App_Resources/Android/src/main/AndroidManifest.xml
@@ -19,8 +19,11 @@
+
+
+
+ android:foregroundServiceType="connectedDevice|microphone" />
0
+ 1
+ 2
+S 3
+T 4
+▁THE 5
+▁A 6
+▁TO 7
+▁AND 8
+N 9
+E 10
+D 11
+ING 12
+' 13
+▁OF 14
+▁I 15
+▁IN 16
+Y 17
+ED 18
+M 19
+▁ 20
+▁THAT 21
+O 22
+C 23
+R 24
+A 25
+P 26
+▁YOU 27
+U 28
+RE 29
+▁IT 30
+▁S 31
+G 32
+L 33
+B 34
+▁IS 35
+I 36
+AL 37
+ER 38
+▁RE 39
+▁C 40
+▁WE 41
+▁SO 42
+▁BE 43
+OR 44
+AR 45
+IN 46
+▁FOR 47
+F 48
+▁HE 49
+LE 50
+LY 51
+▁E 52
+▁WAS 53
+K 54
+▁F 55
+W 56
+VE 57
+RI 58
+CH 59
+LL 60
+▁THIS 61
+▁T 62
+EN 63
+▁ON 64
+IT 65
+CE 66
+▁WITH 67
+▁B 68
+▁W 69
+▁ME 70
+▁DO 71
+▁DE 72
+▁HAVE 73
+▁BUT 74
+ENT 75
+▁P 76
+UR 77
+ON 78
+▁AS 79
+IC 80
+TER 81
+TE 82
+▁THEY 83
+ES 84
+RA 85
+▁NOT 86
+TH 87
+▁ARE 88
+RO 89
+IL 90
+ATION 91
+AN 92
+▁ST 93
+▁WHAT 94
+SE 95
+LI 96
+V 97
+▁CAN 98
+▁G 99
+▁OR 100
+▁ALL 101
+NE 102
+▁AT 103
+▁CON 104
+H 105
+▁LIKE 106
+▁MO 107
+▁HIS 108
+▁AN 109
+▁MA 110
+▁THERE 111
+▁SU 112
+▁MY 113
+NG 114
+- 115
+▁ABOUT 116
+▁SE 117
+LA 118
+▁ONE 119
+LO 120
+▁SHE 121
+CK 122
+▁HAD 123
+AD 124
+MENT 125
+VER 126
+▁CA 127
+IR 128
+▁IF 129
+▁NO 130
+EL 131
+▁O 132
+▁FROM 133
+▁PA 134
+▁YOUR 135
+▁KNOW 136
+RY 137
+ION 138
+UN 139
+▁JUST 140
+▁UP 141
+▁US 142
+▁WHO 143
+▁GO 144
+ME 145
+▁EX 146
+▁OUT 147
+ND 148
+CI 149
+▁PO 150
+▁HER 151
+▁FA 152
+▁PRO 153
+▁WHEN 154
+ATE 155
+AT 156
+▁BY 157
+PE 158
+AS 159
+▁LA 160
+GE 161
+▁CO 162
+▁PEOPLE 163
+▁K 164
+▁SOME 165
+IGHT 166
+▁THINK 167
+VI 168
+DE 169
+▁LE 170
+▁WERE 171
+▁LI 172
+OM 173
+▁WOULD 174
+▁SEE 175
+CO 176
+▁BO 177
+▁OUR 178
+▁HOW 179
+ET 180
+Z 181
+UL 182
+▁HO 183
+IG 184
+▁GET 185
+▁DON 186
+IS 187
+URE 188
+▁WHICH 189
+▁MI 190
+IES 191
+US 192
+X 193
+▁MORE 194
+▁NOW 195
+▁TIME 196
+▁UN 197
+▁FI 198
+TY 199
+▁WILL 200
+RU 201
+IVE 202
+▁THEIR 203
+▁SAY 204
+ITY 205
+▁HA 206
+LU 207
+▁RO 208
+HO 209
+OL 210
+IST 211
+AGE 212
+MA 213
+KE 214
+▁RA 215
+ID 216
+ERS 217
+▁LO 218
+▁SP 219
+▁DA 220
+▁SAID 221
+TION 222
+THER 223
+▁EN 224
+▁BECAUSE 225
+▁HAS 226
+EST 227
+TI 228
+ACT 229
+▁BA 230
+▁VERY 231
+▁THEM 232
+▁BEEN 233
+▁WA 234
+ABLE 235
+▁SA 236
+▁DID 237
+▁HIM 238
+▁DIS 239
+▁WANT 240
+▁OTHER 241
+IM 242
+▁PRE 243
+▁LOOK 244
+▁GOING 245
+▁VI 246
+▁WORK 247
+▁DR 248
+ENCE 249
+▁DI 250
+FF 251
+TA 252
+▁REALLY 253
+▁RIGHT 254
+MO 255
+ALLY 256
+▁INTO 257
+OW 258
+▁WAY 259
+▁FE 260
+▁COULD 261
+▁THEN 262
+TED 263
+▁HERE 264
+▁THESE 265
+▁OVER 266
+▁WELL 267
+▁NA 268
+▁ANY 269
+HE 270
+▁COMP 271
+▁COM 272
+▁TH 273
+▁EVEN 274
+▁TWO 275
+▁JO 276
+▁MAKE 277
+▁BACK 278
+▁WHERE 279
+SH 280
+▁QU 281
+MAN 282
+▁YEAR 283
+▁TRA 284
+▁CHA 285
+QU 286
+CA 287
+▁ALSO 288
+CTION 289
+ANCE 290
+UM 291
+PER 292
+DI 293
+▁HU 294
+▁MU 295
+FUL 296
+▁NEW 297
+ARD 298
+J 299
+ICAL 300
+ANT 301
+END 302
+▁MAR 303
+▁NE 304
+IF 305
+DER 306
+ONE 307
+▁THOSE 308
+PP 309
+▁MAN 310
+▁GOOD 311
+▁TAKE 312
+SIDE 313
+PO 314
+HA 315
+▁LITTLE 316
+▁APP 317
+▁BI 318
+UGH 319
+▁COME 320
+ISH 321
+TIC 322
+▁PER 323
+▁THINGS 324
+▁MUCH 325
+▁FIRST 326
+▁THAN 327
+IA 328
+▁AGAIN 329
+▁STA 330
+▁PART 331
+▁GA 332
+▁MEAN 333
+▁LET 334
+▁DOWN 335
+▁NEED 336
+▁GU 337
+LES 338
+▁RU 339
+ATED 340
+▁SHA 341
+VO 342
+▁TALK 343
+▁DAY 344
+▁CHI 345
+▁SOMETHING 346
+▁AFTER 347
+▁THOUGH 348
+▁CALL 349
+▁GRA 350
+▁EVERY 351
+▁KIND 352
+ROW 353
+AIN 354
+▁BU 355
+▁THROUGH 356
+▁SHOULD 357
+▁ONLY 358
+▁TA 359
+▁MIN 360
+▁GOT 361
+IAN 362
+▁GREAT 363
+▁RI 364
+▁DOES 365
+▁THING 366
+DAY 367
+OUS 368
+▁START 369
+▁MOST 370
+▁COL 371
+PH 372
+▁LOT 373
+IZE 374
+BER 375
+IOUS 376
+▁LIFE 377
+▁MANY 378
+▁FEEL 379
+ITION 380
+▁VA 381
+NESS 382
+▁BEFORE 383
+▁THREE 384
+▁WORLD 385
+▁GIVE 386
+▁WHY 387
+▁PI 388
+▁LONG 389
+PORT 390
+▁SHOW 391
+▁ACTUALLY 392
+▁NEVER 393
+IV 394
+FORM 395
+▁THANK 396
+▁HI 397
+▁PUT 398
+▁LOVE 399
+▁QUESTION 400
+▁HELP 401
+▁HAPPEN 402
+▁MR 403
+▁OWN 404
+WARD 405
+▁MADE 406
+▁HAND 407
+▁STATE 408
+IBLE 409
+▁TELL 410
+▁PH 411
+▁CAME 412
+▁TEA 413
+▁STILL 414
+▁DIFFERENT 415
+▁PLACE 416
+▁MIGHT 417
+▁FIND 418
+▁DU 419
+▁IMP 420
+▁OLD 421
+▁PERSON 422
+▁COUNT 423
+▁JA 424
+▁POINT 425
+▁HIGH 426
+SHIP 427
+▁LAST 428
+▁SAME 429
+▁FOUR 430
+▁AROUND 431
+▁OH 432
+▁CHANGE 433
+▁UNDER 434
+▁HOME 435
+▁ANOTHER 436
+▁FOUND 437
+ATIVE 438
+▁TURN 439
+▁YES 440
+LAND 441
+▁ALWAYS 442
+▁YEAH 443
+▁SCHOOL 444
+▁JE 445
+▁NEXT 446
+▁WORD 447
+▁JU 448
+LINE 449
+▁SIX 450
+▁LEARN 451
+▁BOOK 452
+▁PLAY 453
+▁AWAY 454
+▁HOUSE 455
+▁PLAN 456
+▁WHILE 457
+▁COMMUNI 458
+▁FRIEND 459
+▁AMERICA 460
+▁INTEREST 461
+▁KEEP 462
+▁BETTER 463
+▁IDEA 464
+▁HARD 465
+▁TWENTY 466
+▁THOUSAND 467
+▁HUNDRED 468
+▁WEEK 469
+▁NUMBER 470
+▁COURSE 471
+▁OKAY 472
+PRESS 473
+▁SYSTEM 474
+▁NINE 475
+▁UNDERSTAND 476
+▁WITHOUT 477
+▁BELIEVE 478
+▁POWER 479
+▁MAYBE 480
+▁BUILD 481
+▁MONEY 482
+▁BETWEEN 483
+▁GONNA 484
+▁IMPORTANT 485
+▁ANYTHING 486
+▁LEFT 487
+▁HAVING 488
+▁WHITE 489
+▁PROBLEM 490
+▁SECOND 491
+▁SPEAK 492
+▁TOGETHER 493
+▁BLACK 494
+▁PUBLIC 495
+▁HEALTH 496
+▁NOTHING 497
+▁EVERYTHING 498
+Q 499
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java
index 7c877350..50175243 100644
--- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java
@@ -27,7 +27,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final int WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE;
private static final int IMAGE_WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT;
- private static final int DESIRED_MTU = 247;
+ private static final int DESIRED_MTU = 512;
private static final int CONNECTION_PRIORITY_HIGH = BluetoothGatt.CONNECTION_PRIORITY_HIGH;
private static final int CONNECT_TIMEOUT_MS = 5_000;
@@ -40,7 +40,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final int BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000;
private static final int BATTERY_INPUT_QUIET_MS = 5_000;
private static final int WINDOW_SIZE = 3;
- private static final int IMAGE_FRAGMENT_SIZE = 1000;
+ private static final int IMAGE_FRAGMENT_SIZE = 3800;
private static final int IMAGE_RETRY_DELAY_MS = 10_000;
private static final boolean IMAGE_FRAGMENT_NO_ACK = false;
private static final int WARMUP_FRAGMENT_TIMEOUT_MS = 3_000;
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java
index 8cf6c136..b96672c3 100644
--- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawForegroundService.java
@@ -6,6 +6,7 @@ import android.app.NotificationManager;
import android.app.PendingIntent;
import android.app.Service;
import android.content.Intent;
+import android.content.pm.ServiceInfo;
import android.os.Build;
import android.os.IBinder;
@@ -41,7 +42,11 @@ public class FaceclawForegroundService extends Service {
text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected"
);
- startForeground(NOTIFICATION_ID, notification);
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+ startForeground(NOTIFICATION_ID, notification, foregroundServiceType());
+ } else {
+ startForeground(NOTIFICATION_ID, notification);
+ }
if (ACTION_UPDATE.equals(action)) {
NotificationManager manager = getSystemService(NotificationManager.class);
@@ -95,4 +100,9 @@ public class FaceclawForegroundService extends Service {
.setOnlyAlertOnce(true)
.build();
}
+
+ private int foregroundServiceType() {
+ return ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE
+ | ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE;
+ }
}
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java
new file mode 100644
index 00000000..9597818a
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java
@@ -0,0 +1,341 @@
+package com.faceclaw.app;
+
+import android.Manifest;
+import android.bluetooth.BluetoothAdapter;
+import android.bluetooth.BluetoothDevice;
+import android.bluetooth.BluetoothManager;
+import android.bluetooth.BluetoothProfile;
+import android.content.Context;
+import android.content.pm.PackageManager;
+import android.content.res.AssetManager;
+import android.media.AudioDeviceInfo;
+import android.media.AudioFormat;
+import android.media.AudioManager;
+import android.media.AudioRecord;
+import android.media.MediaRecorder;
+import android.os.Build;
+import android.os.Handler;
+import android.os.Looper;
+import android.util.Log;
+
+import androidx.core.content.ContextCompat;
+
+import com.k2fsa.sherpa.onnx.FeatureConfig;
+import com.k2fsa.sherpa.onnx.KeywordSpotter;
+import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
+import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
+import com.k2fsa.sherpa.onnx.OnlineModelConfig;
+import com.k2fsa.sherpa.onnx.OnlineStream;
+import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+
+public class FaceclawVoiceController {
+ private static final String TAG = "FaceclawVoice";
+ private static final int SAMPLE_RATE = 16000;
+ private static final int FEATURE_DIM = 80;
+ private static final String ASSET_ROOT = "faceclaw-voice";
+ private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
+ private static final String[] MODEL_FILES = {
+ "encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
+ "decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
+ "joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
+ "tokens.txt",
+ "screen-on-keywords.txt"
+ };
+
+ private final Context appContext;
+ private final Handler mainHandler = new Handler(Looper.getMainLooper());
+ private final Object lock = new Object();
+ private volatile FaceclawVoiceControllerListener listener;
+ private Thread workerThread;
+ private volatile boolean started;
+ private AudioRecord audioRecord;
+ private KeywordSpotter keywordSpotter;
+ private OnlineStream stream;
+
+ public FaceclawVoiceController(Context context) {
+ this.appContext = context.getApplicationContext();
+ }
+
+ public void setListener(FaceclawVoiceControllerListener listener) {
+ this.listener = listener;
+ }
+
+ public void start() {
+ synchronized (lock) {
+ if (started) {
+ emitStatus("Voice control is already listening.");
+ return;
+ }
+ if (!hasRecordAudioPermission()) {
+ emitStatus("Voice control needs microphone permission.");
+ return;
+ }
+ if (!hasBluetoothHeadset()) {
+ emitStatus("Voice control needs a paired Bluetooth headset.");
+ return;
+ }
+ started = true;
+ workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
+ workerThread.start();
+ }
+ }
+
+ public void stop() {
+ Thread threadToJoin;
+ synchronized (lock) {
+ if (!started) {
+ return;
+ }
+ started = false;
+ threadToJoin = workerThread;
+ }
+ stopAudio();
+ if (threadToJoin != null) {
+ threadToJoin.interrupt();
+ }
+ }
+
+ public void close() {
+ stop();
+ }
+
+ private void runLoop() {
+ try {
+ emitStatus("Voice control loading wake-word model...");
+ File modelDir = installModelFiles();
+ keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
+ stream = keywordSpotter.createStream();
+ if (!startAudio()) {
+ emitStatus("Voice control could not start microphone input.");
+ return;
+ }
+
+ emitStatus("Voice control listening for \"screen on\".");
+ processAudio();
+ } catch (Throwable error) {
+ Log.e(TAG, "Voice control failed", error);
+ emitStatus("Voice control failed: " + error.getMessage());
+ } finally {
+ releaseSherpa();
+ stopAudio();
+ synchronized (lock) {
+ started = false;
+ workerThread = null;
+ }
+ }
+ }
+
+ private KeywordSpotterConfig buildConfig(File modelDir) {
+ return KeywordSpotterConfig.builder()
+ .setFeatureConfig(FeatureConfig.builder()
+ .setSampleRate(SAMPLE_RATE)
+ .setFeatureDim(FEATURE_DIM)
+ .build())
+ .setOnlineModelConfig(OnlineModelConfig.builder()
+ .setTransducer(OnlineTransducerModelConfig.builder()
+ .setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
+ .setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
+ .setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
+ .build())
+ .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
+ .setModelType("zipformer2")
+ .setNumThreads(1)
+ .build())
+ .setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
+ .setKeywordsScore(1.5f)
+ .setKeywordsThreshold(0.35f)
+ .build();
+ }
+
+ private File installModelFiles() throws IOException {
+ File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
+ if (!modelDir.exists() && !modelDir.mkdirs()) {
+ throw new IOException("Could not create " + modelDir.getAbsolutePath());
+ }
+ AssetManager assets = appContext.getAssets();
+ for (String fileName : MODEL_FILES) {
+ copyAssetIfNeeded(
+ assets,
+ ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
+ new File(modelDir, fileName)
+ );
+ }
+ return modelDir;
+ }
+
+ private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
+ if (destination.exists() && destination.length() > 0) {
+ return;
+ }
+ try (InputStream input = assets.open(assetPath);
+ FileOutputStream output = new FileOutputStream(destination)) {
+ byte[] buffer = new byte[64 * 1024];
+ int read;
+ while ((read = input.read(buffer)) >= 0) {
+ output.write(buffer, 0, read);
+ }
+ }
+ }
+
+ private boolean startAudio() {
+ routeToBluetoothHeadset();
+ int minBufferBytes = AudioRecord.getMinBufferSize(
+ SAMPLE_RATE,
+ AudioFormat.CHANNEL_IN_MONO,
+ AudioFormat.ENCODING_PCM_16BIT
+ );
+ if (minBufferBytes <= 0) {
+ return false;
+ }
+ if (!hasRecordAudioPermission()) {
+ return false;
+ }
+ audioRecord = new AudioRecord(
+ MediaRecorder.AudioSource.VOICE_RECOGNITION,
+ SAMPLE_RATE,
+ AudioFormat.CHANNEL_IN_MONO,
+ AudioFormat.ENCODING_PCM_16BIT,
+ minBufferBytes * 2
+ );
+ if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
+ audioRecord.release();
+ audioRecord = null;
+ return false;
+ }
+ audioRecord.startRecording();
+ return true;
+ }
+
+ private void processAudio() {
+ short[] buffer = new short[SAMPLE_RATE / 10];
+ while (started && !Thread.currentThread().isInterrupted()) {
+ AudioRecord currentAudio = audioRecord;
+ OnlineStream currentStream = stream;
+ KeywordSpotter currentSpotter = keywordSpotter;
+ if (currentAudio == null || currentStream == null || currentSpotter == null) {
+ return;
+ }
+
+ int count = currentAudio.read(buffer, 0, buffer.length);
+ if (count <= 0) {
+ continue;
+ }
+
+ float[] samples = new float[count];
+ for (int i = 0; i < count; i++) {
+ samples[i] = buffer[i] / 32768.0f;
+ }
+ currentStream.acceptWaveform(samples, SAMPLE_RATE);
+
+ while (currentSpotter.isReady(currentStream)) {
+ currentSpotter.decode(currentStream);
+ KeywordSpotterResult result = currentSpotter.getResult(currentStream);
+ String keyword = result == null ? "" : result.getKeyword();
+ if (keyword != null && keyword.trim().length() > 0) {
+ currentSpotter.reset(currentStream);
+ emitWakeWord(keyword);
+ }
+ }
+ }
+ }
+
+ private void stopAudio() {
+ AudioRecord record = audioRecord;
+ audioRecord = null;
+ if (record != null) {
+ try {
+ record.stop();
+ } catch (IllegalStateException ignored) {
+ }
+ record.release();
+ }
+ AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
+ if (audioManager != null) {
+ if (Build.VERSION.SDK_INT >= 31) {
+ audioManager.clearCommunicationDevice();
+ }
+ audioManager.stopBluetoothSco();
+ audioManager.setBluetoothScoOn(false);
+ }
+ }
+
+ private void releaseSherpa() {
+ if (stream != null) {
+ stream.release();
+ stream = null;
+ }
+ if (keywordSpotter != null) {
+ keywordSpotter.release();
+ keywordSpotter = null;
+ }
+ }
+
+ private void routeToBluetoothHeadset() {
+ AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
+ if (audioManager == null) {
+ return;
+ }
+ audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION);
+ audioManager.startBluetoothSco();
+ audioManager.setBluetoothScoOn(true);
+ if (Build.VERSION.SDK_INT >= 31) {
+ for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) {
+ if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
+ || device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) {
+ audioManager.setCommunicationDevice(device);
+ return;
+ }
+ }
+ }
+ }
+
+ private boolean hasBluetoothHeadset() {
+ if (Build.VERSION.SDK_INT >= 31
+ && ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT)
+ != PackageManager.PERMISSION_GRANTED) {
+ return false;
+ }
+ BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
+ BluetoothAdapter adapter = manager == null ? null : manager.getAdapter();
+ if (adapter == null) {
+ return false;
+ }
+ try {
+ int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET);
+ if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) {
+ return true;
+ }
+ Set bondedDevices = adapter.getBondedDevices();
+ return bondedDevices != null && !bondedDevices.isEmpty();
+ } catch (SecurityException ignored) {
+ return false;
+ }
+ }
+
+ private boolean hasRecordAudioPermission() {
+ return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
+ == PackageManager.PERMISSION_GRANTED;
+ }
+
+ private void emitStatus(String status) {
+ FaceclawVoiceControllerListener currentListener = listener;
+ if (currentListener == null) {
+ return;
+ }
+ mainHandler.post(() -> currentListener.onStatus(status));
+ }
+
+ private void emitWakeWord(String keyword) {
+ FaceclawVoiceControllerListener currentListener = listener;
+ if (currentListener == null) {
+ return;
+ }
+ mainHandler.post(() -> currentListener.onWakeWord(keyword));
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java
new file mode 100644
index 00000000..60b306b5
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java
@@ -0,0 +1,7 @@
+package com.faceclaw.app;
+
+public interface FaceclawVoiceControllerListener {
+ void onStatus(String status);
+
+ void onWakeWord(String keyword);
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.java
new file mode 100644
index 00000000..1dd5c677
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.java
@@ -0,0 +1,54 @@
+package com.k2fsa.sherpa.onnx;
+
+public class FeatureConfig {
+ private final int sampleRate;
+ private final int featureDim;
+ private final float dither;
+
+ private FeatureConfig(Builder builder) {
+ this.sampleRate = builder.sampleRate;
+ this.featureDim = builder.featureDim;
+ this.dither = builder.dither;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public int getSampleRate() {
+ return sampleRate;
+ }
+
+ public int getFeatureDim() {
+ return featureDim;
+ }
+
+ public float getDither() {
+ return dither;
+ }
+
+ public static class Builder {
+ private int sampleRate = 16000;
+ private int featureDim = 80;
+ private float dither = 0.0f;
+
+ public Builder setSampleRate(int sampleRate) {
+ this.sampleRate = sampleRate;
+ return this;
+ }
+
+ public Builder setFeatureDim(int featureDim) {
+ this.featureDim = featureDim;
+ return this;
+ }
+
+ public Builder setDither(float dither) {
+ this.dither = dither;
+ return this;
+ }
+
+ public FeatureConfig build() {
+ return new FeatureConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java
new file mode 100644
index 00000000..836a07c2
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotter.java
@@ -0,0 +1,66 @@
+package com.k2fsa.sherpa.onnx;
+
+public class KeywordSpotter {
+ private long ptr;
+
+ public KeywordSpotter(KeywordSpotterConfig config) {
+ System.loadLibrary("onnxruntime");
+ System.loadLibrary("sherpa-onnx-jni");
+ ptr = newFromFile(config);
+ if (ptr == 0) {
+ throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
+ }
+ }
+
+ public OnlineStream createStream() {
+ return createStream("");
+ }
+
+ public OnlineStream createStream(String keywords) {
+ return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
+ }
+
+ public boolean isReady(OnlineStream stream) {
+ return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
+ }
+
+ public void decode(OnlineStream stream) {
+ decode(ptr, stream.getPtr());
+ }
+
+ public void reset(OnlineStream stream) {
+ reset(ptr, stream.getPtr());
+ }
+
+ public KeywordSpotterResult getResult(OnlineStream stream) {
+ return getResult(ptr, stream.getPtr());
+ }
+
+ public void release() {
+ if (ptr == 0) {
+ return;
+ }
+ delete(ptr);
+ ptr = 0;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ release();
+ super.finalize();
+ }
+
+ private native long newFromFile(KeywordSpotterConfig config);
+
+ private native void delete(long ptr);
+
+ private native long createStream(long ptr, String keywords);
+
+ private native boolean isReady(long ptr, long streamPtr);
+
+ private native void decode(long ptr, long streamPtr);
+
+ private native void reset(long ptr, long streamPtr);
+
+ private native KeywordSpotterResult getResult(long ptr, long streamPtr);
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java
new file mode 100644
index 00000000..3c4ce6f3
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterConfig.java
@@ -0,0 +1,92 @@
+package com.k2fsa.sherpa.onnx;
+
+public class KeywordSpotterConfig {
+ private final FeatureConfig featConfig;
+ private final OnlineModelConfig modelConfig;
+ private final int maxActivePaths;
+ private final String keywordsFile;
+ private final float keywordsScore;
+ private final float keywordsThreshold;
+ private final int numTrailingBlanks;
+
+ private KeywordSpotterConfig(Builder builder) {
+ this.featConfig = builder.featConfig;
+ this.modelConfig = builder.modelConfig;
+ this.maxActivePaths = builder.maxActivePaths;
+ this.keywordsFile = builder.keywordsFile;
+ this.keywordsScore = builder.keywordsScore;
+ this.keywordsThreshold = builder.keywordsThreshold;
+ this.numTrailingBlanks = builder.numTrailingBlanks;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public FeatureConfig getFeatConfig() {
+ return featConfig;
+ }
+
+ public OnlineModelConfig getModelConfig() {
+ return modelConfig;
+ }
+
+ public int getMaxActivePaths() {
+ return maxActivePaths;
+ }
+
+ public String getKeywordsFile() {
+ return keywordsFile;
+ }
+
+ public float getKeywordsScore() {
+ return keywordsScore;
+ }
+
+ public float getKeywordsThreshold() {
+ return keywordsThreshold;
+ }
+
+ public int getNumTrailingBlanks() {
+ return numTrailingBlanks;
+ }
+
+ public static class Builder {
+ private FeatureConfig featConfig = FeatureConfig.builder().build();
+ private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
+ private int maxActivePaths = 4;
+ private String keywordsFile = "keywords.txt";
+ private float keywordsScore = 1.5f;
+ private float keywordsThreshold = 0.25f;
+ private int numTrailingBlanks = 2;
+
+ public Builder setFeatureConfig(FeatureConfig featConfig) {
+ this.featConfig = featConfig;
+ return this;
+ }
+
+ public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
+ this.modelConfig = modelConfig;
+ return this;
+ }
+
+ public Builder setKeywordsFile(String keywordsFile) {
+ this.keywordsFile = keywordsFile;
+ return this;
+ }
+
+ public Builder setKeywordsScore(float keywordsScore) {
+ this.keywordsScore = keywordsScore;
+ return this;
+ }
+
+ public Builder setKeywordsThreshold(float keywordsThreshold) {
+ this.keywordsThreshold = keywordsThreshold;
+ return this;
+ }
+
+ public KeywordSpotterConfig build() {
+ return new KeywordSpotterConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java
new file mode 100644
index 00000000..77e3a72e
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/KeywordSpotterResult.java
@@ -0,0 +1,25 @@
+package com.k2fsa.sherpa.onnx;
+
+public class KeywordSpotterResult {
+ private final String keyword;
+ private final String[] tokens;
+ private final float[] timestamps;
+
+ public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
+ this.keyword = keyword == null ? "" : keyword;
+ this.tokens = tokens == null ? new String[0] : tokens;
+ this.timestamps = timestamps == null ? new float[0] : timestamps;
+ }
+
+ public String getKeyword() {
+ return keyword;
+ }
+
+ public String[] getTokens() {
+ return tokens;
+ }
+
+ public float[] getTimestamps() {
+ return timestamps;
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java
new file mode 100644
index 00000000..d3f0db66
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineModelConfig.java
@@ -0,0 +1,137 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineModelConfig {
+ private final OnlineTransducerModelConfig transducer;
+ private final OnlineParaformerModelConfig paraformer;
+ private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
+ private final OnlineNeMoCtcModelConfig neMoCtc;
+ private final OnlineToneCtcModelConfig toneCtc;
+ private final String tokens;
+ private final int numThreads;
+ private final boolean debug;
+ private final String provider;
+ private final String modelType;
+ private final String modelingUnit;
+ private final String bpeVocab;
+
+ private OnlineModelConfig(Builder builder) {
+ this.transducer = builder.transducer;
+ this.paraformer = builder.paraformer;
+ this.zipformer2Ctc = builder.zipformer2Ctc;
+ this.neMoCtc = builder.neMoCtc;
+ this.toneCtc = builder.toneCtc;
+ this.tokens = builder.tokens;
+ this.numThreads = builder.numThreads;
+ this.debug = builder.debug;
+ this.provider = builder.provider;
+ this.modelType = builder.modelType;
+ this.modelingUnit = builder.modelingUnit;
+ this.bpeVocab = builder.bpeVocab;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public OnlineTransducerModelConfig getTransducer() {
+ return transducer;
+ }
+
+ public OnlineParaformerModelConfig getParaformer() {
+ return paraformer;
+ }
+
+ public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
+ return zipformer2Ctc;
+ }
+
+ public OnlineNeMoCtcModelConfig getNeMoCtc() {
+ return neMoCtc;
+ }
+
+ public OnlineToneCtcModelConfig getToneCtc() {
+ return toneCtc;
+ }
+
+ public String getTokens() {
+ return tokens;
+ }
+
+ public int getNumThreads() {
+ return numThreads;
+ }
+
+ public boolean getDebug() {
+ return debug;
+ }
+
+ public String getProvider() {
+ return provider;
+ }
+
+ public String getModelType() {
+ return modelType;
+ }
+
+ public String getModelingUnit() {
+ return modelingUnit;
+ }
+
+ public String getBpeVocab() {
+ return bpeVocab;
+ }
+
+ public static class Builder {
+ private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
+ private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
+ private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
+ private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
+ private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
+ private String tokens = "";
+ private int numThreads = 1;
+ private boolean debug = false;
+ private String provider = "cpu";
+ private String modelType = "";
+ private String modelingUnit = "cjkchar";
+ private String bpeVocab = "";
+
+ public Builder setTransducer(OnlineTransducerModelConfig transducer) {
+ this.transducer = transducer;
+ return this;
+ }
+
+ public Builder setTokens(String tokens) {
+ this.tokens = tokens;
+ return this;
+ }
+
+ public Builder setModelType(String modelType) {
+ this.modelType = modelType;
+ return this;
+ }
+
+ public Builder setNumThreads(int numThreads) {
+ this.numThreads = numThreads;
+ return this;
+ }
+
+ public Builder setProvider(String provider) {
+ this.provider = provider;
+ return this;
+ }
+
+ public Builder setModelingUnit(String modelingUnit) {
+ this.modelingUnit = modelingUnit;
+ return this;
+ }
+
+ public Builder setBpeVocab(String bpeVocab) {
+ this.bpeVocab = bpeVocab;
+ return this;
+ }
+
+ public OnlineModelConfig build() {
+ return new OnlineModelConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java
new file mode 100644
index 00000000..1982804c
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineNeMoCtcModelConfig.java
@@ -0,0 +1,30 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineNeMoCtcModelConfig {
+ private final String model;
+
+ private OnlineNeMoCtcModelConfig(Builder builder) {
+ this.model = builder.model;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getModel() {
+ return model;
+ }
+
+ public static class Builder {
+ private String model = "";
+
+ public Builder setModel(String model) {
+ this.model = model;
+ return this;
+ }
+
+ public OnlineNeMoCtcModelConfig build() {
+ return new OnlineNeMoCtcModelConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java
new file mode 100644
index 00000000..3e44e2c5
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineParaformerModelConfig.java
@@ -0,0 +1,42 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineParaformerModelConfig {
+ private final String encoder;
+ private final String decoder;
+
+ private OnlineParaformerModelConfig(Builder builder) {
+ this.encoder = builder.encoder;
+ this.decoder = builder.decoder;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getEncoder() {
+ return encoder;
+ }
+
+ public String getDecoder() {
+ return decoder;
+ }
+
+ public static class Builder {
+ private String encoder = "";
+ private String decoder = "";
+
+ public Builder setEncoder(String encoder) {
+ this.encoder = encoder;
+ return this;
+ }
+
+ public Builder setDecoder(String decoder) {
+ this.decoder = decoder;
+ return this;
+ }
+
+ public OnlineParaformerModelConfig build() {
+ return new OnlineParaformerModelConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java
new file mode 100644
index 00000000..90a249cc
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.java
@@ -0,0 +1,35 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineStream {
+ private long ptr;
+
+ public OnlineStream(long ptr) {
+ this.ptr = ptr;
+ }
+
+ public long getPtr() {
+ return ptr;
+ }
+
+ public void acceptWaveform(float[] samples, int sampleRate) {
+ acceptWaveform(ptr, samples, sampleRate);
+ }
+
+ public void release() {
+ if (ptr == 0) {
+ return;
+ }
+ delete(ptr);
+ ptr = 0;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ release();
+ super.finalize();
+ }
+
+ private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
+
+ private native void delete(long ptr);
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java
new file mode 100644
index 00000000..a847e79f
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineToneCtcModelConfig.java
@@ -0,0 +1,30 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineToneCtcModelConfig {
+ private final String model;
+
+ private OnlineToneCtcModelConfig(Builder builder) {
+ this.model = builder.model;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getModel() {
+ return model;
+ }
+
+ public static class Builder {
+ private String model = "";
+
+ public Builder setModel(String model) {
+ this.model = model;
+ return this;
+ }
+
+ public OnlineToneCtcModelConfig build() {
+ return new OnlineToneCtcModelConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java
new file mode 100644
index 00000000..ca003533
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineTransducerModelConfig.java
@@ -0,0 +1,54 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineTransducerModelConfig {
+ private final String encoder;
+ private final String decoder;
+ private final String joiner;
+
+ private OnlineTransducerModelConfig(Builder builder) {
+ this.encoder = builder.encoder;
+ this.decoder = builder.decoder;
+ this.joiner = builder.joiner;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getEncoder() {
+ return encoder;
+ }
+
+ public String getDecoder() {
+ return decoder;
+ }
+
+ public String getJoiner() {
+ return joiner;
+ }
+
+ public static class Builder {
+ private String encoder = "";
+ private String decoder = "";
+ private String joiner = "";
+
+ public Builder setEncoder(String encoder) {
+ this.encoder = encoder;
+ return this;
+ }
+
+ public Builder setDecoder(String decoder) {
+ this.decoder = decoder;
+ return this;
+ }
+
+ public Builder setJoiner(String joiner) {
+ this.joiner = joiner;
+ return this;
+ }
+
+ public OnlineTransducerModelConfig build() {
+ return new OnlineTransducerModelConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java
new file mode 100644
index 00000000..d73f73dd
--- /dev/null
+++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineZipformer2CtcModelConfig.java
@@ -0,0 +1,30 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OnlineZipformer2CtcModelConfig {
+ private final String model;
+
+ private OnlineZipformer2CtcModelConfig(Builder builder) {
+ this.model = builder.model;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getModel() {
+ return model;
+ }
+
+ public static class Builder {
+ private String model = "";
+
+ public Builder setModel(String model) {
+ this.model = model;
+ return this;
+ }
+
+ public OnlineZipformer2CtcModelConfig build() {
+ return new OnlineZipformer2CtcModelConfig(this);
+ }
+ }
+}
diff --git a/App_Resources/Android/src/main/jniLibs/arm64-v8a/libonnxruntime.so b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libonnxruntime.so
new file mode 100644
index 00000000..873e54e2
Binary files /dev/null and b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libonnxruntime.so differ
diff --git a/App_Resources/Android/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so
new file mode 100644
index 00000000..754aa66d
Binary files /dev/null and b/App_Resources/Android/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so differ
diff --git a/app/g2/android-permissions.ts b/app/g2/android-permissions.ts
index 58a409b5..c58d2247 100644
--- a/app/g2/android-permissions.ts
+++ b/app/g2/android-permissions.ts
@@ -1,7 +1,9 @@
import { Application, Utils } from "@nativescript/core";
const PERMISSION_REQUEST_CODE = 4242;
+const VOICE_PERMISSION_REQUEST_CODE = 4243;
const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS";
+const RECORD_AUDIO_PERMISSION = "android.permission.RECORD_AUDIO";
function getActivity(): androidx.appcompat.app.AppCompatActivity {
const activity = Application.android.foregroundActivity ?? Application.android.startActivity;
@@ -43,17 +45,21 @@ function getRequiredPermissions(): string[] {
return permissions;
}
-export async function ensureBlePermissions(): Promise {
+function isPermissionGranted(permission: string): boolean {
+ const context = getContext();
+ return androidx.core.content.ContextCompat.checkSelfPermission(context, permission) ===
+ android.content.pm.PackageManager.PERMISSION_GRANTED;
+}
+
+async function ensurePermissions(
+ permissions: string[],
+ requestCode: number,
+ denialLabel: string,
+): Promise {
if (!global.isAndroid) return;
- const context = getContext();
const activity = getActivity();
- const required = getRequiredPermissions();
- const missing = required.filter(
- (permission) =>
- androidx.core.content.ContextCompat.checkSelfPermission(context, permission) !==
- android.content.pm.PackageManager.PERMISSION_GRANTED,
- );
+ const missing = permissions.filter((permission) => !isPermissionGranted(permission));
if (missing.length === 0) return;
@@ -63,7 +69,7 @@ export async function ensureBlePermissions(): Promise {
permissions: string[];
grantResults: number[];
}) => {
- if (args.requestCode !== PERMISSION_REQUEST_CODE) return;
+ if (args.requestCode !== requestCode) return;
Application.android.off(Application.android.activityRequestPermissionsEvent, callback);
const denied = missing.filter(
@@ -72,7 +78,7 @@ export async function ensureBlePermissions(): Promise {
);
if (denied.length > 0) {
- reject(new Error(`Bluetooth permissions denied: ${denied.join(", ")}`));
+ reject(new Error(`${denialLabel} permissions denied: ${denied.join(", ")}`));
return;
}
resolve();
@@ -82,7 +88,15 @@ export async function ensureBlePermissions(): Promise {
androidx.core.app.ActivityCompat.requestPermissions(
activity,
toJavaStringArray(missing),
- PERMISSION_REQUEST_CODE,
+ requestCode,
);
});
}
+
+export async function ensureBlePermissions(): Promise {
+ await ensurePermissions(getRequiredPermissions(), PERMISSION_REQUEST_CODE, "Bluetooth");
+}
+
+export async function ensureVoicePermissions(): Promise {
+ await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
+}
diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts
index f509a1d3..d8287b81 100644
--- a/app/g2/dashboard-controller.ts
+++ b/app/g2/dashboard-controller.ts
@@ -1,19 +1,21 @@
import { ImageSource } from "@nativescript/core";
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { loadDeviceAddresses } from "./device-addresses";
-import { ensureBlePermissions } from "./android-permissions";
+import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service";
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { grayImageToPreviewSource } from "../native/gray-image-preview";
+import { voiceControlBridge } from "../native/voice-control";
import {
applyDashboardScreenTimeout,
consumeDashboardTiledWakePaint,
drawDashboard,
getDashboardNightscoutSettings,
getDashboardSystemCardName,
+ isDashboardVoiceControlEnabled,
receiveInput,
resetDashboardSleepTimerAndWake,
setDashboardBatteryLevels,
@@ -21,6 +23,7 @@ import {
setDashboardNightscoutApiToken,
setDashboardNightscoutSiteUrl,
setDashboardSystemCardName,
+ setDashboardVoiceControlEnabled,
} from "../ui/dashboard";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -118,6 +121,8 @@ class DashboardController {
private offFrameMetrics: (() => void) | null = null;
private offMedia: (() => void) | null = null;
private offNightscout: (() => void) | null = null;
+ private offVoiceStatus: (() => void) | null = null;
+ private offVoiceWakeWord: (() => void) | null = null;
private lastInput = "waiting...";
private lastSys = "none yet";
private renderInProgress = false;
@@ -132,6 +137,7 @@ class DashboardController {
startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(),
startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(),
endTextSettingEdit: () => this.endTextSettingEdit(),
+ setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
});
}
@@ -322,11 +328,20 @@ class DashboardController {
});
}
});
+ this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
+ this.appendLog(state.status);
+ });
+ this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => {
+ void this.handleWakeWord(keyword).catch((error) => {
+ this.appendLog(`wake-word handler failed: ${this.formatError(error)}`);
+ });
+ });
await mediaControllerBridge.start();
await nightscoutBridge.start();
await communicator.start();
await this.requestRender("initial");
+ this.startVoiceControlIfEnabled();
this.dashboardTimer = setInterval(() => {
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
@@ -360,8 +375,13 @@ class DashboardController {
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
+ this.offVoiceStatus?.();
+ this.offVoiceStatus = null;
+ this.offVoiceWakeWord?.();
+ this.offVoiceWakeWord = null;
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
+ voiceControlBridge.stop();
if (communicator) {
await communicator.close().catch(() => {});
}
@@ -397,6 +417,10 @@ class DashboardController {
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
+ this.offVoiceStatus?.();
+ this.offVoiceStatus = null;
+ this.offVoiceWakeWord?.();
+ this.offVoiceWakeWord = null;
const communicator = this.communicator;
this.communicator = null;
@@ -413,6 +437,7 @@ class DashboardController {
}
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
+ voiceControlBridge.stop();
await communicator?.close().catch(() => {});
} finally {
stopForegroundNotification();
@@ -445,6 +470,30 @@ class DashboardController {
this.emit();
}
+ private async setVoiceControlEnabled(enabled: boolean): Promise {
+ if (enabled) {
+ try {
+ await ensureVoicePermissions();
+ } catch (error) {
+ this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
+ return;
+ }
+ }
+
+ setDashboardVoiceControlEnabled(enabled);
+ this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`);
+ if (enabled) {
+ this.startVoiceControlIfEnabled();
+ } else {
+ voiceControlBridge.stop();
+ }
+ if (this.phase === "connected" && this.communicator) {
+ await this.requestRender("interval").catch((error) => {
+ this.appendLog(`voice setting render failed: ${this.formatError(error)}`);
+ });
+ }
+ }
+
private endTextSettingEdit(): void {
const finishedKind = this.activeTextSettingEditorKind;
this.editingSystemCardName = false;
@@ -580,6 +629,29 @@ class DashboardController {
}
}
+ private async handleWakeWord(keyword: string): Promise {
+ const normalized = keyword.trim();
+ if (normalized.length === 0) return;
+ this.appendLog(`wake-word detected: ${normalized}`);
+ const changed = resetDashboardSleepTimerAndWake();
+ if (this.phase === "connected" && this.communicator) {
+ if (changed) {
+ await this.requestRender("initial");
+ }
+ return;
+ }
+ if (changed) {
+ const image = drawDashboard();
+ this.setDisplayPreview(grayImageToPreviewSource(image));
+ }
+ }
+
+ private startVoiceControlIfEnabled(): void {
+ if (!this.communicator) return;
+ if (!isDashboardVoiceControlEnabled()) return;
+ voiceControlBridge.start();
+ }
+
private clearDashboardTimer(): void {
if (this.dashboardTimer) {
clearInterval(this.dashboardTimer);
diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts
new file mode 100644
index 00000000..91b190ae
--- /dev/null
+++ b/app/native/voice-control.ts
@@ -0,0 +1,70 @@
+import { Utils } from "@nativescript/core";
+
+declare const com: any;
+
+export type VoiceControlState = {
+ status: string;
+};
+
+export class FaceclawVoiceControlBridge {
+ private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
+ private readonly wakeWordListeners = new Set<(keyword: string) => void>();
+ private controller: any | null = null;
+ private listenerProxy: any | null = null;
+ private status = "Voice control stopped.";
+ private started = false;
+
+ onStatus(listener: (state: VoiceControlState) => void): () => void {
+ this.statusListeners.add(listener);
+ listener({ status: this.status });
+ return () => this.statusListeners.delete(listener);
+ }
+
+ onWakeWord(listener: (keyword: string) => void): () => void {
+ this.wakeWordListeners.add(listener);
+ return () => this.wakeWordListeners.delete(listener);
+ }
+
+ start(): void {
+ if (this.started || !global.isAndroid) return;
+ this.ensureController();
+ this.started = true;
+ this.controller?.start();
+ }
+
+ stop(): void {
+ if (!this.started || !global.isAndroid) return;
+ this.started = false;
+ this.controller?.stop();
+ this.setStatus("Voice control stopped.");
+ }
+
+ private ensureController(): void {
+ if (!global.isAndroid || this.controller) return;
+ const context = Utils.android.getApplicationContext();
+ if (!context) {
+ throw new Error("Android application context unavailable");
+ }
+ this.controller = new com.faceclaw.app.FaceclawVoiceController(context);
+ this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({
+ onStatus: (status: string) => {
+ this.setStatus(String(status));
+ },
+ onWakeWord: (keyword: string) => {
+ for (const listener of this.wakeWordListeners) {
+ listener(String(keyword));
+ }
+ },
+ });
+ this.controller.setListener(this.listenerProxy);
+ }
+
+ private setStatus(status: string): void {
+ this.status = status;
+ for (const listener of this.statusListeners) {
+ listener({ status });
+ }
+ }
+}
+
+export const voiceControlBridge = new FaceclawVoiceControlBridge();
diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts
index e462f8a1..78f8ed80 100644
--- a/app/ui/dashboard-settings.ts
+++ b/app/ui/dashboard-settings.ts
@@ -21,9 +21,11 @@ export type WakeModeSetting = "tiled" | "synchronized";
export const DEFAULT_SYSTEM_CARD_NAME = "Faceclaw";
export const DEFAULT_SCREEN_TIMEOUT: ScreenTimeoutSetting = "30s";
export const DEFAULT_WAKE_MODE: WakeModeSetting = "tiled";
+export const DEFAULT_VOICE_CONTROL_ENABLED = false;
const SYSTEM_CARD_NAME_KEY = "dashboard.systemCardName";
const SCREEN_TIMEOUT_KEY = "display.screenTimeout";
const WAKE_MODE_KEY = "display.wakeMode";
+const VOICE_CONTROL_ENABLED_KEY = "voice.enabled";
const SYSTEM_CARD_SETTING_KEYS: Record = {
showFaceclawLogo: "dashboard.systemCard.showFaceclawLogo",
showBatteryIndicators: "dashboard.systemCard.showBatteryIndicators",
@@ -138,6 +140,15 @@ export function saveWakeModeSetting(value: WakeModeSetting): WakeModeSetting {
return normalized;
}
+export function loadVoiceControlEnabled(): boolean {
+ return ApplicationSettings.getBoolean(VOICE_CONTROL_ENABLED_KEY, DEFAULT_VOICE_CONTROL_ENABLED);
+}
+
+export function saveVoiceControlEnabled(value: boolean): boolean {
+ ApplicationSettings.setBoolean(VOICE_CONTROL_ENABLED_KEY, value);
+ return value;
+}
+
export function nextScreenTimeoutSetting(value: ScreenTimeoutSetting): ScreenTimeoutSetting {
switch (value) {
case "15s":
diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts
index da74a0db..0da8ab8b 100644
--- a/app/ui/dashboard.ts
+++ b/app/ui/dashboard.ts
@@ -25,6 +25,7 @@ import {
loadScreenTimeoutSetting,
loadSystemCardSettings,
loadSystemCardName,
+ loadVoiceControlEnabled,
loadWakeModeSetting,
nextScreenTimeoutSetting,
nextWakeModeSetting,
@@ -37,6 +38,7 @@ import {
screenTimeoutMs,
saveSystemCardSetting,
saveSystemCardName,
+ saveVoiceControlEnabled,
wakeModeLabel,
type DashboardSlotId,
type NightscoutSettings,
@@ -63,6 +65,7 @@ let dashboardState = {
systemCardSettings: loadSystemCardSettings(),
screenTimeout: loadScreenTimeoutSetting(),
wakeMode: loadWakeModeSetting(),
+ voiceControlEnabled: loadVoiceControlEnabled(),
tiledWakePaintPending: false,
nightscoutSettings: loadNightscoutSettings(),
battery: {
@@ -80,6 +83,7 @@ const dashboardActions: LayerActions = {
startNightscoutSiteUrlEdit: () => {},
startNightscoutApiTokenEdit: () => {},
endTextSettingEdit: () => {},
+ setVoiceControlEnabled: () => {},
};
const dashboardFont = loadEmbeddedTerminus12();
const dashboardSystemFont = loadEmbeddedTerminus16();
@@ -210,6 +214,14 @@ export function setDashboardSystemCardSetting(key: SystemCardSettingKey, value:
saveSystemCardSetting(key, value);
}
+export function isDashboardVoiceControlEnabled(): boolean {
+ return dashboardState.voiceControlEnabled;
+}
+
+export function setDashboardVoiceControlEnabled(value: boolean): void {
+ dashboardState.voiceControlEnabled = saveVoiceControlEnabled(value);
+}
+
export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean {
const timeoutMs = screenTimeoutMs(dashboardState.screenTimeout);
if (timeoutMs === null || !dashboardState.screenOn) return false;
@@ -511,6 +523,12 @@ function createSettingsMenuLayer(): MenuLayer {
ctx.stack.push(createDashboardSettingsMenuLayer());
},
},
+ {
+ label: "Voice",
+ onSelect: (ctx) => {
+ ctx.stack.push(createVoiceSettingsMenuLayer());
+ },
+ },
{
label: "Integrations",
onSelect: (ctx) => {
@@ -558,6 +576,24 @@ function setWakeMode(value: WakeModeSetting): void {
dashboardState.wakeMode = saveWakeModeSetting(value);
}
+function createVoiceSettingsMenuLayer(): MenuLayer {
+ return new MenuLayer(
+ "Voice",
+ [
+ {
+ label: "Enable",
+ onSelect: async (ctx) => {
+ await ctx.actions.setVoiceControlEnabled(!dashboardState.voiceControlEnabled);
+ },
+ render: ({ image, x, y, width, selected }) => {
+ drawToggleMenuItem(image, dashboardFont, x, y, width, "Enable", dashboardState.voiceControlEnabled, selected);
+ },
+ },
+ ],
+ TOP_LEFT_MENU_LAYOUT,
+ );
+}
+
function createDashboardSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Dashboard",
diff --git a/app/ui/layers.ts b/app/ui/layers.ts
index 857e2402..f949fd5c 100644
--- a/app/ui/layers.ts
+++ b/app/ui/layers.ts
@@ -15,6 +15,7 @@ export type LayerActions = {
startNightscoutSiteUrlEdit: () => Promise | void;
startNightscoutApiTokenEdit: () => Promise | void;
endTextSettingEdit: () => Promise | void;
+ setVoiceControlEnabled: (enabled: boolean) => Promise | void;
};
export type PaintBelow = () => GrayImage;