Add initial voice-command support

Adds a single wakeword-style voice command, "screen on", and deals with
all the wiring and permissions that entails. Currently this only
supports using the phone's mic (or a bluetooth headset), not the G2 mic.
This commit is contained in:
James Babcock committed 2026-04-27 16:45:09 -07:00
1 parent c4cb4f9f01
commit 9fbb985db6
35 files changed
+1722 -16

No files matched your search

+3
View File
@@ -2,3 +2,6 @@ Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficia
and others who contributed to documenting the G2's bluetooth protocol. It has enabled me and others who contributed to documenting the G2's bluetooth protocol. It has enabled me
to keep my room clean, so to speak. to keep my room clean, so to speak.
Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
community.
+7
View File
@@ -8,9 +8,16 @@ android {
buildToolsVersion "35" buildToolsVersion "35"
// ndkVersion "" // ndkVersion ""
sourceSets.main {
jniLibs.srcDirs += ["$projectDir/src/main/jniLibs"]
}
defaultConfig { defaultConfig {
minSdkVersion 24 minSdkVersion 24
targetSdkVersion 35 targetSdkVersion 35
ndk {
abiFilters "arm64-v8a"
}
// Version Information // Version Information
versionCode 1 versionCode 1
@@ -19,8 +19,11 @@
<uses-permission android:name="android.permission.BLUETOOTH_CONNECT"/> <uses-permission android:name="android.permission.BLUETOOTH_CONNECT"/>
<uses-permission android:name="android.permission.BLUETOOTH_SCAN" android:usesPermissionFlags="neverForLocation"/> <uses-permission android:name="android.permission.BLUETOOTH_SCAN" android:usesPermissionFlags="neverForLocation"/>
<uses-permission android:name="android.permission.POST_NOTIFICATIONS"/> <uses-permission android:name="android.permission.POST_NOTIFICATIONS"/>
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE"/> <uses-permission android:name="android.permission.FOREGROUND_SERVICE"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CONNECTED_DEVICE"/> <uses-permission android:name="android.permission.FOREGROUND_SERVICE_CONNECTED_DEVICE"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE"/>
<application <application
android:name="com.tns.NativeScriptApplication" android:name="com.tns.NativeScriptApplication"
@@ -51,7 +54,7 @@
android:name="com.faceclaw.app.FaceclawForegroundService" android:name="com.faceclaw.app.FaceclawForegroundService"
android:enabled="true" android:enabled="true"
android:exported="false" android:exported="false"
android:foregroundServiceType="connectedDevice" /> android:foregroundServiceType="connectedDevice|microphone" />
<service <service
android:name="com.faceclaw.app.FaceclawMediaNotificationListenerService" android:name="com.faceclaw.app.FaceclawMediaNotificationListenerService"
android:enabled="true" android:enabled="true"
@@ -0,0 +1,16 @@
---
frameworks:
- 其他
license: Apache License 2.0
tasks:
- keyword-spotting
---
sherpa 中自定义唤醒词模型,训练数据为 gigaspeech XL (10000 小时)数据,模型大小约为 3.3 M, 建模单元为 BPE 。 使用 icefall 训练,已转换为 onnx 格式,此仓库主要是给 sherpa-onnx 这个 inference 引擎使用。
模型结构为 zipformer 模型, 本质是一个非常小的语音识别模型,为了实现唤醒词的功能,我们在解码端做了一些修改和约束。 支持自定义唤醒词,数量不限,效果需要单个调整参数。
#### Clone with HTTP
```bash
git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.git
```
@@ -0,0 +1,10 @@
▁HE LL O ▁WORLD
▁HI ▁GO O G LE
▁HE Y ▁S I RI
▁A LE X A
▁LOVE ▁AND ▁P E A CE
▁PLAY ▁MU S IC
▁GO ▁HOME
▁HA PP Y ▁NEW ▁YEAR
▁ME R RY ▁ CH R IST MA S
▁S C RE EN ▁ON :1.5 #0.35
@@ -0,0 +1,9 @@
HELLO WORLD
HI GOOGLE
HEY SIRI
ALEXA
LOVE AND PEACE
PLAY MUSIC
GO HOME
HAPPY NEW YEAR
MERRY CHRISTMAS
@@ -0,0 +1,500 @@
<blk> 0
<sos/eos> 1
<unk> 2
S 3
T 4
▁THE 5
▁A 6
▁TO 7
▁AND 8
N 9
E 10
D 11
ING 12
' 13
▁OF 14
▁I 15
▁IN 16
Y 17
ED 18
M 19
▁ 20
▁THAT 21
O 22
C 23
R 24
A 25
P 26
▁YOU 27
U 28
RE 29
▁IT 30
▁S 31
G 32
L 33
B 34
▁IS 35
I 36
AL 37
ER 38
▁RE 39
▁C 40
▁WE 41
▁SO 42
▁BE 43
OR 44
AR 45
IN 46
▁FOR 47
F 48
▁HE 49
LE 50
LY 51
▁E 52
▁WAS 53
K 54
▁F 55
W 56
VE 57
RI 58
CH 59
LL 60
▁THIS 61
▁T 62
EN 63
▁ON 64
IT 65
CE 66
▁WITH 67
▁B 68
▁W 69
▁ME 70
▁DO 71
▁DE 72
▁HAVE 73
▁BUT 74
ENT 75
▁P 76
UR 77
ON 78
▁AS 79
IC 80
TER 81
TE 82
▁THEY 83
ES 84
RA 85
▁NOT 86
TH 87
▁ARE 88
RO 89
IL 90
ATION 91
AN 92
▁ST 93
▁WHAT 94
SE 95
LI 96
V 97
▁CAN 98
▁G 99
▁OR 100
▁ALL 101
NE 102
▁AT 103
▁CON 104
H 105
▁LIKE 106
▁MO 107
▁HIS 108
▁AN 109
▁MA 110
▁THERE 111
▁SU 112
▁MY 113
NG 114
- 115
▁ABOUT 116
▁SE 117
LA 118
▁ONE 119
LO 120
▁SHE 121
CK 122
▁HAD 123
AD 124
MENT 125
VER 126
▁CA 127
IR 128
▁IF 129
▁NO 130
EL 131
▁O 132
▁FROM 133
▁PA 134
▁YOUR 135
▁KNOW 136
RY 137
ION 138
UN 139
▁JUST 140
▁UP 141
▁US 142
▁WHO 143
▁GO 144
ME 145
▁EX 146
▁OUT 147
ND 148
CI 149
▁PO 150
▁HER 151
▁FA 152
▁PRO 153
▁WHEN 154
ATE 155
AT 156
▁BY 157
PE 158
AS 159
▁LA 160
GE 161
▁CO 162
▁PEOPLE 163
▁K 164
▁SOME 165
IGHT 166
▁THINK 167
VI 168
DE 169
▁LE 170
▁WERE 171
▁LI 172
OM 173
▁WOULD 174
▁SEE 175
CO 176
▁BO 177
▁OUR 178
▁HOW 179
ET 180
Z 181
UL 182
▁HO 183
IG 184
▁GET 185
▁DON 186
IS 187
URE 188
▁WHICH 189
▁MI 190
IES 191
US 192
X 193
▁MORE 194
▁NOW 195
▁TIME 196
▁UN 197
▁FI 198
TY 199
▁WILL 200
RU 201
IVE 202
▁THEIR 203
▁SAY 204
ITY 205
▁HA 206
LU 207
▁RO 208
HO 209
OL 210
IST 211
AGE 212
MA 213
KE 214
▁RA 215
ID 216
ERS 217
▁LO 218
▁SP 219
▁DA 220
▁SAID 221
TION 222
THER 223
▁EN 224
▁BECAUSE 225
▁HAS 226
EST 227
TI 228
ACT 229
▁BA 230
▁VERY 231
▁THEM 232
▁BEEN 233
▁WA 234
ABLE 235
▁SA 236
▁DID 237
▁HIM 238
▁DIS 239
▁WANT 240
▁OTHER 241
IM 242
▁PRE 243
▁LOOK 244
▁GOING 245
▁VI 246
▁WORK 247
▁DR 248
ENCE 249
▁DI 250
FF 251
TA 252
▁REALLY 253
▁RIGHT 254
MO 255
ALLY 256
▁INTO 257
OW 258
▁WAY 259
▁FE 260
▁COULD 261
▁THEN 262
TED 263
▁HERE 264
▁THESE 265
▁OVER 266
▁WELL 267
▁NA 268
▁ANY 269
HE 270
▁COMP 271
▁COM 272
▁TH 273
▁EVEN 274
▁TWO 275
▁JO 276
▁MAKE 277
▁BACK 278
▁WHERE 279
SH 280
▁QU 281
MAN 282
▁YEAR 283
▁TRA 284
▁CHA 285
QU 286
CA 287
▁ALSO 288
CTION 289
ANCE 290
UM 291
PER 292
DI 293
▁HU 294
▁MU 295
FUL 296
▁NEW 297
ARD 298
J 299
ICAL 300
ANT 301
END 302
▁MAR 303
▁NE 304
IF 305
DER 306
ONE 307
▁THOSE 308
PP 309
▁MAN 310
▁GOOD 311
▁TAKE 312
SIDE 313
PO 314
HA 315
▁LITTLE 316
▁APP 317
▁BI 318
UGH 319
▁COME 320
ISH 321
TIC 322
▁PER 323
▁THINGS 324
▁MUCH 325
▁FIRST 326
▁THAN 327
IA 328
▁AGAIN 329
▁STA 330
▁PART 331
▁GA 332
▁MEAN 333
▁LET 334
▁DOWN 335
▁NEED 336
▁GU 337
LES 338
▁RU 339
ATED 340
▁SHA 341
VO 342
▁TALK 343
▁DAY 344
▁CHI 345
▁SOMETHING 346
▁AFTER 347
▁THOUGH 348
▁CALL 349
▁GRA 350
▁EVERY 351
▁KIND 352
ROW 353
AIN 354
▁BU 355
▁THROUGH 356
▁SHOULD 357
▁ONLY 358
▁TA 359
▁MIN 360
▁GOT 361
IAN 362
▁GREAT 363
▁RI 364
▁DOES 365
▁THING 366
DAY 367
OUS 368
▁START 369
▁MOST 370
▁COL 371
PH 372
▁LOT 373
IZE 374
BER 375
IOUS 376
▁LIFE 377
▁MANY 378
▁FEEL 379
ITION 380
▁VA 381
NESS 382
▁BEFORE 383
▁THREE 384
▁WORLD 385
▁GIVE 386
▁WHY 387
▁PI 388
▁LONG 389
PORT 390
▁SHOW 391
▁ACTUALLY 392
▁NEVER 393
IV 394
FORM 395
▁THANK 396
▁HI 397
▁PUT 398
▁LOVE 399
▁QUESTION 400
▁HELP 401
▁HAPPEN 402
▁MR 403
▁OWN 404
WARD 405
▁MADE 406
▁HAND 407
▁STATE 408
IBLE 409
▁TELL 410
▁PH 411
▁CAME 412
▁TEA 413
▁STILL 414
▁DIFFERENT 415
▁PLACE 416
▁MIGHT 417
▁FIND 418
▁DU 419
▁IMP 420
▁OLD 421
▁PERSON 422
▁COUNT 423
▁JA 424
▁POINT 425
▁HIGH 426
SHIP 427
▁LAST 428
▁SAME 429
▁FOUR 430
▁AROUND 431
▁OH 432
▁CHANGE 433
▁UNDER 434
▁HOME 435
▁ANOTHER 436
▁FOUND 437
ATIVE 438
▁TURN 439
▁YES 440
LAND 441
▁ALWAYS 442
▁YEAH 443
▁SCHOOL 444
▁JE 445
▁NEXT 446
▁WORD 447
▁JU 448
LINE 449
▁SIX 450
▁LEARN 451
▁BOOK 452
▁PLAY 453
▁AWAY 454
▁HOUSE 455
▁PLAN 456
▁WHILE 457
▁COMMUNI 458
▁FRIEND 459
▁AMERICA 460
▁INTEREST 461
▁KEEP 462
▁BETTER 463
▁IDEA 464
▁HARD 465
▁TWENTY 466
▁THOUSAND 467
▁HUNDRED 468
▁WEEK 469
▁NUMBER 470
▁COURSE 471
▁OKAY 472
PRESS 473
▁SYSTEM 474
▁NINE 475
▁UNDERSTAND 476
▁WITHOUT 477
▁BELIEVE 478
▁POWER 479
▁MAYBE 480
▁BUILD 481
▁MONEY 482
▁BETWEEN 483
▁GONNA 484
▁IMPORTANT 485
▁ANYTHING 486
▁LEFT 487
▁HAVING 488
▁WHITE 489
▁PROBLEM 490
▁SECOND 491
▁SPEAK 492
▁TOGETHER 493
▁BLACK 494
▁PUBLIC 495
▁HEALTH 496
▁NOTHING 497
▁EVERYTHING 498
Q 499
@@ -27,7 +27,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final int WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE; private static final int WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE;
private static final int IMAGE_WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT; private static final int IMAGE_WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT;
private static final int DESIRED_MTU = 247; private static final int DESIRED_MTU = 512;
private static final int CONNECTION_PRIORITY_HIGH = BluetoothGatt.CONNECTION_PRIORITY_HIGH; private static final int CONNECTION_PRIORITY_HIGH = BluetoothGatt.CONNECTION_PRIORITY_HIGH;
private static final int CONNECT_TIMEOUT_MS = 5_000; private static final int CONNECT_TIMEOUT_MS = 5_000;
@@ -40,7 +40,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final int BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000; private static final int BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000;
private static final int BATTERY_INPUT_QUIET_MS = 5_000; private static final int BATTERY_INPUT_QUIET_MS = 5_000;
private static final int WINDOW_SIZE = 3; private static final int WINDOW_SIZE = 3;
private static final int IMAGE_FRAGMENT_SIZE = 1000; private static final int IMAGE_FRAGMENT_SIZE = 3800;
private static final int IMAGE_RETRY_DELAY_MS = 10_000; private static final int IMAGE_RETRY_DELAY_MS = 10_000;
private static final boolean IMAGE_FRAGMENT_NO_ACK = false; private static final boolean IMAGE_FRAGMENT_NO_ACK = false;
private static final int WARMUP_FRAGMENT_TIMEOUT_MS = 3_000; private static final int WARMUP_FRAGMENT_TIMEOUT_MS = 3_000;
@@ -6,6 +6,7 @@ import android.app.NotificationManager;
import android.app.PendingIntent; import android.app.PendingIntent;
import android.app.Service; import android.app.Service;
import android.content.Intent; import android.content.Intent;
import android.content.pm.ServiceInfo;
import android.os.Build; import android.os.Build;
import android.os.IBinder; import android.os.IBinder;
@@ -41,7 +42,11 @@ public class FaceclawForegroundService extends Service {
text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected" text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected"
); );
startForeground(NOTIFICATION_ID, notification); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(NOTIFICATION_ID, notification, foregroundServiceType());
} else {
startForeground(NOTIFICATION_ID, notification);
}
if (ACTION_UPDATE.equals(action)) { if (ACTION_UPDATE.equals(action)) {
NotificationManager manager = getSystemService(NotificationManager.class); NotificationManager manager = getSystemService(NotificationManager.class);
@@ -95,4 +100,9 @@ public class FaceclawForegroundService extends Service {
.setOnlyAlertOnce(true) .setOnlyAlertOnce(true)
.build(); .build();
} }
private int foregroundServiceType() {
return ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE
| ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE;
}
} }
@@ -0,0 +1,341 @@
package com.faceclaw.app;
import android.Manifest;
import android.bluetooth.BluetoothAdapter;
import android.bluetooth.BluetoothDevice;
import android.bluetooth.BluetoothManager;
import android.bluetooth.BluetoothProfile;
import android.content.Context;
import android.content.pm.PackageManager;
import android.content.res.AssetManager;
import android.media.AudioDeviceInfo;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import androidx.core.content.ContextCompat;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotter;
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
public class FaceclawVoiceController {
private static final String TAG = "FaceclawVoice";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
private static final String[] MODEL_FILES = {
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
"tokens.txt",
"screen-on-keywords.txt"
};
private final Context appContext;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private volatile FaceclawVoiceControllerListener listener;
private Thread workerThread;
private volatile boolean started;
private AudioRecord audioRecord;
private KeywordSpotter keywordSpotter;
private OnlineStream stream;
public FaceclawVoiceController(Context context) {
this.appContext = context.getApplicationContext();
}
public void setListener(FaceclawVoiceControllerListener listener) {
this.listener = listener;
}
public void start() {
synchronized (lock) {
if (started) {
emitStatus("Voice control is already listening.");
return;
}
if (!hasRecordAudioPermission()) {
emitStatus("Voice control needs microphone permission.");
return;
}
if (!hasBluetoothHeadset()) {
emitStatus("Voice control needs a paired Bluetooth headset.");
return;
}
started = true;
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
workerThread.start();
}
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
if (!started) {
return;
}
started = false;
threadToJoin = workerThread;
}
stopAudio();
if (threadToJoin != null) {
threadToJoin.interrupt();
}
}
public void close() {
stop();
}
private void runLoop() {
try {
emitStatus("Voice control loading wake-word model...");
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
if (!startAudio()) {
emitStatus("Voice control could not start microphone input.");
return;
}
emitStatus("Voice control listening for \"screen on\".");
processAudio();
} catch (Throwable error) {
Log.e(TAG, "Voice control failed", error);
emitStatus("Voice control failed: " + error.getMessage());
} finally {
releaseSherpa();
stopAudio();
synchronized (lock) {
started = false;
workerThread = null;
}
}
}
private KeywordSpotterConfig buildConfig(File modelDir) {
return KeywordSpotterConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setOnlineModelConfig(OnlineModelConfig.builder()
.setTransducer(OnlineTransducerModelConfig.builder()
.setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setModelType("zipformer2")
.setNumThreads(1)
.build())
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
.setKeywordsScore(1.5f)
.setKeywordsThreshold(0.35f)
.build();
}
private File installModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
for (String fileName : MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
if (destination.exists() && destination.length() > 0) {
return;
}
try (InputStream input = assets.open(assetPath);
FileOutputStream output = new FileOutputStream(destination)) {
byte[] buffer = new byte[64 * 1024];
int read;
while ((read = input.read(buffer)) >= 0) {
output.write(buffer, 0, read);
}
}
}
private boolean startAudio() {
routeToBluetoothHeadset();
int minBufferBytes = AudioRecord.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
);
if (minBufferBytes <= 0) {
return false;
}
if (!hasRecordAudioPermission()) {
return false;
}
audioRecord = new AudioRecord(
MediaRecorder.AudioSource.VOICE_RECOGNITION,
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
minBufferBytes * 2
);
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
audioRecord.release();
audioRecord = null;
return false;
}
audioRecord.startRecording();
return true;
}
private void processAudio() {
short[] buffer = new short[SAMPLE_RATE / 10];
while (started && !Thread.currentThread().isInterrupted()) {
AudioRecord currentAudio = audioRecord;
OnlineStream currentStream = stream;
KeywordSpotter currentSpotter = keywordSpotter;
if (currentAudio == null || currentStream == null || currentSpotter == null) {
return;
}
int count = currentAudio.read(buffer, 0, buffer.length);
if (count <= 0) {
continue;
}
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = buffer[i] / 32768.0f;
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
while (currentSpotter.isReady(currentStream)) {
currentSpotter.decode(currentStream);
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
String keyword = result == null ? "" : result.getKeyword();
if (keyword != null && keyword.trim().length() > 0) {
currentSpotter.reset(currentStream);
emitWakeWord(keyword);
}
}
}
}
private void stopAudio() {
AudioRecord record = audioRecord;
audioRecord = null;
if (record != null) {
try {
record.stop();
} catch (IllegalStateException ignored) {
}
record.release();
}
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
if (audioManager != null) {
if (Build.VERSION.SDK_INT >= 31) {
audioManager.clearCommunicationDevice();
}
audioManager.stopBluetoothSco();
audioManager.setBluetoothScoOn(false);
}
}
private void releaseSherpa() {
if (stream != null) {
stream.release();
stream = null;
}
if (keywordSpotter != null) {
keywordSpotter.release();
keywordSpotter = null;
}
}
private void routeToBluetoothHeadset() {
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
if (audioManager == null) {
return;
}
audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION);
audioManager.startBluetoothSco();
audioManager.setBluetoothScoOn(true);
if (Build.VERSION.SDK_INT >= 31) {
for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) {
if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
|| device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) {
audioManager.setCommunicationDevice(device);
return;
}
}
}
}
private boolean hasBluetoothHeadset() {
if (Build.VERSION.SDK_INT >= 31
&& ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT)
!= PackageManager.PERMISSION_GRANTED) {
return false;
}
BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
BluetoothAdapter adapter = manager == null ? null : manager.getAdapter();
if (adapter == null) {
return false;
}
try {
int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET);
if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) {
return true;
}
Set<BluetoothDevice> bondedDevices = adapter.getBondedDevices();
return bondedDevices != null && !bondedDevices.isEmpty();
} catch (SecurityException ignored) {
return false;
}
}
private boolean hasRecordAudioPermission() {
return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
== PackageManager.PERMISSION_GRANTED;
}
private void emitStatus(String status) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onStatus(status));
}
private void emitWakeWord(String keyword) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onWakeWord(keyword));
}
}
@@ -0,0 +1,7 @@
package com.faceclaw.app;
public interface FaceclawVoiceControllerListener {
void onStatus(String status);
void onWakeWord(String keyword);
}
@@ -0,0 +1,54 @@
package com.k2fsa.sherpa.onnx;
public class FeatureConfig {
private final int sampleRate;
private final int featureDim;
private final float dither;
private FeatureConfig(Builder builder) {
this.sampleRate = builder.sampleRate;
this.featureDim = builder.featureDim;
this.dither = builder.dither;
}
public static Builder builder() {
return new Builder();
}
public int getSampleRate() {
return sampleRate;
}
public int getFeatureDim() {
return featureDim;
}
public float getDither() {
return dither;
}
public static class Builder {
private int sampleRate = 16000;
private int featureDim = 80;
private float dither = 0.0f;
public Builder setSampleRate(int sampleRate) {
this.sampleRate = sampleRate;
return this;
}
public Builder setFeatureDim(int featureDim) {
this.featureDim = featureDim;
return this;
}
public Builder setDither(float dither) {
this.dither = dither;
return this;
}
public FeatureConfig build() {
return new FeatureConfig(this);
}
}
}
@@ -0,0 +1,66 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotter {
private long ptr;
public KeywordSpotter(KeywordSpotterConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
}
}
public OnlineStream createStream() {
return createStream("");
}
public OnlineStream createStream(String keywords) {
return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
}
public boolean isReady(OnlineStream stream) {
return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
}
public void decode(OnlineStream stream) {
decode(ptr, stream.getPtr());
}
public void reset(OnlineStream stream) {
reset(ptr, stream.getPtr());
}
public KeywordSpotterResult getResult(OnlineStream stream) {
return getResult(ptr, stream.getPtr());
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native long newFromFile(KeywordSpotterConfig config);
private native void delete(long ptr);
private native long createStream(long ptr, String keywords);
private native boolean isReady(long ptr, long streamPtr);
private native void decode(long ptr, long streamPtr);
private native void reset(long ptr, long streamPtr);
private native KeywordSpotterResult getResult(long ptr, long streamPtr);
}
@@ -0,0 +1,92 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterConfig {
private final FeatureConfig featConfig;
private final OnlineModelConfig modelConfig;
private final int maxActivePaths;
private final String keywordsFile;
private final float keywordsScore;
private final float keywordsThreshold;
private final int numTrailingBlanks;
private KeywordSpotterConfig(Builder builder) {
this.featConfig = builder.featConfig;
this.modelConfig = builder.modelConfig;
this.maxActivePaths = builder.maxActivePaths;
this.keywordsFile = builder.keywordsFile;
this.keywordsScore = builder.keywordsScore;
this.keywordsThreshold = builder.keywordsThreshold;
this.numTrailingBlanks = builder.numTrailingBlanks;
}
public static Builder builder() {
return new Builder();
}
public FeatureConfig getFeatConfig() {
return featConfig;
}
public OnlineModelConfig getModelConfig() {
return modelConfig;
}
public int getMaxActivePaths() {
return maxActivePaths;
}
public String getKeywordsFile() {
return keywordsFile;
}
public float getKeywordsScore() {
return keywordsScore;
}
public float getKeywordsThreshold() {
return keywordsThreshold;
}
public int getNumTrailingBlanks() {
return numTrailingBlanks;
}
public static class Builder {
private FeatureConfig featConfig = FeatureConfig.builder().build();
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
private int maxActivePaths = 4;
private String keywordsFile = "keywords.txt";
private float keywordsScore = 1.5f;
private float keywordsThreshold = 0.25f;
private int numTrailingBlanks = 2;
public Builder setFeatureConfig(FeatureConfig featConfig) {
this.featConfig = featConfig;
return this;
}
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
this.modelConfig = modelConfig;
return this;
}
public Builder setKeywordsFile(String keywordsFile) {
this.keywordsFile = keywordsFile;
return this;
}
public Builder setKeywordsScore(float keywordsScore) {
this.keywordsScore = keywordsScore;
return this;
}
public Builder setKeywordsThreshold(float keywordsThreshold) {
this.keywordsThreshold = keywordsThreshold;
return this;
}
public KeywordSpotterConfig build() {
return new KeywordSpotterConfig(this);
}
}
}
@@ -0,0 +1,25 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterResult {
private final String keyword;
private final String[] tokens;
private final float[] timestamps;
public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
this.keyword = keyword == null ? "" : keyword;
this.tokens = tokens == null ? new String[0] : tokens;
this.timestamps = timestamps == null ? new float[0] : timestamps;
}
public String getKeyword() {
return keyword;
}
public String[] getTokens() {
return tokens;
}
public float[] getTimestamps() {
return timestamps;
}
}
@@ -0,0 +1,137 @@
package com.k2fsa.sherpa.onnx;
public class OnlineModelConfig {
private final OnlineTransducerModelConfig transducer;
private final OnlineParaformerModelConfig paraformer;
private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
private final OnlineNeMoCtcModelConfig neMoCtc;
private final OnlineToneCtcModelConfig toneCtc;
private final String tokens;
private final int numThreads;
private final boolean debug;
private final String provider;
private final String modelType;
private final String modelingUnit;
private final String bpeVocab;
private OnlineModelConfig(Builder builder) {
this.transducer = builder.transducer;
this.paraformer = builder.paraformer;
this.zipformer2Ctc = builder.zipformer2Ctc;
this.neMoCtc = builder.neMoCtc;
this.toneCtc = builder.toneCtc;
this.tokens = builder.tokens;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
this.modelType = builder.modelType;
this.modelingUnit = builder.modelingUnit;
this.bpeVocab = builder.bpeVocab;
}
public static Builder builder() {
return new Builder();
}
public OnlineTransducerModelConfig getTransducer() {
return transducer;
}
public OnlineParaformerModelConfig getParaformer() {
return paraformer;
}
public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
return zipformer2Ctc;
}
public OnlineNeMoCtcModelConfig getNeMoCtc() {
return neMoCtc;
}
public OnlineToneCtcModelConfig getToneCtc() {
return toneCtc;
}
public String getTokens() {
return tokens;
}
public int getNumThreads() {
return numThreads;
}
public boolean getDebug() {
return debug;
}
public String getProvider() {
return provider;
}
public String getModelType() {
return modelType;
}
public String getModelingUnit() {
return modelingUnit;
}
public String getBpeVocab() {
return bpeVocab;
}
public static class Builder {
private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
private String tokens = "";
private int numThreads = 1;
private boolean debug = false;
private String provider = "cpu";
private String modelType = "";
private String modelingUnit = "cjkchar";
private String bpeVocab = "";
public Builder setTransducer(OnlineTransducerModelConfig transducer) {
this.transducer = transducer;
return this;
}
public Builder setTokens(String tokens) {
this.tokens = tokens;
return this;
}
public Builder setModelType(String modelType) {
this.modelType = modelType;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
public Builder setModelingUnit(String modelingUnit) {
this.modelingUnit = modelingUnit;
return this;
}
public Builder setBpeVocab(String bpeVocab) {
this.bpeVocab = bpeVocab;
return this;
}
public OnlineModelConfig build() {
return new OnlineModelConfig(this);
}
}
}
@@ -0,0 +1,30 @@
package com.k2fsa.sherpa.onnx;
public class OnlineNeMoCtcModelConfig {
private final String model;
private OnlineNeMoCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineNeMoCtcModelConfig build() {
return new OnlineNeMoCtcModelConfig(this);
}
}
}
@@ -0,0 +1,42 @@
package com.k2fsa.sherpa.onnx;
public class OnlineParaformerModelConfig {
private final String encoder;
private final String decoder;
private OnlineParaformerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public OnlineParaformerModelConfig build() {
return new OnlineParaformerModelConfig(this);
}
}
}
@@ -0,0 +1,35 @@
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr;
public OnlineStream(long ptr) {
this.ptr = ptr;
}
public long getPtr() {
return ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(ptr, samples, sampleRate);
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void delete(long ptr);
}
@@ -0,0 +1,30 @@
package com.k2fsa.sherpa.onnx;
public class OnlineToneCtcModelConfig {
private final String model;
private OnlineToneCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineToneCtcModelConfig build() {
return new OnlineToneCtcModelConfig(this);
}
}
}
@@ -0,0 +1,54 @@
package com.k2fsa.sherpa.onnx;
public class OnlineTransducerModelConfig {
private final String encoder;
private final String decoder;
private final String joiner;
private OnlineTransducerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
this.joiner = builder.joiner;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public String getJoiner() {
return joiner;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
private String joiner = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public Builder setJoiner(String joiner) {
this.joiner = joiner;
return this;
}
public OnlineTransducerModelConfig build() {
return new OnlineTransducerModelConfig(this);
}
}
}
@@ -0,0 +1,30 @@
package com.k2fsa.sherpa.onnx;
public class OnlineZipformer2CtcModelConfig {
private final String model;
private OnlineZipformer2CtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineZipformer2CtcModelConfig build() {
return new OnlineZipformer2CtcModelConfig(this);
}
}
}
+25 -11
View File
@@ -1,7 +1,9 @@
import { Application, Utils } from "@nativescript/core"; import { Application, Utils } from "@nativescript/core";
const PERMISSION_REQUEST_CODE = 4242; const PERMISSION_REQUEST_CODE = 4242;
const VOICE_PERMISSION_REQUEST_CODE = 4243;
const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS"; const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS";
const RECORD_AUDIO_PERMISSION = "android.permission.RECORD_AUDIO";
function getActivity(): androidx.appcompat.app.AppCompatActivity { function getActivity(): androidx.appcompat.app.AppCompatActivity {
const activity = Application.android.foregroundActivity ?? Application.android.startActivity; const activity = Application.android.foregroundActivity ?? Application.android.startActivity;
@@ -43,17 +45,21 @@ function getRequiredPermissions(): string[] {
return permissions; return permissions;
} }
export async function ensureBlePermissions(): Promise<void> { function isPermissionGranted(permission: string): boolean {
const context = getContext();
return androidx.core.content.ContextCompat.checkSelfPermission(context, permission) ===
android.content.pm.PackageManager.PERMISSION_GRANTED;
}
async function ensurePermissions(
permissions: string[],
requestCode: number,
denialLabel: string,
): Promise<void> {
if (!global.isAndroid) return; if (!global.isAndroid) return;
const context = getContext();
const activity = getActivity(); const activity = getActivity();
const required = getRequiredPermissions(); const missing = permissions.filter((permission) => !isPermissionGranted(permission));
const missing = required.filter(
(permission) =>
androidx.core.content.ContextCompat.checkSelfPermission(context, permission) !==
android.content.pm.PackageManager.PERMISSION_GRANTED,
);
if (missing.length === 0) return; if (missing.length === 0) return;
@@ -63,7 +69,7 @@ export async function ensureBlePermissions(): Promise<void> {
permissions: string[]; permissions: string[];
grantResults: number[]; grantResults: number[];
}) => { }) => {
if (args.requestCode !== PERMISSION_REQUEST_CODE) return; if (args.requestCode !== requestCode) return;
Application.android.off(Application.android.activityRequestPermissionsEvent, callback); Application.android.off(Application.android.activityRequestPermissionsEvent, callback);
const denied = missing.filter( const denied = missing.filter(
@@ -72,7 +78,7 @@ export async function ensureBlePermissions(): Promise<void> {
); );
if (denied.length > 0) { if (denied.length > 0) {
reject(new Error(`Bluetooth permissions denied: ${denied.join(", ")}`)); reject(new Error(`${denialLabel} permissions denied: ${denied.join(", ")}`));
return; return;
} }
resolve(); resolve();
@@ -82,7 +88,15 @@ export async function ensureBlePermissions(): Promise<void> {
androidx.core.app.ActivityCompat.requestPermissions( androidx.core.app.ActivityCompat.requestPermissions(
activity, activity,
toJavaStringArray(missing), toJavaStringArray(missing),
PERMISSION_REQUEST_CODE, requestCode,
); );
}); });
} }
export async function ensureBlePermissions(): Promise<void> {
await ensurePermissions(getRequiredPermissions(), PERMISSION_REQUEST_CODE, "Bluetooth");
}
export async function ensureVoicePermissions(): Promise<void> {
await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
}
+73 -1
View File
@@ -1,19 +1,21 @@
import { ImageSource } from "@nativescript/core"; import { ImageSource } from "@nativescript/core";
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events"; import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { loadDeviceAddresses } from "./device-addresses"; import { loadDeviceAddresses } from "./device-addresses";
import { ensureBlePermissions } from "./android-permissions"; import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator"; import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service"; import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service";
import { mediaControllerBridge } from "../native/media-controller"; import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout"; import { nightscoutBridge } from "../native/nightscout";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict"; import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { grayImageToPreviewSource } from "../native/gray-image-preview"; import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { voiceControlBridge } from "../native/voice-control";
import { import {
applyDashboardScreenTimeout, applyDashboardScreenTimeout,
consumeDashboardTiledWakePaint, consumeDashboardTiledWakePaint,
drawDashboard, drawDashboard,
getDashboardNightscoutSettings, getDashboardNightscoutSettings,
getDashboardSystemCardName, getDashboardSystemCardName,
isDashboardVoiceControlEnabled,
receiveInput, receiveInput,
resetDashboardSleepTimerAndWake, resetDashboardSleepTimerAndWake,
setDashboardBatteryLevels, setDashboardBatteryLevels,
@@ -21,6 +23,7 @@ import {
setDashboardNightscoutApiToken, setDashboardNightscoutApiToken,
setDashboardNightscoutSiteUrl, setDashboardNightscoutSiteUrl,
setDashboardSystemCardName, setDashboardSystemCardName,
setDashboardVoiceControlEnabled,
} from "../ui/dashboard"; } from "../ui/dashboard";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -118,6 +121,8 @@ class DashboardController {
private offFrameMetrics: (() => void) | null = null; private offFrameMetrics: (() => void) | null = null;
private offMedia: (() => void) | null = null; private offMedia: (() => void) | null = null;
private offNightscout: (() => void) | null = null; private offNightscout: (() => void) | null = null;
private offVoiceStatus: (() => void) | null = null;
private offVoiceWakeWord: (() => void) | null = null;
private lastInput = "waiting..."; private lastInput = "waiting...";
private lastSys = "none yet"; private lastSys = "none yet";
private renderInProgress = false; private renderInProgress = false;
@@ -132,6 +137,7 @@ class DashboardController {
startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(), startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(),
startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(), startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(),
endTextSettingEdit: () => this.endTextSettingEdit(), endTextSettingEdit: () => this.endTextSettingEdit(),
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
}); });
} }
@@ -322,11 +328,20 @@ class DashboardController {
}); });
} }
}); });
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
this.appendLog(state.status);
});
this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => {
void this.handleWakeWord(keyword).catch((error) => {
this.appendLog(`wake-word handler failed: ${this.formatError(error)}`);
});
});
await mediaControllerBridge.start(); await mediaControllerBridge.start();
await nightscoutBridge.start(); await nightscoutBridge.start();
await communicator.start(); await communicator.start();
await this.requestRender("initial"); await this.requestRender("initial");
this.startVoiceControlIfEnabled();
this.dashboardTimer = setInterval(() => { this.dashboardTimer = setInterval(() => {
void this.requestRender("interval").catch((error) => { void this.requestRender("interval").catch((error) => {
const message = this.formatError(error); const message = this.formatError(error);
@@ -360,8 +375,13 @@ class DashboardController {
this.offMedia = null; this.offMedia = null;
this.offNightscout?.(); this.offNightscout?.();
this.offNightscout = null; this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
await mediaControllerBridge.stop().catch(() => {}); await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
if (communicator) { if (communicator) {
await communicator.close().catch(() => {}); await communicator.close().catch(() => {});
} }
@@ -397,6 +417,10 @@ class DashboardController {
this.offMedia = null; this.offMedia = null;
this.offNightscout?.(); this.offNightscout?.();
this.offNightscout = null; this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
const communicator = this.communicator; const communicator = this.communicator;
this.communicator = null; this.communicator = null;
@@ -413,6 +437,7 @@ class DashboardController {
} }
await mediaControllerBridge.stop().catch(() => {}); await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
await communicator?.close().catch(() => {}); await communicator?.close().catch(() => {});
} finally { } finally {
stopForegroundNotification(); stopForegroundNotification();
@@ -445,6 +470,30 @@ class DashboardController {
this.emit(); this.emit();
} }
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
if (enabled) {
try {
await ensureVoicePermissions();
} catch (error) {
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
return;
}
}
setDashboardVoiceControlEnabled(enabled);
this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`);
if (enabled) {
this.startVoiceControlIfEnabled();
} else {
voiceControlBridge.stop();
}
if (this.phase === "connected" && this.communicator) {
await this.requestRender("interval").catch((error) => {
this.appendLog(`voice setting render failed: ${this.formatError(error)}`);
});
}
}
private endTextSettingEdit(): void { private endTextSettingEdit(): void {
const finishedKind = this.activeTextSettingEditorKind; const finishedKind = this.activeTextSettingEditorKind;
this.editingSystemCardName = false; this.editingSystemCardName = false;
@@ -580,6 +629,29 @@ class DashboardController {
} }
} }
private async handleWakeWord(keyword: string): Promise<void> {
const normalized = keyword.trim();
if (normalized.length === 0) return;
this.appendLog(`wake-word detected: ${normalized}`);
const changed = resetDashboardSleepTimerAndWake();
if (this.phase === "connected" && this.communicator) {
if (changed) {
await this.requestRender("initial");
}
return;
}
if (changed) {
const image = drawDashboard();
this.setDisplayPreview(grayImageToPreviewSource(image));
}
}
private startVoiceControlIfEnabled(): void {
if (!this.communicator) return;
if (!isDashboardVoiceControlEnabled()) return;
voiceControlBridge.start();
}
private clearDashboardTimer(): void { private clearDashboardTimer(): void {
if (this.dashboardTimer) { if (this.dashboardTimer) {
clearInterval(this.dashboardTimer); clearInterval(this.dashboardTimer);
+70
View File
@@ -0,0 +1,70 @@
import { Utils } from "@nativescript/core";
declare const com: any;
export type VoiceControlState = {
status: string;
};
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
private status = "Voice control stopped.";
private started = false;
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
listener({ status: this.status });
return () => this.statusListeners.delete(listener);
}
onWakeWord(listener: (keyword: string) => void): () => void {
this.wakeWordListeners.add(listener);
return () => this.wakeWordListeners.delete(listener);
}
start(): void {
if (this.started || !global.isAndroid) return;
this.ensureController();
this.started = true;
this.controller?.start();
}
stop(): void {
if (!this.started || !global.isAndroid) return;
this.started = false;
this.controller?.stop();
this.setStatus("Voice control stopped.");
}
private ensureController(): void {
if (!global.isAndroid || this.controller) return;
const context = Utils.android.getApplicationContext();
if (!context) {
throw new Error("Android application context unavailable");
}
this.controller = new com.faceclaw.app.FaceclawVoiceController(context);
this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({
onStatus: (status: string) => {
this.setStatus(String(status));
},
onWakeWord: (keyword: string) => {
for (const listener of this.wakeWordListeners) {
listener(String(keyword));
}
},
});
this.controller.setListener(this.listenerProxy);
}
private setStatus(status: string): void {
this.status = status;
for (const listener of this.statusListeners) {
listener({ status });
}
}
}
export const voiceControlBridge = new FaceclawVoiceControlBridge();
+11
View File
@@ -21,9 +21,11 @@ export type WakeModeSetting = "tiled" | "synchronized";
export const DEFAULT_SYSTEM_CARD_NAME = "Faceclaw"; export const DEFAULT_SYSTEM_CARD_NAME = "Faceclaw";
export const DEFAULT_SCREEN_TIMEOUT: ScreenTimeoutSetting = "30s"; export const DEFAULT_SCREEN_TIMEOUT: ScreenTimeoutSetting = "30s";
export const DEFAULT_WAKE_MODE: WakeModeSetting = "tiled"; export const DEFAULT_WAKE_MODE: WakeModeSetting = "tiled";
export const DEFAULT_VOICE_CONTROL_ENABLED = false;
const SYSTEM_CARD_NAME_KEY = "dashboard.systemCardName"; const SYSTEM_CARD_NAME_KEY = "dashboard.systemCardName";
const SCREEN_TIMEOUT_KEY = "display.screenTimeout"; const SCREEN_TIMEOUT_KEY = "display.screenTimeout";
const WAKE_MODE_KEY = "display.wakeMode"; const WAKE_MODE_KEY = "display.wakeMode";
const VOICE_CONTROL_ENABLED_KEY = "voice.enabled";
const SYSTEM_CARD_SETTING_KEYS: Record<SystemCardSettingKey, string> = { const SYSTEM_CARD_SETTING_KEYS: Record<SystemCardSettingKey, string> = {
showFaceclawLogo: "dashboard.systemCard.showFaceclawLogo", showFaceclawLogo: "dashboard.systemCard.showFaceclawLogo",
showBatteryIndicators: "dashboard.systemCard.showBatteryIndicators", showBatteryIndicators: "dashboard.systemCard.showBatteryIndicators",
@@ -138,6 +140,15 @@ export function saveWakeModeSetting(value: WakeModeSetting): WakeModeSetting {
return normalized; return normalized;
} }
export function loadVoiceControlEnabled(): boolean {
return ApplicationSettings.getBoolean(VOICE_CONTROL_ENABLED_KEY, DEFAULT_VOICE_CONTROL_ENABLED);
}
export function saveVoiceControlEnabled(value: boolean): boolean {
ApplicationSettings.setBoolean(VOICE_CONTROL_ENABLED_KEY, value);
return value;
}
export function nextScreenTimeoutSetting(value: ScreenTimeoutSetting): ScreenTimeoutSetting { export function nextScreenTimeoutSetting(value: ScreenTimeoutSetting): ScreenTimeoutSetting {
switch (value) { switch (value) {
case "15s": case "15s":
+36
View File
@@ -25,6 +25,7 @@ import {
loadScreenTimeoutSetting, loadScreenTimeoutSetting,
loadSystemCardSettings, loadSystemCardSettings,
loadSystemCardName, loadSystemCardName,
loadVoiceControlEnabled,
loadWakeModeSetting, loadWakeModeSetting,
nextScreenTimeoutSetting, nextScreenTimeoutSetting,
nextWakeModeSetting, nextWakeModeSetting,
@@ -37,6 +38,7 @@ import {
screenTimeoutMs, screenTimeoutMs,
saveSystemCardSetting, saveSystemCardSetting,
saveSystemCardName, saveSystemCardName,
saveVoiceControlEnabled,
wakeModeLabel, wakeModeLabel,
type DashboardSlotId, type DashboardSlotId,
type NightscoutSettings, type NightscoutSettings,
@@ -63,6 +65,7 @@ let dashboardState = {
systemCardSettings: loadSystemCardSettings(), systemCardSettings: loadSystemCardSettings(),
screenTimeout: loadScreenTimeoutSetting(), screenTimeout: loadScreenTimeoutSetting(),
wakeMode: loadWakeModeSetting(), wakeMode: loadWakeModeSetting(),
voiceControlEnabled: loadVoiceControlEnabled(),
tiledWakePaintPending: false, tiledWakePaintPending: false,
nightscoutSettings: loadNightscoutSettings(), nightscoutSettings: loadNightscoutSettings(),
battery: { battery: {
@@ -80,6 +83,7 @@ const dashboardActions: LayerActions = {
startNightscoutSiteUrlEdit: () => {}, startNightscoutSiteUrlEdit: () => {},
startNightscoutApiTokenEdit: () => {}, startNightscoutApiTokenEdit: () => {},
endTextSettingEdit: () => {}, endTextSettingEdit: () => {},
setVoiceControlEnabled: () => {},
}; };
const dashboardFont = loadEmbeddedTerminus12(); const dashboardFont = loadEmbeddedTerminus12();
const dashboardSystemFont = loadEmbeddedTerminus16(); const dashboardSystemFont = loadEmbeddedTerminus16();
@@ -210,6 +214,14 @@ export function setDashboardSystemCardSetting(key: SystemCardSettingKey, value:
saveSystemCardSetting(key, value); saveSystemCardSetting(key, value);
} }
export function isDashboardVoiceControlEnabled(): boolean {
return dashboardState.voiceControlEnabled;
}
export function setDashboardVoiceControlEnabled(value: boolean): void {
dashboardState.voiceControlEnabled = saveVoiceControlEnabled(value);
}
export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean { export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean {
const timeoutMs = screenTimeoutMs(dashboardState.screenTimeout); const timeoutMs = screenTimeoutMs(dashboardState.screenTimeout);
if (timeoutMs === null || !dashboardState.screenOn) return false; if (timeoutMs === null || !dashboardState.screenOn) return false;
@@ -511,6 +523,12 @@ function createSettingsMenuLayer(): MenuLayer {
ctx.stack.push(createDashboardSettingsMenuLayer()); ctx.stack.push(createDashboardSettingsMenuLayer());
}, },
}, },
{
label: "Voice",
onSelect: (ctx) => {
ctx.stack.push(createVoiceSettingsMenuLayer());
},
},
{ {
label: "Integrations", label: "Integrations",
onSelect: (ctx) => { onSelect: (ctx) => {
@@ -558,6 +576,24 @@ function setWakeMode(value: WakeModeSetting): void {
dashboardState.wakeMode = saveWakeModeSetting(value); dashboardState.wakeMode = saveWakeModeSetting(value);
} }
function createVoiceSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Voice",
[
{
label: "Enable",
onSelect: async (ctx) => {
await ctx.actions.setVoiceControlEnabled(!dashboardState.voiceControlEnabled);
},
render: ({ image, x, y, width, selected }) => {
drawToggleMenuItem(image, dashboardFont, x, y, width, "Enable", dashboardState.voiceControlEnabled, selected);
},
},
],
TOP_LEFT_MENU_LAYOUT,
);
}
function createDashboardSettingsMenuLayer(): MenuLayer { function createDashboardSettingsMenuLayer(): MenuLayer {
return new MenuLayer( return new MenuLayer(
"Dashboard", "Dashboard",
+1
View File
@@ -15,6 +15,7 @@ export type LayerActions = {
startNightscoutSiteUrlEdit: () => Promise<void> | void; startNightscoutSiteUrlEdit: () => Promise<void> | void;
startNightscoutApiTokenEdit: () => Promise<void> | void; startNightscoutApiTokenEdit: () => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void; endTextSettingEdit: () => Promise<void> | void;
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
}; };
export type PaintBelow = () => GrayImage; export type PaintBelow = () => GrayImage;