Add initial voice-command support

Adds a single wakeword-style voice command, "screen on", and deals with
all the wiring and permissions that entails. Currently this only
supports using the phone's mic (or a bluetooth headset), not the G2 mic.
This commit is contained in:
James Babcock committed 2026-04-27 16:45:09 -07:00
1 parent c4cb4f9f01
commit 9fbb985db6
35 files changed
+1722 -16

No files matched your search

+3
View File
@@ -2,3 +2,6 @@ Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficia
and others who contributed to documenting the G2's bluetooth protocol. It has enabled me
to keep my room clean, so to speak.
Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
community.
+7
View File
@@ -8,9 +8,16 @@ android {
buildToolsVersion "35"
// ndkVersion ""
sourceSets.main {
jniLibs.srcDirs += ["$projectDir/src/main/jniLibs"]
}
defaultConfig {
minSdkVersion 24
targetSdkVersion 35
ndk {
abiFilters "arm64-v8a"
}
// Version Information
versionCode 1
@@ -19,8 +19,11 @@
<uses-permission android:name="android.permission.BLUETOOTH_CONNECT"/>
<uses-permission android:name="android.permission.BLUETOOTH_SCAN" android:usesPermissionFlags="neverForLocation"/>
<uses-permission android:name="android.permission.POST_NOTIFICATIONS"/>
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CONNECTED_DEVICE"/>
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE"/>
<application
android:name="com.tns.NativeScriptApplication"
@@ -51,7 +54,7 @@
android:name="com.faceclaw.app.FaceclawForegroundService"
android:enabled="true"
android:exported="false"
android:foregroundServiceType="connectedDevice" />
android:foregroundServiceType="connectedDevice|microphone" />
<service
android:name="com.faceclaw.app.FaceclawMediaNotificationListenerService"
android:enabled="true"
@@ -0,0 +1,16 @@
---
frameworks:
- 其他
license: Apache License 2.0
tasks:
- keyword-spotting
---
sherpa 中自定义唤醒词模型,训练数据为 gigaspeech XL (10000 小时)数据,模型大小约为 3.3 M, 建模单元为 BPE 。 使用 icefall 训练,已转换为 onnx 格式,此仓库主要是给 sherpa-onnx 这个 inference 引擎使用。
模型结构为 zipformer 模型, 本质是一个非常小的语音识别模型,为了实现唤醒词的功能,我们在解码端做了一些修改和约束。 支持自定义唤醒词,数量不限,效果需要单个调整参数。
#### Clone with HTTP
```bash
git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.git
```
@@ -0,0 +1,10 @@
▁HE LL O ▁WORLD
▁HI ▁GO O G LE
▁HE Y ▁S I RI
▁A LE X A
▁LOVE ▁AND ▁P E A CE
▁PLAY ▁MU S IC
▁GO ▁HOME
▁HA PP Y ▁NEW ▁YEAR
▁ME R RY ▁ CH R IST MA S
▁S C RE EN ▁ON :1.5 #0.35
@@ -0,0 +1,9 @@
HELLO WORLD
HI GOOGLE
HEY SIRI
ALEXA
LOVE AND PEACE
PLAY MUSIC
GO HOME
HAPPY NEW YEAR
MERRY CHRISTMAS
@@ -0,0 +1,500 @@
<blk> 0
<sos/eos> 1
<unk> 2
S 3
T 4
▁THE 5
▁A 6
▁TO 7
▁AND 8
N 9
E 10
D 11
ING 12
' 13
▁OF 14
▁I 15
▁IN 16
Y 17
ED 18
M 19
▁ 20
▁THAT 21
O 22
C 23
R 24
A 25
P 26
▁YOU 27
U 28
RE 29
▁IT 30
▁S 31
G 32
L 33
B 34
▁IS 35
I 36
AL 37
ER 38
▁RE 39
▁C 40
▁WE 41
▁SO 42
▁BE 43
OR 44
AR 45
IN 46
▁FOR 47
F 48
▁HE 49
LE 50
LY 51
▁E 52
▁WAS 53
K 54
▁F 55
W 56
VE 57
RI 58
CH 59
LL 60
▁THIS 61
▁T 62
EN 63
▁ON 64
IT 65
CE 66
▁WITH 67
▁B 68
▁W 69
▁ME 70
▁DO 71
▁DE 72
▁HAVE 73
▁BUT 74
ENT 75
▁P 76
UR 77
ON 78
▁AS 79
IC 80
TER 81
TE 82
▁THEY 83
ES 84
RA 85
▁NOT 86
TH 87
▁ARE 88
RO 89
IL 90
ATION 91
AN 92
▁ST 93
▁WHAT 94
SE 95
LI 96
V 97
▁CAN 98
▁G 99
▁OR 100
▁ALL 101
NE 102
▁AT 103
▁CON 104
H 105
▁LIKE 106
▁MO 107
▁HIS 108
▁AN 109
▁MA 110
▁THERE 111
▁SU 112
▁MY 113
NG 114
- 115
▁ABOUT 116
▁SE 117
LA 118
▁ONE 119
LO 120
▁SHE 121
CK 122
▁HAD 123
AD 124
MENT 125
VER 126
▁CA 127
IR 128
▁IF 129
▁NO 130
EL 131
▁O 132
▁FROM 133
▁PA 134
▁YOUR 135
▁KNOW 136
RY 137
ION 138
UN 139
▁JUST 140
▁UP 141
▁US 142
▁WHO 143
▁GO 144
ME 145
▁EX 146
▁OUT 147
ND 148
CI 149
▁PO 150
▁HER 151
▁FA 152
▁PRO 153
▁WHEN 154
ATE 155
AT 156
▁BY 157
PE 158
AS 159
▁LA 160
GE 161
▁CO 162
▁PEOPLE 163
▁K 164
▁SOME 165
IGHT 166
▁THINK 167
VI 168
DE 169
▁LE 170
▁WERE 171
▁LI 172
OM 173
▁WOULD 174
▁SEE 175
CO 176
▁BO 177
▁OUR 178
▁HOW 179
ET 180
Z 181
UL 182
▁HO 183
IG 184
▁GET 185
▁DON 186
IS 187
URE 188
▁WHICH 189
▁MI 190
IES 191
US 192
X 193
▁MORE 194
▁NOW 195
▁TIME 196
▁UN 197
▁FI 198
TY 199
▁WILL 200
RU 201
IVE 202
▁THEIR 203
▁SAY 204
ITY 205
▁HA 206
LU 207
▁RO 208
HO 209
OL 210
IST 211
AGE 212
MA 213
KE 214
▁RA 215
ID 216
ERS 217
▁LO 218
▁SP 219
▁DA 220
▁SAID 221
TION 222
THER 223
▁EN 224
▁BECAUSE 225
▁HAS 226
EST 227
TI 228
ACT 229
▁BA 230
▁VERY 231
▁THEM 232
▁BEEN 233
▁WA 234
ABLE 235
▁SA 236
▁DID 237
▁HIM 238
▁DIS 239
▁WANT 240
▁OTHER 241
IM 242
▁PRE 243
▁LOOK 244
▁GOING 245
▁VI 246
▁WORK 247
▁DR 248
ENCE 249
▁DI 250
FF 251
TA 252
▁REALLY 253
▁RIGHT 254
MO 255
ALLY 256
▁INTO 257
OW 258
▁WAY 259
▁FE 260
▁COULD 261
▁THEN 262
TED 263
▁HERE 264
▁THESE 265
▁OVER 266
▁WELL 267
▁NA 268
▁ANY 269
HE 270
▁COMP 271
▁COM 272
▁TH 273
▁EVEN 274
▁TWO 275
▁JO 276
▁MAKE 277
▁BACK 278
▁WHERE 279
SH 280
▁QU 281
MAN 282
▁YEAR 283
▁TRA 284
▁CHA 285
QU 286
CA 287
▁ALSO 288
CTION 289
ANCE 290
UM 291
PER 292
DI 293
▁HU 294
▁MU 295
FUL 296
▁NEW 297
ARD 298
J 299
ICAL 300
ANT 301
END 302
▁MAR 303
▁NE 304
IF 305
DER 306
ONE 307
▁THOSE 308
PP 309
▁MAN 310
▁GOOD 311
▁TAKE 312
SIDE 313
PO 314
HA 315
▁LITTLE 316
▁APP 317
▁BI 318
UGH 319
▁COME 320
ISH 321
TIC 322
▁PER 323
▁THINGS 324
▁MUCH 325
▁FIRST 326
▁THAN 327
IA 328
▁AGAIN 329
▁STA 330
▁PART 331
▁GA 332
▁MEAN 333
▁LET 334
▁DOWN 335
▁NEED 336
▁GU 337
LES 338
▁RU 339
ATED 340
▁SHA 341
VO 342
▁TALK 343
▁DAY 344
▁CHI 345
▁SOMETHING 346
▁AFTER 347
▁THOUGH 348
▁CALL 349
▁GRA 350
▁EVERY 351
▁KIND 352
ROW 353
AIN 354
▁BU 355
▁THROUGH 356
▁SHOULD 357
▁ONLY 358
▁TA 359
▁MIN 360
▁GOT 361
IAN 362
▁GREAT 363
▁RI 364
▁DOES 365
▁THING 366
DAY 367
OUS 368
▁START 369
▁MOST 370
▁COL 371
PH 372
▁LOT 373
IZE 374
BER 375
IOUS 376
▁LIFE 377
▁MANY 378
▁FEEL 379
ITION 380
▁VA 381
NESS 382
▁BEFORE 383
▁THREE 384
▁WORLD 385
▁GIVE 386
▁WHY 387
▁PI 388
▁LONG 389
PORT 390
▁SHOW 391
▁ACTUALLY 392
▁NEVER 393
IV 394
FORM 395
▁THANK 396
▁HI 397
▁PUT 398
▁LOVE 399
▁QUESTION 400
▁HELP 401
▁HAPPEN 402
▁MR 403
▁OWN 404
WARD 405
▁MADE 406
▁HAND 407
▁STATE 408
IBLE 409
▁TELL 410
▁PH 411
▁CAME 412
▁TEA 413
▁STILL 414
▁DIFFERENT 415
▁PLACE 416
▁MIGHT 417
▁FIND 418
▁DU 419
▁IMP 420
▁OLD 421
▁PERSON 422
▁COUNT 423
▁JA 424
▁POINT 425
▁HIGH 426
SHIP 427
▁LAST 428
▁SAME 429
▁FOUR 430
▁AROUND 431
▁OH 432
▁CHANGE 433
▁UNDER 434
▁HOME 435
▁ANOTHER 436
▁FOUND 437
ATIVE 438
▁TURN 439
▁YES 440
LAND 441
▁ALWAYS 442
▁YEAH 443
▁SCHOOL 444
▁JE 445
▁NEXT 446
▁WORD 447
▁JU 448
LINE 449
▁SIX 450
▁LEARN 451
▁BOOK 452
▁PLAY 453
▁AWAY 454
▁HOUSE 455
▁PLAN 456
▁WHILE 457
▁COMMUNI 458
▁FRIEND 459
▁AMERICA 460
▁INTEREST 461
▁KEEP 462
▁BETTER 463
▁IDEA 464
▁HARD 465
▁TWENTY 466
▁THOUSAND 467
▁HUNDRED 468
▁WEEK 469
▁NUMBER 470
▁COURSE 471
▁OKAY 472
PRESS 473
▁SYSTEM 474
▁NINE 475
▁UNDERSTAND 476
▁WITHOUT 477
▁BELIEVE 478
▁POWER 479
▁MAYBE 480
▁BUILD 481
▁MONEY 482
▁BETWEEN 483
▁GONNA 484
▁IMPORTANT 485
▁ANYTHING 486
▁LEFT 487
▁HAVING 488
▁WHITE 489
▁PROBLEM 490
▁SECOND 491
▁SPEAK 492
▁TOGETHER 493
▁BLACK 494
▁PUBLIC 495
▁HEALTH 496
▁NOTHING 497
▁EVERYTHING 498
Q 499
@@ -27,7 +27,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final int WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE;
private static final int IMAGE_WRITE_TYPE = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT;
private static final int DESIRED_MTU = 247;
private static final int DESIRED_MTU = 512;
private static final int CONNECTION_PRIORITY_HIGH = BluetoothGatt.CONNECTION_PRIORITY_HIGH;
private static final int CONNECT_TIMEOUT_MS = 5_000;
@@ -40,7 +40,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
private static final int BATTERY_REFRESH_INTERVAL_MS = 5 * 60_000;
private static final int BATTERY_INPUT_QUIET_MS = 5_000;
private static final int WINDOW_SIZE = 3;
private static final int IMAGE_FRAGMENT_SIZE = 1000;
private static final int IMAGE_FRAGMENT_SIZE = 3800;
private static final int IMAGE_RETRY_DELAY_MS = 10_000;
private static final boolean IMAGE_FRAGMENT_NO_ACK = false;
private static final int WARMUP_FRAGMENT_TIMEOUT_MS = 3_000;
@@ -6,6 +6,7 @@ import android.app.NotificationManager;
import android.app.PendingIntent;
import android.app.Service;
import android.content.Intent;
import android.content.pm.ServiceInfo;
import android.os.Build;
import android.os.IBinder;
@@ -41,7 +42,11 @@ public class FaceclawForegroundService extends Service {
text != null && !text.trim().isEmpty() ? text : "Keeping the dashboard connected"
);
startForeground(NOTIFICATION_ID, notification);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(NOTIFICATION_ID, notification, foregroundServiceType());
} else {
startForeground(NOTIFICATION_ID, notification);
}
if (ACTION_UPDATE.equals(action)) {
NotificationManager manager = getSystemService(NotificationManager.class);
@@ -95,4 +100,9 @@ public class FaceclawForegroundService extends Service {
.setOnlyAlertOnce(true)
.build();
}
private int foregroundServiceType() {
return ServiceInfo.FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE
| ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE;
}
}
@@ -0,0 +1,341 @@
package com.faceclaw.app;
import android.Manifest;
import android.bluetooth.BluetoothAdapter;
import android.bluetooth.BluetoothDevice;
import android.bluetooth.BluetoothManager;
import android.bluetooth.BluetoothProfile;
import android.content.Context;
import android.content.pm.PackageManager;
import android.content.res.AssetManager;
import android.media.AudioDeviceInfo;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import androidx.core.content.ContextCompat;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotter;
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
public class FaceclawVoiceController {
private static final String TAG = "FaceclawVoice";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
private static final String[] MODEL_FILES = {
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
"tokens.txt",
"screen-on-keywords.txt"
};
private final Context appContext;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private volatile FaceclawVoiceControllerListener listener;
private Thread workerThread;
private volatile boolean started;
private AudioRecord audioRecord;
private KeywordSpotter keywordSpotter;
private OnlineStream stream;
public FaceclawVoiceController(Context context) {
this.appContext = context.getApplicationContext();
}
public void setListener(FaceclawVoiceControllerListener listener) {
this.listener = listener;
}
public void start() {
synchronized (lock) {
if (started) {
emitStatus("Voice control is already listening.");
return;
}
if (!hasRecordAudioPermission()) {
emitStatus("Voice control needs microphone permission.");
return;
}
if (!hasBluetoothHeadset()) {
emitStatus("Voice control needs a paired Bluetooth headset.");
return;
}
started = true;
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
workerThread.start();
}
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
if (!started) {
return;
}
started = false;
threadToJoin = workerThread;
}
stopAudio();
if (threadToJoin != null) {
threadToJoin.interrupt();
}
}
public void close() {
stop();
}
private void runLoop() {
try {
emitStatus("Voice control loading wake-word model...");
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
if (!startAudio()) {
emitStatus("Voice control could not start microphone input.");
return;
}
emitStatus("Voice control listening for \"screen on\".");
processAudio();
} catch (Throwable error) {
Log.e(TAG, "Voice control failed", error);
emitStatus("Voice control failed: " + error.getMessage());
} finally {
releaseSherpa();
stopAudio();
synchronized (lock) {
started = false;
workerThread = null;
}
}
}
private KeywordSpotterConfig buildConfig(File modelDir) {
return KeywordSpotterConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setOnlineModelConfig(OnlineModelConfig.builder()
.setTransducer(OnlineTransducerModelConfig.builder()
.setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setModelType("zipformer2")
.setNumThreads(1)
.build())
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
.setKeywordsScore(1.5f)
.setKeywordsThreshold(0.35f)
.build();
}
private File installModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
for (String fileName : MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
if (destination.exists() && destination.length() > 0) {
return;
}
try (InputStream input = assets.open(assetPath);
FileOutputStream output = new FileOutputStream(destination)) {
byte[] buffer = new byte[64 * 1024];
int read;
while ((read = input.read(buffer)) >= 0) {
output.write(buffer, 0, read);
}
}
}
private boolean startAudio() {
routeToBluetoothHeadset();
int minBufferBytes = AudioRecord.getMinBufferSize(
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT
);
if (minBufferBytes <= 0) {
return false;
}
if (!hasRecordAudioPermission()) {
return false;
}
audioRecord = new AudioRecord(
MediaRecorder.AudioSource.VOICE_RECOGNITION,
SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO,
AudioFormat.ENCODING_PCM_16BIT,
minBufferBytes * 2
);
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
audioRecord.release();
audioRecord = null;
return false;
}
audioRecord.startRecording();
return true;
}
private void processAudio() {
short[] buffer = new short[SAMPLE_RATE / 10];
while (started && !Thread.currentThread().isInterrupted()) {
AudioRecord currentAudio = audioRecord;
OnlineStream currentStream = stream;
KeywordSpotter currentSpotter = keywordSpotter;
if (currentAudio == null || currentStream == null || currentSpotter == null) {
return;
}
int count = currentAudio.read(buffer, 0, buffer.length);
if (count <= 0) {
continue;
}
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = buffer[i] / 32768.0f;
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
while (currentSpotter.isReady(currentStream)) {
currentSpotter.decode(currentStream);
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
String keyword = result == null ? "" : result.getKeyword();
if (keyword != null && keyword.trim().length() > 0) {
currentSpotter.reset(currentStream);
emitWakeWord(keyword);
}
}
}
}
private void stopAudio() {
AudioRecord record = audioRecord;
audioRecord = null;
if (record != null) {
try {
record.stop();
} catch (IllegalStateException ignored) {
}
record.release();
}
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
if (audioManager != null) {
if (Build.VERSION.SDK_INT >= 31) {
audioManager.clearCommunicationDevice();
}
audioManager.stopBluetoothSco();
audioManager.setBluetoothScoOn(false);
}
}
private void releaseSherpa() {
if (stream != null) {
stream.release();
stream = null;
}
if (keywordSpotter != null) {
keywordSpotter.release();
keywordSpotter = null;
}
}
private void routeToBluetoothHeadset() {
AudioManager audioManager = (AudioManager) appContext.getSystemService(Context.AUDIO_SERVICE);
if (audioManager == null) {
return;
}
audioManager.setMode(AudioManager.MODE_IN_COMMUNICATION);
audioManager.startBluetoothSco();
audioManager.setBluetoothScoOn(true);
if (Build.VERSION.SDK_INT >= 31) {
for (AudioDeviceInfo device : audioManager.getAvailableCommunicationDevices()) {
if (device.getType() == AudioDeviceInfo.TYPE_BLUETOOTH_SCO
|| device.getType() == AudioDeviceInfo.TYPE_BLE_HEADSET) {
audioManager.setCommunicationDevice(device);
return;
}
}
}
}
private boolean hasBluetoothHeadset() {
if (Build.VERSION.SDK_INT >= 31
&& ContextCompat.checkSelfPermission(appContext, Manifest.permission.BLUETOOTH_CONNECT)
!= PackageManager.PERMISSION_GRANTED) {
return false;
}
BluetoothManager manager = (BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
BluetoothAdapter adapter = manager == null ? null : manager.getAdapter();
if (adapter == null) {
return false;
}
try {
int state = adapter.getProfileConnectionState(BluetoothProfile.HEADSET);
if (state == BluetoothProfile.STATE_CONNECTED || state == BluetoothProfile.STATE_CONNECTING) {
return true;
}
Set<BluetoothDevice> bondedDevices = adapter.getBondedDevices();
return bondedDevices != null && !bondedDevices.isEmpty();
} catch (SecurityException ignored) {
return false;
}
}
private boolean hasRecordAudioPermission() {
return ContextCompat.checkSelfPermission(appContext, Manifest.permission.RECORD_AUDIO)
== PackageManager.PERMISSION_GRANTED;
}
private void emitStatus(String status) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onStatus(status));
}
private void emitWakeWord(String keyword) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onWakeWord(keyword));
}
}
@@ -0,0 +1,7 @@
package com.faceclaw.app;
public interface FaceclawVoiceControllerListener {
void onStatus(String status);
void onWakeWord(String keyword);
}
@@ -0,0 +1,54 @@
package com.k2fsa.sherpa.onnx;
public class FeatureConfig {
private final int sampleRate;
private final int featureDim;
private final float dither;
private FeatureConfig(Builder builder) {
this.sampleRate = builder.sampleRate;
this.featureDim = builder.featureDim;
this.dither = builder.dither;
}
public static Builder builder() {
return new Builder();
}
public int getSampleRate() {
return sampleRate;
}
public int getFeatureDim() {
return featureDim;
}
public float getDither() {
return dither;
}
public static class Builder {
private int sampleRate = 16000;
private int featureDim = 80;
private float dither = 0.0f;
public Builder setSampleRate(int sampleRate) {
this.sampleRate = sampleRate;
return this;
}
public Builder setFeatureDim(int featureDim) {
this.featureDim = featureDim;
return this;
}
public Builder setDither(float dither) {
this.dither = dither;
return this;
}
public FeatureConfig build() {
return new FeatureConfig(this);
}
}
}
@@ -0,0 +1,66 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotter {
private long ptr;
public KeywordSpotter(KeywordSpotterConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
}
}
public OnlineStream createStream() {
return createStream("");
}
public OnlineStream createStream(String keywords) {
return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
}
public boolean isReady(OnlineStream stream) {
return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
}
public void decode(OnlineStream stream) {
decode(ptr, stream.getPtr());
}
public void reset(OnlineStream stream) {
reset(ptr, stream.getPtr());
}
public KeywordSpotterResult getResult(OnlineStream stream) {
return getResult(ptr, stream.getPtr());
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native long newFromFile(KeywordSpotterConfig config);
private native void delete(long ptr);
private native long createStream(long ptr, String keywords);
private native boolean isReady(long ptr, long streamPtr);
private native void decode(long ptr, long streamPtr);
private native void reset(long ptr, long streamPtr);
private native KeywordSpotterResult getResult(long ptr, long streamPtr);
}
@@ -0,0 +1,92 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterConfig {
private final FeatureConfig featConfig;
private final OnlineModelConfig modelConfig;
private final int maxActivePaths;
private final String keywordsFile;
private final float keywordsScore;
private final float keywordsThreshold;
private final int numTrailingBlanks;
private KeywordSpotterConfig(Builder builder) {
this.featConfig = builder.featConfig;
this.modelConfig = builder.modelConfig;
this.maxActivePaths = builder.maxActivePaths;
this.keywordsFile = builder.keywordsFile;
this.keywordsScore = builder.keywordsScore;
this.keywordsThreshold = builder.keywordsThreshold;
this.numTrailingBlanks = builder.numTrailingBlanks;
}
public static Builder builder() {
return new Builder();
}
public FeatureConfig getFeatConfig() {
return featConfig;
}
public OnlineModelConfig getModelConfig() {
return modelConfig;
}
public int getMaxActivePaths() {
return maxActivePaths;
}
public String getKeywordsFile() {
return keywordsFile;
}
public float getKeywordsScore() {
return keywordsScore;
}
public float getKeywordsThreshold() {
return keywordsThreshold;
}
public int getNumTrailingBlanks() {
return numTrailingBlanks;
}
public static class Builder {
private FeatureConfig featConfig = FeatureConfig.builder().build();
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
private int maxActivePaths = 4;
private String keywordsFile = "keywords.txt";
private float keywordsScore = 1.5f;
private float keywordsThreshold = 0.25f;
private int numTrailingBlanks = 2;
public Builder setFeatureConfig(FeatureConfig featConfig) {
this.featConfig = featConfig;
return this;
}
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
this.modelConfig = modelConfig;
return this;
}
public Builder setKeywordsFile(String keywordsFile) {
this.keywordsFile = keywordsFile;
return this;
}
public Builder setKeywordsScore(float keywordsScore) {
this.keywordsScore = keywordsScore;
return this;
}
public Builder setKeywordsThreshold(float keywordsThreshold) {
this.keywordsThreshold = keywordsThreshold;
return this;
}
public KeywordSpotterConfig build() {
return new KeywordSpotterConfig(this);
}
}
}
@@ -0,0 +1,25 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterResult {
private final String keyword;
private final String[] tokens;
private final float[] timestamps;
public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
this.keyword = keyword == null ? "" : keyword;
this.tokens = tokens == null ? new String[0] : tokens;
this.timestamps = timestamps == null ? new float[0] : timestamps;
}
public String getKeyword() {
return keyword;
}
public String[] getTokens() {
return tokens;
}
public float[] getTimestamps() {
return timestamps;
}
}
@@ -0,0 +1,137 @@
package com.k2fsa.sherpa.onnx;
public class OnlineModelConfig {
private final OnlineTransducerModelConfig transducer;
private final OnlineParaformerModelConfig paraformer;
private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
private final OnlineNeMoCtcModelConfig neMoCtc;
private final OnlineToneCtcModelConfig toneCtc;
private final String tokens;
private final int numThreads;
private final boolean debug;
private final String provider;
private final String modelType;
private final String modelingUnit;
private final String bpeVocab;
private OnlineModelConfig(Builder builder) {
this.transducer = builder.transducer;
this.paraformer = builder.paraformer;
this.zipformer2Ctc = builder.zipformer2Ctc;
this.neMoCtc = builder.neMoCtc;
this.toneCtc = builder.toneCtc;
this.tokens = builder.tokens;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
this.modelType = builder.modelType;
this.modelingUnit = builder.modelingUnit;
this.bpeVocab = builder.bpeVocab;
}
public static Builder builder() {
return new Builder();
}
public OnlineTransducerModelConfig getTransducer() {
return transducer;
}
public OnlineParaformerModelConfig getParaformer() {
return paraformer;
}
public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
return zipformer2Ctc;
}
public OnlineNeMoCtcModelConfig getNeMoCtc() {
return neMoCtc;
}
public OnlineToneCtcModelConfig getToneCtc() {
return toneCtc;
}
public String getTokens() {
return tokens;
}
public int getNumThreads() {
return numThreads;
}
public boolean getDebug() {
return debug;
}
public String getProvider() {
return provider;
}
public String getModelType() {
return modelType;
}
public String getModelingUnit() {
return modelingUnit;
}
public String getBpeVocab() {
return bpeVocab;
}
public static class Builder {
private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
private String tokens = "";
private int numThreads = 1;
private boolean debug = false;
private String provider = "cpu";
private String modelType = "";
private String modelingUnit = "cjkchar";
private String bpeVocab = "";
public Builder setTransducer(OnlineTransducerModelConfig transducer) {
this.transducer = transducer;
return this;
}
public Builder setTokens(String tokens) {
this.tokens = tokens;
return this;
}
public Builder setModelType(String modelType) {
this.modelType = modelType;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
public Builder setModelingUnit(String modelingUnit) {
this.modelingUnit = modelingUnit;
return this;
}
public Builder setBpeVocab(String bpeVocab) {
this.bpeVocab = bpeVocab;
return this;
}
public OnlineModelConfig build() {
return new OnlineModelConfig(this);
}
}
}
@@ -0,0 +1,30 @@
package com.k2fsa.sherpa.onnx;
public class OnlineNeMoCtcModelConfig {
private final String model;
private OnlineNeMoCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineNeMoCtcModelConfig build() {
return new OnlineNeMoCtcModelConfig(this);
}
}
}
@@ -0,0 +1,42 @@
package com.k2fsa.sherpa.onnx;
public class OnlineParaformerModelConfig {
private final String encoder;
private final String decoder;
private OnlineParaformerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public OnlineParaformerModelConfig build() {
return new OnlineParaformerModelConfig(this);
}
}
}
@@ -0,0 +1,35 @@
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr;
public OnlineStream(long ptr) {
this.ptr = ptr;
}
public long getPtr() {
return ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(ptr, samples, sampleRate);
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void delete(long ptr);
}
@@ -0,0 +1,30 @@
package com.k2fsa.sherpa.onnx;
public class OnlineToneCtcModelConfig {
private final String model;
private OnlineToneCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineToneCtcModelConfig build() {
return new OnlineToneCtcModelConfig(this);
}
}
}
@@ -0,0 +1,54 @@
package com.k2fsa.sherpa.onnx;
public class OnlineTransducerModelConfig {
private final String encoder;
private final String decoder;
private final String joiner;
private OnlineTransducerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
this.joiner = builder.joiner;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public String getJoiner() {
return joiner;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
private String joiner = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public Builder setJoiner(String joiner) {
this.joiner = joiner;
return this;
}
public OnlineTransducerModelConfig build() {
return new OnlineTransducerModelConfig(this);
}
}
}
@@ -0,0 +1,30 @@
package com.k2fsa.sherpa.onnx;
public class OnlineZipformer2CtcModelConfig {
private final String model;
private OnlineZipformer2CtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineZipformer2CtcModelConfig build() {
return new OnlineZipformer2CtcModelConfig(this);
}
}
}
+25 -11
View File
@@ -1,7 +1,9 @@
import { Application, Utils } from "@nativescript/core";
const PERMISSION_REQUEST_CODE = 4242;
const VOICE_PERMISSION_REQUEST_CODE = 4243;
const POST_NOTIFICATIONS_PERMISSION = "android.permission.POST_NOTIFICATIONS";
const RECORD_AUDIO_PERMISSION = "android.permission.RECORD_AUDIO";
function getActivity(): androidx.appcompat.app.AppCompatActivity {
const activity = Application.android.foregroundActivity ?? Application.android.startActivity;
@@ -43,17 +45,21 @@ function getRequiredPermissions(): string[] {
return permissions;
}
export async function ensureBlePermissions(): Promise<void> {
function isPermissionGranted(permission: string): boolean {
const context = getContext();
return androidx.core.content.ContextCompat.checkSelfPermission(context, permission) ===
android.content.pm.PackageManager.PERMISSION_GRANTED;
}
async function ensurePermissions(
permissions: string[],
requestCode: number,
denialLabel: string,
): Promise<void> {
if (!global.isAndroid) return;
const context = getContext();
const activity = getActivity();
const required = getRequiredPermissions();
const missing = required.filter(
(permission) =>
androidx.core.content.ContextCompat.checkSelfPermission(context, permission) !==
android.content.pm.PackageManager.PERMISSION_GRANTED,
);
const missing = permissions.filter((permission) => !isPermissionGranted(permission));
if (missing.length === 0) return;
@@ -63,7 +69,7 @@ export async function ensureBlePermissions(): Promise<void> {
permissions: string[];
grantResults: number[];
}) => {
if (args.requestCode !== PERMISSION_REQUEST_CODE) return;
if (args.requestCode !== requestCode) return;
Application.android.off(Application.android.activityRequestPermissionsEvent, callback);
const denied = missing.filter(
@@ -72,7 +78,7 @@ export async function ensureBlePermissions(): Promise<void> {
);
if (denied.length > 0) {
reject(new Error(`Bluetooth permissions denied: ${denied.join(", ")}`));
reject(new Error(`${denialLabel} permissions denied: ${denied.join(", ")}`));
return;
}
resolve();
@@ -82,7 +88,15 @@ export async function ensureBlePermissions(): Promise<void> {
androidx.core.app.ActivityCompat.requestPermissions(
activity,
toJavaStringArray(missing),
PERMISSION_REQUEST_CODE,
requestCode,
);
});
}
export async function ensureBlePermissions(): Promise<void> {
await ensurePermissions(getRequiredPermissions(), PERMISSION_REQUEST_CODE, "Bluetooth");
}
export async function ensureVoicePermissions(): Promise<void> {
await ensurePermissions([RECORD_AUDIO_PERMISSION], VOICE_PERMISSION_REQUEST_CODE, "Voice control");
}
+73 -1
View File
@@ -1,19 +1,21 @@
import { ImageSource } from "@nativescript/core";
import { EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events";
import { loadDeviceAddresses } from "./device-addresses";
import { ensureBlePermissions } from "./android-permissions";
import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions";
import { FaceclawCommunicatorBridge, type FrameMetrics, type RawInputEvent } from "../native/faceclaw-communicator";
import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service";
import { mediaControllerBridge } from "../native/media-controller";
import { nightscoutBridge } from "../native/nightscout";
import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict";
import { grayImageToPreviewSource } from "../native/gray-image-preview";
import { voiceControlBridge } from "../native/voice-control";
import {
applyDashboardScreenTimeout,
consumeDashboardTiledWakePaint,
drawDashboard,
getDashboardNightscoutSettings,
getDashboardSystemCardName,
isDashboardVoiceControlEnabled,
receiveInput,
resetDashboardSleepTimerAndWake,
setDashboardBatteryLevels,
@@ -21,6 +23,7 @@ import {
setDashboardNightscoutApiToken,
setDashboardNightscoutSiteUrl,
setDashboardSystemCardName,
setDashboardVoiceControlEnabled,
} from "../ui/dashboard";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -118,6 +121,8 @@ class DashboardController {
private offFrameMetrics: (() => void) | null = null;
private offMedia: (() => void) | null = null;
private offNightscout: (() => void) | null = null;
private offVoiceStatus: (() => void) | null = null;
private offVoiceWakeWord: (() => void) | null = null;
private lastInput = "waiting...";
private lastSys = "none yet";
private renderInProgress = false;
@@ -132,6 +137,7 @@ class DashboardController {
startNightscoutSiteUrlEdit: () => this.startNightscoutSiteUrlEdit(),
startNightscoutApiTokenEdit: () => this.startNightscoutApiTokenEdit(),
endTextSettingEdit: () => this.endTextSettingEdit(),
setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled),
});
}
@@ -322,11 +328,20 @@ class DashboardController {
});
}
});
this.offVoiceStatus = voiceControlBridge.onStatus((state) => {
this.appendLog(state.status);
});
this.offVoiceWakeWord = voiceControlBridge.onWakeWord((keyword) => {
void this.handleWakeWord(keyword).catch((error) => {
this.appendLog(`wake-word handler failed: ${this.formatError(error)}`);
});
});
await mediaControllerBridge.start();
await nightscoutBridge.start();
await communicator.start();
await this.requestRender("initial");
this.startVoiceControlIfEnabled();
this.dashboardTimer = setInterval(() => {
void this.requestRender("interval").catch((error) => {
const message = this.formatError(error);
@@ -360,8 +375,13 @@ class DashboardController {
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
if (communicator) {
await communicator.close().catch(() => {});
}
@@ -397,6 +417,10 @@ class DashboardController {
this.offMedia = null;
this.offNightscout?.();
this.offNightscout = null;
this.offVoiceStatus?.();
this.offVoiceStatus = null;
this.offVoiceWakeWord?.();
this.offVoiceWakeWord = null;
const communicator = this.communicator;
this.communicator = null;
@@ -413,6 +437,7 @@ class DashboardController {
}
await mediaControllerBridge.stop().catch(() => {});
await nightscoutBridge.stop().catch(() => {});
voiceControlBridge.stop();
await communicator?.close().catch(() => {});
} finally {
stopForegroundNotification();
@@ -445,6 +470,30 @@ class DashboardController {
this.emit();
}
private async setVoiceControlEnabled(enabled: boolean): Promise<void> {
if (enabled) {
try {
await ensureVoicePermissions();
} catch (error) {
this.appendLog(`voice control permission failed: ${this.formatError(error)}`);
return;
}
}
setDashboardVoiceControlEnabled(enabled);
this.appendLog(`Voice control ${enabled ? "enabled" : "disabled"}.`);
if (enabled) {
this.startVoiceControlIfEnabled();
} else {
voiceControlBridge.stop();
}
if (this.phase === "connected" && this.communicator) {
await this.requestRender("interval").catch((error) => {
this.appendLog(`voice setting render failed: ${this.formatError(error)}`);
});
}
}
private endTextSettingEdit(): void {
const finishedKind = this.activeTextSettingEditorKind;
this.editingSystemCardName = false;
@@ -580,6 +629,29 @@ class DashboardController {
}
}
private async handleWakeWord(keyword: string): Promise<void> {
const normalized = keyword.trim();
if (normalized.length === 0) return;
this.appendLog(`wake-word detected: ${normalized}`);
const changed = resetDashboardSleepTimerAndWake();
if (this.phase === "connected" && this.communicator) {
if (changed) {
await this.requestRender("initial");
}
return;
}
if (changed) {
const image = drawDashboard();
this.setDisplayPreview(grayImageToPreviewSource(image));
}
}
private startVoiceControlIfEnabled(): void {
if (!this.communicator) return;
if (!isDashboardVoiceControlEnabled()) return;
voiceControlBridge.start();
}
private clearDashboardTimer(): void {
if (this.dashboardTimer) {
clearInterval(this.dashboardTimer);
+70
View File
@@ -0,0 +1,70 @@
import { Utils } from "@nativescript/core";
declare const com: any;
export type VoiceControlState = {
status: string;
};
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly wakeWordListeners = new Set<(keyword: string) => void>();
private controller: any | null = null;
private listenerProxy: any | null = null;
private status = "Voice control stopped.";
private started = false;
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
listener({ status: this.status });
return () => this.statusListeners.delete(listener);
}
onWakeWord(listener: (keyword: string) => void): () => void {
this.wakeWordListeners.add(listener);
return () => this.wakeWordListeners.delete(listener);
}
start(): void {
if (this.started || !global.isAndroid) return;
this.ensureController();
this.started = true;
this.controller?.start();
}
stop(): void {
if (!this.started || !global.isAndroid) return;
this.started = false;
this.controller?.stop();
this.setStatus("Voice control stopped.");
}
private ensureController(): void {
if (!global.isAndroid || this.controller) return;
const context = Utils.android.getApplicationContext();
if (!context) {
throw new Error("Android application context unavailable");
}
this.controller = new com.faceclaw.app.FaceclawVoiceController(context);
this.listenerProxy = new com.faceclaw.app.FaceclawVoiceControllerListener({
onStatus: (status: string) => {
this.setStatus(String(status));
},
onWakeWord: (keyword: string) => {
for (const listener of this.wakeWordListeners) {
listener(String(keyword));
}
},
});
this.controller.setListener(this.listenerProxy);
}
private setStatus(status: string): void {
this.status = status;
for (const listener of this.statusListeners) {
listener({ status });
}
}
}
export const voiceControlBridge = new FaceclawVoiceControlBridge();
+11
View File
@@ -21,9 +21,11 @@ export type WakeModeSetting = "tiled" | "synchronized";
export const DEFAULT_SYSTEM_CARD_NAME = "Faceclaw";
export const DEFAULT_SCREEN_TIMEOUT: ScreenTimeoutSetting = "30s";
export const DEFAULT_WAKE_MODE: WakeModeSetting = "tiled";
export const DEFAULT_VOICE_CONTROL_ENABLED = false;
const SYSTEM_CARD_NAME_KEY = "dashboard.systemCardName";
const SCREEN_TIMEOUT_KEY = "display.screenTimeout";
const WAKE_MODE_KEY = "display.wakeMode";
const VOICE_CONTROL_ENABLED_KEY = "voice.enabled";
const SYSTEM_CARD_SETTING_KEYS: Record<SystemCardSettingKey, string> = {
showFaceclawLogo: "dashboard.systemCard.showFaceclawLogo",
showBatteryIndicators: "dashboard.systemCard.showBatteryIndicators",
@@ -138,6 +140,15 @@ export function saveWakeModeSetting(value: WakeModeSetting): WakeModeSetting {
return normalized;
}
export function loadVoiceControlEnabled(): boolean {
return ApplicationSettings.getBoolean(VOICE_CONTROL_ENABLED_KEY, DEFAULT_VOICE_CONTROL_ENABLED);
}
export function saveVoiceControlEnabled(value: boolean): boolean {
ApplicationSettings.setBoolean(VOICE_CONTROL_ENABLED_KEY, value);
return value;
}
export function nextScreenTimeoutSetting(value: ScreenTimeoutSetting): ScreenTimeoutSetting {
switch (value) {
case "15s":
+36
View File
@@ -25,6 +25,7 @@ import {
loadScreenTimeoutSetting,
loadSystemCardSettings,
loadSystemCardName,
loadVoiceControlEnabled,
loadWakeModeSetting,
nextScreenTimeoutSetting,
nextWakeModeSetting,
@@ -37,6 +38,7 @@ import {
screenTimeoutMs,
saveSystemCardSetting,
saveSystemCardName,
saveVoiceControlEnabled,
wakeModeLabel,
type DashboardSlotId,
type NightscoutSettings,
@@ -63,6 +65,7 @@ let dashboardState = {
systemCardSettings: loadSystemCardSettings(),
screenTimeout: loadScreenTimeoutSetting(),
wakeMode: loadWakeModeSetting(),
voiceControlEnabled: loadVoiceControlEnabled(),
tiledWakePaintPending: false,
nightscoutSettings: loadNightscoutSettings(),
battery: {
@@ -80,6 +83,7 @@ const dashboardActions: LayerActions = {
startNightscoutSiteUrlEdit: () => {},
startNightscoutApiTokenEdit: () => {},
endTextSettingEdit: () => {},
setVoiceControlEnabled: () => {},
};
const dashboardFont = loadEmbeddedTerminus12();
const dashboardSystemFont = loadEmbeddedTerminus16();
@@ -210,6 +214,14 @@ export function setDashboardSystemCardSetting(key: SystemCardSettingKey, value:
saveSystemCardSetting(key, value);
}
export function isDashboardVoiceControlEnabled(): boolean {
return dashboardState.voiceControlEnabled;
}
export function setDashboardVoiceControlEnabled(value: boolean): void {
dashboardState.voiceControlEnabled = saveVoiceControlEnabled(value);
}
export function applyDashboardScreenTimeout(nowMs = Date.now()): boolean {
const timeoutMs = screenTimeoutMs(dashboardState.screenTimeout);
if (timeoutMs === null || !dashboardState.screenOn) return false;
@@ -511,6 +523,12 @@ function createSettingsMenuLayer(): MenuLayer {
ctx.stack.push(createDashboardSettingsMenuLayer());
},
},
{
label: "Voice",
onSelect: (ctx) => {
ctx.stack.push(createVoiceSettingsMenuLayer());
},
},
{
label: "Integrations",
onSelect: (ctx) => {
@@ -558,6 +576,24 @@ function setWakeMode(value: WakeModeSetting): void {
dashboardState.wakeMode = saveWakeModeSetting(value);
}
function createVoiceSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Voice",
[
{
label: "Enable",
onSelect: async (ctx) => {
await ctx.actions.setVoiceControlEnabled(!dashboardState.voiceControlEnabled);
},
render: ({ image, x, y, width, selected }) => {
drawToggleMenuItem(image, dashboardFont, x, y, width, "Enable", dashboardState.voiceControlEnabled, selected);
},
},
],
TOP_LEFT_MENU_LAYOUT,
);
}
function createDashboardSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Dashboard",
+1
View File
@@ -15,6 +15,7 @@ export type LayerActions = {
startNightscoutSiteUrlEdit: () => Promise<void> | void;
startNightscoutApiTokenEdit: () => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void;
setVoiceControlEnabled: (enabled: boolean) => Promise<void> | void;
};
export type PaintBelow = () => GrayImage;