This commit is contained in:
James Babcock committed 2026-08-30 00:42:27 -07:00
commit d542acb891
315 files changed
+35117 -2087

No files matched your search

+3 -1
View File
@@ -1,4 +1,6 @@
--ignore-file=match:.mjs
--ignore-file=match:frame-timings*
--ignore-dir=tempPlugin
--ignore-dir=build
--ignore-dir=platforms/android/build-tools
--ignore-dir=old
--ignore-dir=platforms
+40
View File
@@ -0,0 +1,40 @@
name: Deploy website
# Publishes website/ to GitHub Pages. Requires the repository's
# Settings > Pages > Source to be set to "GitHub Actions".
on:
push:
branches: [main]
paths:
- 'website/**'
- 'PRIVACY'
- 'scripts/sync-site.mjs'
- '.github/workflows/pages.yml'
workflow_dispatch:
permissions:
contents: read
pages: write
id-token: write
concurrency:
group: pages
cancel-in-progress: false
jobs:
deploy:
runs-on: ubuntu-latest
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
steps:
- uses: actions/checkout@v4
- name: Check generated blocks are up to date
run: node scripts/sync-site.mjs --check
- uses: actions/configure-pages@v5
- uses: actions/upload-pages-artifact@v3
with:
path: website
- id: deployment
uses: actions/deploy-pages@v4
+11
View File
@@ -3,6 +3,9 @@ hooks/
node_modules/
platforms/
# Release APKs from scripts/release.sh
dist/
# Logs
logs
*.log
@@ -10,6 +13,9 @@ npm-debug.log*
yarn-debug.log*
yarn-error.log*
# npm test compile output (tests/tsconfig.json outDir)
.test-build/
# General
.DS_Store
.AppleDouble
@@ -22,6 +28,7 @@ typings/
.gradle-cursor
App_Resources/Android/src/main/jniLibs/**/*.so
App_Resources/Android/src/main/assets/faceclaw-voice-asr/
old
# Visual Studio Code
.vscode/*
@@ -32,3 +39,7 @@ App_Resources/Android/src/main/assets/faceclaw-voice-asr/
# pull_config.sh output: the phone's live settings, including API keys
faceclaw_settings.xml
# Developer configs
build_paths.sh
+14 -9
View File
@@ -1,11 +1,16 @@
Acknowledgements
================
Faceclaw is an open source project, at https://github.com/jimrandomh/faceclaw.
Contributors include:
James Babcock <jimrandomh@gmail.com>
Kalani Helekunihi <i@am.guru>
Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficial/,
evenRealities-openCFW https://github.com/kalanihelekunihi/evenRealities-openCFW/,
and others who contributed to documenting the G2's bluetooth protocol and firmware.
Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
community.
G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3
The Terminus font is by Dimitar Zhekov's Terminus: https://framagit.org/ohnonot/terv-terc
@@ -15,8 +20,8 @@ The TerminusV proportional font is by ohnonot, derived from Terminus:
https://framagit.org/ohnonot/terv-terc. It is also distributed under the SIL
Open Font License; see app/fonts/terminusv/LICENSE.
The CJK font used for EvenHub compatibility is Source Han Sans SC Light by
Adobe: https://github.com/adobe-fonts/source-han-sans. It is distributed under
the SIL Open Font License, Version 1.1; see app/fonts/source-han-sans/LICENSE.txt.
Faceclaw includes the G2's serialized 20 px LVGL build of the font so phone-side
rendering matches the glasses.
The CJK font is Source Han Sans SC Light by Adobe:
https://github.com/adobe-fonts/source-han-sans. It is distributed under the SIL
Open Font License, Version 1.1; see app/fonts/source-han-sans/LICENSE.txt.
Faceclaw includes the G2's serialized 20 px LVGL build of the font so
phone-side rendering matches the glasses.
+32 -24
View File
@@ -12,14 +12,19 @@ dependencies {
// code" entry. Scanner UI and camera live in Play Services, so Faceclaw
// needs no camera permission and no third-party scanner app.
implementation 'com.google.android.gms:play-services-code-scanner:16.1.0'
// Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for
// the Wear OS watch remote; see FaceclawWearBridge and wear/.
implementation 'com.google.android.gms:play-services-wearable:19.0.0'
// On-device language identification and translation for the Microphones
// app's captions (translation models download on demand, per language).
implementation 'com.google.mlkit:language-id:17.0.6'
implementation 'com.google.mlkit:translate:17.0.3'
}
def faceclawSherpaOnnxVersion = "1.13.0"
def faceclawLibLc3Version = "1.1.3"
def faceclawAsrModelName = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"
def faceclawAbi = "arm64-v8a"
def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi")
def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr")
def faceclawNativeBuildDir = file("$buildDir/faceclaw-native")
def faceclawDownload = { String url, File destination ->
@@ -67,15 +72,11 @@ tasks.register("prepareFaceclawNativeLibs") {
outputs.files(
file("$faceclawJniLibsDir/libonnxruntime.so"),
file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"),
file("$faceclawJniLibsDir/libfaceclaw_lc3.so"),
file("$faceclawAssetsDir/$faceclawAsrModelName/encoder_model.ort"),
file("$faceclawAssetsDir/$faceclawAsrModelName/decoder_model_merged.ort"),
file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt")
file("$faceclawJniLibsDir/libfaceclaw_lc3.so")
)
doLast {
faceclawJniLibsDir.mkdirs()
faceclawAssetsDir.mkdirs()
def downloadsDir = file("$faceclawNativeBuildDir/downloads")
def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2")
@@ -92,20 +93,6 @@ tasks.register("prepareFaceclawNativeLibs") {
into faceclawJniLibsDir
}
def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2")
faceclawDownload(
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2",
asrArchive
)
copy {
from tarTree(resources.bzip2(asrArchive))
include "$faceclawAsrModelName/encoder_model.ort"
include "$faceclawAsrModelName/decoder_model_merged.ort"
include "$faceclawAsrModelName/tokens.txt"
includeEmptyDirs = false
into faceclawAssetsDir
}
def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz")
faceclawDownload(
"https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz",
@@ -252,6 +239,22 @@ tasks.register("prepareFaceclawLlama") {
}
}
// versionName tracks FACECLAW_VERSION in app/version.ts, the single source of
// truth for the Faceclaw version. The gradle root project is platforms/android,
// so the repo root is two levels up from rootDir.
def faceclawVersionName = "0.0"
def faceclawVersionFile = file("$rootDir/../../app/version.ts")
if (faceclawVersionFile.exists()) {
def faceclawVersionMatch = (faceclawVersionFile.text =~ /FACECLAW_VERSION = "([^"]+)"/)
if (faceclawVersionMatch.find()) {
faceclawVersionName = faceclawVersionMatch.group(1)
}
}
def faceclawVersionCode = {
def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger()
return parts[0] * 10000 + parts[1] * 100 + parts[2]
}()
android {
compileSdkVersion 35
buildToolsVersion "35"
@@ -265,12 +268,17 @@ android {
minSdkVersion 24
targetSdkVersion 35
ndk {
// The NativeScript platform build.gradle has already added all four
// ABIs by the time app.gradle is applied, and abiFilters is additive,
// so the set must be cleared for this filter to take effect. Only
// arm64 devices are supported (all our own JNI libs are arm64-only).
abiFilters.clear()
abiFilters "arm64-v8a"
}
// Version Information
versionCode 1
versionName "1.0.0"
// Version Information (derived from app/version.ts above)
versionCode faceclawVersionCode
versionName faceclawVersionName
generatedDensities = []
}
@@ -77,6 +77,7 @@
android:name="com.tns.NativeScriptActivity"
android:label="@string/title_activity_kimera"
android:configChanges="keyboard|keyboardHidden|orientation|screenSize|smallestScreenSize|screenLayout|locale|uiMode"
android:windowSoftInputMode="adjustResize"
android:theme="@style/LaunchScreenTheme"
android:hardwareAccelerated="true"
android:launchMode="singleTask"
@@ -110,6 +111,22 @@
<action android:name="android.service.notification.NotificationListenerService" />
</intent-filter>
</service>
<!-- Wear OS: Play services routes watch-app messages (paths under
/faceclaw) and watch reachability changes here, starting the app
process if needed. See FaceclawWearBridge. -->
<service
android:name="com.faceclaw.app.FaceclawWearListenerService"
android:enabled="true"
android:exported="true">
<intent-filter>
<action android:name="com.google.android.gms.wearable.MESSAGE_RECEIVED" />
<data android:scheme="wear" android:host="*" android:pathPrefix="/faceclaw" />
</intent-filter>
<intent-filter>
<action android:name="com.google.android.gms.wearable.CAPABILITY_CHANGED" />
<data android:scheme="wear" android:host="*" android:pathPrefix="/" />
</intent-filter>
</service>
<receiver
android:name="com.faceclaw.app.FaceclawTimerReceiver"
android:enabled="true"
@@ -1,16 +0,0 @@
---
frameworks:
- 其他
license: Apache License 2.0
tasks:
- keyword-spotting
---
sherpa 中自定义唤醒词模型,训练数据为 gigaspeech XL (10000 小时)数据,模型大小约为 3.3 M, 建模单元为 BPE 。 使用 icefall 训练,已转换为 onnx 格式,此仓库主要是给 sherpa-onnx 这个 inference 引擎使用。
模型结构为 zipformer 模型, 本质是一个非常小的语音识别模型,为了实现唤醒词的功能,我们在解码端做了一些修改和约束。 支持自定义唤醒词,数量不限,效果需要单个调整参数。
#### Clone with HTTP
```bash
git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.git
```
@@ -1,10 +0,0 @@
▁HE LL O ▁WORLD
▁HI ▁GO O G LE
▁HE Y ▁S I RI
▁A LE X A
▁LOVE ▁AND ▁P E A CE
▁PLAY ▁MU S IC
▁GO ▁HOME
▁HA PP Y ▁NEW ▁YEAR
▁ME R RY ▁ CH R IST MA S
▁S C RE EN ▁ON :1.5 #0.35
@@ -1,9 +0,0 @@
HELLO WORLD
HI GOOGLE
HEY SIRI
ALEXA
LOVE AND PEACE
PLAY MUSIC
GO HOME
HAPPY NEW YEAR
MERRY CHRISTMAS
@@ -1,500 +0,0 @@
<blk> 0
<sos/eos> 1
<unk> 2
S 3
T 4
▁THE 5
▁A 6
▁TO 7
▁AND 8
N 9
E 10
D 11
ING 12
' 13
▁OF 14
▁I 15
▁IN 16
Y 17
ED 18
M 19
▁ 20
▁THAT 21
O 22
C 23
R 24
A 25
P 26
▁YOU 27
U 28
RE 29
▁IT 30
▁S 31
G 32
L 33
B 34
▁IS 35
I 36
AL 37
ER 38
▁RE 39
▁C 40
▁WE 41
▁SO 42
▁BE 43
OR 44
AR 45
IN 46
▁FOR 47
F 48
▁HE 49
LE 50
LY 51
▁E 52
▁WAS 53
K 54
▁F 55
W 56
VE 57
RI 58
CH 59
LL 60
▁THIS 61
▁T 62
EN 63
▁ON 64
IT 65
CE 66
▁WITH 67
▁B 68
▁W 69
▁ME 70
▁DO 71
▁DE 72
▁HAVE 73
▁BUT 74
ENT 75
▁P 76
UR 77
ON 78
▁AS 79
IC 80
TER 81
TE 82
▁THEY 83
ES 84
RA 85
▁NOT 86
TH 87
▁ARE 88
RO 89
IL 90
ATION 91
AN 92
▁ST 93
▁WHAT 94
SE 95
LI 96
V 97
▁CAN 98
▁G 99
▁OR 100
▁ALL 101
NE 102
▁AT 103
▁CON 104
H 105
▁LIKE 106
▁MO 107
▁HIS 108
▁AN 109
▁MA 110
▁THERE 111
▁SU 112
▁MY 113
NG 114
- 115
▁ABOUT 116
▁SE 117
LA 118
▁ONE 119
LO 120
▁SHE 121
CK 122
▁HAD 123
AD 124
MENT 125
VER 126
▁CA 127
IR 128
▁IF 129
▁NO 130
EL 131
▁O 132
▁FROM 133
▁PA 134
▁YOUR 135
▁KNOW 136
RY 137
ION 138
UN 139
▁JUST 140
▁UP 141
▁US 142
▁WHO 143
▁GO 144
ME 145
▁EX 146
▁OUT 147
ND 148
CI 149
▁PO 150
▁HER 151
▁FA 152
▁PRO 153
▁WHEN 154
ATE 155
AT 156
▁BY 157
PE 158
AS 159
▁LA 160
GE 161
▁CO 162
▁PEOPLE 163
▁K 164
▁SOME 165
IGHT 166
▁THINK 167
VI 168
DE 169
▁LE 170
▁WERE 171
▁LI 172
OM 173
▁WOULD 174
▁SEE 175
CO 176
▁BO 177
▁OUR 178
▁HOW 179
ET 180
Z 181
UL 182
▁HO 183
IG 184
▁GET 185
▁DON 186
IS 187
URE 188
▁WHICH 189
▁MI 190
IES 191
US 192
X 193
▁MORE 194
▁NOW 195
▁TIME 196
▁UN 197
▁FI 198
TY 199
▁WILL 200
RU 201
IVE 202
▁THEIR 203
▁SAY 204
ITY 205
▁HA 206
LU 207
▁RO 208
HO 209
OL 210
IST 211
AGE 212
MA 213
KE 214
▁RA 215
ID 216
ERS 217
▁LO 218
▁SP 219
▁DA 220
▁SAID 221
TION 222
THER 223
▁EN 224
▁BECAUSE 225
▁HAS 226
EST 227
TI 228
ACT 229
▁BA 230
▁VERY 231
▁THEM 232
▁BEEN 233
▁WA 234
ABLE 235
▁SA 236
▁DID 237
▁HIM 238
▁DIS 239
▁WANT 240
▁OTHER 241
IM 242
▁PRE 243
▁LOOK 244
▁GOING 245
▁VI 246
▁WORK 247
▁DR 248
ENCE 249
▁DI 250
FF 251
TA 252
▁REALLY 253
▁RIGHT 254
MO 255
ALLY 256
▁INTO 257
OW 258
▁WAY 259
▁FE 260
▁COULD 261
▁THEN 262
TED 263
▁HERE 264
▁THESE 265
▁OVER 266
▁WELL 267
▁NA 268
▁ANY 269
HE 270
▁COMP 271
▁COM 272
▁TH 273
▁EVEN 274
▁TWO 275
▁JO 276
▁MAKE 277
▁BACK 278
▁WHERE 279
SH 280
▁QU 281
MAN 282
▁YEAR 283
▁TRA 284
▁CHA 285
QU 286
CA 287
▁ALSO 288
CTION 289
ANCE 290
UM 291
PER 292
DI 293
▁HU 294
▁MU 295
FUL 296
▁NEW 297
ARD 298
J 299
ICAL 300
ANT 301
END 302
▁MAR 303
▁NE 304
IF 305
DER 306
ONE 307
▁THOSE 308
PP 309
▁MAN 310
▁GOOD 311
▁TAKE 312
SIDE 313
PO 314
HA 315
▁LITTLE 316
▁APP 317
▁BI 318
UGH 319
▁COME 320
ISH 321
TIC 322
▁PER 323
▁THINGS 324
▁MUCH 325
▁FIRST 326
▁THAN 327
IA 328
▁AGAIN 329
▁STA 330
▁PART 331
▁GA 332
▁MEAN 333
▁LET 334
▁DOWN 335
▁NEED 336
▁GU 337
LES 338
▁RU 339
ATED 340
▁SHA 341
VO 342
▁TALK 343
▁DAY 344
▁CHI 345
▁SOMETHING 346
▁AFTER 347
▁THOUGH 348
▁CALL 349
▁GRA 350
▁EVERY 351
▁KIND 352
ROW 353
AIN 354
▁BU 355
▁THROUGH 356
▁SHOULD 357
▁ONLY 358
▁TA 359
▁MIN 360
▁GOT 361
IAN 362
▁GREAT 363
▁RI 364
▁DOES 365
▁THING 366
DAY 367
OUS 368
▁START 369
▁MOST 370
▁COL 371
PH 372
▁LOT 373
IZE 374
BER 375
IOUS 376
▁LIFE 377
▁MANY 378
▁FEEL 379
ITION 380
▁VA 381
NESS 382
▁BEFORE 383
▁THREE 384
▁WORLD 385
▁GIVE 386
▁WHY 387
▁PI 388
▁LONG 389
PORT 390
▁SHOW 391
▁ACTUALLY 392
▁NEVER 393
IV 394
FORM 395
▁THANK 396
▁HI 397
▁PUT 398
▁LOVE 399
▁QUESTION 400
▁HELP 401
▁HAPPEN 402
▁MR 403
▁OWN 404
WARD 405
▁MADE 406
▁HAND 407
▁STATE 408
IBLE 409
▁TELL 410
▁PH 411
▁CAME 412
▁TEA 413
▁STILL 414
▁DIFFERENT 415
▁PLACE 416
▁MIGHT 417
▁FIND 418
▁DU 419
▁IMP 420
▁OLD 421
▁PERSON 422
▁COUNT 423
▁JA 424
▁POINT 425
▁HIGH 426
SHIP 427
▁LAST 428
▁SAME 429
▁FOUR 430
▁AROUND 431
▁OH 432
▁CHANGE 433
▁UNDER 434
▁HOME 435
▁ANOTHER 436
▁FOUND 437
ATIVE 438
▁TURN 439
▁YES 440
LAND 441
▁ALWAYS 442
▁YEAH 443
▁SCHOOL 444
▁JE 445
▁NEXT 446
▁WORD 447
▁JU 448
LINE 449
▁SIX 450
▁LEARN 451
▁BOOK 452
▁PLAY 453
▁AWAY 454
▁HOUSE 455
▁PLAN 456
▁WHILE 457
▁COMMUNI 458
▁FRIEND 459
▁AMERICA 460
▁INTEREST 461
▁KEEP 462
▁BETTER 463
▁IDEA 464
▁HARD 465
▁TWENTY 466
▁THOUSAND 467
▁HUNDRED 468
▁WEEK 469
▁NUMBER 470
▁COURSE 471
▁OKAY 472
PRESS 473
▁SYSTEM 474
▁NINE 475
▁UNDERSTAND 476
▁WITHOUT 477
▁BELIEVE 478
▁POWER 479
▁MAYBE 480
▁BUILD 481
▁MONEY 482
▁BETWEEN 483
▁GONNA 484
▁IMPORTANT 485
▁ANYTHING 486
▁LEFT 487
▁HAVING 488
▁WHITE 489
▁PROBLEM 490
▁SECOND 491
▁SPEAK 492
▁TOGETHER 493
▁BLACK 494
▁PUBLIC 495
▁HEALTH 496
▁NOTHING 497
▁EVERYTHING 498
Q 499
@@ -0,0 +1,242 @@
package com.faceclaw.app;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
* container, cutting speech recordings to roughly a tenth of their WAV size.
* Runs on its own thread; the source WAV is deleted on success when requested.
*/
public class FaceclawAudioTranscoder {
private static final String TAG = "FaceclawTranscode";
private static final long CODEC_TIMEOUT_US = 10_000;
public interface Listener {
void onDone(String outputPath, long outputBytes);
void onError(String message);
}
private final String wavPath;
private final String outputPath;
private final int bitrateBps;
private final boolean deleteSourceOnSuccess;
private final Listener listener;
public FaceclawAudioTranscoder(
String wavPath,
String outputPath,
int bitrateBps,
boolean deleteSourceOnSuccess,
Listener listener) {
this.wavPath = wavPath;
this.outputPath = outputPath;
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawTranscode");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
private void run() {
try {
transcode();
long size = new File(outputPath).length();
if (deleteSourceOnSuccess) {
new File(wavPath).delete();
}
if (listener != null) {
listener.onDone(outputPath, size);
}
} catch (Throwable t) {
Log.w(TAG, "transcode failed for " + wavPath, t);
new File(outputPath).delete();
if (listener != null) {
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
}
}
}
private void transcode() throws IOException {
WavInfo wav = readWavHeader(wavPath);
MediaFormat format = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
MediaMuxer muxer = null;
FileInputStream input = null;
try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
codec.start();
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
input = new FileInputStream(wavPath);
long skipped = 0;
while (skipped < wav.dataOffset) {
long step = input.skip(wav.dataOffset - skipped);
if (step <= 0) {
throw new IOException("could not seek past WAV header");
}
skipped += step;
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
byte[] chunk = new byte[8192];
int track = -1;
boolean muxerStarted = false;
boolean inputDone = false;
boolean outputDone = false;
long remaining = wav.dataBytes;
long presentationUs = 0;
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
if (read <= 0) {
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
inBuf.clear();
inBuf.put(chunk, 0, read);
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
remaining -= read;
presentationUs += read * 1_000_000L / bytesPerSecond;
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
track = muxer.addTrack(codec.getOutputFormat());
muxer.start();
muxerStarted = true;
} else if (outIndex >= 0) {
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
if (!muxerStarted) {
throw new IOException("encoder produced data before format");
}
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
outBuf.position(info.offset);
outBuf.limit(info.offset + info.size);
muxer.writeSampleData(track, outBuf, info);
}
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
if (input != null) {
try {
input.close();
} catch (IOException ignored) {
}
}
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
if (muxer != null) {
try {
muxer.stop();
} catch (Throwable ignored) {
}
muxer.release();
}
}
}
private static final class WavInfo {
int sampleRate;
int channels;
long dataOffset;
long dataBytes;
}
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
private static WavInfo readWavHeader(String path) throws IOException {
try (FileInputStream in = new FileInputStream(path)) {
byte[] head = new byte[12];
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
throw new IOException("not a WAV file");
}
WavInfo info = new WavInfo();
long offset = 12;
byte[] chunkHead = new byte[8];
while (in.read(chunkHead) == 8) {
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
int id = b.getInt();
long size = b.getInt() & 0xffffffffL;
offset += 8;
if (id == 0x20746d66) { // "fmt "
byte[] fmt = new byte[(int) Math.min(size, 16)];
if (in.read(fmt) != fmt.length) {
throw new IOException("truncated fmt chunk");
}
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
int audioFormat = f.getShort();
info.channels = f.getShort();
info.sampleRate = f.getInt();
f.getInt();
f.getShort();
int bits = f.getShort();
if (audioFormat != 1 || bits != 16) {
throw new IOException("only 16-bit PCM WAV is supported");
}
long skip = size - fmt.length;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
} else if (id == 0x61746164) { // "data"
info.dataOffset = offset;
// A zero size means the header was never patched (crash
// mid-recording); recover the length from the file size.
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
break;
} else {
long skip = size;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
}
}
if (info.sampleRate == 0 || info.dataOffset == 0) {
throw new IOException("missing fmt or data chunk");
}
return info;
}
}
}
@@ -60,9 +60,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawCompassListener> compassListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private volatile String lastFirmwareCapabilities = "";
private volatile Thread workerThread;
private volatile boolean running;
private volatile boolean userDisconnectRequested;
// Set when a connect attempt failed while an arm's Android bond is gone:
// retrying is pointless until the user re-pairs, so the worker loop parks
// instead of redialing. Cleared by start() (a fresh explicit connect).
private volatile boolean reconnectHalted;
private String phase = "disconnected";
private String status = "Disconnected.";
@@ -239,6 +246,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
running = true;
userDisconnectRequested = false;
reconnectHalted = false;
shutdownRequested = false;
activeInstance = this;
workerThread = new Thread(this, "FaceclawBleCommunicator");
@@ -355,6 +363,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/**
* Whether the glasses mic is enabled right now. The enable lives in the
* current EvenHub session, so it dies with a transport drop, the charging
* case, or a suspend — silently, from the phone's point of view. Callers
* that track a capture across those events must check this rather than
* assume their earlier enable still holds.
*/
public boolean isAudioCaptureActive() {
synchronized (lock) {
return running && sessionReady && !shutdownRequested && audioCaptureActive;
}
}
/**
* Acquire/renew or release CFW's volatile wake-takeover lease on both
* arms. Delivery (not a protocol ACK) is awaited so a caller can ensure
@@ -530,6 +551,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
public String getFirmwareCapabilities() {
return lastFirmwareCapabilities;
}
public void addMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.add(listener);
}
}
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.remove(listener);
}
}
private void emitMicStatus(byte[] body, String address) {
if (micStatusListeners.isEmpty()) {
return;
}
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
byte[] copy = java.util.Arrays.copyOf(body, body.length);
mainHandler.post(() -> {
for (FaceclawMicStatusListener micListener : micStatusListeners) {
try {
micListener.onMicStatus(copy, arm);
} catch (Throwable t) {
Log.w(TAG, "mic status listener failed", t);
}
}
});
}
/**
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
* the firmware answers with a field-104 status notify per temple, which
* arrives through addMicStatusListener.
*/
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
if (record == null || record.length < 4) {
return;
}
synchronized (lock) {
if (!running || !sessionReady) {
logLine("skip mic control (" + label + "); session not ready");
return;
}
if (rightTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
}
if (leftTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
}
logLine("queue mic control " + label);
}
interruptibleSleep.interrupt();
}
/**
* Forward render-characteristic audio packets to the listener WITHOUT
* sending the stock EvenHub audio-control enable. Used for the CFW
* mic_control streaming path, where capture is armed through settings
* field 103 and the temples emit 'SM' frames on the same characteristic
* that stock mono LC3 uses. Returns false when no session is up.
*/
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
synchronized (lock) {
if (!running || !sessionReady || shutdownRequested) {
logLine("skip G2 audio forwarding; session not ready");
return false;
}
audioPacketListener = listener;
audioCaptureActive = true;
logLine("G2 audio forwarding enabled");
}
return true;
}
public void stopG2AudioForwarding() {
synchronized (lock) {
audioPacketListener = null;
audioCaptureActive = false;
logLine("G2 audio forwarding disabled");
}
}
public void addCompassListener(FaceclawCompassListener listener) {
if (listener != null) {
compassListeners.add(listener);
@@ -564,12 +677,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
* foreground), including worker-app frames the TS side never sees.
*/
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) {
return getCompositePreviewBitmap(brightenGamma, false);
}
/** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) {
SurfaceCompositor.Composite composite = compositor.previewComposite();
if (composite == null) {
return null;
}
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma);
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green);
}
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
@@ -1118,6 +1236,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
emitPhoneLockStateIfChanged(false);
if (!sessionReady) {
if (reconnectHalted) {
interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS);
continue;
}
long now = SystemClock.elapsedRealtime();
if (now < reconnectAfterMs) {
interruptibleSleep.sleep(Math.min(ConnectionOptions.IDLE_SLEEP_MS, reconnectAfterMs - now));
@@ -1216,6 +1338,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
0
);
}
if (!faceclawWakeNotification
&& frame.ok
&& frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW mic status (field 104) rides both standalone pushes and
// settings read acks, from each temple on its own link.
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
if (micStatus != null) {
emitMicStatus(micStatus, address);
}
}
if (!faceclawWakeNotification
&& decodedWearState < 0
&& frame.ok
@@ -1365,13 +1497,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
audioPacketListener = null;
clearAllMessagesLocked("connection lost");
displayedFingerprint = "";
reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS;
if (!reconnectHalted) {
reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS;
}
}
}
interruptibleSleep.interrupt();
if (connected) {
setStateDisplay("connected", "Connected.");
} else {
} else if (!reconnectHalted) {
// While parked on a missing bond, keep the "unpaired" display: this
// callback is just the teardown of the arm that did connect.
setStateDisplay("connecting", "Connecting to the glasses...");
}
}
@@ -1384,6 +1520,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
if (!sleepDuringConnectSettling(800)) {
return;
}
authenticateArms();
sendPrelude();
synchronized (lock) {
@@ -1440,7 +1577,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
tryConnectRing("initial");
} catch (Throwable t) {
logLine("connect failed: " + safeMessage(t));
handleTransportFailure("connect failed");
String unpairedArm = firstUnpairedArm();
if (unpairedArm != null) {
handleUnpairedFailure(unpairedArm);
} else {
handleTransportFailure("connect failed");
}
}
}
@@ -1559,6 +1701,55 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/**
* Complete the sid-0x80 security-auth exchange on both freshly opened arm
* connections. Firmware 2.2.9 answers no queries until it completes over an
* encrypted link and closes unauthenticated links after ~30 s (see
* ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is
* also what triggers SMP pairing. Deliberately soft: on timeout we log and
* continue rather than fail the connect — the custom firmware's response
* behavior is not yet hardware-verified, and on stock firmware an
* unanswered auth just means the prelude fails exactly as it did before.
* A pairing prompt accepted after our window still bonds at the OS level,
* so the next reconnect attempt authenticates promptly.
*/
private void authenticateArms() throws InterruptedException {
OutboundMessage right = messageBuilder.securityAuth(false);
OutboundMessage left = messageBuilder.securityAuth(true);
long now = SystemClock.elapsedRealtime();
for (OutboundMessage message : new OutboundMessage[] {right, left}) {
message.onAck = () -> {
};
message.onTimeout = () -> {
};
message.sentAtMs = now;
writeMessage(message);
}
long deadline = SystemClock.elapsedRealtime() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS;
while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) {
synchronized (lock) {
if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) {
break;
}
}
long remaining = deadline - SystemClock.elapsedRealtime();
if (remaining <= 0) {
break;
}
interruptibleSleep.sleep(Math.min(remaining, 100));
}
synchronized (lock) {
if (!inFlightMessages.isEmpty()) {
clearInFlightMessagesLocked("security auth timeout");
logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)");
return;
}
}
boolean rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic);
boolean leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic);
logLine("security auth R=" + (rightOk ? "ok" : "unconfirmed") + " L=" + (leftOk ? "ok" : "unconfirmed"));
}
private void sendPrelude() throws InterruptedException {
sendPrelude(false);
}
@@ -2429,6 +2620,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
if (firmwareInfo != null) {
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
&& hasCapability(firmwareInfo.capabilities, "texstr14");
@@ -2928,6 +3120,49 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
+ " release=" + previous.reason);
}
/** The address of a configured arm Android no longer holds a bond for, or null. */
private String firstUnpairedArm() {
if (!bleManager.isBonded(rightAddress)) return rightAddress;
if (!bleManager.isBonded(leftAddress)) return leftAddress;
return null;
}
/**
* A connect failure while an arm's bond is missing (the pairing was
* forgotten in Android settings, or the address was typed in by hand and
* never paired) will repeat forever, so instead of scheduling a retry,
* park the worker loop and tell the user to re-pair. Only an explicit
* connect (which builds a fresh communicator) starts a new attempt.
*/
private void handleUnpairedFailure(String address) {
Log.e(TAG, "Connect failed and " + address + " is not paired; suspending reconnect");
synchronized (lock) {
reconnectHalted = true;
sessionReady = false;
fixedLayoutCreated = false;
startupProbePending = false;
shutdownRequested = false;
chargingMode = false;
imageRetryAfterMs = 0;
displayedFingerprint = "";
faceclawWakePendingNonce = -1;
lastFaceclawWakeLeaseQueuedAtMs = 0;
faceclawWakeControlSentCount = 0;
clearAllMessagesLocked("arm not paired: " + address);
reconnectAfterMs = Long.MAX_VALUE;
bleManager.disconnect(rightAddress);
bleManager.disconnect(leftAddress);
}
if (!userDisconnectRequested) {
setStateDisplay(
"unpaired",
"The glasses (" + address + ") are not paired with this phone."
+ " Use \"Pair glasses\" to pair them again, then connect."
);
}
interruptibleSleep.interrupt();
}
private void handleTransportFailure(String reason) {
Log.e(TAG, "Transport failure: "+reason);
synchronized (lock) {
@@ -2963,6 +3198,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
ringConnected = false;
ringNotificationsReady = false;
reconnectAfterMs = 0;
reconnectHalted = false;
ringReconnectAfterMs = 0;
lastAckAtMs = 0;
lastIncomingAtMs = 0;
@@ -64,6 +64,23 @@ public class FaceclawBleManager {
this.listener = listener;
}
/**
* Whether Android still holds a bond (pairing) for this address. Returns
* true when the state cannot be determined, so callers only act on a
* definite BOND_NONE.
*/
public boolean isBonded(String address) {
if (address == null || address.trim().isEmpty()) {
return true;
}
try {
BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address);
return device == null || device.getBondState() != BluetoothDevice.BOND_NONE;
} catch (Throwable t) {
return true;
}
}
public boolean connect(String address, int timeoutMs) {
if (address == null || address.trim().isEmpty()) {
throw new IllegalArgumentException("address is required");
@@ -0,0 +1,446 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import java.io.File;
import java.util.ArrayDeque;
import java.util.Arrays;
/**
* Continuous captioning over the decoded mic PCM: segments speech into
* utterances with an adaptive energy gate, transcribes each utterance with
* the on-device Moonshine model, and attaches a speaker voice-print per
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
* and any beam-direction gating), so this engine has no BLE dependencies.
*
* Utterance boundaries are measured on the sample clock, not the wall clock:
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
* audio heard. The TS side converts startMs/endMs to wall-clock times using
* the engine start timestamp.
*/
public class FaceclawCaptionEngine {
private static final String TAG = "FaceclawCaptions";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final int MAX_QUEUE_PACKETS = 200;
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
private static final double ONSET_FACTOR = 3.0;
private static final double RELEASE_FACTOR = 1.8;
private static final double MIN_RMS = 220.0;
private static final double NOISE_EMA_ALPHA = 0.05;
private static final int PRE_ROLL_MS = 400;
private static final int DEFAULT_SILENCE_MS = 800;
private static final int MIN_UTTERANCE_MS = 350;
private static final int MAX_UTTERANCE_MS = 15000;
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
// segments cut at the quietest window (mirrors FaceclawVoiceController).
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
private static final float NORMALIZE_MAX_GAIN = 30f;
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private final Object queueLock = new Object();
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
private volatile FaceclawCaptionEngineListener listener;
private volatile String asrModelDir;
private volatile String speakerModelPath;
private volatile int silenceMs = DEFAULT_SILENCE_MS;
private Thread workerThread;
private volatile boolean started;
private OfflineRecognizer recognizer;
private FaceclawSpeakerId speakerId;
// Segmentation state (worker thread only).
private long totalSamples;
private double noiseFloor;
private boolean inUtterance;
private long utteranceStartSample;
private long silenceRunSamples;
private double utterancePeakRms;
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
private int utteranceLength;
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
private int preRollLength;
public void setListener(FaceclawCaptionEngineListener listener) {
this.listener = listener;
}
/** Directory holding the Moonshine model files, or null to disable ASR. */
public void setAsrModelDir(String dir) {
this.asrModelDir = dir;
}
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
public void setSpeakerModelPath(String path) {
this.speakerModelPath = path;
}
public void setSilenceMs(int ms) {
this.silenceMs = Math.max(200, Math.min(3000, ms));
}
public void start() {
synchronized (lock) {
if (started) {
return;
}
started = true;
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
workerThread.start();
}
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
if (!started) {
return;
}
started = false;
threadToJoin = workerThread;
workerThread = null;
}
synchronized (queueLock) {
queueLock.notifyAll();
}
if (threadToJoin != null) {
threadToJoin.interrupt();
if (Thread.currentThread() != threadToJoin) {
try {
threadToJoin.join(2000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
public void acceptPcm(byte[] pcm16le) {
if (!started || pcm16le == null || pcm16le.length < 2) {
return;
}
synchronized (queueLock) {
if (queue.size() >= MAX_QUEUE_PACKETS) {
queue.removeFirst();
}
queue.addLast(pcm16le);
queueLock.notifyAll();
}
}
private void runLoop() {
try {
loadModels();
resetSegmentation();
while (started && !Thread.currentThread().isInterrupted()) {
byte[] chunk = takeChunk();
if (chunk == null) {
continue;
}
processChunk(chunk);
}
// Flush a trailing utterance so its text isn't lost on stop.
if (inUtterance && utteranceLength > 0) {
finalizeUtterance();
}
} catch (Throwable t) {
Log.e(TAG, "caption engine failed", t);
emitStatus("Captions failed: " + t.getMessage());
} finally {
releaseModels();
}
}
private void loadModels() {
String modelDir = asrModelDir;
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
emitStatus("Loading caption model...");
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setNumThreads(1)
.build())
.build());
} else {
recognizer = null;
}
String embedModel = speakerModelPath;
if (embedModel != null && new File(embedModel).exists()) {
speakerId = new FaceclawSpeakerId(embedModel);
speakerId.ensureLoaded();
} else {
speakerId = null;
}
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
}
private void releaseModels() {
if (recognizer != null) {
recognizer.release();
recognizer = null;
}
if (speakerId != null) {
speakerId.close();
speakerId = null;
}
}
private byte[] takeChunk() {
synchronized (queueLock) {
while (started && queue.isEmpty()) {
try {
queueLock.wait(250);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
}
return queue.pollFirst();
}
}
private void resetSegmentation() {
totalSamples = 0;
noiseFloor = 0;
inUtterance = false;
utteranceLength = 0;
preRollLength = 0;
silenceRunSamples = 0;
utterancePeakRms = 0;
}
private void processChunk(byte[] chunk) {
int count = chunk.length / 2;
short[] pcm = new short[count];
double sumSquares = 0;
for (int i = 0; i < count; i++) {
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
pcm[i] = s;
sumSquares += (double) s * s;
}
double rms = Math.sqrt(sumSquares / Math.max(1, count));
totalSamples += count;
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true;
utteranceLength = 0;
silenceRunSamples = 0;
utterancePeakRms = rms;
appendUtterance(preRoll, preRollLength);
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
appendUtterance(pcm, count);
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
} else {
appendPreRoll(pcm, count);
}
return;
}
appendUtterance(pcm, count);
utterancePeakRms = Math.max(utterancePeakRms, rms);
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count;
} else {
silenceRunSamples = 0;
}
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance();
inUtterance = false;
preRollLength = 0;
}
}
private void appendPreRoll(short[] pcm, int count) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.length) {
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
preRollLength = preRoll.length;
return;
}
int keep = Math.min(preRollLength, preRoll.length - count);
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
System.arraycopy(pcm, 0, preRoll, keep, count);
preRollLength = keep + count;
}
private void appendUtterance(short[] pcm, int count) {
int room = utterance.length - utteranceLength;
int copied = Math.min(room, count);
if (copied > 0) {
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
utteranceLength += copied;
}
}
private void finalizeUtterance() {
int length = utteranceLength;
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
if (endMs - startMs < MIN_UTTERANCE_MS) {
return;
}
String text = recognizeUtterance(length);
float[] embedding = embedUtterance(length);
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
}
private String recognizeUtterance(int length) {
if (recognizer == null || length <= 0) {
return "";
}
StringBuilder joined = new StringBuilder();
int offset = 0;
while (offset < length) {
int remaining = length - offset;
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = findQuietCut(offset, segment);
}
String part = recognizeRange(offset, segment);
if (part.length() > 0) {
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
joined.append(' ');
}
joined.append(part);
}
offset += segment;
}
return joined.toString().trim();
}
/**
* End a decode segment at the center of the quietest window near its end
* so the cut lands between words (same approach as the PTT controller).
*/
private int findQuietCut(int offset, int segment) {
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
int win = CUT_WINDOW_SAMPLES;
if (segment - searchStart <= win) {
return segment;
}
double sum = 0;
for (int i = searchStart; i < searchStart + win; i++) {
double s = utterance[offset + i];
sum += s * s;
}
double best = sum;
int bestStart = searchStart;
for (int start = searchStart + 1; start + win <= segment; start++) {
double dropped = utterance[offset + start - 1];
double added = utterance[offset + start + win - 1];
sum += added * added - dropped * dropped;
if (sum < best) {
best = sum;
bestStart = start;
}
}
return bestStart + win / 2;
}
private String recognizeRange(int offset, int count) {
float[] samples = new float[count];
float peak = 0f;
for (int i = 0; i < count; i++) {
float v = utterance[offset + i] / 32768.0f;
samples[i] = v;
float a = Math.abs(v);
if (a > peak) {
peak = a;
}
}
if (peak > 0f) {
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
if (gain > 1f) {
for (int i = 0; i < count; i++) {
samples[i] *= gain;
}
}
}
OfflineStream stream = recognizer.createStream();
try {
stream.acceptWaveform(samples, SAMPLE_RATE);
recognizer.decode(stream);
OfflineRecognizerResult result = recognizer.getResult(stream);
String raw = result == null ? "" : result.getText();
return raw == null ? "" : raw.trim();
} finally {
stream.release();
}
}
private float[] embedUtterance(int length) {
FaceclawSpeakerId currentSpeakerId = speakerId;
if (currentSpeakerId == null || length <= 0) {
return null;
}
int count = Math.min(length, EMBED_MAX_SAMPLES);
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
short s = utterance[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
return currentSpeakerId.embed(le, SAMPLE_RATE);
}
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
}
private void emitSpeechStart(long startMs) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
}
private void emitStatus(String status) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onStatus(status));
}
}
@@ -0,0 +1,17 @@
package com.faceclaw.app;
public interface FaceclawCaptionEngineListener {
/**
* One finished utterance. text may be empty when no ASR model is loaded
* or recognition produced nothing; embedding is the L2-normalized
* voice-print or null when the speaker model is unavailable or the
* utterance was too short. startMs/endMs are on the engine's sample
* clock (milliseconds of audio since start()).
*/
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
/** A speech onset was detected; the utterance is now being collected. */
void onSpeechStart(long startMs);
void onStatus(String status);
}
@@ -0,0 +1,778 @@
package com.faceclaw.app;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.util.Base64;
import android.util.Log;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Persistent store for caption sessions, transcript segments, and speaker
* voice profiles, backing the Microphones app and its phone-side review UI.
* All complex values cross the JS bridge as JSON strings; voice-print
* embeddings are float32 LE blobs, exposed as base64.
*/
public class FaceclawConversationStore extends SQLiteOpenHelper {
private static final String TAG = "FaceclawConvStore";
private static final String DB_NAME = "faceclaw-conversations.db";
private static final int DB_VERSION = 2;
private static FaceclawConversationStore instance;
public static synchronized FaceclawConversationStore getInstance(Context context) {
if (instance == null) {
instance = new FaceclawConversationStore(context.getApplicationContext());
}
return instance;
}
private FaceclawConversationStore(Context context) {
super(context, DB_NAME, null, DB_VERSION);
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL("CREATE TABLE speakers ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "name TEXT NOT NULL,"
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
+ "embedding BLOB,"
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
+ "created_at INTEGER NOT NULL,"
+ "last_heard_at INTEGER,"
+ "tag TEXT,"
+ "last_recap TEXT,"
+ "action_items TEXT,"
+ "facts TEXT,"
+ "insights_updated_at INTEGER,"
+ "insights_session_id INTEGER)");
db.execSQL("CREATE TABLE sessions ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER,"
+ "title TEXT,"
+ "audio_path TEXT,"
+ "audio_codec TEXT,"
+ "avg_sentiment REAL,"
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
db.execSQL("CREATE TABLE segments ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "session_id INTEGER NOT NULL,"
+ "speaker_id INTEGER,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER NOT NULL,"
+ "audio_offset_ms INTEGER,"
+ "text TEXT NOT NULL,"
+ "lang TEXT,"
+ "translation TEXT,"
+ "translation_lang TEXT,"
+ "sentiment REAL,"
+ "emotion TEXT,"
+ "search_meta TEXT,"
+ "angle INTEGER,"
+ "embedding BLOB)");
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion < 2) {
// v2: per-speaker conversation insights (recap of the most recent
// conversation, its action items, and accumulated inferred facts).
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
}
}
// ---- sessions ----
public long startSession(long startedAtMs, String title) {
ContentValues values = new ContentValues();
values.put("started_at", startedAtMs);
values.put("title", title == null || title.isEmpty() ? null : title);
return getWritableDatabase().insert("sessions", null, values);
}
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
ContentValues values = new ContentValues();
values.put("ended_at", endedAtMs);
if (audioPath != null && !audioPath.isEmpty()) {
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
}
values.put("avg_sentiment", avgSentiment);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
ContentValues values = new ContentValues();
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public String getSessionAudioPath(long sessionId) {
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
if (c.moveToFirst()) {
return c.isNull(0) ? "" : c.getString(0);
}
}
return "";
}
/**
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
* speakerId, emotion (matches any segment emotion in the session), query
* (LIKE over title and segment text/search metadata), limit.
*/
public String querySessions(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
+ " s.avg_sentiment, s.segment_count FROM sessions s");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
if (joinSegments) {
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
}
if (filter.has("sinceMs")) {
where.add("s.started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("s.started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (filter.has("speakerId")) {
where.add("g.speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("emotion")) {
where.add("g.emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("query")) {
String like = "%" + filter.getString("query") + "%";
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY s.started_at DESC");
int limit = filter.optInt("limit", 200);
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
long id = c.getLong(0);
row.put("id", id);
row.put("startedAt", c.getLong(1));
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
row.put("title", c.isNull(3) ? "" : c.getString(3));
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
row.put("segmentCount", c.getLong(7));
row.put("speakers", sessionSpeakersJson(id));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "querySessions failed", e);
return "[]";
}
}
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
out.put(row);
}
}
return out;
}
// ---- segments ----
/**
* Insert one transcript segment. json fields: sessionId, speakerId?,
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
* embeddingBase64?. Returns the new row id, or -1.
*/
public long insertSegment(String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
long sessionId = s.getLong("sessionId");
values.put("session_id", sessionId);
if (s.has("speakerId")) {
values.put("speaker_id", s.getLong("speakerId"));
}
values.put("started_at", s.getLong("startedAt"));
values.put("ended_at", s.getLong("endedAt"));
if (s.has("audioOffsetMs")) {
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
}
values.put("text", s.getString("text"));
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (s.has("angle")) {
values.put("angle", s.getInt("angle"));
}
if (s.has("embeddingBase64")) {
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
}
SQLiteDatabase db = getWritableDatabase();
long id = db.insert("segments", null, values);
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
new Object[]{sessionId});
return id;
} catch (Throwable t) {
Log.w(TAG, "insertSegment failed", t);
return -1;
}
}
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
throws JSONException {
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
values.put(column, s.getString(jsonKey));
}
}
/** Update mutable fields of a segment (same keys as insertSegment). */
public void updateSegment(long segmentId, String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
if (s.has("speakerId")) {
if (s.isNull("speakerId")) {
values.putNull("speaker_id");
} else {
values.put("speaker_id", s.getLong("speakerId"));
}
}
if (s.has("text")) {
values.put("text", s.getString("text"));
}
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (values.size() > 0) {
getWritableDatabase().update("segments", values, "id=?",
new String[]{String.valueOf(segmentId)});
}
} catch (Throwable t) {
Log.w(TAG, "updateSegment failed", t);
}
}
public String querySegments(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle"
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
out.put(segmentRowJson(c));
}
} catch (JSONException e) {
Log.w(TAG, "querySegments failed", e);
}
return out.toString();
}
private JSONObject segmentRowJson(Cursor c) throws JSONException {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
row.put("startedAt", c.getLong(2));
row.put("endedAt", c.getLong(3));
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
row.put("text", c.getString(5));
row.put("lang", c.isNull(6) ? "" : c.getString(6));
row.put("translation", c.isNull(7) ? "" : c.getString(7));
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
return row;
}
/**
* Search segments across sessions. filterJson: query?, emotion?,
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
* the sentiment search metadata.
*/
public String searchSegments(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
+ " FROM segments");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
if (filter.has("query") && !filter.getString("query").isEmpty()) {
String like = "%" + filter.getString("query") + "%";
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (filter.has("emotion")) {
where.add("emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("speakerId")) {
where.add("speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("sinceMs")) {
where.add("started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY started_at DESC LIMIT ")
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = segmentRowJson(c);
row.put("sessionId", c.getLong(12));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "searchSegments failed", e);
return "[]";
}
}
/** Per-segment voice-print embeddings of a session (for re-diarization). */
public String querySegmentEmbeddings(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySegmentEmbeddings failed", e);
}
return out.toString();
}
// ---- speakers ----
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
ContentValues values = new ContentValues();
values.put("name", name);
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
values.put("is_wearer", isWearer ? 1 : 0);
values.put("created_at", System.currentTimeMillis());
values.put("last_heard_at", System.currentTimeMillis());
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
values.put("embedding_count", 1);
}
return getWritableDatabase().insert("speakers", null, values);
}
public String querySpeakers() {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
+ " p.last_heard_at, p.tag,"
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
row.put("isWearer", c.getInt(3) != 0);
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
row.put("embeddingCount", c.getInt(5));
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
row.put("tag", c.isNull(7) ? "" : c.getString(7));
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
row.put("segmentCount", c.getLong(13));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySpeakers failed", e);
}
return out.toString();
}
public void renameSpeaker(long speakerId, String name) {
ContentValues values = new ContentValues();
values.put("name", name);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerColor(long speakerId, String color) {
ContentValues values = new ContentValues();
values.put("color", color);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerTag(long speakerId, String tag) {
ContentValues values = new ContentValues();
values.put("tag", tag);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Store LLM-derived conversation insights for a speaker: a recap of their
* most recent conversation, its open action items (JSON string array), and
* the accumulated inferred facts about them (JSON string array).
*/
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
String factsJson, long sessionId) {
ContentValues values = new ContentValues();
values.put("last_recap", recap == null ? "" : recap);
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
values.put("facts", factsJson == null ? "" : factsJson);
values.put("insights_updated_at", System.currentTimeMillis());
values.put("insights_session_id", sessionId);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerWearer(long speakerId, boolean isWearer) {
SQLiteDatabase db = getWritableDatabase();
if (isWearer) {
// Exactly one wearer profile at a time.
db.execSQL("UPDATE speakers SET is_wearer = 0");
}
ContentValues values = new ContentValues();
values.put("is_wearer", isWearer ? 1 : 0);
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Running-mean centroid update for a speaker's voice-print, capped so a
* long session cannot drown the enrolled voice.
*/
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
try {
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
float[] update = blobToFloats(incoming);
if (update == null) {
return;
}
float[] current = null;
int count = 0;
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
new String[]{String.valueOf(speakerId)})) {
if (c.moveToFirst()) {
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
count = c.getInt(1);
}
}
float[] merged;
if (current == null || current.length != update.length) {
merged = update;
count = 1;
} else {
int effective = Math.min(count, Math.max(1, maxCount));
merged = new float[current.length];
for (int i = 0; i < merged.length; i++) {
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
}
l2NormalizeInPlace(merged);
count = count + 1;
}
ContentValues values = new ContentValues();
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", count);
values.put("last_heard_at", System.currentTimeMillis());
getWritableDatabase().update("speakers", values, "id=?",
new String[]{String.valueOf(speakerId)});
} catch (Throwable t) {
Log.w(TAG, "updateSpeakerEmbedding failed", t);
}
}
/** Move every segment from one speaker onto another and drop the source. */
public void mergeSpeakers(long fromId, long intoId) {
if (fromId == intoId) {
return;
}
SQLiteDatabase db = getWritableDatabase();
db.beginTransaction();
try {
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
new Object[]{intoId, fromId});
// Blend the two centroids weighted by their sample counts.
float[] from = null;
float[] into = null;
int fromCount = 0;
int intoCount = 0;
boolean fromWearer = false;
try (Cursor c = db.rawQuery(
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
while (c.moveToNext()) {
long id = c.getLong(0);
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
int count = c.getInt(2);
if (id == fromId) {
from = embedding;
fromCount = count;
fromWearer = c.getInt(3) != 0;
} else {
into = embedding;
intoCount = count;
}
}
}
float[] merged = into;
if (from != null && into != null && from.length == into.length) {
merged = new float[into.length];
int total = Math.max(1, fromCount + intoCount);
for (int i = 0; i < merged.length; i++) {
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
}
l2NormalizeInPlace(merged);
} else if (into == null) {
merged = from;
}
ContentValues values = new ContentValues();
if (merged != null) {
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", fromCount + intoCount);
}
if (fromWearer) {
values.put("is_wearer", 1);
}
values.put("last_heard_at", System.currentTimeMillis());
// Keep the target's conversation insights; adopt the source's only
// where the target has none (e.g. merging a rich profile into a
// freshly auto-created one).
try (Cursor c = db.rawQuery(
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
boolean targetHasInsights = false;
while (c.moveToNext()) {
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
if (c.getPosition() == 0) {
targetHasInsights = hasInsights;
} else if (!targetHasInsights && hasInsights) {
values.put("last_recap", c.getString(0));
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
values.put("facts", c.isNull(2) ? "" : c.getString(2));
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
}
}
}
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
}
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
ContentValues values = new ContentValues();
values.put("speaker_id", speakerId);
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
}
/** Reassign every segment of a speaker within one session (split support). */
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
getWritableDatabase().execSQL(
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
}
public void deleteSpeaker(long speakerId) {
SQLiteDatabase db = getWritableDatabase();
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
}
// ---- retention ----
/**
* Delete caption sessions older than the cutoff (0 disables), and strip
* recordings older than their own cutoff while keeping the transcript.
* Returns a JSON summary of what was removed.
*/
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
SQLiteDatabase db = getWritableDatabase();
int sessionsDeleted = 0;
int recordingsDeleted = 0;
if (recordingCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(recordingCutoffMs)})) {
while (c.moveToNext()) {
if (!c.isNull(1)) {
paths.add(c.getString(1));
}
}
}
for (String path : paths) {
if (new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
new Object[]{recordingCutoffMs});
}
if (captionCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(captionCutoffMs)})) {
while (c.moveToNext()) {
paths.add(c.getString(0));
}
}
for (String path : paths) {
if (path != null && new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
new Object[]{captionCutoffMs});
Cursor count = db.rawQuery("SELECT changes()", null);
count.close();
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
if (c.moveToFirst()) {
sessionsDeleted = c.getInt(0);
}
}
}
try {
JSONObject summary = new JSONObject();
summary.put("sessionsDeleted", sessionsDeleted);
summary.put("recordingsDeleted", recordingsDeleted);
return summary.toString();
} catch (JSONException e) {
return "{}";
}
}
public void deleteSession(long sessionId) {
SQLiteDatabase db = getWritableDatabase();
String path = getSessionAudioPath(sessionId);
if (!path.isEmpty()) {
new File(path).delete();
}
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
}
// ---- blob helpers ----
private static float[] blobToFloats(byte[] blob) {
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
return null;
}
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
float[] out = new float[blob.length / 4];
for (int i = 0; i < out.length; i++) {
out[i] = buffer.getFloat();
}
return out;
}
private static byte[] floatsToBlob(float[] values) {
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
for (float v : values) {
buffer.putFloat(v);
}
return buffer.array();
}
private static void l2NormalizeInPlace(float[] v) {
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
}
}
@@ -6,151 +6,381 @@ import android.bluetooth.BluetoothDevice;
import android.bluetooth.BluetoothManager;
import android.bluetooth.le.BluetoothLeScanner;
import android.bluetooth.le.ScanCallback;
import android.bluetooth.le.ScanRecord;
import android.bluetooth.le.ScanResult;
import android.bluetooth.le.ScanSettings;
import android.content.Context;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.util.SparseArray;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
/**
* Finds Even Realities hardware over BLE and hands every advertisement up to
* TypeScript with the raw fields the pairing screen needs: the LIVE local name
* from the scan record (BluetoothDevice.getName() is a cache that can hold a
* pre-reset value indefinitely), the full manufacturer-specific payload with
* its company identifier restored so the "ER"+serial+MAC layout survives
* intact, RSSI, advertised TX power, and connectability.
*
* Admission is deliberately loose — a name containing G2, the stock
* "EVEN R1…" ring prefix, or the Even "ER" manufacturer signature — because
* renamed custom firmware may drop the stock name. Side, serial, and pair
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
* must stay a superset of the TS classifier's admission.
*/
@SuppressLint("MissingPermission")
public class FaceclawDeviceDiscovery {
private static final Pattern RING_NAME = Pattern.compile("^EVEN R1_[0-9A-F]{6}$");
private static final Pattern RIGHT_NAME = Pattern.compile("^Even G2_32_R_[0-9A-F]{6}$");
private static final Pattern LEFT_NAME = Pattern.compile("^Even G2_32_L_[0-9A-F]{6}$");
/** Even Realities' company identifier: the ASCII bytes "ER" read little-endian. */
public static final int EVEN_COMPANY_ID = 0x5245;
private final Context appContext;
private final BluetoothAdapter bluetoothAdapter;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private ScanCallback activeCallback;
private volatile FaceclawDeviceDiscoveryListener listener;
public FaceclawDeviceDiscovery(Context context) {
this.appContext = context.getApplicationContext();
Context appContext = context.getApplicationContext();
BluetoothManager bluetoothManager =
(BluetoothManager) this.appContext.getSystemService(Context.BLUETOOTH_SERVICE);
(BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
if (bluetoothManager == null || bluetoothManager.getAdapter() == null) {
throw new IllegalStateException("Bluetooth adapter unavailable");
}
this.bluetoothAdapter = bluetoothManager.getAdapter();
}
public String getBondedCandidatesJson() {
LinkedHashMap<String, CandidateDevice> matches = new LinkedHashMap<>();
Set<BluetoothDevice> bonded = bluetoothAdapter.getBondedDevices();
if (bonded != null) {
for (BluetoothDevice device : bonded) {
maybeAddCandidate(matches, device, "paired");
}
}
return encodeCandidates(matches);
public boolean isBluetoothEnabled() {
return bluetoothAdapter.isEnabled();
}
public String scanCandidatesJson(int timeoutMs) throws InterruptedException {
LinkedHashMap<String, CandidateDevice> matches = new LinkedHashMap<>();
BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner();
if (scanner == null) {
return encodeCandidates(matches);
}
final ScanCallback callback = new ScanCallback() {
@Override
public void onScanResult(int callbackType, ScanResult result) {
if (result == null) {
return;
}
maybeAddCandidate(matches, result.getDevice(), "scan");
}
public void setListener(FaceclawDeviceDiscoveryListener newListener) {
this.listener = newListener;
}
@Override
public void onBatchScanResults(java.util.List<ScanResult> results) {
if (results == null) {
return;
}
for (ScanResult result : results) {
// ------------------------------------------------------------------
// Live scan
// ------------------------------------------------------------------
/**
* Start streaming advertisements to the listener. Returns false when the
* radio is off or the scanner is unavailable; a later platform refusal
* arrives through onScanFailed. Calling while a scan is running is a no-op.
*/
public boolean startScan() {
synchronized (lock) {
if (activeCallback != null) {
return true;
}
if (!bluetoothAdapter.isEnabled()) {
emitLog("scan not started: Bluetooth is off");
return false;
}
BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner();
if (scanner == null) {
emitLog("scan not started: no BLE scanner");
return false;
}
final ScanCallback callback = new ScanCallback() {
@Override
public void onScanResult(int callbackType, ScanResult result) {
if (result != null) {
maybeAddCandidate(matches, result.getDevice(), "scan");
emitAdvertisement(result);
}
}
@Override
public void onBatchScanResults(List<ScanResult> results) {
if (results == null) {
return;
}
for (ScanResult result : results) {
if (result != null) {
emitAdvertisement(result);
}
}
}
@Override
public void onScanFailed(int errorCode) {
synchronized (lock) {
if (activeCallback == this) {
activeCallback = null;
}
}
final FaceclawDeviceDiscoveryListener l = listener;
if (l != null) {
final String message = describeScanError(errorCode);
mainHandler.post(() -> l.onScanFailed(errorCode, message));
}
}
};
activeCallback = callback;
try {
scanner.startScan(null, buildScanSettings(), callback);
} catch (Throwable t) {
activeCallback = null;
emitLog("scan not started: " + t);
return false;
}
};
scanner.startScan(
null,
new ScanSettings.Builder().setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY).build(),
callback
);
try {
Thread.sleep(Math.max(500, timeoutMs));
} finally {
emitLog("scan started (unfiltered, low latency)");
return true;
}
}
public void stopScan() {
ScanCallback callback;
synchronized (lock) {
callback = activeCallback;
activeCallback = null;
}
if (callback == null) {
return;
}
BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner();
if (scanner != null) {
try {
scanner.stopScan(callback);
} catch (Throwable ignored) {
}
}
return encodeCandidates(matches);
emitLog("scan stopped");
}
private void maybeAddCandidate(Map<String, CandidateDevice> matches, BluetoothDevice device, String source) {
if (device == null) {
return;
public boolean isScanning() {
synchronized (lock) {
return activeCallback != null;
}
String name = device.getName();
String role = classifyRole(name);
if (role == null) {
return;
}
String address = device.getAddress();
if (address == null || address.isEmpty()) {
return;
}
matches.put(address, new CandidateDevice(address, name == null ? "" : name, role, source));
}
private String classifyRole(String name) {
if (name == null) {
return null;
/**
* Replay bonded Even devices to the listener. A bonded listing has no
* signal sample and no manufacturer data, so its serial stays unknown until
* the device is also heard advertising.
*/
public void emitBondedDevices() {
final FaceclawDeviceDiscoveryListener l = listener;
if (l == null) {
return;
}
if (RIGHT_NAME.matcher(name).matches()) {
return "right";
for (String json : bondedCandidateJsonList()) {
l.onAdvertisement(json);
}
if (LEFT_NAME.matcher(name).matches()) {
return "left";
}
if (RING_NAME.matcher(name).matches()) {
return "ring";
}
return null;
}
private String encodeCandidates(Map<String, CandidateDevice> matches) {
/** Bonded Even devices as one JSON array, for callers without a listener. */
public String getBondedCandidatesJson() {
JSONArray array = new JSONArray();
for (CandidateDevice candidate : matches.values()) {
JSONObject object = new JSONObject();
for (String json : bondedCandidateJsonList()) {
try {
object.put("address", candidate.address);
object.put("name", candidate.name);
object.put("role", candidate.role);
object.put("source", candidate.source);
array.put(object);
array.put(new JSONObject(json));
} catch (JSONException ignored) {
}
}
return array.toString();
}
private static final class CandidateDevice {
final String address;
final String name;
final String role;
final String source;
// ------------------------------------------------------------------
// Internals
// ------------------------------------------------------------------
CandidateDevice(String address, String name, String role, String source) {
this.address = address;
this.name = name;
this.role = role;
this.source = source;
private static ScanSettings buildScanSettings() {
ScanSettings.Builder builder = new ScanSettings.Builder()
.setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY)
.setReportDelay(0);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
builder.setCallbackType(ScanSettings.CALLBACK_TYPE_ALL_MATCHES);
builder.setMatchMode(ScanSettings.MATCH_MODE_AGGRESSIVE);
builder.setNumOfMatches(ScanSettings.MATCH_NUM_MAX_ADVERTISEMENT);
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
// Accept extended advertising too; setLegacy(true) would hide it.
builder.setLegacy(false);
}
return builder.build();
}
private void emitAdvertisement(ScanResult result) {
final JSONObject object = toJson(result);
if (object == null) {
return;
}
final FaceclawDeviceDiscoveryListener l = listener;
if (l == null) {
return;
}
final String json = object.toString();
if (Looper.myLooper() == Looper.getMainLooper()) {
l.onAdvertisement(json);
} else {
mainHandler.post(() -> {
FaceclawDeviceDiscoveryListener current = listener;
if (current != null) {
current.onAdvertisement(json);
}
});
}
}
private List<String> bondedCandidateJsonList() {
List<String> out = new ArrayList<>();
Set<BluetoothDevice> bonded = bluetoothAdapter.getBondedDevices();
if (bonded == null) {
return out;
}
long now = System.currentTimeMillis();
for (BluetoothDevice device : bonded) {
if (device == null) {
continue;
}
String name = device.getName();
if (!isAdmissible(name, null)) {
continue;
}
JSONObject object = buildJson(
device.getAddress(), name, "", null, null, null, true, "paired", now);
if (object != null) {
out.add(object.toString());
}
}
return out;
}
/** Null when the result is not something we pair with. */
private static JSONObject toJson(ScanResult result) {
BluetoothDevice device = result.getDevice();
if (device == null) {
return null;
}
ScanRecord record = result.getScanRecord();
String liveName = record == null ? null : record.getDeviceName();
String name = liveName != null && !liveName.isEmpty() ? liveName : device.getName();
byte[] manufacturerData = extractEvenManufacturerData(record);
if (!isAdmissible(name, manufacturerData)) {
return null;
}
Integer txPower = null;
if (record != null && record.getTxPowerLevel() != Integer.MIN_VALUE) {
txPower = record.getTxPowerLevel();
}
if (txPower == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O
&& result.getTxPower() != ScanResult.TX_POWER_NOT_PRESENT) {
txPower = result.getTxPower();
}
Boolean connectable = null;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
connectable = result.isConnectable();
}
int rssi = result.getRssi();
// 127 is the "unavailable" sentinel; 0 dBm never happens for a real sample.
Integer rssiValue = (rssi == 127 || rssi == 0) ? null : rssi;
boolean bonded = device.getBondState() == BluetoothDevice.BOND_BONDED;
return buildJson(
device.getAddress(), name, FaceclawFirmwareUtil.bytesToHex(manufacturerData), rssiValue, txPower,
connectable, bonded, "scan", System.currentTimeMillis());
}
/**
* Android strips the 2-byte company identifier from each manufacturer
* record and keys the SparseArray by it. Put the bytes back (little-endian)
* so TypeScript sees the same "ER"+… layout the HCI captures
* document. Prefers Even's own id; otherwise returns the first record so an
* unexpected company id still reaches the diagnostics log.
*/
private static byte[] extractEvenManufacturerData(ScanRecord record) {
if (record == null) {
return new byte[0];
}
SparseArray<byte[]> all = record.getManufacturerSpecificData();
if (all == null || all.size() == 0) {
return new byte[0];
}
byte[] payload = all.get(EVEN_COMPANY_ID);
int companyId = EVEN_COMPANY_ID;
if (payload == null) {
companyId = all.keyAt(0);
payload = all.valueAt(0);
}
if (payload == null) {
return new byte[0];
}
byte[] out = new byte[payload.length + 2];
out[0] = (byte) (companyId & 0xff);
out[1] = (byte) ((companyId >> 8) & 0xff);
System.arraycopy(payload, 0, out, 2, payload.length);
return out;
}
private static boolean isAdmissible(String name, byte[] manufacturerData) {
if (manufacturerData != null && manufacturerData.length >= 2
&& manufacturerData[0] == 0x45 && manufacturerData[1] == 0x52) {
return true;
}
if (name == null) {
return false;
}
String upper = name.toUpperCase();
// A bare "R1" substring also matches unrelated hardware ("Oppo Enco
// R1"), and TS only classifies rings from the stock name prefix.
return upper.contains("G2") || upper.startsWith("EVEN R1");
}
private static JSONObject buildJson(
String address, String name, String manufacturerHex, Integer rssi, Integer txPower,
Boolean connectable, boolean bonded, String source, long seenAtMs) {
if (address == null || address.isEmpty()) {
return null;
}
try {
JSONObject object = new JSONObject();
object.put("address", address);
object.put("name", name == null ? "" : name);
object.put("manufacturerData", manufacturerHex == null ? "" : manufacturerHex);
object.put("rssi", rssi == null ? JSONObject.NULL : rssi);
object.put("txPower", txPower == null ? JSONObject.NULL : txPower);
object.put("connectable", connectable == null ? JSONObject.NULL : connectable);
object.put("bonded", bonded);
object.put("source", source);
object.put("seenAtMs", seenAtMs);
return object;
} catch (JSONException e) {
return null;
}
}
private static String describeScanError(int errorCode) {
switch (errorCode) {
case ScanCallback.SCAN_FAILED_ALREADY_STARTED:
return "a scan is already running";
case ScanCallback.SCAN_FAILED_APPLICATION_REGISTRATION_FAILED:
return "the Bluetooth stack refused to register the scan (try toggling Bluetooth)";
case ScanCallback.SCAN_FAILED_FEATURE_UNSUPPORTED:
return "BLE scanning is not supported on this device";
case ScanCallback.SCAN_FAILED_INTERNAL_ERROR:
return "internal Bluetooth error";
case 5: // SCAN_FAILED_OUT_OF_HARDWARE_RESOURCES (API 31)
return "out of Bluetooth hardware resources";
case 6: // SCAN_FAILED_SCANNING_TOO_FREQUENTLY (API 31)
return "scanning too frequently; wait a moment and try again";
default:
return "scan failed (code " + errorCode + ")";
}
}
private void emitLog(String line) {
FaceclawDeviceDiscoveryListener l = listener;
if (l != null) {
l.onLog(line);
}
}
}
@@ -0,0 +1,18 @@
package com.faceclaw.app;
/** Callbacks from FaceclawDeviceDiscovery's live scan to the TypeScript layer. */
public interface FaceclawDeviceDiscoveryListener {
/**
* One advertisement (or one bonded device) as a JSON object:
* {address, name, manufacturerData (hex, company id included), rssi|null,
* txPower|null, connectable|null, bonded, source, seenAtMs}.
* Protocol decoding happens in TypeScript (see app/g2/even-advertisement.ts).
*/
void onAdvertisement(String json);
/** The platform refused to start or continue the scan. */
void onScanFailed(int errorCode, String message);
/** Diagnostic line for the pairing log. */
void onLog(String line);
}
@@ -23,6 +23,7 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
private final Context context;
private final String rightAddress;
private final String leftAddress;
private final FaceclawBleManager bleManager;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
@@ -35,13 +36,21 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
private byte[] awaitPb = null;
private CountDownLatch awaitLatch = null;
/**
* A sid-0x09 frame that carried firmware versions without matching the
* awaited magic — some firmware pushes the settings snapshot on a magic it
* picked itself instead of (or in addition to) acking the read.
*/
private volatile byte[] unsolicitedSettingsPb;
private volatile FaceclawDeviceInfoProbeListener listener;
private volatile Thread worker;
private volatile boolean cancelled = false;
public FaceclawDeviceInfoProbe(Context context, String rightAddress) {
public FaceclawDeviceInfoProbe(Context context, String rightAddress, String leftAddress) {
this.context = context.getApplicationContext();
this.rightAddress = rightAddress == null ? "" : rightAddress;
this.leftAddress = leftAddress == null ? "" : leftAddress;
this.bleManager = new FaceclawBleManager(this.context);
this.bleManager.setListener(this);
}
@@ -83,27 +92,28 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
return;
}
emitState("connecting", "");
connectArm(rightAddress);
byte[] ack = probeArm(rightAddress, "right");
// The right lens is the documented control endpoint, but a silent
// right lens has been observed on stock 2.2.9 even after a
// successful security auth — run the same probe against the left
// lens rather than giving up, and log which lens answered.
if (ack == null && !cancelled && !leftAddress.trim().isEmpty()
&& !leftAddress.equalsIgnoreCase(rightAddress)) {
emitLog("right lens did not answer the settings query; probing the left lens");
try {
ack = probeArm(leftAddress, "left");
} catch (Exception e) {
emitLog("left-lens probe failed: " + (e.getMessage() == null ? e.toString() : e.getMessage()));
}
}
if (cancelled) {
emitError("Cancelled.");
return;
}
emitState("querying", "");
// Session prelude, then a settings/device-info read (both arms'
// versions and the CFW capability string ride back in one response).
if (writeAndAwaitAck(rightAddress, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw new IllegalStateException("session prelude not acked");
}
int magic = allocMagic();
byte[] ack = writeAndAwaitAck(rightAddress, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST,
magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS);
if (ack == null) {
throw new IllegalStateException("no response to the device-info query");
throw new IllegalStateException(
"no response to the device-info query on either lens — check `adb logcat -s "
+ TAG + "` for the frame trace");
}
BleProtocol.FirmwareInfo info = BleProtocol.parseSettingsFirmwareInfo(ack);
@@ -123,6 +133,73 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
}
}
/**
* Connect + authenticate + prelude + settings read on one lens. Returns the
* settings ack protobuf, an unsolicited settings push that carried firmware
* versions, or null when the lens never answered the read. Throws on
* connect or prelude failure; the caller wraps the fallback lens's attempt
* so its failure cannot mask the primary lens's outcome.
*/
private byte[] probeArm(String address, String label) throws InterruptedException {
emitState("connecting", label);
connectArm(address);
if (cancelled) {
return null;
}
// Firmware 2.2.9 answers no queries until the security-auth exchange
// completes over an encrypted link; on a phone with no existing bond
// this is also what triggers SMP pairing (and its OS prompt), so it
// must come before the prelude and query. Soft: this probe also runs
// against the custom firmware, whose response to the exchange is not
// yet hardware-verified, so an unconfirmed auth falls through.
emitState("authenticating", label);
boolean authenticated = authenticate(address);
if (cancelled) {
return null;
}
emitState("querying", label);
// Session prelude, then a settings/device-info read (both arms'
// versions and the CFW capability string ride back in one response).
if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw new IllegalStateException("session prelude not acked (" + label + " lens)"
+ (authenticated ? "" : "; authentication did not complete —"
+ " if Android shows a Bluetooth pairing request, accept it and try again"));
}
// Two attempts: the first read straight after a fresh pairing has been
// seen to go unanswered while a later one succeeds.
for (int attempt = 0; attempt < 2 && !cancelled; attempt++) {
int magic = allocMagic();
byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST,
magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS);
if (ack != null) {
return ack;
}
// A push with the firmware versions on the device's own magic is
// as good as the ack we asked for.
byte[] pushed = unsolicitedSettingsPb;
if (pushed != null) {
emitLog("using unsolicited settings push instead of the read ack (" + label + " lens)");
return pushed;
}
emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)");
}
return null;
}
private boolean authenticate(String address) throws InterruptedException {
int magic = allocMagic();
byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH,
magic, BleProtocol.buildAuthenticationRequest(magic), ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS);
boolean success = ack != null && BleProtocol.isAuthenticationSuccess(ack, magic);
emitLog(success ? "security auth complete: " + address : "security auth unconfirmed: " + address);
return success;
}
private void connectArm(String address) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw new IllegalStateException("connect failed");
@@ -150,10 +227,13 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
synchronized (lock) {
seq = nextSeq++ & 0xff;
}
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
address, sid, flag, magic, seq, payload.length));
List<byte[]> frames = BleProtocol.framePb(payload, sid, flag, seq);
boolean written = bleManager.writeFrames(
address, BleProtocol.WRITE_CHAR_UUID, frames, ConnectionOptions.WRITE_TYPE, ConnectionOptions.WRITE_TIMEOUT_MS);
if (!written) {
emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid));
synchronized (lock) {
awaitLatch = null;
}
@@ -171,10 +251,38 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) {
return;
}
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
// One notification value can carry several envelope frames back to
// back; reading only the first would silently drop the rest.
List<byte[]> frames = BleProtocol.splitFrames(data);
if (frames.size() > 1) {
emitLog("rx " + address + " value carries " + frames.size() + " frames (raw " + data.length + " bytes)");
}
for (byte[] buf : frames) {
handleFrame(address, buf, data.length);
}
}
private void handleFrame(String address, byte[] buf, int rawValueLength) {
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf);
if (!frame.ok) {
emitLog("rx " + address + " unparseable frame len=" + buf.length + " (raw value " + rawValueLength + ")"
+ " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.length))));
return;
}
// Log every control frame while diagnosing 2.2.9: sid/flag/type/magic
// plus a payload prefix is enough to reconstruct what the lens said.
int declared = buf.length > 3 ? buf[3] & 0xff : 0;
String truncated = buf.length < 8 + declared ? " TRUNCATED(declared=" + declared + ")" : "";
emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s",
address, frame.sid, frame.flag, frame.msgType, frame.msgSeq,
buf.length > 5 ? buf[5] & 0xff : 0, buf.length > 4 ? buf[4] & 0xff : 0, frame.pb.length, truncated,
FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.length)))));
if (frame.sid == BleProtocol.SID_UI_SETTING
&& BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) {
// Any settings frame carrying firmware versions answers the probe's
// question, whether or not it matches the magic we asked with.
unsolicitedSettingsPb = frame.pb;
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
return; // async event, not an ack
}
@@ -0,0 +1,262 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* Offline re-diarization of a stored conversation recording (16-bit mono
* WAV): pyannote segmentation plus the shared speaker-embedding model with
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
* TS as JSON turns for reconciliation against the transcript segments.
*/
public class FaceclawDiarizer {
private static final String TAG = "FaceclawDiarizer";
public interface Listener {
/** progress in [0, 1]. */
void onProgress(double progress);
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
void onDone(String turnsJson);
void onError(String message);
}
private final String segmentationModelPath;
private final String embeddingModelPath;
private final String wavPath;
private final int numClusters;
private final float threshold;
private final Listener listener;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private volatile boolean cancelled;
public FaceclawDiarizer(
String segmentationModelPath,
String embeddingModelPath,
String wavPath,
int numClusters,
float threshold,
Listener listener) {
this.segmentationModelPath = segmentationModelPath;
this.embeddingModelPath = embeddingModelPath;
this.wavPath = wavPath;
this.numClusters = numClusters;
this.threshold = threshold <= 0 ? 0.5f : threshold;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawDiarizer");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
public void cancel() {
cancelled = true;
}
private void run() {
OfflineSpeakerDiarization diarization = null;
try {
float[] samples = wavPath.endsWith(".wav")
? readWavMono16(wavPath)
: decodeCompressedMono(wavPath);
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
.setModel(segmentationModelPath)
.build())
.setNumThreads(1)
.build())
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
.setModel(embeddingModelPath)
.setNumThreads(1)
.build())
.setClustering(FastClusteringConfig.builder()
.setNumClusters(numClusters > 0 ? numClusters : -1)
.setThreshold(threshold)
.build())
.setMinDurationOn(0.3f)
.setMinDurationOff(0.5f)
.build();
diarization = new OfflineSpeakerDiarization(config);
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
samples,
(numProcessedChunks, numTotalChunks, arg) -> {
emitProgress(numTotalChunks <= 0 ? 0
: (double) numProcessedChunks / numTotalChunks);
return cancelled ? 1 : 0;
});
JSONArray turns = new JSONArray();
if (segments != null) {
for (OfflineSpeakerDiarizationSegment segment : segments) {
JSONObject turn = new JSONObject();
turn.put("startMs", Math.round(segment.getStart() * 1000));
turn.put("endMs", Math.round(segment.getEnd() * 1000));
turn.put("cluster", segment.getSpeaker());
turns.put(turn);
}
}
String result = turns.toString();
mainHandler.post(() -> listener.onDone(result));
} catch (Throwable t) {
Log.w(TAG, "diarization failed for " + wavPath, t);
String message = t.getMessage() == null ? t.toString() : t.getMessage();
mainHandler.post(() -> listener.onError(message));
} finally {
if (diarization != null) {
diarization.release();
}
}
}
private void emitProgress(double progress) {
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
}
/**
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
* as 16 kHz mono, so no resampling is needed.
*/
private static float[] decodeCompressedMono(String path) throws IOException {
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
android.media.MediaCodec codec = null;
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
try {
extractor.setDataSource(path);
int trackIndex = -1;
android.media.MediaFormat format = null;
for (int i = 0; i < extractor.getTrackCount(); i++) {
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith("audio/")) {
trackIndex = i;
format = candidate;
break;
}
}
if (trackIndex < 0 || format == null) {
throw new IOException("no audio track in " + path);
}
extractor.selectTrack(trackIndex);
codec = android.media.MediaCodec.createDecoderByType(
format.getString(android.media.MediaFormat.KEY_MIME));
codec.configure(format, null, null, 0);
codec.start();
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
boolean inputDone = false;
boolean outputDone = false;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(10_000);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int size = extractor.readSampleData(inBuf, 0);
if (size < 0) {
codec.queueInputBuffer(inIndex, 0, 0, 0,
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
extractor.advance();
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
if (outIndex >= 0) {
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
byte[] chunk = new byte[info.size];
outBuf.position(info.offset);
outBuf.get(chunk);
pcm.write(chunk);
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
extractor.release();
if (codec != null) {
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
}
}
byte[] bytes = pcm.toByteArray();
int count = bytes.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
}
return samples;
}
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
private static float[] readWavMono16(String path) throws IOException {
File file = new File(path);
long fileLength = file.length();
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
byte[] header = new byte[44];
in.readFully(header);
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
throw new IOException("not a WAV file");
}
int channels = (header[22] & 0xff) | (header[23] << 8);
int bits = (header[34] & 0xff) | (header[35] << 8);
if (channels != 1 || bits != 16) {
throw new IOException("only 16-bit mono WAV is supported");
}
long dataBytes = fileLength - 44;
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
float[] samples = new float[count];
byte[] chunk = new byte[65536];
int sampleIndex = 0;
int carried = 0;
byte carriedByte = 0;
int read;
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
int offset = 0;
if (carried == 1) {
samples[sampleIndex++] =
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
offset = 1;
carried = 0;
}
int pairs = (read - offset) / 2;
for (int i = 0; i < pairs && sampleIndex < count; i++) {
int base = offset + i * 2;
samples[sampleIndex++] =
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
}
if (((read - offset) & 1) == 1) {
carried = 1;
carriedByte = chunk[read - 1];
}
}
return samples;
}
}
}
@@ -14,17 +14,23 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA
* firmware container to each lens over the firmware-data service using the
* aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol, with
* a heartbeat on the EvenHub control char to keep the session alive.
* aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol.
*
* Each lens connection completes the sid-0x80 security-auth exchange before
* OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing
* the transfer mid-component. No control-channel traffic is injected during
* the transfer itself: official OTA captures show none between BEGIN and the
* final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a
* concurrent control writer would interleave with the marker/data
* transactions. (See ../notes/ble-connections-2.2.9.md.)
*
* Lenses are flashed one at a time (left, then right). Finishing either lens
* reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect
@@ -40,7 +46,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
// OTA message types (envelope sid byte) and control opcodes.
private static final int SID_CTRL = 0xc0;
private static final int SID_DATA = 0xc1;
private static final int SID_HEARTBEAT = 0x80;
private static final int FLAG_OTA = 0x00;
private static final int OP_BEGIN = 0x00;
private static final int OP_FILE_CHECK = 0x01;
@@ -52,9 +57,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
private static final int CTRL_ACK_TIMEOUT_MS = 8_000;
private static final int BLOCK_NAK_RETRIES = 3;
private static final int COMPONENT_RETRIES = 3;
private static final int HEARTBEAT_INTERVAL_MS = 12_000;
private static final byte[] HEARTBEAT_PAYLOAD =
new byte[] {0x08, 0x0e, 0x10, 0x26, 0x6a, 0x00};
// Reconnect windows. The first lens connects from an idle device; the second
// must ride out the post-first-lens reboot of both lenses.
@@ -89,10 +91,12 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
private volatile FaceclawFirmwareFlasherListener listener;
private volatile Thread worker;
private volatile boolean cancelled = false;
private volatile ScheduledExecutorService heartbeatExecutor;
private volatile String heartbeatAddress;
private int nextSeq = 0;
private int nextAuthMagic = 0x60;
private int pendingAuthMagic = -1;
private CountDownLatch authLatch;
private byte[] authAckPb;
public FaceclawFirmwareFlasher(Context context, String rightAddress, String leftAddress, String firmwarePath) {
this.context = context.getApplicationContext();
@@ -127,7 +131,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
public void close() {
cancel();
stopHeartbeat();
try {
bleManager.close();
} catch (Exception ignored) {
@@ -172,27 +175,34 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
private void flashLens(String lens, String address, byte[] img, List<Segment> segs, int connectWindowMs)
throws TimeoutException {
resetSeq();
emitState("connecting", lens);
connectLensResilient(lens, address, connectWindowMs);
sleepInterruptibly(NOTIFY_SETTLE_MS);
drainAcks();
// Fresh envelope counter for the OTA stream (the auth exchange during
// bring-up used its own); mirrors the stock sequence.
resetSeq();
emitState("flashing", lens);
startHeartbeat(address);
try {
int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS);
if (!isEndOk(beginStatus)) {
emitLog("warning: unexpected begin status " + beginStatus + "; continuing");
int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS);
if (!isEndOk(beginStatus)) {
emitLog("warning: unexpected begin status " + beginStatus + "; continuing");
}
// Progress is reported in bytes across the whole image: the segments
// are one large firmware blob plus several small ones, so a bar
// stepped per segment would sit still through the big one and then
// sprint through the rest.
long totalBytes = 0;
for (Segment seg : segs) {
totalBytes += seg.ps;
}
long bytesBefore = 0;
for (int i = 0; i < segs.size(); i++) {
if (cancelled) {
throw new IllegalStateException("Cancelled.");
}
for (int i = 0; i < segs.size(); i++) {
if (cancelled) {
throw new IllegalStateException("Cancelled.");
}
flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img);
}
} finally {
stopHeartbeat();
flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img, bytesBefore, totalBytes);
bytesBefore += segs.get(i).ps;
}
emitLog(lens + " lens: all components verified");
try {
@@ -201,14 +211,15 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
}
}
private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img) {
private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img,
long bytesBefore, long totalBytes) {
for (int attempt = 0; attempt < COMPONENT_RETRIES; attempt++) {
if (attempt > 0) {
emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES);
}
int endStatus;
try {
endStatus = flashComponent(lens, address, index, count, seg, img);
endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes);
} catch (TimeoutException | RuntimeException e) {
emitLog(seg.name + ": block phase failed: " + e.getMessage());
endStatus = -1;
@@ -226,8 +237,8 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
throw new IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts");
}
private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img)
throws TimeoutException {
private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img,
long bytesBefore, long totalBytes) throws TimeoutException {
byte[] sub = Arrays.copyOfRange(img, seg.off, seg.off + 128);
int ps = seg.ps;
int payloadStart = seg.off + 128;
@@ -260,7 +271,8 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
throw new IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times");
}
if (b % 20 == 0 || b == blockCount - 1) {
emitProgress(lens, index + 1, count, b + 1, blockCount);
long bytesSent = bytesBefore + Math.min((long) (b + 1) * BLOCK_SIZE, (long) ps);
emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes);
}
}
emitLog(seg.name + ": data phase done; sending END");
@@ -356,50 +368,64 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
return false;
}
// Control-channel notify carries the auth response; mandatory.
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
return false;
}
if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
return false;
}
// Control-channel notify (heartbeat responses); best-effort.
bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS);
if (!authenticate(address)) {
throw new IllegalStateException(
"security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it");
}
return true;
}
// ---- heartbeat -----------------------------------------------------------
private void startHeartbeat(String address) {
stopHeartbeat();
heartbeatAddress = address;
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(runnable -> {
Thread t = new Thread(runnable, "faceclaw-flash-hb");
t.setDaemon(true);
return t;
});
heartbeatExecutor = executor;
executor.scheduleWithFixedDelay(
this::sendHeartbeat, HEARTBEAT_INTERVAL_MS, HEARTBEAT_INTERVAL_MS, TimeUnit.MILLISECONDS);
}
private void sendHeartbeat() {
if (cancelled) {
return;
}
String address = heartbeatAddress;
if (address == null) {
return;
/**
* Complete the sid-0x80 security-auth exchange on this connection. On an
* unbonded phone this triggers SMP pairing (possibly with an OS prompt),
* so the wait is generous. Must succeed before any OTA traffic.
*/
private boolean authenticate(String address) {
int magic;
CountDownLatch latch = new CountDownLatch(1);
synchronized (lock) {
magic = nextAuthMagic;
nextAuthMagic = nextAuthMagic >= 0x7f ? 0x60 : nextAuthMagic + 1;
pendingAuthMagic = magic;
authAckPb = null;
authLatch = latch;
}
try {
writeOta(address, BleProtocol.WRITE_CHAR_UUID, SID_HEARTBEAT, HEARTBEAT_PAYLOAD, nextSeq());
} catch (Exception e) {
Log.w(TAG, "heartbeat failed: " + e.getMessage());
}
}
private void stopHeartbeat() {
ScheduledExecutorService executor = heartbeatExecutor;
heartbeatExecutor = null;
heartbeatAddress = null;
if (executor != null) {
executor.shutdownNow();
if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH,
BleProtocol.buildAuthenticationRequest(magic), nextSeq())) {
emitLog("auth write failed: " + address);
return false;
}
boolean signalled;
try {
signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
byte[] pb;
synchronized (lock) {
pb = authAckPb;
}
if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) {
emitLog("security auth not acknowledged (" + address
+ ") — if Android shows a Bluetooth pairing request, accept it");
return false;
}
emitLog("security auth complete: " + address);
return true;
} finally {
synchronized (lock) {
pendingAuthMagic = -1;
authLatch = null;
}
}
}
@@ -497,8 +523,25 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
@Override
public void onNotification(String address, String characteristicUuid, byte[] data) {
if (BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) {
// Control channel: only the security-auth response is awaited here.
// A value can carry several envelope frames back to back (e.g. the
// response packed with one of the lens's own sid-0x80 notifies).
for (byte[] buf : BleProtocol.splitFrames(data)) {
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf);
if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) {
synchronized (lock) {
if (authLatch != null && frame.msgSeq == pendingAuthMagic) {
authAckPb = frame.pb;
authLatch.countDown();
}
}
}
}
return;
}
if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equalsIgnoreCase(characteristicUuid)) {
return; // acks arrive on the data-notify char; ignore heartbeat responses
return; // OTA acks arrive on the data-notify char
}
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
if (!frame.ok || frame.pb.length < 2) {
@@ -517,7 +560,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
// ---- helpers -------------------------------------------------------------
private void teardown() {
stopHeartbeat();
try {
bleManager.close();
} catch (Exception ignored) {
@@ -621,11 +663,12 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
});
}
private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount) {
private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount,
long bytesSent, long bytesTotal) {
mainHandler.post(() -> {
FaceclawFirmwareFlasherListener current = listener;
if (current != null) {
current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount);
current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal);
}
});
}
@@ -6,9 +6,12 @@ public interface FaceclawFirmwareFlasherListener {
/**
* Fine-grained progress for the UI bar. `lens` is "left"/"right";
* component/block indices are 1-based against their counts.
* component/block indices are 1-based against their counts. `bytesSent` /
* `bytesTotal` cover the whole lens image, so a bar driven by them moves
* evenly even though the components differ wildly in size.
*/
void onProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount);
void onProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount,
long bytesSent, long bytesTotal);
/**
* Coarse lifecycle. `state` is one of: validating, connecting, flashing,
@@ -11,24 +11,32 @@ import java.security.NoSuchAlgorithmException;
* Native helpers for the on-device custom-firmware build: SHA-256 hashing and
* binary file writing. Both take a java.nio.ByteBuffer so NativeScript can
* marshal a JS ArrayBuffer straight across without a per-element copy of the
* ~4 MB firmware image (and with no overload ambiguity).
* ~4 MB firmware image (and with no overload ambiguity). Also home of the one
* shared byte→hex encoder for com.faceclaw.app.
*/
public final class FaceclawFirmwareUtil {
private FaceclawFirmwareUtil() {}
/** Lowercase hex of the bytes. The TS-side decoder is case-insensitive. */
public static String bytesToHex(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return "";
}
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
int v = b & 0xff;
sb.append(Character.forDigit(v >>> 4, 16));
sb.append(Character.forDigit(v & 0x0f, 16));
}
return sb.toString();
}
/** Lowercase hex SHA-256 of the buffer's remaining bytes. */
public static String sha256Hex(ByteBuffer data) {
try {
MessageDigest md = MessageDigest.getInstance("SHA-256");
md.update(data.duplicate());
byte[] digest = md.digest();
StringBuilder sb = new StringBuilder(digest.length * 2);
for (byte b : digest) {
int v = b & 0xff;
sb.append(Character.forDigit(v >>> 4, 16));
sb.append(Character.forDigit(v & 0x0f, 16));
}
return sb.toString();
return bytesToHex(md.digest());
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException("SHA-256 unavailable", e);
}
@@ -26,6 +26,10 @@ import java.util.concurrent.TimeUnit;
* protobuf helpers (BleProtocol), but owns its own GATT connection and runs its
* whole lifecycle on a single worker thread. It expects the main app to be
* disconnected while it runs (onboarding, before flashing).
*
* Only the right arm is used: the lenses relay messages to each other, so
* anything sent to the right arm reaches both, and acks/events only ever come
* from the right arm anyway.
*/
public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private static final String TAG = "FaceclawFlashPrompt";
@@ -43,13 +47,13 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private final Context context;
private final String rightAddress;
private final String leftAddress;
private final String warningText;
private final FaceclawBleManager bleManager;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private final ConcurrentHashMap<String, CountDownLatch> pendingAcks = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, byte[]> ackPayloads = new ConcurrentHashMap<>();
private final CountDownLatch selectionLatch = new CountDownLatch(1);
private volatile FaceclawFlashPromptListener listener;
@@ -57,7 +61,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private volatile boolean cancelled = false;
private volatile boolean finished = false;
private volatile boolean rightConnected = false;
private volatile boolean leftConnected = false;
private volatile boolean rightLost = false;
private volatile Boolean approved = null;
private volatile ScheduledExecutorService heartbeatExecutor;
@@ -65,10 +68,9 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
private int nextMagic = 100;
private int nextSeq = 0x40;
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String leftAddress, String warningText) {
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String warningText) {
this.context = context.getApplicationContext();
this.rightAddress = rightAddress == null ? "" : rightAddress;
this.leftAddress = leftAddress == null ? "" : leftAddress;
this.warningText = warningText == null ? "" : warningText;
this.bleManager = new FaceclawBleManager(this.context);
this.bleManager.setListener(this);
@@ -115,18 +117,8 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
}
emitState("connecting", "");
connectArm(rightAddress, "right");
connectArm(rightAddress);
rightConnected = true;
boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress);
if (haveLeft) {
try {
connectArm(leftAddress, "left");
leftConnected = true;
} catch (Exception e) {
emitLog("left arm connect failed (continuing on right only): " + e.getMessage());
}
}
if (cancelled) {
teardown();
return;
@@ -134,14 +126,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
emitState("connected", "");
bringUpArm(rightAddress);
if (leftConnected) {
try {
bringUpArm(leftAddress);
} catch (Exception e) {
emitLog("left arm prompt failed (continuing on right only): " + e.getMessage());
leftConnected = false;
}
}
startHeartbeat();
emitState("prompting", "");
@@ -157,7 +141,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (approved != null) {
boolean result = Boolean.TRUE.equals(approved);
try {
sendShutdownBoth();
sendShutdown();
} catch (Exception ignored) {
}
emitResult(result);
@@ -181,7 +165,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
}
}
private void connectArm(String address, String labelForLog) {
private void connectArm(String address) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw new IllegalStateException("connect failed: " + address);
}
@@ -193,42 +177,75 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw new IllegalStateException("enableNotifications failed: " + address);
}
emitLog("connected " + labelForLog + " arm");
emitLog("connected right arm");
}
private void bringUpArm(String address) throws InterruptedException {
// 1. Mandatory session prelude on sid=0x01 (app-launch).
if (!writeAndAwaitAck(
// 1. Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a
// session (and closes the link after ~30 s) without it, and on an
// unbonded phone this is what triggers SMP pairing, so it may sit
// waiting on an OS pairing prompt.
int authMagic = allocMagic();
byte[] authAck = writeAndAwaitAck(
address,
BleProtocol.SID_SECURITY_AUTH,
BleProtocol.FLAG_SECURITY_AUTH,
authMagic,
BleProtocol.buildAuthenticationRequest(authMagic),
ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS);
if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) {
throw new IllegalStateException(
"could not authenticate with the glasses (" + address
+ ") — if Android shows a Bluetooth pairing request, accept it and try again");
}
// 2. Mandatory session prelude on sid=0x01 (app-launch).
if (writeAndAwaitAck(
address,
BleProtocol.PRELUDE_ACK_SID,
BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC,
BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS)) {
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw new IllegalStateException("session prelude not acked: " + address);
}
// 2. Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0.
int magic = allocMagic();
byte[] page = BleProtocol.buildCreatePromptPage(
magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS);
if (!writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS)) {
// 3. Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0.
// Two attempts: on 2.2.9 the first request sent right after the
// prelude can be dropped while the lens emits its own sid-0x80
// notifications (observed with the device-info settings read).
byte[] pageAck = null;
for (int attempt = 0; attempt < 2 && pageAck == null && !cancelled; attempt++) {
int magic = allocMagic();
byte[] page = BleProtocol.buildCreatePromptPage(
magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS);
pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS);
if (pageAck == null) {
emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address);
}
}
if (pageAck == null) {
throw new IllegalStateException("prompt page not acked: " + address);
}
emitLog("prompt page shown on " + address);
}
private boolean writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs)
/** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */
private byte[] writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs)
throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
String key = ackKey(sid, magic);
pendingAcks.put(key, latch);
try {
if (!writeFrame(address, sid, flag, payload)) {
return false;
return null;
}
return latch.await(timeoutMs, TimeUnit.MILLISECONDS);
if (!latch.await(timeoutMs, TimeUnit.MILLISECONDS)) {
return null;
}
byte[] pb = ackPayloads.get(key);
return pb == null ? new byte[0] : pb;
} finally {
pendingAcks.remove(key, latch);
ackPayloads.remove(key);
}
}
@@ -267,9 +284,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
}
if (leftConnected) {
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
}
} catch (Exception e) {
Log.w(TAG, "heartbeat failed: " + e.getMessage());
}
@@ -283,14 +297,11 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
}
}
private void sendShutdownBoth() {
private void sendShutdown() {
byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0);
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
}
if (leftConnected) {
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
}
}
private void teardown() {
@@ -301,7 +312,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
} catch (Exception ignored) {
}
rightConnected = false;
leftConnected = false;
}
@Override
@@ -309,20 +319,26 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) {
return;
}
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
// A value can carry several envelope frames back to back.
for (byte[] buf : BleProtocol.splitFrames(data)) {
handleFrame(address, buf);
}
}
private void handleFrame(String address, byte[] buf) {
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf);
if (!frame.ok) {
return;
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
// Async events (taps/selections) only ever come from the right arm.
if (address.equalsIgnoreCase(rightAddress)) {
handleEvent(frame);
}
handleEvent(frame);
return;
}
if (frame.msgSeq >= 0) {
CountDownLatch latch = pendingAcks.get(ackKey(frame.sid, frame.msgSeq));
String key = ackKey(frame.sid, frame.msgSeq);
CountDownLatch latch = pendingAcks.get(key);
if (latch != null) {
ackPayloads.put(key, frame.pb);
latch.countDown();
}
}
@@ -364,8 +380,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
selectionLatch.countDown();
}
}
} else if (address.equalsIgnoreCase(leftAddress)) {
leftConnected = false;
}
}
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
public static final int COUNTER_OFFSET = 204;
public static final int SAMPLES_PER_FRAME = 160;
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
public static final int SSR_OFFSET = 200;
public static final int ANGLE_OFFSET = 202;
static {
System.loadLibrary("faceclaw_lc3");
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
private long duplicatePackets;
private long missingPackets;
private long decodeErrors;
private int lastSsr;
private int lastAngleDegrees;
public FaceclawLc3Decoder() {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
}
realPackets++;
lastCounter = counter;
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
return decoded;
}
/** Firmware signal-strength ratio from the most recent decoded packet. */
public synchronized int getLastSsr() {
return lastSsr;
}
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
public synchronized int getLastAngleDegrees() {
return lastAngleDegrees;
}
public synchronized long getRealPackets() {
return realPackets;
}
@@ -0,0 +1,10 @@
package com.faceclaw.app;
public interface FaceclawMicStatusListener {
/**
* A CFW mic_control field-104 status record from one temple. body is the
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
*/
void onMicStatus(byte[] body, String arm);
}
@@ -144,7 +144,7 @@ public class FaceclawModelDownloader {
+ expectedTotalBytes + " bytes); try again to resume");
return;
}
String actualSha = toHex(digest.digest());
String actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest());
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
partFile.delete();
postError("Model download was corrupted (checksum mismatch); download it again");
@@ -177,12 +177,6 @@ public class FaceclawModelDownloader {
return hashed;
}
private static String toHex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) sb.append(String.format("%02x", b));
return sb.toString();
}
private void post(Runnable runnable) {
callbackHandler.post(runnable);
}
@@ -0,0 +1,266 @@
package com.faceclaw.app;
import android.media.audiofx.AcousticEchoCanceler;
import android.media.audiofx.AudioEffect;
import android.media.audiofx.AutomaticGainControl;
import android.media.audiofx.NoiseSuppressor;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Phone-side noise suppression for the microphone stream.
*
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
* arrives as raw PCM over BLE from the glasses, which that effect chain never
* sees, so this class picks the best engine that can actually reach the
* samples:
*
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
* whatever the Android build ships) on the AudioRecord session.
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
* over-subtracted Wiener gain with a residual floor so speech never pumps
* in and out of silence.
*/
public final class FaceclawNoiseSuppressor {
private static final int FFT_SIZE = 256;
private static final int HOP = FFT_SIZE / 2;
private static final float OVERSUBTRACTION = 1.6f;
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
private static final float GAIN_FLOOR = 0.15f;
private static final float GAIN_SMOOTHING = 0.6f;
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
private static final int NOISE_WARMUP_FRAMES = 12;
private final int sampleRate;
private final float[] window = new float[FFT_SIZE];
private final float[] history = new float[FFT_SIZE];
private final float[] overlap = new float[HOP];
private final float[] noise = new float[FFT_SIZE / 2 + 1];
private final float[] gain = new float[FFT_SIZE / 2 + 1];
private final float[] re = new float[FFT_SIZE];
private final float[] im = new float[FFT_SIZE];
private final short[] pending = new short[HOP];
private int pendingCount = 0;
private int warmupFrames = 0;
public FaceclawNoiseSuppressor(int sampleRate) {
this.sampleRate = sampleRate;
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
for (int i = 0; i < FFT_SIZE; i++) {
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
}
reset();
}
public int getSampleRate() {
return sampleRate;
}
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
public void reset() {
Arrays.fill(history, 0f);
Arrays.fill(overlap, 0f);
Arrays.fill(noise, 0f);
Arrays.fill(gain, 1f);
pendingCount = 0;
warmupFrames = 0;
}
/**
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
* filling a whole 8 ms hop is carried into the next call, so the output
* may be up to one hop shorter than the input (and longer on a later
* call); total latency is one hop.
*/
public byte[] process(byte[] pcm16le) {
if (pcm16le == null || pcm16le.length < 2) {
return pcm16le;
}
int sampleCount = pcm16le.length / 2;
int hops = (pendingCount + sampleCount) / HOP;
byte[] out = new byte[hops * HOP * 2];
int outPos = 0;
int inPos = 0;
while (inPos < sampleCount) {
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
for (int i = 0; i < take; i++) {
int lo = pcm16le[(inPos + i) * 2] & 0xff;
int hi = pcm16le[(inPos + i) * 2 + 1];
pending[pendingCount + i] = (short) ((hi << 8) | lo);
}
pendingCount += take;
inPos += take;
if (pendingCount == HOP) {
outPos = processHop(out, outPos);
pendingCount = 0;
}
}
return out;
}
private int processHop(byte[] out, int outPos) {
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
for (int i = 0; i < HOP; i++) {
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
}
for (int i = 0; i < FFT_SIZE; i++) {
re[i] = history[i] * window[i];
im[i] = 0f;
}
fft(re, im, false);
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
if (warming) {
warmupFrames++;
}
for (int b = 0; b <= FFT_SIZE / 2; b++) {
float power = re[b] * re[b] + im[b] * im[b];
if (warming) {
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
} else if (power < noise[b]) {
// Track the floor down quickly; drift up slowly so speech
// energy never gets learned as noise.
noise[b] += (power - noise[b]) * 0.2f;
} else {
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
}
float clean = power - OVERSUBTRACTION * noise[b];
float g = clean > 0f ? clean / power : 0f;
if (g < GAIN_FLOOR) {
g = GAIN_FLOOR;
}
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
gain[b] = g;
re[b] *= g;
im[b] *= g;
if (b > 0 && b < FFT_SIZE / 2) {
re[FFT_SIZE - b] *= g;
im[FFT_SIZE - b] *= g;
}
}
fft(re, im, true);
for (int i = 0; i < HOP; i++) {
float value = (overlap[i] + re[i] * window[i]) * 32767f;
int sample = Math.round(value);
if (sample > 32767) {
sample = 32767;
}
if (sample < -32768) {
sample = -32768;
}
out[outPos + i * 2] = (byte) (sample & 0xff);
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
}
for (int i = 0; i < HOP; i++) {
overlap[i] = re[HOP + i] * window[HOP + i];
}
return outPos + HOP * 2;
}
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
private static void fft(float[] re, float[] im, boolean inverse) {
int n = re.length;
for (int i = 1, j = 0; i < n; i++) {
int bit = n >> 1;
for (; (j & bit) != 0; bit >>= 1) {
j ^= bit;
}
j ^= bit;
if (i < j) {
float t = re[i];
re[i] = re[j];
re[j] = t;
t = im[i];
im[i] = im[j];
im[j] = t;
}
}
for (int len = 2; len <= n; len <<= 1) {
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
float wRe = (float) Math.cos(ang);
float wIm = (float) Math.sin(ang);
for (int i = 0; i < n; i += len) {
float curRe = 1f;
float curIm = 0f;
for (int k = 0; k < len / 2; k++) {
int a = i + k;
int b = a + len / 2;
float xr = re[b] * curRe - im[b] * curIm;
float xi = re[b] * curIm + im[b] * curRe;
re[b] = re[a] - xr;
im[b] = im[a] - xi;
re[a] += xr;
im[a] += xi;
float nextRe = curRe * wRe - curIm * wIm;
curIm = curRe * wIm + curIm * wRe;
curRe = nextRe;
}
}
}
if (inverse) {
for (int i = 0; i < n; i++) {
re[i] /= n;
im[i] /= n;
}
}
}
/** Whether this Android build ships the built-in capture noise suppressor. */
public static boolean platformSuppressorAvailable() {
try {
return NoiseSuppressor.isAvailable();
} catch (Throwable t) {
return false;
}
}
/** Short engine name for the UI. */
public static String engineDescription() {
return "spectral (phone DSP)";
}
/**
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
* build provides) to a phone AudioRecord session. Only applicable when the
* phone itself captures the audio; the BLE glasses stream must go through
* {@link #process(byte[])} instead. The caller keeps the returned effects
* for the life of the recording and release()s them with it.
*/
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
ArrayList<AudioEffect> effects = new ArrayList<>();
try {
if (NoiseSuppressor.isAvailable()) {
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
if (ns != null) {
ns.setEnabled(true);
effects.add(ns);
}
}
if (AcousticEchoCanceler.isAvailable()) {
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
if (aec != null) {
aec.setEnabled(true);
effects.add(aec);
}
}
if (AutomaticGainControl.isAvailable()) {
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
if (agc != null) {
agc.setEnabled(true);
effects.add(agc);
}
}
} catch (Throwable t) {
// A missing effect just means this tier is unavailable.
}
return effects.toArray(new AudioEffect[0]);
}
}
@@ -0,0 +1,111 @@
package com.faceclaw.app;
import android.util.Log;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
/**
* Speaker voice-print extraction over sherpa-onnx. One embedding per
* utterance; matching/enrollment happens on the TS side (cosine similarity
* against the stored speaker profiles).
*/
public class FaceclawSpeakerId {
private static final String TAG = "FaceclawSpeakerId";
private SpeakerEmbeddingExtractor extractor;
private final String modelPath;
public FaceclawSpeakerId(String modelPath) {
this.modelPath = modelPath;
}
/** Lazily loads the model; returns false when it cannot be loaded. */
public synchronized boolean ensureLoaded() {
if (extractor != null) {
return true;
}
try {
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
.setModel(modelPath)
.setNumThreads(1)
.setDebug(false)
.setProvider("cpu")
.build();
extractor = new SpeakerEmbeddingExtractor(config);
return true;
} catch (Throwable t) {
Log.w(TAG, "could not load speaker model at " + modelPath, t);
extractor = null;
return false;
}
}
public synchronized int embeddingDim() {
return extractor == null ? 0 : extractor.getDim();
}
/**
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
* S16LE PCM. Returns null when the model is unavailable or the utterance
* is too short to embed (under ~0.5 s).
*/
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
if (pcm16le == null || pcm16le.length < sampleRate) {
return null;
}
if (!ensureLoaded()) {
return null;
}
int count = pcm16le.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
samples[i] = s / 32768.0f;
}
OnlineStream stream = null;
try {
stream = extractor.createStream();
stream.acceptWaveform(samples, sampleRate);
stream.inputFinished();
if (!extractor.isReady(stream)) {
return null;
}
float[] embedding = extractor.compute(stream);
return l2Normalize(embedding);
} catch (Throwable t) {
Log.w(TAG, "embedding failed", t);
return null;
} finally {
if (stream != null) {
stream.release();
}
}
}
private static float[] l2Normalize(float[] v) {
if (v == null) {
return null;
}
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return v;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
return v;
}
public synchronized void close() {
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
@@ -1,30 +1,20 @@
package com.faceclaw.app;
import android.content.Context;
import android.content.res.AssetManager;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotter;
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.Arrays;
@@ -57,17 +47,15 @@ public class FaceclawVoiceController {
private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f;
private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f;
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
private static final String ASSET_ROOT = "faceclaw-voice";
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr";
// Model directory shared with the TS-side download flow (asr-model.ts),
// which fetches the Moonshine files here on demand; they are no longer
// bundled in the APK.
private static final String ASR_ROOT = "faceclaw-voice-asr";
private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27";
private static final String[] MODEL_FILES = {
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
"joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
"tokens.txt",
"screen-on-keywords.txt"
};
// Model files for the retired on-phone wake-word spotter, copied to
// filesDir by earlier releases; deleted on sight to reclaim the space.
// (The wakeword is now detected by the glasses firmware itself.)
private static final String LEGACY_KWS_ROOT = "faceclaw-voice";
private static final String[] ASR_MODEL_FILES = {
"encoder_model.ort",
"decoder_model_merged.ort",
@@ -75,7 +63,6 @@ public class FaceclawVoiceController {
};
private enum VoiceInputMode {
WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired)
ONBOARD, // on-phone Moonshine transcription
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
}
@@ -89,10 +76,11 @@ public class FaceclawVoiceController {
private volatile FaceclawBleCommunicator communicator;
private Thread workerThread;
private volatile boolean started;
private VoiceInputMode mode = VoiceInputMode.WAKEWORD;
private KeywordSpotter keywordSpotter;
// Set once the worker has the glasses mic enabled for this session.
// Read and written under `lock`, so it flips with `started` atomically.
private boolean audioStarted;
private VoiceInputMode mode = VoiceInputMode.CLOUD;
private OfflineRecognizer recognizer;
private OnlineStream stream;
private FaceclawLc3Decoder lc3Decoder;
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
private int transcriptSampleCount;
@@ -105,6 +93,26 @@ public class FaceclawVoiceController {
private volatile boolean endpointing;
private final EndpointDetector endpointDetector = new EndpointDetector();
private java.io.ByteArrayOutputStream recordingPcm;
// Speaker verification against the enrolled wearer voice-print ("my voice
// only" command gating). Configured before start(); the utterance PCM is
// buffered (capped) and verified once at session end.
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
private volatile String verifySpeakerModelPath;
private volatile float[] verifyWearerEmbedding;
private volatile float verifyThreshold = 0.8f;
private short[] verifyBuffer;
private int verifyCount;
// Global mic processing (Microphones app config): spectral noise
// suppression and firmware-DoA beam gating, applied to every capture
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
// The raw tap opts out — raw means raw, and the Microphones session does
// its own beam-compensated processing on that path.
private volatile boolean suppressionEnabled;
private volatile boolean beamFilterEnabled;
private volatile int beamCenterDeg;
private volatile int beamHalfWidthDeg = 180;
private FaceclawNoiseSuppressor suppressor;
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
@@ -140,8 +148,66 @@ public class FaceclawVoiceController {
this.endpointing = endpointing;
}
public void start() {
start("wakeword");
/**
* Verify this session's speaker against the enrolled wearer voice-print
* and report the result via onSpeakerVerified just before the final
* transcript. Must be set before {@link #start}; pass a null model path
* to disable.
*/
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
this.verifySpeakerModelPath = speakerModelPath;
this.verifyWearerEmbedding = wearerEmbedding;
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
}
public void clearSpeakerVerification() {
this.verifySpeakerModelPath = null;
this.verifyWearerEmbedding = null;
}
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
public void setNoiseSuppression(boolean enabled) {
this.suppressionEnabled = enabled;
}
/**
* Direction gating from the Sonic Radar beam: packets whose firmware
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
* frame, 0 = straight ahead, positive right) are dropped before any
* consumer sees them. Safe to update mid-run.
*/
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
this.beamFilterEnabled = enabled;
this.beamCenterDeg = centerDeg;
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
}
private boolean withinBeam(int angleDegrees) {
int delta = angleDegrees - beamCenterDeg;
while (delta > 180) delta -= 360;
while (delta < -180) delta += 360;
return Math.abs(delta) <= beamHalfWidthDeg;
}
private void applySuppression(short[] pcm, int count) {
try {
if (suppressor == null) {
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
}
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
le[i * 2] = (byte) (pcm[i] & 0xff);
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
}
byte[] cleaned = suppressor.process(le);
int cleanedCount = Math.min(count, cleaned.length / 2);
for (int i = 0; i < cleanedCount; i++) {
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
}
} catch (Throwable t) {
Log.w(TAG, "noise suppression failed; passing audio through", t);
suppressionEnabled = false;
}
}
public void start(String requestedMode) {
@@ -156,11 +222,36 @@ public class FaceclawVoiceController {
}
mode = parseMode(requestedMode);
started = true;
audioStarted = false;
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
workerThread.start();
}
}
/**
* Whether mic audio is actually flowing. {@link #start} only records
* intent: the enable lives in the glasses' EvenHub session, so a transport
* drop or a session suspend can leave this controller started with a
* worker that will never see another packet. Anything deciding whether to
* (re)start capture must ask this rather than assume its own bookkeeping.
*/
public boolean isCapturing() {
boolean audioUp;
synchronized (lock) {
if (!started) {
return false;
}
audioUp = audioStarted;
}
if (!audioUp) {
// The worker is still bringing the mic up; report it as running so
// a concurrent request shares it instead of restarting it.
return true;
}
FaceclawBleCommunicator currentCommunicator = communicator;
return currentCommunicator != null && currentCommunicator.isAudioCaptureActive();
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
@@ -194,41 +285,49 @@ public class FaceclawVoiceController {
if ("cloud".equals(requestedMode)) {
return VoiceInputMode.CLOUD;
}
if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) {
return VoiceInputMode.ONBOARD;
}
return VoiceInputMode.WAKEWORD;
return VoiceInputMode.ONBOARD;
}
private void runLoop() {
try {
deleteLegacyKwsFiles();
VoiceInputMode currentMode = mode;
if (currentMode == VoiceInputMode.ONBOARD) {
File modelDir = findAsrModelDir();
if (modelDir == null) {
emitStatus("Voice model not downloaded (see Settings > Voice).");
return;
}
emitStatus("Loading transcription model...");
File modelDir = installAsrModelFiles();
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
resetTranscriptState();
lastTranscript = "";
} else if (currentMode == VoiceInputMode.WAKEWORD) {
emitStatus("Loading wake-word model...");
File modelDir = installModelFiles();
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
stream = keywordSpotter.createStream();
}
lc3Decoder = new FaceclawLc3Decoder();
endpointDetector.reset();
if (suppressor != null) {
suppressor.reset();
}
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
verifyCount = 0;
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.");
return;
}
synchronized (lock) {
audioStarted = true;
}
emitStatus(currentMode == VoiceInputMode.CLOUD
? "Listening (cloud)..."
: currentMode == VoiceInputMode.ONBOARD
? "Listening..."
: "Listening for \"screen on\"...");
: "Listening...");
processG2Audio();
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
// on (the callbacks are posted in order to the main handler).
runSpeakerVerification();
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
if (currentMode == VoiceInputMode.ONBOARD) {
@@ -244,6 +343,7 @@ public class FaceclawVoiceController {
releaseLc3();
synchronized (lock) {
started = false;
audioStarted = false;
workerThread = null;
}
}
@@ -312,29 +412,6 @@ public class FaceclawVoiceController {
return b.array();
}
private KeywordSpotterConfig buildConfig(File modelDir) {
return KeywordSpotterConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setOnlineModelConfig(OnlineModelConfig.builder()
.setTransducer(OnlineTransducerModelConfig.builder()
.setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setModelType("zipformer2")
.setModelingUnit("")
.setNumThreads(1)
.build())
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
.setKeywordsScore(1.5f)
.setKeywordsThreshold(0.35f)
.build();
}
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
return OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
@@ -352,50 +429,42 @@ public class FaceclawVoiceController {
.build();
}
private File installModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
for (String fileName : MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private File installAsrModelFiles() throws IOException {
File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR);
if (!modelDir.exists() && !modelDir.mkdirs()) {
throw new IOException("Could not create " + modelDir.getAbsolutePath());
}
AssetManager assets = appContext.getAssets();
/**
* The Moonshine model directory, populated by the download flow in
* asr-model.ts (releases before 0.5.0 copied the same files out of the
* APK, so upgraded installs are already complete). Null when any file is
* missing, i.e. the model still needs to be downloaded.
*/
private File findAsrModelDir() {
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR);
for (String fileName : ASR_MODEL_FILES) {
copyAssetIfNeeded(
assets,
ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName,
new File(modelDir, fileName)
);
}
return modelDir;
}
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
if (destination.exists() && destination.length() > 0) {
return;
}
try (InputStream input = assets.open(assetPath);
FileOutputStream output = new FileOutputStream(destination)) {
byte[] buffer = new byte[64 * 1024];
int read;
while ((read = input.read(buffer)) >= 0) {
output.write(buffer, 0, read);
File file = new File(modelDir, fileName);
if (!file.exists() || file.length() == 0) {
return null;
}
}
return modelDir;
}
private void deleteLegacyKwsFiles() {
try {
deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT));
} catch (Throwable t) {
Log.w(TAG, "failed to delete legacy wake-word files", t);
}
}
private static void deleteRecursively(File file) {
if (!file.exists()) {
return;
}
File[] children = file.listFiles();
if (children != null) {
for (File child : children) {
deleteRecursively(child);
}
}
file.delete();
}
private boolean startG2Audio() {
@@ -413,9 +482,8 @@ public class FaceclawVoiceController {
private void processG2Audio() {
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
while (started && !Thread.currentThread().isInterrupted()) {
OnlineStream currentStream = stream;
FaceclawLc3Decoder currentDecoder = lc3Decoder;
if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) {
if (currentDecoder == null) {
return;
}
@@ -430,48 +498,48 @@ public class FaceclawVoiceController {
continue;
}
decodedSamples += count;
// Global mic processing from the Microphones app config: the
// beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker
// for every consumer, recognition included), and the spectral
// noise suppressor cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
int angleDegrees = currentDecoder.getLastAngleDegrees();
int ssr = currentDecoder.getLastSsr();
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr);
maybeEmitAudioStats(false);
continue;
}
if (suppressionEnabled) {
applySuppression(pcm, count);
}
if (recordingPcm != null) {
appendRecording(pcm, count);
}
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
verifyCount += copied;
}
if (endpointing && endpointDetector.accept(pcm, count)) {
emitSpeechEnd();
}
if (mode == VoiceInputMode.CLOUD) {
emitPcm(pcm, count);
} else if (mode == VoiceInputMode.ONBOARD) {
// PCM and frame metadata flow in every mode so levels, recording,
// and the Microphones radar keep working alongside onboard ASR.
emitPcm(pcm, count);
emitFrameMeta(angleDegrees, ssr);
if (mode != VoiceInputMode.CLOUD) {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
processRecognizer(samples);
} else {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
}
currentStream.acceptWaveform(samples, SAMPLE_RATE);
processKeywordSpotter(currentStream);
}
maybeEmitAudioStats(false);
}
}
private void processKeywordSpotter(OnlineStream currentStream) {
KeywordSpotter currentSpotter = keywordSpotter;
if (currentSpotter == null) {
return;
}
while (currentSpotter.isReady(currentStream)) {
currentSpotter.decode(currentStream);
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
String keyword = result == null ? "" : result.getKeyword();
if (keyword != null && keyword.trim().length() > 0) {
currentSpotter.reset(currentStream);
emitWakeWord(keyword);
}
}
}
private void processRecognizer(float[] samples) {
appendTranscriptSamples(samples);
long now = SystemClock.elapsedRealtime();
@@ -661,6 +729,83 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onPcm(le));
}
/**
* Embed the session's buffered utterance and compare it to the enrolled
* wearer voice-print. Fails open: a session too short to verify, or a
* model that will not load, counts as the wearer rather than silencing
* every command.
*/
private void runSpeakerVerification() {
short[] buffer = verifyBuffer;
float[] wearer = verifyWearerEmbedding;
String modelPath = verifySpeakerModelPath;
verifyBuffer = null;
if (buffer == null || wearer == null || modelPath == null) {
return;
}
if (verifyCount < VERIFY_MIN_SAMPLES) {
emitSpeakerVerified(true, 0f);
return;
}
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
try {
byte[] le = new byte[verifyCount * 2];
for (int i = 0; i < verifyCount; i++) {
short s = buffer[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
if (embedding == null || embedding.length != wearer.length) {
emitSpeakerVerified(true, 0f);
return;
}
double dot = 0;
for (int i = 0; i < embedding.length; i++) {
dot += (double) embedding[i] * wearer[i];
}
boolean isWearer = dot >= verifyThreshold;
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
emitSpeakerVerified(isWearer, (float) dot);
} catch (Throwable t) {
Log.w(TAG, "speaker verification failed", t);
emitSpeakerVerified(true, 0f);
}
}
// The 28 MB embedding model takes seconds to load; keep one instance
// across capture sessions so verification adds only the embed time.
private static FaceclawSpeakerId sharedSpeakerId;
private static String sharedSpeakerIdPath;
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
if (sharedSpeakerId != null) {
sharedSpeakerId.close();
}
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
sharedSpeakerIdPath = modelPath;
}
return sharedSpeakerId;
}
private void emitSpeakerVerified(boolean isWearer, float similarity) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
}
private void emitFrameMeta(int angleDegrees, int ssr) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
}
private void emitSpeechEnd() {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
@@ -783,14 +928,6 @@ public class FaceclawVoiceController {
}
private void releaseSherpa() {
if (stream != null) {
stream.release();
stream = null;
}
if (keywordSpotter != null) {
keywordSpotter.release();
keywordSpotter = null;
}
if (recognizer != null) {
recognizer.release();
recognizer = null;
@@ -893,14 +1030,6 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onStatus(status));
}
private void emitWakeWord(String keyword) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onWakeWord(keyword));
}
private void emitTranscript(String text, boolean isFinal) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
@@ -3,8 +3,6 @@ package com.faceclaw.app;
public interface FaceclawVoiceControllerListener {
void onStatus(String status);
void onWakeWord(String keyword);
/**
* Best transcript of the current utterance so far. REPLACE semantics: text
* is the complete transcript, not a delta — display it verbatim, replacing
@@ -13,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
void onTranscript(String text, boolean isFinal);
/**
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
* little-endian PCM. Empty/absent in onboard mode.
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
* keep working while the onboard recognizer owns the transcript.
*/
void onPcm(byte[] pcm16le);
/**
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
* packet, computed on the raw stereo capture before its mono downmix: the
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
* the wearer's right) and a signal-strength ratio that serves as a
* confidence/energy proxy for the angle.
*/
void onFrameMeta(int angleDegrees, int ssr);
/**
* The speaker stopped, in a hands-free session that has no button release
* to end it. Only fires when endpointing was enabled for the session; see
* FaceclawVoiceController.setEndpointing.
*/
void onSpeechEnd();
/**
* Result of speaker verification against the enrolled wearer voice-print,
* emitted once per capture session just before the final transcript.
* Only fires when verification was configured for the session; fails
* open (isWearer=true) when the utterance was too short to verify or the
* speaker model could not be loaded.
*/
void onSpeakerVerified(boolean isWearer, float similarity);
}
@@ -0,0 +1,100 @@
package com.faceclaw.app;
import android.util.Log;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
/**
* Streaming WAV writer for conversation recordings: the header is written up
* front with zero sizes and patched on finish, so a crash mid-session leaves
* a recoverable file (data length can be derived from the file size).
*/
public class FaceclawWavRecorder {
private static final String TAG = "FaceclawWavRecorder";
private final String path;
private final int sampleRate;
private final int channels;
private RandomAccessFile file;
private long dataBytes;
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
this.path = path;
this.sampleRate = sampleRate;
this.channels = channels;
File parent = new File(path).getParentFile();
if (parent != null && !parent.exists() && !parent.mkdirs()) {
throw new IOException("could not create " + parent);
}
file = new RandomAccessFile(path, "rw");
file.setLength(0);
file.write(buildHeader(0));
}
public synchronized void append(byte[] pcm16le) {
if (file == null || pcm16le == null || pcm16le.length == 0) {
return;
}
try {
file.write(pcm16le);
dataBytes += pcm16le.length;
} catch (IOException e) {
Log.w(TAG, "append failed", e);
}
}
public synchronized long getDurationMs() {
return dataBytes * 1000L / (sampleRate * channels * 2L);
}
public String getPath() {
return path;
}
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
public synchronized long finish() {
RandomAccessFile current = file;
file = null;
if (current == null) {
return dataBytes;
}
try {
current.seek(0);
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
current.close();
} catch (IOException e) {
Log.w(TAG, "finish failed", e);
}
return dataBytes;
}
/** Delete the (finished or abandoned) recording. */
public synchronized void discard() {
finish();
new File(path).delete();
}
private byte[] buildHeader(int pcmBytes) {
int byteRate = sampleRate * channels * 2;
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
b.putInt(36 + pcmBytes);
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
b.putInt(16);
b.putShort((short) 1);
b.putShort((short) channels);
b.putInt(sampleRate);
b.putInt(byteRate);
b.putShort((short) (channels * 2));
b.putShort((short) 16);
b.put("data".getBytes(StandardCharsets.US_ASCII));
b.putInt(pcmBytes);
return b.array();
}
}
@@ -0,0 +1,383 @@
package com.faceclaw.app;
import android.content.Context;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.google.android.gms.common.ConnectionResult;
import com.google.android.gms.common.GoogleApiAvailability;
import com.google.android.gms.wearable.CapabilityClient;
import com.google.android.gms.wearable.CapabilityInfo;
import com.google.android.gms.wearable.DataMap;
import com.google.android.gms.wearable.MessageEvent;
import com.google.android.gms.wearable.Node;
import com.google.android.gms.wearable.PutDataMapRequest;
import com.google.android.gms.wearable.PutDataRequest;
import com.google.android.gms.wearable.Wearable;
import org.json.JSONException;
import org.json.JSONObject;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
/**
* Phone side of the Wear OS integration: the single place that talks to the
* Wearable Data Layer (Google Play services), so the JS side only ever sees
* JSON strings and never touches Play services types.
*
* Inbound: FaceclawWearListenerService hands every watch message to
* {@link #handleMessage}; it is forwarded to the JS listener on the thread
* that registered it (the main isolate), mirroring FaceclawSettings. Messages
* that arrive before a listener exists (the watch poked a phone whose JS
* dashboard is not up yet) are acked with jsReady=false so the watch can say
* "open Faceclaw on your phone"; only idempotent state requests are kept for
* replay on registration — the watch already reported gestures and commands
* as failed, and executing them tens of seconds later would act on whatever
* the glasses happen to show by then.
*
* Outbound: {@link #publishState} mirrors the dashboard state into a Data
* Layer item (delivered even if the watch is out of range right now, and
* available to the watch app the moment it opens), while {@link #sendToWatch}
* sends fire-and-forget messages (acks, assistant text) to every reachable
* watch that advertises the Faceclaw watch-app capability.
*
* Everything is a no-op on phones without Google Play services.
*/
public final class FaceclawWearBridge {
private static final String TAG = "FaceclawWear";
/** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */
public static final String CAPABILITY_WATCH = "faceclaw_watch";
/** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */
public static final String CAPABILITY_PHONE = "faceclaw_phone";
public static final String PATH_PREFIX = "/faceclaw";
public static final String PATH_STATE = "/faceclaw/state";
public static final String PATH_STATE_REQUEST = "/faceclaw/state/request";
public static final String PATH_ACK = "/faceclaw/ack";
public static final String PATH_EVENT = "/faceclaw/event";
private static final String STATE_KEY_JSON = "json";
private static final String STATE_KEY_UPDATED_AT = "updatedAt";
/** Messages kept for a listener that is not registered yet. */
private static final int PENDING_LIMIT = 16;
private static final long PENDING_MAX_AGE_MS = 30_000;
private static volatile FaceclawWearBridge instance;
private final Context context;
private final boolean available;
private final Object lock = new Object();
private FaceclawWearListener listener;
private Handler listenerHandler;
private final ArrayDeque<PendingMessage> pending = new ArrayDeque<>();
private final List<Node> watchNodes = new ArrayList<>();
private String lastPublishedState = null;
private static final class PendingMessage {
final String path;
final String json;
final String nodeId;
final long receivedAt;
PendingMessage(String path, String json, String nodeId, long receivedAt) {
this.path = path;
this.json = json;
this.nodeId = nodeId;
this.receivedAt = receivedAt;
}
}
private FaceclawWearBridge(Context context) {
this.context = context.getApplicationContext();
boolean playServices;
try {
playServices = GoogleApiAvailability.getInstance()
.isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS;
} catch (Throwable error) {
Log.w(TAG, "Play services availability check failed", error);
playServices = false;
}
this.available = playServices;
if (!available) {
Log.i(TAG, "Google Play services unavailable; watch integration disabled");
}
}
/** Initialize (idempotent) and return the singleton. */
public static FaceclawWearBridge getInstance(Context context) {
if (instance == null) {
synchronized (FaceclawWearBridge.class) {
if (instance == null) {
instance = new FaceclawWearBridge(context);
}
}
}
return instance;
}
/** Whether the Wearable Data Layer can be used on this phone at all. */
public boolean isAvailable() {
return available;
}
/**
* Register the JS listener. Must be called from the thread whose isolate
* owns it; that thread's Looper is captured for dispatch. Replays any
* recently queued watch messages and refreshes the reachable-watch set.
*/
public void setListener(FaceclawWearListener newListener) {
Looper looper = Looper.myLooper();
if (looper == null) {
Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified");
}
List<PendingMessage> replay;
synchronized (lock) {
listener = newListener;
listenerHandler = looper != null ? new Handler(looper) : null;
replay = new ArrayList<>(pending);
pending.clear();
}
long now = System.currentTimeMillis();
for (PendingMessage message : replay) {
if (now - message.receivedAt <= PENDING_MAX_AGE_MS) {
dispatchMessage(message.path, message.json, message.nodeId);
}
}
refreshWatchNodes();
}
public void clearListener() {
synchronized (lock) {
listener = null;
listenerHandler = null;
}
}
public boolean isWatchReachable() {
synchronized (lock) {
return !watchNodes.isEmpty();
}
}
/** Display name of a reachable watch ("" if none). */
public String getWatchName() {
synchronized (lock) {
return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
}
}
/** Re-query which watches advertise the Faceclaw watch-app capability. */
public void refreshWatchNodes() {
if (!available) return;
try {
Wearable.getCapabilityClient(context)
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
.addOnSuccessListener(this::updateWatchNodes)
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
} catch (Throwable error) {
Log.w(TAG, "capability query failed", error);
}
}
/**
* Mirror the dashboard state to the watch. Identical consecutive states
* are not re-sent (the Data Layer would drop them anyway).
*/
public void publishState(String json) {
publishState(json, false);
}
/** As {@link #publishState(String)}; `force` re-sends an unchanged state. */
public void publishState(String json, boolean force) {
if (!available || json == null) return;
synchronized (lock) {
if (!force && json.equals(lastPublishedState)) return;
lastPublishedState = json;
}
try {
PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE);
DataMap map = request.getDataMap();
map.putString(STATE_KEY_JSON, json);
map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis());
PutDataRequest put = request.asPutDataRequest().setUrgent();
Wearable.getDataClient(context)
.putDataItem(put)
.addOnFailureListener(error -> Log.w(TAG, "state publish failed", error));
} catch (Throwable error) {
Log.w(TAG, "state publish failed", error);
}
}
/** Send a message to every reachable watch running the Faceclaw watch app. */
public void sendToWatch(String path, String json) {
if (!available) return;
List<Node> targets;
synchronized (lock) {
targets = new ArrayList<>(watchNodes);
}
if (targets.isEmpty()) {
// Nothing known yet (first message after boot): look the watch up, then send.
try {
Wearable.getCapabilityClient(context)
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
.addOnSuccessListener(info -> {
updateWatchNodes(info);
for (Node node : info.getNodes()) {
sendToNode(node.getId(), path, json);
}
})
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
} catch (Throwable error) {
Log.w(TAG, "capability query failed", error);
}
return;
}
for (Node node : targets) {
sendToNode(node.getId(), path, json);
}
}
/** Send a message to one specific node (e.g. an ack back to the sender). */
public void sendToNode(String nodeId, String path, String json) {
if (!available || nodeId == null || nodeId.isEmpty()) return;
try {
byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8);
Wearable.getMessageClient(context)
.sendMessage(nodeId, path, payload)
.addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error));
} catch (Throwable error) {
Log.w(TAG, "send " + path + " failed", error);
}
}
/** Reply to a watch message; `seq` echoes the watch's sequence number. */
public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) {
JSONObject ack = new JSONObject();
try {
ack.put("seq", seq);
ack.put("ok", ok);
ack.put("jsReady", jsReady);
ack.put("message", message != null ? message : "");
} catch (JSONException ignored) {
// The keys are constants; this cannot happen.
}
sendToNode(nodeId, PATH_ACK, ack.toString());
}
/** Entry point for FaceclawWearListenerService; may run on any thread. */
void handleMessage(MessageEvent event) {
if (event == null) return;
String path = event.getPath();
if (path == null || !path.startsWith(PATH_PREFIX)) return;
byte[] data = event.getData();
String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}";
String nodeId = event.getSourceNodeId();
Log.d(TAG, "recv " + path + " seq=" + readSeq(json));
boolean delivered = dispatchMessage(path, json, nodeId);
if (delivered) return;
// No JS listener yet: tell the watch so it can explain why nothing is
// happening. Only state requests are kept for replay (see class doc);
// everything else is dropped after the failure ack.
if (PATH_STATE_REQUEST.equals(path)) {
synchronized (lock) {
long now = System.currentTimeMillis();
while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) {
pending.pollFirst();
}
if (pending.size() >= PENDING_LIMIT) {
pending.pollFirst();
}
pending.addLast(new PendingMessage(path, json, nodeId, now));
}
}
sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses.");
}
/** Entry point for FaceclawWearListenerService's capability callback. */
void handleCapabilityChanged(CapabilityInfo info) {
if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return;
updateWatchNodes(info);
}
private void updateWatchNodes(CapabilityInfo info) {
Set<Node> nodes = info != null ? info.getNodes() : Collections.<Node>emptySet();
// Every reachable watch gets messages; a nearby (Bluetooth-linked)
// one is listed first so it is the name the phone UI shows.
List<Node> reachable = new ArrayList<>();
for (Node node : nodes) {
if (node.isNearby()) reachable.add(0, node); else reachable.add(node);
}
boolean changed;
String name;
synchronized (lock) {
changed = !sameNodeIds(watchNodes, reachable);
watchNodes.clear();
watchNodes.addAll(reachable);
name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
}
if (changed) {
Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name);
boolean reachableNow = !reachable.isEmpty();
FaceclawWearListener target;
Handler handler;
synchronized (lock) {
target = listener;
handler = listenerHandler;
}
if (target != null && handler != null) {
handler.post(() -> {
try {
target.onWatchConnection(reachableNow, name);
} catch (Exception error) {
Log.w(TAG, "watch connection listener failed", error);
}
});
}
}
}
private static boolean sameNodeIds(List<Node> a, List<Node> b) {
if (a.size() != b.size()) return false;
for (int i = 0; i < a.size(); i++) {
if (!a.get(i).getId().equals(b.get(i).getId())) return false;
}
return true;
}
/** Post to the JS listener; false when there is none to post to. */
private boolean dispatchMessage(String path, String json, String nodeId) {
FaceclawWearListener target;
Handler handler;
synchronized (lock) {
target = listener;
handler = listenerHandler;
}
if (target == null || handler == null) return false;
handler.post(() -> {
try {
target.onMessage(path, json, nodeId);
} catch (Exception error) {
Log.w(TAG, "wear message listener failed for " + path, error);
}
});
return true;
}
private static long readSeq(String json) {
try {
return new JSONObject(json).optLong("seq", 0);
} catch (JSONException error) {
return 0;
}
}
}
@@ -0,0 +1,19 @@
package com.faceclaw.app;
/**
* Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every
* call is delivered on the thread that registered the listener (see
* FaceclawWearBridge.setListener).
*/
public interface FaceclawWearListener {
/**
* A message from the watch app. `path` is the Data Layer message path
* (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON
* object, "{}" when the watch sent none); `nodeId` identifies the watch
* so a reply can be addressed to it.
*/
void onMessage(String path, String json, String nodeId);
/** The set of reachable watches running the Faceclaw watch app changed. */
void onWatchConnection(boolean reachable, String watchName);
}
@@ -0,0 +1,26 @@
package com.faceclaw.app;
import android.util.Log;
import com.google.android.gms.wearable.CapabilityInfo;
import com.google.android.gms.wearable.MessageEvent;
import com.google.android.gms.wearable.WearableListenerService;
/**
* Play services starts this service (in the app process) for every Data
* Layer message under /faceclaw and for capability changes, whether or not
* the app is running. It only forwards to FaceclawWearBridge, which owns the
* hand-off to the JS side.
*/
public class FaceclawWearListenerService extends WearableListenerService {
@Override
public void onMessageReceived(MessageEvent event) {
Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId());
FaceclawWearBridge.getInstance(this).handleMessage(event);
}
@Override
public void onCapabilityChanged(CapabilityInfo info) {
FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info);
}
}
@@ -15,13 +15,18 @@ public final class PreviewBitmapUtil {
private PreviewBitmapUtil() {}
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) {
return fromGray(gray, width, height, brightenGamma, false);
}
/** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) {
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
throw new IllegalArgumentException("invalid gray preview buffer");
}
int[] lut = new int[256];
for (int g = 0; g < 256; g++) {
int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma))));
lut[g] = 0xff000000 | (v << 16) | (v << 8) | v;
lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v);
}
byte[] bytes = new byte[width * height];
gray.get(bytes);
@@ -28,6 +28,16 @@ public class BleProtocol {
public static final int PRELUDE_ACK_SID = 0x01;
public static final int PRELUDE_ACK_MAGIC = 156;
/**
* DEVICE_SETTINGS security-auth channel. Firmware 2.2.9 enforces completing
* this exchange shortly after a GATT connection opens (a ~30 s deadline
* closes unauthenticated links) and gates query responses on it; the
* message itself exists back to the 2.2.4 era, so sending it is safe on all
* stock versions. See ../notes/ble-connections-2.2.9.md.
*/
public static final int SID_SECURITY_AUTH = 0x80;
public static final int FLAG_SECURITY_AUTH = 0x00;
private static final int SECURITY_AUTH_CMD = 4;
public static final int SID_APP_LAUNCH = 0x01;
public static final int SID_EVENHUB = 0xe0;
public static final int SID_UI_SETTING = 0x09;
@@ -39,6 +49,14 @@ public class BleProtocol {
// protobuf fields to stock implementations, so they are safely ignored.
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
/**
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
* to settings read responses and as standalone commandId=3 pushes.
*/
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
@@ -323,6 +341,48 @@ public class BleProtocol {
));
}
/**
* The application authentication request: DEVICE_SETTINGS/AUTHENTICATION(4)
* on sid 0x80, flag 0x00. Carries no key material — it arms the firmware's
* "security auth" flag; the firmware sends the success notification once
* the flag is set AND the BLE link is encrypted (SMP pairing/bond
* restoration, handled by the OS below GATT, possibly after a pairing
* prompt). Protobuf: f1=4 (command), f2=magic, f3={f1=1, f2=4}.
*/
public static byte[] buildAuthenticationRequest(int magic) {
byte[] auth = concat(CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, SECURITY_AUTH_CMD)
));
return concat(CollectionUtils.listOf(
encodeVarintField(1, SECURITY_AUTH_CMD),
encodeVarintField(2, magic),
encodeMessageField(3, auth)
));
}
/**
* True when `pb` (trailing CRC still attached) is the success notification
* for the authentication request sent with `magic`: command 4, the same
* magic, and an EMPTY field-3 result message (`1a 00` — the omitted result
* code defaults to zero). A GATT write completing is NOT success; only this
* notification is.
*/
public static boolean isAuthenticationSuccess(byte[] pb, int magic) {
if (pb == null) {
return false;
}
byte[] root = stripTrailingCrc(pb);
if (readVarintFieldValue(root, 1, -1) != SECURITY_AUTH_CMD) {
return false;
}
if (readVarintFieldValue(root, 2, -1) != magic) {
return false;
}
byte[] result = readFieldBytes(root, 3);
return result != null && result.length == 0;
}
public static byte[] buildSettingsQuery(int magic) {
byte[] request = encodeVarintField(1, 1);
return concat(CollectionUtils.listOf(
@@ -353,6 +413,40 @@ public class BleProtocol {
));
}
/**
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
* settings write carrying CFW field 103. Magic zero makes it
* fire-and-forget like the wake lease: the firmware consumes the field
* before its stock nanopb decoder discards the unknown tag, and the
* "ack" is the field-104 status notify that follows.
*/
public static byte[] buildFaceclawMicControl(byte[] record) {
return concat(CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, 0),
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
));
}
/**
* Extract the CFW mic status record (field 104) from a settings frame, or
* null when absent. The 21-byte body layout is decoded on the TS side;
* here we only validate the 'M','C',version prelude.
*/
public static byte[] parseFaceclawMicStatus(byte[] pb) {
if (pb == null) {
return null;
}
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
if (body == null
|| body.length < 3
|| body[0] != 'M'
|| body[1] != 'C') {
return null;
}
return body;
}
/**
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
* when this is an ordinary settings frame.
@@ -552,6 +646,35 @@ public class BleProtocol {
}
/**
* Split one notification value into individual `aa21`/`aa12` envelope
* frames. A value normally holds exactly one frame, but the firmware can
* pack several (each self-describing via the length byte at offset 3), and
* a frame that only reads the first would silently drop the rest. A value
* that does not start with an envelope, or a trailing partial frame
* (truncated by the stack — observed on a Samsung tablet for values over
* 64 bytes), is returned as-is so the caller can log it.
*/
public static List<byte[]> splitFrames(byte[] value) {
List<byte[]> out = new ArrayList<>();
if (value == null || value.length == 0) {
return out;
}
int offset = 0;
while (offset + 8 <= value.length
&& value[offset] == (byte) 0xaa
&& (value[offset + 1] == 0x21 || value[offset + 1] == 0x12)) {
int len = value[offset + 3] & 0xff;
int end = Math.min(value.length, offset + 8 + len);
out.add(Arrays.copyOfRange(value, offset, end));
offset = end;
}
if (offset < value.length) {
out.add(Arrays.copyOfRange(value, offset, value.length));
}
return out;
}
public static ParsedFrame parseFrame(byte[] buf) {
if (buf == null || buf.length < 10 || buf[0] != (byte) 0xaa || (buf[1] != 0x21 && buf[1] != 0x12)) {
return new ParsedFrame(false, 0, 0, new byte[0], -1, -1);
@@ -12,6 +12,14 @@ public class ConnectionOptions {
public static final int RING_RECONNECT_DELAY_MS = 2_000;
public static final int WRITE_TIMEOUT_MS = 2_000;
public static final int PRELUDE_TIMEOUT_MS = 2_000;
// Application security-auth exchange (sid 0x80). The success notification
// only arrives once the BLE link is encrypted, which on a first-time
// connection means SMP pairing — possibly with an OS dialog the user has
// to accept — so the stock-firmware paths wait generously.
public static final int SECURITY_AUTH_TIMEOUT_MS = 30_000;
// The main communicator's wait is soft (it proceeds on timeout so firmware
// that never answers can't stall every reconnect), so it is kept short.
public static final int SECURITY_AUTH_SOFT_TIMEOUT_MS = 6_000;
public static final int ACK_TIMEOUT_MS = 3_500;
public static final int HEARTBEAT_FAILURE_DEADLINE_MS = 10_000;
public static final int HEARTBEAT_READY_MS = 4_000;
@@ -23,6 +23,22 @@ public class MessageBuilder {
);
}
/** The per-connection sid-0x80 security-auth request (one per arm). */
public OutboundMessage securityAuth(boolean leftArm) {
int magic = magicPool.allocate();
return new OutboundMessage(
"security-auth",
"security auth " + (leftArm ? "L" : "R"),
BleProtocol.SID_SECURITY_AUTH,
BleProtocol.FLAG_SECURITY_AUTH,
magic,
BleProtocol.buildAuthenticationRequest(magic),
ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS,
-1,
leftArm
);
}
public OutboundMessage shutdown(int exitMode) {
int magic = magicPool.allocate();
return new OutboundMessage(
@@ -251,6 +267,21 @@ public class MessageBuilder {
);
}
/** CFW mic_control write (settings field 103) targeted at one temple. */
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
return new OutboundMessage(
"mic-control",
"mic control " + label + (leftArm ? " L" : " R"),
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
0,
BleProtocol.buildFaceclawMicControl(record),
ACK_TIMEOUT_MS,
-1,
leftArm
);
}
public OutboundMessage faceclawWearQuery(boolean leftArm) {
return new OutboundMessage(
"wear-query-control",
@@ -0,0 +1,44 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class FastClusteringConfig {
private final int numClusters;
private final float threshold;
private FastClusteringConfig(Builder builder) {
this.numClusters = builder.numClusters;
this.threshold = builder.threshold;
}
public static Builder builder() {
return new Builder();
}
public int getNumClusters() {
return numClusters;
}
public float getThreshold() {
return threshold;
}
public static class Builder {
private int numClusters = -1;
private float threshold = 0.5f;
public FastClusteringConfig build() {
return new FastClusteringConfig(this);
}
public Builder setNumClusters(int numClusters) {
this.numClusters = numClusters;
return this;
}
public Builder setThreshold(float threshold) {
this.threshold = threshold;
return this;
}
}
}
@@ -1,66 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotter {
private long ptr;
public KeywordSpotter(KeywordSpotterConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
}
}
public OnlineStream createStream() {
return createStream("");
}
public OnlineStream createStream(String keywords) {
return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
}
public boolean isReady(OnlineStream stream) {
return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
}
public void decode(OnlineStream stream) {
decode(ptr, stream.getPtr());
}
public void reset(OnlineStream stream) {
reset(ptr, stream.getPtr());
}
public KeywordSpotterResult getResult(OnlineStream stream) {
return getResult(ptr, stream.getPtr());
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
super.finalize();
}
private native long newFromFile(KeywordSpotterConfig config);
private native void delete(long ptr);
private native long createStream(long ptr, String keywords);
private native boolean isReady(long ptr, long streamPtr);
private native void decode(long ptr, long streamPtr);
private native void reset(long ptr, long streamPtr);
private native KeywordSpotterResult getResult(long ptr, long streamPtr);
}
@@ -1,92 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterConfig {
private final FeatureConfig featConfig;
private final OnlineModelConfig modelConfig;
private final int maxActivePaths;
private final String keywordsFile;
private final float keywordsScore;
private final float keywordsThreshold;
private final int numTrailingBlanks;
private KeywordSpotterConfig(Builder builder) {
this.featConfig = builder.featConfig;
this.modelConfig = builder.modelConfig;
this.maxActivePaths = builder.maxActivePaths;
this.keywordsFile = builder.keywordsFile;
this.keywordsScore = builder.keywordsScore;
this.keywordsThreshold = builder.keywordsThreshold;
this.numTrailingBlanks = builder.numTrailingBlanks;
}
public static Builder builder() {
return new Builder();
}
public FeatureConfig getFeatConfig() {
return featConfig;
}
public OnlineModelConfig getModelConfig() {
return modelConfig;
}
public int getMaxActivePaths() {
return maxActivePaths;
}
public String getKeywordsFile() {
return keywordsFile;
}
public float getKeywordsScore() {
return keywordsScore;
}
public float getKeywordsThreshold() {
return keywordsThreshold;
}
public int getNumTrailingBlanks() {
return numTrailingBlanks;
}
public static class Builder {
private FeatureConfig featConfig = FeatureConfig.builder().build();
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
private int maxActivePaths = 4;
private String keywordsFile = "keywords.txt";
private float keywordsScore = 1.5f;
private float keywordsThreshold = 0.25f;
private int numTrailingBlanks = 2;
public Builder setFeatureConfig(FeatureConfig featConfig) {
this.featConfig = featConfig;
return this;
}
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
this.modelConfig = modelConfig;
return this;
}
public Builder setKeywordsFile(String keywordsFile) {
this.keywordsFile = keywordsFile;
return this;
}
public Builder setKeywordsScore(float keywordsScore) {
this.keywordsScore = keywordsScore;
return this;
}
public Builder setKeywordsThreshold(float keywordsThreshold) {
this.keywordsThreshold = keywordsThreshold;
return this;
}
public KeywordSpotterConfig build() {
return new KeywordSpotterConfig(this);
}
}
}
@@ -1,25 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class KeywordSpotterResult {
private final String keyword;
private final String[] tokens;
private final float[] timestamps;
public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
this.keyword = keyword == null ? "" : keyword;
this.tokens = tokens == null ? new String[0] : tokens;
this.timestamps = timestamps == null ? new float[0] : timestamps;
}
public String getKeyword() {
return keyword;
}
public String[] getTokens() {
return tokens;
}
public float[] getTimestamps() {
return timestamps;
}
}
@@ -0,0 +1,62 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarization {
private long ptr = 0;
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
}
}
public int getSampleRate() {
return getSampleRate(ptr);
}
// Only config.clustering is used. All other fields are ignored
public void setConfig(OfflineSpeakerDiarizationConfig config) {
setConfig(ptr, config);
}
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
return process(ptr, samples);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
return processWithCallback(ptr, samples, callback, 0);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
return processWithCallback(ptr, samples, callback, arg);
}
protected void finalize() throws Throwable {
release();
}
// You'd better call it manually if it is not used anymore
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
private native int getSampleRate(long ptr);
private native void delete(long ptr);
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
}
@@ -0,0 +1,8 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
@FunctionalInterface
public interface OfflineSpeakerDiarizationCallback {
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
}
@@ -0,0 +1,79 @@
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationConfig {
private final OfflineSpeakerSegmentationModelConfig segmentation;
private final SpeakerEmbeddingExtractorConfig embedding;
private final FastClusteringConfig clustering;
private final float minDurationOn;
private final float minDurationOff;
private OfflineSpeakerDiarizationConfig(Builder builder) {
this.segmentation = builder.segmentation;
this.embedding = builder.embedding;
this.clustering = builder.clustering;
this.minDurationOff = builder.minDurationOff;
this.minDurationOn = builder.minDurationOn;
}
public static Builder builder() {
return new Builder();
}
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
return segmentation;
}
public SpeakerEmbeddingExtractorConfig getEmbedding() {
return embedding;
}
public FastClusteringConfig getClustering() {
return clustering;
}
public float getMinDurationOff() {
return minDurationOff;
}
public float getMinDurationOn() {
return minDurationOn;
}
public static class Builder {
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
private float minDurationOn = 0.2f;
private float minDurationOff = 0.5f;
public OfflineSpeakerDiarizationConfig build() {
return new OfflineSpeakerDiarizationConfig(this);
}
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
this.segmentation = segmentation;
return this;
}
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
this.embedding = embedding;
return this;
}
public Builder setClustering(FastClusteringConfig clustering) {
this.clustering = clustering;
return this;
}
public Builder setMinDurationOff(float minDurationOff) {
this.minDurationOff = minDurationOff;
return this;
}
public Builder setMinDurationOn(float minDurationOn) {
this.minDurationOn = minDurationOn;
return this;
}
}
}
@@ -0,0 +1,27 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationSegment {
private final float start;
private final float end;
private final int speaker;
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
this.start = start;
this.end = end;
this.speaker = speaker;
}
public float getStart() {
return start;
}
public float getEnd() {
return end;
}
public int getSpeaker() {
return speaker;
}
}
@@ -0,0 +1,52 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationModelConfig {
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
private final int numThreads;
private final boolean debug;
private final String provider;
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
this.pyannote = builder.pyannote;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public OfflineSpeakerSegmentationModelConfig build() {
return new OfflineSpeakerSegmentationModelConfig(this);
}
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
this.pyannote = pyannote;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -1,9 +1,11 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OnlineToneCtcModelConfig {
public class OfflineSpeakerSegmentationPyannoteModelConfig {
private final String model;
private OnlineToneCtcModelConfig(Builder builder) {
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
this.model = builder.model;
}
@@ -18,13 +20,13 @@ public class OnlineToneCtcModelConfig {
public static class Builder {
private String model = "";
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineToneCtcModelConfig build() {
return new OnlineToneCtcModelConfig(this);
}
}
}
@@ -1,137 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineModelConfig {
private final OnlineTransducerModelConfig transducer;
private final OnlineParaformerModelConfig paraformer;
private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
private final OnlineNeMoCtcModelConfig neMoCtc;
private final OnlineToneCtcModelConfig toneCtc;
private final String tokens;
private final int numThreads;
private final boolean debug;
private final String provider;
private final String modelType;
private final String modelingUnit;
private final String bpeVocab;
private OnlineModelConfig(Builder builder) {
this.transducer = builder.transducer;
this.paraformer = builder.paraformer;
this.zipformer2Ctc = builder.zipformer2Ctc;
this.neMoCtc = builder.neMoCtc;
this.toneCtc = builder.toneCtc;
this.tokens = builder.tokens;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
this.modelType = builder.modelType;
this.modelingUnit = builder.modelingUnit;
this.bpeVocab = builder.bpeVocab;
}
public static Builder builder() {
return new Builder();
}
public OnlineTransducerModelConfig getTransducer() {
return transducer;
}
public OnlineParaformerModelConfig getParaformer() {
return paraformer;
}
public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
return zipformer2Ctc;
}
public OnlineNeMoCtcModelConfig getNeMoCtc() {
return neMoCtc;
}
public OnlineToneCtcModelConfig getToneCtc() {
return toneCtc;
}
public String getTokens() {
return tokens;
}
public int getNumThreads() {
return numThreads;
}
public boolean getDebug() {
return debug;
}
public String getProvider() {
return provider;
}
public String getModelType() {
return modelType;
}
public String getModelingUnit() {
return modelingUnit;
}
public String getBpeVocab() {
return bpeVocab;
}
public static class Builder {
private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
private String tokens = "";
private int numThreads = 1;
private boolean debug = false;
private String provider = "cpu";
private String modelType = "";
private String modelingUnit = "cjkchar";
private String bpeVocab = "";
public Builder setTransducer(OnlineTransducerModelConfig transducer) {
this.transducer = transducer;
return this;
}
public Builder setTokens(String tokens) {
this.tokens = tokens;
return this;
}
public Builder setModelType(String modelType) {
this.modelType = modelType;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
public Builder setModelingUnit(String modelingUnit) {
this.modelingUnit = modelingUnit;
return this;
}
public Builder setBpeVocab(String bpeVocab) {
this.bpeVocab = bpeVocab;
return this;
}
public OnlineModelConfig build() {
return new OnlineModelConfig(this);
}
}
}
@@ -1,30 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineNeMoCtcModelConfig {
private final String model;
private OnlineNeMoCtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineNeMoCtcModelConfig build() {
return new OnlineNeMoCtcModelConfig(this);
}
}
}
@@ -1,42 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineParaformerModelConfig {
private final String encoder;
private final String decoder;
private OnlineParaformerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public OnlineParaformerModelConfig build() {
return new OnlineParaformerModelConfig(this);
}
}
}
@@ -1,7 +1,16 @@
// Copyright 2022-2023 by zhaoming
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr;
private long ptr = 0;
public OnlineStream() {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
this.ptr = 0;
}
public OnlineStream(long ptr) {
this.ptr = ptr;
@@ -11,25 +20,58 @@ public class OnlineStream {
return ptr;
}
public void setPtr(long ptr) {
this.ptr = ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(ptr, samples, sampleRate);
acceptWaveform(this.ptr, samples, sampleRate);
}
public void inputFinished() {
inputFinished(this.ptr);
}
public void setOption(String key, String value) {
setOption(this.ptr, key, value);
}
public String getOption(String key) {
return getOption(this.ptr, key);
}
public boolean hasOption(String key) {
return hasOption(this.ptr, key);
}
public void release() {
if (ptr == 0) {
return;
}
delete(ptr);
ptr = 0;
close();
}
public void close() {
// stream object must be release after used
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
@Override
protected void finalize() throws Throwable {
release();
close();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void inputFinished(long ptr);
private native void setOption(long ptr, String key, String value);
private native String getOption(long ptr, String key);
private native boolean hasOption(long ptr, String key);
private native void delete(long ptr);
}
}
@@ -1,54 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineTransducerModelConfig {
private final String encoder;
private final String decoder;
private final String joiner;
private OnlineTransducerModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
this.joiner = builder.joiner;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public String getJoiner() {
return joiner;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
private String joiner = "";
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public Builder setJoiner(String joiner) {
this.joiner = joiner;
return this;
}
public OnlineTransducerModelConfig build() {
return new OnlineTransducerModelConfig(this);
}
}
}
@@ -1,30 +0,0 @@
package com.k2fsa.sherpa.onnx;
public class OnlineZipformer2CtcModelConfig {
private final String model;
private OnlineZipformer2CtcModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public Builder setModel(String model) {
this.model = model;
return this;
}
public OnlineZipformer2CtcModelConfig build() {
return new OnlineZipformer2CtcModelConfig(this);
}
}
}
@@ -0,0 +1,58 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractor {
private long ptr = 0;
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
}
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public OnlineStream createStream() {
long p = createStream(ptr);
return new OnlineStream(p);
}
public boolean isReady(OnlineStream s) {
return isReady(ptr, s.getPtr());
}
public float[] compute(OnlineStream s) {
return compute(ptr, s.getPtr());
}
public int getDim() {
return dim(ptr);
}
private native void delete(long ptr);
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
private native long createStream(long ptr);
private native boolean isReady(long ptr, long streamPtr);
private native float[] compute(long ptr, long streamPtr);
private native int dim(long ptr);
}
@@ -0,0 +1,53 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractorConfig {
private final String model;
private final int numThreads;
private final boolean debug;
private final String provider;
private SpeakerEmbeddingExtractorConfig(Builder builder) {
this.model = builder.model;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String model = "";
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public SpeakerEmbeddingExtractorConfig build() {
return new SpeakerEmbeddingExtractorConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,78 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingManager {
private long ptr = 0;
public SpeakerEmbeddingManager(int dim) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = create(dim);
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public boolean add(String name, float[] embedding) {
return add(ptr, name, embedding);
}
public boolean add(String name, float[][] embedding) {
return addList(ptr, name, embedding);
}
public boolean remove(String name) {
return remove(ptr, name);
}
public String search(float[] embedding, float threshold) {
return search(ptr, embedding, threshold);
}
public boolean verify(String name, float[] embedding, float threshold) {
return verify(ptr, name, embedding, threshold);
}
public boolean contains(String name) {
return contains(ptr, name);
}
public int getNumSpeakers() {
return numSpeakers(ptr);
}
public String[] getAllSpeakerNames() {
return allSpeakerNames(ptr);
}
private native long create(int dim);
private native void delete(long ptr);
private native boolean add(long ptr, String name, float[] embedding);
private native boolean addList(long ptr, String name, float[][] embedding);
private native boolean remove(long ptr, String name);
private native String search(long ptr, float[] embedding, float threshold);
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
private native boolean contains(long ptr, String name);
private native int numSpeakers(long ptr);
private native String[] allSpeakerNames(long ptr);
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.0 KiB

After

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.5 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.4 KiB

After

Width:  |  Height:  |  Size: 22 KiB

@@ -1,8 +1,8 @@
<layer-list xmlns:android="http://schemas.android.com/apk/res/android" android:gravity="fill">
<item>
<bitmap android:gravity="fill" android:src="@drawable/background" />
<color android:color="#F6F6F6" />
</item>
<item>
<bitmap android:gravity="center" android:src="@drawable/logo" />
</item>
</layer-list>
</layer-list>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.7 KiB

After

Width:  |  Height:  |  Size: 70 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 146 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 256 KiB

@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Microphone icon for the main page's tab-bar wakeword button.
Converted from ~/Downloads/noun-microphone-1368757.svg (Noun Project
icon #1368757); base fill is overridden by tint-color in app.css. -->
<vector xmlns:android="http://schemas.android.com/apk/res/android"
android:width="24dp"
android:height="24dp"
android:viewportWidth="1200"
android:viewportHeight="1200">
<path
android:fillColor="#FF333333"
android:fillType="evenOdd"
android:pathData="m821.23 215.45v382.24c0 289.57-442.47 291.89-442.47 0v-382.24c0-287.26 442.47-287.26 442.47 0z" />
<path
android:fillColor="#FF333333"
android:fillType="evenOdd"
android:pathData="m543.24 963.71c-294.21-27.801-328.96-298.84-328.96-546.72 0-41.699 88.031-41.699 97.297-4.6328 2.3164 199.23 4.6328 454.05 287.26 454.05 284.94 0 287.26-254.83 287.26-454.05 6.9492-39.383 99.613-37.066 99.613 4.6328 0 247.88-34.75 518.92-333.59 546.72v139h152.89c67.18 0 69.5 97.297 0 97.297h-414.67c-64.863 0-62.547-97.297 0-97.297h152.89v-139z" />
</vector>
@@ -25,6 +25,17 @@
<item name="colorPrimary">@color/ns_primary</item>
<item name="colorPrimaryDark">@color/ns_primaryDark</item>
<item name="colorAccent">@color/ns_accent</item>
<!-- NativeScript builds the toolbar programmatically, so the
NativeScriptToolbarTheme overlay below (an inflation-time "theme"
item) never applies and the overflow icon falls back to the light
activity theme's dark textColorSecondary: black-on-dark. Tint the
overflow button itself instead; every page's bar is dark. -->
<item name="actionOverflowButtonStyle">@style/FaceclawOverflowButtonStyle</item>
</style>
<style name="FaceclawOverflowButtonStyle" parent="Widget.AppCompat.ActionButton.Overflow">
<item name="android:tint">#FFFFFF</item>
</style>
<style name="AppTheme" parent="AppThemeBase">
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Wearable Data Layer capability advertised to the watch, so the Faceclaw
watch app can find the phone that has Faceclaw installed. The watch app
advertises "faceclaw_watch" in return (see wear/). -->
<resources>
<string-array name="android_wear_capabilities">
<item>faceclaw_phone</item>
</string-array>
</resources>
+22 -10
View File
@@ -5,16 +5,22 @@ for interfacing with Android SDK).
Typescript parts are in app/. Java parts are in App_Resources/Android/src/main/java/com/faceclaw/app/.
If build.sh exists, it's a machine-specific wrapper that sets some environment
variables to point to where tools are installed, then runs `npm run build`.
If build_and_run.sh exists, it does everything build.sh does plus
`nativescript run android --device <id> --justlaunch`,
which will launch the app on the developer's phone if it is plugged in and has
adb enabled. If no phone is connected, it will still build but the install step
will fail and the output will contain the string "Cannot find connected
devices." To lint and typecheck, prefer to use build.sh; if the user asks you
to run or test the app, use build_and_run.sh and then use `adb logcat` to view
the results.
wear/ is a separate Gradle project (Kotlin + Compose for Wear OS): the watch
remote that drives Faceclaw over the Wearable Data Layer. Its phone-side
counterparts are FaceclawWearBridge.java, app/native/wear-bridge.ts and
app/g2/wear-remote.ts; the message format is in wear/PROTOCOL.md.
`build.sh` and `build_wear.sh` build the Android and Wear OS apps respectively.
`build_and_run.sh` builds the Android app then also installs it on a phone
paired with `adb`. Building may require configuring environment variables
(JAVA_HOME, ANDROID_HOME, PATH, ANDROID_KEYSTORE, ANDROID_KEYSTORE) to point at
relevant tools and an Android device ID to install on; do this in
build_paths.sh, which is created from build_paths.sh.template and not checked
into git.
To lint and typecheck, prefer to use build.sh; if the user asks you to run or
test the app, use build_and_run.sh and then use `adb logcat` to view the
results.
If you are working on low-level communication bits, consider checking out
https://github.com/Commute773/g2-kit-unofficial/ and referring to ble/docs/
@@ -23,4 +29,10 @@ well as some communication test scripts and documentation of caveats that
come up when communicating with the headset. There may already be an existing
local checkout above this workspace at ../g2-kit-unofficial.
This Android app uses a modded firmware for the G2 smart glasses. Source code
to the corresponding firmware modifications lives in
https://github.com/jimrandomh/g2flash; there is a compiled version of the
firmware patch-set in app/g2/firmware/cfw-patches.ts. Strongly prefer making
changes to the Android app over making changes to changes to the firmware, as
firmware development is considerably more fraught.
+15
View File
@@ -74,3 +74,18 @@ https://support.evenrealities.com/hc/en-us/articles/14127464826511-Privacy-Polic
Apps downloaded from the EvenHub app store come with their own privacy
policies.
## Wear OS Watch
The optional Faceclaw watch app (wear/) talks to the phone over Google Play
services' Wearable Data Layer, the same channel the watch uses for
notifications and other companion apps. Gestures, commands, the watch's own
speech-recognition results, and the mirrored glasses status and assistant
replies travel between your watch and your phone over their Bluetooth (or,
when paired that way, Wi-Fi) link; Faceclaw does not send any of it to a
server. Speech recognized on the watch uses the watch's system speech
recognizer, which may send audio to Google under Google's privacy policy:
Google: https://policies.google.com/privacy
The phone does nothing watch-related on phones without Google Play services.
+82 -46
View File
@@ -9,13 +9,15 @@ glasses. The Android app itself can install and uninstall the custom firmware;
it downloads the stock firmware from Even and applies patches generated from
https://github.com/jimrandomh/g2flash.
User-facing documentation lives at https://faceclaw.org/.
## Screenshots
![App launcher](screenshots/launcher.png)
![Music player](screenshots/music-player.png)
![Assistant settings](screenshots/settings-assistant.png)
![Display settings](screenshots/settings-display.png)
![App launcher](website/screenshots/launcher.png)
![Music player](website/screenshots/music-player.png)
![Assistant settings](website/screenshots/settings-assistant.png)
![Display settings](website/screenshots/settings-display.png)
## Installation
@@ -23,52 +25,86 @@ To compile, you will need an Android SDK environment with SDK version 35
installed and licenses accepted, and the ANDROID_HOME pointing to the install.
You will need Nativescript installed, and a JDK environment (v21), with the
JAVA_HOME environment variable pointed at it. You will also need a reasonably
up to date npm and nodejs installed.
up to date npm and nodejs installed. You can put environment variables in
build_paths.sh and they will be used by all build commands.
To install a version that you compiled, you will need `adb` connected. Go to
Settings>About phone and tap the "Build number" field seven times. Then plug
the phone's USB-C port into your computer, authorize access on the phone, then
run `nativescript run android
The compile, run
'npm run build'
To install, run
`nativescript run android --juslaunch`
To install a version that you compiled, you will need developer mode enabled on
your phone, and `adb` connected. To enable developer mode, go to Settings>About
phone and tap the "Build number" field seven times. Then plug the phone's USB-C
port into your computer, authorize access on the phone, and run
`./build_and_run.sh` to install.
## Features
* A voice assistant that wakes up when you say "Hey Even", transcribes text
with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and
responds to queries and commands using an onboard model (Qwen3 4B, slow)
or with an Anthropic or OpenAI model (requires an API key), or using your
own long-running OpenClaw agent (see "Connecting an external agent" below).
* Multitasking, with an app-switcher sidebar and app launcher
* Mostly-compatible with EvenHub apps
* A lock screen; glasses lock automatically when you take them off and unlock
when you unlock your phone
* Full-screen apps can use the full 640x480 display area (rather than the
576x288 that EvenHub apps can use)
* Integration with Android notifications: A top-bar that shows the
same icons your phone does, popups when notifications arrive, and menu
items to dismiss or use Android-app-provided custom actions like mark as
read or quick reply.
* Mirror terminal apps such as Claude Code or Codex CLI with g2mirror
(https://github.com/jimrandomh/g2mirror), view them on the glasses, and
send them inputs with the voice assistant
* Media player controls including playlist and media library navigation,
compatible with most Android media players
* Turn-by-turn directions (requires a Mapbox API token)
* Nightscout, an app for viewing blood-glucose data (requires a cloud server
and API token)
* Power management: the glasses go to sleep properly when the screen is off,
and wake when you double-tap the ring or speak the wakeword, allowing
<!-- Generated from website/content/features.md by scripts/sync-site.mjs; edit there. -->
<!-- BEGIN GENERATED: features -->
* **A voice assistant** that wakes up when you say "Hey Even", transcribes
text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API,
and responds to queries and commands using an onboard model (Qwen3 4B,
slow) or with an Anthropic or OpenAI model (requires an API key), or using
your own long-running OpenClaw agent.
* **Multitasking**, with an app-switcher sidebar and app launcher.
* **Mostly-compatible with EvenHub apps.**
* **A lock screen**; glasses lock automatically when you take them off and
unlock when you unlock your phone.
* **The full display.** Full-screen apps can use the full 640x480 display
area, rather than the 576x288 that EvenHub apps can use.
* **Integration with Android notifications**: a top bar that shows the same
icons your phone does, popups when notifications arrive, and menu items to
dismiss or use Android-app-provided custom actions like mark as read or
quick reply.
* **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex
CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on
the glasses, and send them inputs with the voice assistant.
* **Media player controls** including playlist and media library navigation,
compatible with most Android media players.
* **Turn-by-turn directions** (requires a Mapbox API token).
* **Nightscout**, an app for viewing blood-glucose data (requires a cloud
server and API token).
* **Power management**: the glasses go to sleep properly when the screen is
off, and wake when you double-tap the ring or speak the wakeword, allowing
battery life similar to the stock Even app.
* Connection management with auto-reconnect, and autodetection of conflict
with the official Even Realities app
* On-phone screen mirroring and simulated ring input
* Dual-language NativeScript architecture with Java for the multithreaded
Android API and bluetooth stack bits, Typescript for the bits you want to
hack on
* **Connection management** with auto-reconnect, and autodetection of
conflict with the official Even Realities app.
* **A Wear OS watch app** that replaces (and outdoes) the R1 ring: tap,
swipe, hold and crown gestures, side buttons, app launching and window
switching, voice or keyboard queries to the assistant with the reply on
your wrist, typing into apps, and glasses status/lock/display control.
* **On-phone screen mirroring with touch control**: tap what you see on the
mirror (sidebar icons, launcher cells), or use the phone's own touchpad,
d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a
compact ring simulator. A display-mode picker (576×288 band, 576×480 tall,
or the full 640×480 panel with an auto-hiding sidebar) and a brightness
slider with an Auto toggle sit beside the mirror.
* **Bluetooth pairing** that scans for nearby glasses and identifies each
pair before connecting: model, frame shape, and colour decoded from the
advertised serial (with product photos), left and right arms matched to
each other by that serial, an estimated distance so the pair in your hand
sorts first, and the optional R1 ring.
* **Dual-language NativeScript architecture**, with Java for the
multithreaded Android API and bluetooth stack bits, Typescript for the bits
you want to hack on.
<!-- END GENERATED: features -->
## Watch remote (Wear OS)
The `wear/` directory holds a companion app for Wear OS 3+ watches that turns
the watch into a remote for the glasses. The whole watch face is a touchpad
that mirrors the ring (tap, double-tap, hold, swipe, crown scrolling, side
buttons), and beyond the ring it can launch apps, switch and close windows,
wake/sleep/lock the display, send spoken or typed queries straight to the
assistant (using the watch's own microphone) and show the reply on the wrist,
and type text into apps such as the terminal. Its swipes are a spatial
control scheme the ring never had, with no mode to enter first: the
launcher highlights one app and moves it by rows and columns, right goes
into things (sidebar → window, settings section → items, next track in
Music), left backs out, tap selects, hold opens the context menu, double-tap
or a two-finger tap is back everywhere, and two wrist twists go back; up/down
scroll and right/left select/back everywhere else. The ring's own scheme is
unchanged — watch input is tagged as a separate source. Faceclaw's Settings > Watch
section has the phone-side switches. See [wear/README.md](wear/README.md) for
building and installing it; it is a separate Gradle build, sideloaded over
adb, and needs no Play Store. The phone side requires Google Play services.
## Connecting an external agent (OpenClaw)
@@ -122,7 +158,7 @@ https://github.com/jimrandomh/faceclaw.
The Typescript and Java code in this repository runs on your phone, not on the
glasses themselves, and (with the narrow exception of the firmware-updating
tool), can't hurt your hardware. For changes to the glasses firmware, refer to
[g2flash](https://github.com/jimrandomh/faceclaw); changes there require more
[g2flash](https://github.com/jimrandomh/g2flash); changes there require more
caution.
Faceclaw is Free Software (GPLv3). Please only contribute code that you wrote
+610
View File
@@ -15,11 +15,43 @@ ActionItem {
color: #ffffff;
}
/* The main page's custom titleView: title left, connection status and the
* warning triangle right-aligned beside the overflow menu. */
.action-bar-title {
color: #ffffff;
font-size: 18;
font-weight: bold;
vertical-align: center;
}
.action-bar-status {
color: #ffffff;
font-size: 14;
margin-right: 12;
vertical-align: center;
}
/* The emoji brings its own yellow; the padding widens the tap target. */
.action-bar-warning {
font-size: 18;
padding: 4 6;
vertical-align: center;
}
TextField {
background-color: #ffffff;
color: #222222;
}
/* Without this, the theme's .ns-dark rules recolour the text white while the
* white background above sticks, rendering white-on-white fields.
* Order matters: .system-name-modal TextField must stay after this rule so the
* deliberately light modal wins the equal-specificity tie in dark mode. */
.ns-dark TextField {
background-color: #2e2e2e;
color: #f0f0f0;
}
.system-name-modal TextField {
background-color: #ffffff;
color: #222222;
@@ -29,6 +61,43 @@ Button.-primary {
font-size: 18;
}
/* Body/explanatory text in the onboarding flow; the theme default is too
* small to comfortably read paragraphs of legalese, especially on tablets. */
.body-text {
font-size: 18;
}
/* The logo art is black-on-transparent; tint it light in dark mode so it
* doesn't vanish into the dark page background. */
.ns-dark .onboarding-logo {
tint-color: #e6e6e6;
}
/* License / privacy-policy links in the onboarding disclaimer footer. The
* padding widens the tap target beyond the small text. */
.footer-link {
color: #3b6fd6;
font-size: 15;
text-decoration: underline;
padding: 4 8;
}
.ns-dark .footer-link {
color: #8ab0ff;
}
.footer-link-separator {
color: #777777;
font-size: 15;
padding: 4 0;
}
/* Bundled project docs (phone-ui/document-page). LICENSE and PRIVACY are
* hard-wrapped plain text, so keep the size modest to limit rewrapping. */
.document-text {
font-size: 13;
}
/*
* Stands in for the lens preview when showing it would be misleading (silent
* mode, or a disconnect explained by a flat battery). Sized by the view model
@@ -55,3 +124,544 @@ Button.-primary {
border-width: 1;
border-radius: 8;
}
/* Modal behind the action bar's warning triangle: the problem-status warnings
* that used to sit inline on the main page, each with its fix-it button.
* The width is set from the view model (near-full-width on phones, capped on
* tablets); the padding lives on the content stack because a ScrollView does
* not apply its own padding to its content. */
.warnings-modal {
margin: 16;
background-color: #f6f6f6;
color: #222222;
border-color: #222222;
border-width: 1;
border-radius: 8;
}
.warnings-modal-content {
padding: 16;
}
/* The modal stays light in dark mode; the .ns-dark pairing outguns the
* theme's dark-mode recolour of Label/.h2 so the title doesn't go
* white-on-light-gray. */
.warnings-modal Label,
.ns-dark .warnings-modal Label {
color: #222222;
}
.warnings-modal-item {
margin-bottom: 12;
}
/* The phone's controller for the glasses (phone-ui/remote-controls.xml). */
.remote-controls {
margin: 0 8 0 8;
}
/* The Settings / Watch / Ring tab bar above the controls. */
.controls-tab-bar {
margin-bottom: 8;
}
.tab-button {
font-size: 14;
padding: 8 0;
margin: 0 2;
border-radius: 6;
background-color: #e2e2e2;
color: #333333;
}
.tab-button-selected {
background-color: #424242;
color: #ffffff;
}
/* The wakeword button at the tab bar's right edge: icon-only, narrower than
* the tabs, and never in the selected state (it isn't a tab). It is a
* button-styled GridLayout (Buttons can't hold images), so it borrows the
* .tab-button look and adds the elevation MaterialButtons get for free. */
.tab-mic-button {
width: 56;
/* The real MaterialButtons render 48dp tall inside a 60dp layout box (6dp
* insets top and bottom); the margins and explicit height reproduce that.
* The height also bounds the tile's measure, without which its star row
* would collapse to content and pin the icon to the top. */
height: 48;
margin: 6 2;
android-elevation: 2;
}
.tab-mic-icon {
height: 20;
tint-color: #333333;
}
.ns-dark .tab-mic-icon {
tint-color: #cccccc;
}
.ns-dark .tab-button {
background-color: #333333;
color: #cccccc;
}
.ns-dark .tab-button-selected {
background-color: #e2e2e2;
color: #222222;
}
/* Settings tab rows. */
.settings-row {
margin-bottom: 8;
}
.settings-row-label {
font-size: 14;
margin-right: 10;
}
.mode-button {
font-size: 13;
padding: 6 10;
margin-right: 6;
}
.brightness-icon {
font-size: 16;
margin: 0 4 0 4;
color: #444444;
}
.auto-toggle Label {
font-size: 13;
margin-right: 2;
}
.touchpad {
height: 230;
background-color: #1f1f1f;
border-radius: 14;
border-width: 1;
border-color: #3a3a3a;
color: #f0f0f0;
margin-top: 2;
}
.pad-focus {
font-size: 20;
color: #ffffff;
}
.pad-dpad-button {
width: 46;
height: 46;
margin: 6;
padding: 0;
font-size: 16;
border-radius: 23;
background-color: #333333;
color: #e8e8e8;
}
.pad-dpad-down {
margin-bottom: 30;
}
.pad-legend {
font-size: 11;
color: #8a8a8a;
margin-bottom: 8;
}
.action-row {
margin-top: 6;
}
.action-button {
margin: 2;
font-size: 14;
}
/* Ring tab: simulated R1 inputs. */
.ring-pad-button {
margin: 3;
padding: 0;
height: 52;
font-size: 14;
}
/*
* Pairing scan list (phone-ui/pairing-page). Rows are Repeater items: product
* art (or a colourway swatch) on the left, decoded SKU + proximity + serial in
* the middle, a chevron/check on the right.
*/
.pairing-instructions {
color: #555555;
font-size: 14;
}
.pairing-status {
font-size: 14;
margin-left: 8;
}
.pairing-section {
font-size: 16;
font-weight: bold;
margin-bottom: 4;
}
.pairing-empty {
color: #777777;
font-size: 14;
padding: 8 0;
}
.pairing-row {
background-color: #f4f4f4;
border-color: #dddddd;
border-width: 1;
border-radius: 10;
padding: 10;
margin-bottom: 8;
}
.pairing-row-selected {
background-color: #e6f0ff;
border-color: #3b6fd6;
}
.pairing-row-incomplete {
background-color: #fafafa;
border-color: #e6e6e6;
}
.pairing-art {
vertical-align: center;
}
.pairing-title {
font-size: 16;
font-weight: bold;
color: #222222;
}
.pairing-badge {
font-size: 11;
font-weight: bold;
color: #2a5bc4;
background-color: #dbe6fb;
border-radius: 9;
padding: 1 7;
}
.pairing-variant {
font-size: 13;
font-weight: bold;
color: #333333;
}
.pairing-proximity {
font-size: 12;
color: #666666;
}
.pairing-proximity-immediate {
color: #2a5bc4;
font-weight: bold;
}
.pairing-serial {
font-size: 12;
color: #666666;
font-family: monospace;
}
.pairing-arms {
font-size: 12;
color: #444444;
}
.pairing-warning {
font-size: 12;
color: #a33a1f;
margin-top: 4;
}
.pairing-swatch-label {
font-size: 11;
color: #555555;
}
.pairing-chevron {
font-size: 22;
color: #444444;
margin-left: 6;
}
.pairing-selection {
background-color: #eef4ff;
border-radius: 10;
padding: 10;
}
Button.pairing-link {
background-color: transparent;
color: #2a5bc4;
font-size: 14;
border-width: 0;
}
/*
* Permissions screen (phone-ui/permissions-page). Cards reuse the pairing-row
* / pairing-title / pairing-badge look above; these add the explanation body
* and the already-granted checkmark.
*/
.permissions-description {
font-size: 13;
color: #555555;
margin-top: 2;
}
.permissions-check {
font-size: 24;
color: #2e7d32;
margin-left: 10;
}
/* Continue with required permissions still missing: looks disabled but stays
* tappable (the tap raises an are-you-sure prompt instead of being ignored). */
Button.permissions-primary-dimmed {
opacity: 0.45;
}
/*
* Dark-mode pairing page. The cards keep their light-mode look above; here
* they go dark-gray, and every text colour set above gets a light counterpart
* so nothing renders dark-on-dark (the instructions) or dark-on-a-dark-card.
* Order matters: the -selected/-incomplete rules must follow .pairing-row so
* they win at equal specificity.
*/
.ns-dark .pairing-instructions {
color: #b0b0b0;
}
.ns-dark .pairing-empty {
color: #999999;
}
.ns-dark .pairing-row {
background-color: #2e2e2e;
border-color: #4a4a4a;
}
.ns-dark .pairing-row-selected {
background-color: #333c4c;
border-color: #5b8def;
}
.ns-dark .pairing-row-incomplete {
background-color: #282828;
border-color: #3c3c3c;
}
.ns-dark .pairing-title {
color: #f0f0f0;
}
.ns-dark .pairing-badge {
color: #a8c4f5;
background-color: #2a3b5e;
}
.ns-dark .pairing-variant {
color: #cccccc;
}
.ns-dark .pairing-proximity {
color: #a0a0a0;
}
.ns-dark .pairing-proximity-immediate {
color: #7da7f5;
}
.ns-dark .pairing-serial {
color: #a0a0a0;
}
.ns-dark .pairing-arms {
color: #c0c0c0;
}
.ns-dark .pairing-warning {
color: #e08a66;
}
.ns-dark .pairing-swatch-label {
color: #aaaaaa;
}
.ns-dark .pairing-chevron {
color: #bbbbbb;
}
.ns-dark .pairing-selection {
background-color: #2e2e2e;
}
.ns-dark Button.pairing-link {
color: #7da7f5;
}
.ns-dark .permissions-description {
color: #b0b0b0;
}
.ns-dark .permissions-check {
color: #7bc47f;
}
/*
* Conversation review (phone-ui/conversations-page, conversation-page,
* speakers-page). Cards follow the pairing-row look; speaker/emotion chips
* get their background colour inline (per-speaker / per-emotion), so the
* classes only carry shape and dark text that reads on the pastel palette.
*/
.conversation-row {
background-color: #f4f4f4;
border-color: #dddddd;
border-width: 1;
border-radius: 10;
padding: 10;
margin-bottom: 8;
}
.conversation-title {
font-size: 16;
font-weight: bold;
color: #222222;
}
.conversation-meta {
font-size: 12;
color: #666666;
}
.conversation-empty {
color: #777777;
font-size: 14;
padding: 8 0;
}
.ask-answer {
font-size: 15;
color: #222222;
padding: 4 0;
}
.ns-dark .ask-answer {
color: #e6e6e6;
}
.conversation-status {
font-size: 13;
color: #2a5bc4;
}
.speaker-chip {
font-size: 12;
color: #222222;
border-radius: 10;
padding: 2 8;
margin: 0 6 4 0;
}
.speaker-chip-selected {
border-width: 2;
border-color: #222222;
font-weight: bold;
}
.speaker-tag {
font-size: 11;
color: #555555;
background-color: #e6e6e6;
border-radius: 8;
padding: 1 6;
}
.speaker-wearer-badge {
font-size: 11;
font-weight: bold;
color: #2a5bc4;
background-color: #dbe6fb;
border-radius: 8;
padding: 1 6;
}
.transcript-row {
border-radius: 8;
padding: 8;
margin-bottom: 4;
}
.transcript-row-playing {
background-color: #e6f0ff;
}
.transcript-translation {
font-size: 13;
font-style: italic;
color: #777777;
}
.transcript-time {
font-size: 11;
color: #888888;
}
.transcript-emotion {
font-size: 11;
color: #222222;
border-radius: 8;
padding: 1 6;
}
.transcript-play {
font-size: 16;
color: #2a5bc4;
margin-left: 6;
}
.ns-dark .conversation-row {
background-color: #2e2e2e;
border-color: #4a4a4a;
}
.ns-dark .conversation-title {
color: #f0f0f0;
}
.ns-dark .conversation-meta {
color: #a0a0a0;
}
.ns-dark .conversation-empty {
color: #999999;
}
.ns-dark .conversation-status {
color: #7da7f5;
}
.ns-dark .speaker-chip-selected {
border-color: #f0f0f0;
}
.ns-dark .speaker-tag {
color: #cccccc;
background-color: #3c3c3c;
}
.ns-dark .speaker-wearer-badge {
color: #a8c4f5;
background-color: #2a3b5e;
}
.ns-dark .transcript-row-playing {
background-color: #333c4c;
}
.ns-dark .transcript-translation {
color: #999999;
}
.ns-dark .transcript-time {
color: #909090;
}
.ns-dark .transcript-play {
color: #7da7f5;
}
+4
View File
@@ -1,11 +1,13 @@
import { type AppDefinition } from "./app-definition";
import launcherApp from "./launcher";
import timerApp from "./timer";
import calculatorApp from "./calculator";
import terminalApp from "./terminal";
import filesApp from "./files";
import musicApp from "./music";
import nightscoutApp from "./nightscout";
import transcribeApp from "./transcribe";
import microphonesApp from "./microphones";
import notificationsApp from "./notifications";
import calendarApp from "./calendar";
import weatherApp from "./weather";
@@ -28,11 +30,13 @@ import settingsApp from "./settings";
export const ALL_APPS: readonly AppDefinition[] = [
launcherApp,
timerApp,
calculatorApp,
terminalApp,
filesApp,
musicApp,
nightscoutApp,
transcribeApp,
microphonesApp,
notificationsApp,
calendarApp,
weatherApp,
+150
View File
@@ -0,0 +1,150 @@
/**
* Calculator window wiring: the in-process window, its long-press menu (mode
* follow-ups, clear, settings), the assistant tools, and the voice/text
* plumbing into the layer.
*
* The coordinator is a module singleton on purpose: a
* problem stated through the assistant's calculate tool is still standing
* when the wearer opens the window, and "graph it" there refers to it. The
* same session also survives closing and reopening the window.
*/
import {
createInProcessWindow,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { type MenuItem } from "../../ui/menu";
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { shell } from "../../ui/shell/shell";
import { toolRegistry } from "../../assistant/tool-registry";
import { type ToolSpec } from "../../assistant/tool-registry";
import { MathCoordinator } from "./math/coordinator";
import { rewriteSpokenProblem } from "./calculator-llm";
import {
CalculatorLayer,
calculatorDegreesSetting,
calculatorListeningSetting,
calculatorModeSetting,
} from "./calculator";
import { CALCULATOR_MODES, modeLabel } from "./calculator-commands";
export const CALCULATOR_WINDOW_ID = "calculator";
export const CALCULATOR_SURFACE_ID = "window:calculator";
/** One standing maths session, shared by the window and the assistant tools. */
const coordinator = new MathCoordinator();
coordinator.setAICompletion(rewriteSpokenProblem);
const CALCULATOR_TOOLS: ToolSpec[] = [
{
name: "calculate",
description:
"Solve a maths problem stated in words or notation ('2x + 7 = 19', 'what is 15 percent of 240', " +
"'is 91 prime', 'where does 11 first appear in pi', 'area of a circle with radius 3'). " +
"Do NOT work it out yourself first — hand the words over verbatim; the calculator answers " +
"exactly and shows the result on the glasses. Follow-ups like 'graph it' and 'explain it' " +
"act on the problem already standing.",
inputSchema: {
type: "object",
properties: {
problem: { type: "string", description: "The problem or follow-up, exactly as asked." },
},
required: ["problem"],
additionalProperties: false,
},
availability: "open",
},
];
export function createCalculatorAppWindow(options: InProcessAppOptions): InProcessWindow {
const layer = new CalculatorLayer(coordinator, options.actions);
const menuItems = (): MenuItem[] => [
...CALCULATOR_MODES.map(
(mode): MenuItem => ({
label: `${modeLabel(mode)} it`,
onSelect: (ctx) => {
ctx.stack.pop();
layer.performFollowUp(mode);
},
}),
),
{
label: "Clear",
onSelect: (ctx) => {
ctx.stack.pop();
layer.clear();
},
},
{
label: "Calculator settings",
onSelect: (ctx) => {
ctx.stack.pop();
openSettingsSubMenu(ctx, "Calculator settings", [
enumSettingMenuItem(calculatorModeSetting, {
onChange: () => layer.applySettings(),
}),
enumSettingMenuItem(calculatorListeningSetting, {
onChange: () => layer.applySettings(),
}),
toggleSettingMenuItem(calculatorDegreesSetting, {
onChange: () => layer.applySettings(),
}),
]);
},
},
];
let app: InProcessWindow;
app = createInProcessWindow({
appId: "calculator",
windowId: CALCULATOR_WINDOW_ID,
title: "Calculator",
iconLetter: "=",
icon: "calculator",
closeable: true,
menuItems,
actions: options.actions,
receiveTextInput: (text) => {
if (app.stack.receiveTextInput(text)) app.requestRender();
},
baseLayer: layer,
submitFrame: options.submitFrame,
setSurfaceVisible: (visible) => {
layer.setForeground(visible);
options.setSurfaceVisible(visible);
},
removeSurface: options.removeSurface,
onClosed: () => {
toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID);
options.onClosed();
},
});
layer.requestRender = app.requestRender;
layer.setForeground(shell.isWindowVisible(CALCULATOR_WINDOW_ID));
// Apply the persisted settings now the window exists: degrees onto the
// session, and continuous listening if the wearer left it on.
layer.applySettings();
toolRegistry.setAppTools({
windowId: CALCULATOR_WINDOW_ID,
appId: "calculator",
specs: CALCULATOR_TOOLS,
isForeground: () => shell.isWindowFocused(CALCULATOR_WINDOW_ID),
invoke: async (toolName, args) => {
if (toolName !== "calculate") {
return { ok: false, error: `Unknown calculator tool: ${toolName}` };
}
const problem = String((args as { problem?: unknown })?.problem ?? "").trim();
if (!problem) return { ok: false, error: "calculate requires a problem" };
const outcome = await coordinator.answerSpoken(problem);
app.requestRender();
if (outcome.kind === "failure") return { ok: false, error: outcome.reason };
const detail = coordinator.lastResult?.detail ?? "";
return { ok: true, content: detail || outcome.text };
},
});
return app;
}
+117
View File
@@ -0,0 +1,117 @@
// The pure classification half of the calculator's command handling. Kept
// free of any UI import so the tests can exercise exactly the
// decisions that matter: which utterances are commands, which unprompted room
// speech is worth answering, and how the modes map onto the shared session.
import { type MathBranchMode } from "./math/coordinator";
import { parseCommand } from "./math/session";
/**
* What a freshly captured problem does the moment it lands.
*
* Three, not more, because these are the three the wearer is TOLD about —
* "Solve", "Explain", "Graph" are the spoken commands, so making them the
* selectable modes means the touchpad and the voice are steering the same
* thing rather than two parallel vocabularies. Differentiation stays
* reachable by saying it ("the derivative of…"), which is how it is actually
* asked for.
*/
export type CalculatorMode = "solve" | "explain" | "graph";
export const CALCULATOR_MODES: readonly CalculatorMode[] = ["solve", "explain", "graph"];
export function modeLabel(mode: CalculatorMode): string {
switch (mode) {
case "solve":
return "Solve";
case "explain":
return "Explain";
case "graph":
return "Graph";
}
}
export function modeDetail(mode: CalculatorMode): string {
switch (mode) {
case "solve":
return "Answer it, exactly. 3/2 stays 3/2.";
case "explain":
return "Show the working step by step; scroll walks the rest.";
case "graph":
return "Plot it.";
}
}
export function modeBranch(mode: CalculatorMode): MathBranchMode {
return mode;
}
/**
* The spoken commands that select this mode. Both the bare verb and the
* "it" form, because someone shown "Solve" says both.
*/
export function modeSpokenCommands(mode: CalculatorMode): string[] {
switch (mode) {
case "solve":
return ["solve", "solve it"];
case "explain":
return ["explain", "explain it"];
case "graph":
return ["graph", "graph it"];
}
}
/**
* How the Calculator listens.
*
* "Tap to talk" arms exactly one finalized utterance per tap, so the app is
* not standing in the room claiming every sentence; "continuous" listens the
* whole time the app is foreground, gated by the looks-like-maths test
* below. The reason there are two input modes: a wearer in a
* quiet room must be able to take the microphone away from the calculator.
*/
export type CalculatorListening = "tap" | "continuous";
export function listeningLabel(listening: CalculatorListening): string {
switch (listening) {
case "tap":
return "Tap to talk";
case "continuous":
return "Continuous";
}
}
/**
* Recognise a bare mode command in an utterance. Exact match on the whole
* utterance, not a substring: "I need to solve this problem with the tax
* people" is conversation, and a substring test turns it into a command.
*/
export function modeCommand(text: string): CalculatorMode | null {
const normalized = text.toLowerCase().replace(/^[ .,!?"']+|[ .,!?"']+$/g, "");
for (const mode of CALCULATOR_MODES) {
if (modeSpokenCommands(mode).includes(normalized)) return mode;
}
return null;
}
export function isDictationCancellation(text: string): boolean {
const normalized = text.toLowerCase().replace(/^[ .,!?"']+|[ .,!?"']+$/g, "");
return ["cancel", "cancel listening", "stop listening", "never mind", "nevermind"].includes(normalized);
}
/**
* Whether unprompted room speech is worth handing to the parser. In
* continuous mode the app sees every finalized utterance in the room;
* without this gate, every sentence becomes a failed calculation on the
* lens.
*/
export function isWorthAnswering(text: string): boolean {
if (modeCommand(text) !== null) return true;
return parseCommand(text) !== null;
}
export function cycleMode(mode: CalculatorMode, forward: boolean): CalculatorMode {
const index = CALCULATOR_MODES.indexOf(mode);
const step = forward ? 1 : CALCULATOR_MODES.length - 1;
return CALCULATOR_MODES[(index + step) % CALCULATOR_MODES.length]!;
}
+96
View File
@@ -0,0 +1,96 @@
// Draws the maths engine's already-sampled graph into the app's GrayImage viewport.
// The renderer never re-evaluates the expression: segment breaks, viewport
// clipping, roots, and extrema therefore stay exactly what the plotter
// decided, and an asymptote correctly stays an open gap rather than a
// vertical line — the single most recognisable "bad graphing app" artefact.
import { GrayImage } from "../../graphics/image";
import { type Graph, type GraphPoint, viewportContainsY } from "./math/plotter";
// Brightness levels tuned for the G2's grayscale panel: the curve stays
// visually dominant over the axes, and the axes over the tick grid.
const GRID_VALUE = 38;
const AXIS_VALUE = 150;
const BORDER_VALUE = 105;
const CURVE_VALUES = [245, 200, 160];
const MARKER_VALUE = 255;
/**
* Draw the graph into `rect` (x, y, width, height) of the image. The caller
* owns the header and footer rows.
*/
export function drawGraph(
image: GrayImage,
graph: Graph,
rect: { x: number; y: number; width: number; height: number },
): void {
if (rect.width <= 1 || rect.height <= 1) return;
const viewport = graph.viewport;
const spanX = Math.max(viewport.xMax - viewport.xMin, Number.EPSILON);
const spanY = Math.max(viewport.yMax - viewport.yMin, Number.EPSILON);
const px = (x: number): number => rect.x + ((x - viewport.xMin) / spanX) * rect.width;
const py = (y: number): number => rect.y + rect.height - ((y - viewport.yMin) / spanY) * rect.height;
const clampX = (x: number): number => Math.max(rect.x, Math.min(rect.x + rect.width - 1, x));
const clampY = (y: number): number => Math.max(rect.y, Math.min(rect.y + rect.height - 1, y));
// Tick grid first so the curve and axes draw over it.
for (const tick of graph.xAxis.ticks) {
if (tick < viewport.xMin || tick > viewport.xMax) continue;
const x = Math.round(px(tick));
image.drawLine(x, rect.y, x, rect.y + rect.height - 1, GRID_VALUE);
}
for (const tick of graph.yAxis.ticks) {
if (tick < viewport.yMin || tick > viewport.yMax) continue;
const y = Math.round(py(tick));
image.drawLine(rect.x, y, rect.x + rect.width - 1, y, GRID_VALUE);
}
// Axes.
if (viewport.yMin <= 0 && viewport.yMax >= 0) {
const y = Math.round(py(0));
image.drawLine(rect.x, y, rect.x + rect.width - 1, y, AXIS_VALUE);
}
if (viewport.xMin <= 0 && viewport.xMax >= 0) {
const x = Math.round(px(0));
image.drawLine(x, rect.y, x, rect.y + rect.height - 1, AXIS_VALUE);
}
// Curves, brightest first plot.
for (let plotIndex = 0; plotIndex < graph.plots.length; plotIndex++) {
const plot = graph.plots[plotIndex]!;
const value = CURVE_VALUES[plotIndex % CURVE_VALUES.length]!;
for (const segment of plot.segments) {
let previous: GraphPoint | null = null;
for (const point of segment.points) {
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) {
previous = null;
continue;
}
if (previous) {
image.drawLine(
Math.round(clampX(px(previous.x))),
Math.round(clampY(py(previous.y))),
Math.round(clampX(px(point.x))),
Math.round(clampY(py(point.y))),
value,
);
}
previous = point;
}
}
// Markers: roots and intercepts as small filled dots.
for (const marker of plot.markers) {
if (!viewportContainsY(viewport, marker.point.y)) continue;
if (marker.point.x < viewport.xMin || marker.point.x > viewport.xMax) continue;
const cx = Math.round(px(marker.point.x));
const cy = Math.round(py(marker.point.y));
image.fillRoundedRect(cx - 3, cy - 3, 6, 6, MARKER_VALUE, 3);
}
}
// Frame the plot area.
image.drawRect(rect.x, rect.y, rect.width, rect.height, BORDER_VALUE);
}
+51
View File
@@ -0,0 +1,51 @@
/**
* One-shot completion against the assistant's configured model, used by the
* Calculator's optional rewrite fallback (see MathCoordinator.answerSpoken:
* the deterministic parser runs first; the model only ever translates words
* into notation). Resolves to null when no provider is configured or the
* call fails — the calculator then reports "could not read that" instead of
* surfacing a transport error.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel } from "../../assistant/models";
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
import {
anthropicApiKeySetting,
assistantModelSetting,
openAiApiKeySetting,
} from "../../ui/dashboard-settings";
import { CALCULATOR_REWRITE_SYSTEM_PROMPT, buildCalculatorRewriteUserMessage } from "../../prompts";
export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
const resolved = resolveAssistantModel(assistantModelSetting.get(), {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
if (!resolved) return Promise.resolve(null);
return new Promise((resolve) => {
const options: LlmStreamOptions = {
apiKey: resolved.apiKey,
model: resolved.model,
system: CALCULATOR_REWRITE_SYSTEM_PROMPT,
effort: "low",
maxTokens: 200,
messages: [{ role: "user", content: buildCalculatorRewriteUserMessage(spoken) }],
onDone: (result) => resolve(result.text ?? null),
onError: () => resolve(null),
};
switch (resolved.provider) {
case "openai":
streamOpenAiResponse(options);
break;
case "local":
streamLocalQwen(options);
break;
default:
streamAnthropicMessage(options);
break;
}
});
}
+606
View File
@@ -0,0 +1,606 @@
// The calculator app layer for FaceClaw: a spoken (or typed) problem in, an
// exact answer on the lens.
//
// Deliberately thin. Everything mathematical belongs to math/ and everything
// about the standing problem, follow-ups, and long runs belongs to
// MathCoordinator. This layer owns only what is genuinely the app's: which
// mode a fresh capture lands in, what the viewport shows, and how the
// touchpad and the microphone are shared.
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultLargeFont, getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { wrapText, truncateText } from "../../graphics/textwrap";
import { lineStep } from "../../ui/metrics";
import { gestureHints, GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../../ui/gestures";
import { type DashboardInputEvent, type Layer, type LayerActions, type LayerContext } from "../../ui/layers";
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
import { shell } from "../../ui/shell/shell";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator";
import {
type LongRunOffer,
type LongRunProgress,
offerGlassesPrompt,
progressText,
} from "./math/workload";
import { plain } from "./math/text-renderer";
import { drawGraph } from "./calculator-graph";
import {
type CalculatorListening,
type CalculatorMode,
CALCULATOR_MODES,
cycleMode,
isDictationCancellation,
isWorthAnswering,
listeningLabel,
modeCommand,
modeLabel,
} from "./calculator-commands";
// ---------------------------------------------------------------------------
// Settings
/** The mode a fresh capture lands in. Also what scrolling cycles. */
export const calculatorModeSetting = new ConfigSettingEnum<CalculatorMode>({
id: "calculator-mode",
label: "Default mode",
storageKey: "calculator.defaultMode",
defaultValue: "solve",
values: [...CALCULATOR_MODES],
formatValue: modeLabel,
description:
"What a freshly stated problem does. On the glasses, scroll applies another mode — or walks Explain steps — and tap toggles the mic.",
});
export const calculatorListeningSetting = new ConfigSettingEnum<CalculatorListening>({
id: "calculator-listening",
label: "Listening",
storageKey: "calculator.listening",
defaultValue: "tap",
values: ["tap", "continuous"],
formatValue: listeningLabel,
description:
"Tap to talk captures one problem per tap. Continuous listens the whole time the calculator is in front, answering anything that sounds like maths.",
});
export const calculatorDegreesSetting = new ConfigSettingBoolean({
id: "calculator-degrees",
label: "Trigonometry in degrees",
storageKey: "calculator.usesDegrees",
defaultValue: false,
description: "Evaluate sin/cos/tan in degrees instead of radians.",
});
// ---------------------------------------------------------------------------
// Layer
export class CalculatorLayer implements Layer {
/** Rebound by the window factory once the window exists. */
requestRender: () => void = () => {};
/**
* The last thing said to the calculator, so the view can show what it
* heard next to what it made of it — the only way a wearer can tell a
* mis-heard problem from a wrong answer.
*/
private lastInput = "";
private lastAnswer = "";
private lastFailure: string | null = null;
private working = false;
/** True while one finalized utterance is armed for capture. */
private dictating = false;
private foreground = true;
private continuousActive = false;
private unsubscribeTranscript: (() => void) | null = null;
private lastDeepAnswer: MathBranchOutcome | null = null;
constructor(
private readonly coordinator: MathCoordinator,
private readonly actions: LayerActions,
) {
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
}
private get mode(): CalculatorMode {
return calculatorModeSetting.get();
}
private get listening(): CalculatorListening {
return calculatorListeningSetting.get();
}
// -------------------------------------------------------------------------
// Lifecycle
onRemoved(): void {
this.stopListening();
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.coordinator.onChanged = null;
}
/** Foreground/background transitions, forwarded by the window factory. */
setForeground(foreground: boolean): void {
if (this.foreground === foreground) return;
this.foreground = foreground;
if (!foreground) {
this.stopListening();
} else {
this.refreshListening();
}
}
/** Apply a settings change: degrees and the listening mode both act immediately. */
applySettings(): void {
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
this.refreshListening();
this.requestRender();
}
private refreshListening(): void {
if (!this.foreground) return;
if (this.listening === "continuous") {
if (!this.continuousActive) {
this.continuousActive = true;
void this.actions.startContinuousVoiceCapture();
}
} else if (this.continuousActive) {
this.continuousActive = false;
void this.actions.stopContinuousVoiceCapture();
}
}
private stopListening(): void {
if (this.dictating) {
this.dictating = false;
void this.actions.stopVoiceCapture();
}
if (this.continuousActive) {
this.continuousActive = false;
void this.actions.stopContinuousVoiceCapture();
}
}
// -------------------------------------------------------------------------
// Asking
/**
* Put a problem (or a follow-up command) to the calculator.
*
* Command first, problem second. "graph it" contains no operator and would
* fail as an expression, and a wearer who says it expects the plot of what
* is already on the lens rather than an error about the word "it".
*/
submit(text: string): void {
const trimmed = text.trim();
if (!trimmed) return;
this.lastInput = trimmed;
this.lastFailure = null;
const selected = modeCommand(trimmed);
if (selected) {
this.performFollowUp(selected);
return;
}
const target = this.mode;
this.working = true;
this.requestRender();
void this.coordinator.answerSpoken(trimmed).then((outcome) => {
// A fresh problem lands in the app's mode: solving is the default, but
// a wearer whose default is "graph" said the problem in order to see it
// plotted, not to read its root.
if (outcome.kind === "value" && target !== "solve") {
outcome = this.coordinator.run(target, "");
}
this.working = false;
this.applyOutcome(outcome);
});
}
/**
* Re-run the standing problem through a different presentation branch.
* This is the one action used by scroll cycling, spoken "… it" commands,
* and the window menu, so changing mode can never mean "change a setting
* but leave the old answer up".
*/
performFollowUp(selected: CalculatorMode): void {
calculatorModeSetting.set(selected);
if (!this.coordinator.hasProblem) {
this.lastFailure = "No problem yet";
this.requestRender();
return;
}
this.working = false;
this.lastFailure = null;
this.applyOutcome(this.coordinator.run(selected, ""));
}
clear(): void {
this.working = false;
this.lastInput = "";
this.lastAnswer = "";
this.lastFailure = null;
this.coordinator.handleRingAction("exit");
this.requestRender();
}
private applyOutcome(outcome: MathBranchOutcome | null): void {
if (outcome?.kind === "value") {
this.lastAnswer = outcome.text;
this.lastFailure = null;
} else if (outcome?.kind === "failure") {
this.lastFailure = outcome.reason;
}
this.requestRender();
}
private handleCoordinatorChanged(): void {
// The answer a deeper pass produced arrives on its own channel; fold it
// into the standing answer the moment it lands.
const deep = this.coordinator.deepAnswer;
if (deep && deep !== this.lastDeepAnswer) {
this.lastDeepAnswer = deep;
this.applyOutcome(deep);
return;
}
this.requestRender();
}
// -------------------------------------------------------------------------
// Dictation
/**
* Arm exactly one finalized utterance. One, not a listening window: the
* indicator on the lens is what makes this honest, and a calculator that
* quietly held the microphone open would be indistinguishable from one
* that did not.
*/
beginDictation(): void {
if (this.dictating) return;
this.dictating = true;
void this.actions.startVoiceCapture(true);
this.requestRender();
}
cancelDictation(): void {
if (!this.dictating) return;
this.dictating = false;
void this.actions.stopVoiceCapture();
this.requestRender();
}
/**
* A finalized utterance reached the app.
*
* In tap mode this only ever arrives because the wearer armed it, so it is
* always taken. In continuous mode it also arrives from ordinary room
* speech, and there it has to clear the looks-like-maths gate first —
* otherwise every sentence in the room becomes a failed calculation on the
* lens.
*/
private onTranscript(event: VoiceTranscriptEvent): void {
if (!event.isFinal) return;
const text = event.text.trim();
if (!text) return;
const armed = this.dictating;
const continuous = this.continuousActive && this.foreground;
if (!armed && !continuous) return;
if (armed) {
this.dictating = false;
void this.actions.stopVoiceCapture();
}
// A standing offer or a run in flight owns the next yes/no said in the
// room ("keep going", "never mind", "stop").
const longRunAnswer = this.coordinator.handleLongRunUtterance(text);
if (longRunAnswer) {
this.applyOutcome(longRunAnswer);
return;
}
if (armed && isDictationCancellation(text)) {
this.requestRender();
return;
}
if (!armed && !isWorthAnswering(text)) return;
this.submit(text);
}
// -------------------------------------------------------------------------
// Input
receiveTextInput(text: string): void {
this.submit(text);
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
case "scroll-down": {
const forward = event.type === "scroll-down";
// While Explain is showing a step sequence the scroll walks it — the
// coordinator owns that, and asking it first is what keeps one
// gesture from meaning two things at once. Otherwise scroll cycles
// the MODE, because the mode is what the wearer changes between
// problems and the steps only exist inside one of the three modes.
if (this.mode === "explain" && this.coordinator.stepPosition !== null) {
this.applyOutcome(this.coordinator.handleRingAction(forward ? "forward" : "back"));
return;
}
const next = cycleMode(this.mode, forward);
if (this.coordinator.hasProblem) {
this.performFollowUp(next);
} else {
calculatorModeSetting.set(next);
this.requestRender();
}
return;
}
case "click": {
// A standing offer owns the tap: accept and start the long run.
if (this.coordinator.hasLongRunInteraction) {
this.applyOutcome(this.coordinator.handleRingAction("commit"));
return;
}
if (this.dictating) {
this.cancelDictation();
} else {
this.beginDictation();
}
return;
}
case "double-click": {
if (this.dictating) {
this.cancelDictation();
return;
}
if (this.coordinator.isRunningLong || this.coordinator.pendingLongRun) {
// Stop the run, or turn the offer down; the shallow answer stands.
this.applyOutcome(this.coordinator.handleRingAction("exit"));
return;
}
// The standard leave-the-app gesture.
shell.yieldFocusToSidebar();
return;
}
default:
return;
}
}
// -------------------------------------------------------------------------
// Paint
paint(ctx: LayerContext): GrayImage {
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const small = getDefaultSmallFont();
this.paintTabs(image, small, width);
const footerY = height - small.lineHeight - 6;
const bodyTop = 46;
let bodyBottom = footerY - 4;
// A standing offer or a running deep pass takes the rows just above the
// footer, whatever view is behind it.
const offer = this.coordinator.pendingLongRun;
const progress = this.coordinator.longRunProgress;
if (offer || progress) {
bodyBottom = this.paintLongRunPanel(image, small, width, bodyBottom, offer, progress);
}
if (this.working) {
this.drawCentered(image, small, Math.round((bodyTop + bodyBottom) / 2) - 8, "Working…", 180);
} else if (this.lastFailure !== null) {
this.paintFailure(image, small, width, bodyTop, bodyBottom);
} else if (this.mode === "explain" && this.coordinator.stepPosition !== null) {
this.paintExplain(image, width, bodyTop, bodyBottom);
} else if (this.mode === "graph" && this.coordinator.lastGraph) {
this.paintGraph(image, small, width, bodyTop, bodyBottom);
} else if (this.lastAnswer) {
this.paintAnswer(image, width, bodyTop, bodyBottom);
} else {
this.paintEmpty(image, small, width, bodyTop, bodyBottom);
}
image.drawText(small, 24, footerY, this.footerHints(), 110);
return image;
}
/** The three modes as tabs, plus the deg/listening status chips. */
private paintTabs(image: GrayImage, font: UiFont, width: number): void {
let x = 24;
for (const mode of CALCULATOR_MODES) {
const label = modeLabel(mode);
const active = mode === this.mode;
image.drawText(font, x, 15, label, active ? 245 : 105);
image.drawLine(x, 35, x + font.measureText(label), 35, active ? 200 : 30);
x += font.measureText(label) + 18;
}
const chips: string[] = [];
if (calculatorDegreesSetting.get()) chips.push("deg");
if (this.dictating) chips.push("listening…");
else if (this.continuousActive && this.foreground) chips.push("mic on");
if (chips.length > 0) {
const text = chips.join(" · ");
image.drawText(font, width - 24 - font.measureText(text), 15, text, this.dictating ? 220 : 110);
}
}
private paintFailure(
image: GrayImage,
font: UiFont,
width: number,
bodyTop: number,
bodyBottom: number,
): void {
this.paintHeardLine(image, font, width, bodyTop);
const top = this.lastInput ? bodyTop + lineStep(font) + 6 : bodyTop + 8;
const lines = wrapText(font, this.lastFailure ?? "", width - 64);
const step = lineStep(font);
const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step));
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
image.drawText(font, 32, top + index * step, lines[index]!, 200);
}
}
private paintExplain(image: GrayImage, width: number, bodyTop: number, bodyBottom: number): void {
const small = getDefaultSmallFont();
const medium = getDefaultMediumFont();
const position = this.coordinator.stepPosition;
const steps = this.coordinator.explanationSteps;
const step = position ? steps[position.index] : undefined;
if (!position || !step) return;
let y = bodyTop + 2;
image.drawText(small, 24, y, `Step ${position.index + 1} of ${position.total}`, 130);
y += lineStep(small) + 2;
image.drawText(medium, 24, y, truncateText(medium, step.title, width - 48), 245);
y += lineStep(medium) + 2;
image.drawText(medium, 24, y, truncateText(medium, plain(step.expression), width - 48), 220);
y += lineStep(medium) + 6;
const detailLines = wrapText(small, step.detail, width - 64);
const stepPitch = lineStep(small);
const maxLines = Math.max(0, Math.floor((bodyBottom - y) / stepPitch));
for (let index = 0; index < Math.min(detailLines.length, maxLines); index++) {
image.drawText(small, 32, y + index * stepPitch, detailLines[index]!, 150);
}
}
private paintGraph(
image: GrayImage,
font: UiFont,
width: number,
bodyTop: number,
bodyBottom: number,
): void {
const graph = this.coordinator.lastGraph;
if (!graph) return;
const headline = this.lastAnswer.split("\n")[0] ?? "";
image.drawText(font, 24, bodyTop, truncateText(font, headline, width - 48), 200);
const plotTop = bodyTop + lineStep(font) + 4;
drawGraph(image, graph, {
x: 18,
y: plotTop,
width: width - 36,
height: Math.max(2, bodyBottom - plotTop),
});
}
private paintAnswer(image: GrayImage, width: number, bodyTop: number, bodyBottom: number): void {
const small = getDefaultSmallFont();
this.paintHeardLine(image, small, width, bodyTop);
const top = this.lastInput ? bodyTop + lineStep(small) + 4 : bodyTop;
// A short single-line answer earns the large font; anything longer drops
// to the medium font and wraps.
const lines = this.lastAnswer.split("\n");
const large = getDefaultLargeFont();
if (lines.length === 1 && large.measureText(lines[0]!) <= width - 64) {
const y = Math.round((top + bodyBottom - large.lineHeight) / 2);
this.drawCentered(image, large, y, lines[0]!, 245);
return;
}
const medium = getDefaultMediumFont();
const wrapped = lines.flatMap((line) => wrapText(medium, line, width - 64));
const step = lineStep(medium);
const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step));
const shown = wrapped.slice(0, maxLines);
let y = Math.max(top, Math.round((top + bodyBottom - shown.length * step) / 2));
for (const line of shown) {
this.drawCentered(image, medium, y, line, 235);
y += step;
}
}
private paintEmpty(
image: GrayImage,
font: UiFont,
width: number,
bodyTop: number,
bodyBottom: number,
): void {
const prompt =
this.listening === "continuous"
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
: "Tap to say a problem — or use Voice input to type one";
const lines = wrapText(font, prompt, width - 96);
const step = lineStep(font);
let y = Math.round((bodyTop + bodyBottom - lines.length * step) / 2);
for (const line of lines) {
this.drawCentered(image, font, y, line, 150);
y += step;
}
}
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
if (!this.lastInput) return;
const label = "Heard";
image.drawText(font, 24, y, label, 110);
const x = 24 + font.measureText(label) + 10;
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
}
/**
* The standing offer, or the live progress of an accepted run, in the two
* rows above the footer. Returns the new body bottom.
*/
private paintLongRunPanel(
image: GrayImage,
font: UiFont,
width: number,
bodyBottom: number,
offer: LongRunOffer | null,
progress: LongRunProgress | null,
): number {
const step = lineStep(font);
const lines = progress
? [`${progressText(progress)} · ${Math.round(progress.elapsedSeconds)}s`]
: offer
? offerGlassesPrompt(offer).split("\n")
: [];
const panelTop = bodyBottom - lines.length * step - 6;
image.drawLine(16, panelTop, width - 16, panelTop, 40);
for (let index = 0; index < lines.length; index++) {
image.drawText(font, 24, panelTop + 4 + index * step, truncateText(font, lines[index]!, width - 48), 190);
}
return panelTop - 2;
}
private footerHints(): string {
if (this.coordinator.longRunProgress) {
return gestureHints([[GESTURE_DOUBLE_CLICK, "stop"]]);
}
if (this.coordinator.pendingLongRun) {
return gestureHints([
[GESTURE_CLICK, "keep going"],
[GESTURE_DOUBLE_CLICK, "leave it"],
]);
}
if (this.dictating) {
return gestureHints([
[GESTURE_CLICK, "cancel mic"],
[GESTURE_DOUBLE_CLICK, "cancel"],
]);
}
const scrollTarget =
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
return gestureHints([
[GESTURE_SCROLL, scrollTarget],
[GESTURE_CLICK, "mic"],
[GESTURE_DOUBLE_CLICK, "back"],
]);
}
private drawCentered(image: GrayImage, font: UiFont, y: number, text: string, value: number): void {
const x = Math.max(0, Math.round((image.width - font.measureText(text)) / 2));
image.drawText(font, x, y, text, value);
}
}
+20
View File
@@ -0,0 +1,20 @@
import { type AppDefinition } from "../app-definition";
import {
CALCULATOR_SURFACE_ID,
CALCULATOR_WINDOW_ID,
createCalculatorAppWindow,
} from "./calculator-app";
/**
* A spoken (or typed) problem in, an exact answer on the lens — with
* step-by-step explanations, graphs, and the advanced spoken-question
* topics (primality, integrals, digit searches in π, geometry).
*/
const calculatorApp: AppDefinition = {
appId: "calculator",
title: "Calculator",
icon: "calculator",
launch: (ctx) => ctx.launchInProcessApp(CALCULATOR_WINDOW_ID, CALCULATOR_SURFACE_ID, createCalculatorAppWindow),
};
export default calculatorApp;
+796
View File
@@ -0,0 +1,796 @@
// Routes a spoken question to the right advanced-maths topic. Separate from
// the command parser because these questions have a different shape: a
// command is about an EXPRESSION and what to do with it, whereas "is 91
// prime", "factor 360", "area of a circle with radius 3" name a topic and
// carry their own arguments. Trying to force both through one parser made
// the expression grammar swallow phrases it had no business touching.
import { type MathExpression, variablesOf } from "./expression";
import { parseSpoken } from "./parser";
import { applyNumberWords } from "./normalizer";
import { plain } from "./text-renderer";
import { describeNumber, formatInt, fromDecimal } from "./math-number";
import {
type MathConstantName,
MAXIMUM_COMPUTED_DIGITS,
constantDisplayName,
constantNamed,
constantSpokenName,
} from "./constant-digits";
import {
countDigits as countDigitsIn,
digitsAt,
expectedFirstPosition,
findDigits,
highlightedMatch,
} from "./digit-search";
import { type MathWorkload, longRunOffer as makeLongRunOffer, type LongRunOffer } from "./workload";
import {
binomial,
factorizationText,
fibonacci,
greatestCommonDivisor,
integerReport,
isPrime,
leastCommonMultiple,
modularInverse,
} from "./number-theory";
import {
type Shape,
pythagoras,
shapeName,
shapeSummary,
triangleAnglesInDegrees,
triangleArea,
triangleFromSides,
triangleKind,
trianglePerimeter,
} from "./geometry";
import { TRIG_IDENTITIES, familyText, identityMatching, solveTrig } from "./trigonometry";
import { criticalPointText, criticalPoints, limit, limitText, taylor } from "./calculus";
import { definite, integrateWithConstant } from "./symbolic-integral";
// ---------------------------------------------------------------------------
// Topics
export type AdvancedTopic =
| { kind: "integerReport"; value: number }
| { kind: "primality"; value: number }
| { kind: "factorize"; value: number }
| { kind: "gcd"; a: number; b: number }
| { kind: "lcm"; a: number; b: number }
| { kind: "modularInverse"; value: number; modulus: number }
| { kind: "fibonacci"; n: number }
| { kind: "binomial"; n: number; k: number }
| { kind: "symbolicIntegral"; expression: MathExpression; variable: string }
| { kind: "definiteIntegral"; expression: MathExpression; variable: string; from: number; to: number }
| { kind: "limit"; expression: MathExpression; variable: string; at: number }
| { kind: "taylor"; expression: MathExpression; variable: string; order: number }
| { kind: "criticalPoints"; expression: MathExpression; variable: string }
| { kind: "trigSolve"; fn: "sin" | "cos" | "tan"; value: number }
| { kind: "trigIdentity"; query: string }
/** "where does 11 first appear in pi" */
| { kind: "digitSearch"; pattern: string; constant: MathConstantName; occurrence: number }
/** "how many times does 42 appear in pi" */
| { kind: "digitCount"; pattern: string; constant: MathConstantName }
/** "what are digits 100 to 110 of pi" */
| { kind: "digitsAt"; constant: MathConstantName; from: number; to: number }
| { kind: "shapeArea"; shape: Shape }
| { kind: "triangleFromSides"; a: number; b: number; c: number }
| { kind: "pythagoras"; a: number | null; b: number | null; c: number | null };
export type AdvancedAnswer = {
headline: string;
detail: string;
/** Extra rows for the answer view, and for a longer readback. */
fields: [string, string][];
/**
* Set when this answer is only as good as the fast expansion allowed, and
* a deeper opted-in run would do better. The router itself never starts
* one: it is pure and synchronous, and the wait is a decision belonging to
* the caller.
*/
unresolvedWorkload: MathWorkload | null;
};
// ---------------------------------------------------------------------------
// Parsing
/**
* null when the utterance is not an advanced-maths question, so the caller
* can fall through to the ordinary expression path.
*/
export function parseAdvanced(utterance: string): AdvancedTopic | null {
const text = applyNumberWords(utterance.toLowerCase()).replace(/^[ .,!?]+|[ .,!?]+$/g, "");
// Constants first: "the first 11 in pi" contains digits and the word "in",
// which the geometry and number-theory matchers would both take a swing at.
const constantTopic = parseConstantDigits(text);
if (constantTopic) return constantTopic;
const numberTopic = parseNumberTheory(text);
if (numberTopic) return numberTopic;
const calculusTopic = parseCalculus(text, utterance);
if (calculusTopic) return calculusTopic;
const trigTopic = parseTrigonometry(text);
if (trigTopic) return trigTopic;
return parseGeometry(text);
}
// Constant digits ------------------------------------------------------------
function parseConstantDigits(text: string): AdvancedTopic | null {
// Must name a constant AND talk about its expansion, or "area of a circle"
// (which contains pi implicitly) would land here.
const constant = constantNamed(text);
if (!constant) return null;
const mentionsDigits =
text.includes("digit") ||
text.includes("appear") ||
text.includes("occur") ||
text.includes("within") ||
text.includes("inside") ||
text.includes("expansion") ||
text.includes("decimal place") ||
text.includes("position") ||
text.includes("show up") ||
text.includes("found in");
if (!mentionsDigits) return null;
const numbers = allInts(text);
// "digits 100 to 110 of pi"
if (
text.includes("digit") &&
text.includes(" to ") &&
numbers.length >= 2 &&
!text.includes("appear") &&
!text.includes("occur")
) {
const sorted = [numbers[0]!, numbers[1]!].sort((a, b) => a - b);
return { kind: "digitsAt", constant, from: sorted[0]!, to: sorted[1]! };
}
// "how many times does 42 appear"
if (text.includes("how many") || text.includes("count")) {
const pattern = numbers[0];
if (pattern === undefined) return null;
return { kind: "digitCount", pattern: String(pattern), constant };
}
const pattern = numbers[0];
if (pattern === undefined) return null;
// "the third time 11 appears"
const occurrence = ordinalIn(text) ?? 1;
return { kind: "digitSearch", pattern: String(pattern), constant, occurrence };
}
/**
* "3rd", not "3th". Cheap to get right and jarring to get wrong on a line
* someone reads aloud.
*/
export function ordinalWord(value: number): string {
switch (value) {
case 1:
return "first";
case 2:
return "second";
case 3:
return "third";
default:
break;
}
let suffix: string;
const mod100 = value % 100;
const mod10 = value % 10;
if (mod100 === 11 || mod100 === 12 || mod100 === 13) suffix = "th";
else if (mod10 === 1) suffix = "st";
else if (mod10 === 2) suffix = "nd";
else if (mod10 === 3) suffix = "rd";
else suffix = "th";
return `${value}${suffix}`;
}
function ordinalIn(text: string): number | null {
const words: [string, number][] = [
["first", 1],
["second", 2],
["third", 3],
["fourth", 4],
["fifth", 5],
["sixth", 6],
["seventh", 7],
["eighth", 8],
["ninth", 9],
["tenth", 10],
];
for (const [word, value] of words) {
if (text.includes(word)) return value;
}
return null;
}
// Number theory --------------------------------------------------------------
function parseNumberTheory(text: string): AdvancedTopic | null {
if (text.includes("prime") && !text.includes("factor")) {
const value = firstInt(text);
if (value !== null) return { kind: "primality", value };
}
if (text.includes("factor")) {
const value = firstInt(text);
if (value !== null) return { kind: "factorize", value };
}
if (text.includes("gcd") || text.includes("greatest common") || text.includes("highest common")) {
const values = allInts(text);
if (values.length >= 2) return { kind: "gcd", a: values[0]!, b: values[1]! };
}
if (
text.includes("lcm") ||
text.includes("lowest common multiple") ||
text.includes("least common multiple")
) {
const values = allInts(text);
if (values.length >= 2) return { kind: "lcm", a: values[0]!, b: values[1]! };
}
if (text.includes("inverse") && text.includes("mod")) {
const values = allInts(text);
if (values.length >= 2) return { kind: "modularInverse", value: values[0]!, modulus: values[1]! };
}
if (text.includes("fibonacci")) {
const value = firstInt(text);
if (value !== null) return { kind: "fibonacci", n: value };
}
if (text.includes("choose") || text.includes("binomial") || text.includes("combinations")) {
const values = allInts(text);
if (values.length >= 2) return { kind: "binomial", n: values[0]!, k: values[1]! };
}
if (text.includes("tell me about") || text.includes("facts about")) {
const value = firstInt(text);
if (value !== null) return { kind: "integerReport", value };
}
return null;
}
// Calculus -------------------------------------------------------------------
function parseCalculus(text: string, original: string): AdvancedTopic | null {
if (text.includes("integral") || text.includes("integrate") || text.includes("antiderivative")) {
const expression = expressionTail(original, ["integral of", "integrate", "antiderivative of", "integral"]);
if (!expression) return null;
const variable = variablesOf(expression)[0] ?? "x";
const bounds = boundsIn(text);
if (bounds) {
return { kind: "definiteIntegral", expression, variable, from: bounds[0], to: bounds[1] };
}
return { kind: "symbolicIntegral", expression, variable };
}
if (text.includes("limit")) {
const expression = expressionTail(original, ["limit of", "the limit of", "limit"]);
if (!expression) return null;
// "as x approaches 0"
const point = firstDoubleAfter(["approaches", "approaching", "tends to", "goes to"], text) ?? 0;
return { kind: "limit", expression, variable: variablesOf(expression)[0] ?? "x", at: point };
}
if (text.includes("taylor") || text.includes("maclaurin") || text.includes("series")) {
const expression = expressionTail(original, [
"series of",
"series for",
"expansion of",
"taylor",
"maclaurin",
]);
if (!expression) return null;
const orderValue = firstDoubleAfter(["order", "to order", "terms"], text);
const order = orderValue !== null ? Math.trunc(orderValue) : 5;
return { kind: "taylor", expression, variable: variablesOf(expression)[0] ?? "x", order };
}
if (
text.includes("critical point") ||
text.includes("turning point") ||
text.includes("maximum") ||
text.includes("minimum") ||
text.includes("extrema")
) {
const expression = expressionTail(original, [
"critical points of",
"turning points of",
"extrema of",
"maximum of",
"minimum of",
"of",
]);
if (!expression) return null;
return { kind: "criticalPoints", expression, variable: variablesOf(expression)[0] ?? "x" };
}
return null;
}
// Trigonometry ---------------------------------------------------------------
function parseTrigonometry(text: string): AdvancedTopic | null {
if (text.includes("identity") || text.includes("law of")) {
return { kind: "trigIdentity", query: text };
}
// "solve sin x = 0.5"
for (const name of ["sin", "cos", "tan"] as const) {
if (!text.includes(name)) continue;
if (!text.includes("=") && !text.includes("equals")) continue;
const value = lastDouble(text);
if (value === null) continue;
return { kind: "trigSolve", fn: name, value };
}
return null;
}
// Geometry -------------------------------------------------------------------
function parseGeometry(text: string): AdvancedTopic | null {
const numbers = allDoubles(text);
if (text.includes("pythagoras") || text.includes("hypotenuse")) {
if (numbers.length < 2) return null;
return text.includes("hypotenuse") && text.includes("find")
? { kind: "pythagoras", a: numbers[0]!, b: numbers[1]!, c: null }
: { kind: "pythagoras", a: numbers[0]!, b: null, c: numbers[1]! };
}
if (text.includes("triangle") && numbers.length >= 3 && text.includes("side")) {
return { kind: "triangleFromSides", a: numbers[0]!, b: numbers[1]!, c: numbers[2]! };
}
// Shapes, longest name first so "rectangular prism" is not read as
// "rectangle".
const shapes: [string, (values: number[]) => Shape | null][] = [
[
"rectangular prism",
(values) =>
values.length >= 3
? { kind: "rectangularPrism", length: values[0]!, width: values[1]!, height: values[2]! }
: null,
],
["sphere", (values) => (values.length >= 1 ? { kind: "sphere", radius: values[0]! } : null)],
[
"cylinder",
(values) => (values.length >= 2 ? { kind: "cylinder", radius: values[0]!, height: values[1]! } : null),
],
["cone", (values) => (values.length >= 2 ? { kind: "cone", radius: values[0]!, height: values[1]! } : null)],
["cube", (values) => (values.length >= 1 ? { kind: "cube", side: values[0]! } : null)],
["circle", (values) => (values.length >= 1 ? { kind: "circle", radius: values[0]! } : null)],
[
"rectangle",
(values) => (values.length >= 2 ? { kind: "rectangle", width: values[0]!, height: values[1]! } : null),
],
["square", (values) => (values.length >= 1 ? { kind: "square", side: values[0]! } : null)],
[
"trapezoid",
(values) =>
values.length >= 3 ? { kind: "trapezoid", a: values[0]!, b: values[1]!, height: values[2]! } : null,
],
["ellipse", (values) => (values.length >= 2 ? { kind: "ellipse", a: values[0]!, b: values[1]! } : null)],
];
const wantsMeasurement =
text.includes("area") ||
text.includes("volume") ||
text.includes("perimeter") ||
text.includes("circumference") ||
text.includes("surface");
if (!wantsMeasurement) return null;
for (const [name, build] of shapes) {
if (!text.includes(name)) continue;
const shape = build(numbers);
if (shape) return { kind: "shapeArea", shape };
}
return null;
}
// ---------------------------------------------------------------------------
// Answering
export function answerAdvanced(topic: AdvancedTopic): AdvancedAnswer {
switch (topic.kind) {
case "integerReport": {
const report = integerReport(topic.value);
const fields: [string, string][] = [
["Prime", report.isPrime ? "yes" : "no"],
["Factorisation", factorizationText(report)],
["Divisors", String(report.divisorCount)],
["Divisor sum", String(report.divisorSum)],
["Totient", String(report.totient)],
["Digit sum", String(report.digitSum)],
["Digital root", String(report.digitalRoot)],
];
if (report.isPerfect) fields.push(["Perfect", "yes"]);
return answer(
`${topic.value} = ${factorizationText(report)}`,
`${topic.value} is ${report.isPrime ? "prime" : "composite"}, ` +
`with ${report.divisorCount} divisors summing to ${report.divisorSum}.`,
fields,
);
}
case "primality": {
const prime = isPrime(topic.value);
const report = prime ? "" : factorizationText(integerReport(topic.value));
return answer(
`${topic.value} is ${prime ? "prime" : "not prime"}`,
prime
? `${topic.value} has no divisors other than 1 and itself.`
: `${topic.value} = ${report}.`,
);
}
case "factorize": {
const report = integerReport(topic.value);
return answer(
`${topic.value} = ${factorizationText(report)}`,
`${topic.value} factors into ${factorizationText(report)}.`,
);
}
case "gcd": {
const gcd = greatestCommonDivisor(topic.a, topic.b);
return answer(
`gcd(${topic.a}, ${topic.b}) = ${gcd}`,
`The greatest common divisor of ${topic.a} and ${topic.b} is ${gcd}.`,
);
}
case "lcm": {
const lcm = leastCommonMultiple(topic.a, topic.b);
return answer(
`lcm(${topic.a}, ${topic.b}) = ${lcm}`,
`The least common multiple of ${topic.a} and ${topic.b} is ${lcm}.`,
);
}
case "modularInverse": {
const inverse = modularInverse(topic.value, topic.modulus);
if (inverse === null) {
return answer(
"No inverse",
`${topic.value} and ${topic.modulus} share a factor, so no modular inverse exists.`,
);
}
return answer(
`${topic.value}⁻¹ ≡ ${inverse} (mod ${topic.modulus})`,
`${topic.value} × ${inverse} ≡ 1 (mod ${topic.modulus}).`,
);
}
case "fibonacci": {
const value = fibonacci(topic.n);
if (value === null) {
return answer("Too large", `F(${topic.n}) does not fit in an exact integer.`);
}
return answer(`F(${topic.n}) = ${value}`, `The ${topic.n}th Fibonacci number is ${value}.`);
}
case "binomial": {
const value = binomial(topic.n, topic.k);
if (value === null) {
return answer("Undefined", `${topic.n} choose ${topic.k} is not defined.`);
}
return answer(
`C(${topic.n}, ${topic.k}) = ${value}`,
`There are ${value} ways to choose ${topic.k} from ${topic.n}.`,
);
}
case "symbolicIntegral": {
const antiderivative = integrateWithConstant(topic.expression, topic.variable);
if (!antiderivative) {
return answer(
"No closed form",
"This has no elementary antiderivative that this build can find. " +
"Ask for a definite integral and it will be computed numerically.",
);
}
return answer(
`∫ = ${plain(antiderivative)}`,
`∫ ${plain(topic.expression)} d${topic.variable} = ${plain(antiderivative)}`,
);
}
case "definiteIntegral": {
const result = definite(topic.expression, topic.variable, topic.from, topic.to);
switch (result.kind) {
case "exact":
return answer(
`∫ = ${describeNumber(result.value)}`,
`Using F = ${plain(result.antiderivative)}, the definite integral is ${describeNumber(result.value)}.`,
);
case "numeric":
return answer(
`∫ ≈ ${describeNumber(fromDecimal(result.value))}`,
"No closed form was found, so this was integrated numerically.",
);
case "failed":
return answer("Could not integrate", result.reason);
}
break;
}
case "limit": {
const result = limit(topic.expression, topic.variable, topic.at);
let detail =
`As ${topic.variable} → ${format(topic.at)}, ` +
`${plain(topic.expression)} → ${limitText(result)}.`;
if (result.kind === "doesNotExist") {
detail =
`The one-sided limits disagree ` +
`(${format(result.left ?? 0)} from the left, ${format(result.right ?? 0)} from the right), ` +
`so the limit does not exist.`;
}
return answer(`limit = ${limitText(result)}`, detail);
}
case "taylor": {
const series = taylor(topic.expression, topic.variable, topic.order);
if (!series) {
return answer("No series", "Could not expand that about 0.");
}
return answer(
plain(series),
`${plain(topic.expression)} ≈ ${plain(series)} to order ${topic.order}.`,
);
}
case "criticalPoints": {
const points = criticalPoints(topic.expression, topic.variable);
if (points.length === 0) {
return answer("No critical points", "The derivative never vanishes in the search window.");
}
return answer(
points
.slice(0, 2)
.map(criticalPointText)
.join(", "),
points.map(criticalPointText).join("; "),
points.map((point) => [
point.kind.charAt(0).toUpperCase() + point.kind.slice(1),
`(${format(point.x)}, ${format(point.y)})`,
]),
);
}
case "trigSolve": {
const solution = solveTrig(topic.fn, topic.value);
switch (solution.kind) {
case "none":
return answer("No solution", `${topic.fn} never reaches ${format(topic.value)}.`);
case "all":
return answer("True for every angle", "Any value works.");
case "families": {
const text = solution.families.map(familyText).join(", ");
return answer(
`x = ${text}`,
`${topic.fn} x = ${format(topic.value)} for x = ${text}, where n is any integer.`,
solution.families.map((family, index) => [`Family ${index + 1}`, familyText(family)]),
);
}
}
break;
}
case "trigIdentity": {
const identity = identityMatching(topic.query);
if (!identity) {
return answer(
"Not in the sheet",
"Known identities: " + TRIG_IDENTITIES.map((entry) => entry.name).join(", "),
);
}
return answer(identity.plain, `${identity.name}: ${identity.plain}`);
}
case "digitSearch": {
const outcome = findDigits(topic.pattern, topic.constant, topic.occurrence);
switch (outcome.kind) {
case "invalidPattern":
return answer("Cannot search", outcome.reason);
case "notFound": {
// "Not in pi" and "not in the first 10,000 digits of pi" are very
// different claims, and only the second one is true.
let detail =
`${topic.pattern} does not appear in the first ` +
`${formatInt(outcome.searchedDigits)} digits of ${constantSpokenName(topic.constant)}.`;
const expected = expectedFirstPosition(topic.pattern.length);
if (expected !== null && expected > outcome.searchedDigits) {
const article = [8, 11, 18].includes(topic.pattern.length) ? "An" : "A";
detail +=
` ${article} ${topic.pattern.length}-digit run turns up around the ` +
`${formatInt(expected)}th place on average, so this needs a deeper expansion.`;
}
return {
headline: `Not in the first ${formatInt(outcome.searchedDigits)} digits`,
detail,
fields: [],
unresolvedWorkload: {
kind: "digitSearch",
constant: topic.constant,
pattern: topic.pattern,
occurrence: topic.occurrence,
},
};
}
case "found": {
const match = outcome.match;
const ordinalText = ordinalWord(topic.occurrence);
const places =
match.start === match.end ? `place ${match.start}` : `places ${match.start}–${match.end}`;
return answer(
`${constantDisplayName(topic.constant)}: ${places}`,
`The ${ordinalText} appearance of ${topic.pattern} in ${constantSpokenName(topic.constant)} ` +
`is at decimal ${places}, in …${highlightedMatch(match)}…`,
[
["Start", String(match.start)],
["End", String(match.end)],
["Context", highlightedMatch(match)],
],
);
}
}
break;
}
case "digitCount": {
const counted = countDigitsIn(topic.pattern, topic.constant);
return {
headline: `${counted.occurrences} time${counted.occurrences === 1 ? "" : "s"}`,
detail:
`${topic.pattern} appears ${counted.occurrences} time${counted.occurrences === 1 ? "" : "s"} ` +
`in the first ${formatInt(counted.searchedDigits)} digits of ${constantSpokenName(topic.constant)}.`,
fields: [["Searched", `${formatInt(counted.searchedDigits)} digits`]],
unresolvedWorkload: { kind: "digitCount", constant: topic.constant, pattern: topic.pattern },
};
}
case "digitsAt": {
if (topic.to - topic.from >= 200) {
return answer("Too many digits", "Ask for a span under 200 digits.");
}
const digits = digitsAt(topic.constant, topic.from, topic.to);
if (!digits) {
return {
headline: "Out of range",
detail:
`That is past the ${formatInt(MAXIMUM_COMPUTED_DIGITS)} ` +
`digits computed on device.`,
fields: [],
unresolvedWorkload: { kind: "constantDigits", constant: topic.constant, count: topic.to },
};
}
return answer(
digits,
`Decimal places ${topic.from}–${topic.to} of ${constantSpokenName(topic.constant)} are ${digits}.`,
[["Places", `${topic.from}–${topic.to}`]],
);
}
case "shapeArea": {
const parts = shapeSummary(topic.shape);
if (parts.length === 0) {
return answer("Not enough measurements", "That shape needs more values.");
}
return answer(
parts.map(([label, value]) => `${label} ${format(value)}`).join(", "),
`For that ${shapeName(topic.shape)}: ` +
parts.map(([label, value]) => `${label.toLowerCase()} ${format(value)}`).join(", ") +
".",
parts.map(([label, value]) => [label, format(value)]),
);
}
case "triangleFromSides": {
const result = triangleFromSides(topic.a, topic.b, topic.c);
switch (result.kind) {
case "impossible":
return answer("No such triangle", result.reason);
case "one":
case "two": {
const triangle = result.kind === "one" ? result.triangle : result.first;
const [angleA, angleB, angleC] = triangleAnglesInDegrees(triangle);
return answer(
`Area ${format(triangleArea(triangle))}, ${triangleKind(triangle)}`,
`Angles ${format(angleA)}°, ${format(angleB)}°, ${format(angleC)}°; ` +
`area ${format(triangleArea(triangle))}; perimeter ${format(trianglePerimeter(triangle))}.`,
[
["Angle A", `${format(angleA)}°`],
["Angle B", `${format(angleB)}°`],
["Angle C", `${format(angleC)}°`],
["Area", format(triangleArea(triangle))],
["Perimeter", format(trianglePerimeter(triangle))],
["Type", triangleKind(triangle)],
],
);
}
}
break;
}
case "pythagoras": {
const missing = pythagoras(topic.a, topic.b, topic.c);
if (missing === null) {
return answer(
"Cannot solve",
"Give two of the three sides, and a leg must be shorter than the hypotenuse.",
);
}
return answer(format(missing), `The missing side is ${format(missing)}.`);
}
}
}
function answer(headline: string, detail: string, fields: [string, string][] = []): AdvancedAnswer {
return { headline, detail, fields, unresolvedWorkload: null };
}
// ---------------------------------------------------------------------------
// Extraction helpers
export function firstInt(text: string): number | null {
return allInts(text)[0] ?? null;
}
export function allInts(text: string): number[] {
return (text.match(/-?\d+/g) ?? []).map(Number).filter(Number.isSafeInteger);
}
export function allDoubles(text: string): number[] {
return (text.match(/-?\d+(?:\.\d+)?/g) ?? []).map(Number).filter(Number.isFinite);
}
export function lastDouble(text: string): number | null {
const values = allDoubles(text);
return values.length > 0 ? values[values.length - 1]! : null;
}
function firstDoubleAfter(keywords: string[], text: string): number | null {
for (const keyword of keywords) {
const index = text.indexOf(keyword);
if (index < 0) continue;
const tail = text.slice(index + keyword.length);
const value = allDoubles(tail)[0];
if (value !== undefined) return value;
}
return null;
}
/** "from 0 to 5" */
export function boundsIn(text: string): [number, number] | null {
const match = text.match(/from\s+(-?[0-9.]+)\s+to\s+(-?[0-9.]+)/);
if (!match) return null;
const lower = Number(match[1]);
const upper = Number(match[2]);
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
return [lower, upper];
}
/** Take everything after the topic keyword and parse it as an expression. */
function expressionTail(original: string, keywords: string[]): MathExpression | null {
const lowered = original.toLowerCase();
for (const keyword of [...keywords].sort((a, b) => b.length - a.length)) {
const index = lowered.indexOf(keyword);
if (index < 0) continue;
let tail = original.slice(index + keyword.length);
// Drop trailing clauses that belong to the command, not the maths.
for (const terminator of [" from ", " as ", " with respect to ", " to order ", " dx"]) {
const cut = tail.toLowerCase().indexOf(terminator);
if (cut >= 0) tail = tail.slice(0, cut);
}
const trimmed = tail.replace(/^[ .,?]+|[ .,?]+$/g, "");
if (!trimmed) continue;
try {
return parseSpoken(trimmed);
} catch {
continue;
}
}
return null;
}
function format(value: number): string {
return describeNumber(fromDecimal(value));
}
+359
View File
@@ -0,0 +1,359 @@
// Limits, Taylor series, and curve analysis.
import {
type MathExpression,
addExpr,
divideExpr,
multiplyExpr,
num,
numberValue,
powerExpr,
subtractExpr,
variable as variableExpr,
ONE,
ZERO,
} from "./expression";
import { describeNumber, divide as divideNumber, fromDecimal, integer, isNegative, isOne, negated } from "./math-number";
import { evaluateValue } from "./evaluator";
import { differentiate } from "./derivative";
import { simplify } from "./simplifier";
import { Polynomial } from "./polynomial";
import { numericRoots as polynomialNumericRoots } from "./solver";
// ---------------------------------------------------------------------------
// Limits
export type LimitApproach = "both" | "fromLeft" | "fromRight";
export type LimitResult =
| { kind: "value"; value: number }
| { kind: "positiveInfinity" }
| { kind: "negativeInfinity" }
/** One-sided limits disagree, so the two-sided limit does not exist. */
| { kind: "doesNotExist"; left: number | null; right: number | null }
| { kind: "indeterminate" };
export function limitText(result: LimitResult): string {
switch (result.kind) {
case "value":
return describeNumber(fromDecimal(result.value));
case "positiveInfinity":
return "+∞";
case "negativeInfinity":
return "−∞";
case "doesNotExist":
return "does not exist";
case "indeterminate":
return "indeterminate";
}
}
/**
* Limit of `expression` as `variable` → `point`.
*
* Direct substitution first, then L'Hôpital for 0/0 and ∞/∞, then a numeric
* squeeze. The order matters: substitution is exact when it works, L'Hôpital
* is exact when it applies, and only the leftovers get the approximate
* treatment.
*/
export function limit(
expression: MathExpression,
variable: string,
point: number,
approach: LimitApproach = "both",
): LimitResult {
if (approach === "both" && Number.isFinite(point)) {
const direct = evaluateValue(expression, { [variable]: point });
if (direct !== null && Number.isFinite(direct)) {
return { kind: "value", value: direct };
}
}
const hospital = lHopital(expression, variable, point, 0);
if (hospital) return hospital;
return numericLimit(expression, variable, point, approach);
}
/** Detect an indeterminate quotient and differentiate top and bottom. */
function lHopital(
expression: MathExpression,
variable: string,
point: number,
depth: number,
): LimitResult | null {
// Bounded: a quotient that stays indeterminate forever (sin(1/x)/x) must
// give up rather than recurse without end.
if (depth >= 8 || !Number.isFinite(point)) return null;
const quotient = asQuotient(expression);
if (!quotient) return null;
const [numerator, denominator] = quotient;
const top = evaluateValue(numerator, { [variable]: point });
const bottom = evaluateValue(denominator, { [variable]: point });
if (top === null || bottom === null) return null;
const isZeroOverZero = Math.abs(top) < 1e-12 && Math.abs(bottom) < 1e-12;
const isInfinityOverInfinity = !Number.isFinite(top) && !Number.isFinite(bottom);
if (!isZeroOverZero && !isInfinityOverInfinity) return null;
const dTop = differentiate(numerator, variable);
const dBottom = differentiate(denominator, variable);
// A vanishing derivative of the denominator means the rule does not apply
// again; fall through to the numeric path.
const bottomValue = evaluateValue(dBottom, { [variable]: point });
if (bottomValue === null || !Number.isFinite(bottomValue)) return null;
if (Math.abs(bottomValue) > 1e-12) {
const topValue = evaluateValue(dTop, { [variable]: point });
if (topValue !== null && Number.isFinite(topValue)) {
return { kind: "value", value: topValue / bottomValue };
}
}
return lHopital(divideExpr(dTop, dBottom), variable, point, depth + 1);
}
/** Recognise `a / b`, which the tree stores as `a · b^-1`. */
export function asQuotient(expression: MathExpression): [MathExpression, MathExpression] | null {
if (expression.type !== "multiply") return null;
const numerator: MathExpression[] = [];
const denominator: MathExpression[] = [];
for (const factor of expression.factors) {
if (factor.type === "power") {
const power = numberValue(factor.exponent);
if (power && isNegative(power)) {
const positive = negated(power);
denominator.push(isOne(positive) ? factor.base : powerExpr(factor.base, num(positive)));
continue;
}
}
numerator.push(factor);
}
if (denominator.length === 0) return null;
return [
numerator.length === 0 ? ONE : numerator.length === 1 ? numerator[0]! : multiplyExpr(numerator),
denominator.length === 1 ? denominator[0]! : multiplyExpr(denominator),
];
}
function numericLimit(
expression: MathExpression,
variable: string,
point: number,
approach: LimitApproach,
): LimitResult {
const sampleSide = (side: LimitApproach): number | null => {
if (!Number.isFinite(point)) {
// At infinity, march outward instead of shrinking a step.
const sign = point > 0 ? 1 : -1;
let last: number | null = null;
for (const magnitude of [1e3, 1e4, 1e5, 1e6, 1e7]) {
const value = evaluateValue(expression, { [variable]: sign * magnitude });
if (value === null || !Number.isFinite(value)) return last;
last = value;
}
return last;
}
const samples: number[] = [];
for (const step of [1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7]) {
const offset = side === "fromLeft" ? -step : step;
const value = evaluateValue(expression, { [variable]: point + offset });
if (value === null || !Number.isFinite(value)) continue;
samples.push(value);
}
const last = samples[samples.length - 1];
if (last === undefined) return null;
// Divergence is a TREND, not a threshold. 1/x reaches only 1e7 at the
// smallest step sampled here, so a fixed "> 1e12" cutoff calls it finite;
// magnitudes growing by an order per step do not.
if (samples.length >= 3) {
const tail = samples.slice(-3);
const magnitudes = tail.map((value) => Math.abs(value));
const growing = magnitudes.every(
(value, index) => index === 0 || value > magnitudes[index - 1]! * 5,
);
if (growing && (magnitudes[magnitudes.length - 1] ?? 0) > 1e4) {
return last > 0 ? Infinity : -Infinity;
}
}
return last;
};
switch (approach) {
case "fromLeft": {
const value = sampleSide("fromLeft");
return value === null ? { kind: "indeterminate" } : classify(value);
}
case "fromRight": {
const value = sampleSide("fromRight");
return value === null ? { kind: "indeterminate" } : classify(value);
}
case "both": {
const left = sampleSide("fromLeft");
const right = sampleSide("fromRight");
if (left === null || right === null) return { kind: "indeterminate" };
// Scale-relative comparison: two values near 1e6 differing by 1 are the
// same limit; two near 0 differing by 1 are not.
const scale = Math.max(Math.abs(left), Math.abs(right), 1);
if (Math.abs(left - right) >= 1e-4 * scale) {
return { kind: "doesNotExist", left, right };
}
return classify((left + right) / 2);
}
}
}
function classify(value: number): LimitResult {
if (!Number.isFinite(value)) {
return value > 0 ? { kind: "positiveInfinity" } : { kind: "negativeInfinity" };
}
if (value > 1e12) return { kind: "positiveInfinity" };
if (value < -1e12) return { kind: "negativeInfinity" };
return { kind: "value", value };
}
// ---------------------------------------------------------------------------
// Series
/** Taylor expansion about `centre`, to `order` terms. */
export function taylor(
expression: MathExpression,
variable: string,
order = 5,
centre = 0,
): MathExpression | null {
if (order < 0 || order > 12) return null;
const terms: MathExpression[] = [];
let derivative = expression;
let factorial = 1;
for (let degree = 0; degree <= order; degree++) {
if (degree > 0) {
derivative = differentiate(derivative, variable);
factorial *= degree;
}
const coefficient = evaluateValue(derivative, { [variable]: centre });
if (coefficient === null || !Number.isFinite(coefficient)) return null;
if (Math.abs(coefficient) <= 1e-12) continue;
const scaled = divideNumber(fromDecimal(coefficient), integer(factorial));
const shifted: MathExpression =
centre === 0
? variableExpr(variable)
: subtractExpr(variableExpr(variable), num(fromDecimal(centre)));
const power: MathExpression =
degree === 0 ? ONE : degree === 1 ? shifted : powerExpr(shifted, num(integer(degree)));
terms.push(degree === 0 ? num(scaled) : multiplyExpr([num(scaled), power]));
}
if (terms.length === 0) return ZERO;
return simplify(addExpr(terms));
}
// ---------------------------------------------------------------------------
// Curve analysis
export type CriticalPointKind = "minimum" | "maximum" | "inflection" | "undetermined";
export type CriticalPoint = {
x: number;
y: number;
kind: CriticalPointKind;
};
export function criticalPointText(point: CriticalPoint): string {
const xText = describeNumber(fromDecimal(point.x));
const yText = describeNumber(fromDecimal(point.y));
return `${point.kind} at (${xText}, ${yText})`;
}
/**
* Critical points classified by the second derivative, falling back to a
* sign test where the second derivative vanishes.
*/
export function criticalPoints(
expression: MathExpression,
variable: string,
range: [number, number] = [-50, 50],
): CriticalPoint[] {
const first = differentiate(expression, variable);
const second = differentiate(first, variable);
let roots: number[];
const polynomial = Polynomial.extract(first, variable);
if (polynomial) {
roots = polynomialNumericRoots(polynomial, range, 20_000);
} else {
roots = numericRootsOfExpression(first, variable, range);
}
const points: CriticalPoint[] = [];
for (const x of roots) {
const y = evaluateValue(expression, { [variable]: x });
if (y === null || !Number.isFinite(y)) continue;
const curvature = evaluateValue(second, { [variable]: x }) ?? 0;
let kind: CriticalPointKind;
if (curvature > 1e-9) {
kind = "minimum";
} else if (curvature < -1e-9) {
kind = "maximum";
} else {
// Second-derivative test is inconclusive; look at how the first
// derivative behaves either side.
const left = evaluateValue(first, { [variable]: x - 1e-4 }) ?? 0;
const right = evaluateValue(first, { [variable]: x + 1e-4 }) ?? 0;
if (left < 0 && right > 0) kind = "minimum";
else if (left > 0 && right < 0) kind = "maximum";
else kind = "inflection";
}
points.push({ x, y, kind });
}
return points;
}
export function numericRootsOfExpression(
expression: MathExpression,
variable: string,
range: [number, number],
samples = 20_000,
): number[] {
const [lower, upper] = range;
const roots: number[] = [];
const step = (upper - lower) / samples;
let previousX = lower;
let previousY = evaluateValue(expression, { [variable]: previousX });
for (let index = 1; index <= samples; index++) {
const x = lower + index * step;
const y = evaluateValue(expression, { [variable]: x });
const lastX = previousX;
const lastY = previousY;
previousX = x;
previousY = y;
if (y === null || lastY === null) continue;
if (Math.sign(lastY) === Math.sign(y)) continue;
if (Math.abs(y) >= 1e6 || Math.abs(lastY) >= 1e6) continue;
let low = lastX;
let high = x;
for (let iteration = 0; iteration < 60; iteration++) {
const mid = (low + high) / 2;
const value = evaluateValue(expression, { [variable]: mid });
const lowValue = evaluateValue(expression, { [variable]: low });
if (value === null || lowValue === null) break;
if (value === 0) {
low = mid;
high = mid;
break;
}
if (Math.sign(value) === Math.sign(lowValue)) {
low = mid;
} else {
high = mid;
}
}
const root = (low + high) / 2;
// Same pole guard as the main solver: verify the residual.
const residual = evaluateValue(expression, { [variable]: root });
if (residual === null || Math.abs(residual) >= 1e-6) continue;
if (!roots.some((existing) => Math.abs(existing - root) < 1e-6)) roots.push(root);
}
return roots;
}
+591
View File
@@ -0,0 +1,591 @@
// Digit expansions of the well-known constants, computed exactly. Computed
// rather than tabulated: a hardcoded string of π is a thing nobody can check,
// and a single transposed digit would silently shift every position this
// feature reports. Everything here comes out of integer arithmetic that can
// be re-derived and tested.
//
// JavaScript's native BigInt supplies the operation set the generators need
// (add, subtract, compare, multiply/divide by a small scalar).
// The deep paths are async and yield to the event loop periodically — a
// multi-minute synchronous grind would freeze the app's whole UI thread.
export type MathConstantName = "pi" | "e" | "phi" | "sqrt2" | "sqrt3" | "sqrt5";
export const ALL_CONSTANT_NAMES: readonly MathConstantName[] = ["pi", "e", "phi", "sqrt2", "sqrt3", "sqrt5"];
export function constantDisplayName(constant: MathConstantName): string {
switch (constant) {
case "pi":
return "π";
case "e":
return "e";
case "phi":
return "φ";
case "sqrt2":
return "√2";
case "sqrt3":
return "√3";
case "sqrt5":
return "√5";
}
}
export function constantSpokenName(constant: MathConstantName): string {
switch (constant) {
case "pi":
return "pi";
case "e":
return "e";
case "phi":
return "the golden ratio";
case "sqrt2":
return "the square root of 2";
case "sqrt3":
return "the square root of 3";
case "sqrt5":
return "the square root of 5";
}
}
/** Digits before the decimal point — 1 for all of these. */
export const CONSTANT_INTEGER_PART_DIGITS = 1;
export function constantNamed(text: string): MathConstantName | null {
const lowered = text.toLowerCase();
if (lowered.includes("golden") || lowered.includes("phi")) return "phi";
if (lowered.includes("root of 2") || lowered.includes("root two") || lowered.includes("sqrt2")) {
return "sqrt2";
}
if (lowered.includes("root of 3") || lowered.includes("root three")) return "sqrt3";
if (lowered.includes("root of 5") || lowered.includes("root five")) return "sqrt5";
// Word-boundary matched, not substring: `includes("pi")` also fires on
// "pie", "spin", and "copying", and `=== "e"` only ever matched a
// single-character utterance, so "the digits of e" never resolved.
if (lowered.includes("π") || hasWord("pi", lowered)) return "pi";
if (hasWord("e", lowered) || lowered.includes("euler")) return "e";
return null;
}
function hasWord(word: string, text: string): boolean {
return new RegExp(`\\b${word}\\b`).test(text);
}
// ---------------------------------------------------------------------------
// Cancellation and progress
/** Cooperative cancellation for the long paths. */
export type ComputeCancellation = () => boolean;
export const NEVER_CANCELLED: ComputeCancellation = () => false;
/** Raised when a long run is cancelled partway. */
export class ComputeCancelled extends Error {
constructor() {
super("Cancelled");
this.name = "ComputeCancelled";
}
}
export type ComputeProgress = (fraction: number, stage: string) => void;
// setTimeout via globalThis so this module needs no platform lib: it runs
// under NativeScript's globals in the app and under Node in the tests.
const scheduleMacrotask: (callback: () => void) => void = (callback) =>
(globalThis as unknown as { setTimeout: (fn: () => void, ms: number) => unknown }).setTimeout(callback, 0);
/**
* Yield to the event loop when the current slice has run long enough. The
* generators call this between series terms so a deep run stays cancellable
* and the rest of the app keeps painting.
*/
class Yielder {
private lastYieldAt = Date.now();
async maybeYield(): Promise<void> {
if (Date.now() - this.lastYieldAt < 50) return;
await new Promise<void>((resolve) => scheduleMacrotask(resolve));
this.lastYieldAt = Date.now();
}
}
// ---------------------------------------------------------------------------
// Limits
/**
* How far the on-device generator will go for an ordinary question.
*
* Machin's formula is O(n²) in the digit count with scalar-only big
* arithmetic. 10,000 digits is well under a second and finds any two- or
* three-digit run plus most four-digit ones; past this, the wearer opts into
* a long run explicitly.
*/
export const MAXIMUM_COMPUTED_DIGITS = 10_000;
/**
* The furthest an opted-in long run will go, however much time the wearer
* offers. Not a time limit — the caller has already accepted the wait — but
* a memory-and-honesty one: the O(n²) cost puts this at hours on a phone,
* past which nobody is still waiting.
*/
export const MAXIMUM_FEASIBLE_DIGITS = 2_000_000;
// ---------------------------------------------------------------------------
// Cache
/**
* The same constant is asked for repeatedly within one session (search, then
* context, then a second search), and recomputing 10,000 digits each time
* would be pointlessly slow. Grow-only.
*/
const cache = new Map<MathConstantName, string>();
function cachedDigits(constant: MathConstantName, atLeast: number): string | null {
const cached = cache.get(constant);
if (!cached || cached.length < atLeast) return null;
return cached;
}
function storeDigits(constant: MathConstantName, digits: string): void {
const existing = cache.get(constant);
if (existing && existing.length >= digits.length) return;
cache.set(constant, digits);
}
// ---------------------------------------------------------------------------
// Public entry points
/**
* Fractional digits only — the part after the decimal point, which is what
* "position N of π" universally means. Clamped to MAXIMUM_COMPUTED_DIGITS;
* the deeper run is a separate call the wearer has to opt into, so no
* ordinary caller can accidentally block the app for minutes.
*/
export function fractionalDigits(constant: MathConstantName, count: number): string | null {
const requested = Math.min(Math.max(count, 1), MAXIMUM_COMPUTED_DIGITS);
const cached = cachedDigits(constant, requested);
if (cached) return cached.slice(0, requested);
// The shallow path runs synchronously; at 10k digits it is fast enough that
// an async hop would only complicate every caller.
const digits = computeSync(constant, requested);
if (digits === null) return null;
storeDigits(constant, digits);
return digits.slice(0, requested);
}
/**
* The opted-in deep run. Cancellable, and reports progress, because a
* multi-minute grind with no way out and nothing on screen is
* indistinguishable from a hang.
*/
export async function deepFractionalDigits(
constant: MathConstantName,
count: number,
cancellation: ComputeCancellation = NEVER_CANCELLED,
progress: ComputeProgress | null = null,
): Promise<string | null> {
const requested = Math.min(Math.max(count, 1), MAXIMUM_FEASIBLE_DIGITS);
const cached = cachedDigits(constant, requested);
if (cached) return cached.slice(0, requested);
const digits = await compute(constant, requested, cancellation, progress, new Yielder());
if (digits === null) return null;
storeDigits(constant, digits);
return digits.slice(0, requested);
}
function computeSync(constant: MathConstantName, requested: number): string | null {
// A resolved-synchronously wrapper would still be async; instead run the
// shared generator with a yielder that never fires (50ms slices cannot
// elapse mid-loop when the whole computation is sub-second) by executing
// the promise eagerly. The generators only await through Yielder.maybeYield,
// which resolves immediately below the time threshold, so the promise
// settles synchronously enough — but rather than rely on that, the shallow
// path uses the dedicated synchronous variants.
return computeCore(constant, requested, NEVER_CANCELLED, null, null);
}
async function compute(
constant: MathConstantName,
requested: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<string | null> {
return computeCoreAsync(constant, requested, cancellation, progress, yielder);
}
// ---------------------------------------------------------------------------
// Core generators
//
// Each exists in a synchronous and an asynchronous variant sharing the same
// arithmetic. The async ones interleave event-loop yields; the sync ones are
// used only under MAXIMUM_COMPUTED_DIGITS where the total cost is small.
/**
* Guard digits absorb the truncation error the series accumulates in its
* least significant places; without them the last handful of reported digits
* are wrong, which for a position-reporting feature is the whole ballgame.
*/
const GUARD_DIGITS = 20;
function computeCore(
constant: MathConstantName,
requested: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
_yielder: null,
): string | null {
const scale = requested + GUARD_DIGITS;
let scaled: bigint;
switch (constant) {
case "pi":
scaled = machinPiSync(scale, cancellation, progress);
break;
case "e":
scaled = eulerESync(scale, cancellation, progress);
break;
case "phi": {
// φ = (1 + √5) / 2
const root = integerSquareRootSync(5, scale, cancellation, progress);
scaled = (root + powerOfTen(scale)) / 2n;
break;
}
case "sqrt2":
scaled = integerSquareRootSync(2, scale, cancellation, progress);
break;
case "sqrt3":
scaled = integerSquareRootSync(3, scale, cancellation, progress);
break;
case "sqrt5":
scaled = integerSquareRootSync(5, scale, cancellation, progress);
break;
}
return finishDigits(scaled, requested);
}
async function computeCoreAsync(
constant: MathConstantName,
requested: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<string | null> {
const scale = requested + GUARD_DIGITS;
let scaled: bigint;
switch (constant) {
case "pi":
scaled = await machinPi(scale, cancellation, progress, yielder);
break;
case "e":
scaled = await eulerE(scale, cancellation, progress, yielder);
break;
case "phi": {
const root = await integerSquareRoot(5, scale, cancellation, progress, yielder);
scaled = (root + powerOfTen(scale)) / 2n;
break;
}
case "sqrt2":
scaled = await integerSquareRoot(2, scale, cancellation, progress, yielder);
break;
case "sqrt3":
scaled = await integerSquareRoot(3, scale, cancellation, progress, yielder);
break;
case "sqrt5":
scaled = await integerSquareRoot(5, scale, cancellation, progress, yielder);
break;
}
return finishDigits(scaled, requested);
}
/**
* The value is the constant × 10^scale. Strip the integer part, then the
* guard digits.
*/
function finishDigits(scaled: bigint, requested: number): string | null {
const text = scaled.toString();
if (text.length <= CONSTANT_INTEGER_PART_DIGITS) return null;
const fractional = text.slice(CONSTANT_INTEGER_PART_DIGITS);
if (fractional.length < requested) return null;
const trimmed = fractional.slice(0, fractional.length - GUARD_DIGITS);
return trimmed.slice(0, requested);
}
export function powerOfTen(exponent: number): bigint {
return 10n ** BigInt(Math.max(0, exponent));
}
// ---------------------------------------------------------------------------
// π — Machin's formula: π = 16·arctan(1/5) − 4·arctan(1/239).
//
// Chosen over Chudnovsky because it needs only scalar big-integer operations
// — no big×big multiply, no big÷big division, no square roots of big values.
// arctan(1/5) needs log10(239)/log10(5) ≈ 3.4× as many terms as arctan(1/239)
// at the same scale, and each term costs the same, so the first phase is
// ~77% of the work. Splitting the reported fraction that way keeps the
// progress bar honest instead of stalling at half.
const MACHIN_FIRST_SHARE = 0.77;
function machinPiSync(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
): bigint {
const first =
arctanReciprocalSync(5, scale, cancellation, (fraction) =>
progress?.(fraction * MACHIN_FIRST_SHARE, "π series 1 of 2"),
) * 16n;
const second =
arctanReciprocalSync(239, scale, cancellation, (fraction) =>
progress?.(MACHIN_FIRST_SHARE + fraction * (1 - MACHIN_FIRST_SHARE), "π series 2 of 2"),
) * 4n;
return first - second;
}
async function machinPi(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<bigint> {
const first =
(await arctanReciprocal(
5,
scale,
cancellation,
(fraction) => progress?.(fraction * MACHIN_FIRST_SHARE, "π series 1 of 2"),
yielder,
)) * 16n;
const second =
(await arctanReciprocal(
239,
scale,
cancellation,
(fraction) => progress?.(MACHIN_FIRST_SHARE + fraction * (1 - MACHIN_FIRST_SHARE), "π series 2 of 2"),
yielder,
)) * 4n;
return first - second;
}
/** arctan(1/x), scaled by 10^scale, by its alternating Taylor series. */
function arctanReciprocalSync(
x: number,
scale: number,
cancellation: ComputeCancellation,
progress: ((fraction: number) => void) | null,
): bigint {
const bigX = BigInt(x);
const xSquared = bigX * bigX;
let power = powerOfTen(scale) / bigX;
let sum = power;
let divisor = 3n;
let subtractNext = true;
// Terms shrink by a factor of x² each step, so the series dies after about
// scale / (2·log10 x) of them. Known up front, which is what makes a real
// fraction possible rather than a spinner.
const expectedTerms = Math.max(1, scale / (2 * Math.log10(x)));
let iterations = 0;
while (power !== 0n) {
// Checked every term: one closure call is nothing beside a big-integer
// division, and the alternative is an uncancellable multi-minute loop.
if (cancellation()) throw new ComputeCancelled();
power /= xSquared;
const term = power / divisor;
if (term === 0n) break;
sum = subtractNext ? sum - term : sum + term;
subtractNext = !subtractNext;
divisor += 2n;
iterations += 1;
// Throttled: a million progress callbacks would cost more than the
// arithmetic they are reporting on.
if (progress && iterations % 256 === 0) {
progress(Math.min(1, iterations / expectedTerms));
}
}
return sum;
}
async function arctanReciprocal(
x: number,
scale: number,
cancellation: ComputeCancellation,
progress: ((fraction: number) => void) | null,
yielder: Yielder,
): Promise<bigint> {
const bigX = BigInt(x);
const xSquared = bigX * bigX;
let power = powerOfTen(scale) / bigX;
let sum = power;
let divisor = 3n;
let subtractNext = true;
const expectedTerms = Math.max(1, scale / (2 * Math.log10(x)));
let iterations = 0;
while (power !== 0n) {
if (cancellation()) throw new ComputeCancelled();
power /= xSquared;
const term = power / divisor;
if (term === 0n) break;
sum = subtractNext ? sum - term : sum + term;
subtractNext = !subtractNext;
divisor += 2n;
iterations += 1;
if (progress && iterations % 256 === 0) {
progress(Math.min(1, iterations / expectedTerms));
}
if (iterations % 64 === 0) await yielder.maybeYield();
}
return sum;
}
// ---------------------------------------------------------------------------
// e = Σ 1/k!, scaled. Each term is the previous divided by k, so the whole
// series needs only scalar division.
function eulerESync(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
): bigint {
let term = powerOfTen(scale);
let sum = term;
let k = 1n;
// The term loses log10(k) digits each step; the series ends when the
// accumulated loss passes `scale`. Tracking that sum gives a fraction
// without needing Stirling's approximation.
let digitsShed = 0;
let iterations = 0;
while (term !== 0n) {
if (cancellation()) throw new ComputeCancelled();
term /= k;
sum += term;
digitsShed += Math.log10(Number(k));
k += 1n;
iterations += 1;
if (progress && iterations % 256 === 0) {
progress(Math.min(1, digitsShed / Math.max(scale, 1)), "e series");
}
// 10^scale / k! underflows long before this, but a runaway loop is not an
// acceptable failure mode.
if (k > 1_000_000n) break;
}
return sum;
}
async function eulerE(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<bigint> {
let term = powerOfTen(scale);
let sum = term;
let k = 1n;
let digitsShed = 0;
let iterations = 0;
while (term !== 0n) {
if (cancellation()) throw new ComputeCancelled();
term /= k;
sum += term;
digitsShed += Math.log10(Number(k));
k += 1n;
iterations += 1;
if (progress && iterations % 256 === 0) {
progress(Math.min(1, digitsShed / Math.max(scale, 1)), "e series");
}
if (iterations % 64 === 0) await yielder.maybeYield();
if (k > 1_000_000n) break;
}
return sum;
}
// ---------------------------------------------------------------------------
// Square roots — ⌊√(value · 10^(2·scale))⌋, i.e. √value scaled by 10^scale.
//
// Digit-by-digit ("long division") extraction, which needs only add,
// subtract, compare, and multiply-by-scalar.
function integerSquareRootSync(
value: number,
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
): bigint {
// Work in pairs of decimal digits, which is what the algorithm consumes.
const totalPairs = scale + 1;
let remainder = 0n;
let root = 0n;
for (let pairIndex = 0; pairIndex < totalPairs; pairIndex++) {
if (cancellation()) throw new ComputeCancelled();
if (progress && pairIndex % 256 === 0) {
progress(pairIndex / totalPairs, "root extraction");
}
// Bring down the next two digits: the first pair is the integer part,
// every later pair is zero because the radicand's fractional expansion is
// all zeros.
const bringDown = pairIndex === 0 ? BigInt(value) : 0n;
remainder = remainder * 100n + bringDown;
// Find the largest d in 0...9 with (20·root + d)·d ≤ remainder.
const twentyRoot = root * 20n;
let chosen = 0n;
let chosenProduct = 0n;
for (let digit = 9n; digit >= 1n; digit--) {
const candidate = (twentyRoot + digit) * digit;
if (candidate <= remainder) {
chosen = digit;
chosenProduct = candidate;
break;
}
}
remainder -= chosenProduct;
root = root * 10n + chosen;
}
return root;
}
async function integerSquareRoot(
value: number,
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<bigint> {
const totalPairs = scale + 1;
let remainder = 0n;
let root = 0n;
for (let pairIndex = 0; pairIndex < totalPairs; pairIndex++) {
if (cancellation()) throw new ComputeCancelled();
if (progress && pairIndex % 256 === 0) {
progress(pairIndex / totalPairs, "root extraction");
}
const bringDown = pairIndex === 0 ? BigInt(value) : 0n;
remainder = remainder * 100n + bringDown;
const twentyRoot = root * 20n;
let chosen = 0n;
let chosenProduct = 0n;
for (let digit = 9n; digit >= 1n; digit--) {
const candidate = (twentyRoot + digit) * digit;
if (candidate <= remainder) {
chosen = digit;
chosenProduct = candidate;
break;
}
}
remainder -= chosenProduct;
root = root * 10n + chosen;
if (pairIndex % 64 === 0) await yielder.maybeYield();
}
return root;
}
+602
View File
@@ -0,0 +1,602 @@
// Owns the one live maths problem and everything derived from it. Shared
// state rather than per-request because the whole point of "graph it" and
// "explain it" is that they refer to the problem already on the lens; a
// caller that re-parsed its own capture each time would make those
// follow-ups impossible.
//
// Deep digit work runs on the phone as an explicit, cancellable long run
// the wearer opts into. The
// optional AI pass is injected as a hook so this module stays free of any
// networking and testable in plain Node.
import { type MathConstantName, ComputeCancelled, constantDisplayName, constantNamed } from "./constant-digits";
import { highlightedMatch } from "./digit-search";
import { formatInt } from "./math-number";
import {
type LongRunOffer,
type LongRunProgress,
type MathWorkload,
compactDuration,
longRunOffer as makeLongRunOffer,
runAcceptedCount,
runAcceptedDigits,
runAcceptedSearch,
} from "./workload";
import { type MathSessionResult, MathSession, parseCommand } from "./session";
import { type Graph } from "./plotter";
import { type SolutionStep } from "./solver";
import { plain } from "./text-renderer";
// ---------------------------------------------------------------------------
// Branch outcome
/**
* A branch result. The value/failure split matters: a parse error must never
* be posted onward as if it were the answer.
*/
export type MathBranchOutcome =
| { kind: "value"; text: string }
| { kind: "failure"; reason: string };
export type MathBranchMode = "solve" | "explain" | "graph" | "derivative" | "digitSearch" | "digitCount";
/**
* The app's uniform gesture contract: forward/back step through the
* working, commit
* graphs, alternate explains, exit clears.
*/
export type RingAction = "forward" | "back" | "commit" | "alternate" | "exit";
/**
* Ask the configured assistant to render a spoken problem as an expression
* (the prompt text lives in app/prompts.ts with every other prompt).
* Injected by the app; null when no provider is configured. Resolves to the
* model's raw reply, or null on failure — the coordinator sanitises it.
*/
export type AICompletion = (spokenText: string) => Promise<string | null>;
export class MathCoordinator {
/** The last thing produced, for the answer view. */
lastResult: MathSessionResult | null = null;
lastGraph: Graph | null = null;
/** Position in the working, for the step indicator. */
stepPosition: { index: number; total: number } | null = null;
/**
* A standing offer to grind the full-depth answer out on the phone.
* Non-null means every surface should be offering it.
*/
pendingLongRun: LongRunOffer | null = null;
/** Live progress while an accepted run is going. Null when nothing is running. */
longRunProgress: LongRunProgress | null = null;
/**
* The answer a deeper pass produced. One channel, because from the
* wearer's side it is one event: the better answer arrived.
*/
deepAnswer: MathBranchOutcome | null = null;
/** Repaint hook, set by the owning layer. */
onChanged: (() => void) | null = null;
private readonly session = new MathSession();
private completeWithAI: AICompletion | null = null;
private longRunCancelled = false;
private longRunRunning = false;
private longRunStartedAtMs = 0;
get hasProblem(): boolean {
return this.session.hasProblem;
}
get currentSolution() {
return this.session.currentSolution;
}
get explanationSteps(): SolutionStep[] {
return this.session.currentSolution?.steps ?? [];
}
get isRunningLong(): boolean {
return this.longRunRunning;
}
setUsesDegrees(value: boolean): void {
this.session.setUsesDegrees(value);
}
setAICompletion(completion: AICompletion | null): void {
this.completeWithAI = completion;
}
private changed(): void {
this.onChanged?.();
}
// -------------------------------------------------------------------------
// Branch execution
/**
* Run one branch. Returns the glasses text, or a failure string — the
* caller distinguishes them.
*/
run(mode: MathBranchMode, input: string): MathBranchOutcome {
const trimmed = input.trim();
let sessionResult: MathSessionResult;
switch (mode) {
case "solve":
// "Solve it" after Graph/Explain means show the standing answer
// again. Requiring fresh input here made the third follow-up mode
// uniquely fail.
if (!trimmed && this.session.hasProblem) {
sessionResult = this.session.execute({ kind: "restate" });
} else if (!trimmed) {
return { kind: "failure", reason: "Nothing was captured to solve" };
} else {
sessionResult = this.session.solve(trimmed);
}
break;
case "graph":
// A graph branch may fire with a fresh capture ("graph x squared") or
// as a follow-up on the standing problem ("graph it"). Both have to
// work from the same branch.
if (trimmed && (parseCommand(trimmed) === null || this.isProblem(trimmed))) {
this.session.solve(trimmed);
}
sessionResult = this.session.graph();
break;
case "explain":
if (trimmed && this.isProblem(trimmed)) this.session.solve(trimmed);
sessionResult = this.session.explain();
break;
case "derivative":
if (trimmed && this.isProblem(trimmed)) this.session.solve(trimmed);
sessionResult = this.session.differentiate(null);
break;
case "digitSearch":
case "digitCount": {
const digits = MathCoordinator.digitRun(trimmed);
if (!digits) return { kind: "failure", reason: "No digits were captured to look for" };
// The constant is read from the capture when the wearer named one
// ("…in the golden ratio") and is π otherwise.
const constant: MathConstantName = constantNamed(trimmed) ?? "pi";
sessionResult = this.session.execute({
kind: "advanced",
topic:
mode === "digitCount"
? { kind: "digitCount", pattern: digits, constant }
: { kind: "digitSearch", pattern: digits, constant, occurrence: 1 },
});
break;
}
}
return this.publish(sessionResult);
}
/**
* Every digit in the capture, in order, with the separators dropped.
*
* Joined rather than "the first run" because ASR punctuates a spoken date
* freely — "07 04 84", "070484", "7/4/84" are all the same six digits —
* and taking the first run would search π for "07", which appears within
* the first dozen places and looks for all the world like a working
* answer.
*/
static digitRun(text: string): string | null {
const joined = text.replace(/\D/g, "");
return joined ? joined : null;
}
private isProblem(text: string): boolean {
return parseCommand(text)?.kind === "problem";
}
// -------------------------------------------------------------------------
// Spoken questions
/**
* Answer a question stated in words, whatever shape it arrived in.
*
* Two passes, in this order and never the other way round:
*
* 1. The deterministic engine alone. The spoken normaliser already turns
* "two x squared plus three x" into `2x²+3x`, and every answer it gives
* is exact.
* 2. Only if that fails to parse, ask the model to REWRITE the words as an
* expression — and then solve that expression here, deterministically.
*
* The model never computes anything. It is used as a translator from
* English into notation, which is the part it is genuinely good at and the
* part where a mistake is visible: the rewritten expression is returned
* alongside the answer, so a wrong reading shows up as a wrong-looking
* equation rather than as a plausible wrong number.
*/
async answerSpoken(text: string): Promise<MathBranchOutcome> {
const trimmed = text.trim();
if (!trimmed) return { kind: "failure", reason: "Nothing was said to calculate" };
const command = parseCommand(trimmed);
if (command) {
const outcome = this.publish(this.session.execute(command));
if (outcome.kind === "value") return outcome;
// A FOLLOW-UP that failed keeps its own message. "Graph it" with
// nothing on the lens fails because there is nothing to graph, and
// handing those two words to a rewriter would replace that with "could
// not read that as a calculation" — true of the words, and useless
// about the actual problem.
if (command.kind !== "problem") return outcome;
}
const rewritten = await this.expressionFromAI(trimmed);
if (!rewritten) {
return { kind: "failure", reason: "Could not read that as a calculation" };
}
const outcome = this.publish(this.session.solve(rewritten));
if (outcome.kind !== "value") return outcome;
// Show the reading, not just the result. A wearer glancing at the lens
// has to be able to see that the machine understood the right question
// before they repeat the number.
return { kind: "value", text: `${rewritten}\n${outcome.text}` };
}
/**
* Ask the configured assistant to render a spoken problem as an
* expression. Returns null when no provider is configured, when it
* declines, or when it answers with anything that is not one.
*/
private async expressionFromAI(text: string): Promise<string | null> {
if (!this.completeWithAI) return null;
let raw: string | null;
try {
raw = await this.completeWithAI(text);
} catch {
return null;
}
if (!raw) return null;
let candidate = raw.trim();
// Models fence notation out of habit even when told not to.
if (candidate.startsWith("```")) {
candidate = candidate
.split("\n")
.filter((line) => !line.startsWith("```"))
.join(" ")
.trim();
}
candidate = candidate.replace(/^[ `.,]+|[ `.,]+$/g, "");
if (!candidate || candidate.toLowerCase() === "none") return null;
// A refusal or an apology is prose, and prose fed to the parser fails in
// a way that reads like the maths was wrong rather than the routing.
if (candidate.length > 120) return null;
if (!/[+\-*/^=\d]/.test(candidate)) return null;
return candidate;
}
// -------------------------------------------------------------------------
// Voice follow-ups
/**
* Handle a spoken follow-up. Returns null when the utterance is not a
* maths command, so ordinary conversation flows past untouched — this is
* called from the transcript stream, which sees everything said in the
* room in continuous mode.
*/
handleUtterance(utterance: string): MathBranchOutcome | null {
// A standing offer or a run in flight owns the next yes/no said in the
// room. Checked before the maths parser, because "keep going" is not an
// equation and would otherwise fall straight through.
const longRunAnswer = this.handleLongRunUtterance(utterance);
if (longRunAnswer) return longRunAnswer;
const command = parseCommand(utterance);
if (!command) return null;
// A bare problem with no standing session is a new problem; a follow-up
// with no session is a no-op rather than an error card in mid-air.
if (command.kind !== "problem" && command.kind !== "advanced" && !this.session.hasProblem) {
return null;
}
return this.publish(this.session.execute(command));
}
/**
* Accept or decline a long run by voice. Only fires while an offer stands
* or a run is going, so these phrases stay out of the way of ordinary
* conversation the rest of the time.
*/
handleLongRunUtterance(utterance: string): MathBranchOutcome | null {
const text = utterance.toLowerCase();
if (this.longRunRunning) {
if (!STOP_PHRASES.some((phrase) => text.includes(phrase))) return null;
this.cancelLongRun();
return { kind: "failure", reason: "Stopping" };
}
const offer = this.pendingLongRun;
if (!offer) return null;
if (ACCEPT_PHRASES.some((phrase) => text.includes(phrase))) {
void this.acceptLongRun();
return { kind: "value", text: `Working — ${compactDuration(offer.estimatedSeconds)}` };
}
if (DECLINE_PHRASES.some((phrase) => text.includes(phrase))) {
this.declineLongRun();
return { kind: "failure", reason: "Left it there" };
}
return null;
}
get hasLongRunInteraction(): boolean {
return this.pendingLongRun !== null || this.longRunRunning;
}
// -------------------------------------------------------------------------
// Uniform gesture contract
handleRingAction(action: RingAction): MathBranchOutcome | null {
// The offer and the running job both pre-empt the step navigation: while
// either is on screen, the input is answering that question and nothing
// else. Same gestures, no new vocabulary.
if (this.longRunRunning) {
if (action !== "exit") return null;
this.cancelLongRun();
return { kind: "failure", reason: "Stopping" };
}
const offer = this.pendingLongRun;
if (offer) {
switch (action) {
case "commit":
void this.acceptLongRun();
return { kind: "value", text: `Working — ${compactDuration(offer.estimatedSeconds)}` };
case "exit":
this.declineLongRun();
return { kind: "failure", reason: "Left it there" };
default:
return null;
}
}
if (!this.session.hasProblem) return null;
switch (action) {
case "forward":
return this.publish(this.session.execute({ kind: "nextStep" }));
case "back":
return this.publish(this.session.execute({ kind: "previousStep" }));
case "commit":
return this.publish(this.session.graph());
case "alternate":
return this.publish(this.session.explain());
case "exit":
return this.publish(this.session.execute({ kind: "clear" }));
}
}
// -------------------------------------------------------------------------
// Long runs
/** Take the standing offer. Returns false when there was nothing to accept. */
acceptLongRun(): boolean {
const offer = this.pendingLongRun;
if (!offer || this.longRunRunning) return false;
this.pendingLongRun = null;
this.deepAnswer = null;
this.longRunCancelled = false;
this.longRunRunning = true;
this.longRunStartedAtMs = Date.now();
this.longRunProgress = { fraction: 0, stage: "starting", elapsedSeconds: 0 };
this.changed();
const cancellation = () => this.longRunCancelled;
const progress = (fraction: number, stage: string) => {
this.longRunProgress = {
fraction,
stage,
elapsedSeconds: (Date.now() - this.longRunStartedAtMs) / 1000,
};
this.changed();
};
void (async () => {
let outcome: MathBranchOutcome;
try {
switch (offer.workload.kind) {
case "digitSearch": {
const found = await runAcceptedSearch(offer, cancellation, progress);
outcome = describeDeepSearch(found, offer.workload.pattern, offer.workload.constant);
break;
}
case "constantDigits": {
const digits = await runAcceptedDigits(offer, cancellation, progress);
outcome = digits
? {
kind: "value",
text: `${constantDisplayName(offer.workload.constant)}: ${formatInt(digits.length)} digits ready`,
}
: { kind: "failure", reason: "Could not generate that many digits" };
break;
}
case "digitCount": {
const counted = await runAcceptedCount(offer, cancellation, progress);
outcome = {
kind: "value",
text: describeCount(counted.occurrences, counted.searchedDigits, offer.workload.pattern, offer.workload.constant),
};
break;
}
}
} catch (error) {
outcome =
error instanceof ComputeCancelled
? { kind: "failure", reason: "Stopped" }
: { kind: "failure", reason: String((error as Error)?.message ?? error) };
}
this.longRunRunning = false;
this.longRunProgress = null;
this.deepAnswer = outcome;
this.changed();
})();
return true;
}
/** Turn the offer down. The shallow answer already given stands. */
declineLongRun(): void {
this.pendingLongRun = null;
this.changed();
}
/**
* Stop a run in flight. The digits computed so far are discarded — a
* partial expansion cannot answer a position question.
*/
cancelLongRun(): void {
this.longRunCancelled = true;
}
// -------------------------------------------------------------------------
// Publishing
private publish(sessionResult: MathSessionResult): MathBranchOutcome {
this.lastResult = sessionResult;
// The shallow answer goes out immediately; anything better arrives on
// `deepAnswer` if the wearer accepts the standing offer. Never block the
// first response on the second — silence is worse than a partial.
if (sessionResult.unresolvedWorkload) {
this.pendingLongRun = makeLongRunOffer(
sessionResult.unresolvedWorkload,
"Needs a deeper expansion than the fast pass holds",
shallowDescription(sessionResult.unresolvedWorkload, sessionResult),
);
this.deepAnswer = null;
}
switch (sessionResult.payload.kind) {
case "graph":
this.lastGraph = sessionResult.payload.graph;
this.stepPosition = null;
break;
case "step":
this.lastGraph = null;
this.stepPosition = { index: sessionResult.payload.index, total: sessionResult.payload.total };
break;
case "solution":
case "message":
case "topic":
this.lastGraph = null;
// An advanced topic answers in one shot — there is no working to step
// through, so the step indicator must clear.
this.stepPosition = null;
break;
case "failure":
break;
}
this.changed();
if (sessionResult.payload.kind === "failure") {
return { kind: "failure", reason: sessionResult.payload.message };
}
return { kind: "value", text: this.glassesText(sessionResult) };
}
/**
* Two or three short rows. The lens has no room for the detail sentence,
* which is why `detail` stays in the fuller answer view.
*/
private glassesText(sessionResult: MathSessionResult): string {
switch (sessionResult.payload.kind) {
case "step": {
const payload = sessionResult.payload;
return `${payload.index + 1}/${payload.total} ${payload.step.title}\n${plain(payload.step.expression)}`;
}
case "graph":
// The bitmap carries the picture; this is the text fallback.
return sessionResult.headline;
case "topic": {
// Two field rows at most: the window is short and the headline
// already took a line.
const payload = sessionResult.payload;
return [payload.headline, ...payload.fields.slice(0, 2)].join("\n");
}
default:
return sessionResult.headline;
}
}
}
// ---------------------------------------------------------------------------
// Long-run phrases
const ACCEPT_PHRASES = [
"keep going",
"run it anyway",
"run it any way",
"take your time",
"go deeper",
"keep looking",
"do it anyway",
"yes go on",
];
const DECLINE_PHRASES = [
"never mind",
"nevermind",
"forget it",
"leave it",
"don't bother",
"do not bother",
"skip it",
];
const STOP_PHRASES = ["stop", "cancel", "give up", "that's enough", "thats enough"];
// ---------------------------------------------------------------------------
// Descriptions
function describeDeepSearch(
outcome: Awaited<ReturnType<typeof runAcceptedSearch>>,
pattern: string,
constant: MathConstantName,
): MathBranchOutcome {
switch (outcome.kind) {
case "found":
return {
kind: "value",
text:
`${constantDisplayName(constant)}: places ${outcome.match.start}–${outcome.match.end}\n` +
`…${highlightedMatch(outcome.match)}…`,
};
case "notFound":
// Now a much stronger claim than the shallow pass could make, so it is
// worth stating the depth that was actually reached.
return {
kind: "failure",
reason:
`${pattern} is not in the first ${formatInt(outcome.searchedDigits)} digits ` +
`of ${constantDisplayName(constant)}`,
};
case "invalidPattern":
return { kind: "failure", reason: outcome.reason };
}
}
function describeCount(
occurrences: number,
searchedDigits: number,
pattern: string,
constant: MathConstantName,
): string {
const times = occurrences === 1 ? "time" : "times";
return `${pattern}: ${occurrences} ${times}\nfirst ${formatInt(searchedDigits)} digits of ${constantDisplayName(constant)}`;
}
function shallowDescription(workload: MathWorkload, sessionResult: MathSessionResult): string | null {
switch (workload.kind) {
case "digitSearch":
return sessionResult.headline;
case "digitCount":
return sessionResult.headline;
case "constantDigits":
return null;
}
}
+234
View File
@@ -0,0 +1,234 @@
// Symbolic differentiation. Every rule is applied structurally and the result
// is handed to the simplifier, so `d/dx (x^2)` comes back as `2x` rather than
// `2 · x^(2-1) · 1`. That matters because the derivative is user-visible — it
// is what "explain it" shows.
import {
type MathExpression,
type MathFunction,
ONE,
ZERO,
addExpr,
divideExpr,
fn,
int,
multiplyExpr,
negate,
powerExpr,
relation,
subtractExpr,
variablesOf,
} from "./expression";
import { simplify } from "./simplifier";
import { evaluateValue } from "./evaluator";
export function differentiate(expression: MathExpression, withRespectTo: string): MathExpression {
return simplify(raw(expression, withRespectTo));
}
/** Nth derivative. */
export function differentiateOrder(
expression: MathExpression,
withRespectTo: string,
order: number,
): MathExpression {
let current = expression;
for (let step = 0; step < Math.max(0, order); step++) {
current = differentiate(current, withRespectTo);
}
return current;
}
function raw(expression: MathExpression, variable: string): MathExpression {
switch (expression.type) {
case "number":
case "constant":
return ZERO;
case "variable":
return expression.name === variable ? ONE : ZERO;
case "negate":
return negate(raw(expression.inner, variable));
case "add":
return addExpr(expression.terms.map((term) => raw(term, variable)));
case "multiply": {
// Product rule, generalised: d(abc) = a'bc + ab'c + abc'.
const terms: MathExpression[] = [];
for (let index = 0; index < expression.factors.length; index++) {
const replaced = [...expression.factors];
replaced[index] = raw(expression.factors[index]!, variable);
terms.push(multiplyExpr(replaced));
}
return addExpr(terms);
}
case "power":
return differentiatePower(expression.base, expression.exponent, variable);
case "function": {
const argument = expression.args[0];
if (!argument) return ZERO;
const outer = derivativeOfFunction(expression.fn, argument);
// Chain rule. Skipping this when the argument is the bare variable
// would be a needless special case — the simplifier removes the `· 1`
// anyway.
return multiplyExpr([outer, raw(argument, variable)]);
}
case "relation":
// Differentiating both sides of an equation is implicit
// differentiation's first step; useful, and harmless here.
return relation(expression.relation, raw(expression.lhs, variable), raw(expression.rhs, variable));
}
}
function differentiatePower(
base: MathExpression,
exponent: MathExpression,
variable: string,
): MathExpression {
const baseHasVariable = variablesOf(base).includes(variable);
const exponentHasVariable = variablesOf(exponent).includes(variable);
if (!baseHasVariable && !exponentHasVariable) return ZERO;
if (baseHasVariable && !exponentHasVariable) {
// Power rule: d(u^n) = n · u^(n-1) · u'
const reduced = subtractExpr(exponent, ONE);
return multiplyExpr([exponent, powerExpr(base, reduced), raw(base, variable)]);
}
if (!baseHasVariable && exponentHasVariable) {
// Exponential rule: d(a^v) = a^v · ln(a) · v'
return multiplyExpr([powerExpr(base, exponent), fn("ln", [base]), raw(exponent, variable)]);
}
// General case via logarithmic differentiation:
// d(u^v) = u^v · (v' · ln u + v · u'/u)
const uv = powerExpr(base, exponent);
const firstTerm = multiplyExpr([raw(exponent, variable), fn("ln", [base])]);
const secondTerm = multiplyExpr([exponent, divideExpr(raw(base, variable), base)]);
return multiplyExpr([uv, addExpr([firstTerm, secondTerm])]);
}
/** The outer derivative only; the caller applies the chain rule. */
function derivativeOfFunction(name: MathFunction, argument: MathExpression): MathExpression {
switch (name) {
case "sin":
return fn("cos", [argument]);
case "cos":
return negate(fn("sin", [argument]));
case "tan":
// sec² = 1 / cos²
return divideExpr(ONE, powerExpr(fn("cos", [argument]), int(2)));
case "asin":
return divideExpr(ONE, fn("sqrt", [subtractExpr(ONE, powerExpr(argument, int(2)))]));
case "acos":
return negate(divideExpr(ONE, fn("sqrt", [subtractExpr(ONE, powerExpr(argument, int(2)))])));
case "atan":
return divideExpr(ONE, addExpr([ONE, powerExpr(argument, int(2))]));
case "sinh":
return fn("cosh", [argument]);
case "cosh":
return fn("sinh", [argument]);
case "tanh":
return divideExpr(ONE, powerExpr(fn("cosh", [argument]), int(2)));
case "ln":
return divideExpr(ONE, argument);
case "log10":
return divideExpr(ONE, multiplyExpr([argument, fn("ln", [int(10)])]));
case "log2":
return divideExpr(ONE, multiplyExpr([argument, fn("ln", [int(2)])]));
case "sqrt":
return divideExpr(ONE, multiplyExpr([int(2), fn("sqrt", [argument])]));
case "cbrt":
return divideExpr(ONE, multiplyExpr([int(3), powerExpr(fn("cbrt", [argument]), int(2))]));
case "exp":
return fn("exp", [argument]);
case "abs":
// d|u|/du = u/|u| — undefined at zero, which the plotter's
// discontinuity handling already copes with.
return divideExpr(argument, fn("abs", [argument]));
case "floor":
case "ceil":
case "round":
// Zero almost everywhere; the jumps are measure-zero and are not
// representable here.
return ZERO;
case "factorial":
// The derivative of the gamma function needs the digamma function,
// which this build does not carry. Returning zero would be a lie, so
// the caller gets an unevaluated marker instead.
return fn("factorial", [argument]);
}
}
// ---------------------------------------------------------------------------
// Numeric integration
/**
* Definite integral by adaptive Simpson's rule.
*
* Numeric rather than symbolic: symbolic integration needs a Risch-style
* algorithm that is far out of proportion to the feature, and a definite
* value is what a calculator user actually asks for. (The symbolic table in
* symbolic-integral.ts is tried first where it applies.)
*/
export function integrateNumeric(
expression: MathExpression,
variable: string,
lower: number,
upper: number,
tolerance = 1e-9,
): number | null {
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
if (lower === upper) return 0;
if (lower > upper) {
const flipped = integrateNumeric(expression, variable, upper, lower, tolerance);
return flipped === null ? null : -flipped;
}
const f = (x: number): number | null => evaluateValue(expression, { [variable]: x });
const simpson = (a: number, b: number, fa: number, fb: number, fm: number): number =>
((b - a) / 6) * (fa + 4 * fm + fb);
const adaptive = (
a: number,
b: number,
fa: number,
fb: number,
fm: number,
whole: number,
depth: number,
): number | null => {
const m = (a + b) / 2;
const leftMid = (a + m) / 2;
const rightMid = (m + b) / 2;
const flm = f(leftMid);
const frm = f(rightMid);
if (flm === null || frm === null) return null;
const left = simpson(a, m, fa, fm, flm);
const right = simpson(m, b, fm, fb, frm);
const total = left + right;
// Richardson: the Simpson error term is (total - whole) / 15.
if (depth <= 0 || Math.abs(total - whole) <= 15 * tolerance) {
return total + (total - whole) / 15;
}
const leftResult = adaptive(a, m, fa, fm, flm, left, depth - 1);
if (leftResult === null) return null;
const rightResult = adaptive(m, b, fm, fb, frm, right, depth - 1);
if (rightResult === null) return null;
return leftResult + rightResult;
};
const fa = f(lower);
const fb = f(upper);
const fm = f((lower + upper) / 2);
if (fa === null || fb === null || fm === null) return null;
const whole = simpson(lower, upper, fa, fb, fm);
return adaptive(lower, upper, fa, fb, fm, whole, 24);
}
+184
View File
@@ -0,0 +1,184 @@
// Finds a digit sequence inside the expansion of a constant.
//
// The position convention is fixed and stated everywhere it is reported:
// **decimal places after the point, 1-based**. So the first `11` in π is at
// 94–95, because π = 3.(1)4159… and the 94th digit after the point is the
// first `1` of that pair. Every off-by-one complaint about this kind of
// feature is a convention that was never written down, so it is written down.
import {
type MathConstantName,
MAXIMUM_COMPUTED_DIGITS,
fractionalDigits,
} from "./constant-digits";
export type DigitCountResult = {
occurrences: number;
searchedDigits: number;
};
export type DigitMatch = {
/** 1-based decimal place of the first digit of the match. */
start: number;
/** 1-based decimal place of the last digit. */
end: number;
/** The digits either side, for the reveal. */
context: string;
/** Where the match sits inside `context`, as an offset. */
contextOffset: number;
};
/** `...3421 ·11· 70679...` with the match marked. */
export function highlightedMatch(match: DigitMatch, marker = "·"): string {
const length = match.end - match.start + 1;
if (match.contextOffset < 0 || match.contextOffset + length > match.context.length) {
return match.context;
}
const before = match.context.slice(0, match.contextOffset);
const middle = match.context.slice(match.contextOffset, match.contextOffset + length);
const after = match.context.slice(match.contextOffset + length);
return `${before}${marker}${middle}${marker}${after}`;
}
export type DigitSearchOutcome =
| { kind: "found"; match: DigitMatch }
/**
* Searched the whole available expansion and did not find it. Carries how
* far the search actually reached, because "not in π" and "not in the first
* 20,000 digits of π" are very different claims and only one of them is
* true.
*/
| { kind: "notFound"; searchedDigits: number }
| { kind: "invalidPattern"; reason: string };
/** How many digits either side of a hit to return. */
export const DEFAULT_CONTEXT_RADIUS = 6;
export type DigitSource = (constant: MathConstantName, count: number) => string | null;
const DEFAULT_SOURCE: DigitSource = (constant, count) => fractionalDigits(constant, count);
/** Find the `occurrence`-th appearance of `pattern`. */
export function findDigits(
pattern: string,
constant: MathConstantName,
occurrence = 1,
requested: number = MAXIMUM_COMPUTED_DIGITS,
contextRadius: number = DEFAULT_CONTEXT_RADIUS,
digitSource: DigitSource = DEFAULT_SOURCE,
): DigitSearchOutcome {
const cleaned = pattern.replace(/\D/g, "");
if (!cleaned) return { kind: "invalidPattern", reason: "Give a sequence of digits to look for" };
if (occurrence < 1) return { kind: "invalidPattern", reason: "Occurrence must be 1 or more" };
const expansion = digitSource(constant, requested);
if (!expansion) return { kind: "notFound", searchedDigits: 0 };
return locateDigits(cleaned, expansion, occurrence, contextRadius);
}
/**
* Search an expansion the caller already has — the path an accepted long run
* uses, where the digits come from the deep generator rather than the
* shallow cache.
*/
export function locateDigits(
pattern: string,
expansion: string,
occurrence = 1,
contextRadius: number = DEFAULT_CONTEXT_RADIUS,
): DigitSearchOutcome {
if (pattern.length > expansion.length) {
return { kind: "notFound", searchedDigits: expansion.length };
}
let seen = 0;
// Plain scan rather than a substring search: overlapping matches count
// ("111" contains "11" twice), and indexOf-with-skip would miss the
// overlap.
for (let index = 0; index <= expansion.length - pattern.length; index++) {
let matched = true;
for (let offset = 0; offset < pattern.length; offset++) {
if (expansion[index + offset] !== pattern[offset]) {
matched = false;
break;
}
}
if (!matched) continue;
seen += 1;
if (seen !== occurrence) continue;
const contextStart = Math.max(0, index - contextRadius);
const contextEnd = Math.min(expansion.length, index + pattern.length + contextRadius);
return {
kind: "found",
match: {
start: index + 1,
end: index + pattern.length,
context: expansion.slice(contextStart, contextEnd),
contextOffset: index - contextStart,
},
};
}
return { kind: "notFound", searchedDigits: expansion.length };
}
/** How many times the pattern appears in the available expansion. */
export function countDigits(
pattern: string,
constant: MathConstantName,
requested: number = MAXIMUM_COMPUTED_DIGITS,
digitSource: DigitSource = DEFAULT_SOURCE,
): DigitCountResult {
const cleaned = pattern.replace(/\D/g, "");
if (!cleaned) return { occurrences: 0, searchedDigits: 0 };
const expansion = digitSource(constant, requested);
if (!expansion) return { occurrences: 0, searchedDigits: 0 };
return countInExpansion(cleaned, expansion);
}
/** Count overlapping occurrences inside an expansion the caller already has. */
export function countInExpansion(pattern: string, expansion: string): DigitCountResult {
if (pattern.length > expansion.length) {
return { occurrences: 0, searchedDigits: expansion.length };
}
let total = 0;
for (let index = 0; index <= expansion.length - pattern.length; index++) {
let matched = true;
for (let offset = 0; offset < pattern.length; offset++) {
if (expansion[index + offset] !== pattern[offset]) {
matched = false;
break;
}
}
if (matched) total += 1;
}
return { occurrences: total, searchedDigits: expansion.length };
}
/** The digits at a given range of decimal places, 1-based inclusive. */
export function digitsAt(
constant: MathConstantName,
start: number,
end: number,
digitSource: DigitSource = DEFAULT_SOURCE,
): string | null {
if (start < 1 || end < start) return null;
const expansion = digitSource(constant, end);
if (!expansion || expansion.length < end) return null;
return expansion.slice(start - 1, end);
}
/**
* How deep you would expect to search before a random `length`-digit string
* turns up: 10^length places, on average.
*
* Reported alongside a notFound so the answer is honest about whether the
* search was ever likely to succeed — an eight-digit birthday first appears
* around the hundred-millionth digit, which no on-device expansion is going
* to reach.
*/
export function expectedFirstPosition(patternLength: number): number | null {
if (patternLength <= 0 || patternLength > 18) return null;
return Math.pow(10, patternLength);
}
+189
View File
@@ -0,0 +1,189 @@
// Numeric evaluation of an expression tree. Two return paths on purpose:
// `evaluateExact` keeps rationals as rationals (so a solution can be shown as
// `3/2`), while `evaluateValue` collapses to a plain number for the plotter,
// which samples thousands of points and has no use for exactness.
import {
type MathExpression,
constantValue,
evaluateFunction,
functionIsDefined,
} from "./expression";
import {
type MathNumber,
add,
doubleValue,
integer,
integerValue,
isNegative,
isZero,
multiply,
negated,
raisedTo,
real,
} from "./math-number";
export type EvaluatorContext = {
variables: Record<string, MathNumber>;
/**
* Radians unless the caller says otherwise. Trig in degrees is a real user
* expectation for a calculator, and getting it silently wrong produces
* answers that look plausible.
*/
usesDegrees: boolean;
};
export function radiansContext(): EvaluatorContext {
return { variables: {}, usesDegrees: false };
}
export class EvaluationError extends Error {
constructor(message: string) {
super(message);
this.name = "EvaluationError";
}
}
/**
* Evaluate keeping exactness where possible. Throws on a free variable, so
* the caller can distinguish "not a number yet" from "evaluates to NaN".
*/
export function evaluateExact(expression: MathExpression, context: EvaluatorContext = radiansContext()): MathNumber {
switch (expression.type) {
case "number":
return expression.value;
case "variable": {
const value = context.variables[expression.name];
if (value === undefined) {
throw new EvaluationError(`No value given for ${expression.name}`);
}
return value;
}
case "constant":
return real(constantValue(expression.constant));
case "add":
return expression.terms.reduce<MathNumber>(
(total, term) => add(total, evaluateExact(term, context)),
integer(0),
);
case "multiply":
return expression.factors.reduce<MathNumber>(
(total, factor) => multiply(total, evaluateExact(factor, context)),
integer(1),
);
case "negate":
return negated(evaluateExact(expression.inner, context));
case "power": {
const baseValue = evaluateExact(expression.base, context);
const exponentValue = evaluateExact(expression.exponent, context);
if (isZero(baseValue) && isNegative(exponentValue)) {
throw new EvaluationError("Division by zero");
}
return raisedTo(baseValue, exponentValue);
}
case "function": {
const first = expression.args[0];
if (!first) throw new EvaluationError(`${expression.fn} needs an argument`);
const argument = evaluateExact(first, context);
// Integer factorial stays exact; everything else goes through the
// floating-point implementation.
const argumentInt = integerValue(argument);
if (expression.fn === "factorial" && argumentInt !== null && argumentInt >= 0 && argumentInt <= 20) {
let total = 1;
for (let value = 2; value <= argumentInt; value++) total *= value;
return integer(total);
}
const raw = adjustForDegrees(expression.fn, doubleValue(argument), context);
if (!functionIsDefined(expression.fn, raw)) {
throw new EvaluationError(`${expression.fn} is undefined there`);
}
const result = evaluateFunction(expression.fn, raw);
if (!Number.isFinite(result)) {
throw new EvaluationError(`${expression.fn} is undefined there`);
}
return adjustResultForDegrees(expression.fn, result, context);
}
case "relation": {
const left = doubleValue(evaluateExact(expression.lhs, context));
const right = doubleValue(evaluateExact(expression.rhs, context));
let holds: boolean;
switch (expression.relation) {
case "=":
holds = left === right;
break;
case "<":
holds = left < right;
break;
case "<=":
holds = left <= right;
break;
case ">":
holds = left > right;
break;
case ">=":
holds = left >= right;
break;
}
return integer(holds ? 1 : 0);
}
}
}
/**
* Plotting path: never throws, returns null where the function is undefined
* so the renderer can break the curve instead of drawing a line across the
* gap.
*/
export function evaluateValue(
expression: MathExpression,
variables: Record<string, number>,
usesDegrees = false,
): number | null {
const context: EvaluatorContext = { variables: {}, usesDegrees };
for (const name of Object.keys(variables)) {
context.variables[name] = real(variables[name]!);
}
let result: MathNumber;
try {
result = evaluateExact(expression, context);
} catch {
return null;
}
const value = doubleValue(result);
return Number.isFinite(value) ? value : null;
}
// ---------------------------------------------------------------------------
// Degrees
function adjustForDegrees(name: string, argument: number, context: EvaluatorContext): number {
if (!context.usesDegrees) return argument;
switch (name) {
case "sin":
case "cos":
case "tan":
return (argument * Math.PI) / 180;
default:
return argument;
}
}
function adjustResultForDegrees(name: string, result: number, context: EvaluatorContext): MathNumber {
if (!context.usesDegrees) return real(result);
switch (name) {
case "asin":
case "acos":
case "atan":
return real((result * 180) / Math.PI);
default:
return real(result);
}
}
+487
View File
@@ -0,0 +1,487 @@
// The expression tree everything else operates on: evaluation, simplification,
// differentiation, solving, rendering, and plotting. A plain discriminated
// union so the whole tree is a value — simplification and differentiation can
// rebuild subtrees freely, and structural comparison (expressionEquals) gives
// the simplifier the like-term matching it leans on heavily.
import {
type MathNumber,
integer as makeInteger,
fromDecimal,
isZero as numberIsZero,
isOne as numberIsOne,
numberEquals as numbersEqual,
} from "./math-number";
export type MathRelation = "=" | "<" | "<=" | ">" | ">=";
export type MathConstant = "pi" | "e" | "tau";
export type MathFunction =
| "sin"
| "cos"
| "tan"
| "asin"
| "acos"
| "atan"
| "sinh"
| "cosh"
| "tanh"
| "ln"
| "log10"
| "log2"
| "sqrt"
| "cbrt"
| "abs"
| "exp"
| "floor"
| "ceil"
| "round"
| "factorial";
export const ALL_FUNCTIONS: readonly MathFunction[] = [
"sin",
"cos",
"tan",
"asin",
"acos",
"atan",
"sinh",
"cosh",
"tanh",
"ln",
"log10",
"log2",
"sqrt",
"cbrt",
"abs",
"exp",
"floor",
"ceil",
"round",
"factorial",
];
export const ALL_CONSTANTS: readonly MathConstant[] = ["pi", "e", "tau"];
export type MathExpression =
| { type: "number"; value: MathNumber }
/** A free variable — `x`, `y`, `theta`. */
| { type: "variable"; name: string }
/**
* A named constant with a known value (`pi`, `e`). Kept distinct from
* `variable` so "solve for pi" is impossible and `2π` stays exact instead of
* becoming 6.283…
*/
| { type: "constant"; constant: MathConstant }
| { type: "add"; terms: MathExpression[] }
| { type: "multiply"; factors: MathExpression[] }
| { type: "power"; base: MathExpression; exponent: MathExpression }
/**
* Unary negation. Not folded into `multiply([-1, x])` because the renderers
* and the step explanations both want to say "negative x".
*/
| { type: "negate"; inner: MathExpression }
| { type: "function"; fn: MathFunction; args: MathExpression[] }
/** An equation or inequality — the root node of anything solvable. */
| { type: "relation"; relation: MathRelation; lhs: MathExpression; rhs: MathExpression };
// ---------------------------------------------------------------------------
// Convenience constructors
export function num(value: MathNumber): MathExpression {
return { type: "number", value };
}
export function int(value: number): MathExpression {
return num(makeInteger(value));
}
export function realExpr(value: number): MathExpression {
return num(fromDecimal(value));
}
export function variable(name: string): MathExpression {
return { type: "variable", name };
}
export function constant(value: MathConstant): MathExpression {
return { type: "constant", constant: value };
}
export function addExpr(terms: MathExpression[]): MathExpression {
return { type: "add", terms };
}
export function multiplyExpr(factors: MathExpression[]): MathExpression {
return { type: "multiply", factors };
}
export function powerExpr(base: MathExpression, exponent: MathExpression): MathExpression {
return { type: "power", base, exponent };
}
export function negate(inner: MathExpression): MathExpression {
return { type: "negate", inner };
}
export function fn(name: MathFunction, args: MathExpression[]): MathExpression {
return { type: "function", fn: name, args };
}
export function relation(rel: MathRelation, lhs: MathExpression, rhs: MathExpression): MathExpression {
return { type: "relation", relation: rel, lhs, rhs };
}
export function subtractExpr(lhs: MathExpression, rhs: MathExpression): MathExpression {
return addExpr([lhs, negate(rhs)]);
}
/**
* Division is `a * b^-1` in the tree, which is what makes the simplifier and
* the differentiator tractable — the renderers detect the pattern and print a
* real fraction, so nothing user-visible degrades.
*/
export function divideExpr(lhs: MathExpression, rhs: MathExpression): MathExpression {
return multiplyExpr([lhs, powerExpr(rhs, int(-1))]);
}
export const ZERO: MathExpression = int(0);
export const ONE: MathExpression = int(1);
// ---------------------------------------------------------------------------
// Introspection
export function isZeroExpr(expression: MathExpression): boolean {
return expression.type === "number" && numberIsZero(expression.value);
}
export function isOneExpr(expression: MathExpression): boolean {
return expression.type === "number" && numberIsOne(expression.value);
}
export function numberValue(expression: MathExpression): MathNumber | null {
return expression.type === "number" ? expression.value : null;
}
/**
* Every free variable in the tree, in first-appearance order. Order matters:
* it decides which variable "solve for x" defaults to and which axis a
* single-variable graph uses.
*/
export function variablesOf(expression: MathExpression): string[] {
const seen = new Set<string>();
const ordered: string[] = [];
collectVariables(expression, ordered, seen);
return ordered;
}
function collectVariables(expression: MathExpression, ordered: string[], seen: Set<string>): void {
switch (expression.type) {
case "variable":
if (!seen.has(expression.name)) {
seen.add(expression.name);
ordered.push(expression.name);
}
break;
case "number":
case "constant":
break;
case "add":
for (const term of expression.terms) collectVariables(term, ordered, seen);
break;
case "multiply":
for (const factor of expression.factors) collectVariables(factor, ordered, seen);
break;
case "power":
collectVariables(expression.base, ordered, seen);
collectVariables(expression.exponent, ordered, seen);
break;
case "negate":
collectVariables(expression.inner, ordered, seen);
break;
case "function":
for (const argument of expression.args) collectVariables(argument, ordered, seen);
break;
case "relation":
collectVariables(expression.lhs, ordered, seen);
collectVariables(expression.rhs, ordered, seen);
break;
}
}
export function isRelation(expression: MathExpression): boolean {
return expression.type === "relation";
}
/**
* Node count — used to pick the simpler of two equivalent forms and to bound
* runaway expansion in the simplifier.
*/
export function complexity(expression: MathExpression): number {
switch (expression.type) {
case "number":
case "variable":
case "constant":
return 1;
case "add":
return 1 + expression.terms.reduce((total, term) => total + complexity(term), 0);
case "multiply":
return 1 + expression.factors.reduce((total, factor) => total + complexity(factor), 0);
case "power":
return 1 + complexity(expression.base) + complexity(expression.exponent);
case "negate":
return 1 + complexity(expression.inner);
case "function":
return 1 + expression.args.reduce((total, argument) => total + complexity(argument), 0);
case "relation":
return 1 + complexity(expression.lhs) + complexity(expression.rhs);
}
}
/** Replace every occurrence of a variable with an expression. */
export function substituting(
expression: MathExpression,
name: string,
replacement: MathExpression,
): MathExpression {
switch (expression.type) {
case "variable":
return expression.name === name ? replacement : expression;
case "number":
case "constant":
return expression;
case "add":
return addExpr(expression.terms.map((term) => substituting(term, name, replacement)));
case "multiply":
return multiplyExpr(expression.factors.map((factor) => substituting(factor, name, replacement)));
case "power":
return powerExpr(
substituting(expression.base, name, replacement),
substituting(expression.exponent, name, replacement),
);
case "negate":
return negate(substituting(expression.inner, name, replacement));
case "function":
return fn(
expression.fn,
expression.args.map((argument) => substituting(argument, name, replacement)),
);
case "relation":
return relation(
expression.relation,
substituting(expression.lhs, name, replacement),
substituting(expression.rhs, name, replacement),
);
}
}
/**
* Structural equality with value-equal numbers (MathNumber compares by
* value).
*/
export function expressionEquals(a: MathExpression, b: MathExpression): boolean {
if (a === b) return true;
if (a.type !== b.type) return false;
switch (a.type) {
case "number": {
const other = b as Extract<MathExpression, { type: "number" }>;
return numbersEqual(a.value, other.value);
}
case "variable":
return a.name === (b as Extract<MathExpression, { type: "variable" }>).name;
case "constant":
return a.constant === (b as Extract<MathExpression, { type: "constant" }>).constant;
case "add": {
const other = b as Extract<MathExpression, { type: "add" }>;
return listEquals(a.terms, other.terms);
}
case "multiply": {
const other = b as Extract<MathExpression, { type: "multiply" }>;
return listEquals(a.factors, other.factors);
}
case "power": {
const other = b as Extract<MathExpression, { type: "power" }>;
return expressionEquals(a.base, other.base) && expressionEquals(a.exponent, other.exponent);
}
case "negate":
return expressionEquals(a.inner, (b as Extract<MathExpression, { type: "negate" }>).inner);
case "function": {
const other = b as Extract<MathExpression, { type: "function" }>;
return a.fn === other.fn && listEquals(a.args, other.args);
}
case "relation": {
const other = b as Extract<MathExpression, { type: "relation" }>;
return (
a.relation === other.relation &&
expressionEquals(a.lhs, other.lhs) &&
expressionEquals(a.rhs, other.rhs)
);
}
}
}
function listEquals(a: MathExpression[], b: MathExpression[]): boolean {
if (a.length !== b.length) return false;
for (let index = 0; index < a.length; index++) {
if (!expressionEquals(a[index]!, b[index]!)) return false;
}
return true;
}
// ---------------------------------------------------------------------------
// Relations
/**
* The relation you get by multiplying or dividing both sides by a negative —
* the classic inequality mistake, handled explicitly.
*/
export function flippedRelation(rel: MathRelation): MathRelation {
switch (rel) {
case "=":
return "=";
case "<":
return ">";
case "<=":
return ">=";
case ">":
return "<";
case ">=":
return "<=";
}
}
export function spokenRelation(rel: MathRelation): string {
switch (rel) {
case "=":
return "equals";
case "<":
return "is less than";
case "<=":
return "is at most";
case ">":
return "is greater than";
case ">=":
return "is at least";
}
}
// ---------------------------------------------------------------------------
// Constants
export function constantValue(value: MathConstant): number {
switch (value) {
case "pi":
return Math.PI;
case "e":
return Math.E;
case "tau":
return 2 * Math.PI;
}
}
// ---------------------------------------------------------------------------
// Functions
export function evaluateFunction(name: MathFunction, argument: number): number {
switch (name) {
case "sin":
return Math.sin(argument);
case "cos":
return Math.cos(argument);
case "tan":
return Math.tan(argument);
case "asin":
return Math.asin(argument);
case "acos":
return Math.acos(argument);
case "atan":
return Math.atan(argument);
case "sinh":
return Math.sinh(argument);
case "cosh":
return Math.cosh(argument);
case "tanh":
return Math.tanh(argument);
case "ln":
return Math.log(argument);
case "log10":
return Math.log10(argument);
case "log2":
return Math.log2(argument);
case "sqrt":
return Math.sqrt(argument);
case "cbrt":
return Math.cbrt(argument);
case "abs":
return Math.abs(argument);
case "exp":
return Math.exp(argument);
case "floor":
return Math.floor(argument);
case "ceil":
return Math.ceil(argument);
case "round":
return Math.round(argument);
case "factorial":
// Real factorial via the gamma function, so `2.5!` is defined rather
// than an error — and so the evaluator has one code path for both.
return gamma(argument + 1);
}
}
/**
* Domain check used by the plotter to break a curve rather than draw a line
* through a gap (`ln` left of zero, `tan` at its asymptotes).
*/
export function functionIsDefined(name: MathFunction, argument: number): boolean {
switch (name) {
case "ln":
case "log10":
case "log2":
return argument > 0;
case "sqrt":
return argument >= 0;
case "asin":
case "acos":
return argument >= -1 && argument <= 1;
case "factorial":
return argument > -1 || Math.round(argument) !== argument;
default:
return true;
}
}
/**
* Lanczos approximation of the gamma function (JavaScript has no tgamma).
* Accurate to well past the display precision used anywhere in this app.
*/
export function gamma(x: number): number {
if (Number.isNaN(x)) return NaN;
if (x === Math.round(x)) {
if (x <= 0) return NaN; // poles at non-positive integers
if (x <= 171) {
let result = 1;
for (let k = 2; k < x; k++) result *= k;
return result;
}
return Infinity;
}
if (x < 0.5) {
// Reflection formula.
return Math.PI / (Math.sin(Math.PI * x) * gamma(1 - x));
}
const g = 7;
const coefficients = [
0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,
-176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,
1.5056327351493116e-7,
];
const z = x - 1;
let sum = coefficients[0]!;
for (let index = 1; index < g + 2; index++) {
sum += coefficients[index]! / (z + index);
}
const t = z + g + 0.5;
return Math.sqrt(2 * Math.PI) * Math.pow(t, z + 0.5) * Math.exp(-t) * sum;
}
+274
View File
@@ -0,0 +1,274 @@
// The parts of geometry the calculator's spoken-question surface reaches:
// shapes, triangle-from-sides, and Pythagoras.
// ---------------------------------------------------------------------------
// Triangles
/** A triangle solved from whatever three parts were given. */
export type Triangle = {
/** Side lengths, opposite the same-named angles. */
a: number;
b: number;
c: number;
/** Angles in radians. */
angleA: number;
angleB: number;
angleC: number;
};
export function triangleArea(triangle: Triangle): number {
// Heron's formula, in the numerically stable ordering: the naive form
// loses most of its precision on a needle-thin triangle.
const sides = [triangle.a, triangle.b, triangle.c].sort((p, q) => q - p);
const [x, y, z] = [sides[0]!, sides[1]!, sides[2]!];
const term = (x + (y + z)) * (z - (x - y)) * (z + (x - y)) * (x + (y - z));
return term <= 0 ? 0 : Math.sqrt(term) / 4;
}
export function trianglePerimeter(triangle: Triangle): number {
return triangle.a + triangle.b + triangle.c;
}
export function triangleKind(triangle: Triangle): string {
const tolerance = 1e-9;
const sides = [triangle.a, triangle.b, triangle.c].sort((p, q) => p - q);
const descriptors: string[] = [];
if (Math.abs(sides[0]! - sides[2]!) < tolerance) {
descriptors.push("equilateral");
} else if (Math.abs(sides[0]! - sides[1]!) < tolerance || Math.abs(sides[1]! - sides[2]!) < tolerance) {
descriptors.push("isosceles");
} else {
descriptors.push("scalene");
}
const largest = Math.max(triangle.angleA, triangle.angleB, triangle.angleC);
if (Math.abs(largest - Math.PI / 2) < 1e-6) {
descriptors.push("right");
} else if (largest > Math.PI / 2) {
descriptors.push("obtuse");
} else {
descriptors.push("acute");
}
return descriptors.join(", ");
}
export function triangleAnglesInDegrees(triangle: Triangle): [number, number, number] {
return [
(triangle.angleA * 180) / Math.PI,
(triangle.angleB * 180) / Math.PI,
(triangle.angleC * 180) / Math.PI,
];
}
export type TriangleResult =
| { kind: "one"; triangle: Triangle }
/**
* SSA can genuinely produce two valid triangles. Returning one and calling
* it solved is the classic wrong answer in this topic.
*/
| { kind: "two"; first: Triangle; second: Triangle }
| { kind: "impossible"; reason: string };
/**
* Guards acos/asin against a value a hair outside [-1, 1] from rounding,
* which returns NaN and poisons everything downstream.
*/
function clampedToUnit(value: number): number {
return Math.min(1, Math.max(-1, value));
}
/** Three sides. */
export function triangleFromSides(a: number, b: number, c: number): TriangleResult {
if (a <= 0 || b <= 0 || c <= 0) return { kind: "impossible", reason: "Sides must be positive" };
if (a + b <= c || b + c <= a || a + c <= b) {
return { kind: "impossible", reason: "Those sides cannot close a triangle" };
}
const angleA = Math.acos(clampedToUnit((b * b + c * c - a * a) / (2 * b * c)));
const angleB = Math.acos(clampedToUnit((a * a + c * c - b * b) / (2 * a * c)));
return {
kind: "one",
triangle: { a, b, c, angleA, angleB, angleC: Math.PI - angleA - angleB },
};
}
// ---------------------------------------------------------------------------
// Shapes
export type Shape =
| { kind: "circle"; radius: number }
| { kind: "rectangle"; width: number; height: number }
| { kind: "square"; side: number }
| { kind: "triangleBaseHeight"; base: number; height: number }
| { kind: "trapezoid"; a: number; b: number; height: number }
| { kind: "parallelogram"; base: number; height: number }
| { kind: "regularPolygon"; sides: number; sideLength: number }
| { kind: "ellipse"; a: number; b: number }
// Solids
| { kind: "sphere"; radius: number }
| { kind: "cylinder"; radius: number; height: number }
| { kind: "cone"; radius: number; height: number }
| { kind: "cube"; side: number }
| { kind: "rectangularPrism"; length: number; width: number; height: number }
| { kind: "pyramid"; baseArea: number; height: number };
export function shapeName(shape: Shape): string {
switch (shape.kind) {
case "circle":
return "circle";
case "rectangle":
return "rectangle";
case "square":
return "square";
case "triangleBaseHeight":
return "triangle";
case "trapezoid":
return "trapezoid";
case "parallelogram":
return "parallelogram";
case "regularPolygon":
return `regular ${shape.sides}-gon`;
case "ellipse":
return "ellipse";
case "sphere":
return "sphere";
case "cylinder":
return "cylinder";
case "cone":
return "cone";
case "cube":
return "cube";
case "rectangularPrism":
return "rectangular prism";
case "pyramid":
return "pyramid";
}
}
export function shapeIsSolid(shape: Shape): boolean {
switch (shape.kind) {
case "sphere":
case "cylinder":
case "cone":
case "cube":
case "rectangularPrism":
case "pyramid":
return true;
default:
return false;
}
}
/** Area for a plane figure; surface area for a solid. */
export function shapeArea(shape: Shape): number | null {
switch (shape.kind) {
case "circle":
return Math.PI * shape.radius * shape.radius;
case "rectangle":
return shape.width * shape.height;
case "square":
return shape.side * shape.side;
case "triangleBaseHeight":
return (shape.base * shape.height) / 2;
case "trapezoid":
return ((shape.a + shape.b) * shape.height) / 2;
case "parallelogram":
return shape.base * shape.height;
case "regularPolygon":
return shape.sides >= 3
? (shape.sides * shape.sideLength * shape.sideLength) / (4 * Math.tan(Math.PI / shape.sides))
: null;
case "ellipse":
return Math.PI * shape.a * shape.b;
case "sphere":
return 4 * Math.PI * shape.radius * shape.radius;
case "cylinder":
return 2 * Math.PI * shape.radius * (shape.radius + shape.height);
case "cone":
return (
Math.PI *
shape.radius *
(shape.radius + Math.sqrt(shape.radius * shape.radius + shape.height * shape.height))
);
case "cube":
return 6 * shape.side * shape.side;
case "rectangularPrism":
return (
2 *
(shape.length * shape.width + shape.length * shape.height + shape.width * shape.height)
);
case "pyramid":
return null; // needs the base shape, not just its area
}
}
export function shapePerimeter(shape: Shape): number | null {
switch (shape.kind) {
case "circle":
return 2 * Math.PI * shape.radius;
case "rectangle":
return 2 * (shape.width + shape.height);
case "square":
return 4 * shape.side;
case "regularPolygon":
return shape.sides >= 3 ? shape.sides * shape.sideLength : null;
case "ellipse": {
// Ramanujan's second approximation — accurate to ~1e-9 for any
// realistic eccentricity, where the naive π(a+b) is not close.
const h = ((shape.a - shape.b) * (shape.a - shape.b)) / ((shape.a + shape.b) * (shape.a + shape.b));
return Math.PI * (shape.a + shape.b) * (1 + (3 * h) / (10 + Math.sqrt(4 - 3 * h)));
}
default:
// Trapezoid/triangle/parallelogram need every side, and the constructor
// only took the measurements the area formula uses. Solids have none.
return null;
}
}
export function shapeVolume(shape: Shape): number | null {
switch (shape.kind) {
case "sphere":
return (4 / 3) * Math.PI * shape.radius * shape.radius * shape.radius;
case "cylinder":
return Math.PI * shape.radius * shape.radius * shape.height;
case "cone":
return (Math.PI * shape.radius * shape.radius * shape.height) / 3;
case "cube":
return shape.side * shape.side * shape.side;
case "rectangularPrism":
return shape.length * shape.width * shape.height;
case "pyramid":
return (shape.baseArea * shape.height) / 3;
default:
return null;
}
}
/** Everything known, for the readback. */
export function shapeSummary(shape: Shape): [string, number][] {
const parts: [string, number][] = [];
const area = shapeArea(shape);
if (area !== null) parts.push([shapeIsSolid(shape) ? "Surface area" : "Area", area]);
const perimeter = shapePerimeter(shape);
if (perimeter !== null) parts.push(["Perimeter", perimeter]);
const volume = shapeVolume(shape);
if (volume !== null) parts.push(["Volume", volume]);
return parts;
}
// ---------------------------------------------------------------------------
// Coordinate geometry
/** Pythagoras, either direction: give any two, get the third. */
export function pythagoras(
a: number | null,
b: number | null,
hypotenuse: number | null,
): number | null {
if (a !== null && b !== null && hypotenuse === null) return Math.sqrt(a * a + b * b);
if (a !== null && b === null && hypotenuse !== null) {
return hypotenuse > a ? Math.sqrt(hypotenuse * hypotenuse - a * a) : null;
}
if (a === null && b !== null && hypotenuse !== null) {
return hypotenuse > b ? Math.sqrt(hypotenuse * hypotenuse - b * b) : null;
}
return null;
}
+219
View File
@@ -0,0 +1,219 @@
// Tokeniser for typed and normalised spoken maths expressions.
import { type MathNumber, fromDecimal, integer, real } from "./math-number";
import { type MathRelation } from "./expression";
import { isLetter } from "../../../util/unicode-class";
export type MathToken =
| { type: "number"; value: MathNumber }
| { type: "identifier"; name: string }
| { type: "plus" }
| { type: "minus" }
| { type: "star" }
| { type: "slash" }
| { type: "percent" }
| { type: "caret" }
| { type: "bang" }
| { type: "leftParen" }
| { type: "rightParen" }
| { type: "pipe" }
| { type: "comma" }
| { type: "relation"; relation: MathRelation };
export type MathParseErrorKind =
| { kind: "unexpectedCharacter"; character: string }
| { kind: "unexpectedToken"; token: string }
| { kind: "unexpectedEnd" }
| { kind: "unbalancedParenthesis" }
| { kind: "unknownFunction"; name: string }
| { kind: "emptyInput" }
| { kind: "tooComplex" };
/**
* A parse failure with the user's own words attached, so an error can quote
* back what was actually said rather than leaving the wearer guessing which
* word broke.
*/
export class MathParseError extends Error {
readonly errorKind: MathParseErrorKind;
/** What the user actually said. */
readonly context: string;
constructor(errorKind: MathParseErrorKind, context = "") {
super(describeParseError(errorKind, context));
this.name = "MathParseError";
this.errorKind = errorKind;
this.context = context;
}
}
/** Written for someone reading a lens mid-task, not for a log. */
export function describeParseError(kind: MathParseErrorKind, context: string): string {
const quoted = context ? `“${context}”` : "that expression";
switch (kind.kind) {
case "unexpectedCharacter":
return `Could not read "${kind.character}" in ${quoted}`;
case "unexpectedToken":
return `Did not expect "${kind.token}" in ${quoted}`;
case "unexpectedEnd":
return `${quoted} is incomplete`;
case "unbalancedParenthesis":
return `Brackets do not balance in ${quoted}`;
case "unknownFunction":
return `"${kind.name}" is not a function this build knows`;
case "emptyInput":
return "Nothing to solve";
case "tooComplex":
return "That expression is too deeply nested";
}
}
function isDigit(character: string): boolean {
return character >= "0" && character <= "9";
}
export function tokenize(input: string): MathToken[] {
const tokens: MathToken[] = [];
const characters = Array.from(input);
let index = 0;
while (index < characters.length) {
const character = characters[index]!;
if (/\s/.test(character)) {
index += 1;
continue;
}
if (isDigit(character) || (character === "." && index + 1 < characters.length && isDigit(characters[index + 1]!))) {
let literal = "";
let sawDot = false;
while (index < characters.length) {
const next = characters[index]!;
if (isDigit(next)) {
literal += next;
} else if (next === "." && !sawDot) {
sawDot = true;
literal += next;
} else if (next === "," && index + 1 < characters.length && isDigit(characters[index + 1]!)) {
// Thousands separator inside a literal: 1,000
index += 1;
continue;
} else {
break;
}
index += 1;
}
// Decimals become exact rationals so later arithmetic stays exact:
// 0.25 behaves as 1/4, not as a float.
if (sawDot) {
tokens.push({ type: "number", value: fromDecimal(Number(literal) || 0) });
} else {
const parsed = Number(literal);
// Exact for every safe literal; anything larger degrades to a real
// instead of losing digits silently.
tokens.push({
type: "number",
value: Number.isSafeInteger(parsed) ? integer(parsed) : real(parsed),
});
}
continue;
}
if (isLetter(character) || character === "_") {
let name = "";
while (
index < characters.length &&
(isLetter(characters[index]!) || isDigit(characters[index]!) || characters[index] === "_")
) {
name += characters[index];
index += 1;
}
tokens.push({ type: "identifier", name });
continue;
}
switch (character) {
case "+":
tokens.push({ type: "plus" });
break;
case "-":
case "−": // ASCII and Unicode minus
tokens.push({ type: "minus" });
break;
case "*":
case "×":
case "⋅":
tokens.push({ type: "star" });
break;
case "/":
case "÷":
tokens.push({ type: "slash" });
break;
case "%":
tokens.push({ type: "percent" });
break;
case "^":
tokens.push({ type: "caret" });
break;
case "!":
tokens.push({ type: "bang" });
break;
case "(":
case "[":
case "{":
tokens.push({ type: "leftParen" });
break;
case ")":
case "]":
case "}":
tokens.push({ type: "rightParen" });
break;
case "|":
tokens.push({ type: "pipe" });
break;
case ",":
tokens.push({ type: "comma" });
break;
case "=":
// "==" from a typed source is still equality.
if (index + 1 < characters.length && characters[index + 1] === "=") index += 1;
tokens.push({ type: "relation", relation: "=" });
break;
case "<":
if (index + 1 < characters.length && characters[index + 1] === "=") {
tokens.push({ type: "relation", relation: "<=" });
index += 1;
} else {
tokens.push({ type: "relation", relation: "<" });
}
break;
case ">":
if (index + 1 < characters.length && characters[index + 1] === "=") {
tokens.push({ type: "relation", relation: ">=" });
index += 1;
} else {
tokens.push({ type: "relation", relation: ">" });
}
break;
case "≤":
tokens.push({ type: "relation", relation: "<=" });
break;
case "≥":
tokens.push({ type: "relation", relation: ">=" });
break;
case "√":
tokens.push({ type: "identifier", name: "sqrt" });
break;
case "π":
tokens.push({ type: "identifier", name: "pi" });
break;
default:
throw new MathParseError({ kind: "unexpectedCharacter", character }, input);
}
index += 1;
}
if (tokens.length === 0) throw new MathParseError({ kind: "emptyInput" }, input);
return tokens;
}
Loaded 100 of 315 files, more files were not shown because too many files have changed in this diff. Show more