Merge branch 'main' of https://github.com/jimrandomh/faceclaw
No files matched your search
@@ -1,4 +1,6 @@
|
||||
--ignore-file=match:.mjs
|
||||
--ignore-file=match:frame-timings*
|
||||
--ignore-dir=tempPlugin
|
||||
--ignore-dir=build
|
||||
--ignore-dir=platforms/android/build-tools
|
||||
--ignore-dir=old
|
||||
--ignore-dir=platforms
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Deploy website
|
||||
|
||||
# Publishes website/ to GitHub Pages. Requires the repository's
|
||||
# Settings > Pages > Source to be set to "GitHub Actions".
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'website/**'
|
||||
- 'PRIVACY'
|
||||
- 'scripts/sync-site.mjs'
|
||||
- '.github/workflows/pages.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
group: pages
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check generated blocks are up to date
|
||||
run: node scripts/sync-site.mjs --check
|
||||
- uses: actions/configure-pages@v5
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: website
|
||||
- id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
@@ -3,6 +3,9 @@ hooks/
|
||||
node_modules/
|
||||
platforms/
|
||||
|
||||
# Release APKs from scripts/release.sh
|
||||
dist/
|
||||
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
@@ -10,6 +13,9 @@ npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# npm test compile output (tests/tsconfig.json outDir)
|
||||
.test-build/
|
||||
|
||||
# General
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
@@ -22,6 +28,7 @@ typings/
|
||||
.gradle-cursor
|
||||
App_Resources/Android/src/main/jniLibs/**/*.so
|
||||
App_Resources/Android/src/main/assets/faceclaw-voice-asr/
|
||||
old
|
||||
|
||||
# Visual Studio Code
|
||||
.vscode/*
|
||||
@@ -32,3 +39,7 @@ App_Resources/Android/src/main/assets/faceclaw-voice-asr/
|
||||
|
||||
# pull_config.sh output: the phone's live settings, including API keys
|
||||
faceclaw_settings.xml
|
||||
|
||||
# Developer configs
|
||||
build_paths.sh
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
Acknowledgements
|
||||
================
|
||||
|
||||
Faceclaw is an open source project, at https://github.com/jimrandomh/faceclaw.
|
||||
Contributors include:
|
||||
James Babcock <jimrandomh@gmail.com>
|
||||
Kalani Helekunihi <i@am.guru>
|
||||
|
||||
|
||||
Many thanks to g2-kit-unofficial: https://github.com/Commute773/g2-kit-unofficial/,
|
||||
evenRealities-openCFW https://github.com/kalanihelekunihi/evenRealities-openCFW/,
|
||||
and others who contributed to documenting the G2's bluetooth protocol and firmware.
|
||||
|
||||
Voice wake-word detection uses sherpa-onnx: https://github.com/k2-fsa/sherpa-onnx/
|
||||
and the sherpa-onnx KWS Zipformer GigaSpeech model published by the k2-fsa/sherpa
|
||||
community.
|
||||
|
||||
G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3
|
||||
|
||||
The Terminus font is by Dimitar Zhekov's Terminus: https://framagit.org/ohnonot/terv-terc
|
||||
@@ -15,8 +20,8 @@ The TerminusV proportional font is by ohnonot, derived from Terminus:
|
||||
https://framagit.org/ohnonot/terv-terc. It is also distributed under the SIL
|
||||
Open Font License; see app/fonts/terminusv/LICENSE.
|
||||
|
||||
The CJK font used for EvenHub compatibility is Source Han Sans SC Light by
|
||||
Adobe: https://github.com/adobe-fonts/source-han-sans. It is distributed under
|
||||
the SIL Open Font License, Version 1.1; see app/fonts/source-han-sans/LICENSE.txt.
|
||||
Faceclaw includes the G2's serialized 20 px LVGL build of the font so phone-side
|
||||
rendering matches the glasses.
|
||||
The CJK font is Source Han Sans SC Light by Adobe:
|
||||
https://github.com/adobe-fonts/source-han-sans. It is distributed under the SIL
|
||||
Open Font License, Version 1.1; see app/fonts/source-han-sans/LICENSE.txt.
|
||||
Faceclaw includes the G2's serialized 20 px LVGL build of the font so
|
||||
phone-side rendering matches the glasses.
|
||||
@@ -12,14 +12,19 @@ dependencies {
|
||||
// code" entry. Scanner UI and camera live in Play Services, so Faceclaw
|
||||
// needs no camera permission and no third-party scanner app.
|
||||
implementation 'com.google.android.gms:play-services-code-scanner:16.1.0'
|
||||
// Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for
|
||||
// the Wear OS watch remote; see FaceclawWearBridge and wear/.
|
||||
implementation 'com.google.android.gms:play-services-wearable:19.0.0'
|
||||
// On-device language identification and translation for the Microphones
|
||||
// app's captions (translation models download on demand, per language).
|
||||
implementation 'com.google.mlkit:language-id:17.0.6'
|
||||
implementation 'com.google.mlkit:translate:17.0.3'
|
||||
}
|
||||
|
||||
def faceclawSherpaOnnxVersion = "1.13.0"
|
||||
def faceclawLibLc3Version = "1.1.3"
|
||||
def faceclawAsrModelName = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27"
|
||||
def faceclawAbi = "arm64-v8a"
|
||||
def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi")
|
||||
def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr")
|
||||
def faceclawNativeBuildDir = file("$buildDir/faceclaw-native")
|
||||
|
||||
def faceclawDownload = { String url, File destination ->
|
||||
@@ -67,15 +72,11 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
outputs.files(
|
||||
file("$faceclawJniLibsDir/libonnxruntime.so"),
|
||||
file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"),
|
||||
file("$faceclawJniLibsDir/libfaceclaw_lc3.so"),
|
||||
file("$faceclawAssetsDir/$faceclawAsrModelName/encoder_model.ort"),
|
||||
file("$faceclawAssetsDir/$faceclawAsrModelName/decoder_model_merged.ort"),
|
||||
file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt")
|
||||
file("$faceclawJniLibsDir/libfaceclaw_lc3.so")
|
||||
)
|
||||
|
||||
doLast {
|
||||
faceclawJniLibsDir.mkdirs()
|
||||
faceclawAssetsDir.mkdirs()
|
||||
|
||||
def downloadsDir = file("$faceclawNativeBuildDir/downloads")
|
||||
def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2")
|
||||
@@ -92,20 +93,6 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
into faceclawJniLibsDir
|
||||
}
|
||||
|
||||
def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2")
|
||||
faceclawDownload(
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2",
|
||||
asrArchive
|
||||
)
|
||||
copy {
|
||||
from tarTree(resources.bzip2(asrArchive))
|
||||
include "$faceclawAsrModelName/encoder_model.ort"
|
||||
include "$faceclawAsrModelName/decoder_model_merged.ort"
|
||||
include "$faceclawAsrModelName/tokens.txt"
|
||||
includeEmptyDirs = false
|
||||
into faceclawAssetsDir
|
||||
}
|
||||
|
||||
def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz")
|
||||
faceclawDownload(
|
||||
"https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz",
|
||||
@@ -252,6 +239,22 @@ tasks.register("prepareFaceclawLlama") {
|
||||
}
|
||||
}
|
||||
|
||||
// versionName tracks FACECLAW_VERSION in app/version.ts, the single source of
|
||||
// truth for the Faceclaw version. The gradle root project is platforms/android,
|
||||
// so the repo root is two levels up from rootDir.
|
||||
def faceclawVersionName = "0.0"
|
||||
def faceclawVersionFile = file("$rootDir/../../app/version.ts")
|
||||
if (faceclawVersionFile.exists()) {
|
||||
def faceclawVersionMatch = (faceclawVersionFile.text =~ /FACECLAW_VERSION = "([^"]+)"/)
|
||||
if (faceclawVersionMatch.find()) {
|
||||
faceclawVersionName = faceclawVersionMatch.group(1)
|
||||
}
|
||||
}
|
||||
def faceclawVersionCode = {
|
||||
def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger()
|
||||
return parts[0] * 10000 + parts[1] * 100 + parts[2]
|
||||
}()
|
||||
|
||||
android {
|
||||
compileSdkVersion 35
|
||||
buildToolsVersion "35"
|
||||
@@ -265,12 +268,17 @@ android {
|
||||
minSdkVersion 24
|
||||
targetSdkVersion 35
|
||||
ndk {
|
||||
// The NativeScript platform build.gradle has already added all four
|
||||
// ABIs by the time app.gradle is applied, and abiFilters is additive,
|
||||
// so the set must be cleared for this filter to take effect. Only
|
||||
// arm64 devices are supported (all our own JNI libs are arm64-only).
|
||||
abiFilters.clear()
|
||||
abiFilters "arm64-v8a"
|
||||
}
|
||||
|
||||
// Version Information
|
||||
versionCode 1
|
||||
versionName "1.0.0"
|
||||
// Version Information (derived from app/version.ts above)
|
||||
versionCode faceclawVersionCode
|
||||
versionName faceclawVersionName
|
||||
|
||||
generatedDensities = []
|
||||
}
|
||||
|
||||
@@ -77,6 +77,7 @@
|
||||
android:name="com.tns.NativeScriptActivity"
|
||||
android:label="@string/title_activity_kimera"
|
||||
android:configChanges="keyboard|keyboardHidden|orientation|screenSize|smallestScreenSize|screenLayout|locale|uiMode"
|
||||
android:windowSoftInputMode="adjustResize"
|
||||
android:theme="@style/LaunchScreenTheme"
|
||||
android:hardwareAccelerated="true"
|
||||
android:launchMode="singleTask"
|
||||
@@ -110,6 +111,22 @@
|
||||
<action android:name="android.service.notification.NotificationListenerService" />
|
||||
</intent-filter>
|
||||
</service>
|
||||
<!-- Wear OS: Play services routes watch-app messages (paths under
|
||||
/faceclaw) and watch reachability changes here, starting the app
|
||||
process if needed. See FaceclawWearBridge. -->
|
||||
<service
|
||||
android:name="com.faceclaw.app.FaceclawWearListenerService"
|
||||
android:enabled="true"
|
||||
android:exported="true">
|
||||
<intent-filter>
|
||||
<action android:name="com.google.android.gms.wearable.MESSAGE_RECEIVED" />
|
||||
<data android:scheme="wear" android:host="*" android:pathPrefix="/faceclaw" />
|
||||
</intent-filter>
|
||||
<intent-filter>
|
||||
<action android:name="com.google.android.gms.wearable.CAPABILITY_CHANGED" />
|
||||
<data android:scheme="wear" android:host="*" android:pathPrefix="/" />
|
||||
</intent-filter>
|
||||
</service>
|
||||
<receiver
|
||||
android:name="com.faceclaw.app.FaceclawTimerReceiver"
|
||||
android:enabled="true"
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
---
|
||||
frameworks:
|
||||
- 其他
|
||||
license: Apache License 2.0
|
||||
tasks:
|
||||
- keyword-spotting
|
||||
---
|
||||
|
||||
sherpa 中自定义唤醒词模型,训练数据为 gigaspeech XL (10000 小时)数据,模型大小约为 3.3 M, 建模单元为 BPE 。 使用 icefall 训练,已转换为 onnx 格式,此仓库主要是给 sherpa-onnx 这个 inference 引擎使用。
|
||||
|
||||
模型结构为 zipformer 模型, 本质是一个非常小的语音识别模型,为了实现唤醒词的功能,我们在解码端做了一些修改和约束。 支持自定义唤醒词,数量不限,效果需要单个调整参数。
|
||||
|
||||
#### Clone with HTTP
|
||||
```bash
|
||||
git clone https://www.modelscope.cn/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.git
|
||||
```
|
||||
@@ -1,10 +0,0 @@
|
||||
▁HE LL O ▁WORLD
|
||||
▁HI ▁GO O G LE
|
||||
▁HE Y ▁S I RI
|
||||
▁A LE X A
|
||||
▁LOVE ▁AND ▁P E A CE
|
||||
▁PLAY ▁MU S IC
|
||||
▁GO ▁HOME
|
||||
▁HA PP Y ▁NEW ▁YEAR
|
||||
▁ME R RY ▁ CH R IST MA S
|
||||
▁S C RE EN ▁ON :1.5 #0.35
|
||||
@@ -1,9 +0,0 @@
|
||||
HELLO WORLD
|
||||
HI GOOGLE
|
||||
HEY SIRI
|
||||
ALEXA
|
||||
LOVE AND PEACE
|
||||
PLAY MUSIC
|
||||
GO HOME
|
||||
HAPPY NEW YEAR
|
||||
MERRY CHRISTMAS
|
||||
@@ -1 +0,0 @@
|
||||
▁S C RE EN ▁ON :1.5 #0.35
|
||||
@@ -1,500 +0,0 @@
|
||||
<blk> 0
|
||||
<sos/eos> 1
|
||||
<unk> 2
|
||||
S 3
|
||||
T 4
|
||||
▁THE 5
|
||||
▁A 6
|
||||
▁TO 7
|
||||
▁AND 8
|
||||
N 9
|
||||
E 10
|
||||
D 11
|
||||
ING 12
|
||||
' 13
|
||||
▁OF 14
|
||||
▁I 15
|
||||
▁IN 16
|
||||
Y 17
|
||||
ED 18
|
||||
M 19
|
||||
▁ 20
|
||||
▁THAT 21
|
||||
O 22
|
||||
C 23
|
||||
R 24
|
||||
A 25
|
||||
P 26
|
||||
▁YOU 27
|
||||
U 28
|
||||
RE 29
|
||||
▁IT 30
|
||||
▁S 31
|
||||
G 32
|
||||
L 33
|
||||
B 34
|
||||
▁IS 35
|
||||
I 36
|
||||
AL 37
|
||||
ER 38
|
||||
▁RE 39
|
||||
▁C 40
|
||||
▁WE 41
|
||||
▁SO 42
|
||||
▁BE 43
|
||||
OR 44
|
||||
AR 45
|
||||
IN 46
|
||||
▁FOR 47
|
||||
F 48
|
||||
▁HE 49
|
||||
LE 50
|
||||
LY 51
|
||||
▁E 52
|
||||
▁WAS 53
|
||||
K 54
|
||||
▁F 55
|
||||
W 56
|
||||
VE 57
|
||||
RI 58
|
||||
CH 59
|
||||
LL 60
|
||||
▁THIS 61
|
||||
▁T 62
|
||||
EN 63
|
||||
▁ON 64
|
||||
IT 65
|
||||
CE 66
|
||||
▁WITH 67
|
||||
▁B 68
|
||||
▁W 69
|
||||
▁ME 70
|
||||
▁DO 71
|
||||
▁DE 72
|
||||
▁HAVE 73
|
||||
▁BUT 74
|
||||
ENT 75
|
||||
▁P 76
|
||||
UR 77
|
||||
ON 78
|
||||
▁AS 79
|
||||
IC 80
|
||||
TER 81
|
||||
TE 82
|
||||
▁THEY 83
|
||||
ES 84
|
||||
RA 85
|
||||
▁NOT 86
|
||||
TH 87
|
||||
▁ARE 88
|
||||
RO 89
|
||||
IL 90
|
||||
ATION 91
|
||||
AN 92
|
||||
▁ST 93
|
||||
▁WHAT 94
|
||||
SE 95
|
||||
LI 96
|
||||
V 97
|
||||
▁CAN 98
|
||||
▁G 99
|
||||
▁OR 100
|
||||
▁ALL 101
|
||||
NE 102
|
||||
▁AT 103
|
||||
▁CON 104
|
||||
H 105
|
||||
▁LIKE 106
|
||||
▁MO 107
|
||||
▁HIS 108
|
||||
▁AN 109
|
||||
▁MA 110
|
||||
▁THERE 111
|
||||
▁SU 112
|
||||
▁MY 113
|
||||
NG 114
|
||||
- 115
|
||||
▁ABOUT 116
|
||||
▁SE 117
|
||||
LA 118
|
||||
▁ONE 119
|
||||
LO 120
|
||||
▁SHE 121
|
||||
CK 122
|
||||
▁HAD 123
|
||||
AD 124
|
||||
MENT 125
|
||||
VER 126
|
||||
▁CA 127
|
||||
IR 128
|
||||
▁IF 129
|
||||
▁NO 130
|
||||
EL 131
|
||||
▁O 132
|
||||
▁FROM 133
|
||||
▁PA 134
|
||||
▁YOUR 135
|
||||
▁KNOW 136
|
||||
RY 137
|
||||
ION 138
|
||||
UN 139
|
||||
▁JUST 140
|
||||
▁UP 141
|
||||
▁US 142
|
||||
▁WHO 143
|
||||
▁GO 144
|
||||
ME 145
|
||||
▁EX 146
|
||||
▁OUT 147
|
||||
ND 148
|
||||
CI 149
|
||||
▁PO 150
|
||||
▁HER 151
|
||||
▁FA 152
|
||||
▁PRO 153
|
||||
▁WHEN 154
|
||||
ATE 155
|
||||
AT 156
|
||||
▁BY 157
|
||||
PE 158
|
||||
AS 159
|
||||
▁LA 160
|
||||
GE 161
|
||||
▁CO 162
|
||||
▁PEOPLE 163
|
||||
▁K 164
|
||||
▁SOME 165
|
||||
IGHT 166
|
||||
▁THINK 167
|
||||
VI 168
|
||||
DE 169
|
||||
▁LE 170
|
||||
▁WERE 171
|
||||
▁LI 172
|
||||
OM 173
|
||||
▁WOULD 174
|
||||
▁SEE 175
|
||||
CO 176
|
||||
▁BO 177
|
||||
▁OUR 178
|
||||
▁HOW 179
|
||||
ET 180
|
||||
Z 181
|
||||
UL 182
|
||||
▁HO 183
|
||||
IG 184
|
||||
▁GET 185
|
||||
▁DON 186
|
||||
IS 187
|
||||
URE 188
|
||||
▁WHICH 189
|
||||
▁MI 190
|
||||
IES 191
|
||||
US 192
|
||||
X 193
|
||||
▁MORE 194
|
||||
▁NOW 195
|
||||
▁TIME 196
|
||||
▁UN 197
|
||||
▁FI 198
|
||||
TY 199
|
||||
▁WILL 200
|
||||
RU 201
|
||||
IVE 202
|
||||
▁THEIR 203
|
||||
▁SAY 204
|
||||
ITY 205
|
||||
▁HA 206
|
||||
LU 207
|
||||
▁RO 208
|
||||
HO 209
|
||||
OL 210
|
||||
IST 211
|
||||
AGE 212
|
||||
MA 213
|
||||
KE 214
|
||||
▁RA 215
|
||||
ID 216
|
||||
ERS 217
|
||||
▁LO 218
|
||||
▁SP 219
|
||||
▁DA 220
|
||||
▁SAID 221
|
||||
TION 222
|
||||
THER 223
|
||||
▁EN 224
|
||||
▁BECAUSE 225
|
||||
▁HAS 226
|
||||
EST 227
|
||||
TI 228
|
||||
ACT 229
|
||||
▁BA 230
|
||||
▁VERY 231
|
||||
▁THEM 232
|
||||
▁BEEN 233
|
||||
▁WA 234
|
||||
ABLE 235
|
||||
▁SA 236
|
||||
▁DID 237
|
||||
▁HIM 238
|
||||
▁DIS 239
|
||||
▁WANT 240
|
||||
▁OTHER 241
|
||||
IM 242
|
||||
▁PRE 243
|
||||
▁LOOK 244
|
||||
▁GOING 245
|
||||
▁VI 246
|
||||
▁WORK 247
|
||||
▁DR 248
|
||||
ENCE 249
|
||||
▁DI 250
|
||||
FF 251
|
||||
TA 252
|
||||
▁REALLY 253
|
||||
▁RIGHT 254
|
||||
MO 255
|
||||
ALLY 256
|
||||
▁INTO 257
|
||||
OW 258
|
||||
▁WAY 259
|
||||
▁FE 260
|
||||
▁COULD 261
|
||||
▁THEN 262
|
||||
TED 263
|
||||
▁HERE 264
|
||||
▁THESE 265
|
||||
▁OVER 266
|
||||
▁WELL 267
|
||||
▁NA 268
|
||||
▁ANY 269
|
||||
HE 270
|
||||
▁COMP 271
|
||||
▁COM 272
|
||||
▁TH 273
|
||||
▁EVEN 274
|
||||
▁TWO 275
|
||||
▁JO 276
|
||||
▁MAKE 277
|
||||
▁BACK 278
|
||||
▁WHERE 279
|
||||
SH 280
|
||||
▁QU 281
|
||||
MAN 282
|
||||
▁YEAR 283
|
||||
▁TRA 284
|
||||
▁CHA 285
|
||||
QU 286
|
||||
CA 287
|
||||
▁ALSO 288
|
||||
CTION 289
|
||||
ANCE 290
|
||||
UM 291
|
||||
PER 292
|
||||
DI 293
|
||||
▁HU 294
|
||||
▁MU 295
|
||||
FUL 296
|
||||
▁NEW 297
|
||||
ARD 298
|
||||
J 299
|
||||
ICAL 300
|
||||
ANT 301
|
||||
END 302
|
||||
▁MAR 303
|
||||
▁NE 304
|
||||
IF 305
|
||||
DER 306
|
||||
ONE 307
|
||||
▁THOSE 308
|
||||
PP 309
|
||||
▁MAN 310
|
||||
▁GOOD 311
|
||||
▁TAKE 312
|
||||
SIDE 313
|
||||
PO 314
|
||||
HA 315
|
||||
▁LITTLE 316
|
||||
▁APP 317
|
||||
▁BI 318
|
||||
UGH 319
|
||||
▁COME 320
|
||||
ISH 321
|
||||
TIC 322
|
||||
▁PER 323
|
||||
▁THINGS 324
|
||||
▁MUCH 325
|
||||
▁FIRST 326
|
||||
▁THAN 327
|
||||
IA 328
|
||||
▁AGAIN 329
|
||||
▁STA 330
|
||||
▁PART 331
|
||||
▁GA 332
|
||||
▁MEAN 333
|
||||
▁LET 334
|
||||
▁DOWN 335
|
||||
▁NEED 336
|
||||
▁GU 337
|
||||
LES 338
|
||||
▁RU 339
|
||||
ATED 340
|
||||
▁SHA 341
|
||||
VO 342
|
||||
▁TALK 343
|
||||
▁DAY 344
|
||||
▁CHI 345
|
||||
▁SOMETHING 346
|
||||
▁AFTER 347
|
||||
▁THOUGH 348
|
||||
▁CALL 349
|
||||
▁GRA 350
|
||||
▁EVERY 351
|
||||
▁KIND 352
|
||||
ROW 353
|
||||
AIN 354
|
||||
▁BU 355
|
||||
▁THROUGH 356
|
||||
▁SHOULD 357
|
||||
▁ONLY 358
|
||||
▁TA 359
|
||||
▁MIN 360
|
||||
▁GOT 361
|
||||
IAN 362
|
||||
▁GREAT 363
|
||||
▁RI 364
|
||||
▁DOES 365
|
||||
▁THING 366
|
||||
DAY 367
|
||||
OUS 368
|
||||
▁START 369
|
||||
▁MOST 370
|
||||
▁COL 371
|
||||
PH 372
|
||||
▁LOT 373
|
||||
IZE 374
|
||||
BER 375
|
||||
IOUS 376
|
||||
▁LIFE 377
|
||||
▁MANY 378
|
||||
▁FEEL 379
|
||||
ITION 380
|
||||
▁VA 381
|
||||
NESS 382
|
||||
▁BEFORE 383
|
||||
▁THREE 384
|
||||
▁WORLD 385
|
||||
▁GIVE 386
|
||||
▁WHY 387
|
||||
▁PI 388
|
||||
▁LONG 389
|
||||
PORT 390
|
||||
▁SHOW 391
|
||||
▁ACTUALLY 392
|
||||
▁NEVER 393
|
||||
IV 394
|
||||
FORM 395
|
||||
▁THANK 396
|
||||
▁HI 397
|
||||
▁PUT 398
|
||||
▁LOVE 399
|
||||
▁QUESTION 400
|
||||
▁HELP 401
|
||||
▁HAPPEN 402
|
||||
▁MR 403
|
||||
▁OWN 404
|
||||
WARD 405
|
||||
▁MADE 406
|
||||
▁HAND 407
|
||||
▁STATE 408
|
||||
IBLE 409
|
||||
▁TELL 410
|
||||
▁PH 411
|
||||
▁CAME 412
|
||||
▁TEA 413
|
||||
▁STILL 414
|
||||
▁DIFFERENT 415
|
||||
▁PLACE 416
|
||||
▁MIGHT 417
|
||||
▁FIND 418
|
||||
▁DU 419
|
||||
▁IMP 420
|
||||
▁OLD 421
|
||||
▁PERSON 422
|
||||
▁COUNT 423
|
||||
▁JA 424
|
||||
▁POINT 425
|
||||
▁HIGH 426
|
||||
SHIP 427
|
||||
▁LAST 428
|
||||
▁SAME 429
|
||||
▁FOUR 430
|
||||
▁AROUND 431
|
||||
▁OH 432
|
||||
▁CHANGE 433
|
||||
▁UNDER 434
|
||||
▁HOME 435
|
||||
▁ANOTHER 436
|
||||
▁FOUND 437
|
||||
ATIVE 438
|
||||
▁TURN 439
|
||||
▁YES 440
|
||||
LAND 441
|
||||
▁ALWAYS 442
|
||||
▁YEAH 443
|
||||
▁SCHOOL 444
|
||||
▁JE 445
|
||||
▁NEXT 446
|
||||
▁WORD 447
|
||||
▁JU 448
|
||||
LINE 449
|
||||
▁SIX 450
|
||||
▁LEARN 451
|
||||
▁BOOK 452
|
||||
▁PLAY 453
|
||||
▁AWAY 454
|
||||
▁HOUSE 455
|
||||
▁PLAN 456
|
||||
▁WHILE 457
|
||||
▁COMMUNI 458
|
||||
▁FRIEND 459
|
||||
▁AMERICA 460
|
||||
▁INTEREST 461
|
||||
▁KEEP 462
|
||||
▁BETTER 463
|
||||
▁IDEA 464
|
||||
▁HARD 465
|
||||
▁TWENTY 466
|
||||
▁THOUSAND 467
|
||||
▁HUNDRED 468
|
||||
▁WEEK 469
|
||||
▁NUMBER 470
|
||||
▁COURSE 471
|
||||
▁OKAY 472
|
||||
PRESS 473
|
||||
▁SYSTEM 474
|
||||
▁NINE 475
|
||||
▁UNDERSTAND 476
|
||||
▁WITHOUT 477
|
||||
▁BELIEVE 478
|
||||
▁POWER 479
|
||||
▁MAYBE 480
|
||||
▁BUILD 481
|
||||
▁MONEY 482
|
||||
▁BETWEEN 483
|
||||
▁GONNA 484
|
||||
▁IMPORTANT 485
|
||||
▁ANYTHING 486
|
||||
▁LEFT 487
|
||||
▁HAVING 488
|
||||
▁WHITE 489
|
||||
▁PROBLEM 490
|
||||
▁SECOND 491
|
||||
▁SPEAK 492
|
||||
▁TOGETHER 493
|
||||
▁BLACK 494
|
||||
▁PUBLIC 495
|
||||
▁HEALTH 496
|
||||
▁NOTHING 497
|
||||
▁EVERYTHING 498
|
||||
Q 499
|
||||
@@ -0,0 +1,242 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaFormat;
|
||||
import android.media.MediaMuxer;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
/**
|
||||
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
|
||||
* container, cutting speech recordings to roughly a tenth of their WAV size.
|
||||
* Runs on its own thread; the source WAV is deleted on success when requested.
|
||||
*/
|
||||
public class FaceclawAudioTranscoder {
|
||||
private static final String TAG = "FaceclawTranscode";
|
||||
private static final long CODEC_TIMEOUT_US = 10_000;
|
||||
|
||||
public interface Listener {
|
||||
void onDone(String outputPath, long outputBytes);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
|
||||
private final String wavPath;
|
||||
private final String outputPath;
|
||||
private final int bitrateBps;
|
||||
private final boolean deleteSourceOnSuccess;
|
||||
private final Listener listener;
|
||||
|
||||
public FaceclawAudioTranscoder(
|
||||
String wavPath,
|
||||
String outputPath,
|
||||
int bitrateBps,
|
||||
boolean deleteSourceOnSuccess,
|
||||
Listener listener) {
|
||||
this.wavPath = wavPath;
|
||||
this.outputPath = outputPath;
|
||||
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
|
||||
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
Thread thread = new Thread(this::run, "FaceclawTranscode");
|
||||
thread.setPriority(Thread.MIN_PRIORITY);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
private void run() {
|
||||
try {
|
||||
transcode();
|
||||
long size = new File(outputPath).length();
|
||||
if (deleteSourceOnSuccess) {
|
||||
new File(wavPath).delete();
|
||||
}
|
||||
if (listener != null) {
|
||||
listener.onDone(outputPath, size);
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "transcode failed for " + wavPath, t);
|
||||
new File(outputPath).delete();
|
||||
if (listener != null) {
|
||||
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void transcode() throws IOException {
|
||||
WavInfo wav = readWavHeader(wavPath);
|
||||
MediaFormat format = MediaFormat.createAudioFormat(
|
||||
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
|
||||
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
|
||||
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
|
||||
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
|
||||
|
||||
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
|
||||
MediaMuxer muxer = null;
|
||||
FileInputStream input = null;
|
||||
try {
|
||||
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
|
||||
codec.start();
|
||||
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
|
||||
input = new FileInputStream(wavPath);
|
||||
long skipped = 0;
|
||||
while (skipped < wav.dataOffset) {
|
||||
long step = input.skip(wav.dataOffset - skipped);
|
||||
if (step <= 0) {
|
||||
throw new IOException("could not seek past WAV header");
|
||||
}
|
||||
skipped += step;
|
||||
}
|
||||
|
||||
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||
byte[] chunk = new byte[8192];
|
||||
int track = -1;
|
||||
boolean muxerStarted = false;
|
||||
boolean inputDone = false;
|
||||
boolean outputDone = false;
|
||||
long remaining = wav.dataBytes;
|
||||
long presentationUs = 0;
|
||||
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
|
||||
|
||||
while (!outputDone) {
|
||||
if (!inputDone) {
|
||||
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
|
||||
if (inIndex >= 0) {
|
||||
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
|
||||
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
|
||||
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
|
||||
if (read <= 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
|
||||
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
|
||||
inputDone = true;
|
||||
} else {
|
||||
inBuf.clear();
|
||||
inBuf.put(chunk, 0, read);
|
||||
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
|
||||
remaining -= read;
|
||||
presentationUs += read * 1_000_000L / bytesPerSecond;
|
||||
}
|
||||
}
|
||||
}
|
||||
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
|
||||
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
track = muxer.addTrack(codec.getOutputFormat());
|
||||
muxer.start();
|
||||
muxerStarted = true;
|
||||
} else if (outIndex >= 0) {
|
||||
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
|
||||
if (!muxerStarted) {
|
||||
throw new IOException("encoder produced data before format");
|
||||
}
|
||||
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
|
||||
outBuf.position(info.offset);
|
||||
outBuf.limit(info.offset + info.size);
|
||||
muxer.writeSampleData(track, outBuf, info);
|
||||
}
|
||||
codec.releaseOutputBuffer(outIndex, false);
|
||||
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (input != null) {
|
||||
try {
|
||||
input.close();
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
}
|
||||
try {
|
||||
codec.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
codec.release();
|
||||
if (muxer != null) {
|
||||
try {
|
||||
muxer.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
muxer.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static final class WavInfo {
|
||||
int sampleRate;
|
||||
int channels;
|
||||
long dataOffset;
|
||||
long dataBytes;
|
||||
}
|
||||
|
||||
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
|
||||
private static WavInfo readWavHeader(String path) throws IOException {
|
||||
try (FileInputStream in = new FileInputStream(path)) {
|
||||
byte[] head = new byte[12];
|
||||
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
|
||||
throw new IOException("not a WAV file");
|
||||
}
|
||||
WavInfo info = new WavInfo();
|
||||
long offset = 12;
|
||||
byte[] chunkHead = new byte[8];
|
||||
while (in.read(chunkHead) == 8) {
|
||||
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
|
||||
int id = b.getInt();
|
||||
long size = b.getInt() & 0xffffffffL;
|
||||
offset += 8;
|
||||
if (id == 0x20746d66) { // "fmt "
|
||||
byte[] fmt = new byte[(int) Math.min(size, 16)];
|
||||
if (in.read(fmt) != fmt.length) {
|
||||
throw new IOException("truncated fmt chunk");
|
||||
}
|
||||
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
|
||||
int audioFormat = f.getShort();
|
||||
info.channels = f.getShort();
|
||||
info.sampleRate = f.getInt();
|
||||
f.getInt();
|
||||
f.getShort();
|
||||
int bits = f.getShort();
|
||||
if (audioFormat != 1 || bits != 16) {
|
||||
throw new IOException("only 16-bit PCM WAV is supported");
|
||||
}
|
||||
long skip = size - fmt.length;
|
||||
while (skip > 0) {
|
||||
long step = in.skip(skip);
|
||||
if (step <= 0) {
|
||||
break;
|
||||
}
|
||||
skip -= step;
|
||||
}
|
||||
offset += size;
|
||||
} else if (id == 0x61746164) { // "data"
|
||||
info.dataOffset = offset;
|
||||
// A zero size means the header was never patched (crash
|
||||
// mid-recording); recover the length from the file size.
|
||||
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
|
||||
break;
|
||||
} else {
|
||||
long skip = size;
|
||||
while (skip > 0) {
|
||||
long step = in.skip(skip);
|
||||
if (step <= 0) {
|
||||
break;
|
||||
}
|
||||
skip -= step;
|
||||
}
|
||||
offset += size;
|
||||
}
|
||||
}
|
||||
if (info.sampleRate == 0 || info.dataOffset == 0) {
|
||||
throw new IOException("missing fmt or data chunk");
|
||||
}
|
||||
return info;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,9 +60,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private final java.util.List<FaceclawCompassListener> compassListeners =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
|
||||
new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
private volatile String lastFirmwareCapabilities = "";
|
||||
private volatile Thread workerThread;
|
||||
private volatile boolean running;
|
||||
private volatile boolean userDisconnectRequested;
|
||||
// Set when a connect attempt failed while an arm's Android bond is gone:
|
||||
// retrying is pointless until the user re-pairs, so the worker loop parks
|
||||
// instead of redialing. Cleared by start() (a fresh explicit connect).
|
||||
private volatile boolean reconnectHalted;
|
||||
|
||||
private String phase = "disconnected";
|
||||
private String status = "Disconnected.";
|
||||
@@ -239,6 +246,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
running = true;
|
||||
userDisconnectRequested = false;
|
||||
reconnectHalted = false;
|
||||
shutdownRequested = false;
|
||||
activeInstance = this;
|
||||
workerThread = new Thread(this, "FaceclawBleCommunicator");
|
||||
@@ -355,6 +363,19 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the glasses mic is enabled right now. The enable lives in the
|
||||
* current EvenHub session, so it dies with a transport drop, the charging
|
||||
* case, or a suspend — silently, from the phone's point of view. Callers
|
||||
* that track a capture across those events must check this rather than
|
||||
* assume their earlier enable still holds.
|
||||
*/
|
||||
public boolean isAudioCaptureActive() {
|
||||
synchronized (lock) {
|
||||
return running && sessionReady && !shutdownRequested && audioCaptureActive;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire/renew or release CFW's volatile wake-takeover lease on both
|
||||
* arms. Delivery (not a protocol ACK) is awaited so a caller can ensure
|
||||
@@ -530,6 +551,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
|
||||
public String getFirmwareCapabilities() {
|
||||
return lastFirmwareCapabilities;
|
||||
}
|
||||
|
||||
public void addMicStatusListener(FaceclawMicStatusListener listener) {
|
||||
if (listener != null) {
|
||||
micStatusListeners.add(listener);
|
||||
}
|
||||
}
|
||||
|
||||
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
|
||||
if (listener != null) {
|
||||
micStatusListeners.remove(listener);
|
||||
}
|
||||
}
|
||||
|
||||
private void emitMicStatus(byte[] body, String address) {
|
||||
if (micStatusListeners.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
|
||||
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
|
||||
byte[] copy = java.util.Arrays.copyOf(body, body.length);
|
||||
mainHandler.post(() -> {
|
||||
for (FaceclawMicStatusListener micListener : micStatusListeners) {
|
||||
try {
|
||||
micListener.onMicStatus(copy, arm);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "mic status listener failed", t);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
|
||||
* field-103 write to both temples, or to a single one. Fire-and-forget:
|
||||
* the firmware answers with a field-104 status notify per temple, which
|
||||
* arrives through addMicStatusListener.
|
||||
*/
|
||||
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
|
||||
if (record == null || record.length < 4) {
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady) {
|
||||
logLine("skip mic control (" + label + "); session not ready");
|
||||
return;
|
||||
}
|
||||
if (rightTemple) {
|
||||
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
|
||||
}
|
||||
if (leftTemple) {
|
||||
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
|
||||
}
|
||||
logLine("queue mic control " + label);
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward render-characteristic audio packets to the listener WITHOUT
|
||||
* sending the stock EvenHub audio-control enable. Used for the CFW
|
||||
* mic_control streaming path, where capture is armed through settings
|
||||
* field 103 and the temples emit 'SM' frames on the same characteristic
|
||||
* that stock mono LC3 uses. Returns false when no session is up.
|
||||
*/
|
||||
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
|
||||
if (listener == null) {
|
||||
throw new IllegalArgumentException("listener is required");
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!running || !sessionReady || shutdownRequested) {
|
||||
logLine("skip G2 audio forwarding; session not ready");
|
||||
return false;
|
||||
}
|
||||
audioPacketListener = listener;
|
||||
audioCaptureActive = true;
|
||||
logLine("G2 audio forwarding enabled");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public void stopG2AudioForwarding() {
|
||||
synchronized (lock) {
|
||||
audioPacketListener = null;
|
||||
audioCaptureActive = false;
|
||||
logLine("G2 audio forwarding disabled");
|
||||
}
|
||||
}
|
||||
|
||||
public void addCompassListener(FaceclawCompassListener listener) {
|
||||
if (listener != null) {
|
||||
compassListeners.add(listener);
|
||||
@@ -564,12 +677,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
* foreground), including worker-app frames the TS side never sees.
|
||||
*/
|
||||
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma) {
|
||||
return getCompositePreviewBitmap(brightenGamma, false);
|
||||
}
|
||||
|
||||
/** As above; `green` renders the preview green-on-black (Settings > Phone display > Preview color). */
|
||||
public android.graphics.Bitmap getCompositePreviewBitmap(double brightenGamma, boolean green) {
|
||||
SurfaceCompositor.Composite composite = compositor.previewComposite();
|
||||
if (composite == null) {
|
||||
return null;
|
||||
}
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma);
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green);
|
||||
}
|
||||
|
||||
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
||||
@@ -1118,6 +1236,10 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
emitPhoneLockStateIfChanged(false);
|
||||
if (!sessionReady) {
|
||||
if (reconnectHalted) {
|
||||
interruptibleSleep.sleep(ConnectionOptions.IDLE_SLEEP_MS);
|
||||
continue;
|
||||
}
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
if (now < reconnectAfterMs) {
|
||||
interruptibleSleep.sleep(Math.min(ConnectionOptions.IDLE_SLEEP_MS, reconnectAfterMs - now));
|
||||
@@ -1216,6 +1338,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
0
|
||||
);
|
||||
}
|
||||
if (!faceclawWakeNotification
|
||||
&& frame.ok
|
||||
&& frame.sid == BleProtocol.SID_UI_SETTING) {
|
||||
// CFW mic status (field 104) rides both standalone pushes and
|
||||
// settings read acks, from each temple on its own link.
|
||||
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
|
||||
if (micStatus != null) {
|
||||
emitMicStatus(micStatus, address);
|
||||
}
|
||||
}
|
||||
if (!faceclawWakeNotification
|
||||
&& decodedWearState < 0
|
||||
&& frame.ok
|
||||
@@ -1365,13 +1497,17 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
audioPacketListener = null;
|
||||
clearAllMessagesLocked("connection lost");
|
||||
displayedFingerprint = "";
|
||||
reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS;
|
||||
if (!reconnectHalted) {
|
||||
reconnectAfterMs = SystemClock.elapsedRealtime() + ConnectionOptions.RECONNECT_DELAY_MS;
|
||||
}
|
||||
}
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
if (connected) {
|
||||
setStateDisplay("connected", "Connected.");
|
||||
} else {
|
||||
} else if (!reconnectHalted) {
|
||||
// While parked on a missing bond, keep the "unpaired" display: this
|
||||
// callback is just the teardown of the arm that did connect.
|
||||
setStateDisplay("connecting", "Connecting to the glasses...");
|
||||
}
|
||||
}
|
||||
@@ -1384,6 +1520,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
if (!sleepDuringConnectSettling(800)) {
|
||||
return;
|
||||
}
|
||||
authenticateArms();
|
||||
sendPrelude();
|
||||
|
||||
synchronized (lock) {
|
||||
@@ -1440,7 +1577,12 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
tryConnectRing("initial");
|
||||
} catch (Throwable t) {
|
||||
logLine("connect failed: " + safeMessage(t));
|
||||
handleTransportFailure("connect failed");
|
||||
String unpairedArm = firstUnpairedArm();
|
||||
if (unpairedArm != null) {
|
||||
handleUnpairedFailure(unpairedArm);
|
||||
} else {
|
||||
handleTransportFailure("connect failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1559,6 +1701,55 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete the sid-0x80 security-auth exchange on both freshly opened arm
|
||||
* connections. Firmware 2.2.9 answers no queries until it completes over an
|
||||
* encrypted link and closes unauthenticated links after ~30 s (see
|
||||
* ../notes/ble-connections-2.2.9.md); on an unbonded phone the exchange is
|
||||
* also what triggers SMP pairing. Deliberately soft: on timeout we log and
|
||||
* continue rather than fail the connect — the custom firmware's response
|
||||
* behavior is not yet hardware-verified, and on stock firmware an
|
||||
* unanswered auth just means the prelude fails exactly as it did before.
|
||||
* A pairing prompt accepted after our window still bonds at the OS level,
|
||||
* so the next reconnect attempt authenticates promptly.
|
||||
*/
|
||||
private void authenticateArms() throws InterruptedException {
|
||||
OutboundMessage right = messageBuilder.securityAuth(false);
|
||||
OutboundMessage left = messageBuilder.securityAuth(true);
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
for (OutboundMessage message : new OutboundMessage[] {right, left}) {
|
||||
message.onAck = () -> {
|
||||
};
|
||||
message.onTimeout = () -> {
|
||||
};
|
||||
message.sentAtMs = now;
|
||||
writeMessage(message);
|
||||
}
|
||||
long deadline = SystemClock.elapsedRealtime() + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS;
|
||||
while (running && !userDisconnectRequested && !inFlightMessages.isEmpty()) {
|
||||
synchronized (lock) {
|
||||
if (!running || userDisconnectRequested || inFlightMessages.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
long remaining = deadline - SystemClock.elapsedRealtime();
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
interruptibleSleep.sleep(Math.min(remaining, 100));
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (!inFlightMessages.isEmpty()) {
|
||||
clearInFlightMessagesLocked("security auth timeout");
|
||||
logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)");
|
||||
return;
|
||||
}
|
||||
}
|
||||
boolean rightOk = BleProtocol.isAuthenticationSuccess(right.ackPayload, right.magic);
|
||||
boolean leftOk = BleProtocol.isAuthenticationSuccess(left.ackPayload, left.magic);
|
||||
logLine("security auth R=" + (rightOk ? "ok" : "unconfirmed") + " L=" + (leftOk ? "ok" : "unconfirmed"));
|
||||
}
|
||||
|
||||
private void sendPrelude() throws InterruptedException {
|
||||
sendPrelude(false);
|
||||
}
|
||||
@@ -2429,6 +2620,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
}
|
||||
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
|
||||
if (firmwareInfo != null) {
|
||||
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
|
||||
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
|
||||
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
|
||||
&& hasCapability(firmwareInfo.capabilities, "texstr14");
|
||||
@@ -2928,6 +3120,49 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
+ " release=" + previous.reason);
|
||||
}
|
||||
|
||||
/** The address of a configured arm Android no longer holds a bond for, or null. */
|
||||
private String firstUnpairedArm() {
|
||||
if (!bleManager.isBonded(rightAddress)) return rightAddress;
|
||||
if (!bleManager.isBonded(leftAddress)) return leftAddress;
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A connect failure while an arm's bond is missing (the pairing was
|
||||
* forgotten in Android settings, or the address was typed in by hand and
|
||||
* never paired) will repeat forever, so instead of scheduling a retry,
|
||||
* park the worker loop and tell the user to re-pair. Only an explicit
|
||||
* connect (which builds a fresh communicator) starts a new attempt.
|
||||
*/
|
||||
private void handleUnpairedFailure(String address) {
|
||||
Log.e(TAG, "Connect failed and " + address + " is not paired; suspending reconnect");
|
||||
synchronized (lock) {
|
||||
reconnectHalted = true;
|
||||
sessionReady = false;
|
||||
fixedLayoutCreated = false;
|
||||
startupProbePending = false;
|
||||
shutdownRequested = false;
|
||||
chargingMode = false;
|
||||
imageRetryAfterMs = 0;
|
||||
displayedFingerprint = "";
|
||||
faceclawWakePendingNonce = -1;
|
||||
lastFaceclawWakeLeaseQueuedAtMs = 0;
|
||||
faceclawWakeControlSentCount = 0;
|
||||
clearAllMessagesLocked("arm not paired: " + address);
|
||||
reconnectAfterMs = Long.MAX_VALUE;
|
||||
bleManager.disconnect(rightAddress);
|
||||
bleManager.disconnect(leftAddress);
|
||||
}
|
||||
if (!userDisconnectRequested) {
|
||||
setStateDisplay(
|
||||
"unpaired",
|
||||
"The glasses (" + address + ") are not paired with this phone."
|
||||
+ " Use \"Pair glasses\" to pair them again, then connect."
|
||||
);
|
||||
}
|
||||
interruptibleSleep.interrupt();
|
||||
}
|
||||
|
||||
private void handleTransportFailure(String reason) {
|
||||
Log.e(TAG, "Transport failure: "+reason);
|
||||
synchronized (lock) {
|
||||
@@ -2963,6 +3198,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
|
||||
ringConnected = false;
|
||||
ringNotificationsReady = false;
|
||||
reconnectAfterMs = 0;
|
||||
reconnectHalted = false;
|
||||
ringReconnectAfterMs = 0;
|
||||
lastAckAtMs = 0;
|
||||
lastIncomingAtMs = 0;
|
||||
|
||||
@@ -64,6 +64,23 @@ public class FaceclawBleManager {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether Android still holds a bond (pairing) for this address. Returns
|
||||
* true when the state cannot be determined, so callers only act on a
|
||||
* definite BOND_NONE.
|
||||
*/
|
||||
public boolean isBonded(String address) {
|
||||
if (address == null || address.trim().isEmpty()) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
BluetoothDevice device = bluetoothAdapter.getRemoteDevice(address);
|
||||
return device == null || device.getBondState() != BluetoothDevice.BOND_NONE;
|
||||
} catch (Throwable t) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean connect(String address, int timeoutMs) {
|
||||
if (address == null || address.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("address is required");
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Continuous captioning over the decoded mic PCM: segments speech into
|
||||
* utterances with an adaptive energy gate, transcribes each utterance with
|
||||
* the on-device Moonshine model, and attaches a speaker voice-print per
|
||||
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
|
||||
* and any beam-direction gating), so this engine has no BLE dependencies.
|
||||
*
|
||||
* Utterance boundaries are measured on the sample clock, not the wall clock:
|
||||
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
|
||||
* audio heard. The TS side converts startMs/endMs to wall-clock times using
|
||||
* the engine start timestamp.
|
||||
*/
|
||||
public class FaceclawCaptionEngine {
|
||||
private static final String TAG = "FaceclawCaptions";
|
||||
private static final int SAMPLE_RATE = 16000;
|
||||
private static final int FEATURE_DIM = 80;
|
||||
private static final int MAX_QUEUE_PACKETS = 200;
|
||||
|
||||
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
|
||||
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
|
||||
private static final double ONSET_FACTOR = 3.0;
|
||||
private static final double RELEASE_FACTOR = 1.8;
|
||||
private static final double MIN_RMS = 220.0;
|
||||
private static final double NOISE_EMA_ALPHA = 0.05;
|
||||
private static final int PRE_ROLL_MS = 400;
|
||||
private static final int DEFAULT_SILENCE_MS = 800;
|
||||
private static final int MIN_UTTERANCE_MS = 350;
|
||||
private static final int MAX_UTTERANCE_MS = 15000;
|
||||
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
|
||||
// segments cut at the quietest window (mirrors FaceclawVoiceController).
|
||||
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
|
||||
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
|
||||
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
|
||||
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
|
||||
private static final float NORMALIZE_MAX_GAIN = 30f;
|
||||
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
|
||||
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
|
||||
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private final Object lock = new Object();
|
||||
private final Object queueLock = new Object();
|
||||
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
|
||||
|
||||
private volatile FaceclawCaptionEngineListener listener;
|
||||
private volatile String asrModelDir;
|
||||
private volatile String speakerModelPath;
|
||||
private volatile int silenceMs = DEFAULT_SILENCE_MS;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
|
||||
private OfflineRecognizer recognizer;
|
||||
private FaceclawSpeakerId speakerId;
|
||||
|
||||
// Segmentation state (worker thread only).
|
||||
private long totalSamples;
|
||||
private double noiseFloor;
|
||||
private boolean inUtterance;
|
||||
private long utteranceStartSample;
|
||||
private long silenceRunSamples;
|
||||
private double utterancePeakRms;
|
||||
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
|
||||
private int utteranceLength;
|
||||
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
|
||||
private int preRollLength;
|
||||
|
||||
public void setListener(FaceclawCaptionEngineListener listener) {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
/** Directory holding the Moonshine model files, or null to disable ASR. */
|
||||
public void setAsrModelDir(String dir) {
|
||||
this.asrModelDir = dir;
|
||||
}
|
||||
|
||||
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
|
||||
public void setSpeakerModelPath(String path) {
|
||||
this.speakerModelPath = path;
|
||||
}
|
||||
|
||||
public void setSilenceMs(int ms) {
|
||||
this.silenceMs = Math.max(200, Math.min(3000, ms));
|
||||
}
|
||||
|
||||
public void start() {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
Thread threadToJoin;
|
||||
synchronized (lock) {
|
||||
if (!started) {
|
||||
return;
|
||||
}
|
||||
started = false;
|
||||
threadToJoin = workerThread;
|
||||
workerThread = null;
|
||||
}
|
||||
synchronized (queueLock) {
|
||||
queueLock.notifyAll();
|
||||
}
|
||||
if (threadToJoin != null) {
|
||||
threadToJoin.interrupt();
|
||||
if (Thread.currentThread() != threadToJoin) {
|
||||
try {
|
||||
threadToJoin.join(2000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
|
||||
public void acceptPcm(byte[] pcm16le) {
|
||||
if (!started || pcm16le == null || pcm16le.length < 2) {
|
||||
return;
|
||||
}
|
||||
synchronized (queueLock) {
|
||||
if (queue.size() >= MAX_QUEUE_PACKETS) {
|
||||
queue.removeFirst();
|
||||
}
|
||||
queue.addLast(pcm16le);
|
||||
queueLock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
try {
|
||||
loadModels();
|
||||
resetSegmentation();
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
byte[] chunk = takeChunk();
|
||||
if (chunk == null) {
|
||||
continue;
|
||||
}
|
||||
processChunk(chunk);
|
||||
}
|
||||
// Flush a trailing utterance so its text isn't lost on stop.
|
||||
if (inUtterance && utteranceLength > 0) {
|
||||
finalizeUtterance();
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.e(TAG, "caption engine failed", t);
|
||||
emitStatus("Captions failed: " + t.getMessage());
|
||||
} finally {
|
||||
releaseModels();
|
||||
}
|
||||
}
|
||||
|
||||
private void loadModels() {
|
||||
String modelDir = asrModelDir;
|
||||
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
|
||||
emitStatus("Loading caption model...");
|
||||
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setModelConfig(OfflineModelConfig.builder()
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
|
||||
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.build());
|
||||
} else {
|
||||
recognizer = null;
|
||||
}
|
||||
String embedModel = speakerModelPath;
|
||||
if (embedModel != null && new File(embedModel).exists()) {
|
||||
speakerId = new FaceclawSpeakerId(embedModel);
|
||||
speakerId.ensureLoaded();
|
||||
} else {
|
||||
speakerId = null;
|
||||
}
|
||||
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
|
||||
}
|
||||
|
||||
private void releaseModels() {
|
||||
if (recognizer != null) {
|
||||
recognizer.release();
|
||||
recognizer = null;
|
||||
}
|
||||
if (speakerId != null) {
|
||||
speakerId.close();
|
||||
speakerId = null;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] takeChunk() {
|
||||
synchronized (queueLock) {
|
||||
while (started && queue.isEmpty()) {
|
||||
try {
|
||||
queueLock.wait(250);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return queue.pollFirst();
|
||||
}
|
||||
}
|
||||
|
||||
private void resetSegmentation() {
|
||||
totalSamples = 0;
|
||||
noiseFloor = 0;
|
||||
inUtterance = false;
|
||||
utteranceLength = 0;
|
||||
preRollLength = 0;
|
||||
silenceRunSamples = 0;
|
||||
utterancePeakRms = 0;
|
||||
}
|
||||
|
||||
private void processChunk(byte[] chunk) {
|
||||
int count = chunk.length / 2;
|
||||
short[] pcm = new short[count];
|
||||
double sumSquares = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
|
||||
pcm[i] = s;
|
||||
sumSquares += (double) s * s;
|
||||
}
|
||||
double rms = Math.sqrt(sumSquares / Math.max(1, count));
|
||||
totalSamples += count;
|
||||
|
||||
if (!inUtterance) {
|
||||
// Track the noise floor only while idle so speech doesn't raise it.
|
||||
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
|
||||
double threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
if (rms >= threshold * ONSET_FACTOR) {
|
||||
inUtterance = true;
|
||||
utteranceLength = 0;
|
||||
silenceRunSamples = 0;
|
||||
utterancePeakRms = rms;
|
||||
appendUtterance(preRoll, preRollLength);
|
||||
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
|
||||
appendUtterance(pcm, count);
|
||||
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
|
||||
} else {
|
||||
appendPreRoll(pcm, count);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
appendUtterance(pcm, count);
|
||||
utterancePeakRms = Math.max(utterancePeakRms, rms);
|
||||
double threshold = Math.max(noiseFloor, MIN_RMS);
|
||||
if (rms < threshold * RELEASE_FACTOR) {
|
||||
silenceRunSamples += count;
|
||||
} else {
|
||||
silenceRunSamples = 0;
|
||||
}
|
||||
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
|
||||
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
|
||||
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
|
||||
finalizeUtterance();
|
||||
inUtterance = false;
|
||||
preRollLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private void appendPreRoll(short[] pcm, int count) {
|
||||
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
|
||||
if (count >= preRoll.length) {
|
||||
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
|
||||
preRollLength = preRoll.length;
|
||||
return;
|
||||
}
|
||||
int keep = Math.min(preRollLength, preRoll.length - count);
|
||||
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
|
||||
System.arraycopy(pcm, 0, preRoll, keep, count);
|
||||
preRollLength = keep + count;
|
||||
}
|
||||
|
||||
private void appendUtterance(short[] pcm, int count) {
|
||||
int room = utterance.length - utteranceLength;
|
||||
int copied = Math.min(room, count);
|
||||
if (copied > 0) {
|
||||
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
|
||||
utteranceLength += copied;
|
||||
}
|
||||
}
|
||||
|
||||
private void finalizeUtterance() {
|
||||
int length = utteranceLength;
|
||||
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
|
||||
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
|
||||
if (endMs - startMs < MIN_UTTERANCE_MS) {
|
||||
return;
|
||||
}
|
||||
String text = recognizeUtterance(length);
|
||||
float[] embedding = embedUtterance(length);
|
||||
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
|
||||
}
|
||||
|
||||
private String recognizeUtterance(int length) {
|
||||
if (recognizer == null || length <= 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder joined = new StringBuilder();
|
||||
int offset = 0;
|
||||
while (offset < length) {
|
||||
int remaining = length - offset;
|
||||
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
|
||||
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
|
||||
segment = findQuietCut(offset, segment);
|
||||
}
|
||||
String part = recognizeRange(offset, segment);
|
||||
if (part.length() > 0) {
|
||||
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
|
||||
joined.append(' ');
|
||||
}
|
||||
joined.append(part);
|
||||
}
|
||||
offset += segment;
|
||||
}
|
||||
return joined.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* End a decode segment at the center of the quietest window near its end
|
||||
* so the cut lands between words (same approach as the PTT controller).
|
||||
*/
|
||||
private int findQuietCut(int offset, int segment) {
|
||||
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
|
||||
int win = CUT_WINDOW_SAMPLES;
|
||||
if (segment - searchStart <= win) {
|
||||
return segment;
|
||||
}
|
||||
double sum = 0;
|
||||
for (int i = searchStart; i < searchStart + win; i++) {
|
||||
double s = utterance[offset + i];
|
||||
sum += s * s;
|
||||
}
|
||||
double best = sum;
|
||||
int bestStart = searchStart;
|
||||
for (int start = searchStart + 1; start + win <= segment; start++) {
|
||||
double dropped = utterance[offset + start - 1];
|
||||
double added = utterance[offset + start + win - 1];
|
||||
sum += added * added - dropped * dropped;
|
||||
if (sum < best) {
|
||||
best = sum;
|
||||
bestStart = start;
|
||||
}
|
||||
}
|
||||
return bestStart + win / 2;
|
||||
}
|
||||
|
||||
private String recognizeRange(int offset, int count) {
|
||||
float[] samples = new float[count];
|
||||
float peak = 0f;
|
||||
for (int i = 0; i < count; i++) {
|
||||
float v = utterance[offset + i] / 32768.0f;
|
||||
samples[i] = v;
|
||||
float a = Math.abs(v);
|
||||
if (a > peak) {
|
||||
peak = a;
|
||||
}
|
||||
}
|
||||
if (peak > 0f) {
|
||||
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
|
||||
if (gain > 1f) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] *= gain;
|
||||
}
|
||||
}
|
||||
}
|
||||
OfflineStream stream = recognizer.createStream();
|
||||
try {
|
||||
stream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
recognizer.decode(stream);
|
||||
OfflineRecognizerResult result = recognizer.getResult(stream);
|
||||
String raw = result == null ? "" : result.getText();
|
||||
return raw == null ? "" : raw.trim();
|
||||
} finally {
|
||||
stream.release();
|
||||
}
|
||||
}
|
||||
|
||||
private float[] embedUtterance(int length) {
|
||||
FaceclawSpeakerId currentSpeakerId = speakerId;
|
||||
if (currentSpeakerId == null || length <= 0) {
|
||||
return null;
|
||||
}
|
||||
int count = Math.min(length, EMBED_MAX_SAMPLES);
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = utterance[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
return currentSpeakerId.embed(le, SAMPLE_RATE);
|
||||
}
|
||||
|
||||
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
|
||||
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
|
||||
}
|
||||
|
||||
private void emitSpeechStart(long startMs) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
|
||||
}
|
||||
|
||||
private void emitStatus(String status) {
|
||||
FaceclawCaptionEngineListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onStatus(status));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawCaptionEngineListener {
|
||||
/**
|
||||
* One finished utterance. text may be empty when no ASR model is loaded
|
||||
* or recognition produced nothing; embedding is the L2-normalized
|
||||
* voice-print or null when the speaker model is unavailable or the
|
||||
* utterance was too short. startMs/endMs are on the engine's sample
|
||||
* clock (milliseconds of audio since start()).
|
||||
*/
|
||||
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
|
||||
|
||||
/** A speech onset was detected; the utterance is now being collected. */
|
||||
void onSpeechStart(long startMs);
|
||||
|
||||
void onStatus(String status);
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.ContentValues;
|
||||
import android.content.Context;
|
||||
import android.database.Cursor;
|
||||
import android.database.sqlite.SQLiteDatabase;
|
||||
import android.database.sqlite.SQLiteOpenHelper;
|
||||
import android.util.Base64;
|
||||
import android.util.Log;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Persistent store for caption sessions, transcript segments, and speaker
|
||||
* voice profiles, backing the Microphones app and its phone-side review UI.
|
||||
* All complex values cross the JS bridge as JSON strings; voice-print
|
||||
* embeddings are float32 LE blobs, exposed as base64.
|
||||
*/
|
||||
public class FaceclawConversationStore extends SQLiteOpenHelper {
|
||||
private static final String TAG = "FaceclawConvStore";
|
||||
private static final String DB_NAME = "faceclaw-conversations.db";
|
||||
private static final int DB_VERSION = 2;
|
||||
|
||||
private static FaceclawConversationStore instance;
|
||||
|
||||
public static synchronized FaceclawConversationStore getInstance(Context context) {
|
||||
if (instance == null) {
|
||||
instance = new FaceclawConversationStore(context.getApplicationContext());
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
private FaceclawConversationStore(Context context) {
|
||||
super(context, DB_NAME, null, DB_VERSION);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCreate(SQLiteDatabase db) {
|
||||
db.execSQL("CREATE TABLE speakers ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "name TEXT NOT NULL,"
|
||||
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
|
||||
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
|
||||
+ "embedding BLOB,"
|
||||
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
|
||||
+ "created_at INTEGER NOT NULL,"
|
||||
+ "last_heard_at INTEGER,"
|
||||
+ "tag TEXT,"
|
||||
+ "last_recap TEXT,"
|
||||
+ "action_items TEXT,"
|
||||
+ "facts TEXT,"
|
||||
+ "insights_updated_at INTEGER,"
|
||||
+ "insights_session_id INTEGER)");
|
||||
db.execSQL("CREATE TABLE sessions ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "started_at INTEGER NOT NULL,"
|
||||
+ "ended_at INTEGER,"
|
||||
+ "title TEXT,"
|
||||
+ "audio_path TEXT,"
|
||||
+ "audio_codec TEXT,"
|
||||
+ "avg_sentiment REAL,"
|
||||
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
|
||||
db.execSQL("CREATE TABLE segments ("
|
||||
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
+ "session_id INTEGER NOT NULL,"
|
||||
+ "speaker_id INTEGER,"
|
||||
+ "started_at INTEGER NOT NULL,"
|
||||
+ "ended_at INTEGER NOT NULL,"
|
||||
+ "audio_offset_ms INTEGER,"
|
||||
+ "text TEXT NOT NULL,"
|
||||
+ "lang TEXT,"
|
||||
+ "translation TEXT,"
|
||||
+ "translation_lang TEXT,"
|
||||
+ "sentiment REAL,"
|
||||
+ "emotion TEXT,"
|
||||
+ "search_meta TEXT,"
|
||||
+ "angle INTEGER,"
|
||||
+ "embedding BLOB)");
|
||||
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
|
||||
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
|
||||
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
|
||||
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
|
||||
if (oldVersion < 2) {
|
||||
// v2: per-speaker conversation insights (recap of the most recent
|
||||
// conversation, its action items, and accumulated inferred facts).
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
|
||||
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
|
||||
}
|
||||
}
|
||||
|
||||
// ---- sessions ----
|
||||
|
||||
public long startSession(long startedAtMs, String title) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("started_at", startedAtMs);
|
||||
values.put("title", title == null || title.isEmpty() ? null : title);
|
||||
return getWritableDatabase().insert("sessions", null, values);
|
||||
}
|
||||
|
||||
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("ended_at", endedAtMs);
|
||||
if (audioPath != null && !audioPath.isEmpty()) {
|
||||
values.put("audio_path", audioPath);
|
||||
values.put("audio_codec", audioCodec);
|
||||
}
|
||||
values.put("avg_sentiment", avgSentiment);
|
||||
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("audio_path", audioPath);
|
||||
values.put("audio_codec", audioCodec);
|
||||
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
public String getSessionAudioPath(long sessionId) {
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
|
||||
if (c.moveToFirst()) {
|
||||
return c.isNull(0) ? "" : c.getString(0);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
|
||||
* speakerId, emotion (matches any segment emotion in the session), query
|
||||
* (LIKE over title and segment text/search metadata), limit.
|
||||
*/
|
||||
public String querySessions(String filterJson) {
|
||||
try {
|
||||
JSONObject filter = filterJson == null || filterJson.isEmpty()
|
||||
? new JSONObject() : new JSONObject(filterJson);
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
|
||||
+ " s.avg_sentiment, s.segment_count FROM sessions s");
|
||||
List<String> args = new ArrayList<>();
|
||||
List<String> where = new ArrayList<>();
|
||||
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
|
||||
if (joinSegments) {
|
||||
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
|
||||
}
|
||||
if (filter.has("sinceMs")) {
|
||||
where.add("s.started_at >= ?");
|
||||
args.add(String.valueOf(filter.getLong("sinceMs")));
|
||||
}
|
||||
if (filter.has("untilMs")) {
|
||||
where.add("s.started_at <= ?");
|
||||
args.add(String.valueOf(filter.getLong("untilMs")));
|
||||
}
|
||||
if (filter.has("speakerId")) {
|
||||
where.add("g.speaker_id = ?");
|
||||
args.add(String.valueOf(filter.getLong("speakerId")));
|
||||
}
|
||||
if (filter.has("emotion")) {
|
||||
where.add("g.emotion = ?");
|
||||
args.add(filter.getString("emotion"));
|
||||
}
|
||||
if (filter.has("query")) {
|
||||
String like = "%" + filter.getString("query") + "%";
|
||||
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
}
|
||||
if (!where.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", where));
|
||||
}
|
||||
sql.append(" ORDER BY s.started_at DESC");
|
||||
int limit = filter.optInt("limit", 200);
|
||||
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
|
||||
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
long id = c.getLong(0);
|
||||
row.put("id", id);
|
||||
row.put("startedAt", c.getLong(1));
|
||||
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
|
||||
row.put("title", c.isNull(3) ? "" : c.getString(3));
|
||||
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
|
||||
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
|
||||
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
|
||||
row.put("segmentCount", c.getLong(7));
|
||||
row.put("speakers", sessionSpeakersJson(id));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySessions failed", e);
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
|
||||
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
|
||||
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("name", c.getString(1));
|
||||
row.put("color", c.getString(2));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---- segments ----
|
||||
|
||||
/**
|
||||
* Insert one transcript segment. json fields: sessionId, speakerId?,
|
||||
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
|
||||
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
|
||||
* embeddingBase64?. Returns the new row id, or -1.
|
||||
*/
|
||||
public long insertSegment(String json) {
|
||||
try {
|
||||
JSONObject s = new JSONObject(json);
|
||||
ContentValues values = new ContentValues();
|
||||
long sessionId = s.getLong("sessionId");
|
||||
values.put("session_id", sessionId);
|
||||
if (s.has("speakerId")) {
|
||||
values.put("speaker_id", s.getLong("speakerId"));
|
||||
}
|
||||
values.put("started_at", s.getLong("startedAt"));
|
||||
values.put("ended_at", s.getLong("endedAt"));
|
||||
if (s.has("audioOffsetMs")) {
|
||||
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
|
||||
}
|
||||
values.put("text", s.getString("text"));
|
||||
putOptString(values, s, "lang", "lang");
|
||||
putOptString(values, s, "translation", "translation");
|
||||
putOptString(values, s, "translationLang", "translation_lang");
|
||||
if (s.has("sentiment")) {
|
||||
values.put("sentiment", s.getDouble("sentiment"));
|
||||
}
|
||||
putOptString(values, s, "emotion", "emotion");
|
||||
putOptString(values, s, "searchMeta", "search_meta");
|
||||
if (s.has("angle")) {
|
||||
values.put("angle", s.getInt("angle"));
|
||||
}
|
||||
if (s.has("embeddingBase64")) {
|
||||
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
|
||||
}
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
long id = db.insert("segments", null, values);
|
||||
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
|
||||
new Object[]{sessionId});
|
||||
return id;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "insertSegment failed", t);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
|
||||
throws JSONException {
|
||||
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
|
||||
values.put(column, s.getString(jsonKey));
|
||||
}
|
||||
}
|
||||
|
||||
/** Update mutable fields of a segment (same keys as insertSegment). */
|
||||
public void updateSegment(long segmentId, String json) {
|
||||
try {
|
||||
JSONObject s = new JSONObject(json);
|
||||
ContentValues values = new ContentValues();
|
||||
if (s.has("speakerId")) {
|
||||
if (s.isNull("speakerId")) {
|
||||
values.putNull("speaker_id");
|
||||
} else {
|
||||
values.put("speaker_id", s.getLong("speakerId"));
|
||||
}
|
||||
}
|
||||
if (s.has("text")) {
|
||||
values.put("text", s.getString("text"));
|
||||
}
|
||||
putOptString(values, s, "lang", "lang");
|
||||
putOptString(values, s, "translation", "translation");
|
||||
putOptString(values, s, "translationLang", "translation_lang");
|
||||
if (s.has("sentiment")) {
|
||||
values.put("sentiment", s.getDouble("sentiment"));
|
||||
}
|
||||
putOptString(values, s, "emotion", "emotion");
|
||||
putOptString(values, s, "searchMeta", "search_meta");
|
||||
if (values.size() > 0) {
|
||||
getWritableDatabase().update("segments", values, "id=?",
|
||||
new String[]{String.valueOf(segmentId)});
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "updateSegment failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
public String querySegments(long sessionId) {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
|
||||
+ " translation, translation_lang, sentiment, emotion, angle"
|
||||
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
out.put(segmentRowJson(c));
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySegments failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
private JSONObject segmentRowJson(Cursor c) throws JSONException {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
|
||||
row.put("startedAt", c.getLong(2));
|
||||
row.put("endedAt", c.getLong(3));
|
||||
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
|
||||
row.put("text", c.getString(5));
|
||||
row.put("lang", c.isNull(6) ? "" : c.getString(6));
|
||||
row.put("translation", c.isNull(7) ? "" : c.getString(7));
|
||||
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
|
||||
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
|
||||
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
|
||||
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
|
||||
return row;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search segments across sessions. filterJson: query?, emotion?,
|
||||
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
|
||||
* the sentiment search metadata.
|
||||
*/
|
||||
public String searchSegments(String filterJson) {
|
||||
try {
|
||||
JSONObject filter = filterJson == null || filterJson.isEmpty()
|
||||
? new JSONObject() : new JSONObject(filterJson);
|
||||
StringBuilder sql = new StringBuilder(
|
||||
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
|
||||
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
|
||||
+ " FROM segments");
|
||||
List<String> args = new ArrayList<>();
|
||||
List<String> where = new ArrayList<>();
|
||||
if (filter.has("query") && !filter.getString("query").isEmpty()) {
|
||||
String like = "%" + filter.getString("query") + "%";
|
||||
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
args.add(like);
|
||||
}
|
||||
if (filter.has("emotion")) {
|
||||
where.add("emotion = ?");
|
||||
args.add(filter.getString("emotion"));
|
||||
}
|
||||
if (filter.has("speakerId")) {
|
||||
where.add("speaker_id = ?");
|
||||
args.add(String.valueOf(filter.getLong("speakerId")));
|
||||
}
|
||||
if (filter.has("sinceMs")) {
|
||||
where.add("started_at >= ?");
|
||||
args.add(String.valueOf(filter.getLong("sinceMs")));
|
||||
}
|
||||
if (filter.has("untilMs")) {
|
||||
where.add("started_at <= ?");
|
||||
args.add(String.valueOf(filter.getLong("untilMs")));
|
||||
}
|
||||
if (!where.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", where));
|
||||
}
|
||||
sql.append(" ORDER BY started_at DESC LIMIT ")
|
||||
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = segmentRowJson(c);
|
||||
row.put("sessionId", c.getLong(12));
|
||||
out.put(row);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "searchSegments failed", e);
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
|
||||
/** Per-segment voice-print embeddings of a session (for re-diarization). */
|
||||
public String querySegmentEmbeddings(long sessionId) {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
|
||||
new String[]{String.valueOf(sessionId)})) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
|
||||
out.put(row);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySegmentEmbeddings failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
// ---- speakers ----
|
||||
|
||||
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("name", name);
|
||||
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
|
||||
values.put("is_wearer", isWearer ? 1 : 0);
|
||||
values.put("created_at", System.currentTimeMillis());
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
|
||||
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
|
||||
values.put("embedding_count", 1);
|
||||
}
|
||||
return getWritableDatabase().insert("speakers", null, values);
|
||||
}
|
||||
|
||||
public String querySpeakers() {
|
||||
JSONArray out = new JSONArray();
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
|
||||
+ " p.last_heard_at, p.tag,"
|
||||
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
|
||||
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
|
||||
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
|
||||
while (c.moveToNext()) {
|
||||
JSONObject row = new JSONObject();
|
||||
row.put("id", c.getLong(0));
|
||||
row.put("name", c.getString(1));
|
||||
row.put("color", c.getString(2));
|
||||
row.put("isWearer", c.getInt(3) != 0);
|
||||
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
|
||||
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
|
||||
row.put("embeddingCount", c.getInt(5));
|
||||
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
|
||||
row.put("tag", c.isNull(7) ? "" : c.getString(7));
|
||||
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
|
||||
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
|
||||
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
|
||||
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
|
||||
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
|
||||
row.put("segmentCount", c.getLong(13));
|
||||
out.put(row);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
Log.w(TAG, "querySpeakers failed", e);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
public void renameSpeaker(long speakerId, String name) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("name", name);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerColor(long speakerId, String color) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("color", color);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerTag(long speakerId, String tag) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("tag", tag);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
/**
|
||||
* Store LLM-derived conversation insights for a speaker: a recap of their
|
||||
* most recent conversation, its open action items (JSON string array), and
|
||||
* the accumulated inferred facts about them (JSON string array).
|
||||
*/
|
||||
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
|
||||
String factsJson, long sessionId) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("last_recap", recap == null ? "" : recap);
|
||||
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
|
||||
values.put("facts", factsJson == null ? "" : factsJson);
|
||||
values.put("insights_updated_at", System.currentTimeMillis());
|
||||
values.put("insights_session_id", sessionId);
|
||||
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
public void setSpeakerWearer(long speakerId, boolean isWearer) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
if (isWearer) {
|
||||
// Exactly one wearer profile at a time.
|
||||
db.execSQL("UPDATE speakers SET is_wearer = 0");
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("is_wearer", isWearer ? 1 : 0);
|
||||
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
/**
|
||||
* Running-mean centroid update for a speaker's voice-print, capped so a
|
||||
* long session cannot drown the enrolled voice.
|
||||
*/
|
||||
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
|
||||
try {
|
||||
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
|
||||
float[] update = blobToFloats(incoming);
|
||||
if (update == null) {
|
||||
return;
|
||||
}
|
||||
float[] current = null;
|
||||
int count = 0;
|
||||
try (Cursor c = getReadableDatabase().rawQuery(
|
||||
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
|
||||
new String[]{String.valueOf(speakerId)})) {
|
||||
if (c.moveToFirst()) {
|
||||
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
|
||||
count = c.getInt(1);
|
||||
}
|
||||
}
|
||||
float[] merged;
|
||||
if (current == null || current.length != update.length) {
|
||||
merged = update;
|
||||
count = 1;
|
||||
} else {
|
||||
int effective = Math.min(count, Math.max(1, maxCount));
|
||||
merged = new float[current.length];
|
||||
for (int i = 0; i < merged.length; i++) {
|
||||
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
|
||||
}
|
||||
l2NormalizeInPlace(merged);
|
||||
count = count + 1;
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("embedding", floatsToBlob(merged));
|
||||
values.put("embedding_count", count);
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
getWritableDatabase().update("speakers", values, "id=?",
|
||||
new String[]{String.valueOf(speakerId)});
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "updateSpeakerEmbedding failed", t);
|
||||
}
|
||||
}
|
||||
|
||||
/** Move every segment from one speaker onto another and drop the source. */
|
||||
public void mergeSpeakers(long fromId, long intoId) {
|
||||
if (fromId == intoId) {
|
||||
return;
|
||||
}
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
db.beginTransaction();
|
||||
try {
|
||||
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
|
||||
new Object[]{intoId, fromId});
|
||||
// Blend the two centroids weighted by their sample counts.
|
||||
float[] from = null;
|
||||
float[] into = null;
|
||||
int fromCount = 0;
|
||||
int intoCount = 0;
|
||||
boolean fromWearer = false;
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
|
||||
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
|
||||
while (c.moveToNext()) {
|
||||
long id = c.getLong(0);
|
||||
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
|
||||
int count = c.getInt(2);
|
||||
if (id == fromId) {
|
||||
from = embedding;
|
||||
fromCount = count;
|
||||
fromWearer = c.getInt(3) != 0;
|
||||
} else {
|
||||
into = embedding;
|
||||
intoCount = count;
|
||||
}
|
||||
}
|
||||
}
|
||||
float[] merged = into;
|
||||
if (from != null && into != null && from.length == into.length) {
|
||||
merged = new float[into.length];
|
||||
int total = Math.max(1, fromCount + intoCount);
|
||||
for (int i = 0; i < merged.length; i++) {
|
||||
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
|
||||
}
|
||||
l2NormalizeInPlace(merged);
|
||||
} else if (into == null) {
|
||||
merged = from;
|
||||
}
|
||||
ContentValues values = new ContentValues();
|
||||
if (merged != null) {
|
||||
values.put("embedding", floatsToBlob(merged));
|
||||
values.put("embedding_count", fromCount + intoCount);
|
||||
}
|
||||
if (fromWearer) {
|
||||
values.put("is_wearer", 1);
|
||||
}
|
||||
values.put("last_heard_at", System.currentTimeMillis());
|
||||
// Keep the target's conversation insights; adopt the source's only
|
||||
// where the target has none (e.g. merging a rich profile into a
|
||||
// freshly auto-created one).
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
|
||||
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
|
||||
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
|
||||
boolean targetHasInsights = false;
|
||||
while (c.moveToNext()) {
|
||||
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
|
||||
if (c.getPosition() == 0) {
|
||||
targetHasInsights = hasInsights;
|
||||
} else if (!targetHasInsights && hasInsights) {
|
||||
values.put("last_recap", c.getString(0));
|
||||
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
|
||||
values.put("facts", c.isNull(2) ? "" : c.getString(2));
|
||||
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
|
||||
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
|
||||
}
|
||||
}
|
||||
}
|
||||
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
|
||||
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
|
||||
db.setTransactionSuccessful();
|
||||
} finally {
|
||||
db.endTransaction();
|
||||
}
|
||||
}
|
||||
|
||||
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
|
||||
ContentValues values = new ContentValues();
|
||||
values.put("speaker_id", speakerId);
|
||||
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
|
||||
}
|
||||
|
||||
/** Reassign every segment of a speaker within one session (split support). */
|
||||
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
|
||||
getWritableDatabase().execSQL(
|
||||
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
|
||||
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
|
||||
}
|
||||
|
||||
public void deleteSpeaker(long speakerId) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
|
||||
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
|
||||
}
|
||||
|
||||
// ---- retention ----
|
||||
|
||||
/**
|
||||
* Delete caption sessions older than the cutoff (0 disables), and strip
|
||||
* recordings older than their own cutoff while keeping the transcript.
|
||||
* Returns a JSON summary of what was removed.
|
||||
*/
|
||||
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
int sessionsDeleted = 0;
|
||||
int recordingsDeleted = 0;
|
||||
if (recordingCutoffMs > 0) {
|
||||
List<String> paths = new ArrayList<>();
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
|
||||
new String[]{String.valueOf(recordingCutoffMs)})) {
|
||||
while (c.moveToNext()) {
|
||||
if (!c.isNull(1)) {
|
||||
paths.add(c.getString(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String path : paths) {
|
||||
if (new File(path).delete()) {
|
||||
recordingsDeleted++;
|
||||
}
|
||||
}
|
||||
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
|
||||
new Object[]{recordingCutoffMs});
|
||||
}
|
||||
if (captionCutoffMs > 0) {
|
||||
List<String> paths = new ArrayList<>();
|
||||
try (Cursor c = db.rawQuery(
|
||||
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
|
||||
new String[]{String.valueOf(captionCutoffMs)})) {
|
||||
while (c.moveToNext()) {
|
||||
paths.add(c.getString(0));
|
||||
}
|
||||
}
|
||||
for (String path : paths) {
|
||||
if (path != null && new File(path).delete()) {
|
||||
recordingsDeleted++;
|
||||
}
|
||||
}
|
||||
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
|
||||
new Object[]{captionCutoffMs});
|
||||
Cursor count = db.rawQuery("SELECT changes()", null);
|
||||
count.close();
|
||||
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
|
||||
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
|
||||
if (c.moveToFirst()) {
|
||||
sessionsDeleted = c.getInt(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
JSONObject summary = new JSONObject();
|
||||
summary.put("sessionsDeleted", sessionsDeleted);
|
||||
summary.put("recordingsDeleted", recordingsDeleted);
|
||||
return summary.toString();
|
||||
} catch (JSONException e) {
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteSession(long sessionId) {
|
||||
SQLiteDatabase db = getWritableDatabase();
|
||||
String path = getSessionAudioPath(sessionId);
|
||||
if (!path.isEmpty()) {
|
||||
new File(path).delete();
|
||||
}
|
||||
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
|
||||
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
|
||||
}
|
||||
|
||||
// ---- blob helpers ----
|
||||
|
||||
private static float[] blobToFloats(byte[] blob) {
|
||||
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
|
||||
return null;
|
||||
}
|
||||
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
|
||||
float[] out = new float[blob.length / 4];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
out[i] = buffer.getFloat();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static byte[] floatsToBlob(float[] values) {
|
||||
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
|
||||
for (float v : values) {
|
||||
buffer.putFloat(v);
|
||||
}
|
||||
return buffer.array();
|
||||
}
|
||||
|
||||
private static void l2NormalizeInPlace(float[] v) {
|
||||
double sum = 0;
|
||||
for (float x : v) {
|
||||
sum += (double) x * x;
|
||||
}
|
||||
double norm = Math.sqrt(sum);
|
||||
if (norm <= 0) {
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < v.length; i++) {
|
||||
v[i] = (float) (v[i] / norm);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6,151 +6,381 @@ import android.bluetooth.BluetoothDevice;
|
||||
import android.bluetooth.BluetoothManager;
|
||||
import android.bluetooth.le.BluetoothLeScanner;
|
||||
import android.bluetooth.le.ScanCallback;
|
||||
import android.bluetooth.le.ScanRecord;
|
||||
import android.bluetooth.le.ScanResult;
|
||||
import android.bluetooth.le.ScanSettings;
|
||||
import android.content.Context;
|
||||
import android.os.Build;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.SparseArray;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Finds Even Realities hardware over BLE and hands every advertisement up to
|
||||
* TypeScript with the raw fields the pairing screen needs: the LIVE local name
|
||||
* from the scan record (BluetoothDevice.getName() is a cache that can hold a
|
||||
* pre-reset value indefinitely), the full manufacturer-specific payload with
|
||||
* its company identifier restored so the "ER"+serial+MAC layout survives
|
||||
* intact, RSSI, advertised TX power, and connectability.
|
||||
*
|
||||
* Admission is deliberately loose — a name containing G2, the stock
|
||||
* "EVEN R1…" ring prefix, or the Even "ER" manufacturer signature — because
|
||||
* renamed custom firmware may drop the stock name. Side, serial, and pair
|
||||
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
|
||||
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
|
||||
* must stay a superset of the TS classifier's admission.
|
||||
*/
|
||||
@SuppressLint("MissingPermission")
|
||||
public class FaceclawDeviceDiscovery {
|
||||
private static final Pattern RING_NAME = Pattern.compile("^EVEN R1_[0-9A-F]{6}$");
|
||||
private static final Pattern RIGHT_NAME = Pattern.compile("^Even G2_32_R_[0-9A-F]{6}$");
|
||||
private static final Pattern LEFT_NAME = Pattern.compile("^Even G2_32_L_[0-9A-F]{6}$");
|
||||
/** Even Realities' company identifier: the ASCII bytes "ER" read little-endian. */
|
||||
public static final int EVEN_COMPANY_ID = 0x5245;
|
||||
|
||||
private final Context appContext;
|
||||
private final BluetoothAdapter bluetoothAdapter;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
|
||||
private final Object lock = new Object();
|
||||
private ScanCallback activeCallback;
|
||||
private volatile FaceclawDeviceDiscoveryListener listener;
|
||||
|
||||
public FaceclawDeviceDiscovery(Context context) {
|
||||
this.appContext = context.getApplicationContext();
|
||||
Context appContext = context.getApplicationContext();
|
||||
BluetoothManager bluetoothManager =
|
||||
(BluetoothManager) this.appContext.getSystemService(Context.BLUETOOTH_SERVICE);
|
||||
(BluetoothManager) appContext.getSystemService(Context.BLUETOOTH_SERVICE);
|
||||
if (bluetoothManager == null || bluetoothManager.getAdapter() == null) {
|
||||
throw new IllegalStateException("Bluetooth adapter unavailable");
|
||||
}
|
||||
this.bluetoothAdapter = bluetoothManager.getAdapter();
|
||||
}
|
||||
|
||||
public String getBondedCandidatesJson() {
|
||||
LinkedHashMap<String, CandidateDevice> matches = new LinkedHashMap<>();
|
||||
Set<BluetoothDevice> bonded = bluetoothAdapter.getBondedDevices();
|
||||
if (bonded != null) {
|
||||
for (BluetoothDevice device : bonded) {
|
||||
maybeAddCandidate(matches, device, "paired");
|
||||
}
|
||||
}
|
||||
return encodeCandidates(matches);
|
||||
public boolean isBluetoothEnabled() {
|
||||
return bluetoothAdapter.isEnabled();
|
||||
}
|
||||
|
||||
public String scanCandidatesJson(int timeoutMs) throws InterruptedException {
|
||||
LinkedHashMap<String, CandidateDevice> matches = new LinkedHashMap<>();
|
||||
BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner();
|
||||
if (scanner == null) {
|
||||
return encodeCandidates(matches);
|
||||
}
|
||||
final ScanCallback callback = new ScanCallback() {
|
||||
@Override
|
||||
public void onScanResult(int callbackType, ScanResult result) {
|
||||
if (result == null) {
|
||||
return;
|
||||
}
|
||||
maybeAddCandidate(matches, result.getDevice(), "scan");
|
||||
}
|
||||
public void setListener(FaceclawDeviceDiscoveryListener newListener) {
|
||||
this.listener = newListener;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onBatchScanResults(java.util.List<ScanResult> results) {
|
||||
if (results == null) {
|
||||
return;
|
||||
}
|
||||
for (ScanResult result : results) {
|
||||
// ------------------------------------------------------------------
|
||||
// Live scan
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Start streaming advertisements to the listener. Returns false when the
|
||||
* radio is off or the scanner is unavailable; a later platform refusal
|
||||
* arrives through onScanFailed. Calling while a scan is running is a no-op.
|
||||
*/
|
||||
public boolean startScan() {
|
||||
synchronized (lock) {
|
||||
if (activeCallback != null) {
|
||||
return true;
|
||||
}
|
||||
if (!bluetoothAdapter.isEnabled()) {
|
||||
emitLog("scan not started: Bluetooth is off");
|
||||
return false;
|
||||
}
|
||||
BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner();
|
||||
if (scanner == null) {
|
||||
emitLog("scan not started: no BLE scanner");
|
||||
return false;
|
||||
}
|
||||
final ScanCallback callback = new ScanCallback() {
|
||||
@Override
|
||||
public void onScanResult(int callbackType, ScanResult result) {
|
||||
if (result != null) {
|
||||
maybeAddCandidate(matches, result.getDevice(), "scan");
|
||||
emitAdvertisement(result);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onBatchScanResults(List<ScanResult> results) {
|
||||
if (results == null) {
|
||||
return;
|
||||
}
|
||||
for (ScanResult result : results) {
|
||||
if (result != null) {
|
||||
emitAdvertisement(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onScanFailed(int errorCode) {
|
||||
synchronized (lock) {
|
||||
if (activeCallback == this) {
|
||||
activeCallback = null;
|
||||
}
|
||||
}
|
||||
final FaceclawDeviceDiscoveryListener l = listener;
|
||||
if (l != null) {
|
||||
final String message = describeScanError(errorCode);
|
||||
mainHandler.post(() -> l.onScanFailed(errorCode, message));
|
||||
}
|
||||
}
|
||||
};
|
||||
activeCallback = callback;
|
||||
try {
|
||||
scanner.startScan(null, buildScanSettings(), callback);
|
||||
} catch (Throwable t) {
|
||||
activeCallback = null;
|
||||
emitLog("scan not started: " + t);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
scanner.startScan(
|
||||
null,
|
||||
new ScanSettings.Builder().setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY).build(),
|
||||
callback
|
||||
);
|
||||
try {
|
||||
Thread.sleep(Math.max(500, timeoutMs));
|
||||
} finally {
|
||||
emitLog("scan started (unfiltered, low latency)");
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void stopScan() {
|
||||
ScanCallback callback;
|
||||
synchronized (lock) {
|
||||
callback = activeCallback;
|
||||
activeCallback = null;
|
||||
}
|
||||
if (callback == null) {
|
||||
return;
|
||||
}
|
||||
BluetoothLeScanner scanner = bluetoothAdapter.getBluetoothLeScanner();
|
||||
if (scanner != null) {
|
||||
try {
|
||||
scanner.stopScan(callback);
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
}
|
||||
return encodeCandidates(matches);
|
||||
emitLog("scan stopped");
|
||||
}
|
||||
|
||||
private void maybeAddCandidate(Map<String, CandidateDevice> matches, BluetoothDevice device, String source) {
|
||||
if (device == null) {
|
||||
return;
|
||||
public boolean isScanning() {
|
||||
synchronized (lock) {
|
||||
return activeCallback != null;
|
||||
}
|
||||
String name = device.getName();
|
||||
String role = classifyRole(name);
|
||||
if (role == null) {
|
||||
return;
|
||||
}
|
||||
String address = device.getAddress();
|
||||
if (address == null || address.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
matches.put(address, new CandidateDevice(address, name == null ? "" : name, role, source));
|
||||
}
|
||||
|
||||
private String classifyRole(String name) {
|
||||
if (name == null) {
|
||||
return null;
|
||||
/**
|
||||
* Replay bonded Even devices to the listener. A bonded listing has no
|
||||
* signal sample and no manufacturer data, so its serial stays unknown until
|
||||
* the device is also heard advertising.
|
||||
*/
|
||||
public void emitBondedDevices() {
|
||||
final FaceclawDeviceDiscoveryListener l = listener;
|
||||
if (l == null) {
|
||||
return;
|
||||
}
|
||||
if (RIGHT_NAME.matcher(name).matches()) {
|
||||
return "right";
|
||||
for (String json : bondedCandidateJsonList()) {
|
||||
l.onAdvertisement(json);
|
||||
}
|
||||
if (LEFT_NAME.matcher(name).matches()) {
|
||||
return "left";
|
||||
}
|
||||
if (RING_NAME.matcher(name).matches()) {
|
||||
return "ring";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String encodeCandidates(Map<String, CandidateDevice> matches) {
|
||||
/** Bonded Even devices as one JSON array, for callers without a listener. */
|
||||
public String getBondedCandidatesJson() {
|
||||
JSONArray array = new JSONArray();
|
||||
for (CandidateDevice candidate : matches.values()) {
|
||||
JSONObject object = new JSONObject();
|
||||
for (String json : bondedCandidateJsonList()) {
|
||||
try {
|
||||
object.put("address", candidate.address);
|
||||
object.put("name", candidate.name);
|
||||
object.put("role", candidate.role);
|
||||
object.put("source", candidate.source);
|
||||
array.put(object);
|
||||
array.put(new JSONObject(json));
|
||||
} catch (JSONException ignored) {
|
||||
}
|
||||
}
|
||||
return array.toString();
|
||||
}
|
||||
|
||||
private static final class CandidateDevice {
|
||||
final String address;
|
||||
final String name;
|
||||
final String role;
|
||||
final String source;
|
||||
// ------------------------------------------------------------------
|
||||
// Internals
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
CandidateDevice(String address, String name, String role, String source) {
|
||||
this.address = address;
|
||||
this.name = name;
|
||||
this.role = role;
|
||||
this.source = source;
|
||||
private static ScanSettings buildScanSettings() {
|
||||
ScanSettings.Builder builder = new ScanSettings.Builder()
|
||||
.setScanMode(ScanSettings.SCAN_MODE_LOW_LATENCY)
|
||||
.setReportDelay(0);
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
|
||||
builder.setCallbackType(ScanSettings.CALLBACK_TYPE_ALL_MATCHES);
|
||||
builder.setMatchMode(ScanSettings.MATCH_MODE_AGGRESSIVE);
|
||||
builder.setNumOfMatches(ScanSettings.MATCH_NUM_MAX_ADVERTISEMENT);
|
||||
}
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
// Accept extended advertising too; setLegacy(true) would hide it.
|
||||
builder.setLegacy(false);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private void emitAdvertisement(ScanResult result) {
|
||||
final JSONObject object = toJson(result);
|
||||
if (object == null) {
|
||||
return;
|
||||
}
|
||||
final FaceclawDeviceDiscoveryListener l = listener;
|
||||
if (l == null) {
|
||||
return;
|
||||
}
|
||||
final String json = object.toString();
|
||||
if (Looper.myLooper() == Looper.getMainLooper()) {
|
||||
l.onAdvertisement(json);
|
||||
} else {
|
||||
mainHandler.post(() -> {
|
||||
FaceclawDeviceDiscoveryListener current = listener;
|
||||
if (current != null) {
|
||||
current.onAdvertisement(json);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> bondedCandidateJsonList() {
|
||||
List<String> out = new ArrayList<>();
|
||||
Set<BluetoothDevice> bonded = bluetoothAdapter.getBondedDevices();
|
||||
if (bonded == null) {
|
||||
return out;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
for (BluetoothDevice device : bonded) {
|
||||
if (device == null) {
|
||||
continue;
|
||||
}
|
||||
String name = device.getName();
|
||||
if (!isAdmissible(name, null)) {
|
||||
continue;
|
||||
}
|
||||
JSONObject object = buildJson(
|
||||
device.getAddress(), name, "", null, null, null, true, "paired", now);
|
||||
if (object != null) {
|
||||
out.add(object.toString());
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Null when the result is not something we pair with. */
|
||||
private static JSONObject toJson(ScanResult result) {
|
||||
BluetoothDevice device = result.getDevice();
|
||||
if (device == null) {
|
||||
return null;
|
||||
}
|
||||
ScanRecord record = result.getScanRecord();
|
||||
String liveName = record == null ? null : record.getDeviceName();
|
||||
String name = liveName != null && !liveName.isEmpty() ? liveName : device.getName();
|
||||
byte[] manufacturerData = extractEvenManufacturerData(record);
|
||||
if (!isAdmissible(name, manufacturerData)) {
|
||||
return null;
|
||||
}
|
||||
Integer txPower = null;
|
||||
if (record != null && record.getTxPowerLevel() != Integer.MIN_VALUE) {
|
||||
txPower = record.getTxPowerLevel();
|
||||
}
|
||||
if (txPower == null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O
|
||||
&& result.getTxPower() != ScanResult.TX_POWER_NOT_PRESENT) {
|
||||
txPower = result.getTxPower();
|
||||
}
|
||||
Boolean connectable = null;
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
connectable = result.isConnectable();
|
||||
}
|
||||
int rssi = result.getRssi();
|
||||
// 127 is the "unavailable" sentinel; 0 dBm never happens for a real sample.
|
||||
Integer rssiValue = (rssi == 127 || rssi == 0) ? null : rssi;
|
||||
boolean bonded = device.getBondState() == BluetoothDevice.BOND_BONDED;
|
||||
return buildJson(
|
||||
device.getAddress(), name, FaceclawFirmwareUtil.bytesToHex(manufacturerData), rssiValue, txPower,
|
||||
connectable, bonded, "scan", System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/**
|
||||
* Android strips the 2-byte company identifier from each manufacturer
|
||||
* record and keys the SparseArray by it. Put the bytes back (little-endian)
|
||||
* so TypeScript sees the same "ER"+… layout the HCI captures
|
||||
* document. Prefers Even's own id; otherwise returns the first record so an
|
||||
* unexpected company id still reaches the diagnostics log.
|
||||
*/
|
||||
private static byte[] extractEvenManufacturerData(ScanRecord record) {
|
||||
if (record == null) {
|
||||
return new byte[0];
|
||||
}
|
||||
SparseArray<byte[]> all = record.getManufacturerSpecificData();
|
||||
if (all == null || all.size() == 0) {
|
||||
return new byte[0];
|
||||
}
|
||||
byte[] payload = all.get(EVEN_COMPANY_ID);
|
||||
int companyId = EVEN_COMPANY_ID;
|
||||
if (payload == null) {
|
||||
companyId = all.keyAt(0);
|
||||
payload = all.valueAt(0);
|
||||
}
|
||||
if (payload == null) {
|
||||
return new byte[0];
|
||||
}
|
||||
byte[] out = new byte[payload.length + 2];
|
||||
out[0] = (byte) (companyId & 0xff);
|
||||
out[1] = (byte) ((companyId >> 8) & 0xff);
|
||||
System.arraycopy(payload, 0, out, 2, payload.length);
|
||||
return out;
|
||||
}
|
||||
|
||||
private static boolean isAdmissible(String name, byte[] manufacturerData) {
|
||||
if (manufacturerData != null && manufacturerData.length >= 2
|
||||
&& manufacturerData[0] == 0x45 && manufacturerData[1] == 0x52) {
|
||||
return true;
|
||||
}
|
||||
if (name == null) {
|
||||
return false;
|
||||
}
|
||||
String upper = name.toUpperCase();
|
||||
// A bare "R1" substring also matches unrelated hardware ("Oppo Enco
|
||||
// R1"), and TS only classifies rings from the stock name prefix.
|
||||
return upper.contains("G2") || upper.startsWith("EVEN R1");
|
||||
}
|
||||
|
||||
private static JSONObject buildJson(
|
||||
String address, String name, String manufacturerHex, Integer rssi, Integer txPower,
|
||||
Boolean connectable, boolean bonded, String source, long seenAtMs) {
|
||||
if (address == null || address.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
JSONObject object = new JSONObject();
|
||||
object.put("address", address);
|
||||
object.put("name", name == null ? "" : name);
|
||||
object.put("manufacturerData", manufacturerHex == null ? "" : manufacturerHex);
|
||||
object.put("rssi", rssi == null ? JSONObject.NULL : rssi);
|
||||
object.put("txPower", txPower == null ? JSONObject.NULL : txPower);
|
||||
object.put("connectable", connectable == null ? JSONObject.NULL : connectable);
|
||||
object.put("bonded", bonded);
|
||||
object.put("source", source);
|
||||
object.put("seenAtMs", seenAtMs);
|
||||
return object;
|
||||
} catch (JSONException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String describeScanError(int errorCode) {
|
||||
switch (errorCode) {
|
||||
case ScanCallback.SCAN_FAILED_ALREADY_STARTED:
|
||||
return "a scan is already running";
|
||||
case ScanCallback.SCAN_FAILED_APPLICATION_REGISTRATION_FAILED:
|
||||
return "the Bluetooth stack refused to register the scan (try toggling Bluetooth)";
|
||||
case ScanCallback.SCAN_FAILED_FEATURE_UNSUPPORTED:
|
||||
return "BLE scanning is not supported on this device";
|
||||
case ScanCallback.SCAN_FAILED_INTERNAL_ERROR:
|
||||
return "internal Bluetooth error";
|
||||
case 5: // SCAN_FAILED_OUT_OF_HARDWARE_RESOURCES (API 31)
|
||||
return "out of Bluetooth hardware resources";
|
||||
case 6: // SCAN_FAILED_SCANNING_TOO_FREQUENTLY (API 31)
|
||||
return "scanning too frequently; wait a moment and try again";
|
||||
default:
|
||||
return "scan failed (code " + errorCode + ")";
|
||||
}
|
||||
}
|
||||
|
||||
private void emitLog(String line) {
|
||||
FaceclawDeviceDiscoveryListener l = listener;
|
||||
if (l != null) {
|
||||
l.onLog(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/** Callbacks from FaceclawDeviceDiscovery's live scan to the TypeScript layer. */
|
||||
public interface FaceclawDeviceDiscoveryListener {
|
||||
/**
|
||||
* One advertisement (or one bonded device) as a JSON object:
|
||||
* {address, name, manufacturerData (hex, company id included), rssi|null,
|
||||
* txPower|null, connectable|null, bonded, source, seenAtMs}.
|
||||
* Protocol decoding happens in TypeScript (see app/g2/even-advertisement.ts).
|
||||
*/
|
||||
void onAdvertisement(String json);
|
||||
|
||||
/** The platform refused to start or continue the scan. */
|
||||
void onScanFailed(int errorCode, String message);
|
||||
|
||||
/** Diagnostic line for the pairing log. */
|
||||
void onLog(String line);
|
||||
}
|
||||
@@ -23,6 +23,7 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
|
||||
|
||||
private final Context context;
|
||||
private final String rightAddress;
|
||||
private final String leftAddress;
|
||||
private final FaceclawBleManager bleManager;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
|
||||
@@ -35,13 +36,21 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
|
||||
private byte[] awaitPb = null;
|
||||
private CountDownLatch awaitLatch = null;
|
||||
|
||||
/**
|
||||
* A sid-0x09 frame that carried firmware versions without matching the
|
||||
* awaited magic — some firmware pushes the settings snapshot on a magic it
|
||||
* picked itself instead of (or in addition to) acking the read.
|
||||
*/
|
||||
private volatile byte[] unsolicitedSettingsPb;
|
||||
|
||||
private volatile FaceclawDeviceInfoProbeListener listener;
|
||||
private volatile Thread worker;
|
||||
private volatile boolean cancelled = false;
|
||||
|
||||
public FaceclawDeviceInfoProbe(Context context, String rightAddress) {
|
||||
public FaceclawDeviceInfoProbe(Context context, String rightAddress, String leftAddress) {
|
||||
this.context = context.getApplicationContext();
|
||||
this.rightAddress = rightAddress == null ? "" : rightAddress;
|
||||
this.leftAddress = leftAddress == null ? "" : leftAddress;
|
||||
this.bleManager = new FaceclawBleManager(this.context);
|
||||
this.bleManager.setListener(this);
|
||||
}
|
||||
@@ -83,27 +92,28 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
|
||||
return;
|
||||
}
|
||||
|
||||
emitState("connecting", "");
|
||||
connectArm(rightAddress);
|
||||
byte[] ack = probeArm(rightAddress, "right");
|
||||
// The right lens is the documented control endpoint, but a silent
|
||||
// right lens has been observed on stock 2.2.9 even after a
|
||||
// successful security auth — run the same probe against the left
|
||||
// lens rather than giving up, and log which lens answered.
|
||||
if (ack == null && !cancelled && !leftAddress.trim().isEmpty()
|
||||
&& !leftAddress.equalsIgnoreCase(rightAddress)) {
|
||||
emitLog("right lens did not answer the settings query; probing the left lens");
|
||||
try {
|
||||
ack = probeArm(leftAddress, "left");
|
||||
} catch (Exception e) {
|
||||
emitLog("left-lens probe failed: " + (e.getMessage() == null ? e.toString() : e.getMessage()));
|
||||
}
|
||||
}
|
||||
if (cancelled) {
|
||||
emitError("Cancelled.");
|
||||
return;
|
||||
}
|
||||
|
||||
emitState("querying", "");
|
||||
// Session prelude, then a settings/device-info read (both arms'
|
||||
// versions and the CFW capability string ride back in one response).
|
||||
if (writeAndAwaitAck(rightAddress, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
|
||||
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
|
||||
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
|
||||
throw new IllegalStateException("session prelude not acked");
|
||||
}
|
||||
|
||||
int magic = allocMagic();
|
||||
byte[] ack = writeAndAwaitAck(rightAddress, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST,
|
||||
magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS);
|
||||
if (ack == null) {
|
||||
throw new IllegalStateException("no response to the device-info query");
|
||||
throw new IllegalStateException(
|
||||
"no response to the device-info query on either lens — check `adb logcat -s "
|
||||
+ TAG + "` for the frame trace");
|
||||
}
|
||||
|
||||
BleProtocol.FirmwareInfo info = BleProtocol.parseSettingsFirmwareInfo(ack);
|
||||
@@ -123,6 +133,73 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect + authenticate + prelude + settings read on one lens. Returns the
|
||||
* settings ack protobuf, an unsolicited settings push that carried firmware
|
||||
* versions, or null when the lens never answered the read. Throws on
|
||||
* connect or prelude failure; the caller wraps the fallback lens's attempt
|
||||
* so its failure cannot mask the primary lens's outcome.
|
||||
*/
|
||||
private byte[] probeArm(String address, String label) throws InterruptedException {
|
||||
emitState("connecting", label);
|
||||
connectArm(address);
|
||||
if (cancelled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Firmware 2.2.9 answers no queries until the security-auth exchange
|
||||
// completes over an encrypted link; on a phone with no existing bond
|
||||
// this is also what triggers SMP pairing (and its OS prompt), so it
|
||||
// must come before the prelude and query. Soft: this probe also runs
|
||||
// against the custom firmware, whose response to the exchange is not
|
||||
// yet hardware-verified, so an unconfirmed auth falls through.
|
||||
emitState("authenticating", label);
|
||||
boolean authenticated = authenticate(address);
|
||||
if (cancelled) {
|
||||
return null;
|
||||
}
|
||||
|
||||
emitState("querying", label);
|
||||
// Session prelude, then a settings/device-info read (both arms'
|
||||
// versions and the CFW capability string ride back in one response).
|
||||
if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
|
||||
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
|
||||
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
|
||||
throw new IllegalStateException("session prelude not acked (" + label + " lens)"
|
||||
+ (authenticated ? "" : "; authentication did not complete —"
|
||||
+ " if Android shows a Bluetooth pairing request, accept it and try again"));
|
||||
}
|
||||
|
||||
// Two attempts: the first read straight after a fresh pairing has been
|
||||
// seen to go unanswered while a later one succeeds.
|
||||
for (int attempt = 0; attempt < 2 && !cancelled; attempt++) {
|
||||
int magic = allocMagic();
|
||||
byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST,
|
||||
magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS);
|
||||
if (ack != null) {
|
||||
return ack;
|
||||
}
|
||||
// A push with the firmware versions on the device's own magic is
|
||||
// as good as the ack we asked for.
|
||||
byte[] pushed = unsolicitedSettingsPb;
|
||||
if (pushed != null) {
|
||||
emitLog("using unsolicited settings push instead of the read ack (" + label + " lens)");
|
||||
return pushed;
|
||||
}
|
||||
emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean authenticate(String address) throws InterruptedException {
|
||||
int magic = allocMagic();
|
||||
byte[] ack = writeAndAwaitAck(address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH,
|
||||
magic, BleProtocol.buildAuthenticationRequest(magic), ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS);
|
||||
boolean success = ack != null && BleProtocol.isAuthenticationSuccess(ack, magic);
|
||||
emitLog(success ? "security auth complete: " + address : "security auth unconfirmed: " + address);
|
||||
return success;
|
||||
}
|
||||
|
||||
private void connectArm(String address) {
|
||||
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
|
||||
throw new IllegalStateException("connect failed");
|
||||
@@ -150,10 +227,13 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
|
||||
synchronized (lock) {
|
||||
seq = nextSeq++ & 0xff;
|
||||
}
|
||||
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
|
||||
address, sid, flag, magic, seq, payload.length));
|
||||
List<byte[]> frames = BleProtocol.framePb(payload, sid, flag, seq);
|
||||
boolean written = bleManager.writeFrames(
|
||||
address, BleProtocol.WRITE_CHAR_UUID, frames, ConnectionOptions.WRITE_TYPE, ConnectionOptions.WRITE_TIMEOUT_MS);
|
||||
if (!written) {
|
||||
emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid));
|
||||
synchronized (lock) {
|
||||
awaitLatch = null;
|
||||
}
|
||||
@@ -171,10 +251,38 @@ public class FaceclawDeviceInfoProbe implements FaceclawBleListener {
|
||||
if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) {
|
||||
return;
|
||||
}
|
||||
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
|
||||
// One notification value can carry several envelope frames back to
|
||||
// back; reading only the first would silently drop the rest.
|
||||
List<byte[]> frames = BleProtocol.splitFrames(data);
|
||||
if (frames.size() > 1) {
|
||||
emitLog("rx " + address + " value carries " + frames.size() + " frames (raw " + data.length + " bytes)");
|
||||
}
|
||||
for (byte[] buf : frames) {
|
||||
handleFrame(address, buf, data.length);
|
||||
}
|
||||
}
|
||||
|
||||
private void handleFrame(String address, byte[] buf, int rawValueLength) {
|
||||
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf);
|
||||
if (!frame.ok) {
|
||||
emitLog("rx " + address + " unparseable frame len=" + buf.length + " (raw value " + rawValueLength + ")"
|
||||
+ " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.length))));
|
||||
return;
|
||||
}
|
||||
// Log every control frame while diagnosing 2.2.9: sid/flag/type/magic
|
||||
// plus a payload prefix is enough to reconstruct what the lens said.
|
||||
int declared = buf.length > 3 ? buf[3] & 0xff : 0;
|
||||
String truncated = buf.length < 8 + declared ? " TRUNCATED(declared=" + declared + ")" : "";
|
||||
emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s",
|
||||
address, frame.sid, frame.flag, frame.msgType, frame.msgSeq,
|
||||
buf.length > 5 ? buf[5] & 0xff : 0, buf.length > 4 ? buf[4] & 0xff : 0, frame.pb.length, truncated,
|
||||
FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.length)))));
|
||||
if (frame.sid == BleProtocol.SID_UI_SETTING
|
||||
&& BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) {
|
||||
// Any settings frame carrying firmware versions answers the probe's
|
||||
// question, whether or not it matches the magic we asked with.
|
||||
unsolicitedSettingsPb = frame.pb;
|
||||
}
|
||||
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
|
||||
return; // async event, not an ack
|
||||
}
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.DataInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Offline re-diarization of a stored conversation recording (16-bit mono
|
||||
* WAV): pyannote segmentation plus the shared speaker-embedding model with
|
||||
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
|
||||
* TS as JSON turns for reconciliation against the transcript segments.
|
||||
*/
|
||||
public class FaceclawDiarizer {
|
||||
private static final String TAG = "FaceclawDiarizer";
|
||||
|
||||
public interface Listener {
|
||||
/** progress in [0, 1]. */
|
||||
void onProgress(double progress);
|
||||
|
||||
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
|
||||
void onDone(String turnsJson);
|
||||
|
||||
void onError(String message);
|
||||
}
|
||||
|
||||
private final String segmentationModelPath;
|
||||
private final String embeddingModelPath;
|
||||
private final String wavPath;
|
||||
private final int numClusters;
|
||||
private final float threshold;
|
||||
private final Listener listener;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
private volatile boolean cancelled;
|
||||
|
||||
public FaceclawDiarizer(
|
||||
String segmentationModelPath,
|
||||
String embeddingModelPath,
|
||||
String wavPath,
|
||||
int numClusters,
|
||||
float threshold,
|
||||
Listener listener) {
|
||||
this.segmentationModelPath = segmentationModelPath;
|
||||
this.embeddingModelPath = embeddingModelPath;
|
||||
this.wavPath = wavPath;
|
||||
this.numClusters = numClusters;
|
||||
this.threshold = threshold <= 0 ? 0.5f : threshold;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
Thread thread = new Thread(this::run, "FaceclawDiarizer");
|
||||
thread.setPriority(Thread.MIN_PRIORITY);
|
||||
thread.start();
|
||||
}
|
||||
|
||||
public void cancel() {
|
||||
cancelled = true;
|
||||
}
|
||||
|
||||
private void run() {
|
||||
OfflineSpeakerDiarization diarization = null;
|
||||
try {
|
||||
float[] samples = wavPath.endsWith(".wav")
|
||||
? readWavMono16(wavPath)
|
||||
: decodeCompressedMono(wavPath);
|
||||
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
|
||||
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
|
||||
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
|
||||
.setModel(segmentationModelPath)
|
||||
.build())
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
|
||||
.setModel(embeddingModelPath)
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setClustering(FastClusteringConfig.builder()
|
||||
.setNumClusters(numClusters > 0 ? numClusters : -1)
|
||||
.setThreshold(threshold)
|
||||
.build())
|
||||
.setMinDurationOn(0.3f)
|
||||
.setMinDurationOff(0.5f)
|
||||
.build();
|
||||
diarization = new OfflineSpeakerDiarization(config);
|
||||
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
|
||||
samples,
|
||||
(numProcessedChunks, numTotalChunks, arg) -> {
|
||||
emitProgress(numTotalChunks <= 0 ? 0
|
||||
: (double) numProcessedChunks / numTotalChunks);
|
||||
return cancelled ? 1 : 0;
|
||||
});
|
||||
JSONArray turns = new JSONArray();
|
||||
if (segments != null) {
|
||||
for (OfflineSpeakerDiarizationSegment segment : segments) {
|
||||
JSONObject turn = new JSONObject();
|
||||
turn.put("startMs", Math.round(segment.getStart() * 1000));
|
||||
turn.put("endMs", Math.round(segment.getEnd() * 1000));
|
||||
turn.put("cluster", segment.getSpeaker());
|
||||
turns.put(turn);
|
||||
}
|
||||
}
|
||||
String result = turns.toString();
|
||||
mainHandler.post(() -> listener.onDone(result));
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "diarization failed for " + wavPath, t);
|
||||
String message = t.getMessage() == null ? t.toString() : t.getMessage();
|
||||
mainHandler.post(() -> listener.onError(message));
|
||||
} finally {
|
||||
if (diarization != null) {
|
||||
diarization.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void emitProgress(double progress) {
|
||||
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
|
||||
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
|
||||
* as 16 kHz mono, so no resampling is needed.
|
||||
*/
|
||||
private static float[] decodeCompressedMono(String path) throws IOException {
|
||||
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
|
||||
android.media.MediaCodec codec = null;
|
||||
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
|
||||
try {
|
||||
extractor.setDataSource(path);
|
||||
int trackIndex = -1;
|
||||
android.media.MediaFormat format = null;
|
||||
for (int i = 0; i < extractor.getTrackCount(); i++) {
|
||||
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
|
||||
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
|
||||
if (mime != null && mime.startsWith("audio/")) {
|
||||
trackIndex = i;
|
||||
format = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (trackIndex < 0 || format == null) {
|
||||
throw new IOException("no audio track in " + path);
|
||||
}
|
||||
extractor.selectTrack(trackIndex);
|
||||
codec = android.media.MediaCodec.createDecoderByType(
|
||||
format.getString(android.media.MediaFormat.KEY_MIME));
|
||||
codec.configure(format, null, null, 0);
|
||||
codec.start();
|
||||
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
|
||||
boolean inputDone = false;
|
||||
boolean outputDone = false;
|
||||
while (!outputDone) {
|
||||
if (!inputDone) {
|
||||
int inIndex = codec.dequeueInputBuffer(10_000);
|
||||
if (inIndex >= 0) {
|
||||
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
|
||||
int size = extractor.readSampleData(inBuf, 0);
|
||||
if (size < 0) {
|
||||
codec.queueInputBuffer(inIndex, 0, 0, 0,
|
||||
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
|
||||
inputDone = true;
|
||||
} else {
|
||||
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
|
||||
extractor.advance();
|
||||
}
|
||||
}
|
||||
}
|
||||
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
|
||||
if (outIndex >= 0) {
|
||||
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
|
||||
byte[] chunk = new byte[info.size];
|
||||
outBuf.position(info.offset);
|
||||
outBuf.get(chunk);
|
||||
pcm.write(chunk);
|
||||
codec.releaseOutputBuffer(outIndex, false);
|
||||
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputDone = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
extractor.release();
|
||||
if (codec != null) {
|
||||
try {
|
||||
codec.stop();
|
||||
} catch (Throwable ignored) {
|
||||
}
|
||||
codec.release();
|
||||
}
|
||||
}
|
||||
byte[] bytes = pcm.toByteArray();
|
||||
int count = bytes.length / 2;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
|
||||
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
|
||||
private static float[] readWavMono16(String path) throws IOException {
|
||||
File file = new File(path);
|
||||
long fileLength = file.length();
|
||||
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
|
||||
byte[] header = new byte[44];
|
||||
in.readFully(header);
|
||||
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
|
||||
throw new IOException("not a WAV file");
|
||||
}
|
||||
int channels = (header[22] & 0xff) | (header[23] << 8);
|
||||
int bits = (header[34] & 0xff) | (header[35] << 8);
|
||||
if (channels != 1 || bits != 16) {
|
||||
throw new IOException("only 16-bit mono WAV is supported");
|
||||
}
|
||||
long dataBytes = fileLength - 44;
|
||||
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
|
||||
float[] samples = new float[count];
|
||||
byte[] chunk = new byte[65536];
|
||||
int sampleIndex = 0;
|
||||
int carried = 0;
|
||||
byte carriedByte = 0;
|
||||
int read;
|
||||
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
|
||||
int offset = 0;
|
||||
if (carried == 1) {
|
||||
samples[sampleIndex++] =
|
||||
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
|
||||
offset = 1;
|
||||
carried = 0;
|
||||
}
|
||||
int pairs = (read - offset) / 2;
|
||||
for (int i = 0; i < pairs && sampleIndex < count; i++) {
|
||||
int base = offset + i * 2;
|
||||
samples[sampleIndex++] =
|
||||
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
|
||||
}
|
||||
if (((read - offset) & 1) == 1) {
|
||||
carried = 1;
|
||||
carriedByte = chunk[read - 1];
|
||||
}
|
||||
}
|
||||
return samples;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,17 +14,23 @@ import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
/**
|
||||
* Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA
|
||||
* firmware container to each lens over the firmware-data service using the
|
||||
* aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol, with
|
||||
* a heartbeat on the EvenHub control char to keep the session alive.
|
||||
* aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol.
|
||||
*
|
||||
* Each lens connection completes the sid-0x80 security-auth exchange before
|
||||
* OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing
|
||||
* the transfer mid-component. No control-channel traffic is injected during
|
||||
* the transfer itself: official OTA captures show none between BEGIN and the
|
||||
* final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a
|
||||
* concurrent control writer would interleave with the marker/data
|
||||
* transactions. (See ../notes/ble-connections-2.2.9.md.)
|
||||
*
|
||||
* Lenses are flashed one at a time (left, then right). Finishing either lens
|
||||
* reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect
|
||||
@@ -40,7 +46,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
// OTA message types (envelope sid byte) and control opcodes.
|
||||
private static final int SID_CTRL = 0xc0;
|
||||
private static final int SID_DATA = 0xc1;
|
||||
private static final int SID_HEARTBEAT = 0x80;
|
||||
private static final int FLAG_OTA = 0x00;
|
||||
private static final int OP_BEGIN = 0x00;
|
||||
private static final int OP_FILE_CHECK = 0x01;
|
||||
@@ -52,9 +57,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
private static final int CTRL_ACK_TIMEOUT_MS = 8_000;
|
||||
private static final int BLOCK_NAK_RETRIES = 3;
|
||||
private static final int COMPONENT_RETRIES = 3;
|
||||
private static final int HEARTBEAT_INTERVAL_MS = 12_000;
|
||||
private static final byte[] HEARTBEAT_PAYLOAD =
|
||||
new byte[] {0x08, 0x0e, 0x10, 0x26, 0x6a, 0x00};
|
||||
|
||||
// Reconnect windows. The first lens connects from an idle device; the second
|
||||
// must ride out the post-first-lens reboot of both lenses.
|
||||
@@ -89,10 +91,12 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
private volatile FaceclawFirmwareFlasherListener listener;
|
||||
private volatile Thread worker;
|
||||
private volatile boolean cancelled = false;
|
||||
private volatile ScheduledExecutorService heartbeatExecutor;
|
||||
private volatile String heartbeatAddress;
|
||||
|
||||
private int nextSeq = 0;
|
||||
private int nextAuthMagic = 0x60;
|
||||
private int pendingAuthMagic = -1;
|
||||
private CountDownLatch authLatch;
|
||||
private byte[] authAckPb;
|
||||
|
||||
public FaceclawFirmwareFlasher(Context context, String rightAddress, String leftAddress, String firmwarePath) {
|
||||
this.context = context.getApplicationContext();
|
||||
@@ -127,7 +131,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
|
||||
public void close() {
|
||||
cancel();
|
||||
stopHeartbeat();
|
||||
try {
|
||||
bleManager.close();
|
||||
} catch (Exception ignored) {
|
||||
@@ -172,27 +175,34 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
|
||||
private void flashLens(String lens, String address, byte[] img, List<Segment> segs, int connectWindowMs)
|
||||
throws TimeoutException {
|
||||
resetSeq();
|
||||
emitState("connecting", lens);
|
||||
connectLensResilient(lens, address, connectWindowMs);
|
||||
sleepInterruptibly(NOTIFY_SETTLE_MS);
|
||||
drainAcks();
|
||||
// Fresh envelope counter for the OTA stream (the auth exchange during
|
||||
// bring-up used its own); mirrors the stock sequence.
|
||||
resetSeq();
|
||||
|
||||
emitState("flashing", lens);
|
||||
startHeartbeat(address);
|
||||
try {
|
||||
int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS);
|
||||
if (!isEndOk(beginStatus)) {
|
||||
emitLog("warning: unexpected begin status " + beginStatus + "; continuing");
|
||||
int beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS);
|
||||
if (!isEndOk(beginStatus)) {
|
||||
emitLog("warning: unexpected begin status " + beginStatus + "; continuing");
|
||||
}
|
||||
// Progress is reported in bytes across the whole image: the segments
|
||||
// are one large firmware blob plus several small ones, so a bar
|
||||
// stepped per segment would sit still through the big one and then
|
||||
// sprint through the rest.
|
||||
long totalBytes = 0;
|
||||
for (Segment seg : segs) {
|
||||
totalBytes += seg.ps;
|
||||
}
|
||||
long bytesBefore = 0;
|
||||
for (int i = 0; i < segs.size(); i++) {
|
||||
if (cancelled) {
|
||||
throw new IllegalStateException("Cancelled.");
|
||||
}
|
||||
for (int i = 0; i < segs.size(); i++) {
|
||||
if (cancelled) {
|
||||
throw new IllegalStateException("Cancelled.");
|
||||
}
|
||||
flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img);
|
||||
}
|
||||
} finally {
|
||||
stopHeartbeat();
|
||||
flashComponentWithRetry(lens, address, i, segs.size(), segs.get(i), img, bytesBefore, totalBytes);
|
||||
bytesBefore += segs.get(i).ps;
|
||||
}
|
||||
emitLog(lens + " lens: all components verified");
|
||||
try {
|
||||
@@ -201,14 +211,15 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
}
|
||||
}
|
||||
|
||||
private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img) {
|
||||
private void flashComponentWithRetry(String lens, String address, int index, int count, Segment seg, byte[] img,
|
||||
long bytesBefore, long totalBytes) {
|
||||
for (int attempt = 0; attempt < COMPONENT_RETRIES; attempt++) {
|
||||
if (attempt > 0) {
|
||||
emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES);
|
||||
}
|
||||
int endStatus;
|
||||
try {
|
||||
endStatus = flashComponent(lens, address, index, count, seg, img);
|
||||
endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes);
|
||||
} catch (TimeoutException | RuntimeException e) {
|
||||
emitLog(seg.name + ": block phase failed: " + e.getMessage());
|
||||
endStatus = -1;
|
||||
@@ -226,8 +237,8 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
throw new IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts");
|
||||
}
|
||||
|
||||
private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img)
|
||||
throws TimeoutException {
|
||||
private int flashComponent(String lens, String address, int index, int count, Segment seg, byte[] img,
|
||||
long bytesBefore, long totalBytes) throws TimeoutException {
|
||||
byte[] sub = Arrays.copyOfRange(img, seg.off, seg.off + 128);
|
||||
int ps = seg.ps;
|
||||
int payloadStart = seg.off + 128;
|
||||
@@ -260,7 +271,8 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
throw new IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times");
|
||||
}
|
||||
if (b % 20 == 0 || b == blockCount - 1) {
|
||||
emitProgress(lens, index + 1, count, b + 1, blockCount);
|
||||
long bytesSent = bytesBefore + Math.min((long) (b + 1) * BLOCK_SIZE, (long) ps);
|
||||
emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes);
|
||||
}
|
||||
}
|
||||
emitLog(seg.name + ": data phase done; sending END");
|
||||
@@ -356,50 +368,64 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
|
||||
return false;
|
||||
}
|
||||
// Control-channel notify carries the auth response; mandatory.
|
||||
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
|
||||
return false;
|
||||
}
|
||||
if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
|
||||
return false;
|
||||
}
|
||||
// Control-channel notify (heartbeat responses); best-effort.
|
||||
bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS);
|
||||
if (!authenticate(address)) {
|
||||
throw new IllegalStateException(
|
||||
"security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- heartbeat -----------------------------------------------------------
|
||||
|
||||
private void startHeartbeat(String address) {
|
||||
stopHeartbeat();
|
||||
heartbeatAddress = address;
|
||||
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread t = new Thread(runnable, "faceclaw-flash-hb");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
heartbeatExecutor = executor;
|
||||
executor.scheduleWithFixedDelay(
|
||||
this::sendHeartbeat, HEARTBEAT_INTERVAL_MS, HEARTBEAT_INTERVAL_MS, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private void sendHeartbeat() {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
String address = heartbeatAddress;
|
||||
if (address == null) {
|
||||
return;
|
||||
/**
|
||||
* Complete the sid-0x80 security-auth exchange on this connection. On an
|
||||
* unbonded phone this triggers SMP pairing (possibly with an OS prompt),
|
||||
* so the wait is generous. Must succeed before any OTA traffic.
|
||||
*/
|
||||
private boolean authenticate(String address) {
|
||||
int magic;
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
synchronized (lock) {
|
||||
magic = nextAuthMagic;
|
||||
nextAuthMagic = nextAuthMagic >= 0x7f ? 0x60 : nextAuthMagic + 1;
|
||||
pendingAuthMagic = magic;
|
||||
authAckPb = null;
|
||||
authLatch = latch;
|
||||
}
|
||||
try {
|
||||
writeOta(address, BleProtocol.WRITE_CHAR_UUID, SID_HEARTBEAT, HEARTBEAT_PAYLOAD, nextSeq());
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "heartbeat failed: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void stopHeartbeat() {
|
||||
ScheduledExecutorService executor = heartbeatExecutor;
|
||||
heartbeatExecutor = null;
|
||||
heartbeatAddress = null;
|
||||
if (executor != null) {
|
||||
executor.shutdownNow();
|
||||
if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH,
|
||||
BleProtocol.buildAuthenticationRequest(magic), nextSeq())) {
|
||||
emitLog("auth write failed: " + address);
|
||||
return false;
|
||||
}
|
||||
boolean signalled;
|
||||
try {
|
||||
signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS, TimeUnit.MILLISECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return false;
|
||||
}
|
||||
byte[] pb;
|
||||
synchronized (lock) {
|
||||
pb = authAckPb;
|
||||
}
|
||||
if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) {
|
||||
emitLog("security auth not acknowledged (" + address
|
||||
+ ") — if Android shows a Bluetooth pairing request, accept it");
|
||||
return false;
|
||||
}
|
||||
emitLog("security auth complete: " + address);
|
||||
return true;
|
||||
} finally {
|
||||
synchronized (lock) {
|
||||
pendingAuthMagic = -1;
|
||||
authLatch = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,8 +523,25 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
|
||||
@Override
|
||||
public void onNotification(String address, String characteristicUuid, byte[] data) {
|
||||
if (BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) {
|
||||
// Control channel: only the security-auth response is awaited here.
|
||||
// A value can carry several envelope frames back to back (e.g. the
|
||||
// response packed with one of the lens's own sid-0x80 notifies).
|
||||
for (byte[] buf : BleProtocol.splitFrames(data)) {
|
||||
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf);
|
||||
if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) {
|
||||
synchronized (lock) {
|
||||
if (authLatch != null && frame.msgSeq == pendingAuthMagic) {
|
||||
authAckPb = frame.pb;
|
||||
authLatch.countDown();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equalsIgnoreCase(characteristicUuid)) {
|
||||
return; // acks arrive on the data-notify char; ignore heartbeat responses
|
||||
return; // OTA acks arrive on the data-notify char
|
||||
}
|
||||
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
|
||||
if (!frame.ok || frame.pb.length < 2) {
|
||||
@@ -517,7 +560,6 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
// ---- helpers -------------------------------------------------------------
|
||||
|
||||
private void teardown() {
|
||||
stopHeartbeat();
|
||||
try {
|
||||
bleManager.close();
|
||||
} catch (Exception ignored) {
|
||||
@@ -621,11 +663,12 @@ public class FaceclawFirmwareFlasher implements FaceclawBleListener {
|
||||
});
|
||||
}
|
||||
|
||||
private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount) {
|
||||
private void emitProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount,
|
||||
long bytesSent, long bytesTotal) {
|
||||
mainHandler.post(() -> {
|
||||
FaceclawFirmwareFlasherListener current = listener;
|
||||
if (current != null) {
|
||||
current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount);
|
||||
current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -6,9 +6,12 @@ public interface FaceclawFirmwareFlasherListener {
|
||||
|
||||
/**
|
||||
* Fine-grained progress for the UI bar. `lens` is "left"/"right";
|
||||
* component/block indices are 1-based against their counts.
|
||||
* component/block indices are 1-based against their counts. `bytesSent` /
|
||||
* `bytesTotal` cover the whole lens image, so a bar driven by them moves
|
||||
* evenly even though the components differ wildly in size.
|
||||
*/
|
||||
void onProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount);
|
||||
void onProgress(String lens, int componentIndex, int componentCount, int blockIndex, int blockCount,
|
||||
long bytesSent, long bytesTotal);
|
||||
|
||||
/**
|
||||
* Coarse lifecycle. `state` is one of: validating, connecting, flashing,
|
||||
|
||||
@@ -11,24 +11,32 @@ import java.security.NoSuchAlgorithmException;
|
||||
* Native helpers for the on-device custom-firmware build: SHA-256 hashing and
|
||||
* binary file writing. Both take a java.nio.ByteBuffer so NativeScript can
|
||||
* marshal a JS ArrayBuffer straight across without a per-element copy of the
|
||||
* ~4 MB firmware image (and with no overload ambiguity).
|
||||
* ~4 MB firmware image (and with no overload ambiguity). Also home of the one
|
||||
* shared byte→hex encoder for com.faceclaw.app.
|
||||
*/
|
||||
public final class FaceclawFirmwareUtil {
|
||||
private FaceclawFirmwareUtil() {}
|
||||
|
||||
/** Lowercase hex of the bytes. The TS-side decoder is case-insensitive. */
|
||||
public static String bytesToHex(byte[] bytes) {
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
return "";
|
||||
}
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) {
|
||||
int v = b & 0xff;
|
||||
sb.append(Character.forDigit(v >>> 4, 16));
|
||||
sb.append(Character.forDigit(v & 0x0f, 16));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/** Lowercase hex SHA-256 of the buffer's remaining bytes. */
|
||||
public static String sha256Hex(ByteBuffer data) {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
md.update(data.duplicate());
|
||||
byte[] digest = md.digest();
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) {
|
||||
int v = b & 0xff;
|
||||
sb.append(Character.forDigit(v >>> 4, 16));
|
||||
sb.append(Character.forDigit(v & 0x0f, 16));
|
||||
}
|
||||
return sb.toString();
|
||||
return bytesToHex(md.digest());
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
throw new RuntimeException("SHA-256 unavailable", e);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,10 @@ import java.util.concurrent.TimeUnit;
|
||||
* protobuf helpers (BleProtocol), but owns its own GATT connection and runs its
|
||||
* whole lifecycle on a single worker thread. It expects the main app to be
|
||||
* disconnected while it runs (onboarding, before flashing).
|
||||
*
|
||||
* Only the right arm is used: the lenses relay messages to each other, so
|
||||
* anything sent to the right arm reaches both, and acks/events only ever come
|
||||
* from the right arm anyway.
|
||||
*/
|
||||
public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
private static final String TAG = "FaceclawFlashPrompt";
|
||||
@@ -43,13 +47,13 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
|
||||
private final Context context;
|
||||
private final String rightAddress;
|
||||
private final String leftAddress;
|
||||
private final String warningText;
|
||||
private final FaceclawBleManager bleManager;
|
||||
private final Handler mainHandler = new Handler(Looper.getMainLooper());
|
||||
|
||||
private final Object lock = new Object();
|
||||
private final ConcurrentHashMap<String, CountDownLatch> pendingAcks = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, byte[]> ackPayloads = new ConcurrentHashMap<>();
|
||||
private final CountDownLatch selectionLatch = new CountDownLatch(1);
|
||||
|
||||
private volatile FaceclawFlashPromptListener listener;
|
||||
@@ -57,7 +61,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
private volatile boolean cancelled = false;
|
||||
private volatile boolean finished = false;
|
||||
private volatile boolean rightConnected = false;
|
||||
private volatile boolean leftConnected = false;
|
||||
private volatile boolean rightLost = false;
|
||||
private volatile Boolean approved = null;
|
||||
private volatile ScheduledExecutorService heartbeatExecutor;
|
||||
@@ -65,10 +68,9 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
private int nextMagic = 100;
|
||||
private int nextSeq = 0x40;
|
||||
|
||||
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String leftAddress, String warningText) {
|
||||
public FaceclawFlashPromptCommunicator(Context context, String rightAddress, String warningText) {
|
||||
this.context = context.getApplicationContext();
|
||||
this.rightAddress = rightAddress == null ? "" : rightAddress;
|
||||
this.leftAddress = leftAddress == null ? "" : leftAddress;
|
||||
this.warningText = warningText == null ? "" : warningText;
|
||||
this.bleManager = new FaceclawBleManager(this.context);
|
||||
this.bleManager.setListener(this);
|
||||
@@ -115,18 +117,8 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
}
|
||||
|
||||
emitState("connecting", "");
|
||||
connectArm(rightAddress, "right");
|
||||
connectArm(rightAddress);
|
||||
rightConnected = true;
|
||||
|
||||
boolean haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equalsIgnoreCase(rightAddress);
|
||||
if (haveLeft) {
|
||||
try {
|
||||
connectArm(leftAddress, "left");
|
||||
leftConnected = true;
|
||||
} catch (Exception e) {
|
||||
emitLog("left arm connect failed (continuing on right only): " + e.getMessage());
|
||||
}
|
||||
}
|
||||
if (cancelled) {
|
||||
teardown();
|
||||
return;
|
||||
@@ -134,14 +126,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
|
||||
emitState("connected", "");
|
||||
bringUpArm(rightAddress);
|
||||
if (leftConnected) {
|
||||
try {
|
||||
bringUpArm(leftAddress);
|
||||
} catch (Exception e) {
|
||||
emitLog("left arm prompt failed (continuing on right only): " + e.getMessage());
|
||||
leftConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
startHeartbeat();
|
||||
emitState("prompting", "");
|
||||
@@ -157,7 +141,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (approved != null) {
|
||||
boolean result = Boolean.TRUE.equals(approved);
|
||||
try {
|
||||
sendShutdownBoth();
|
||||
sendShutdown();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
emitResult(result);
|
||||
@@ -181,7 +165,7 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
}
|
||||
}
|
||||
|
||||
private void connectArm(String address, String labelForLog) {
|
||||
private void connectArm(String address) {
|
||||
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
|
||||
throw new IllegalStateException("connect failed: " + address);
|
||||
}
|
||||
@@ -193,42 +177,75 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
|
||||
throw new IllegalStateException("enableNotifications failed: " + address);
|
||||
}
|
||||
emitLog("connected " + labelForLog + " arm");
|
||||
emitLog("connected right arm");
|
||||
}
|
||||
|
||||
private void bringUpArm(String address) throws InterruptedException {
|
||||
// 1. Mandatory session prelude on sid=0x01 (app-launch).
|
||||
if (!writeAndAwaitAck(
|
||||
// 1. Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a
|
||||
// session (and closes the link after ~30 s) without it, and on an
|
||||
// unbonded phone this is what triggers SMP pairing, so it may sit
|
||||
// waiting on an OS pairing prompt.
|
||||
int authMagic = allocMagic();
|
||||
byte[] authAck = writeAndAwaitAck(
|
||||
address,
|
||||
BleProtocol.SID_SECURITY_AUTH,
|
||||
BleProtocol.FLAG_SECURITY_AUTH,
|
||||
authMagic,
|
||||
BleProtocol.buildAuthenticationRequest(authMagic),
|
||||
ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS);
|
||||
if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) {
|
||||
throw new IllegalStateException(
|
||||
"could not authenticate with the glasses (" + address
|
||||
+ ") — if Android shows a Bluetooth pairing request, accept it and try again");
|
||||
}
|
||||
// 2. Mandatory session prelude on sid=0x01 (app-launch).
|
||||
if (writeAndAwaitAck(
|
||||
address,
|
||||
BleProtocol.PRELUDE_ACK_SID,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
BleProtocol.PRELUDE_ACK_MAGIC,
|
||||
BleProtocol.PRELUDE_F5872_PAYLOAD,
|
||||
ConnectionOptions.PRELUDE_TIMEOUT_MS)) {
|
||||
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
|
||||
throw new IllegalStateException("session prelude not acked: " + address);
|
||||
}
|
||||
// 2. Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0.
|
||||
int magic = allocMagic();
|
||||
byte[] page = BleProtocol.buildCreatePromptPage(
|
||||
magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS);
|
||||
if (!writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS)) {
|
||||
// 3. Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0.
|
||||
// Two attempts: on 2.2.9 the first request sent right after the
|
||||
// prelude can be dropped while the lens emits its own sid-0x80
|
||||
// notifications (observed with the device-info settings read).
|
||||
byte[] pageAck = null;
|
||||
for (int attempt = 0; attempt < 2 && pageAck == null && !cancelled; attempt++) {
|
||||
int magic = allocMagic();
|
||||
byte[] page = BleProtocol.buildCreatePromptPage(
|
||||
magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS);
|
||||
pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS);
|
||||
if (pageAck == null) {
|
||||
emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address);
|
||||
}
|
||||
}
|
||||
if (pageAck == null) {
|
||||
throw new IllegalStateException("prompt page not acked: " + address);
|
||||
}
|
||||
emitLog("prompt page shown on " + address);
|
||||
}
|
||||
|
||||
private boolean writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs)
|
||||
/** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */
|
||||
private byte[] writeAndAwaitAck(String address, int sid, int flag, int magic, byte[] payload, int timeoutMs)
|
||||
throws InterruptedException {
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
String key = ackKey(sid, magic);
|
||||
pendingAcks.put(key, latch);
|
||||
try {
|
||||
if (!writeFrame(address, sid, flag, payload)) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
return latch.await(timeoutMs, TimeUnit.MILLISECONDS);
|
||||
if (!latch.await(timeoutMs, TimeUnit.MILLISECONDS)) {
|
||||
return null;
|
||||
}
|
||||
byte[] pb = ackPayloads.get(key);
|
||||
return pb == null ? new byte[0] : pb;
|
||||
} finally {
|
||||
pendingAcks.remove(key, latch);
|
||||
ackPayloads.remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,9 +284,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (rightConnected) {
|
||||
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
|
||||
}
|
||||
if (leftConnected) {
|
||||
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Log.w(TAG, "heartbeat failed: " + e.getMessage());
|
||||
}
|
||||
@@ -283,14 +297,11 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
}
|
||||
}
|
||||
|
||||
private void sendShutdownBoth() {
|
||||
private void sendShutdown() {
|
||||
byte[] shutdown = BleProtocol.buildShutdown(allocMagic(), 0);
|
||||
if (rightConnected) {
|
||||
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
|
||||
}
|
||||
if (leftConnected) {
|
||||
writeFrame(leftAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown);
|
||||
}
|
||||
}
|
||||
|
||||
private void teardown() {
|
||||
@@ -301,7 +312,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
rightConnected = false;
|
||||
leftConnected = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -309,20 +319,26 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
if (!BleProtocol.NOTIFY_CHAR_UUID.equalsIgnoreCase(characteristicUuid)) {
|
||||
return;
|
||||
}
|
||||
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(data);
|
||||
// A value can carry several envelope frames back to back.
|
||||
for (byte[] buf : BleProtocol.splitFrames(data)) {
|
||||
handleFrame(address, buf);
|
||||
}
|
||||
}
|
||||
|
||||
private void handleFrame(String address, byte[] buf) {
|
||||
BleProtocol.ParsedFrame frame = BleProtocol.parseFrame(buf);
|
||||
if (!frame.ok) {
|
||||
return;
|
||||
}
|
||||
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
|
||||
// Async events (taps/selections) only ever come from the right arm.
|
||||
if (address.equalsIgnoreCase(rightAddress)) {
|
||||
handleEvent(frame);
|
||||
}
|
||||
handleEvent(frame);
|
||||
return;
|
||||
}
|
||||
if (frame.msgSeq >= 0) {
|
||||
CountDownLatch latch = pendingAcks.get(ackKey(frame.sid, frame.msgSeq));
|
||||
String key = ackKey(frame.sid, frame.msgSeq);
|
||||
CountDownLatch latch = pendingAcks.get(key);
|
||||
if (latch != null) {
|
||||
ackPayloads.put(key, frame.pb);
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
@@ -364,8 +380,6 @@ public class FaceclawFlashPromptCommunicator implements FaceclawBleListener {
|
||||
selectionLatch.countDown();
|
||||
}
|
||||
}
|
||||
} else if (address.equalsIgnoreCase(leftAddress)) {
|
||||
leftConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
|
||||
public static final int COUNTER_OFFSET = 204;
|
||||
public static final int SAMPLES_PER_FRAME = 160;
|
||||
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
|
||||
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
|
||||
// glasses DSP appends two signed 16-bit LE values computed on the raw
|
||||
// stereo capture before the mono downmix — a signal-strength ratio and the
|
||||
// asin-derived direction-of-arrival in degrees — then the packet counter.
|
||||
public static final int SSR_OFFSET = 200;
|
||||
public static final int ANGLE_OFFSET = 202;
|
||||
|
||||
static {
|
||||
System.loadLibrary("faceclaw_lc3");
|
||||
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
|
||||
private long duplicatePackets;
|
||||
private long missingPackets;
|
||||
private long decodeErrors;
|
||||
private int lastSsr;
|
||||
private int lastAngleDegrees;
|
||||
|
||||
public FaceclawLc3Decoder() {
|
||||
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
|
||||
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
|
||||
}
|
||||
realPackets++;
|
||||
lastCounter = counter;
|
||||
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
|
||||
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
|
||||
return decoded;
|
||||
}
|
||||
|
||||
/** Firmware signal-strength ratio from the most recent decoded packet. */
|
||||
public synchronized int getLastSsr() {
|
||||
return lastSsr;
|
||||
}
|
||||
|
||||
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
|
||||
public synchronized int getLastAngleDegrees() {
|
||||
return lastAngleDegrees;
|
||||
}
|
||||
|
||||
public synchronized long getRealPackets() {
|
||||
return realPackets;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
public interface FaceclawMicStatusListener {
|
||||
/**
|
||||
* A CFW mic_control field-104 status record from one temple. body is the
|
||||
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
|
||||
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
|
||||
*/
|
||||
void onMicStatus(byte[] body, String arm);
|
||||
}
|
||||
@@ -144,7 +144,7 @@ public class FaceclawModelDownloader {
|
||||
+ expectedTotalBytes + " bytes); try again to resume");
|
||||
return;
|
||||
}
|
||||
String actualSha = toHex(digest.digest());
|
||||
String actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest());
|
||||
if (!expectedSha256.isEmpty() && !actualSha.equals(expectedSha256)) {
|
||||
partFile.delete();
|
||||
postError("Model download was corrupted (checksum mismatch); download it again");
|
||||
@@ -177,12 +177,6 @@ public class FaceclawModelDownloader {
|
||||
return hashed;
|
||||
}
|
||||
|
||||
private static String toHex(byte[] bytes) {
|
||||
StringBuilder sb = new StringBuilder(bytes.length * 2);
|
||||
for (byte b : bytes) sb.append(String.format("%02x", b));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void post(Runnable runnable) {
|
||||
callbackHandler.post(runnable);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.media.audiofx.AcousticEchoCanceler;
|
||||
import android.media.audiofx.AudioEffect;
|
||||
import android.media.audiofx.AutomaticGainControl;
|
||||
import android.media.audiofx.NoiseSuppressor;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* Phone-side noise suppression for the microphone stream.
|
||||
*
|
||||
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
|
||||
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
|
||||
* arrives as raw PCM over BLE from the glasses, which that effect chain never
|
||||
* sees, so this class picks the best engine that can actually reach the
|
||||
* samples:
|
||||
*
|
||||
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
|
||||
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
|
||||
* whatever the Android build ships) on the AudioRecord session.
|
||||
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
|
||||
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
|
||||
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
|
||||
* over-subtracted Wiener gain with a residual floor so speech never pumps
|
||||
* in and out of silence.
|
||||
*/
|
||||
public final class FaceclawNoiseSuppressor {
|
||||
|
||||
private static final int FFT_SIZE = 256;
|
||||
private static final int HOP = FFT_SIZE / 2;
|
||||
private static final float OVERSUBTRACTION = 1.6f;
|
||||
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
|
||||
private static final float GAIN_FLOOR = 0.15f;
|
||||
private static final float GAIN_SMOOTHING = 0.6f;
|
||||
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
|
||||
private static final int NOISE_WARMUP_FRAMES = 12;
|
||||
|
||||
private final int sampleRate;
|
||||
private final float[] window = new float[FFT_SIZE];
|
||||
private final float[] history = new float[FFT_SIZE];
|
||||
private final float[] overlap = new float[HOP];
|
||||
private final float[] noise = new float[FFT_SIZE / 2 + 1];
|
||||
private final float[] gain = new float[FFT_SIZE / 2 + 1];
|
||||
private final float[] re = new float[FFT_SIZE];
|
||||
private final float[] im = new float[FFT_SIZE];
|
||||
private final short[] pending = new short[HOP];
|
||||
private int pendingCount = 0;
|
||||
private int warmupFrames = 0;
|
||||
|
||||
public FaceclawNoiseSuppressor(int sampleRate) {
|
||||
this.sampleRate = sampleRate;
|
||||
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
|
||||
for (int i = 0; i < FFT_SIZE; i++) {
|
||||
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
|
||||
}
|
||||
reset();
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return sampleRate;
|
||||
}
|
||||
|
||||
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
|
||||
public void reset() {
|
||||
Arrays.fill(history, 0f);
|
||||
Arrays.fill(overlap, 0f);
|
||||
Arrays.fill(noise, 0f);
|
||||
Arrays.fill(gain, 1f);
|
||||
pendingCount = 0;
|
||||
warmupFrames = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
|
||||
* filling a whole 8 ms hop is carried into the next call, so the output
|
||||
* may be up to one hop shorter than the input (and longer on a later
|
||||
* call); total latency is one hop.
|
||||
*/
|
||||
public byte[] process(byte[] pcm16le) {
|
||||
if (pcm16le == null || pcm16le.length < 2) {
|
||||
return pcm16le;
|
||||
}
|
||||
int sampleCount = pcm16le.length / 2;
|
||||
int hops = (pendingCount + sampleCount) / HOP;
|
||||
byte[] out = new byte[hops * HOP * 2];
|
||||
int outPos = 0;
|
||||
int inPos = 0;
|
||||
while (inPos < sampleCount) {
|
||||
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
|
||||
for (int i = 0; i < take; i++) {
|
||||
int lo = pcm16le[(inPos + i) * 2] & 0xff;
|
||||
int hi = pcm16le[(inPos + i) * 2 + 1];
|
||||
pending[pendingCount + i] = (short) ((hi << 8) | lo);
|
||||
}
|
||||
pendingCount += take;
|
||||
inPos += take;
|
||||
if (pendingCount == HOP) {
|
||||
outPos = processHop(out, outPos);
|
||||
pendingCount = 0;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private int processHop(byte[] out, int outPos) {
|
||||
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
|
||||
}
|
||||
for (int i = 0; i < FFT_SIZE; i++) {
|
||||
re[i] = history[i] * window[i];
|
||||
im[i] = 0f;
|
||||
}
|
||||
fft(re, im, false);
|
||||
|
||||
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
|
||||
if (warming) {
|
||||
warmupFrames++;
|
||||
}
|
||||
for (int b = 0; b <= FFT_SIZE / 2; b++) {
|
||||
float power = re[b] * re[b] + im[b] * im[b];
|
||||
if (warming) {
|
||||
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
|
||||
} else if (power < noise[b]) {
|
||||
// Track the floor down quickly; drift up slowly so speech
|
||||
// energy never gets learned as noise.
|
||||
noise[b] += (power - noise[b]) * 0.2f;
|
||||
} else {
|
||||
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
|
||||
}
|
||||
float clean = power - OVERSUBTRACTION * noise[b];
|
||||
float g = clean > 0f ? clean / power : 0f;
|
||||
if (g < GAIN_FLOOR) {
|
||||
g = GAIN_FLOOR;
|
||||
}
|
||||
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
|
||||
gain[b] = g;
|
||||
re[b] *= g;
|
||||
im[b] *= g;
|
||||
if (b > 0 && b < FFT_SIZE / 2) {
|
||||
re[FFT_SIZE - b] *= g;
|
||||
im[FFT_SIZE - b] *= g;
|
||||
}
|
||||
}
|
||||
|
||||
fft(re, im, true);
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
float value = (overlap[i] + re[i] * window[i]) * 32767f;
|
||||
int sample = Math.round(value);
|
||||
if (sample > 32767) {
|
||||
sample = 32767;
|
||||
}
|
||||
if (sample < -32768) {
|
||||
sample = -32768;
|
||||
}
|
||||
out[outPos + i * 2] = (byte) (sample & 0xff);
|
||||
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
|
||||
}
|
||||
for (int i = 0; i < HOP; i++) {
|
||||
overlap[i] = re[HOP + i] * window[HOP + i];
|
||||
}
|
||||
return outPos + HOP * 2;
|
||||
}
|
||||
|
||||
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
|
||||
private static void fft(float[] re, float[] im, boolean inverse) {
|
||||
int n = re.length;
|
||||
for (int i = 1, j = 0; i < n; i++) {
|
||||
int bit = n >> 1;
|
||||
for (; (j & bit) != 0; bit >>= 1) {
|
||||
j ^= bit;
|
||||
}
|
||||
j ^= bit;
|
||||
if (i < j) {
|
||||
float t = re[i];
|
||||
re[i] = re[j];
|
||||
re[j] = t;
|
||||
t = im[i];
|
||||
im[i] = im[j];
|
||||
im[j] = t;
|
||||
}
|
||||
}
|
||||
for (int len = 2; len <= n; len <<= 1) {
|
||||
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
|
||||
float wRe = (float) Math.cos(ang);
|
||||
float wIm = (float) Math.sin(ang);
|
||||
for (int i = 0; i < n; i += len) {
|
||||
float curRe = 1f;
|
||||
float curIm = 0f;
|
||||
for (int k = 0; k < len / 2; k++) {
|
||||
int a = i + k;
|
||||
int b = a + len / 2;
|
||||
float xr = re[b] * curRe - im[b] * curIm;
|
||||
float xi = re[b] * curIm + im[b] * curRe;
|
||||
re[b] = re[a] - xr;
|
||||
im[b] = im[a] - xi;
|
||||
re[a] += xr;
|
||||
im[a] += xi;
|
||||
float nextRe = curRe * wRe - curIm * wIm;
|
||||
curIm = curRe * wIm + curIm * wRe;
|
||||
curRe = nextRe;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (inverse) {
|
||||
for (int i = 0; i < n; i++) {
|
||||
re[i] /= n;
|
||||
im[i] /= n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this Android build ships the built-in capture noise suppressor. */
|
||||
public static boolean platformSuppressorAvailable() {
|
||||
try {
|
||||
return NoiseSuppressor.isAvailable();
|
||||
} catch (Throwable t) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Short engine name for the UI. */
|
||||
public static String engineDescription() {
|
||||
return "spectral (phone DSP)";
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
|
||||
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
|
||||
* build provides) to a phone AudioRecord session. Only applicable when the
|
||||
* phone itself captures the audio; the BLE glasses stream must go through
|
||||
* {@link #process(byte[])} instead. The caller keeps the returned effects
|
||||
* for the life of the recording and release()s them with it.
|
||||
*/
|
||||
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
|
||||
ArrayList<AudioEffect> effects = new ArrayList<>();
|
||||
try {
|
||||
if (NoiseSuppressor.isAvailable()) {
|
||||
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
|
||||
if (ns != null) {
|
||||
ns.setEnabled(true);
|
||||
effects.add(ns);
|
||||
}
|
||||
}
|
||||
if (AcousticEchoCanceler.isAvailable()) {
|
||||
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
|
||||
if (aec != null) {
|
||||
aec.setEnabled(true);
|
||||
effects.add(aec);
|
||||
}
|
||||
}
|
||||
if (AutomaticGainControl.isAvailable()) {
|
||||
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
|
||||
if (agc != null) {
|
||||
agc.setEnabled(true);
|
||||
effects.add(agc);
|
||||
}
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
// A missing effect just means this tier is unavailable.
|
||||
}
|
||||
return effects.toArray(new AudioEffect[0]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
|
||||
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
|
||||
|
||||
/**
|
||||
* Speaker voice-print extraction over sherpa-onnx. One embedding per
|
||||
* utterance; matching/enrollment happens on the TS side (cosine similarity
|
||||
* against the stored speaker profiles).
|
||||
*/
|
||||
public class FaceclawSpeakerId {
|
||||
private static final String TAG = "FaceclawSpeakerId";
|
||||
|
||||
private SpeakerEmbeddingExtractor extractor;
|
||||
private final String modelPath;
|
||||
|
||||
public FaceclawSpeakerId(String modelPath) {
|
||||
this.modelPath = modelPath;
|
||||
}
|
||||
|
||||
/** Lazily loads the model; returns false when it cannot be loaded. */
|
||||
public synchronized boolean ensureLoaded() {
|
||||
if (extractor != null) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
|
||||
.setModel(modelPath)
|
||||
.setNumThreads(1)
|
||||
.setDebug(false)
|
||||
.setProvider("cpu")
|
||||
.build();
|
||||
extractor = new SpeakerEmbeddingExtractor(config);
|
||||
return true;
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "could not load speaker model at " + modelPath, t);
|
||||
extractor = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized int embeddingDim() {
|
||||
return extractor == null ? 0 : extractor.getDim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
|
||||
* S16LE PCM. Returns null when the model is unavailable or the utterance
|
||||
* is too short to embed (under ~0.5 s).
|
||||
*/
|
||||
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
|
||||
if (pcm16le == null || pcm16le.length < sampleRate) {
|
||||
return null;
|
||||
}
|
||||
if (!ensureLoaded()) {
|
||||
return null;
|
||||
}
|
||||
int count = pcm16le.length / 2;
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
|
||||
samples[i] = s / 32768.0f;
|
||||
}
|
||||
OnlineStream stream = null;
|
||||
try {
|
||||
stream = extractor.createStream();
|
||||
stream.acceptWaveform(samples, sampleRate);
|
||||
stream.inputFinished();
|
||||
if (!extractor.isReady(stream)) {
|
||||
return null;
|
||||
}
|
||||
float[] embedding = extractor.compute(stream);
|
||||
return l2Normalize(embedding);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "embedding failed", t);
|
||||
return null;
|
||||
} finally {
|
||||
if (stream != null) {
|
||||
stream.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static float[] l2Normalize(float[] v) {
|
||||
if (v == null) {
|
||||
return null;
|
||||
}
|
||||
double sum = 0;
|
||||
for (float x : v) {
|
||||
sum += (double) x * x;
|
||||
}
|
||||
double norm = Math.sqrt(sum);
|
||||
if (norm <= 0) {
|
||||
return v;
|
||||
}
|
||||
for (int i = 0; i < v.length; i++) {
|
||||
v[i] = (float) (v[i] / norm);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
public synchronized void close() {
|
||||
if (extractor != null) {
|
||||
extractor.release();
|
||||
extractor = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,20 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.Context;
|
||||
import android.content.res.AssetManager;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.os.SystemClock;
|
||||
import android.util.Log;
|
||||
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotter;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig;
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
|
||||
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
|
||||
import com.k2fsa.sherpa.onnx.OfflineStream;
|
||||
import com.k2fsa.sherpa.onnx.OnlineModelConfig;
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream;
|
||||
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Arrays;
|
||||
@@ -57,17 +47,15 @@ public class FaceclawVoiceController {
|
||||
private static final float TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f;
|
||||
private static final float TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f;
|
||||
private static final int TRANSCRIPT_LOG_PREVIEW_CHARS = 80;
|
||||
private static final String ASSET_ROOT = "faceclaw-voice";
|
||||
private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01";
|
||||
private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr";
|
||||
// Model directory shared with the TS-side download flow (asr-model.ts),
|
||||
// which fetches the Moonshine files here on demand; they are no longer
|
||||
// bundled in the APK.
|
||||
private static final String ASR_ROOT = "faceclaw-voice-asr";
|
||||
private static final String ASR_MODEL_DIR = "sherpa-onnx-moonshine-base-en-quantized-2026-02-27";
|
||||
private static final String[] MODEL_FILES = {
|
||||
"encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
"tokens.txt",
|
||||
"screen-on-keywords.txt"
|
||||
};
|
||||
// Model files for the retired on-phone wake-word spotter, copied to
|
||||
// filesDir by earlier releases; deleted on sight to reclaim the space.
|
||||
// (The wakeword is now detected by the glasses firmware itself.)
|
||||
private static final String LEGACY_KWS_ROOT = "faceclaw-voice";
|
||||
private static final String[] ASR_MODEL_FILES = {
|
||||
"encoder_model.ort",
|
||||
"decoder_model_merged.ort",
|
||||
@@ -75,7 +63,6 @@ public class FaceclawVoiceController {
|
||||
};
|
||||
|
||||
private enum VoiceInputMode {
|
||||
WAKEWORD, // on-phone keyword spotting (kept for later; not currently wired)
|
||||
ONBOARD, // on-phone Moonshine transcription
|
||||
CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side
|
||||
}
|
||||
@@ -89,10 +76,11 @@ public class FaceclawVoiceController {
|
||||
private volatile FaceclawBleCommunicator communicator;
|
||||
private Thread workerThread;
|
||||
private volatile boolean started;
|
||||
private VoiceInputMode mode = VoiceInputMode.WAKEWORD;
|
||||
private KeywordSpotter keywordSpotter;
|
||||
// Set once the worker has the glasses mic enabled for this session.
|
||||
// Read and written under `lock`, so it flips with `started` atomically.
|
||||
private boolean audioStarted;
|
||||
private VoiceInputMode mode = VoiceInputMode.CLOUD;
|
||||
private OfflineRecognizer recognizer;
|
||||
private OnlineStream stream;
|
||||
private FaceclawLc3Decoder lc3Decoder;
|
||||
private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES];
|
||||
private int transcriptSampleCount;
|
||||
@@ -105,6 +93,26 @@ public class FaceclawVoiceController {
|
||||
private volatile boolean endpointing;
|
||||
private final EndpointDetector endpointDetector = new EndpointDetector();
|
||||
private java.io.ByteArrayOutputStream recordingPcm;
|
||||
// Speaker verification against the enrolled wearer voice-print ("my voice
|
||||
// only" command gating). Configured before start(); the utterance PCM is
|
||||
// buffered (capped) and verified once at session end.
|
||||
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
|
||||
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
|
||||
private volatile String verifySpeakerModelPath;
|
||||
private volatile float[] verifyWearerEmbedding;
|
||||
private volatile float verifyThreshold = 0.8f;
|
||||
private short[] verifyBuffer;
|
||||
private int verifyCount;
|
||||
// Global mic processing (Microphones app config): spectral noise
|
||||
// suppression and firmware-DoA beam gating, applied to every capture
|
||||
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
|
||||
// The raw tap opts out — raw means raw, and the Microphones session does
|
||||
// its own beam-compensated processing on that path.
|
||||
private volatile boolean suppressionEnabled;
|
||||
private volatile boolean beamFilterEnabled;
|
||||
private volatile int beamCenterDeg;
|
||||
private volatile int beamHalfWidthDeg = 180;
|
||||
private FaceclawNoiseSuppressor suppressor;
|
||||
private long queuedPackets;
|
||||
private long queueDroppedPackets;
|
||||
private long decodedSamples;
|
||||
@@ -140,8 +148,66 @@ public class FaceclawVoiceController {
|
||||
this.endpointing = endpointing;
|
||||
}
|
||||
|
||||
public void start() {
|
||||
start("wakeword");
|
||||
/**
|
||||
* Verify this session's speaker against the enrolled wearer voice-print
|
||||
* and report the result via onSpeakerVerified just before the final
|
||||
* transcript. Must be set before {@link #start}; pass a null model path
|
||||
* to disable.
|
||||
*/
|
||||
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
|
||||
this.verifySpeakerModelPath = speakerModelPath;
|
||||
this.verifyWearerEmbedding = wearerEmbedding;
|
||||
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
|
||||
}
|
||||
|
||||
public void clearSpeakerVerification() {
|
||||
this.verifySpeakerModelPath = null;
|
||||
this.verifyWearerEmbedding = null;
|
||||
}
|
||||
|
||||
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
|
||||
public void setNoiseSuppression(boolean enabled) {
|
||||
this.suppressionEnabled = enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Direction gating from the Sonic Radar beam: packets whose firmware
|
||||
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
|
||||
* frame, 0 = straight ahead, positive right) are dropped before any
|
||||
* consumer sees them. Safe to update mid-run.
|
||||
*/
|
||||
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
|
||||
this.beamFilterEnabled = enabled;
|
||||
this.beamCenterDeg = centerDeg;
|
||||
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
|
||||
}
|
||||
|
||||
private boolean withinBeam(int angleDegrees) {
|
||||
int delta = angleDegrees - beamCenterDeg;
|
||||
while (delta > 180) delta -= 360;
|
||||
while (delta < -180) delta += 360;
|
||||
return Math.abs(delta) <= beamHalfWidthDeg;
|
||||
}
|
||||
|
||||
private void applySuppression(short[] pcm, int count) {
|
||||
try {
|
||||
if (suppressor == null) {
|
||||
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
|
||||
}
|
||||
byte[] le = new byte[count * 2];
|
||||
for (int i = 0; i < count; i++) {
|
||||
le[i * 2] = (byte) (pcm[i] & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
|
||||
}
|
||||
byte[] cleaned = suppressor.process(le);
|
||||
int cleanedCount = Math.min(count, cleaned.length / 2);
|
||||
for (int i = 0; i < cleanedCount; i++) {
|
||||
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
|
||||
}
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "noise suppression failed; passing audio through", t);
|
||||
suppressionEnabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
public void start(String requestedMode) {
|
||||
@@ -156,11 +222,36 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
mode = parseMode(requestedMode);
|
||||
started = true;
|
||||
audioStarted = false;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether mic audio is actually flowing. {@link #start} only records
|
||||
* intent: the enable lives in the glasses' EvenHub session, so a transport
|
||||
* drop or a session suspend can leave this controller started with a
|
||||
* worker that will never see another packet. Anything deciding whether to
|
||||
* (re)start capture must ask this rather than assume its own bookkeeping.
|
||||
*/
|
||||
public boolean isCapturing() {
|
||||
boolean audioUp;
|
||||
synchronized (lock) {
|
||||
if (!started) {
|
||||
return false;
|
||||
}
|
||||
audioUp = audioStarted;
|
||||
}
|
||||
if (!audioUp) {
|
||||
// The worker is still bringing the mic up; report it as running so
|
||||
// a concurrent request shares it instead of restarting it.
|
||||
return true;
|
||||
}
|
||||
FaceclawBleCommunicator currentCommunicator = communicator;
|
||||
return currentCommunicator != null && currentCommunicator.isAudioCaptureActive();
|
||||
}
|
||||
|
||||
public void stop() {
|
||||
Thread threadToJoin;
|
||||
synchronized (lock) {
|
||||
@@ -194,41 +285,49 @@ public class FaceclawVoiceController {
|
||||
if ("cloud".equals(requestedMode)) {
|
||||
return VoiceInputMode.CLOUD;
|
||||
}
|
||||
if ("onboard".equals(requestedMode) || "full".equals(requestedMode)) {
|
||||
return VoiceInputMode.ONBOARD;
|
||||
}
|
||||
return VoiceInputMode.WAKEWORD;
|
||||
return VoiceInputMode.ONBOARD;
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
try {
|
||||
deleteLegacyKwsFiles();
|
||||
VoiceInputMode currentMode = mode;
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
File modelDir = findAsrModelDir();
|
||||
if (modelDir == null) {
|
||||
emitStatus("Voice model not downloaded (see Settings > Voice).");
|
||||
return;
|
||||
}
|
||||
emitStatus("Loading transcription model...");
|
||||
File modelDir = installAsrModelFiles();
|
||||
recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir));
|
||||
resetTranscriptState();
|
||||
lastTranscript = "";
|
||||
} else if (currentMode == VoiceInputMode.WAKEWORD) {
|
||||
emitStatus("Loading wake-word model...");
|
||||
File modelDir = installModelFiles();
|
||||
keywordSpotter = new KeywordSpotter(buildConfig(modelDir));
|
||||
stream = keywordSpotter.createStream();
|
||||
}
|
||||
lc3Decoder = new FaceclawLc3Decoder();
|
||||
endpointDetector.reset();
|
||||
if (suppressor != null) {
|
||||
suppressor.reset();
|
||||
}
|
||||
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
|
||||
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
|
||||
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
|
||||
verifyCount = 0;
|
||||
if (!startG2Audio()) {
|
||||
emitStatus("Could not start G2 microphone input.");
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
audioStarted = true;
|
||||
}
|
||||
|
||||
emitStatus(currentMode == VoiceInputMode.CLOUD
|
||||
? "Listening (cloud)..."
|
||||
: currentMode == VoiceInputMode.ONBOARD
|
||||
? "Listening..."
|
||||
: "Listening for \"screen on\"...");
|
||||
: "Listening...");
|
||||
processG2Audio();
|
||||
// Verification result must precede the final transcript so the
|
||||
// TS bridge can suppress a non-wearer command before it is acted
|
||||
// on (the callbacks are posted in order to the main handler).
|
||||
runSpeakerVerification();
|
||||
// Button released / stop requested: emit one final full-utterance
|
||||
// transcript so the UI can freeze it.
|
||||
if (currentMode == VoiceInputMode.ONBOARD) {
|
||||
@@ -244,6 +343,7 @@ public class FaceclawVoiceController {
|
||||
releaseLc3();
|
||||
synchronized (lock) {
|
||||
started = false;
|
||||
audioStarted = false;
|
||||
workerThread = null;
|
||||
}
|
||||
}
|
||||
@@ -312,29 +412,6 @@ public class FaceclawVoiceController {
|
||||
return b.array();
|
||||
}
|
||||
|
||||
private KeywordSpotterConfig buildConfig(File modelDir) {
|
||||
return KeywordSpotterConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
.setSampleRate(SAMPLE_RATE)
|
||||
.setFeatureDim(FEATURE_DIM)
|
||||
.build())
|
||||
.setOnlineModelConfig(OnlineModelConfig.builder()
|
||||
.setTransducer(OnlineTransducerModelConfig.builder()
|
||||
.setEncoder(new File(modelDir, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
|
||||
.setDecoder(new File(modelDir, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
|
||||
.setJoiner(new File(modelDir, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx").getAbsolutePath())
|
||||
.build())
|
||||
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
|
||||
.setModelType("zipformer2")
|
||||
.setModelingUnit("")
|
||||
.setNumThreads(1)
|
||||
.build())
|
||||
.setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath())
|
||||
.setKeywordsScore(1.5f)
|
||||
.setKeywordsThreshold(0.35f)
|
||||
.build();
|
||||
}
|
||||
|
||||
private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) {
|
||||
return OfflineRecognizerConfig.builder()
|
||||
.setFeatureConfig(FeatureConfig.builder()
|
||||
@@ -352,50 +429,42 @@ public class FaceclawVoiceController {
|
||||
.build();
|
||||
}
|
||||
|
||||
private File installModelFiles() throws IOException {
|
||||
File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR);
|
||||
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
||||
throw new IOException("Could not create " + modelDir.getAbsolutePath());
|
||||
}
|
||||
AssetManager assets = appContext.getAssets();
|
||||
for (String fileName : MODEL_FILES) {
|
||||
copyAssetIfNeeded(
|
||||
assets,
|
||||
ASSET_ROOT + "/" + MODEL_DIR + "/" + fileName,
|
||||
new File(modelDir, fileName)
|
||||
);
|
||||
}
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
private File installAsrModelFiles() throws IOException {
|
||||
File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR);
|
||||
if (!modelDir.exists() && !modelDir.mkdirs()) {
|
||||
throw new IOException("Could not create " + modelDir.getAbsolutePath());
|
||||
}
|
||||
AssetManager assets = appContext.getAssets();
|
||||
/**
|
||||
* The Moonshine model directory, populated by the download flow in
|
||||
* asr-model.ts (releases before 0.5.0 copied the same files out of the
|
||||
* APK, so upgraded installs are already complete). Null when any file is
|
||||
* missing, i.e. the model still needs to be downloaded.
|
||||
*/
|
||||
private File findAsrModelDir() {
|
||||
File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR);
|
||||
for (String fileName : ASR_MODEL_FILES) {
|
||||
copyAssetIfNeeded(
|
||||
assets,
|
||||
ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName,
|
||||
new File(modelDir, fileName)
|
||||
);
|
||||
}
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException {
|
||||
if (destination.exists() && destination.length() > 0) {
|
||||
return;
|
||||
}
|
||||
try (InputStream input = assets.open(assetPath);
|
||||
FileOutputStream output = new FileOutputStream(destination)) {
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int read;
|
||||
while ((read = input.read(buffer)) >= 0) {
|
||||
output.write(buffer, 0, read);
|
||||
File file = new File(modelDir, fileName);
|
||||
if (!file.exists() || file.length() == 0) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return modelDir;
|
||||
}
|
||||
|
||||
private void deleteLegacyKwsFiles() {
|
||||
try {
|
||||
deleteRecursively(new File(appContext.getFilesDir(), LEGACY_KWS_ROOT));
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "failed to delete legacy wake-word files", t);
|
||||
}
|
||||
}
|
||||
|
||||
private static void deleteRecursively(File file) {
|
||||
if (!file.exists()) {
|
||||
return;
|
||||
}
|
||||
File[] children = file.listFiles();
|
||||
if (children != null) {
|
||||
for (File child : children) {
|
||||
deleteRecursively(child);
|
||||
}
|
||||
}
|
||||
file.delete();
|
||||
}
|
||||
|
||||
private boolean startG2Audio() {
|
||||
@@ -413,9 +482,8 @@ public class FaceclawVoiceController {
|
||||
private void processG2Audio() {
|
||||
short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET];
|
||||
while (started && !Thread.currentThread().isInterrupted()) {
|
||||
OnlineStream currentStream = stream;
|
||||
FaceclawLc3Decoder currentDecoder = lc3Decoder;
|
||||
if (currentDecoder == null || (mode == VoiceInputMode.WAKEWORD && currentStream == null)) {
|
||||
if (currentDecoder == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -430,48 +498,48 @@ public class FaceclawVoiceController {
|
||||
continue;
|
||||
}
|
||||
decodedSamples += count;
|
||||
// Global mic processing from the Microphones app config: the
|
||||
// beam filter drops packets whose firmware direction-of-arrival
|
||||
// falls outside the listening wedge (isolating the aimed talker
|
||||
// for every consumer, recognition included), and the spectral
|
||||
// noise suppressor cleans what remains before it reaches the
|
||||
// recognizer, cloud PCM, endpointing, or speaker verification.
|
||||
int angleDegrees = currentDecoder.getLastAngleDegrees();
|
||||
int ssr = currentDecoder.getLastSsr();
|
||||
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
|
||||
emitFrameMeta(angleDegrees, ssr);
|
||||
maybeEmitAudioStats(false);
|
||||
continue;
|
||||
}
|
||||
if (suppressionEnabled) {
|
||||
applySuppression(pcm, count);
|
||||
}
|
||||
if (recordingPcm != null) {
|
||||
appendRecording(pcm, count);
|
||||
}
|
||||
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
|
||||
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
|
||||
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
|
||||
verifyCount += copied;
|
||||
}
|
||||
if (endpointing && endpointDetector.accept(pcm, count)) {
|
||||
emitSpeechEnd();
|
||||
}
|
||||
if (mode == VoiceInputMode.CLOUD) {
|
||||
emitPcm(pcm, count);
|
||||
} else if (mode == VoiceInputMode.ONBOARD) {
|
||||
// PCM and frame metadata flow in every mode so levels, recording,
|
||||
// and the Microphones radar keep working alongside onboard ASR.
|
||||
emitPcm(pcm, count);
|
||||
emitFrameMeta(angleDegrees, ssr);
|
||||
if (mode != VoiceInputMode.CLOUD) {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
processRecognizer(samples);
|
||||
} else {
|
||||
float[] samples = new float[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
samples[i] = pcm[i] / 32768.0f;
|
||||
}
|
||||
currentStream.acceptWaveform(samples, SAMPLE_RATE);
|
||||
processKeywordSpotter(currentStream);
|
||||
}
|
||||
maybeEmitAudioStats(false);
|
||||
}
|
||||
}
|
||||
|
||||
private void processKeywordSpotter(OnlineStream currentStream) {
|
||||
KeywordSpotter currentSpotter = keywordSpotter;
|
||||
if (currentSpotter == null) {
|
||||
return;
|
||||
}
|
||||
while (currentSpotter.isReady(currentStream)) {
|
||||
currentSpotter.decode(currentStream);
|
||||
KeywordSpotterResult result = currentSpotter.getResult(currentStream);
|
||||
String keyword = result == null ? "" : result.getKeyword();
|
||||
if (keyword != null && keyword.trim().length() > 0) {
|
||||
currentSpotter.reset(currentStream);
|
||||
emitWakeWord(keyword);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void processRecognizer(float[] samples) {
|
||||
appendTranscriptSamples(samples);
|
||||
long now = SystemClock.elapsedRealtime();
|
||||
@@ -661,6 +729,83 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(() -> currentListener.onPcm(le));
|
||||
}
|
||||
|
||||
/**
|
||||
* Embed the session's buffered utterance and compare it to the enrolled
|
||||
* wearer voice-print. Fails open: a session too short to verify, or a
|
||||
* model that will not load, counts as the wearer rather than silencing
|
||||
* every command.
|
||||
*/
|
||||
private void runSpeakerVerification() {
|
||||
short[] buffer = verifyBuffer;
|
||||
float[] wearer = verifyWearerEmbedding;
|
||||
String modelPath = verifySpeakerModelPath;
|
||||
verifyBuffer = null;
|
||||
if (buffer == null || wearer == null || modelPath == null) {
|
||||
return;
|
||||
}
|
||||
if (verifyCount < VERIFY_MIN_SAMPLES) {
|
||||
emitSpeakerVerified(true, 0f);
|
||||
return;
|
||||
}
|
||||
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
|
||||
try {
|
||||
byte[] le = new byte[verifyCount * 2];
|
||||
for (int i = 0; i < verifyCount; i++) {
|
||||
short s = buffer[i];
|
||||
le[i * 2] = (byte) (s & 0xff);
|
||||
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
|
||||
}
|
||||
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
|
||||
if (embedding == null || embedding.length != wearer.length) {
|
||||
emitSpeakerVerified(true, 0f);
|
||||
return;
|
||||
}
|
||||
double dot = 0;
|
||||
for (int i = 0; i < embedding.length; i++) {
|
||||
dot += (double) embedding[i] * wearer[i];
|
||||
}
|
||||
boolean isWearer = dot >= verifyThreshold;
|
||||
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
|
||||
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
|
||||
emitSpeakerVerified(isWearer, (float) dot);
|
||||
} catch (Throwable t) {
|
||||
Log.w(TAG, "speaker verification failed", t);
|
||||
emitSpeakerVerified(true, 0f);
|
||||
}
|
||||
}
|
||||
|
||||
// The 28 MB embedding model takes seconds to load; keep one instance
|
||||
// across capture sessions so verification adds only the embed time.
|
||||
private static FaceclawSpeakerId sharedSpeakerId;
|
||||
private static String sharedSpeakerIdPath;
|
||||
|
||||
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
|
||||
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
|
||||
if (sharedSpeakerId != null) {
|
||||
sharedSpeakerId.close();
|
||||
}
|
||||
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
|
||||
sharedSpeakerIdPath = modelPath;
|
||||
}
|
||||
return sharedSpeakerId;
|
||||
}
|
||||
|
||||
private void emitSpeakerVerified(boolean isWearer, float similarity) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
|
||||
}
|
||||
|
||||
private void emitFrameMeta(int angleDegrees, int ssr) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
|
||||
}
|
||||
|
||||
private void emitSpeechEnd() {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
@@ -783,14 +928,6 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
|
||||
private void releaseSherpa() {
|
||||
if (stream != null) {
|
||||
stream.release();
|
||||
stream = null;
|
||||
}
|
||||
if (keywordSpotter != null) {
|
||||
keywordSpotter.release();
|
||||
keywordSpotter = null;
|
||||
}
|
||||
if (recognizer != null) {
|
||||
recognizer.release();
|
||||
recognizer = null;
|
||||
@@ -893,14 +1030,6 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(() -> currentListener.onStatus(status));
|
||||
}
|
||||
|
||||
private void emitWakeWord(String keyword) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
return;
|
||||
}
|
||||
mainHandler.post(() -> currentListener.onWakeWord(keyword));
|
||||
}
|
||||
|
||||
private void emitTranscript(String text, boolean isFinal) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener == null) {
|
||||
|
||||
@@ -3,8 +3,6 @@ package com.faceclaw.app;
|
||||
public interface FaceclawVoiceControllerListener {
|
||||
void onStatus(String status);
|
||||
|
||||
void onWakeWord(String keyword);
|
||||
|
||||
/**
|
||||
* Best transcript of the current utterance so far. REPLACE semantics: text
|
||||
* is the complete transcript, not a delta — display it verbatim, replacing
|
||||
@@ -13,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
|
||||
* little-endian PCM. Empty/absent in onboard mode.
|
||||
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
|
||||
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
|
||||
* keep working while the onboard recognizer owns the transcript.
|
||||
*/
|
||||
void onPcm(byte[] pcm16le);
|
||||
|
||||
/**
|
||||
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
|
||||
* packet, computed on the raw stereo capture before its mono downmix: the
|
||||
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
|
||||
* the wearer's right) and a signal-strength ratio that serves as a
|
||||
* confidence/energy proxy for the angle.
|
||||
*/
|
||||
void onFrameMeta(int angleDegrees, int ssr);
|
||||
|
||||
/**
|
||||
* The speaker stopped, in a hands-free session that has no button release
|
||||
* to end it. Only fires when endpointing was enabled for the session; see
|
||||
* FaceclawVoiceController.setEndpointing.
|
||||
*/
|
||||
void onSpeechEnd();
|
||||
|
||||
/**
|
||||
* Result of speaker verification against the enrolled wearer voice-print,
|
||||
* emitted once per capture session just before the final transcript.
|
||||
* Only fires when verification was configured for the session; fails
|
||||
* open (isWearer=true) when the utterance was too short to verify or the
|
||||
* speaker model could not be loaded.
|
||||
*/
|
||||
void onSpeakerVerified(boolean isWearer, float similarity);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* Streaming WAV writer for conversation recordings: the header is written up
|
||||
* front with zero sizes and patched on finish, so a crash mid-session leaves
|
||||
* a recoverable file (data length can be derived from the file size).
|
||||
*/
|
||||
public class FaceclawWavRecorder {
|
||||
private static final String TAG = "FaceclawWavRecorder";
|
||||
|
||||
private final String path;
|
||||
private final int sampleRate;
|
||||
private final int channels;
|
||||
private RandomAccessFile file;
|
||||
private long dataBytes;
|
||||
|
||||
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
|
||||
this.path = path;
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
File parent = new File(path).getParentFile();
|
||||
if (parent != null && !parent.exists() && !parent.mkdirs()) {
|
||||
throw new IOException("could not create " + parent);
|
||||
}
|
||||
file = new RandomAccessFile(path, "rw");
|
||||
file.setLength(0);
|
||||
file.write(buildHeader(0));
|
||||
}
|
||||
|
||||
public synchronized void append(byte[] pcm16le) {
|
||||
if (file == null || pcm16le == null || pcm16le.length == 0) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
file.write(pcm16le);
|
||||
dataBytes += pcm16le.length;
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "append failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized long getDurationMs() {
|
||||
return dataBytes * 1000L / (sampleRate * channels * 2L);
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
|
||||
public synchronized long finish() {
|
||||
RandomAccessFile current = file;
|
||||
file = null;
|
||||
if (current == null) {
|
||||
return dataBytes;
|
||||
}
|
||||
try {
|
||||
current.seek(0);
|
||||
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
|
||||
current.close();
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "finish failed", e);
|
||||
}
|
||||
return dataBytes;
|
||||
}
|
||||
|
||||
/** Delete the (finished or abandoned) recording. */
|
||||
public synchronized void discard() {
|
||||
finish();
|
||||
new File(path).delete();
|
||||
}
|
||||
|
||||
private byte[] buildHeader(int pcmBytes) {
|
||||
int byteRate = sampleRate * channels * 2;
|
||||
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
|
||||
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(36 + pcmBytes);
|
||||
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
|
||||
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(16);
|
||||
b.putShort((short) 1);
|
||||
b.putShort((short) channels);
|
||||
b.putInt(sampleRate);
|
||||
b.putInt(byteRate);
|
||||
b.putShort((short) (channels * 2));
|
||||
b.putShort((short) 16);
|
||||
b.put("data".getBytes(StandardCharsets.US_ASCII));
|
||||
b.putInt(pcmBytes);
|
||||
return b.array();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.content.Context;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.util.Log;
|
||||
|
||||
import com.google.android.gms.common.ConnectionResult;
|
||||
import com.google.android.gms.common.GoogleApiAvailability;
|
||||
import com.google.android.gms.wearable.CapabilityClient;
|
||||
import com.google.android.gms.wearable.CapabilityInfo;
|
||||
import com.google.android.gms.wearable.DataMap;
|
||||
import com.google.android.gms.wearable.MessageEvent;
|
||||
import com.google.android.gms.wearable.Node;
|
||||
import com.google.android.gms.wearable.PutDataMapRequest;
|
||||
import com.google.android.gms.wearable.PutDataRequest;
|
||||
import com.google.android.gms.wearable.Wearable;
|
||||
|
||||
import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Phone side of the Wear OS integration: the single place that talks to the
|
||||
* Wearable Data Layer (Google Play services), so the JS side only ever sees
|
||||
* JSON strings and never touches Play services types.
|
||||
*
|
||||
* Inbound: FaceclawWearListenerService hands every watch message to
|
||||
* {@link #handleMessage}; it is forwarded to the JS listener on the thread
|
||||
* that registered it (the main isolate), mirroring FaceclawSettings. Messages
|
||||
* that arrive before a listener exists (the watch poked a phone whose JS
|
||||
* dashboard is not up yet) are acked with jsReady=false so the watch can say
|
||||
* "open Faceclaw on your phone"; only idempotent state requests are kept for
|
||||
* replay on registration — the watch already reported gestures and commands
|
||||
* as failed, and executing them tens of seconds later would act on whatever
|
||||
* the glasses happen to show by then.
|
||||
*
|
||||
* Outbound: {@link #publishState} mirrors the dashboard state into a Data
|
||||
* Layer item (delivered even if the watch is out of range right now, and
|
||||
* available to the watch app the moment it opens), while {@link #sendToWatch}
|
||||
* sends fire-and-forget messages (acks, assistant text) to every reachable
|
||||
* watch that advertises the Faceclaw watch-app capability.
|
||||
*
|
||||
* Everything is a no-op on phones without Google Play services.
|
||||
*/
|
||||
public final class FaceclawWearBridge {
|
||||
private static final String TAG = "FaceclawWear";
|
||||
|
||||
/** Advertised by the watch app (wear/app/src/main/res/values/wear.xml). */
|
||||
public static final String CAPABILITY_WATCH = "faceclaw_watch";
|
||||
/** Advertised by this app (res/values/wear.xml) so the watch can find the phone. */
|
||||
public static final String CAPABILITY_PHONE = "faceclaw_phone";
|
||||
|
||||
public static final String PATH_PREFIX = "/faceclaw";
|
||||
public static final String PATH_STATE = "/faceclaw/state";
|
||||
public static final String PATH_STATE_REQUEST = "/faceclaw/state/request";
|
||||
public static final String PATH_ACK = "/faceclaw/ack";
|
||||
public static final String PATH_EVENT = "/faceclaw/event";
|
||||
|
||||
private static final String STATE_KEY_JSON = "json";
|
||||
private static final String STATE_KEY_UPDATED_AT = "updatedAt";
|
||||
|
||||
/** Messages kept for a listener that is not registered yet. */
|
||||
private static final int PENDING_LIMIT = 16;
|
||||
private static final long PENDING_MAX_AGE_MS = 30_000;
|
||||
|
||||
private static volatile FaceclawWearBridge instance;
|
||||
|
||||
private final Context context;
|
||||
private final boolean available;
|
||||
private final Object lock = new Object();
|
||||
private FaceclawWearListener listener;
|
||||
private Handler listenerHandler;
|
||||
private final ArrayDeque<PendingMessage> pending = new ArrayDeque<>();
|
||||
private final List<Node> watchNodes = new ArrayList<>();
|
||||
private String lastPublishedState = null;
|
||||
|
||||
private static final class PendingMessage {
|
||||
final String path;
|
||||
final String json;
|
||||
final String nodeId;
|
||||
final long receivedAt;
|
||||
|
||||
PendingMessage(String path, String json, String nodeId, long receivedAt) {
|
||||
this.path = path;
|
||||
this.json = json;
|
||||
this.nodeId = nodeId;
|
||||
this.receivedAt = receivedAt;
|
||||
}
|
||||
}
|
||||
|
||||
private FaceclawWearBridge(Context context) {
|
||||
this.context = context.getApplicationContext();
|
||||
boolean playServices;
|
||||
try {
|
||||
playServices = GoogleApiAvailability.getInstance()
|
||||
.isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS;
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "Play services availability check failed", error);
|
||||
playServices = false;
|
||||
}
|
||||
this.available = playServices;
|
||||
if (!available) {
|
||||
Log.i(TAG, "Google Play services unavailable; watch integration disabled");
|
||||
}
|
||||
}
|
||||
|
||||
/** Initialize (idempotent) and return the singleton. */
|
||||
public static FaceclawWearBridge getInstance(Context context) {
|
||||
if (instance == null) {
|
||||
synchronized (FaceclawWearBridge.class) {
|
||||
if (instance == null) {
|
||||
instance = new FaceclawWearBridge(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
/** Whether the Wearable Data Layer can be used on this phone at all. */
|
||||
public boolean isAvailable() {
|
||||
return available;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the JS listener. Must be called from the thread whose isolate
|
||||
* owns it; that thread's Looper is captured for dispatch. Replays any
|
||||
* recently queued watch messages and refreshes the reachable-watch set.
|
||||
*/
|
||||
public void setListener(FaceclawWearListener newListener) {
|
||||
Looper looper = Looper.myLooper();
|
||||
if (looper == null) {
|
||||
Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified");
|
||||
}
|
||||
List<PendingMessage> replay;
|
||||
synchronized (lock) {
|
||||
listener = newListener;
|
||||
listenerHandler = looper != null ? new Handler(looper) : null;
|
||||
replay = new ArrayList<>(pending);
|
||||
pending.clear();
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
for (PendingMessage message : replay) {
|
||||
if (now - message.receivedAt <= PENDING_MAX_AGE_MS) {
|
||||
dispatchMessage(message.path, message.json, message.nodeId);
|
||||
}
|
||||
}
|
||||
refreshWatchNodes();
|
||||
}
|
||||
|
||||
public void clearListener() {
|
||||
synchronized (lock) {
|
||||
listener = null;
|
||||
listenerHandler = null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isWatchReachable() {
|
||||
synchronized (lock) {
|
||||
return !watchNodes.isEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
/** Display name of a reachable watch ("" if none). */
|
||||
public String getWatchName() {
|
||||
synchronized (lock) {
|
||||
return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
|
||||
}
|
||||
}
|
||||
|
||||
/** Re-query which watches advertise the Faceclaw watch-app capability. */
|
||||
public void refreshWatchNodes() {
|
||||
if (!available) return;
|
||||
try {
|
||||
Wearable.getCapabilityClient(context)
|
||||
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
|
||||
.addOnSuccessListener(this::updateWatchNodes)
|
||||
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "capability query failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror the dashboard state to the watch. Identical consecutive states
|
||||
* are not re-sent (the Data Layer would drop them anyway).
|
||||
*/
|
||||
public void publishState(String json) {
|
||||
publishState(json, false);
|
||||
}
|
||||
|
||||
/** As {@link #publishState(String)}; `force` re-sends an unchanged state. */
|
||||
public void publishState(String json, boolean force) {
|
||||
if (!available || json == null) return;
|
||||
synchronized (lock) {
|
||||
if (!force && json.equals(lastPublishedState)) return;
|
||||
lastPublishedState = json;
|
||||
}
|
||||
try {
|
||||
PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE);
|
||||
DataMap map = request.getDataMap();
|
||||
map.putString(STATE_KEY_JSON, json);
|
||||
map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis());
|
||||
PutDataRequest put = request.asPutDataRequest().setUrgent();
|
||||
Wearable.getDataClient(context)
|
||||
.putDataItem(put)
|
||||
.addOnFailureListener(error -> Log.w(TAG, "state publish failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "state publish failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
/** Send a message to every reachable watch running the Faceclaw watch app. */
|
||||
public void sendToWatch(String path, String json) {
|
||||
if (!available) return;
|
||||
List<Node> targets;
|
||||
synchronized (lock) {
|
||||
targets = new ArrayList<>(watchNodes);
|
||||
}
|
||||
if (targets.isEmpty()) {
|
||||
// Nothing known yet (first message after boot): look the watch up, then send.
|
||||
try {
|
||||
Wearable.getCapabilityClient(context)
|
||||
.getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE)
|
||||
.addOnSuccessListener(info -> {
|
||||
updateWatchNodes(info);
|
||||
for (Node node : info.getNodes()) {
|
||||
sendToNode(node.getId(), path, json);
|
||||
}
|
||||
})
|
||||
.addOnFailureListener(error -> Log.w(TAG, "capability query failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "capability query failed", error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (Node node : targets) {
|
||||
sendToNode(node.getId(), path, json);
|
||||
}
|
||||
}
|
||||
|
||||
/** Send a message to one specific node (e.g. an ack back to the sender). */
|
||||
public void sendToNode(String nodeId, String path, String json) {
|
||||
if (!available || nodeId == null || nodeId.isEmpty()) return;
|
||||
try {
|
||||
byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8);
|
||||
Wearable.getMessageClient(context)
|
||||
.sendMessage(nodeId, path, payload)
|
||||
.addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error));
|
||||
} catch (Throwable error) {
|
||||
Log.w(TAG, "send " + path + " failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reply to a watch message; `seq` echoes the watch's sequence number. */
|
||||
public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) {
|
||||
JSONObject ack = new JSONObject();
|
||||
try {
|
||||
ack.put("seq", seq);
|
||||
ack.put("ok", ok);
|
||||
ack.put("jsReady", jsReady);
|
||||
ack.put("message", message != null ? message : "");
|
||||
} catch (JSONException ignored) {
|
||||
// The keys are constants; this cannot happen.
|
||||
}
|
||||
sendToNode(nodeId, PATH_ACK, ack.toString());
|
||||
}
|
||||
|
||||
/** Entry point for FaceclawWearListenerService; may run on any thread. */
|
||||
void handleMessage(MessageEvent event) {
|
||||
if (event == null) return;
|
||||
String path = event.getPath();
|
||||
if (path == null || !path.startsWith(PATH_PREFIX)) return;
|
||||
byte[] data = event.getData();
|
||||
String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}";
|
||||
String nodeId = event.getSourceNodeId();
|
||||
Log.d(TAG, "recv " + path + " seq=" + readSeq(json));
|
||||
|
||||
boolean delivered = dispatchMessage(path, json, nodeId);
|
||||
if (delivered) return;
|
||||
|
||||
// No JS listener yet: tell the watch so it can explain why nothing is
|
||||
// happening. Only state requests are kept for replay (see class doc);
|
||||
// everything else is dropped after the failure ack.
|
||||
if (PATH_STATE_REQUEST.equals(path)) {
|
||||
synchronized (lock) {
|
||||
long now = System.currentTimeMillis();
|
||||
while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) {
|
||||
pending.pollFirst();
|
||||
}
|
||||
if (pending.size() >= PENDING_LIMIT) {
|
||||
pending.pollFirst();
|
||||
}
|
||||
pending.addLast(new PendingMessage(path, json, nodeId, now));
|
||||
}
|
||||
}
|
||||
sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses.");
|
||||
}
|
||||
|
||||
/** Entry point for FaceclawWearListenerService's capability callback. */
|
||||
void handleCapabilityChanged(CapabilityInfo info) {
|
||||
if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return;
|
||||
updateWatchNodes(info);
|
||||
}
|
||||
|
||||
private void updateWatchNodes(CapabilityInfo info) {
|
||||
Set<Node> nodes = info != null ? info.getNodes() : Collections.<Node>emptySet();
|
||||
// Every reachable watch gets messages; a nearby (Bluetooth-linked)
|
||||
// one is listed first so it is the name the phone UI shows.
|
||||
List<Node> reachable = new ArrayList<>();
|
||||
for (Node node : nodes) {
|
||||
if (node.isNearby()) reachable.add(0, node); else reachable.add(node);
|
||||
}
|
||||
boolean changed;
|
||||
String name;
|
||||
synchronized (lock) {
|
||||
changed = !sameNodeIds(watchNodes, reachable);
|
||||
watchNodes.clear();
|
||||
watchNodes.addAll(reachable);
|
||||
name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName();
|
||||
}
|
||||
if (changed) {
|
||||
Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name);
|
||||
boolean reachableNow = !reachable.isEmpty();
|
||||
FaceclawWearListener target;
|
||||
Handler handler;
|
||||
synchronized (lock) {
|
||||
target = listener;
|
||||
handler = listenerHandler;
|
||||
}
|
||||
if (target != null && handler != null) {
|
||||
handler.post(() -> {
|
||||
try {
|
||||
target.onWatchConnection(reachableNow, name);
|
||||
} catch (Exception error) {
|
||||
Log.w(TAG, "watch connection listener failed", error);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean sameNodeIds(List<Node> a, List<Node> b) {
|
||||
if (a.size() != b.size()) return false;
|
||||
for (int i = 0; i < a.size(); i++) {
|
||||
if (!a.get(i).getId().equals(b.get(i).getId())) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Post to the JS listener; false when there is none to post to. */
|
||||
private boolean dispatchMessage(String path, String json, String nodeId) {
|
||||
FaceclawWearListener target;
|
||||
Handler handler;
|
||||
synchronized (lock) {
|
||||
target = listener;
|
||||
handler = listenerHandler;
|
||||
}
|
||||
if (target == null || handler == null) return false;
|
||||
handler.post(() -> {
|
||||
try {
|
||||
target.onMessage(path, json, nodeId);
|
||||
} catch (Exception error) {
|
||||
Log.w(TAG, "wear message listener failed for " + path, error);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
private static long readSeq(String json) {
|
||||
try {
|
||||
return new JSONObject(json).optLong("seq", 0);
|
||||
} catch (JSONException error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
/**
|
||||
* Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every
|
||||
* call is delivered on the thread that registered the listener (see
|
||||
* FaceclawWearBridge.setListener).
|
||||
*/
|
||||
public interface FaceclawWearListener {
|
||||
/**
|
||||
* A message from the watch app. `path` is the Data Layer message path
|
||||
* (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON
|
||||
* object, "{}" when the watch sent none); `nodeId` identifies the watch
|
||||
* so a reply can be addressed to it.
|
||||
*/
|
||||
void onMessage(String path, String json, String nodeId);
|
||||
|
||||
/** The set of reachable watches running the Faceclaw watch app changed. */
|
||||
void onWatchConnection(boolean reachable, String watchName);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import com.google.android.gms.wearable.CapabilityInfo;
|
||||
import com.google.android.gms.wearable.MessageEvent;
|
||||
import com.google.android.gms.wearable.WearableListenerService;
|
||||
|
||||
/**
|
||||
* Play services starts this service (in the app process) for every Data
|
||||
* Layer message under /faceclaw and for capability changes, whether or not
|
||||
* the app is running. It only forwards to FaceclawWearBridge, which owns the
|
||||
* hand-off to the JS side.
|
||||
*/
|
||||
public class FaceclawWearListenerService extends WearableListenerService {
|
||||
@Override
|
||||
public void onMessageReceived(MessageEvent event) {
|
||||
Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId());
|
||||
FaceclawWearBridge.getInstance(this).handleMessage(event);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onCapabilityChanged(CapabilityInfo info) {
|
||||
FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info);
|
||||
}
|
||||
}
|
||||
@@ -15,13 +15,18 @@ public final class PreviewBitmapUtil {
|
||||
private PreviewBitmapUtil() {}
|
||||
|
||||
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma) {
|
||||
return fromGray(gray, width, height, brightenGamma, false);
|
||||
}
|
||||
|
||||
/** As above; `green` renders green-on-black (matching the physical glasses) instead of grayscale. */
|
||||
public static Bitmap fromGray(ByteBuffer gray, int width, int height, double brightenGamma, boolean green) {
|
||||
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
|
||||
throw new IllegalArgumentException("invalid gray preview buffer");
|
||||
}
|
||||
int[] lut = new int[256];
|
||||
for (int g = 0; g < 256; g++) {
|
||||
int v = (int) Math.max(0, Math.min(255, Math.round(255 * Math.pow(g / 255.0, brightenGamma))));
|
||||
lut[g] = 0xff000000 | (v << 16) | (v << 8) | v;
|
||||
lut[g] = green ? (0xff000000 | (v << 8)) : (0xff000000 | (v << 16) | (v << 8) | v);
|
||||
}
|
||||
byte[] bytes = new byte[width * height];
|
||||
gray.get(bytes);
|
||||
|
||||
@@ -28,6 +28,16 @@ public class BleProtocol {
|
||||
|
||||
public static final int PRELUDE_ACK_SID = 0x01;
|
||||
public static final int PRELUDE_ACK_MAGIC = 156;
|
||||
/**
|
||||
* DEVICE_SETTINGS security-auth channel. Firmware 2.2.9 enforces completing
|
||||
* this exchange shortly after a GATT connection opens (a ~30 s deadline
|
||||
* closes unauthenticated links) and gates query responses on it; the
|
||||
* message itself exists back to the 2.2.4 era, so sending it is safe on all
|
||||
* stock versions. See ../notes/ble-connections-2.2.9.md.
|
||||
*/
|
||||
public static final int SID_SECURITY_AUTH = 0x80;
|
||||
public static final int FLAG_SECURITY_AUTH = 0x00;
|
||||
private static final int SECURITY_AUTH_CMD = 4;
|
||||
public static final int SID_APP_LAUNCH = 0x01;
|
||||
public static final int SID_EVENHUB = 0xe0;
|
||||
public static final int SID_UI_SETTING = 0x09;
|
||||
@@ -39,6 +49,14 @@ public class BleProtocol {
|
||||
// protobuf fields to stock implementations, so they are safely ignored.
|
||||
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
|
||||
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
|
||||
/**
|
||||
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
|
||||
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
|
||||
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
|
||||
* to settings read responses and as standalone commandId=3 pushes.
|
||||
*/
|
||||
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
|
||||
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
|
||||
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
|
||||
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
|
||||
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
|
||||
@@ -323,6 +341,48 @@ public class BleProtocol {
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* The application authentication request: DEVICE_SETTINGS/AUTHENTICATION(4)
|
||||
* on sid 0x80, flag 0x00. Carries no key material — it arms the firmware's
|
||||
* "security auth" flag; the firmware sends the success notification once
|
||||
* the flag is set AND the BLE link is encrypted (SMP pairing/bond
|
||||
* restoration, handled by the OS below GATT, possibly after a pairing
|
||||
* prompt). Protobuf: f1=4 (command), f2=magic, f3={f1=1, f2=4}.
|
||||
*/
|
||||
public static byte[] buildAuthenticationRequest(int magic) {
|
||||
byte[] auth = concat(CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1),
|
||||
encodeVarintField(2, SECURITY_AUTH_CMD)
|
||||
));
|
||||
return concat(CollectionUtils.listOf(
|
||||
encodeVarintField(1, SECURITY_AUTH_CMD),
|
||||
encodeVarintField(2, magic),
|
||||
encodeMessageField(3, auth)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* True when `pb` (trailing CRC still attached) is the success notification
|
||||
* for the authentication request sent with `magic`: command 4, the same
|
||||
* magic, and an EMPTY field-3 result message (`1a 00` — the omitted result
|
||||
* code defaults to zero). A GATT write completing is NOT success; only this
|
||||
* notification is.
|
||||
*/
|
||||
public static boolean isAuthenticationSuccess(byte[] pb, int magic) {
|
||||
if (pb == null) {
|
||||
return false;
|
||||
}
|
||||
byte[] root = stripTrailingCrc(pb);
|
||||
if (readVarintFieldValue(root, 1, -1) != SECURITY_AUTH_CMD) {
|
||||
return false;
|
||||
}
|
||||
if (readVarintFieldValue(root, 2, -1) != magic) {
|
||||
return false;
|
||||
}
|
||||
byte[] result = readFieldBytes(root, 3);
|
||||
return result != null && result.length == 0;
|
||||
}
|
||||
|
||||
public static byte[] buildSettingsQuery(int magic) {
|
||||
byte[] request = encodeVarintField(1, 1);
|
||||
return concat(CollectionUtils.listOf(
|
||||
@@ -353,6 +413,40 @@ public class BleProtocol {
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
|
||||
* settings write carrying CFW field 103. Magic zero makes it
|
||||
* fire-and-forget like the wake lease: the firmware consumes the field
|
||||
* before its stock nanopb decoder discards the unknown tag, and the
|
||||
* "ack" is the field-104 status notify that follows.
|
||||
*/
|
||||
public static byte[] buildFaceclawMicControl(byte[] record) {
|
||||
return concat(CollectionUtils.listOf(
|
||||
encodeVarintField(1, 1),
|
||||
encodeVarintField(2, 0),
|
||||
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the CFW mic status record (field 104) from a settings frame, or
|
||||
* null when absent. The 21-byte body layout is decoded on the TS side;
|
||||
* here we only validate the 'M','C',version prelude.
|
||||
*/
|
||||
public static byte[] parseFaceclawMicStatus(byte[] pb) {
|
||||
if (pb == null) {
|
||||
return null;
|
||||
}
|
||||
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
|
||||
if (body == null
|
||||
|| body.length < 3
|
||||
|| body[0] != 'M'
|
||||
|| body[1] != 'C') {
|
||||
return null;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
|
||||
* when this is an ordinary settings frame.
|
||||
@@ -552,6 +646,35 @@ public class BleProtocol {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Split one notification value into individual `aa21`/`aa12` envelope
|
||||
* frames. A value normally holds exactly one frame, but the firmware can
|
||||
* pack several (each self-describing via the length byte at offset 3), and
|
||||
* a frame that only reads the first would silently drop the rest. A value
|
||||
* that does not start with an envelope, or a trailing partial frame
|
||||
* (truncated by the stack — observed on a Samsung tablet for values over
|
||||
* 64 bytes), is returned as-is so the caller can log it.
|
||||
*/
|
||||
public static List<byte[]> splitFrames(byte[] value) {
|
||||
List<byte[]> out = new ArrayList<>();
|
||||
if (value == null || value.length == 0) {
|
||||
return out;
|
||||
}
|
||||
int offset = 0;
|
||||
while (offset + 8 <= value.length
|
||||
&& value[offset] == (byte) 0xaa
|
||||
&& (value[offset + 1] == 0x21 || value[offset + 1] == 0x12)) {
|
||||
int len = value[offset + 3] & 0xff;
|
||||
int end = Math.min(value.length, offset + 8 + len);
|
||||
out.add(Arrays.copyOfRange(value, offset, end));
|
||||
offset = end;
|
||||
}
|
||||
if (offset < value.length) {
|
||||
out.add(Arrays.copyOfRange(value, offset, value.length));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
public static ParsedFrame parseFrame(byte[] buf) {
|
||||
if (buf == null || buf.length < 10 || buf[0] != (byte) 0xaa || (buf[1] != 0x21 && buf[1] != 0x12)) {
|
||||
return new ParsedFrame(false, 0, 0, new byte[0], -1, -1);
|
||||
|
||||
@@ -12,6 +12,14 @@ public class ConnectionOptions {
|
||||
public static final int RING_RECONNECT_DELAY_MS = 2_000;
|
||||
public static final int WRITE_TIMEOUT_MS = 2_000;
|
||||
public static final int PRELUDE_TIMEOUT_MS = 2_000;
|
||||
// Application security-auth exchange (sid 0x80). The success notification
|
||||
// only arrives once the BLE link is encrypted, which on a first-time
|
||||
// connection means SMP pairing — possibly with an OS dialog the user has
|
||||
// to accept — so the stock-firmware paths wait generously.
|
||||
public static final int SECURITY_AUTH_TIMEOUT_MS = 30_000;
|
||||
// The main communicator's wait is soft (it proceeds on timeout so firmware
|
||||
// that never answers can't stall every reconnect), so it is kept short.
|
||||
public static final int SECURITY_AUTH_SOFT_TIMEOUT_MS = 6_000;
|
||||
public static final int ACK_TIMEOUT_MS = 3_500;
|
||||
public static final int HEARTBEAT_FAILURE_DEADLINE_MS = 10_000;
|
||||
public static final int HEARTBEAT_READY_MS = 4_000;
|
||||
|
||||
@@ -23,6 +23,22 @@ public class MessageBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
/** The per-connection sid-0x80 security-auth request (one per arm). */
|
||||
public OutboundMessage securityAuth(boolean leftArm) {
|
||||
int magic = magicPool.allocate();
|
||||
return new OutboundMessage(
|
||||
"security-auth",
|
||||
"security auth " + (leftArm ? "L" : "R"),
|
||||
BleProtocol.SID_SECURITY_AUTH,
|
||||
BleProtocol.FLAG_SECURITY_AUTH,
|
||||
magic,
|
||||
BleProtocol.buildAuthenticationRequest(magic),
|
||||
ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS,
|
||||
-1,
|
||||
leftArm
|
||||
);
|
||||
}
|
||||
|
||||
public OutboundMessage shutdown(int exitMode) {
|
||||
int magic = magicPool.allocate();
|
||||
return new OutboundMessage(
|
||||
@@ -251,6 +267,21 @@ public class MessageBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
/** CFW mic_control write (settings field 103) targeted at one temple. */
|
||||
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
|
||||
return new OutboundMessage(
|
||||
"mic-control",
|
||||
"mic control " + label + (leftArm ? " L" : " R"),
|
||||
BleProtocol.SID_UI_SETTING,
|
||||
BleProtocol.FLAG_REQUEST,
|
||||
0,
|
||||
BleProtocol.buildFaceclawMicControl(record),
|
||||
ACK_TIMEOUT_MS,
|
||||
-1,
|
||||
leftArm
|
||||
);
|
||||
}
|
||||
|
||||
public OutboundMessage faceclawWearQuery(boolean leftArm) {
|
||||
return new OutboundMessage(
|
||||
"wear-query-control",
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class FastClusteringConfig {
|
||||
private final int numClusters;
|
||||
private final float threshold;
|
||||
|
||||
private FastClusteringConfig(Builder builder) {
|
||||
this.numClusters = builder.numClusters;
|
||||
this.threshold = builder.threshold;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public int getNumClusters() {
|
||||
return numClusters;
|
||||
}
|
||||
|
||||
public float getThreshold() {
|
||||
return threshold;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private int numClusters = -1;
|
||||
private float threshold = 0.5f;
|
||||
|
||||
public FastClusteringConfig build() {
|
||||
return new FastClusteringConfig(this);
|
||||
}
|
||||
|
||||
public Builder setNumClusters(int numClusters) {
|
||||
this.numClusters = numClusters;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setThreshold(float threshold) {
|
||||
this.threshold = threshold;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class KeywordSpotter {
|
||||
private long ptr;
|
||||
|
||||
public KeywordSpotter(KeywordSpotterConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid KeywordSpotterConfig");
|
||||
}
|
||||
}
|
||||
|
||||
public OnlineStream createStream() {
|
||||
return createStream("");
|
||||
}
|
||||
|
||||
public OnlineStream createStream(String keywords) {
|
||||
return new OnlineStream(createStream(ptr, keywords == null ? "" : keywords));
|
||||
}
|
||||
|
||||
public boolean isReady(OnlineStream stream) {
|
||||
return stream != null && stream.getPtr() != 0 && isReady(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public void decode(OnlineStream stream) {
|
||||
decode(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public void reset(OnlineStream stream) {
|
||||
reset(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public KeywordSpotterResult getResult(OnlineStream stream) {
|
||||
return getResult(ptr, stream.getPtr());
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(ptr);
|
||||
ptr = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private native long newFromFile(KeywordSpotterConfig config);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long createStream(long ptr, String keywords);
|
||||
|
||||
private native boolean isReady(long ptr, long streamPtr);
|
||||
|
||||
private native void decode(long ptr, long streamPtr);
|
||||
|
||||
private native void reset(long ptr, long streamPtr);
|
||||
|
||||
private native KeywordSpotterResult getResult(long ptr, long streamPtr);
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class KeywordSpotterConfig {
|
||||
private final FeatureConfig featConfig;
|
||||
private final OnlineModelConfig modelConfig;
|
||||
private final int maxActivePaths;
|
||||
private final String keywordsFile;
|
||||
private final float keywordsScore;
|
||||
private final float keywordsThreshold;
|
||||
private final int numTrailingBlanks;
|
||||
|
||||
private KeywordSpotterConfig(Builder builder) {
|
||||
this.featConfig = builder.featConfig;
|
||||
this.modelConfig = builder.modelConfig;
|
||||
this.maxActivePaths = builder.maxActivePaths;
|
||||
this.keywordsFile = builder.keywordsFile;
|
||||
this.keywordsScore = builder.keywordsScore;
|
||||
this.keywordsThreshold = builder.keywordsThreshold;
|
||||
this.numTrailingBlanks = builder.numTrailingBlanks;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public FeatureConfig getFeatConfig() {
|
||||
return featConfig;
|
||||
}
|
||||
|
||||
public OnlineModelConfig getModelConfig() {
|
||||
return modelConfig;
|
||||
}
|
||||
|
||||
public int getMaxActivePaths() {
|
||||
return maxActivePaths;
|
||||
}
|
||||
|
||||
public String getKeywordsFile() {
|
||||
return keywordsFile;
|
||||
}
|
||||
|
||||
public float getKeywordsScore() {
|
||||
return keywordsScore;
|
||||
}
|
||||
|
||||
public float getKeywordsThreshold() {
|
||||
return keywordsThreshold;
|
||||
}
|
||||
|
||||
public int getNumTrailingBlanks() {
|
||||
return numTrailingBlanks;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private FeatureConfig featConfig = FeatureConfig.builder().build();
|
||||
private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build();
|
||||
private int maxActivePaths = 4;
|
||||
private String keywordsFile = "keywords.txt";
|
||||
private float keywordsScore = 1.5f;
|
||||
private float keywordsThreshold = 0.25f;
|
||||
private int numTrailingBlanks = 2;
|
||||
|
||||
public Builder setFeatureConfig(FeatureConfig featConfig) {
|
||||
this.featConfig = featConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) {
|
||||
this.modelConfig = modelConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setKeywordsFile(String keywordsFile) {
|
||||
this.keywordsFile = keywordsFile;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setKeywordsScore(float keywordsScore) {
|
||||
this.keywordsScore = keywordsScore;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setKeywordsThreshold(float keywordsThreshold) {
|
||||
this.keywordsThreshold = keywordsThreshold;
|
||||
return this;
|
||||
}
|
||||
|
||||
public KeywordSpotterConfig build() {
|
||||
return new KeywordSpotterConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class KeywordSpotterResult {
|
||||
private final String keyword;
|
||||
private final String[] tokens;
|
||||
private final float[] timestamps;
|
||||
|
||||
public KeywordSpotterResult(String keyword, String[] tokens, float[] timestamps) {
|
||||
this.keyword = keyword == null ? "" : keyword;
|
||||
this.tokens = tokens == null ? new String[0] : tokens;
|
||||
this.timestamps = timestamps == null ? new float[0] : timestamps;
|
||||
}
|
||||
|
||||
public String getKeyword() {
|
||||
return keyword;
|
||||
}
|
||||
|
||||
public String[] getTokens() {
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public float[] getTimestamps() {
|
||||
return timestamps;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarization {
|
||||
private long ptr = 0;
|
||||
|
||||
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
|
||||
}
|
||||
}
|
||||
|
||||
public int getSampleRate() {
|
||||
return getSampleRate(ptr);
|
||||
}
|
||||
|
||||
// Only config.clustering is used. All other fields are ignored
|
||||
public void setConfig(OfflineSpeakerDiarizationConfig config) {
|
||||
setConfig(ptr, config);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
|
||||
return process(ptr, samples);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
|
||||
return processWithCallback(ptr, samples, callback, 0);
|
||||
}
|
||||
|
||||
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
|
||||
return processWithCallback(ptr, samples, callback, arg);
|
||||
}
|
||||
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
// You'd better call it manually if it is not used anymore
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
private native int getSampleRate(long ptr);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
|
||||
|
||||
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
|
||||
|
||||
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
|
||||
|
||||
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface OfflineSpeakerDiarizationCallback {
|
||||
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarizationConfig {
|
||||
private final OfflineSpeakerSegmentationModelConfig segmentation;
|
||||
private final SpeakerEmbeddingExtractorConfig embedding;
|
||||
private final FastClusteringConfig clustering;
|
||||
private final float minDurationOn;
|
||||
private final float minDurationOff;
|
||||
|
||||
private OfflineSpeakerDiarizationConfig(Builder builder) {
|
||||
this.segmentation = builder.segmentation;
|
||||
this.embedding = builder.embedding;
|
||||
this.clustering = builder.clustering;
|
||||
this.minDurationOff = builder.minDurationOff;
|
||||
this.minDurationOn = builder.minDurationOn;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
|
||||
return segmentation;
|
||||
}
|
||||
|
||||
public SpeakerEmbeddingExtractorConfig getEmbedding() {
|
||||
return embedding;
|
||||
}
|
||||
|
||||
public FastClusteringConfig getClustering() {
|
||||
return clustering;
|
||||
}
|
||||
|
||||
public float getMinDurationOff() {
|
||||
return minDurationOff;
|
||||
}
|
||||
|
||||
public float getMinDurationOn() {
|
||||
return minDurationOn;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
|
||||
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
|
||||
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
|
||||
private float minDurationOn = 0.2f;
|
||||
private float minDurationOff = 0.5f;
|
||||
|
||||
public OfflineSpeakerDiarizationConfig build() {
|
||||
return new OfflineSpeakerDiarizationConfig(this);
|
||||
}
|
||||
|
||||
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
|
||||
this.segmentation = segmentation;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
|
||||
this.embedding = embedding;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setClustering(FastClusteringConfig clustering) {
|
||||
this.clustering = clustering;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMinDurationOff(float minDurationOff) {
|
||||
this.minDurationOff = minDurationOff;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setMinDurationOn(float minDurationOn) {
|
||||
this.minDurationOn = minDurationOn;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerDiarizationSegment {
|
||||
private final float start;
|
||||
private final float end;
|
||||
private final int speaker;
|
||||
|
||||
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
this.speaker = speaker;
|
||||
}
|
||||
|
||||
public float getStart() {
|
||||
return start;
|
||||
}
|
||||
|
||||
public float getEnd() {
|
||||
return end;
|
||||
}
|
||||
|
||||
public int getSpeaker() {
|
||||
return speaker;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OfflineSpeakerSegmentationModelConfig {
|
||||
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
|
||||
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
|
||||
this.pyannote = builder.pyannote;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
|
||||
private int numThreads = 1;
|
||||
private boolean debug = true;
|
||||
private String provider = "cpu";
|
||||
|
||||
public OfflineSpeakerSegmentationModelConfig build() {
|
||||
return new OfflineSpeakerSegmentationModelConfig(this);
|
||||
}
|
||||
|
||||
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
|
||||
this.pyannote = pyannote;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDebug(boolean debug) {
|
||||
this.debug = debug;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineToneCtcModelConfig {
|
||||
public class OfflineSpeakerSegmentationPyannoteModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OnlineToneCtcModelConfig(Builder builder) {
|
||||
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
@@ -18,13 +20,13 @@ public class OnlineToneCtcModelConfig {
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
|
||||
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
|
||||
}
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineToneCtcModelConfig build() {
|
||||
return new OnlineToneCtcModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineModelConfig {
|
||||
private final OnlineTransducerModelConfig transducer;
|
||||
private final OnlineParaformerModelConfig paraformer;
|
||||
private final OnlineZipformer2CtcModelConfig zipformer2Ctc;
|
||||
private final OnlineNeMoCtcModelConfig neMoCtc;
|
||||
private final OnlineToneCtcModelConfig toneCtc;
|
||||
private final String tokens;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
private final String modelType;
|
||||
private final String modelingUnit;
|
||||
private final String bpeVocab;
|
||||
|
||||
private OnlineModelConfig(Builder builder) {
|
||||
this.transducer = builder.transducer;
|
||||
this.paraformer = builder.paraformer;
|
||||
this.zipformer2Ctc = builder.zipformer2Ctc;
|
||||
this.neMoCtc = builder.neMoCtc;
|
||||
this.toneCtc = builder.toneCtc;
|
||||
this.tokens = builder.tokens;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
this.modelType = builder.modelType;
|
||||
this.modelingUnit = builder.modelingUnit;
|
||||
this.bpeVocab = builder.bpeVocab;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public OnlineTransducerModelConfig getTransducer() {
|
||||
return transducer;
|
||||
}
|
||||
|
||||
public OnlineParaformerModelConfig getParaformer() {
|
||||
return paraformer;
|
||||
}
|
||||
|
||||
public OnlineZipformer2CtcModelConfig getZipformer2Ctc() {
|
||||
return zipformer2Ctc;
|
||||
}
|
||||
|
||||
public OnlineNeMoCtcModelConfig getNeMoCtc() {
|
||||
return neMoCtc;
|
||||
}
|
||||
|
||||
public OnlineToneCtcModelConfig getToneCtc() {
|
||||
return toneCtc;
|
||||
}
|
||||
|
||||
public String getTokens() {
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public int getNumThreads() {
|
||||
return numThreads;
|
||||
}
|
||||
|
||||
public boolean getDebug() {
|
||||
return debug;
|
||||
}
|
||||
|
||||
public String getProvider() {
|
||||
return provider;
|
||||
}
|
||||
|
||||
public String getModelType() {
|
||||
return modelType;
|
||||
}
|
||||
|
||||
public String getModelingUnit() {
|
||||
return modelingUnit;
|
||||
}
|
||||
|
||||
public String getBpeVocab() {
|
||||
return bpeVocab;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private OnlineTransducerModelConfig transducer = OnlineTransducerModelConfig.builder().build();
|
||||
private OnlineParaformerModelConfig paraformer = OnlineParaformerModelConfig.builder().build();
|
||||
private OnlineZipformer2CtcModelConfig zipformer2Ctc = OnlineZipformer2CtcModelConfig.builder().build();
|
||||
private OnlineNeMoCtcModelConfig neMoCtc = OnlineNeMoCtcModelConfig.builder().build();
|
||||
private OnlineToneCtcModelConfig toneCtc = OnlineToneCtcModelConfig.builder().build();
|
||||
private String tokens = "";
|
||||
private int numThreads = 1;
|
||||
private boolean debug = false;
|
||||
private String provider = "cpu";
|
||||
private String modelType = "";
|
||||
private String modelingUnit = "cjkchar";
|
||||
private String bpeVocab = "";
|
||||
|
||||
public Builder setTransducer(OnlineTransducerModelConfig transducer) {
|
||||
this.transducer = transducer;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setTokens(String tokens) {
|
||||
this.tokens = tokens;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setModelType(String modelType) {
|
||||
this.modelType = modelType;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setModelingUnit(String modelingUnit) {
|
||||
this.modelingUnit = modelingUnit;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setBpeVocab(String bpeVocab) {
|
||||
this.bpeVocab = bpeVocab;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineModelConfig build() {
|
||||
return new OnlineModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineNeMoCtcModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OnlineNeMoCtcModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineNeMoCtcModelConfig build() {
|
||||
return new OnlineNeMoCtcModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineParaformerModelConfig {
|
||||
private final String encoder;
|
||||
private final String decoder;
|
||||
|
||||
private OnlineParaformerModelConfig(Builder builder) {
|
||||
this.encoder = builder.encoder;
|
||||
this.decoder = builder.decoder;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getEncoder() {
|
||||
return encoder;
|
||||
}
|
||||
|
||||
public String getDecoder() {
|
||||
return decoder;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String encoder = "";
|
||||
private String decoder = "";
|
||||
|
||||
public Builder setEncoder(String encoder) {
|
||||
this.encoder = encoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDecoder(String decoder) {
|
||||
this.decoder = decoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineParaformerModelConfig build() {
|
||||
return new OnlineParaformerModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,16 @@
|
||||
// Copyright 2022-2023 by zhaoming
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineStream {
|
||||
private long ptr;
|
||||
private long ptr = 0;
|
||||
|
||||
public OnlineStream() {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public OnlineStream(long ptr) {
|
||||
this.ptr = ptr;
|
||||
@@ -11,25 +20,58 @@ public class OnlineStream {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
public void setPtr(long ptr) {
|
||||
this.ptr = ptr;
|
||||
}
|
||||
|
||||
public void acceptWaveform(float[] samples, int sampleRate) {
|
||||
acceptWaveform(ptr, samples, sampleRate);
|
||||
acceptWaveform(this.ptr, samples, sampleRate);
|
||||
}
|
||||
|
||||
public void inputFinished() {
|
||||
inputFinished(this.ptr);
|
||||
}
|
||||
|
||||
public void setOption(String key, String value) {
|
||||
setOption(this.ptr, key, value);
|
||||
}
|
||||
|
||||
public String getOption(String key) {
|
||||
return getOption(this.ptr, key);
|
||||
}
|
||||
|
||||
public boolean hasOption(String key) {
|
||||
return hasOption(this.ptr, key);
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(ptr);
|
||||
ptr = 0;
|
||||
close();
|
||||
}
|
||||
|
||||
|
||||
public void close() {
|
||||
// stream object must be release after used
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
close();
|
||||
super.finalize();
|
||||
}
|
||||
|
||||
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
|
||||
|
||||
private native void inputFinished(long ptr);
|
||||
|
||||
private native void setOption(long ptr, String key, String value);
|
||||
|
||||
private native String getOption(long ptr, String key);
|
||||
|
||||
private native boolean hasOption(long ptr, String key);
|
||||
|
||||
private native void delete(long ptr);
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineTransducerModelConfig {
|
||||
private final String encoder;
|
||||
private final String decoder;
|
||||
private final String joiner;
|
||||
|
||||
private OnlineTransducerModelConfig(Builder builder) {
|
||||
this.encoder = builder.encoder;
|
||||
this.decoder = builder.decoder;
|
||||
this.joiner = builder.joiner;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getEncoder() {
|
||||
return encoder;
|
||||
}
|
||||
|
||||
public String getDecoder() {
|
||||
return decoder;
|
||||
}
|
||||
|
||||
public String getJoiner() {
|
||||
return joiner;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String encoder = "";
|
||||
private String decoder = "";
|
||||
private String joiner = "";
|
||||
|
||||
public Builder setEncoder(String encoder) {
|
||||
this.encoder = encoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDecoder(String decoder) {
|
||||
this.decoder = decoder;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setJoiner(String joiner) {
|
||||
this.joiner = joiner;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineTransducerModelConfig build() {
|
||||
return new OnlineTransducerModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class OnlineZipformer2CtcModelConfig {
|
||||
private final String model;
|
||||
|
||||
private OnlineZipformer2CtcModelConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public String getModel() {
|
||||
return model;
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OnlineZipformer2CtcModelConfig build() {
|
||||
return new OnlineZipformer2CtcModelConfig(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingExtractor {
|
||||
private long ptr = 0;
|
||||
|
||||
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = newFromFile(config);
|
||||
if (ptr == 0) {
|
||||
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public OnlineStream createStream() {
|
||||
long p = createStream(ptr);
|
||||
return new OnlineStream(p);
|
||||
}
|
||||
|
||||
public boolean isReady(OnlineStream s) {
|
||||
return isReady(ptr, s.getPtr());
|
||||
}
|
||||
|
||||
public float[] compute(OnlineStream s) {
|
||||
return compute(ptr, s.getPtr());
|
||||
}
|
||||
|
||||
public int getDim() {
|
||||
return dim(ptr);
|
||||
}
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
|
||||
|
||||
private native long createStream(long ptr);
|
||||
|
||||
private native boolean isReady(long ptr, long streamPtr);
|
||||
|
||||
private native float[] compute(long ptr, long streamPtr);
|
||||
|
||||
private native int dim(long ptr);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingExtractorConfig {
|
||||
private final String model;
|
||||
private final int numThreads;
|
||||
private final boolean debug;
|
||||
private final String provider;
|
||||
|
||||
private SpeakerEmbeddingExtractorConfig(Builder builder) {
|
||||
this.model = builder.model;
|
||||
this.numThreads = builder.numThreads;
|
||||
this.debug = builder.debug;
|
||||
this.provider = builder.provider;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
private String model = "";
|
||||
private int numThreads = 1;
|
||||
private boolean debug = true;
|
||||
private String provider = "cpu";
|
||||
|
||||
public SpeakerEmbeddingExtractorConfig build() {
|
||||
return new SpeakerEmbeddingExtractorConfig(this);
|
||||
}
|
||||
|
||||
|
||||
public Builder setModel(String model) {
|
||||
this.model = model;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setNumThreads(int numThreads) {
|
||||
this.numThreads = numThreads;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDebug(boolean debug) {
|
||||
this.debug = debug;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setProvider(String provider) {
|
||||
this.provider = provider;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright 2024 Xiaomi Corporation
|
||||
|
||||
package com.k2fsa.sherpa.onnx;
|
||||
|
||||
public class SpeakerEmbeddingManager {
|
||||
private long ptr = 0;
|
||||
|
||||
public SpeakerEmbeddingManager(int dim) {
|
||||
System.loadLibrary("onnxruntime");
|
||||
System.loadLibrary("sherpa-onnx-jni");
|
||||
ptr = create(dim);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
release();
|
||||
}
|
||||
|
||||
public void release() {
|
||||
if (this.ptr == 0) {
|
||||
return;
|
||||
}
|
||||
delete(this.ptr);
|
||||
this.ptr = 0;
|
||||
}
|
||||
|
||||
public boolean add(String name, float[] embedding) {
|
||||
return add(ptr, name, embedding);
|
||||
}
|
||||
|
||||
public boolean add(String name, float[][] embedding) {
|
||||
return addList(ptr, name, embedding);
|
||||
}
|
||||
|
||||
public boolean remove(String name) {
|
||||
return remove(ptr, name);
|
||||
}
|
||||
|
||||
public String search(float[] embedding, float threshold) {
|
||||
return search(ptr, embedding, threshold);
|
||||
}
|
||||
|
||||
public boolean verify(String name, float[] embedding, float threshold) {
|
||||
return verify(ptr, name, embedding, threshold);
|
||||
}
|
||||
|
||||
public boolean contains(String name) {
|
||||
return contains(ptr, name);
|
||||
}
|
||||
|
||||
public int getNumSpeakers() {
|
||||
return numSpeakers(ptr);
|
||||
}
|
||||
|
||||
public String[] getAllSpeakerNames() {
|
||||
return allSpeakerNames(ptr);
|
||||
}
|
||||
|
||||
private native long create(int dim);
|
||||
|
||||
private native void delete(long ptr);
|
||||
|
||||
private native boolean add(long ptr, String name, float[] embedding);
|
||||
|
||||
private native boolean addList(long ptr, String name, float[][] embedding);
|
||||
|
||||
private native boolean remove(long ptr, String name);
|
||||
|
||||
private native String search(long ptr, float[] embedding, float threshold);
|
||||
|
||||
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
|
||||
|
||||
private native boolean contains(long ptr, String name);
|
||||
|
||||
private native int numSpeakers(long ptr);
|
||||
|
||||
private native String[] allSpeakerNames(long ptr);
|
||||
}
|
||||
|
Before Width: | Height: | Size: 3.6 KiB |
|
Before Width: | Height: | Size: 5.0 KiB After Width: | Height: | Size: 43 KiB |
|
Before Width: | Height: | Size: 1.6 KiB |
|
Before Width: | Height: | Size: 2.5 KiB After Width: | Height: | Size: 14 KiB |
|
Before Width: | Height: | Size: 2.2 KiB |
|
Before Width: | Height: | Size: 3.4 KiB After Width: | Height: | Size: 22 KiB |
@@ -1,8 +1,8 @@
|
||||
<layer-list xmlns:android="http://schemas.android.com/apk/res/android" android:gravity="fill">
|
||||
<item>
|
||||
<bitmap android:gravity="fill" android:src="@drawable/background" />
|
||||
<color android:color="#F6F6F6" />
|
||||
</item>
|
||||
<item>
|
||||
<bitmap android:gravity="center" android:src="@drawable/logo" />
|
||||
</item>
|
||||
</layer-list>
|
||||
</layer-list>
|
||||
|
Before Width: | Height: | Size: 5.3 KiB |
|
Before Width: | Height: | Size: 6.7 KiB After Width: | Height: | Size: 70 KiB |
|
Before Width: | Height: | Size: 9.7 KiB |
|
Before Width: | Height: | Size: 10 KiB After Width: | Height: | Size: 146 KiB |
|
Before Width: | Height: | Size: 15 KiB |
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 256 KiB |
@@ -0,0 +1,18 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Microphone icon for the main page's tab-bar wakeword button.
|
||||
Converted from ~/Downloads/noun-microphone-1368757.svg (Noun Project
|
||||
icon #1368757); base fill is overridden by tint-color in app.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="1200"
|
||||
android:viewportHeight="1200">
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:fillType="evenOdd"
|
||||
android:pathData="m821.23 215.45v382.24c0 289.57-442.47 291.89-442.47 0v-382.24c0-287.26 442.47-287.26 442.47 0z" />
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:fillType="evenOdd"
|
||||
android:pathData="m543.24 963.71c-294.21-27.801-328.96-298.84-328.96-546.72 0-41.699 88.031-41.699 97.297-4.6328 2.3164 199.23 4.6328 454.05 287.26 454.05 284.94 0 287.26-254.83 287.26-454.05 6.9492-39.383 99.613-37.066 99.613 4.6328 0 247.88-34.75 518.92-333.59 546.72v139h152.89c67.18 0 69.5 97.297 0 97.297h-414.67c-64.863 0-62.547-97.297 0-97.297h152.89v-139z" />
|
||||
</vector>
|
||||
@@ -25,6 +25,17 @@
|
||||
<item name="colorPrimary">@color/ns_primary</item>
|
||||
<item name="colorPrimaryDark">@color/ns_primaryDark</item>
|
||||
<item name="colorAccent">@color/ns_accent</item>
|
||||
|
||||
<!-- NativeScript builds the toolbar programmatically, so the
|
||||
NativeScriptToolbarTheme overlay below (an inflation-time "theme"
|
||||
item) never applies and the overflow icon falls back to the light
|
||||
activity theme's dark textColorSecondary: black-on-dark. Tint the
|
||||
overflow button itself instead; every page's bar is dark. -->
|
||||
<item name="actionOverflowButtonStyle">@style/FaceclawOverflowButtonStyle</item>
|
||||
</style>
|
||||
|
||||
<style name="FaceclawOverflowButtonStyle" parent="Widget.AppCompat.ActionButton.Overflow">
|
||||
<item name="android:tint">#FFFFFF</item>
|
||||
</style>
|
||||
|
||||
<style name="AppTheme" parent="AppThemeBase">
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Wearable Data Layer capability advertised to the watch, so the Faceclaw
|
||||
watch app can find the phone that has Faceclaw installed. The watch app
|
||||
advertises "faceclaw_watch" in return (see wear/). -->
|
||||
<resources>
|
||||
<string-array name="android_wear_capabilities">
|
||||
<item>faceclaw_phone</item>
|
||||
</string-array>
|
||||
</resources>
|
||||
@@ -5,16 +5,22 @@ for interfacing with Android SDK).
|
||||
|
||||
Typescript parts are in app/. Java parts are in App_Resources/Android/src/main/java/com/faceclaw/app/.
|
||||
|
||||
If build.sh exists, it's a machine-specific wrapper that sets some environment
|
||||
variables to point to where tools are installed, then runs `npm run build`.
|
||||
If build_and_run.sh exists, it does everything build.sh does plus
|
||||
`nativescript run android --device <id> --justlaunch`,
|
||||
which will launch the app on the developer's phone if it is plugged in and has
|
||||
adb enabled. If no phone is connected, it will still build but the install step
|
||||
will fail and the output will contain the string "Cannot find connected
|
||||
devices." To lint and typecheck, prefer to use build.sh; if the user asks you
|
||||
to run or test the app, use build_and_run.sh and then use `adb logcat` to view
|
||||
the results.
|
||||
wear/ is a separate Gradle project (Kotlin + Compose for Wear OS): the watch
|
||||
remote that drives Faceclaw over the Wearable Data Layer. Its phone-side
|
||||
counterparts are FaceclawWearBridge.java, app/native/wear-bridge.ts and
|
||||
app/g2/wear-remote.ts; the message format is in wear/PROTOCOL.md.
|
||||
|
||||
`build.sh` and `build_wear.sh` build the Android and Wear OS apps respectively.
|
||||
`build_and_run.sh` builds the Android app then also installs it on a phone
|
||||
paired with `adb`. Building may require configuring environment variables
|
||||
(JAVA_HOME, ANDROID_HOME, PATH, ANDROID_KEYSTORE, ANDROID_KEYSTORE) to point at
|
||||
relevant tools and an Android device ID to install on; do this in
|
||||
build_paths.sh, which is created from build_paths.sh.template and not checked
|
||||
into git.
|
||||
|
||||
To lint and typecheck, prefer to use build.sh; if the user asks you to run or
|
||||
test the app, use build_and_run.sh and then use `adb logcat` to view the
|
||||
results.
|
||||
|
||||
If you are working on low-level communication bits, consider checking out
|
||||
https://github.com/Commute773/g2-kit-unofficial/ and referring to ble/docs/
|
||||
@@ -23,4 +29,10 @@ well as some communication test scripts and documentation of caveats that
|
||||
come up when communicating with the headset. There may already be an existing
|
||||
local checkout above this workspace at ../g2-kit-unofficial.
|
||||
|
||||
This Android app uses a modded firmware for the G2 smart glasses. Source code
|
||||
to the corresponding firmware modifications lives in
|
||||
https://github.com/jimrandomh/g2flash; there is a compiled version of the
|
||||
firmware patch-set in app/g2/firmware/cfw-patches.ts. Strongly prefer making
|
||||
changes to the Android app over making changes to changes to the firmware, as
|
||||
firmware development is considerably more fraught.
|
||||
|
||||
@@ -74,3 +74,18 @@ https://support.evenrealities.com/hc/en-us/articles/14127464826511-Privacy-Polic
|
||||
Apps downloaded from the EvenHub app store come with their own privacy
|
||||
policies.
|
||||
|
||||
|
||||
## Wear OS Watch
|
||||
|
||||
The optional Faceclaw watch app (wear/) talks to the phone over Google Play
|
||||
services' Wearable Data Layer, the same channel the watch uses for
|
||||
notifications and other companion apps. Gestures, commands, the watch's own
|
||||
speech-recognition results, and the mirrored glasses status and assistant
|
||||
replies travel between your watch and your phone over their Bluetooth (or,
|
||||
when paired that way, Wi-Fi) link; Faceclaw does not send any of it to a
|
||||
server. Speech recognized on the watch uses the watch's system speech
|
||||
recognizer, which may send audio to Google under Google's privacy policy:
|
||||
|
||||
Google: https://policies.google.com/privacy
|
||||
|
||||
The phone does nothing watch-related on phones without Google Play services.
|
||||
@@ -9,13 +9,15 @@ glasses. The Android app itself can install and uninstall the custom firmware;
|
||||
it downloads the stock firmware from Even and applies patches generated from
|
||||
https://github.com/jimrandomh/g2flash.
|
||||
|
||||
User-facing documentation lives at https://faceclaw.org/.
|
||||
|
||||
|
||||
## Screenshots
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
## Installation
|
||||
|
||||
@@ -23,52 +25,86 @@ To compile, you will need an Android SDK environment with SDK version 35
|
||||
installed and licenses accepted, and the ANDROID_HOME pointing to the install.
|
||||
You will need Nativescript installed, and a JDK environment (v21), with the
|
||||
JAVA_HOME environment variable pointed at it. You will also need a reasonably
|
||||
up to date npm and nodejs installed.
|
||||
up to date npm and nodejs installed. You can put environment variables in
|
||||
build_paths.sh and they will be used by all build commands.
|
||||
|
||||
To install a version that you compiled, you will need `adb` connected. Go to
|
||||
Settings>About phone and tap the "Build number" field seven times. Then plug
|
||||
the phone's USB-C port into your computer, authorize access on the phone, then
|
||||
run `nativescript run android
|
||||
|
||||
The compile, run
|
||||
'npm run build'
|
||||
To install, run
|
||||
`nativescript run android --juslaunch`
|
||||
To install a version that you compiled, you will need developer mode enabled on
|
||||
your phone, and `adb` connected. To enable developer mode, go to Settings>About
|
||||
phone and tap the "Build number" field seven times. Then plug the phone's USB-C
|
||||
port into your computer, authorize access on the phone, and run
|
||||
`./build_and_run.sh` to install.
|
||||
|
||||
## Features
|
||||
|
||||
* A voice assistant that wakes up when you say "Hey Even", transcribes text
|
||||
with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API, and
|
||||
responds to queries and commands using an onboard model (Qwen3 4B, slow)
|
||||
or with an Anthropic or OpenAI model (requires an API key), or using your
|
||||
own long-running OpenClaw agent (see "Connecting an external agent" below).
|
||||
* Multitasking, with an app-switcher sidebar and app launcher
|
||||
* Mostly-compatible with EvenHub apps
|
||||
* A lock screen; glasses lock automatically when you take them off and unlock
|
||||
when you unlock your phone
|
||||
* Full-screen apps can use the full 640x480 display area (rather than the
|
||||
576x288 that EvenHub apps can use)
|
||||
* Integration with Android notifications: A top-bar that shows the
|
||||
same icons your phone does, popups when notifications arrive, and menu
|
||||
items to dismiss or use Android-app-provided custom actions like mark as
|
||||
read or quick reply.
|
||||
* Mirror terminal apps such as Claude Code or Codex CLI with g2mirror
|
||||
(https://github.com/jimrandomh/g2mirror), view them on the glasses, and
|
||||
send them inputs with the voice assistant
|
||||
* Media player controls including playlist and media library navigation,
|
||||
compatible with most Android media players
|
||||
* Turn-by-turn directions (requires a Mapbox API token)
|
||||
* Nightscout, an app for viewing blood-glucose data (requires a cloud server
|
||||
and API token)
|
||||
* Power management: the glasses go to sleep properly when the screen is off,
|
||||
and wake when you double-tap the ring or speak the wakeword, allowing
|
||||
<!-- Generated from website/content/features.md by scripts/sync-site.mjs; edit there. -->
|
||||
<!-- BEGIN GENERATED: features -->
|
||||
* **A voice assistant** that wakes up when you say "Hey Even", transcribes
|
||||
text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API,
|
||||
and responds to queries and commands using an onboard model (Qwen3 4B,
|
||||
slow) or with an Anthropic or OpenAI model (requires an API key), or using
|
||||
your own long-running OpenClaw agent.
|
||||
* **Multitasking**, with an app-switcher sidebar and app launcher.
|
||||
* **Mostly-compatible with EvenHub apps.**
|
||||
* **A lock screen**; glasses lock automatically when you take them off and
|
||||
unlock when you unlock your phone.
|
||||
* **The full display.** Full-screen apps can use the full 640x480 display
|
||||
area, rather than the 576x288 that EvenHub apps can use.
|
||||
* **Integration with Android notifications**: a top bar that shows the same
|
||||
icons your phone does, popups when notifications arrive, and menu items to
|
||||
dismiss or use Android-app-provided custom actions like mark as read or
|
||||
quick reply.
|
||||
* **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex
|
||||
CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on
|
||||
the glasses, and send them inputs with the voice assistant.
|
||||
* **Media player controls** including playlist and media library navigation,
|
||||
compatible with most Android media players.
|
||||
* **Turn-by-turn directions** (requires a Mapbox API token).
|
||||
* **Nightscout**, an app for viewing blood-glucose data (requires a cloud
|
||||
server and API token).
|
||||
* **Power management**: the glasses go to sleep properly when the screen is
|
||||
off, and wake when you double-tap the ring or speak the wakeword, allowing
|
||||
battery life similar to the stock Even app.
|
||||
* Connection management with auto-reconnect, and autodetection of conflict
|
||||
with the official Even Realities app
|
||||
* On-phone screen mirroring and simulated ring input
|
||||
* Dual-language NativeScript architecture with Java for the multithreaded
|
||||
Android API and bluetooth stack bits, Typescript for the bits you want to
|
||||
hack on
|
||||
* **Connection management** with auto-reconnect, and autodetection of
|
||||
conflict with the official Even Realities app.
|
||||
* **A Wear OS watch app** that replaces (and outdoes) the R1 ring: tap,
|
||||
swipe, hold and crown gestures, side buttons, app launching and window
|
||||
switching, voice or keyboard queries to the assistant with the reply on
|
||||
your wrist, typing into apps, and glasses status/lock/display control.
|
||||
* **On-phone screen mirroring with touch control**: tap what you see on the
|
||||
mirror (sidebar icons, launcher cells), or use the phone's own touchpad,
|
||||
d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus a
|
||||
compact ring simulator. A display-mode picker (576×288 band, 576×480 tall,
|
||||
or the full 640×480 panel with an auto-hiding sidebar) and a brightness
|
||||
slider with an Auto toggle sit beside the mirror.
|
||||
* **Bluetooth pairing** that scans for nearby glasses and identifies each
|
||||
pair before connecting: model, frame shape, and colour decoded from the
|
||||
advertised serial (with product photos), left and right arms matched to
|
||||
each other by that serial, an estimated distance so the pair in your hand
|
||||
sorts first, and the optional R1 ring.
|
||||
* **Dual-language NativeScript architecture**, with Java for the
|
||||
multithreaded Android API and bluetooth stack bits, Typescript for the bits
|
||||
you want to hack on.
|
||||
<!-- END GENERATED: features -->
|
||||
|
||||
## Watch remote (Wear OS)
|
||||
|
||||
The `wear/` directory holds a companion app for Wear OS 3+ watches that turns
|
||||
the watch into a remote for the glasses. The whole watch face is a touchpad
|
||||
that mirrors the ring (tap, double-tap, hold, swipe, crown scrolling, side
|
||||
buttons), and beyond the ring it can launch apps, switch and close windows,
|
||||
wake/sleep/lock the display, send spoken or typed queries straight to the
|
||||
assistant (using the watch's own microphone) and show the reply on the wrist,
|
||||
and type text into apps such as the terminal. Its swipes are a spatial
|
||||
control scheme the ring never had, with no mode to enter first: the
|
||||
launcher highlights one app and moves it by rows and columns, right goes
|
||||
into things (sidebar → window, settings section → items, next track in
|
||||
Music), left backs out, tap selects, hold opens the context menu, double-tap
|
||||
or a two-finger tap is back everywhere, and two wrist twists go back; up/down
|
||||
scroll and right/left select/back everywhere else. The ring's own scheme is
|
||||
unchanged — watch input is tagged as a separate source. Faceclaw's Settings > Watch
|
||||
section has the phone-side switches. See [wear/README.md](wear/README.md) for
|
||||
building and installing it; it is a separate Gradle build, sideloaded over
|
||||
adb, and needs no Play Store. The phone side requires Google Play services.
|
||||
|
||||
## Connecting an external agent (OpenClaw)
|
||||
|
||||
@@ -122,7 +158,7 @@ https://github.com/jimrandomh/faceclaw.
|
||||
The Typescript and Java code in this repository runs on your phone, not on the
|
||||
glasses themselves, and (with the narrow exception of the firmware-updating
|
||||
tool), can't hurt your hardware. For changes to the glasses firmware, refer to
|
||||
[g2flash](https://github.com/jimrandomh/faceclaw); changes there require more
|
||||
[g2flash](https://github.com/jimrandomh/g2flash); changes there require more
|
||||
caution.
|
||||
|
||||
Faceclaw is Free Software (GPLv3). Please only contribute code that you wrote
|
||||
|
||||
@@ -15,11 +15,43 @@ ActionItem {
|
||||
color: #ffffff;
|
||||
}
|
||||
|
||||
/* The main page's custom titleView: title left, connection status and the
|
||||
* warning triangle right-aligned beside the overflow menu. */
|
||||
.action-bar-title {
|
||||
color: #ffffff;
|
||||
font-size: 18;
|
||||
font-weight: bold;
|
||||
vertical-align: center;
|
||||
}
|
||||
|
||||
.action-bar-status {
|
||||
color: #ffffff;
|
||||
font-size: 14;
|
||||
margin-right: 12;
|
||||
vertical-align: center;
|
||||
}
|
||||
|
||||
/* The emoji brings its own yellow; the padding widens the tap target. */
|
||||
.action-bar-warning {
|
||||
font-size: 18;
|
||||
padding: 4 6;
|
||||
vertical-align: center;
|
||||
}
|
||||
|
||||
TextField {
|
||||
background-color: #ffffff;
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
/* Without this, the theme's .ns-dark rules recolour the text white while the
|
||||
* white background above sticks, rendering white-on-white fields.
|
||||
* Order matters: .system-name-modal TextField must stay after this rule so the
|
||||
* deliberately light modal wins the equal-specificity tie in dark mode. */
|
||||
.ns-dark TextField {
|
||||
background-color: #2e2e2e;
|
||||
color: #f0f0f0;
|
||||
}
|
||||
|
||||
.system-name-modal TextField {
|
||||
background-color: #ffffff;
|
||||
color: #222222;
|
||||
@@ -29,6 +61,43 @@ Button.-primary {
|
||||
font-size: 18;
|
||||
}
|
||||
|
||||
/* Body/explanatory text in the onboarding flow; the theme default is too
|
||||
* small to comfortably read paragraphs of legalese, especially on tablets. */
|
||||
.body-text {
|
||||
font-size: 18;
|
||||
}
|
||||
|
||||
/* The logo art is black-on-transparent; tint it light in dark mode so it
|
||||
* doesn't vanish into the dark page background. */
|
||||
.ns-dark .onboarding-logo {
|
||||
tint-color: #e6e6e6;
|
||||
}
|
||||
|
||||
/* License / privacy-policy links in the onboarding disclaimer footer. The
|
||||
* padding widens the tap target beyond the small text. */
|
||||
.footer-link {
|
||||
color: #3b6fd6;
|
||||
font-size: 15;
|
||||
text-decoration: underline;
|
||||
padding: 4 8;
|
||||
}
|
||||
|
||||
.ns-dark .footer-link {
|
||||
color: #8ab0ff;
|
||||
}
|
||||
|
||||
.footer-link-separator {
|
||||
color: #777777;
|
||||
font-size: 15;
|
||||
padding: 4 0;
|
||||
}
|
||||
|
||||
/* Bundled project docs (phone-ui/document-page). LICENSE and PRIVACY are
|
||||
* hard-wrapped plain text, so keep the size modest to limit rewrapping. */
|
||||
.document-text {
|
||||
font-size: 13;
|
||||
}
|
||||
|
||||
/*
|
||||
* Stands in for the lens preview when showing it would be misleading (silent
|
||||
* mode, or a disconnect explained by a flat battery). Sized by the view model
|
||||
@@ -55,3 +124,544 @@ Button.-primary {
|
||||
border-width: 1;
|
||||
border-radius: 8;
|
||||
}
|
||||
|
||||
/* Modal behind the action bar's warning triangle: the problem-status warnings
|
||||
* that used to sit inline on the main page, each with its fix-it button.
|
||||
* The width is set from the view model (near-full-width on phones, capped on
|
||||
* tablets); the padding lives on the content stack because a ScrollView does
|
||||
* not apply its own padding to its content. */
|
||||
.warnings-modal {
|
||||
margin: 16;
|
||||
background-color: #f6f6f6;
|
||||
color: #222222;
|
||||
border-color: #222222;
|
||||
border-width: 1;
|
||||
border-radius: 8;
|
||||
}
|
||||
|
||||
.warnings-modal-content {
|
||||
padding: 16;
|
||||
}
|
||||
|
||||
/* The modal stays light in dark mode; the .ns-dark pairing outguns the
|
||||
* theme's dark-mode recolour of Label/.h2 so the title doesn't go
|
||||
* white-on-light-gray. */
|
||||
.warnings-modal Label,
|
||||
.ns-dark .warnings-modal Label {
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
.warnings-modal-item {
|
||||
margin-bottom: 12;
|
||||
}
|
||||
|
||||
/* The phone's controller for the glasses (phone-ui/remote-controls.xml). */
|
||||
.remote-controls {
|
||||
margin: 0 8 0 8;
|
||||
}
|
||||
/* The Settings / Watch / Ring tab bar above the controls. */
|
||||
.controls-tab-bar {
|
||||
margin-bottom: 8;
|
||||
}
|
||||
.tab-button {
|
||||
font-size: 14;
|
||||
padding: 8 0;
|
||||
margin: 0 2;
|
||||
border-radius: 6;
|
||||
background-color: #e2e2e2;
|
||||
color: #333333;
|
||||
}
|
||||
.tab-button-selected {
|
||||
background-color: #424242;
|
||||
color: #ffffff;
|
||||
}
|
||||
/* The wakeword button at the tab bar's right edge: icon-only, narrower than
|
||||
* the tabs, and never in the selected state (it isn't a tab). It is a
|
||||
* button-styled GridLayout (Buttons can't hold images), so it borrows the
|
||||
* .tab-button look and adds the elevation MaterialButtons get for free. */
|
||||
.tab-mic-button {
|
||||
width: 56;
|
||||
/* The real MaterialButtons render 48dp tall inside a 60dp layout box (6dp
|
||||
* insets top and bottom); the margins and explicit height reproduce that.
|
||||
* The height also bounds the tile's measure, without which its star row
|
||||
* would collapse to content and pin the icon to the top. */
|
||||
height: 48;
|
||||
margin: 6 2;
|
||||
android-elevation: 2;
|
||||
}
|
||||
.tab-mic-icon {
|
||||
height: 20;
|
||||
tint-color: #333333;
|
||||
}
|
||||
.ns-dark .tab-mic-icon {
|
||||
tint-color: #cccccc;
|
||||
}
|
||||
.ns-dark .tab-button {
|
||||
background-color: #333333;
|
||||
color: #cccccc;
|
||||
}
|
||||
.ns-dark .tab-button-selected {
|
||||
background-color: #e2e2e2;
|
||||
color: #222222;
|
||||
}
|
||||
/* Settings tab rows. */
|
||||
.settings-row {
|
||||
margin-bottom: 8;
|
||||
}
|
||||
.settings-row-label {
|
||||
font-size: 14;
|
||||
margin-right: 10;
|
||||
}
|
||||
.mode-button {
|
||||
font-size: 13;
|
||||
padding: 6 10;
|
||||
margin-right: 6;
|
||||
}
|
||||
.brightness-icon {
|
||||
font-size: 16;
|
||||
margin: 0 4 0 4;
|
||||
color: #444444;
|
||||
}
|
||||
.auto-toggle Label {
|
||||
font-size: 13;
|
||||
margin-right: 2;
|
||||
}
|
||||
.touchpad {
|
||||
height: 230;
|
||||
background-color: #1f1f1f;
|
||||
border-radius: 14;
|
||||
border-width: 1;
|
||||
border-color: #3a3a3a;
|
||||
color: #f0f0f0;
|
||||
margin-top: 2;
|
||||
}
|
||||
.pad-focus {
|
||||
font-size: 20;
|
||||
color: #ffffff;
|
||||
}
|
||||
.pad-dpad-button {
|
||||
width: 46;
|
||||
height: 46;
|
||||
margin: 6;
|
||||
padding: 0;
|
||||
font-size: 16;
|
||||
border-radius: 23;
|
||||
background-color: #333333;
|
||||
color: #e8e8e8;
|
||||
}
|
||||
.pad-dpad-down {
|
||||
margin-bottom: 30;
|
||||
}
|
||||
.pad-legend {
|
||||
font-size: 11;
|
||||
color: #8a8a8a;
|
||||
margin-bottom: 8;
|
||||
}
|
||||
.action-row {
|
||||
margin-top: 6;
|
||||
}
|
||||
.action-button {
|
||||
margin: 2;
|
||||
font-size: 14;
|
||||
}
|
||||
/* Ring tab: simulated R1 inputs. */
|
||||
.ring-pad-button {
|
||||
margin: 3;
|
||||
padding: 0;
|
||||
height: 52;
|
||||
font-size: 14;
|
||||
}
|
||||
/*
|
||||
* Pairing scan list (phone-ui/pairing-page). Rows are Repeater items: product
|
||||
* art (or a colourway swatch) on the left, decoded SKU + proximity + serial in
|
||||
* the middle, a chevron/check on the right.
|
||||
*/
|
||||
.pairing-instructions {
|
||||
color: #555555;
|
||||
font-size: 14;
|
||||
}
|
||||
|
||||
.pairing-status {
|
||||
font-size: 14;
|
||||
margin-left: 8;
|
||||
}
|
||||
|
||||
.pairing-section {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
margin-bottom: 4;
|
||||
}
|
||||
|
||||
.pairing-empty {
|
||||
color: #777777;
|
||||
font-size: 14;
|
||||
padding: 8 0;
|
||||
}
|
||||
|
||||
.pairing-row {
|
||||
background-color: #f4f4f4;
|
||||
border-color: #dddddd;
|
||||
border-width: 1;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
margin-bottom: 8;
|
||||
}
|
||||
|
||||
.pairing-row-selected {
|
||||
background-color: #e6f0ff;
|
||||
border-color: #3b6fd6;
|
||||
}
|
||||
|
||||
.pairing-row-incomplete {
|
||||
background-color: #fafafa;
|
||||
border-color: #e6e6e6;
|
||||
}
|
||||
|
||||
.pairing-art {
|
||||
vertical-align: center;
|
||||
}
|
||||
|
||||
.pairing-title {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
.pairing-badge {
|
||||
font-size: 11;
|
||||
font-weight: bold;
|
||||
color: #2a5bc4;
|
||||
background-color: #dbe6fb;
|
||||
border-radius: 9;
|
||||
padding: 1 7;
|
||||
}
|
||||
|
||||
.pairing-variant {
|
||||
font-size: 13;
|
||||
font-weight: bold;
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
.pairing-proximity {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
}
|
||||
|
||||
.pairing-proximity-immediate {
|
||||
color: #2a5bc4;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.pairing-serial {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
.pairing-arms {
|
||||
font-size: 12;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.pairing-warning {
|
||||
font-size: 12;
|
||||
color: #a33a1f;
|
||||
margin-top: 4;
|
||||
}
|
||||
|
||||
.pairing-swatch-label {
|
||||
font-size: 11;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.pairing-chevron {
|
||||
font-size: 22;
|
||||
color: #444444;
|
||||
margin-left: 6;
|
||||
}
|
||||
|
||||
.pairing-selection {
|
||||
background-color: #eef4ff;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
}
|
||||
|
||||
Button.pairing-link {
|
||||
background-color: transparent;
|
||||
color: #2a5bc4;
|
||||
font-size: 14;
|
||||
border-width: 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Permissions screen (phone-ui/permissions-page). Cards reuse the pairing-row
|
||||
* / pairing-title / pairing-badge look above; these add the explanation body
|
||||
* and the already-granted checkmark.
|
||||
*/
|
||||
.permissions-description {
|
||||
font-size: 13;
|
||||
color: #555555;
|
||||
margin-top: 2;
|
||||
}
|
||||
|
||||
.permissions-check {
|
||||
font-size: 24;
|
||||
color: #2e7d32;
|
||||
margin-left: 10;
|
||||
}
|
||||
|
||||
/* Continue with required permissions still missing: looks disabled but stays
|
||||
* tappable (the tap raises an are-you-sure prompt instead of being ignored). */
|
||||
Button.permissions-primary-dimmed {
|
||||
opacity: 0.45;
|
||||
}
|
||||
|
||||
/*
|
||||
* Dark-mode pairing page. The cards keep their light-mode look above; here
|
||||
* they go dark-gray, and every text colour set above gets a light counterpart
|
||||
* so nothing renders dark-on-dark (the instructions) or dark-on-a-dark-card.
|
||||
* Order matters: the -selected/-incomplete rules must follow .pairing-row so
|
||||
* they win at equal specificity.
|
||||
*/
|
||||
.ns-dark .pairing-instructions {
|
||||
color: #b0b0b0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-empty {
|
||||
color: #999999;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-row {
|
||||
background-color: #2e2e2e;
|
||||
border-color: #4a4a4a;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-row-selected {
|
||||
background-color: #333c4c;
|
||||
border-color: #5b8def;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-row-incomplete {
|
||||
background-color: #282828;
|
||||
border-color: #3c3c3c;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-title {
|
||||
color: #f0f0f0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-badge {
|
||||
color: #a8c4f5;
|
||||
background-color: #2a3b5e;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-variant {
|
||||
color: #cccccc;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-proximity {
|
||||
color: #a0a0a0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-proximity-immediate {
|
||||
color: #7da7f5;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-serial {
|
||||
color: #a0a0a0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-arms {
|
||||
color: #c0c0c0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-warning {
|
||||
color: #e08a66;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-swatch-label {
|
||||
color: #aaaaaa;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-chevron {
|
||||
color: #bbbbbb;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-selection {
|
||||
background-color: #2e2e2e;
|
||||
}
|
||||
|
||||
.ns-dark Button.pairing-link {
|
||||
color: #7da7f5;
|
||||
}
|
||||
|
||||
.ns-dark .permissions-description {
|
||||
color: #b0b0b0;
|
||||
}
|
||||
|
||||
.ns-dark .permissions-check {
|
||||
color: #7bc47f;
|
||||
}
|
||||
|
||||
/*
|
||||
* Conversation review (phone-ui/conversations-page, conversation-page,
|
||||
* speakers-page). Cards follow the pairing-row look; speaker/emotion chips
|
||||
* get their background colour inline (per-speaker / per-emotion), so the
|
||||
* classes only carry shape and dark text that reads on the pastel palette.
|
||||
*/
|
||||
.conversation-row {
|
||||
background-color: #f4f4f4;
|
||||
border-color: #dddddd;
|
||||
border-width: 1;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
margin-bottom: 8;
|
||||
}
|
||||
|
||||
.conversation-title {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
.conversation-meta {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
}
|
||||
|
||||
.conversation-empty {
|
||||
color: #777777;
|
||||
font-size: 14;
|
||||
padding: 8 0;
|
||||
}
|
||||
|
||||
.ask-answer {
|
||||
font-size: 15;
|
||||
color: #222222;
|
||||
padding: 4 0;
|
||||
}
|
||||
|
||||
.ns-dark .ask-answer {
|
||||
color: #e6e6e6;
|
||||
}
|
||||
|
||||
.conversation-status {
|
||||
font-size: 13;
|
||||
color: #2a5bc4;
|
||||
}
|
||||
|
||||
.speaker-chip {
|
||||
font-size: 12;
|
||||
color: #222222;
|
||||
border-radius: 10;
|
||||
padding: 2 8;
|
||||
margin: 0 6 4 0;
|
||||
}
|
||||
|
||||
.speaker-chip-selected {
|
||||
border-width: 2;
|
||||
border-color: #222222;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.speaker-tag {
|
||||
font-size: 11;
|
||||
color: #555555;
|
||||
background-color: #e6e6e6;
|
||||
border-radius: 8;
|
||||
padding: 1 6;
|
||||
}
|
||||
|
||||
.speaker-wearer-badge {
|
||||
font-size: 11;
|
||||
font-weight: bold;
|
||||
color: #2a5bc4;
|
||||
background-color: #dbe6fb;
|
||||
border-radius: 8;
|
||||
padding: 1 6;
|
||||
}
|
||||
|
||||
.transcript-row {
|
||||
border-radius: 8;
|
||||
padding: 8;
|
||||
margin-bottom: 4;
|
||||
}
|
||||
|
||||
.transcript-row-playing {
|
||||
background-color: #e6f0ff;
|
||||
}
|
||||
|
||||
.transcript-translation {
|
||||
font-size: 13;
|
||||
font-style: italic;
|
||||
color: #777777;
|
||||
}
|
||||
|
||||
.transcript-time {
|
||||
font-size: 11;
|
||||
color: #888888;
|
||||
}
|
||||
|
||||
.transcript-emotion {
|
||||
font-size: 11;
|
||||
color: #222222;
|
||||
border-radius: 8;
|
||||
padding: 1 6;
|
||||
}
|
||||
|
||||
.transcript-play {
|
||||
font-size: 16;
|
||||
color: #2a5bc4;
|
||||
margin-left: 6;
|
||||
}
|
||||
|
||||
.ns-dark .conversation-row {
|
||||
background-color: #2e2e2e;
|
||||
border-color: #4a4a4a;
|
||||
}
|
||||
|
||||
.ns-dark .conversation-title {
|
||||
color: #f0f0f0;
|
||||
}
|
||||
|
||||
.ns-dark .conversation-meta {
|
||||
color: #a0a0a0;
|
||||
}
|
||||
|
||||
.ns-dark .conversation-empty {
|
||||
color: #999999;
|
||||
}
|
||||
|
||||
.ns-dark .conversation-status {
|
||||
color: #7da7f5;
|
||||
}
|
||||
|
||||
.ns-dark .speaker-chip-selected {
|
||||
border-color: #f0f0f0;
|
||||
}
|
||||
|
||||
.ns-dark .speaker-tag {
|
||||
color: #cccccc;
|
||||
background-color: #3c3c3c;
|
||||
}
|
||||
|
||||
.ns-dark .speaker-wearer-badge {
|
||||
color: #a8c4f5;
|
||||
background-color: #2a3b5e;
|
||||
}
|
||||
|
||||
.ns-dark .transcript-row-playing {
|
||||
background-color: #333c4c;
|
||||
}
|
||||
|
||||
.ns-dark .transcript-translation {
|
||||
color: #999999;
|
||||
}
|
||||
|
||||
.ns-dark .transcript-time {
|
||||
color: #909090;
|
||||
}
|
||||
|
||||
.ns-dark .transcript-play {
|
||||
color: #7da7f5;
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
import { type AppDefinition } from "./app-definition";
|
||||
import launcherApp from "./launcher";
|
||||
import timerApp from "./timer";
|
||||
import calculatorApp from "./calculator";
|
||||
import terminalApp from "./terminal";
|
||||
import filesApp from "./files";
|
||||
import musicApp from "./music";
|
||||
import nightscoutApp from "./nightscout";
|
||||
import transcribeApp from "./transcribe";
|
||||
import microphonesApp from "./microphones";
|
||||
import notificationsApp from "./notifications";
|
||||
import calendarApp from "./calendar";
|
||||
import weatherApp from "./weather";
|
||||
@@ -28,11 +30,13 @@ import settingsApp from "./settings";
|
||||
export const ALL_APPS: readonly AppDefinition[] = [
|
||||
launcherApp,
|
||||
timerApp,
|
||||
calculatorApp,
|
||||
terminalApp,
|
||||
filesApp,
|
||||
musicApp,
|
||||
nightscoutApp,
|
||||
transcribeApp,
|
||||
microphonesApp,
|
||||
notificationsApp,
|
||||
calendarApp,
|
||||
weatherApp,
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
/**
|
||||
* Calculator window wiring: the in-process window, its long-press menu (mode
|
||||
* follow-ups, clear, settings), the assistant tools, and the voice/text
|
||||
* plumbing into the layer.
|
||||
*
|
||||
* The coordinator is a module singleton on purpose: a
|
||||
* problem stated through the assistant's calculate tool is still standing
|
||||
* when the wearer opens the window, and "graph it" there refers to it. The
|
||||
* same session also survives closing and reopening the window.
|
||||
*/
|
||||
import {
|
||||
createInProcessWindow,
|
||||
type InProcessAppOptions,
|
||||
type InProcessWindow,
|
||||
} from "../../ui/shell/in-process-window";
|
||||
import { type MenuItem } from "../../ui/menu";
|
||||
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
|
||||
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { toolRegistry } from "../../assistant/tool-registry";
|
||||
import { type ToolSpec } from "../../assistant/tool-registry";
|
||||
import { MathCoordinator } from "./math/coordinator";
|
||||
import { rewriteSpokenProblem } from "./calculator-llm";
|
||||
import {
|
||||
CalculatorLayer,
|
||||
calculatorDegreesSetting,
|
||||
calculatorListeningSetting,
|
||||
calculatorModeSetting,
|
||||
} from "./calculator";
|
||||
import { CALCULATOR_MODES, modeLabel } from "./calculator-commands";
|
||||
|
||||
export const CALCULATOR_WINDOW_ID = "calculator";
|
||||
export const CALCULATOR_SURFACE_ID = "window:calculator";
|
||||
|
||||
/** One standing maths session, shared by the window and the assistant tools. */
|
||||
const coordinator = new MathCoordinator();
|
||||
coordinator.setAICompletion(rewriteSpokenProblem);
|
||||
|
||||
const CALCULATOR_TOOLS: ToolSpec[] = [
|
||||
{
|
||||
name: "calculate",
|
||||
description:
|
||||
"Solve a maths problem stated in words or notation ('2x + 7 = 19', 'what is 15 percent of 240', " +
|
||||
"'is 91 prime', 'where does 11 first appear in pi', 'area of a circle with radius 3'). " +
|
||||
"Do NOT work it out yourself first — hand the words over verbatim; the calculator answers " +
|
||||
"exactly and shows the result on the glasses. Follow-ups like 'graph it' and 'explain it' " +
|
||||
"act on the problem already standing.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
problem: { type: "string", description: "The problem or follow-up, exactly as asked." },
|
||||
},
|
||||
required: ["problem"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
availability: "open",
|
||||
},
|
||||
];
|
||||
|
||||
export function createCalculatorAppWindow(options: InProcessAppOptions): InProcessWindow {
|
||||
const layer = new CalculatorLayer(coordinator, options.actions);
|
||||
|
||||
const menuItems = (): MenuItem[] => [
|
||||
...CALCULATOR_MODES.map(
|
||||
(mode): MenuItem => ({
|
||||
label: `${modeLabel(mode)} it`,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.performFollowUp(mode);
|
||||
},
|
||||
}),
|
||||
),
|
||||
{
|
||||
label: "Clear",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
layer.clear();
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Calculator settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
openSettingsSubMenu(ctx, "Calculator settings", [
|
||||
enumSettingMenuItem(calculatorModeSetting, {
|
||||
onChange: () => layer.applySettings(),
|
||||
}),
|
||||
enumSettingMenuItem(calculatorListeningSetting, {
|
||||
onChange: () => layer.applySettings(),
|
||||
}),
|
||||
toggleSettingMenuItem(calculatorDegreesSetting, {
|
||||
onChange: () => layer.applySettings(),
|
||||
}),
|
||||
]);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let app: InProcessWindow;
|
||||
app = createInProcessWindow({
|
||||
appId: "calculator",
|
||||
windowId: CALCULATOR_WINDOW_ID,
|
||||
title: "Calculator",
|
||||
iconLetter: "=",
|
||||
icon: "calculator",
|
||||
closeable: true,
|
||||
menuItems,
|
||||
actions: options.actions,
|
||||
receiveTextInput: (text) => {
|
||||
if (app.stack.receiveTextInput(text)) app.requestRender();
|
||||
},
|
||||
baseLayer: layer,
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: (visible) => {
|
||||
layer.setForeground(visible);
|
||||
options.setSurfaceVisible(visible);
|
||||
},
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID);
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
layer.requestRender = app.requestRender;
|
||||
layer.setForeground(shell.isWindowVisible(CALCULATOR_WINDOW_ID));
|
||||
// Apply the persisted settings now the window exists: degrees onto the
|
||||
// session, and continuous listening if the wearer left it on.
|
||||
layer.applySettings();
|
||||
|
||||
toolRegistry.setAppTools({
|
||||
windowId: CALCULATOR_WINDOW_ID,
|
||||
appId: "calculator",
|
||||
specs: CALCULATOR_TOOLS,
|
||||
isForeground: () => shell.isWindowFocused(CALCULATOR_WINDOW_ID),
|
||||
invoke: async (toolName, args) => {
|
||||
if (toolName !== "calculate") {
|
||||
return { ok: false, error: `Unknown calculator tool: ${toolName}` };
|
||||
}
|
||||
const problem = String((args as { problem?: unknown })?.problem ?? "").trim();
|
||||
if (!problem) return { ok: false, error: "calculate requires a problem" };
|
||||
const outcome = await coordinator.answerSpoken(problem);
|
||||
app.requestRender();
|
||||
if (outcome.kind === "failure") return { ok: false, error: outcome.reason };
|
||||
const detail = coordinator.lastResult?.detail ?? "";
|
||||
return { ok: true, content: detail || outcome.text };
|
||||
},
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// The pure classification half of the calculator's command handling. Kept
|
||||
// free of any UI import so the tests can exercise exactly the
|
||||
// decisions that matter: which utterances are commands, which unprompted room
|
||||
// speech is worth answering, and how the modes map onto the shared session.
|
||||
|
||||
import { type MathBranchMode } from "./math/coordinator";
|
||||
import { parseCommand } from "./math/session";
|
||||
|
||||
/**
|
||||
* What a freshly captured problem does the moment it lands.
|
||||
*
|
||||
* Three, not more, because these are the three the wearer is TOLD about —
|
||||
* "Solve", "Explain", "Graph" are the spoken commands, so making them the
|
||||
* selectable modes means the touchpad and the voice are steering the same
|
||||
* thing rather than two parallel vocabularies. Differentiation stays
|
||||
* reachable by saying it ("the derivative of…"), which is how it is actually
|
||||
* asked for.
|
||||
*/
|
||||
export type CalculatorMode = "solve" | "explain" | "graph";
|
||||
|
||||
export const CALCULATOR_MODES: readonly CalculatorMode[] = ["solve", "explain", "graph"];
|
||||
|
||||
export function modeLabel(mode: CalculatorMode): string {
|
||||
switch (mode) {
|
||||
case "solve":
|
||||
return "Solve";
|
||||
case "explain":
|
||||
return "Explain";
|
||||
case "graph":
|
||||
return "Graph";
|
||||
}
|
||||
}
|
||||
|
||||
export function modeDetail(mode: CalculatorMode): string {
|
||||
switch (mode) {
|
||||
case "solve":
|
||||
return "Answer it, exactly. 3/2 stays 3/2.";
|
||||
case "explain":
|
||||
return "Show the working step by step; scroll walks the rest.";
|
||||
case "graph":
|
||||
return "Plot it.";
|
||||
}
|
||||
}
|
||||
|
||||
export function modeBranch(mode: CalculatorMode): MathBranchMode {
|
||||
return mode;
|
||||
}
|
||||
|
||||
/**
|
||||
* The spoken commands that select this mode. Both the bare verb and the
|
||||
* "it" form, because someone shown "Solve" says both.
|
||||
*/
|
||||
export function modeSpokenCommands(mode: CalculatorMode): string[] {
|
||||
switch (mode) {
|
||||
case "solve":
|
||||
return ["solve", "solve it"];
|
||||
case "explain":
|
||||
return ["explain", "explain it"];
|
||||
case "graph":
|
||||
return ["graph", "graph it"];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* How the Calculator listens.
|
||||
*
|
||||
* "Tap to talk" arms exactly one finalized utterance per tap, so the app is
|
||||
* not standing in the room claiming every sentence; "continuous" listens the
|
||||
* whole time the app is foreground, gated by the looks-like-maths test
|
||||
* below. The reason there are two input modes: a wearer in a
|
||||
* quiet room must be able to take the microphone away from the calculator.
|
||||
*/
|
||||
export type CalculatorListening = "tap" | "continuous";
|
||||
|
||||
export function listeningLabel(listening: CalculatorListening): string {
|
||||
switch (listening) {
|
||||
case "tap":
|
||||
return "Tap to talk";
|
||||
case "continuous":
|
||||
return "Continuous";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognise a bare mode command in an utterance. Exact match on the whole
|
||||
* utterance, not a substring: "I need to solve this problem with the tax
|
||||
* people" is conversation, and a substring test turns it into a command.
|
||||
*/
|
||||
export function modeCommand(text: string): CalculatorMode | null {
|
||||
const normalized = text.toLowerCase().replace(/^[ .,!?"']+|[ .,!?"']+$/g, "");
|
||||
for (const mode of CALCULATOR_MODES) {
|
||||
if (modeSpokenCommands(mode).includes(normalized)) return mode;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isDictationCancellation(text: string): boolean {
|
||||
const normalized = text.toLowerCase().replace(/^[ .,!?"']+|[ .,!?"']+$/g, "");
|
||||
return ["cancel", "cancel listening", "stop listening", "never mind", "nevermind"].includes(normalized);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether unprompted room speech is worth handing to the parser. In
|
||||
* continuous mode the app sees every finalized utterance in the room;
|
||||
* without this gate, every sentence becomes a failed calculation on the
|
||||
* lens.
|
||||
*/
|
||||
export function isWorthAnswering(text: string): boolean {
|
||||
if (modeCommand(text) !== null) return true;
|
||||
return parseCommand(text) !== null;
|
||||
}
|
||||
|
||||
export function cycleMode(mode: CalculatorMode, forward: boolean): CalculatorMode {
|
||||
const index = CALCULATOR_MODES.indexOf(mode);
|
||||
const step = forward ? 1 : CALCULATOR_MODES.length - 1;
|
||||
return CALCULATOR_MODES[(index + step) % CALCULATOR_MODES.length]!;
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Draws the maths engine's already-sampled graph into the app's GrayImage viewport.
|
||||
// The renderer never re-evaluates the expression: segment breaks, viewport
|
||||
// clipping, roots, and extrema therefore stay exactly what the plotter
|
||||
// decided, and an asymptote correctly stays an open gap rather than a
|
||||
// vertical line — the single most recognisable "bad graphing app" artefact.
|
||||
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { type Graph, type GraphPoint, viewportContainsY } from "./math/plotter";
|
||||
|
||||
// Brightness levels tuned for the G2's grayscale panel: the curve stays
|
||||
// visually dominant over the axes, and the axes over the tick grid.
|
||||
const GRID_VALUE = 38;
|
||||
const AXIS_VALUE = 150;
|
||||
const BORDER_VALUE = 105;
|
||||
const CURVE_VALUES = [245, 200, 160];
|
||||
const MARKER_VALUE = 255;
|
||||
|
||||
/**
|
||||
* Draw the graph into `rect` (x, y, width, height) of the image. The caller
|
||||
* owns the header and footer rows.
|
||||
*/
|
||||
export function drawGraph(
|
||||
image: GrayImage,
|
||||
graph: Graph,
|
||||
rect: { x: number; y: number; width: number; height: number },
|
||||
): void {
|
||||
if (rect.width <= 1 || rect.height <= 1) return;
|
||||
const viewport = graph.viewport;
|
||||
const spanX = Math.max(viewport.xMax - viewport.xMin, Number.EPSILON);
|
||||
const spanY = Math.max(viewport.yMax - viewport.yMin, Number.EPSILON);
|
||||
|
||||
const px = (x: number): number => rect.x + ((x - viewport.xMin) / spanX) * rect.width;
|
||||
const py = (y: number): number => rect.y + rect.height - ((y - viewport.yMin) / spanY) * rect.height;
|
||||
|
||||
const clampX = (x: number): number => Math.max(rect.x, Math.min(rect.x + rect.width - 1, x));
|
||||
const clampY = (y: number): number => Math.max(rect.y, Math.min(rect.y + rect.height - 1, y));
|
||||
|
||||
// Tick grid first so the curve and axes draw over it.
|
||||
for (const tick of graph.xAxis.ticks) {
|
||||
if (tick < viewport.xMin || tick > viewport.xMax) continue;
|
||||
const x = Math.round(px(tick));
|
||||
image.drawLine(x, rect.y, x, rect.y + rect.height - 1, GRID_VALUE);
|
||||
}
|
||||
for (const tick of graph.yAxis.ticks) {
|
||||
if (tick < viewport.yMin || tick > viewport.yMax) continue;
|
||||
const y = Math.round(py(tick));
|
||||
image.drawLine(rect.x, y, rect.x + rect.width - 1, y, GRID_VALUE);
|
||||
}
|
||||
|
||||
// Axes.
|
||||
if (viewport.yMin <= 0 && viewport.yMax >= 0) {
|
||||
const y = Math.round(py(0));
|
||||
image.drawLine(rect.x, y, rect.x + rect.width - 1, y, AXIS_VALUE);
|
||||
}
|
||||
if (viewport.xMin <= 0 && viewport.xMax >= 0) {
|
||||
const x = Math.round(px(0));
|
||||
image.drawLine(x, rect.y, x, rect.y + rect.height - 1, AXIS_VALUE);
|
||||
}
|
||||
|
||||
// Curves, brightest first plot.
|
||||
for (let plotIndex = 0; plotIndex < graph.plots.length; plotIndex++) {
|
||||
const plot = graph.plots[plotIndex]!;
|
||||
const value = CURVE_VALUES[plotIndex % CURVE_VALUES.length]!;
|
||||
for (const segment of plot.segments) {
|
||||
let previous: GraphPoint | null = null;
|
||||
for (const point of segment.points) {
|
||||
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) {
|
||||
previous = null;
|
||||
continue;
|
||||
}
|
||||
if (previous) {
|
||||
image.drawLine(
|
||||
Math.round(clampX(px(previous.x))),
|
||||
Math.round(clampY(py(previous.y))),
|
||||
Math.round(clampX(px(point.x))),
|
||||
Math.round(clampY(py(point.y))),
|
||||
value,
|
||||
);
|
||||
}
|
||||
previous = point;
|
||||
}
|
||||
}
|
||||
|
||||
// Markers: roots and intercepts as small filled dots.
|
||||
for (const marker of plot.markers) {
|
||||
if (!viewportContainsY(viewport, marker.point.y)) continue;
|
||||
if (marker.point.x < viewport.xMin || marker.point.x > viewport.xMax) continue;
|
||||
const cx = Math.round(px(marker.point.x));
|
||||
const cy = Math.round(py(marker.point.y));
|
||||
image.fillRoundedRect(cx - 3, cy - 3, 6, 6, MARKER_VALUE, 3);
|
||||
}
|
||||
}
|
||||
|
||||
// Frame the plot area.
|
||||
image.drawRect(rect.x, rect.y, rect.width, rect.height, BORDER_VALUE);
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* One-shot completion against the assistant's configured model, used by the
|
||||
* Calculator's optional rewrite fallback (see MathCoordinator.answerSpoken:
|
||||
* the deterministic parser runs first; the model only ever translates words
|
||||
* into notation). Resolves to null when no provider is configured or the
|
||||
* call fails — the calculator then reports "could not read that" instead of
|
||||
* surfacing a transport error.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantModelSetting,
|
||||
openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { CALCULATOR_REWRITE_SYSTEM_PROMPT, buildCalculatorRewriteUserMessage } from "../../prompts";
|
||||
|
||||
export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
const resolved = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
});
|
||||
if (!resolved) return Promise.resolve(null);
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const options: LlmStreamOptions = {
|
||||
apiKey: resolved.apiKey,
|
||||
model: resolved.model,
|
||||
system: CALCULATOR_REWRITE_SYSTEM_PROMPT,
|
||||
effort: "low",
|
||||
maxTokens: 200,
|
||||
messages: [{ role: "user", content: buildCalculatorRewriteUserMessage(spoken) }],
|
||||
onDone: (result) => resolve(result.text ?? null),
|
||||
onError: () => resolve(null),
|
||||
};
|
||||
switch (resolved.provider) {
|
||||
case "openai":
|
||||
streamOpenAiResponse(options);
|
||||
break;
|
||||
case "local":
|
||||
streamLocalQwen(options);
|
||||
break;
|
||||
default:
|
||||
streamAnthropicMessage(options);
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,606 @@
|
||||
// The calculator app layer for FaceClaw: a spoken (or typed) problem in, an
|
||||
// exact answer on the lens.
|
||||
//
|
||||
// Deliberately thin. Everything mathematical belongs to math/ and everything
|
||||
// about the standing problem, follow-ups, and long runs belongs to
|
||||
// MathCoordinator. This layer owns only what is genuinely the app's: which
|
||||
// mode a fresh capture lands in, what the viewport shows, and how the
|
||||
// touchpad and the microphone are shared.
|
||||
|
||||
import { GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { getDefaultLargeFont, getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { wrapText, truncateText } from "../../graphics/textwrap";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { gestureHints, GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../../ui/gestures";
|
||||
import { type DashboardInputEvent, type Layer, type LayerActions, type LayerContext } from "../../ui/layers";
|
||||
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator";
|
||||
import {
|
||||
type LongRunOffer,
|
||||
type LongRunProgress,
|
||||
offerGlassesPrompt,
|
||||
progressText,
|
||||
} from "./math/workload";
|
||||
import { plain } from "./math/text-renderer";
|
||||
import { drawGraph } from "./calculator-graph";
|
||||
import {
|
||||
type CalculatorListening,
|
||||
type CalculatorMode,
|
||||
CALCULATOR_MODES,
|
||||
cycleMode,
|
||||
isDictationCancellation,
|
||||
isWorthAnswering,
|
||||
listeningLabel,
|
||||
modeCommand,
|
||||
modeLabel,
|
||||
} from "./calculator-commands";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Settings
|
||||
|
||||
/** The mode a fresh capture lands in. Also what scrolling cycles. */
|
||||
export const calculatorModeSetting = new ConfigSettingEnum<CalculatorMode>({
|
||||
id: "calculator-mode",
|
||||
label: "Default mode",
|
||||
storageKey: "calculator.defaultMode",
|
||||
defaultValue: "solve",
|
||||
values: [...CALCULATOR_MODES],
|
||||
formatValue: modeLabel,
|
||||
description:
|
||||
"What a freshly stated problem does. On the glasses, scroll applies another mode — or walks Explain steps — and tap toggles the mic.",
|
||||
});
|
||||
|
||||
export const calculatorListeningSetting = new ConfigSettingEnum<CalculatorListening>({
|
||||
id: "calculator-listening",
|
||||
label: "Listening",
|
||||
storageKey: "calculator.listening",
|
||||
defaultValue: "tap",
|
||||
values: ["tap", "continuous"],
|
||||
formatValue: listeningLabel,
|
||||
description:
|
||||
"Tap to talk captures one problem per tap. Continuous listens the whole time the calculator is in front, answering anything that sounds like maths.",
|
||||
});
|
||||
|
||||
export const calculatorDegreesSetting = new ConfigSettingBoolean({
|
||||
id: "calculator-degrees",
|
||||
label: "Trigonometry in degrees",
|
||||
storageKey: "calculator.usesDegrees",
|
||||
defaultValue: false,
|
||||
description: "Evaluate sin/cos/tan in degrees instead of radians.",
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Layer
|
||||
|
||||
export class CalculatorLayer implements Layer {
|
||||
/** Rebound by the window factory once the window exists. */
|
||||
requestRender: () => void = () => {};
|
||||
|
||||
/**
|
||||
* The last thing said to the calculator, so the view can show what it
|
||||
* heard next to what it made of it — the only way a wearer can tell a
|
||||
* mis-heard problem from a wrong answer.
|
||||
*/
|
||||
private lastInput = "";
|
||||
private lastAnswer = "";
|
||||
private lastFailure: string | null = null;
|
||||
private working = false;
|
||||
/** True while one finalized utterance is armed for capture. */
|
||||
private dictating = false;
|
||||
private foreground = true;
|
||||
private continuousActive = false;
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private lastDeepAnswer: MathBranchOutcome | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly coordinator: MathCoordinator,
|
||||
private readonly actions: LayerActions,
|
||||
) {
|
||||
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
|
||||
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
}
|
||||
|
||||
private get mode(): CalculatorMode {
|
||||
return calculatorModeSetting.get();
|
||||
}
|
||||
|
||||
private get listening(): CalculatorListening {
|
||||
return calculatorListeningSetting.get();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Lifecycle
|
||||
|
||||
onRemoved(): void {
|
||||
this.stopListening();
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.coordinator.onChanged = null;
|
||||
}
|
||||
|
||||
/** Foreground/background transitions, forwarded by the window factory. */
|
||||
setForeground(foreground: boolean): void {
|
||||
if (this.foreground === foreground) return;
|
||||
this.foreground = foreground;
|
||||
if (!foreground) {
|
||||
this.stopListening();
|
||||
} else {
|
||||
this.refreshListening();
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply a settings change: degrees and the listening mode both act immediately. */
|
||||
applySettings(): void {
|
||||
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
|
||||
this.refreshListening();
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
private refreshListening(): void {
|
||||
if (!this.foreground) return;
|
||||
if (this.listening === "continuous") {
|
||||
if (!this.continuousActive) {
|
||||
this.continuousActive = true;
|
||||
void this.actions.startContinuousVoiceCapture();
|
||||
}
|
||||
} else if (this.continuousActive) {
|
||||
this.continuousActive = false;
|
||||
void this.actions.stopContinuousVoiceCapture();
|
||||
}
|
||||
}
|
||||
|
||||
private stopListening(): void {
|
||||
if (this.dictating) {
|
||||
this.dictating = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
}
|
||||
if (this.continuousActive) {
|
||||
this.continuousActive = false;
|
||||
void this.actions.stopContinuousVoiceCapture();
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Asking
|
||||
|
||||
/**
|
||||
* Put a problem (or a follow-up command) to the calculator.
|
||||
*
|
||||
* Command first, problem second. "graph it" contains no operator and would
|
||||
* fail as an expression, and a wearer who says it expects the plot of what
|
||||
* is already on the lens rather than an error about the word "it".
|
||||
*/
|
||||
submit(text: string): void {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return;
|
||||
this.lastInput = trimmed;
|
||||
this.lastFailure = null;
|
||||
|
||||
const selected = modeCommand(trimmed);
|
||||
if (selected) {
|
||||
this.performFollowUp(selected);
|
||||
return;
|
||||
}
|
||||
|
||||
const target = this.mode;
|
||||
this.working = true;
|
||||
this.requestRender();
|
||||
void this.coordinator.answerSpoken(trimmed).then((outcome) => {
|
||||
// A fresh problem lands in the app's mode: solving is the default, but
|
||||
// a wearer whose default is "graph" said the problem in order to see it
|
||||
// plotted, not to read its root.
|
||||
if (outcome.kind === "value" && target !== "solve") {
|
||||
outcome = this.coordinator.run(target, "");
|
||||
}
|
||||
this.working = false;
|
||||
this.applyOutcome(outcome);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-run the standing problem through a different presentation branch.
|
||||
* This is the one action used by scroll cycling, spoken "… it" commands,
|
||||
* and the window menu, so changing mode can never mean "change a setting
|
||||
* but leave the old answer up".
|
||||
*/
|
||||
performFollowUp(selected: CalculatorMode): void {
|
||||
calculatorModeSetting.set(selected);
|
||||
if (!this.coordinator.hasProblem) {
|
||||
this.lastFailure = "No problem yet";
|
||||
this.requestRender();
|
||||
return;
|
||||
}
|
||||
this.working = false;
|
||||
this.lastFailure = null;
|
||||
this.applyOutcome(this.coordinator.run(selected, ""));
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.working = false;
|
||||
this.lastInput = "";
|
||||
this.lastAnswer = "";
|
||||
this.lastFailure = null;
|
||||
this.coordinator.handleRingAction("exit");
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
private applyOutcome(outcome: MathBranchOutcome | null): void {
|
||||
if (outcome?.kind === "value") {
|
||||
this.lastAnswer = outcome.text;
|
||||
this.lastFailure = null;
|
||||
} else if (outcome?.kind === "failure") {
|
||||
this.lastFailure = outcome.reason;
|
||||
}
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
private handleCoordinatorChanged(): void {
|
||||
// The answer a deeper pass produced arrives on its own channel; fold it
|
||||
// into the standing answer the moment it lands.
|
||||
const deep = this.coordinator.deepAnswer;
|
||||
if (deep && deep !== this.lastDeepAnswer) {
|
||||
this.lastDeepAnswer = deep;
|
||||
this.applyOutcome(deep);
|
||||
return;
|
||||
}
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Dictation
|
||||
|
||||
/**
|
||||
* Arm exactly one finalized utterance. One, not a listening window: the
|
||||
* indicator on the lens is what makes this honest, and a calculator that
|
||||
* quietly held the microphone open would be indistinguishable from one
|
||||
* that did not.
|
||||
*/
|
||||
beginDictation(): void {
|
||||
if (this.dictating) return;
|
||||
this.dictating = true;
|
||||
void this.actions.startVoiceCapture(true);
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
cancelDictation(): void {
|
||||
if (!this.dictating) return;
|
||||
this.dictating = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
/**
|
||||
* A finalized utterance reached the app.
|
||||
*
|
||||
* In tap mode this only ever arrives because the wearer armed it, so it is
|
||||
* always taken. In continuous mode it also arrives from ordinary room
|
||||
* speech, and there it has to clear the looks-like-maths gate first —
|
||||
* otherwise every sentence in the room becomes a failed calculation on the
|
||||
* lens.
|
||||
*/
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
if (!event.isFinal) return;
|
||||
const text = event.text.trim();
|
||||
if (!text) return;
|
||||
const armed = this.dictating;
|
||||
const continuous = this.continuousActive && this.foreground;
|
||||
if (!armed && !continuous) return;
|
||||
if (armed) {
|
||||
this.dictating = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
}
|
||||
// A standing offer or a run in flight owns the next yes/no said in the
|
||||
// room ("keep going", "never mind", "stop").
|
||||
const longRunAnswer = this.coordinator.handleLongRunUtterance(text);
|
||||
if (longRunAnswer) {
|
||||
this.applyOutcome(longRunAnswer);
|
||||
return;
|
||||
}
|
||||
if (armed && isDictationCancellation(text)) {
|
||||
this.requestRender();
|
||||
return;
|
||||
}
|
||||
if (!armed && !isWorthAnswering(text)) return;
|
||||
this.submit(text);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Input
|
||||
|
||||
receiveTextInput(text: string): void {
|
||||
this.submit(text);
|
||||
}
|
||||
|
||||
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
|
||||
switch (event.type) {
|
||||
case "scroll-up":
|
||||
case "scroll-down": {
|
||||
const forward = event.type === "scroll-down";
|
||||
// While Explain is showing a step sequence the scroll walks it — the
|
||||
// coordinator owns that, and asking it first is what keeps one
|
||||
// gesture from meaning two things at once. Otherwise scroll cycles
|
||||
// the MODE, because the mode is what the wearer changes between
|
||||
// problems and the steps only exist inside one of the three modes.
|
||||
if (this.mode === "explain" && this.coordinator.stepPosition !== null) {
|
||||
this.applyOutcome(this.coordinator.handleRingAction(forward ? "forward" : "back"));
|
||||
return;
|
||||
}
|
||||
const next = cycleMode(this.mode, forward);
|
||||
if (this.coordinator.hasProblem) {
|
||||
this.performFollowUp(next);
|
||||
} else {
|
||||
calculatorModeSetting.set(next);
|
||||
this.requestRender();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case "click": {
|
||||
// A standing offer owns the tap: accept and start the long run.
|
||||
if (this.coordinator.hasLongRunInteraction) {
|
||||
this.applyOutcome(this.coordinator.handleRingAction("commit"));
|
||||
return;
|
||||
}
|
||||
if (this.dictating) {
|
||||
this.cancelDictation();
|
||||
} else {
|
||||
this.beginDictation();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
case "double-click": {
|
||||
if (this.dictating) {
|
||||
this.cancelDictation();
|
||||
return;
|
||||
}
|
||||
if (this.coordinator.isRunningLong || this.coordinator.pendingLongRun) {
|
||||
// Stop the run, or turn the offer down; the shallow answer stands.
|
||||
this.applyOutcome(this.coordinator.handleRingAction("exit"));
|
||||
return;
|
||||
}
|
||||
// The standard leave-the-app gesture.
|
||||
shell.yieldFocusToSidebar();
|
||||
return;
|
||||
}
|
||||
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Paint
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const small = getDefaultSmallFont();
|
||||
|
||||
this.paintTabs(image, small, width);
|
||||
|
||||
const footerY = height - small.lineHeight - 6;
|
||||
const bodyTop = 46;
|
||||
let bodyBottom = footerY - 4;
|
||||
|
||||
// A standing offer or a running deep pass takes the rows just above the
|
||||
// footer, whatever view is behind it.
|
||||
const offer = this.coordinator.pendingLongRun;
|
||||
const progress = this.coordinator.longRunProgress;
|
||||
if (offer || progress) {
|
||||
bodyBottom = this.paintLongRunPanel(image, small, width, bodyBottom, offer, progress);
|
||||
}
|
||||
|
||||
if (this.working) {
|
||||
this.drawCentered(image, small, Math.round((bodyTop + bodyBottom) / 2) - 8, "Working…", 180);
|
||||
} else if (this.lastFailure !== null) {
|
||||
this.paintFailure(image, small, width, bodyTop, bodyBottom);
|
||||
} else if (this.mode === "explain" && this.coordinator.stepPosition !== null) {
|
||||
this.paintExplain(image, width, bodyTop, bodyBottom);
|
||||
} else if (this.mode === "graph" && this.coordinator.lastGraph) {
|
||||
this.paintGraph(image, small, width, bodyTop, bodyBottom);
|
||||
} else if (this.lastAnswer) {
|
||||
this.paintAnswer(image, width, bodyTop, bodyBottom);
|
||||
} else {
|
||||
this.paintEmpty(image, small, width, bodyTop, bodyBottom);
|
||||
}
|
||||
|
||||
image.drawText(small, 24, footerY, this.footerHints(), 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
/** The three modes as tabs, plus the deg/listening status chips. */
|
||||
private paintTabs(image: GrayImage, font: UiFont, width: number): void {
|
||||
let x = 24;
|
||||
for (const mode of CALCULATOR_MODES) {
|
||||
const label = modeLabel(mode);
|
||||
const active = mode === this.mode;
|
||||
image.drawText(font, x, 15, label, active ? 245 : 105);
|
||||
image.drawLine(x, 35, x + font.measureText(label), 35, active ? 200 : 30);
|
||||
x += font.measureText(label) + 18;
|
||||
}
|
||||
|
||||
const chips: string[] = [];
|
||||
if (calculatorDegreesSetting.get()) chips.push("deg");
|
||||
if (this.dictating) chips.push("listening…");
|
||||
else if (this.continuousActive && this.foreground) chips.push("mic on");
|
||||
if (chips.length > 0) {
|
||||
const text = chips.join(" · ");
|
||||
image.drawText(font, width - 24 - font.measureText(text), 15, text, this.dictating ? 220 : 110);
|
||||
}
|
||||
}
|
||||
|
||||
private paintFailure(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
width: number,
|
||||
bodyTop: number,
|
||||
bodyBottom: number,
|
||||
): void {
|
||||
this.paintHeardLine(image, font, width, bodyTop);
|
||||
const top = this.lastInput ? bodyTop + lineStep(font) + 6 : bodyTop + 8;
|
||||
const lines = wrapText(font, this.lastFailure ?? "", width - 64);
|
||||
const step = lineStep(font);
|
||||
const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step));
|
||||
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
|
||||
image.drawText(font, 32, top + index * step, lines[index]!, 200);
|
||||
}
|
||||
}
|
||||
|
||||
private paintExplain(image: GrayImage, width: number, bodyTop: number, bodyBottom: number): void {
|
||||
const small = getDefaultSmallFont();
|
||||
const medium = getDefaultMediumFont();
|
||||
const position = this.coordinator.stepPosition;
|
||||
const steps = this.coordinator.explanationSteps;
|
||||
const step = position ? steps[position.index] : undefined;
|
||||
if (!position || !step) return;
|
||||
|
||||
let y = bodyTop + 2;
|
||||
image.drawText(small, 24, y, `Step ${position.index + 1} of ${position.total}`, 130);
|
||||
y += lineStep(small) + 2;
|
||||
image.drawText(medium, 24, y, truncateText(medium, step.title, width - 48), 245);
|
||||
y += lineStep(medium) + 2;
|
||||
image.drawText(medium, 24, y, truncateText(medium, plain(step.expression), width - 48), 220);
|
||||
y += lineStep(medium) + 6;
|
||||
|
||||
const detailLines = wrapText(small, step.detail, width - 64);
|
||||
const stepPitch = lineStep(small);
|
||||
const maxLines = Math.max(0, Math.floor((bodyBottom - y) / stepPitch));
|
||||
for (let index = 0; index < Math.min(detailLines.length, maxLines); index++) {
|
||||
image.drawText(small, 32, y + index * stepPitch, detailLines[index]!, 150);
|
||||
}
|
||||
}
|
||||
|
||||
private paintGraph(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
width: number,
|
||||
bodyTop: number,
|
||||
bodyBottom: number,
|
||||
): void {
|
||||
const graph = this.coordinator.lastGraph;
|
||||
if (!graph) return;
|
||||
const headline = this.lastAnswer.split("\n")[0] ?? "";
|
||||
image.drawText(font, 24, bodyTop, truncateText(font, headline, width - 48), 200);
|
||||
const plotTop = bodyTop + lineStep(font) + 4;
|
||||
drawGraph(image, graph, {
|
||||
x: 18,
|
||||
y: plotTop,
|
||||
width: width - 36,
|
||||
height: Math.max(2, bodyBottom - plotTop),
|
||||
});
|
||||
}
|
||||
|
||||
private paintAnswer(image: GrayImage, width: number, bodyTop: number, bodyBottom: number): void {
|
||||
const small = getDefaultSmallFont();
|
||||
this.paintHeardLine(image, small, width, bodyTop);
|
||||
const top = this.lastInput ? bodyTop + lineStep(small) + 4 : bodyTop;
|
||||
|
||||
// A short single-line answer earns the large font; anything longer drops
|
||||
// to the medium font and wraps.
|
||||
const lines = this.lastAnswer.split("\n");
|
||||
const large = getDefaultLargeFont();
|
||||
if (lines.length === 1 && large.measureText(lines[0]!) <= width - 64) {
|
||||
const y = Math.round((top + bodyBottom - large.lineHeight) / 2);
|
||||
this.drawCentered(image, large, y, lines[0]!, 245);
|
||||
return;
|
||||
}
|
||||
const medium = getDefaultMediumFont();
|
||||
const wrapped = lines.flatMap((line) => wrapText(medium, line, width - 64));
|
||||
const step = lineStep(medium);
|
||||
const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step));
|
||||
const shown = wrapped.slice(0, maxLines);
|
||||
let y = Math.max(top, Math.round((top + bodyBottom - shown.length * step) / 2));
|
||||
for (const line of shown) {
|
||||
this.drawCentered(image, medium, y, line, 235);
|
||||
y += step;
|
||||
}
|
||||
}
|
||||
|
||||
private paintEmpty(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
width: number,
|
||||
bodyTop: number,
|
||||
bodyBottom: number,
|
||||
): void {
|
||||
const prompt =
|
||||
this.listening === "continuous"
|
||||
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
|
||||
: "Tap to say a problem — or use Voice input to type one";
|
||||
const lines = wrapText(font, prompt, width - 96);
|
||||
const step = lineStep(font);
|
||||
let y = Math.round((bodyTop + bodyBottom - lines.length * step) / 2);
|
||||
for (const line of lines) {
|
||||
this.drawCentered(image, font, y, line, 150);
|
||||
y += step;
|
||||
}
|
||||
}
|
||||
|
||||
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
|
||||
if (!this.lastInput) return;
|
||||
const label = "Heard";
|
||||
image.drawText(font, 24, y, label, 110);
|
||||
const x = 24 + font.measureText(label) + 10;
|
||||
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
|
||||
}
|
||||
|
||||
/**
|
||||
* The standing offer, or the live progress of an accepted run, in the two
|
||||
* rows above the footer. Returns the new body bottom.
|
||||
*/
|
||||
private paintLongRunPanel(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
width: number,
|
||||
bodyBottom: number,
|
||||
offer: LongRunOffer | null,
|
||||
progress: LongRunProgress | null,
|
||||
): number {
|
||||
const step = lineStep(font);
|
||||
const lines = progress
|
||||
? [`${progressText(progress)} · ${Math.round(progress.elapsedSeconds)}s`]
|
||||
: offer
|
||||
? offerGlassesPrompt(offer).split("\n")
|
||||
: [];
|
||||
const panelTop = bodyBottom - lines.length * step - 6;
|
||||
image.drawLine(16, panelTop, width - 16, panelTop, 40);
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
image.drawText(font, 24, panelTop + 4 + index * step, truncateText(font, lines[index]!, width - 48), 190);
|
||||
}
|
||||
return panelTop - 2;
|
||||
}
|
||||
|
||||
private footerHints(): string {
|
||||
if (this.coordinator.longRunProgress) {
|
||||
return gestureHints([[GESTURE_DOUBLE_CLICK, "stop"]]);
|
||||
}
|
||||
if (this.coordinator.pendingLongRun) {
|
||||
return gestureHints([
|
||||
[GESTURE_CLICK, "keep going"],
|
||||
[GESTURE_DOUBLE_CLICK, "leave it"],
|
||||
]);
|
||||
}
|
||||
if (this.dictating) {
|
||||
return gestureHints([
|
||||
[GESTURE_CLICK, "cancel mic"],
|
||||
[GESTURE_DOUBLE_CLICK, "cancel"],
|
||||
]);
|
||||
}
|
||||
const scrollTarget =
|
||||
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, scrollTarget],
|
||||
[GESTURE_CLICK, "mic"],
|
||||
[GESTURE_DOUBLE_CLICK, "back"],
|
||||
]);
|
||||
}
|
||||
|
||||
private drawCentered(image: GrayImage, font: UiFont, y: number, text: string, value: number): void {
|
||||
const x = Math.max(0, Math.round((image.width - font.measureText(text)) / 2));
|
||||
image.drawText(font, x, y, text, value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import {
|
||||
CALCULATOR_SURFACE_ID,
|
||||
CALCULATOR_WINDOW_ID,
|
||||
createCalculatorAppWindow,
|
||||
} from "./calculator-app";
|
||||
|
||||
/**
|
||||
* A spoken (or typed) problem in, an exact answer on the lens — with
|
||||
* step-by-step explanations, graphs, and the advanced spoken-question
|
||||
* topics (primality, integrals, digit searches in π, geometry).
|
||||
*/
|
||||
const calculatorApp: AppDefinition = {
|
||||
appId: "calculator",
|
||||
title: "Calculator",
|
||||
icon: "calculator",
|
||||
launch: (ctx) => ctx.launchInProcessApp(CALCULATOR_WINDOW_ID, CALCULATOR_SURFACE_ID, createCalculatorAppWindow),
|
||||
};
|
||||
|
||||
export default calculatorApp;
|
||||
@@ -0,0 +1,796 @@
|
||||
// Routes a spoken question to the right advanced-maths topic. Separate from
|
||||
// the command parser because these questions have a different shape: a
|
||||
// command is about an EXPRESSION and what to do with it, whereas "is 91
|
||||
// prime", "factor 360", "area of a circle with radius 3" name a topic and
|
||||
// carry their own arguments. Trying to force both through one parser made
|
||||
// the expression grammar swallow phrases it had no business touching.
|
||||
|
||||
import { type MathExpression, variablesOf } from "./expression";
|
||||
import { parseSpoken } from "./parser";
|
||||
import { applyNumberWords } from "./normalizer";
|
||||
import { plain } from "./text-renderer";
|
||||
import { describeNumber, formatInt, fromDecimal } from "./math-number";
|
||||
import {
|
||||
type MathConstantName,
|
||||
MAXIMUM_COMPUTED_DIGITS,
|
||||
constantDisplayName,
|
||||
constantNamed,
|
||||
constantSpokenName,
|
||||
} from "./constant-digits";
|
||||
import {
|
||||
countDigits as countDigitsIn,
|
||||
digitsAt,
|
||||
expectedFirstPosition,
|
||||
findDigits,
|
||||
highlightedMatch,
|
||||
} from "./digit-search";
|
||||
import { type MathWorkload, longRunOffer as makeLongRunOffer, type LongRunOffer } from "./workload";
|
||||
import {
|
||||
binomial,
|
||||
factorizationText,
|
||||
fibonacci,
|
||||
greatestCommonDivisor,
|
||||
integerReport,
|
||||
isPrime,
|
||||
leastCommonMultiple,
|
||||
modularInverse,
|
||||
} from "./number-theory";
|
||||
import {
|
||||
type Shape,
|
||||
pythagoras,
|
||||
shapeName,
|
||||
shapeSummary,
|
||||
triangleAnglesInDegrees,
|
||||
triangleArea,
|
||||
triangleFromSides,
|
||||
triangleKind,
|
||||
trianglePerimeter,
|
||||
} from "./geometry";
|
||||
import { TRIG_IDENTITIES, familyText, identityMatching, solveTrig } from "./trigonometry";
|
||||
import { criticalPointText, criticalPoints, limit, limitText, taylor } from "./calculus";
|
||||
import { definite, integrateWithConstant } from "./symbolic-integral";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Topics
|
||||
|
||||
export type AdvancedTopic =
|
||||
| { kind: "integerReport"; value: number }
|
||||
| { kind: "primality"; value: number }
|
||||
| { kind: "factorize"; value: number }
|
||||
| { kind: "gcd"; a: number; b: number }
|
||||
| { kind: "lcm"; a: number; b: number }
|
||||
| { kind: "modularInverse"; value: number; modulus: number }
|
||||
| { kind: "fibonacci"; n: number }
|
||||
| { kind: "binomial"; n: number; k: number }
|
||||
| { kind: "symbolicIntegral"; expression: MathExpression; variable: string }
|
||||
| { kind: "definiteIntegral"; expression: MathExpression; variable: string; from: number; to: number }
|
||||
| { kind: "limit"; expression: MathExpression; variable: string; at: number }
|
||||
| { kind: "taylor"; expression: MathExpression; variable: string; order: number }
|
||||
| { kind: "criticalPoints"; expression: MathExpression; variable: string }
|
||||
| { kind: "trigSolve"; fn: "sin" | "cos" | "tan"; value: number }
|
||||
| { kind: "trigIdentity"; query: string }
|
||||
/** "where does 11 first appear in pi" */
|
||||
| { kind: "digitSearch"; pattern: string; constant: MathConstantName; occurrence: number }
|
||||
/** "how many times does 42 appear in pi" */
|
||||
| { kind: "digitCount"; pattern: string; constant: MathConstantName }
|
||||
/** "what are digits 100 to 110 of pi" */
|
||||
| { kind: "digitsAt"; constant: MathConstantName; from: number; to: number }
|
||||
| { kind: "shapeArea"; shape: Shape }
|
||||
| { kind: "triangleFromSides"; a: number; b: number; c: number }
|
||||
| { kind: "pythagoras"; a: number | null; b: number | null; c: number | null };
|
||||
|
||||
export type AdvancedAnswer = {
|
||||
headline: string;
|
||||
detail: string;
|
||||
/** Extra rows for the answer view, and for a longer readback. */
|
||||
fields: [string, string][];
|
||||
/**
|
||||
* Set when this answer is only as good as the fast expansion allowed, and
|
||||
* a deeper opted-in run would do better. The router itself never starts
|
||||
* one: it is pure and synchronous, and the wait is a decision belonging to
|
||||
* the caller.
|
||||
*/
|
||||
unresolvedWorkload: MathWorkload | null;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsing
|
||||
|
||||
/**
|
||||
* null when the utterance is not an advanced-maths question, so the caller
|
||||
* can fall through to the ordinary expression path.
|
||||
*/
|
||||
export function parseAdvanced(utterance: string): AdvancedTopic | null {
|
||||
const text = applyNumberWords(utterance.toLowerCase()).replace(/^[ .,!?]+|[ .,!?]+$/g, "");
|
||||
|
||||
// Constants first: "the first 11 in pi" contains digits and the word "in",
|
||||
// which the geometry and number-theory matchers would both take a swing at.
|
||||
const constantTopic = parseConstantDigits(text);
|
||||
if (constantTopic) return constantTopic;
|
||||
const numberTopic = parseNumberTheory(text);
|
||||
if (numberTopic) return numberTopic;
|
||||
const calculusTopic = parseCalculus(text, utterance);
|
||||
if (calculusTopic) return calculusTopic;
|
||||
const trigTopic = parseTrigonometry(text);
|
||||
if (trigTopic) return trigTopic;
|
||||
return parseGeometry(text);
|
||||
}
|
||||
|
||||
// Constant digits ------------------------------------------------------------
|
||||
|
||||
function parseConstantDigits(text: string): AdvancedTopic | null {
|
||||
// Must name a constant AND talk about its expansion, or "area of a circle"
|
||||
// (which contains pi implicitly) would land here.
|
||||
const constant = constantNamed(text);
|
||||
if (!constant) return null;
|
||||
const mentionsDigits =
|
||||
text.includes("digit") ||
|
||||
text.includes("appear") ||
|
||||
text.includes("occur") ||
|
||||
text.includes("within") ||
|
||||
text.includes("inside") ||
|
||||
text.includes("expansion") ||
|
||||
text.includes("decimal place") ||
|
||||
text.includes("position") ||
|
||||
text.includes("show up") ||
|
||||
text.includes("found in");
|
||||
if (!mentionsDigits) return null;
|
||||
|
||||
const numbers = allInts(text);
|
||||
|
||||
// "digits 100 to 110 of pi"
|
||||
if (
|
||||
text.includes("digit") &&
|
||||
text.includes(" to ") &&
|
||||
numbers.length >= 2 &&
|
||||
!text.includes("appear") &&
|
||||
!text.includes("occur")
|
||||
) {
|
||||
const sorted = [numbers[0]!, numbers[1]!].sort((a, b) => a - b);
|
||||
return { kind: "digitsAt", constant, from: sorted[0]!, to: sorted[1]! };
|
||||
}
|
||||
|
||||
// "how many times does 42 appear"
|
||||
if (text.includes("how many") || text.includes("count")) {
|
||||
const pattern = numbers[0];
|
||||
if (pattern === undefined) return null;
|
||||
return { kind: "digitCount", pattern: String(pattern), constant };
|
||||
}
|
||||
|
||||
const pattern = numbers[0];
|
||||
if (pattern === undefined) return null;
|
||||
// "the third time 11 appears"
|
||||
const occurrence = ordinalIn(text) ?? 1;
|
||||
return { kind: "digitSearch", pattern: String(pattern), constant, occurrence };
|
||||
}
|
||||
|
||||
/**
|
||||
* "3rd", not "3th". Cheap to get right and jarring to get wrong on a line
|
||||
* someone reads aloud.
|
||||
*/
|
||||
export function ordinalWord(value: number): string {
|
||||
switch (value) {
|
||||
case 1:
|
||||
return "first";
|
||||
case 2:
|
||||
return "second";
|
||||
case 3:
|
||||
return "third";
|
||||
default:
|
||||
break;
|
||||
}
|
||||
let suffix: string;
|
||||
const mod100 = value % 100;
|
||||
const mod10 = value % 10;
|
||||
if (mod100 === 11 || mod100 === 12 || mod100 === 13) suffix = "th";
|
||||
else if (mod10 === 1) suffix = "st";
|
||||
else if (mod10 === 2) suffix = "nd";
|
||||
else if (mod10 === 3) suffix = "rd";
|
||||
else suffix = "th";
|
||||
return `${value}${suffix}`;
|
||||
}
|
||||
|
||||
function ordinalIn(text: string): number | null {
|
||||
const words: [string, number][] = [
|
||||
["first", 1],
|
||||
["second", 2],
|
||||
["third", 3],
|
||||
["fourth", 4],
|
||||
["fifth", 5],
|
||||
["sixth", 6],
|
||||
["seventh", 7],
|
||||
["eighth", 8],
|
||||
["ninth", 9],
|
||||
["tenth", 10],
|
||||
];
|
||||
for (const [word, value] of words) {
|
||||
if (text.includes(word)) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Number theory --------------------------------------------------------------
|
||||
|
||||
function parseNumberTheory(text: string): AdvancedTopic | null {
|
||||
if (text.includes("prime") && !text.includes("factor")) {
|
||||
const value = firstInt(text);
|
||||
if (value !== null) return { kind: "primality", value };
|
||||
}
|
||||
if (text.includes("factor")) {
|
||||
const value = firstInt(text);
|
||||
if (value !== null) return { kind: "factorize", value };
|
||||
}
|
||||
if (text.includes("gcd") || text.includes("greatest common") || text.includes("highest common")) {
|
||||
const values = allInts(text);
|
||||
if (values.length >= 2) return { kind: "gcd", a: values[0]!, b: values[1]! };
|
||||
}
|
||||
if (
|
||||
text.includes("lcm") ||
|
||||
text.includes("lowest common multiple") ||
|
||||
text.includes("least common multiple")
|
||||
) {
|
||||
const values = allInts(text);
|
||||
if (values.length >= 2) return { kind: "lcm", a: values[0]!, b: values[1]! };
|
||||
}
|
||||
if (text.includes("inverse") && text.includes("mod")) {
|
||||
const values = allInts(text);
|
||||
if (values.length >= 2) return { kind: "modularInverse", value: values[0]!, modulus: values[1]! };
|
||||
}
|
||||
if (text.includes("fibonacci")) {
|
||||
const value = firstInt(text);
|
||||
if (value !== null) return { kind: "fibonacci", n: value };
|
||||
}
|
||||
if (text.includes("choose") || text.includes("binomial") || text.includes("combinations")) {
|
||||
const values = allInts(text);
|
||||
if (values.length >= 2) return { kind: "binomial", n: values[0]!, k: values[1]! };
|
||||
}
|
||||
if (text.includes("tell me about") || text.includes("facts about")) {
|
||||
const value = firstInt(text);
|
||||
if (value !== null) return { kind: "integerReport", value };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculus -------------------------------------------------------------------
|
||||
|
||||
function parseCalculus(text: string, original: string): AdvancedTopic | null {
|
||||
if (text.includes("integral") || text.includes("integrate") || text.includes("antiderivative")) {
|
||||
const expression = expressionTail(original, ["integral of", "integrate", "antiderivative of", "integral"]);
|
||||
if (!expression) return null;
|
||||
const variable = variablesOf(expression)[0] ?? "x";
|
||||
const bounds = boundsIn(text);
|
||||
if (bounds) {
|
||||
return { kind: "definiteIntegral", expression, variable, from: bounds[0], to: bounds[1] };
|
||||
}
|
||||
return { kind: "symbolicIntegral", expression, variable };
|
||||
}
|
||||
|
||||
if (text.includes("limit")) {
|
||||
const expression = expressionTail(original, ["limit of", "the limit of", "limit"]);
|
||||
if (!expression) return null;
|
||||
// "as x approaches 0"
|
||||
const point = firstDoubleAfter(["approaches", "approaching", "tends to", "goes to"], text) ?? 0;
|
||||
return { kind: "limit", expression, variable: variablesOf(expression)[0] ?? "x", at: point };
|
||||
}
|
||||
|
||||
if (text.includes("taylor") || text.includes("maclaurin") || text.includes("series")) {
|
||||
const expression = expressionTail(original, [
|
||||
"series of",
|
||||
"series for",
|
||||
"expansion of",
|
||||
"taylor",
|
||||
"maclaurin",
|
||||
]);
|
||||
if (!expression) return null;
|
||||
const orderValue = firstDoubleAfter(["order", "to order", "terms"], text);
|
||||
const order = orderValue !== null ? Math.trunc(orderValue) : 5;
|
||||
return { kind: "taylor", expression, variable: variablesOf(expression)[0] ?? "x", order };
|
||||
}
|
||||
|
||||
if (
|
||||
text.includes("critical point") ||
|
||||
text.includes("turning point") ||
|
||||
text.includes("maximum") ||
|
||||
text.includes("minimum") ||
|
||||
text.includes("extrema")
|
||||
) {
|
||||
const expression = expressionTail(original, [
|
||||
"critical points of",
|
||||
"turning points of",
|
||||
"extrema of",
|
||||
"maximum of",
|
||||
"minimum of",
|
||||
"of",
|
||||
]);
|
||||
if (!expression) return null;
|
||||
return { kind: "criticalPoints", expression, variable: variablesOf(expression)[0] ?? "x" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Trigonometry ---------------------------------------------------------------
|
||||
|
||||
function parseTrigonometry(text: string): AdvancedTopic | null {
|
||||
if (text.includes("identity") || text.includes("law of")) {
|
||||
return { kind: "trigIdentity", query: text };
|
||||
}
|
||||
// "solve sin x = 0.5"
|
||||
for (const name of ["sin", "cos", "tan"] as const) {
|
||||
if (!text.includes(name)) continue;
|
||||
if (!text.includes("=") && !text.includes("equals")) continue;
|
||||
const value = lastDouble(text);
|
||||
if (value === null) continue;
|
||||
return { kind: "trigSolve", fn: name, value };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Geometry -------------------------------------------------------------------
|
||||
|
||||
function parseGeometry(text: string): AdvancedTopic | null {
|
||||
const numbers = allDoubles(text);
|
||||
|
||||
if (text.includes("pythagoras") || text.includes("hypotenuse")) {
|
||||
if (numbers.length < 2) return null;
|
||||
return text.includes("hypotenuse") && text.includes("find")
|
||||
? { kind: "pythagoras", a: numbers[0]!, b: numbers[1]!, c: null }
|
||||
: { kind: "pythagoras", a: numbers[0]!, b: null, c: numbers[1]! };
|
||||
}
|
||||
if (text.includes("triangle") && numbers.length >= 3 && text.includes("side")) {
|
||||
return { kind: "triangleFromSides", a: numbers[0]!, b: numbers[1]!, c: numbers[2]! };
|
||||
}
|
||||
|
||||
// Shapes, longest name first so "rectangular prism" is not read as
|
||||
// "rectangle".
|
||||
const shapes: [string, (values: number[]) => Shape | null][] = [
|
||||
[
|
||||
"rectangular prism",
|
||||
(values) =>
|
||||
values.length >= 3
|
||||
? { kind: "rectangularPrism", length: values[0]!, width: values[1]!, height: values[2]! }
|
||||
: null,
|
||||
],
|
||||
["sphere", (values) => (values.length >= 1 ? { kind: "sphere", radius: values[0]! } : null)],
|
||||
[
|
||||
"cylinder",
|
||||
(values) => (values.length >= 2 ? { kind: "cylinder", radius: values[0]!, height: values[1]! } : null),
|
||||
],
|
||||
["cone", (values) => (values.length >= 2 ? { kind: "cone", radius: values[0]!, height: values[1]! } : null)],
|
||||
["cube", (values) => (values.length >= 1 ? { kind: "cube", side: values[0]! } : null)],
|
||||
["circle", (values) => (values.length >= 1 ? { kind: "circle", radius: values[0]! } : null)],
|
||||
[
|
||||
"rectangle",
|
||||
(values) => (values.length >= 2 ? { kind: "rectangle", width: values[0]!, height: values[1]! } : null),
|
||||
],
|
||||
["square", (values) => (values.length >= 1 ? { kind: "square", side: values[0]! } : null)],
|
||||
[
|
||||
"trapezoid",
|
||||
(values) =>
|
||||
values.length >= 3 ? { kind: "trapezoid", a: values[0]!, b: values[1]!, height: values[2]! } : null,
|
||||
],
|
||||
["ellipse", (values) => (values.length >= 2 ? { kind: "ellipse", a: values[0]!, b: values[1]! } : null)],
|
||||
];
|
||||
const wantsMeasurement =
|
||||
text.includes("area") ||
|
||||
text.includes("volume") ||
|
||||
text.includes("perimeter") ||
|
||||
text.includes("circumference") ||
|
||||
text.includes("surface");
|
||||
if (!wantsMeasurement) return null;
|
||||
for (const [name, build] of shapes) {
|
||||
if (!text.includes(name)) continue;
|
||||
const shape = build(numbers);
|
||||
if (shape) return { kind: "shapeArea", shape };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Answering
|
||||
|
||||
export function answerAdvanced(topic: AdvancedTopic): AdvancedAnswer {
|
||||
switch (topic.kind) {
|
||||
case "integerReport": {
|
||||
const report = integerReport(topic.value);
|
||||
const fields: [string, string][] = [
|
||||
["Prime", report.isPrime ? "yes" : "no"],
|
||||
["Factorisation", factorizationText(report)],
|
||||
["Divisors", String(report.divisorCount)],
|
||||
["Divisor sum", String(report.divisorSum)],
|
||||
["Totient", String(report.totient)],
|
||||
["Digit sum", String(report.digitSum)],
|
||||
["Digital root", String(report.digitalRoot)],
|
||||
];
|
||||
if (report.isPerfect) fields.push(["Perfect", "yes"]);
|
||||
return answer(
|
||||
`${topic.value} = ${factorizationText(report)}`,
|
||||
`${topic.value} is ${report.isPrime ? "prime" : "composite"}, ` +
|
||||
`with ${report.divisorCount} divisors summing to ${report.divisorSum}.`,
|
||||
fields,
|
||||
);
|
||||
}
|
||||
|
||||
case "primality": {
|
||||
const prime = isPrime(topic.value);
|
||||
const report = prime ? "" : factorizationText(integerReport(topic.value));
|
||||
return answer(
|
||||
`${topic.value} is ${prime ? "prime" : "not prime"}`,
|
||||
prime
|
||||
? `${topic.value} has no divisors other than 1 and itself.`
|
||||
: `${topic.value} = ${report}.`,
|
||||
);
|
||||
}
|
||||
|
||||
case "factorize": {
|
||||
const report = integerReport(topic.value);
|
||||
return answer(
|
||||
`${topic.value} = ${factorizationText(report)}`,
|
||||
`${topic.value} factors into ${factorizationText(report)}.`,
|
||||
);
|
||||
}
|
||||
|
||||
case "gcd": {
|
||||
const gcd = greatestCommonDivisor(topic.a, topic.b);
|
||||
return answer(
|
||||
`gcd(${topic.a}, ${topic.b}) = ${gcd}`,
|
||||
`The greatest common divisor of ${topic.a} and ${topic.b} is ${gcd}.`,
|
||||
);
|
||||
}
|
||||
|
||||
case "lcm": {
|
||||
const lcm = leastCommonMultiple(topic.a, topic.b);
|
||||
return answer(
|
||||
`lcm(${topic.a}, ${topic.b}) = ${lcm}`,
|
||||
`The least common multiple of ${topic.a} and ${topic.b} is ${lcm}.`,
|
||||
);
|
||||
}
|
||||
|
||||
case "modularInverse": {
|
||||
const inverse = modularInverse(topic.value, topic.modulus);
|
||||
if (inverse === null) {
|
||||
return answer(
|
||||
"No inverse",
|
||||
`${topic.value} and ${topic.modulus} share a factor, so no modular inverse exists.`,
|
||||
);
|
||||
}
|
||||
return answer(
|
||||
`${topic.value}⁻¹ ≡ ${inverse} (mod ${topic.modulus})`,
|
||||
`${topic.value} × ${inverse} ≡ 1 (mod ${topic.modulus}).`,
|
||||
);
|
||||
}
|
||||
|
||||
case "fibonacci": {
|
||||
const value = fibonacci(topic.n);
|
||||
if (value === null) {
|
||||
return answer("Too large", `F(${topic.n}) does not fit in an exact integer.`);
|
||||
}
|
||||
return answer(`F(${topic.n}) = ${value}`, `The ${topic.n}th Fibonacci number is ${value}.`);
|
||||
}
|
||||
|
||||
case "binomial": {
|
||||
const value = binomial(topic.n, topic.k);
|
||||
if (value === null) {
|
||||
return answer("Undefined", `${topic.n} choose ${topic.k} is not defined.`);
|
||||
}
|
||||
return answer(
|
||||
`C(${topic.n}, ${topic.k}) = ${value}`,
|
||||
`There are ${value} ways to choose ${topic.k} from ${topic.n}.`,
|
||||
);
|
||||
}
|
||||
|
||||
case "symbolicIntegral": {
|
||||
const antiderivative = integrateWithConstant(topic.expression, topic.variable);
|
||||
if (!antiderivative) {
|
||||
return answer(
|
||||
"No closed form",
|
||||
"This has no elementary antiderivative that this build can find. " +
|
||||
"Ask for a definite integral and it will be computed numerically.",
|
||||
);
|
||||
}
|
||||
return answer(
|
||||
`∫ = ${plain(antiderivative)}`,
|
||||
`∫ ${plain(topic.expression)} d${topic.variable} = ${plain(antiderivative)}`,
|
||||
);
|
||||
}
|
||||
|
||||
case "definiteIntegral": {
|
||||
const result = definite(topic.expression, topic.variable, topic.from, topic.to);
|
||||
switch (result.kind) {
|
||||
case "exact":
|
||||
return answer(
|
||||
`∫ = ${describeNumber(result.value)}`,
|
||||
`Using F = ${plain(result.antiderivative)}, the definite integral is ${describeNumber(result.value)}.`,
|
||||
);
|
||||
case "numeric":
|
||||
return answer(
|
||||
`∫ ≈ ${describeNumber(fromDecimal(result.value))}`,
|
||||
"No closed form was found, so this was integrated numerically.",
|
||||
);
|
||||
case "failed":
|
||||
return answer("Could not integrate", result.reason);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "limit": {
|
||||
const result = limit(topic.expression, topic.variable, topic.at);
|
||||
let detail =
|
||||
`As ${topic.variable} → ${format(topic.at)}, ` +
|
||||
`${plain(topic.expression)} → ${limitText(result)}.`;
|
||||
if (result.kind === "doesNotExist") {
|
||||
detail =
|
||||
`The one-sided limits disagree ` +
|
||||
`(${format(result.left ?? 0)} from the left, ${format(result.right ?? 0)} from the right), ` +
|
||||
`so the limit does not exist.`;
|
||||
}
|
||||
return answer(`limit = ${limitText(result)}`, detail);
|
||||
}
|
||||
|
||||
case "taylor": {
|
||||
const series = taylor(topic.expression, topic.variable, topic.order);
|
||||
if (!series) {
|
||||
return answer("No series", "Could not expand that about 0.");
|
||||
}
|
||||
return answer(
|
||||
plain(series),
|
||||
`${plain(topic.expression)} ≈ ${plain(series)} to order ${topic.order}.`,
|
||||
);
|
||||
}
|
||||
|
||||
case "criticalPoints": {
|
||||
const points = criticalPoints(topic.expression, topic.variable);
|
||||
if (points.length === 0) {
|
||||
return answer("No critical points", "The derivative never vanishes in the search window.");
|
||||
}
|
||||
return answer(
|
||||
points
|
||||
.slice(0, 2)
|
||||
.map(criticalPointText)
|
||||
.join(", "),
|
||||
points.map(criticalPointText).join("; "),
|
||||
points.map((point) => [
|
||||
point.kind.charAt(0).toUpperCase() + point.kind.slice(1),
|
||||
`(${format(point.x)}, ${format(point.y)})`,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
case "trigSolve": {
|
||||
const solution = solveTrig(topic.fn, topic.value);
|
||||
switch (solution.kind) {
|
||||
case "none":
|
||||
return answer("No solution", `${topic.fn} never reaches ${format(topic.value)}.`);
|
||||
case "all":
|
||||
return answer("True for every angle", "Any value works.");
|
||||
case "families": {
|
||||
const text = solution.families.map(familyText).join(", ");
|
||||
return answer(
|
||||
`x = ${text}`,
|
||||
`${topic.fn} x = ${format(topic.value)} for x = ${text}, where n is any integer.`,
|
||||
solution.families.map((family, index) => [`Family ${index + 1}`, familyText(family)]),
|
||||
);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "trigIdentity": {
|
||||
const identity = identityMatching(topic.query);
|
||||
if (!identity) {
|
||||
return answer(
|
||||
"Not in the sheet",
|
||||
"Known identities: " + TRIG_IDENTITIES.map((entry) => entry.name).join(", "),
|
||||
);
|
||||
}
|
||||
return answer(identity.plain, `${identity.name}: ${identity.plain}`);
|
||||
}
|
||||
|
||||
case "digitSearch": {
|
||||
const outcome = findDigits(topic.pattern, topic.constant, topic.occurrence);
|
||||
switch (outcome.kind) {
|
||||
case "invalidPattern":
|
||||
return answer("Cannot search", outcome.reason);
|
||||
case "notFound": {
|
||||
// "Not in pi" and "not in the first 10,000 digits of pi" are very
|
||||
// different claims, and only the second one is true.
|
||||
let detail =
|
||||
`${topic.pattern} does not appear in the first ` +
|
||||
`${formatInt(outcome.searchedDigits)} digits of ${constantSpokenName(topic.constant)}.`;
|
||||
const expected = expectedFirstPosition(topic.pattern.length);
|
||||
if (expected !== null && expected > outcome.searchedDigits) {
|
||||
const article = [8, 11, 18].includes(topic.pattern.length) ? "An" : "A";
|
||||
detail +=
|
||||
` ${article} ${topic.pattern.length}-digit run turns up around the ` +
|
||||
`${formatInt(expected)}th place on average, so this needs a deeper expansion.`;
|
||||
}
|
||||
return {
|
||||
headline: `Not in the first ${formatInt(outcome.searchedDigits)} digits`,
|
||||
detail,
|
||||
fields: [],
|
||||
unresolvedWorkload: {
|
||||
kind: "digitSearch",
|
||||
constant: topic.constant,
|
||||
pattern: topic.pattern,
|
||||
occurrence: topic.occurrence,
|
||||
},
|
||||
};
|
||||
}
|
||||
case "found": {
|
||||
const match = outcome.match;
|
||||
const ordinalText = ordinalWord(topic.occurrence);
|
||||
const places =
|
||||
match.start === match.end ? `place ${match.start}` : `places ${match.start}–${match.end}`;
|
||||
return answer(
|
||||
`${constantDisplayName(topic.constant)}: ${places}`,
|
||||
`The ${ordinalText} appearance of ${topic.pattern} in ${constantSpokenName(topic.constant)} ` +
|
||||
`is at decimal ${places}, in …${highlightedMatch(match)}…`,
|
||||
[
|
||||
["Start", String(match.start)],
|
||||
["End", String(match.end)],
|
||||
["Context", highlightedMatch(match)],
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "digitCount": {
|
||||
const counted = countDigitsIn(topic.pattern, topic.constant);
|
||||
return {
|
||||
headline: `${counted.occurrences} time${counted.occurrences === 1 ? "" : "s"}`,
|
||||
detail:
|
||||
`${topic.pattern} appears ${counted.occurrences} time${counted.occurrences === 1 ? "" : "s"} ` +
|
||||
`in the first ${formatInt(counted.searchedDigits)} digits of ${constantSpokenName(topic.constant)}.`,
|
||||
fields: [["Searched", `${formatInt(counted.searchedDigits)} digits`]],
|
||||
unresolvedWorkload: { kind: "digitCount", constant: topic.constant, pattern: topic.pattern },
|
||||
};
|
||||
}
|
||||
|
||||
case "digitsAt": {
|
||||
if (topic.to - topic.from >= 200) {
|
||||
return answer("Too many digits", "Ask for a span under 200 digits.");
|
||||
}
|
||||
const digits = digitsAt(topic.constant, topic.from, topic.to);
|
||||
if (!digits) {
|
||||
return {
|
||||
headline: "Out of range",
|
||||
detail:
|
||||
`That is past the ${formatInt(MAXIMUM_COMPUTED_DIGITS)} ` +
|
||||
`digits computed on device.`,
|
||||
fields: [],
|
||||
unresolvedWorkload: { kind: "constantDigits", constant: topic.constant, count: topic.to },
|
||||
};
|
||||
}
|
||||
return answer(
|
||||
digits,
|
||||
`Decimal places ${topic.from}–${topic.to} of ${constantSpokenName(topic.constant)} are ${digits}.`,
|
||||
[["Places", `${topic.from}–${topic.to}`]],
|
||||
);
|
||||
}
|
||||
|
||||
case "shapeArea": {
|
||||
const parts = shapeSummary(topic.shape);
|
||||
if (parts.length === 0) {
|
||||
return answer("Not enough measurements", "That shape needs more values.");
|
||||
}
|
||||
return answer(
|
||||
parts.map(([label, value]) => `${label} ${format(value)}`).join(", "),
|
||||
`For that ${shapeName(topic.shape)}: ` +
|
||||
parts.map(([label, value]) => `${label.toLowerCase()} ${format(value)}`).join(", ") +
|
||||
".",
|
||||
parts.map(([label, value]) => [label, format(value)]),
|
||||
);
|
||||
}
|
||||
|
||||
case "triangleFromSides": {
|
||||
const result = triangleFromSides(topic.a, topic.b, topic.c);
|
||||
switch (result.kind) {
|
||||
case "impossible":
|
||||
return answer("No such triangle", result.reason);
|
||||
case "one":
|
||||
case "two": {
|
||||
const triangle = result.kind === "one" ? result.triangle : result.first;
|
||||
const [angleA, angleB, angleC] = triangleAnglesInDegrees(triangle);
|
||||
return answer(
|
||||
`Area ${format(triangleArea(triangle))}, ${triangleKind(triangle)}`,
|
||||
`Angles ${format(angleA)}°, ${format(angleB)}°, ${format(angleC)}°; ` +
|
||||
`area ${format(triangleArea(triangle))}; perimeter ${format(trianglePerimeter(triangle))}.`,
|
||||
[
|
||||
["Angle A", `${format(angleA)}°`],
|
||||
["Angle B", `${format(angleB)}°`],
|
||||
["Angle C", `${format(angleC)}°`],
|
||||
["Area", format(triangleArea(triangle))],
|
||||
["Perimeter", format(trianglePerimeter(triangle))],
|
||||
["Type", triangleKind(triangle)],
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "pythagoras": {
|
||||
const missing = pythagoras(topic.a, topic.b, topic.c);
|
||||
if (missing === null) {
|
||||
return answer(
|
||||
"Cannot solve",
|
||||
"Give two of the three sides, and a leg must be shorter than the hypotenuse.",
|
||||
);
|
||||
}
|
||||
return answer(format(missing), `The missing side is ${format(missing)}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function answer(headline: string, detail: string, fields: [string, string][] = []): AdvancedAnswer {
|
||||
return { headline, detail, fields, unresolvedWorkload: null };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extraction helpers
|
||||
|
||||
export function firstInt(text: string): number | null {
|
||||
return allInts(text)[0] ?? null;
|
||||
}
|
||||
|
||||
export function allInts(text: string): number[] {
|
||||
return (text.match(/-?\d+/g) ?? []).map(Number).filter(Number.isSafeInteger);
|
||||
}
|
||||
|
||||
export function allDoubles(text: string): number[] {
|
||||
return (text.match(/-?\d+(?:\.\d+)?/g) ?? []).map(Number).filter(Number.isFinite);
|
||||
}
|
||||
|
||||
export function lastDouble(text: string): number | null {
|
||||
const values = allDoubles(text);
|
||||
return values.length > 0 ? values[values.length - 1]! : null;
|
||||
}
|
||||
|
||||
function firstDoubleAfter(keywords: string[], text: string): number | null {
|
||||
for (const keyword of keywords) {
|
||||
const index = text.indexOf(keyword);
|
||||
if (index < 0) continue;
|
||||
const tail = text.slice(index + keyword.length);
|
||||
const value = allDoubles(tail)[0];
|
||||
if (value !== undefined) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** "from 0 to 5" */
|
||||
export function boundsIn(text: string): [number, number] | null {
|
||||
const match = text.match(/from\s+(-?[0-9.]+)\s+to\s+(-?[0-9.]+)/);
|
||||
if (!match) return null;
|
||||
const lower = Number(match[1]);
|
||||
const upper = Number(match[2]);
|
||||
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
|
||||
return [lower, upper];
|
||||
}
|
||||
|
||||
/** Take everything after the topic keyword and parse it as an expression. */
|
||||
function expressionTail(original: string, keywords: string[]): MathExpression | null {
|
||||
const lowered = original.toLowerCase();
|
||||
for (const keyword of [...keywords].sort((a, b) => b.length - a.length)) {
|
||||
const index = lowered.indexOf(keyword);
|
||||
if (index < 0) continue;
|
||||
let tail = original.slice(index + keyword.length);
|
||||
// Drop trailing clauses that belong to the command, not the maths.
|
||||
for (const terminator of [" from ", " as ", " with respect to ", " to order ", " dx"]) {
|
||||
const cut = tail.toLowerCase().indexOf(terminator);
|
||||
if (cut >= 0) tail = tail.slice(0, cut);
|
||||
}
|
||||
const trimmed = tail.replace(/^[ .,?]+|[ .,?]+$/g, "");
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
return parseSpoken(trimmed);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function format(value: number): string {
|
||||
return describeNumber(fromDecimal(value));
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
// Limits, Taylor series, and curve analysis.
|
||||
|
||||
import {
|
||||
type MathExpression,
|
||||
addExpr,
|
||||
divideExpr,
|
||||
multiplyExpr,
|
||||
num,
|
||||
numberValue,
|
||||
powerExpr,
|
||||
subtractExpr,
|
||||
variable as variableExpr,
|
||||
ONE,
|
||||
ZERO,
|
||||
} from "./expression";
|
||||
import { describeNumber, divide as divideNumber, fromDecimal, integer, isNegative, isOne, negated } from "./math-number";
|
||||
import { evaluateValue } from "./evaluator";
|
||||
import { differentiate } from "./derivative";
|
||||
import { simplify } from "./simplifier";
|
||||
import { Polynomial } from "./polynomial";
|
||||
import { numericRoots as polynomialNumericRoots } from "./solver";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Limits
|
||||
|
||||
export type LimitApproach = "both" | "fromLeft" | "fromRight";
|
||||
|
||||
export type LimitResult =
|
||||
| { kind: "value"; value: number }
|
||||
| { kind: "positiveInfinity" }
|
||||
| { kind: "negativeInfinity" }
|
||||
/** One-sided limits disagree, so the two-sided limit does not exist. */
|
||||
| { kind: "doesNotExist"; left: number | null; right: number | null }
|
||||
| { kind: "indeterminate" };
|
||||
|
||||
export function limitText(result: LimitResult): string {
|
||||
switch (result.kind) {
|
||||
case "value":
|
||||
return describeNumber(fromDecimal(result.value));
|
||||
case "positiveInfinity":
|
||||
return "+∞";
|
||||
case "negativeInfinity":
|
||||
return "−∞";
|
||||
case "doesNotExist":
|
||||
return "does not exist";
|
||||
case "indeterminate":
|
||||
return "indeterminate";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Limit of `expression` as `variable` → `point`.
|
||||
*
|
||||
* Direct substitution first, then L'Hôpital for 0/0 and ∞/∞, then a numeric
|
||||
* squeeze. The order matters: substitution is exact when it works, L'Hôpital
|
||||
* is exact when it applies, and only the leftovers get the approximate
|
||||
* treatment.
|
||||
*/
|
||||
export function limit(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
point: number,
|
||||
approach: LimitApproach = "both",
|
||||
): LimitResult {
|
||||
if (approach === "both" && Number.isFinite(point)) {
|
||||
const direct = evaluateValue(expression, { [variable]: point });
|
||||
if (direct !== null && Number.isFinite(direct)) {
|
||||
return { kind: "value", value: direct };
|
||||
}
|
||||
}
|
||||
const hospital = lHopital(expression, variable, point, 0);
|
||||
if (hospital) return hospital;
|
||||
return numericLimit(expression, variable, point, approach);
|
||||
}
|
||||
|
||||
/** Detect an indeterminate quotient and differentiate top and bottom. */
|
||||
function lHopital(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
point: number,
|
||||
depth: number,
|
||||
): LimitResult | null {
|
||||
// Bounded: a quotient that stays indeterminate forever (sin(1/x)/x) must
|
||||
// give up rather than recurse without end.
|
||||
if (depth >= 8 || !Number.isFinite(point)) return null;
|
||||
const quotient = asQuotient(expression);
|
||||
if (!quotient) return null;
|
||||
const [numerator, denominator] = quotient;
|
||||
|
||||
const top = evaluateValue(numerator, { [variable]: point });
|
||||
const bottom = evaluateValue(denominator, { [variable]: point });
|
||||
if (top === null || bottom === null) return null;
|
||||
|
||||
const isZeroOverZero = Math.abs(top) < 1e-12 && Math.abs(bottom) < 1e-12;
|
||||
const isInfinityOverInfinity = !Number.isFinite(top) && !Number.isFinite(bottom);
|
||||
if (!isZeroOverZero && !isInfinityOverInfinity) return null;
|
||||
|
||||
const dTop = differentiate(numerator, variable);
|
||||
const dBottom = differentiate(denominator, variable);
|
||||
// A vanishing derivative of the denominator means the rule does not apply
|
||||
// again; fall through to the numeric path.
|
||||
const bottomValue = evaluateValue(dBottom, { [variable]: point });
|
||||
if (bottomValue === null || !Number.isFinite(bottomValue)) return null;
|
||||
|
||||
if (Math.abs(bottomValue) > 1e-12) {
|
||||
const topValue = evaluateValue(dTop, { [variable]: point });
|
||||
if (topValue !== null && Number.isFinite(topValue)) {
|
||||
return { kind: "value", value: topValue / bottomValue };
|
||||
}
|
||||
}
|
||||
return lHopital(divideExpr(dTop, dBottom), variable, point, depth + 1);
|
||||
}
|
||||
|
||||
/** Recognise `a / b`, which the tree stores as `a · b^-1`. */
|
||||
export function asQuotient(expression: MathExpression): [MathExpression, MathExpression] | null {
|
||||
if (expression.type !== "multiply") return null;
|
||||
const numerator: MathExpression[] = [];
|
||||
const denominator: MathExpression[] = [];
|
||||
for (const factor of expression.factors) {
|
||||
if (factor.type === "power") {
|
||||
const power = numberValue(factor.exponent);
|
||||
if (power && isNegative(power)) {
|
||||
const positive = negated(power);
|
||||
denominator.push(isOne(positive) ? factor.base : powerExpr(factor.base, num(positive)));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
numerator.push(factor);
|
||||
}
|
||||
if (denominator.length === 0) return null;
|
||||
return [
|
||||
numerator.length === 0 ? ONE : numerator.length === 1 ? numerator[0]! : multiplyExpr(numerator),
|
||||
denominator.length === 1 ? denominator[0]! : multiplyExpr(denominator),
|
||||
];
|
||||
}
|
||||
|
||||
function numericLimit(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
point: number,
|
||||
approach: LimitApproach,
|
||||
): LimitResult {
|
||||
const sampleSide = (side: LimitApproach): number | null => {
|
||||
if (!Number.isFinite(point)) {
|
||||
// At infinity, march outward instead of shrinking a step.
|
||||
const sign = point > 0 ? 1 : -1;
|
||||
let last: number | null = null;
|
||||
for (const magnitude of [1e3, 1e4, 1e5, 1e6, 1e7]) {
|
||||
const value = evaluateValue(expression, { [variable]: sign * magnitude });
|
||||
if (value === null || !Number.isFinite(value)) return last;
|
||||
last = value;
|
||||
}
|
||||
return last;
|
||||
}
|
||||
const samples: number[] = [];
|
||||
for (const step of [1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7]) {
|
||||
const offset = side === "fromLeft" ? -step : step;
|
||||
const value = evaluateValue(expression, { [variable]: point + offset });
|
||||
if (value === null || !Number.isFinite(value)) continue;
|
||||
samples.push(value);
|
||||
}
|
||||
const last = samples[samples.length - 1];
|
||||
if (last === undefined) return null;
|
||||
// Divergence is a TREND, not a threshold. 1/x reaches only 1e7 at the
|
||||
// smallest step sampled here, so a fixed "> 1e12" cutoff calls it finite;
|
||||
// magnitudes growing by an order per step do not.
|
||||
if (samples.length >= 3) {
|
||||
const tail = samples.slice(-3);
|
||||
const magnitudes = tail.map((value) => Math.abs(value));
|
||||
const growing = magnitudes.every(
|
||||
(value, index) => index === 0 || value > magnitudes[index - 1]! * 5,
|
||||
);
|
||||
if (growing && (magnitudes[magnitudes.length - 1] ?? 0) > 1e4) {
|
||||
return last > 0 ? Infinity : -Infinity;
|
||||
}
|
||||
}
|
||||
return last;
|
||||
};
|
||||
|
||||
switch (approach) {
|
||||
case "fromLeft": {
|
||||
const value = sampleSide("fromLeft");
|
||||
return value === null ? { kind: "indeterminate" } : classify(value);
|
||||
}
|
||||
case "fromRight": {
|
||||
const value = sampleSide("fromRight");
|
||||
return value === null ? { kind: "indeterminate" } : classify(value);
|
||||
}
|
||||
case "both": {
|
||||
const left = sampleSide("fromLeft");
|
||||
const right = sampleSide("fromRight");
|
||||
if (left === null || right === null) return { kind: "indeterminate" };
|
||||
// Scale-relative comparison: two values near 1e6 differing by 1 are the
|
||||
// same limit; two near 0 differing by 1 are not.
|
||||
const scale = Math.max(Math.abs(left), Math.abs(right), 1);
|
||||
if (Math.abs(left - right) >= 1e-4 * scale) {
|
||||
return { kind: "doesNotExist", left, right };
|
||||
}
|
||||
return classify((left + right) / 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function classify(value: number): LimitResult {
|
||||
if (!Number.isFinite(value)) {
|
||||
return value > 0 ? { kind: "positiveInfinity" } : { kind: "negativeInfinity" };
|
||||
}
|
||||
if (value > 1e12) return { kind: "positiveInfinity" };
|
||||
if (value < -1e12) return { kind: "negativeInfinity" };
|
||||
return { kind: "value", value };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Series
|
||||
|
||||
/** Taylor expansion about `centre`, to `order` terms. */
|
||||
export function taylor(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
order = 5,
|
||||
centre = 0,
|
||||
): MathExpression | null {
|
||||
if (order < 0 || order > 12) return null;
|
||||
const terms: MathExpression[] = [];
|
||||
let derivative = expression;
|
||||
let factorial = 1;
|
||||
|
||||
for (let degree = 0; degree <= order; degree++) {
|
||||
if (degree > 0) {
|
||||
derivative = differentiate(derivative, variable);
|
||||
factorial *= degree;
|
||||
}
|
||||
const coefficient = evaluateValue(derivative, { [variable]: centre });
|
||||
if (coefficient === null || !Number.isFinite(coefficient)) return null;
|
||||
if (Math.abs(coefficient) <= 1e-12) continue;
|
||||
|
||||
const scaled = divideNumber(fromDecimal(coefficient), integer(factorial));
|
||||
const shifted: MathExpression =
|
||||
centre === 0
|
||||
? variableExpr(variable)
|
||||
: subtractExpr(variableExpr(variable), num(fromDecimal(centre)));
|
||||
const power: MathExpression =
|
||||
degree === 0 ? ONE : degree === 1 ? shifted : powerExpr(shifted, num(integer(degree)));
|
||||
terms.push(degree === 0 ? num(scaled) : multiplyExpr([num(scaled), power]));
|
||||
}
|
||||
if (terms.length === 0) return ZERO;
|
||||
return simplify(addExpr(terms));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Curve analysis
|
||||
|
||||
export type CriticalPointKind = "minimum" | "maximum" | "inflection" | "undetermined";
|
||||
|
||||
export type CriticalPoint = {
|
||||
x: number;
|
||||
y: number;
|
||||
kind: CriticalPointKind;
|
||||
};
|
||||
|
||||
export function criticalPointText(point: CriticalPoint): string {
|
||||
const xText = describeNumber(fromDecimal(point.x));
|
||||
const yText = describeNumber(fromDecimal(point.y));
|
||||
return `${point.kind} at (${xText}, ${yText})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Critical points classified by the second derivative, falling back to a
|
||||
* sign test where the second derivative vanishes.
|
||||
*/
|
||||
export function criticalPoints(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
range: [number, number] = [-50, 50],
|
||||
): CriticalPoint[] {
|
||||
const first = differentiate(expression, variable);
|
||||
const second = differentiate(first, variable);
|
||||
|
||||
let roots: number[];
|
||||
const polynomial = Polynomial.extract(first, variable);
|
||||
if (polynomial) {
|
||||
roots = polynomialNumericRoots(polynomial, range, 20_000);
|
||||
} else {
|
||||
roots = numericRootsOfExpression(first, variable, range);
|
||||
}
|
||||
|
||||
const points: CriticalPoint[] = [];
|
||||
for (const x of roots) {
|
||||
const y = evaluateValue(expression, { [variable]: x });
|
||||
if (y === null || !Number.isFinite(y)) continue;
|
||||
const curvature = evaluateValue(second, { [variable]: x }) ?? 0;
|
||||
let kind: CriticalPointKind;
|
||||
if (curvature > 1e-9) {
|
||||
kind = "minimum";
|
||||
} else if (curvature < -1e-9) {
|
||||
kind = "maximum";
|
||||
} else {
|
||||
// Second-derivative test is inconclusive; look at how the first
|
||||
// derivative behaves either side.
|
||||
const left = evaluateValue(first, { [variable]: x - 1e-4 }) ?? 0;
|
||||
const right = evaluateValue(first, { [variable]: x + 1e-4 }) ?? 0;
|
||||
if (left < 0 && right > 0) kind = "minimum";
|
||||
else if (left > 0 && right < 0) kind = "maximum";
|
||||
else kind = "inflection";
|
||||
}
|
||||
points.push({ x, y, kind });
|
||||
}
|
||||
return points;
|
||||
}
|
||||
|
||||
export function numericRootsOfExpression(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
range: [number, number],
|
||||
samples = 20_000,
|
||||
): number[] {
|
||||
const [lower, upper] = range;
|
||||
const roots: number[] = [];
|
||||
const step = (upper - lower) / samples;
|
||||
let previousX = lower;
|
||||
let previousY = evaluateValue(expression, { [variable]: previousX });
|
||||
|
||||
for (let index = 1; index <= samples; index++) {
|
||||
const x = lower + index * step;
|
||||
const y = evaluateValue(expression, { [variable]: x });
|
||||
const lastX = previousX;
|
||||
const lastY = previousY;
|
||||
previousX = x;
|
||||
previousY = y;
|
||||
if (y === null || lastY === null) continue;
|
||||
if (Math.sign(lastY) === Math.sign(y)) continue;
|
||||
if (Math.abs(y) >= 1e6 || Math.abs(lastY) >= 1e6) continue;
|
||||
|
||||
let low = lastX;
|
||||
let high = x;
|
||||
for (let iteration = 0; iteration < 60; iteration++) {
|
||||
const mid = (low + high) / 2;
|
||||
const value = evaluateValue(expression, { [variable]: mid });
|
||||
const lowValue = evaluateValue(expression, { [variable]: low });
|
||||
if (value === null || lowValue === null) break;
|
||||
if (value === 0) {
|
||||
low = mid;
|
||||
high = mid;
|
||||
break;
|
||||
}
|
||||
if (Math.sign(value) === Math.sign(lowValue)) {
|
||||
low = mid;
|
||||
} else {
|
||||
high = mid;
|
||||
}
|
||||
}
|
||||
const root = (low + high) / 2;
|
||||
// Same pole guard as the main solver: verify the residual.
|
||||
const residual = evaluateValue(expression, { [variable]: root });
|
||||
if (residual === null || Math.abs(residual) >= 1e-6) continue;
|
||||
if (!roots.some((existing) => Math.abs(existing - root) < 1e-6)) roots.push(root);
|
||||
}
|
||||
return roots;
|
||||
}
|
||||
@@ -0,0 +1,591 @@
|
||||
// Digit expansions of the well-known constants, computed exactly. Computed
|
||||
// rather than tabulated: a hardcoded string of π is a thing nobody can check,
|
||||
// and a single transposed digit would silently shift every position this
|
||||
// feature reports. Everything here comes out of integer arithmetic that can
|
||||
// be re-derived and tested.
|
||||
//
|
||||
// JavaScript's native BigInt supplies the operation set the generators need
|
||||
// (add, subtract, compare, multiply/divide by a small scalar).
|
||||
// The deep paths are async and yield to the event loop periodically — a
|
||||
// multi-minute synchronous grind would freeze the app's whole UI thread.
|
||||
|
||||
export type MathConstantName = "pi" | "e" | "phi" | "sqrt2" | "sqrt3" | "sqrt5";
|
||||
|
||||
export const ALL_CONSTANT_NAMES: readonly MathConstantName[] = ["pi", "e", "phi", "sqrt2", "sqrt3", "sqrt5"];
|
||||
|
||||
export function constantDisplayName(constant: MathConstantName): string {
|
||||
switch (constant) {
|
||||
case "pi":
|
||||
return "π";
|
||||
case "e":
|
||||
return "e";
|
||||
case "phi":
|
||||
return "φ";
|
||||
case "sqrt2":
|
||||
return "√2";
|
||||
case "sqrt3":
|
||||
return "√3";
|
||||
case "sqrt5":
|
||||
return "√5";
|
||||
}
|
||||
}
|
||||
|
||||
export function constantSpokenName(constant: MathConstantName): string {
|
||||
switch (constant) {
|
||||
case "pi":
|
||||
return "pi";
|
||||
case "e":
|
||||
return "e";
|
||||
case "phi":
|
||||
return "the golden ratio";
|
||||
case "sqrt2":
|
||||
return "the square root of 2";
|
||||
case "sqrt3":
|
||||
return "the square root of 3";
|
||||
case "sqrt5":
|
||||
return "the square root of 5";
|
||||
}
|
||||
}
|
||||
|
||||
/** Digits before the decimal point — 1 for all of these. */
|
||||
export const CONSTANT_INTEGER_PART_DIGITS = 1;
|
||||
|
||||
export function constantNamed(text: string): MathConstantName | null {
|
||||
const lowered = text.toLowerCase();
|
||||
if (lowered.includes("golden") || lowered.includes("phi")) return "phi";
|
||||
if (lowered.includes("root of 2") || lowered.includes("root two") || lowered.includes("sqrt2")) {
|
||||
return "sqrt2";
|
||||
}
|
||||
if (lowered.includes("root of 3") || lowered.includes("root three")) return "sqrt3";
|
||||
if (lowered.includes("root of 5") || lowered.includes("root five")) return "sqrt5";
|
||||
// Word-boundary matched, not substring: `includes("pi")` also fires on
|
||||
// "pie", "spin", and "copying", and `=== "e"` only ever matched a
|
||||
// single-character utterance, so "the digits of e" never resolved.
|
||||
if (lowered.includes("π") || hasWord("pi", lowered)) return "pi";
|
||||
if (hasWord("e", lowered) || lowered.includes("euler")) return "e";
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasWord(word: string, text: string): boolean {
|
||||
return new RegExp(`\\b${word}\\b`).test(text);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cancellation and progress
|
||||
|
||||
/** Cooperative cancellation for the long paths. */
|
||||
export type ComputeCancellation = () => boolean;
|
||||
|
||||
export const NEVER_CANCELLED: ComputeCancellation = () => false;
|
||||
|
||||
/** Raised when a long run is cancelled partway. */
|
||||
export class ComputeCancelled extends Error {
|
||||
constructor() {
|
||||
super("Cancelled");
|
||||
this.name = "ComputeCancelled";
|
||||
}
|
||||
}
|
||||
|
||||
export type ComputeProgress = (fraction: number, stage: string) => void;
|
||||
|
||||
// setTimeout via globalThis so this module needs no platform lib: it runs
|
||||
// under NativeScript's globals in the app and under Node in the tests.
|
||||
const scheduleMacrotask: (callback: () => void) => void = (callback) =>
|
||||
(globalThis as unknown as { setTimeout: (fn: () => void, ms: number) => unknown }).setTimeout(callback, 0);
|
||||
|
||||
/**
|
||||
* Yield to the event loop when the current slice has run long enough. The
|
||||
* generators call this between series terms so a deep run stays cancellable
|
||||
* and the rest of the app keeps painting.
|
||||
*/
|
||||
class Yielder {
|
||||
private lastYieldAt = Date.now();
|
||||
|
||||
async maybeYield(): Promise<void> {
|
||||
if (Date.now() - this.lastYieldAt < 50) return;
|
||||
await new Promise<void>((resolve) => scheduleMacrotask(resolve));
|
||||
this.lastYieldAt = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Limits
|
||||
|
||||
/**
|
||||
* How far the on-device generator will go for an ordinary question.
|
||||
*
|
||||
* Machin's formula is O(n²) in the digit count with scalar-only big
|
||||
* arithmetic. 10,000 digits is well under a second and finds any two- or
|
||||
* three-digit run plus most four-digit ones; past this, the wearer opts into
|
||||
* a long run explicitly.
|
||||
*/
|
||||
export const MAXIMUM_COMPUTED_DIGITS = 10_000;
|
||||
|
||||
/**
|
||||
* The furthest an opted-in long run will go, however much time the wearer
|
||||
* offers. Not a time limit — the caller has already accepted the wait — but
|
||||
* a memory-and-honesty one: the O(n²) cost puts this at hours on a phone,
|
||||
* past which nobody is still waiting.
|
||||
*/
|
||||
export const MAXIMUM_FEASIBLE_DIGITS = 2_000_000;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache
|
||||
|
||||
/**
|
||||
* The same constant is asked for repeatedly within one session (search, then
|
||||
* context, then a second search), and recomputing 10,000 digits each time
|
||||
* would be pointlessly slow. Grow-only.
|
||||
*/
|
||||
const cache = new Map<MathConstantName, string>();
|
||||
|
||||
function cachedDigits(constant: MathConstantName, atLeast: number): string | null {
|
||||
const cached = cache.get(constant);
|
||||
if (!cached || cached.length < atLeast) return null;
|
||||
return cached;
|
||||
}
|
||||
|
||||
function storeDigits(constant: MathConstantName, digits: string): void {
|
||||
const existing = cache.get(constant);
|
||||
if (existing && existing.length >= digits.length) return;
|
||||
cache.set(constant, digits);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public entry points
|
||||
|
||||
/**
|
||||
* Fractional digits only — the part after the decimal point, which is what
|
||||
* "position N of π" universally means. Clamped to MAXIMUM_COMPUTED_DIGITS;
|
||||
* the deeper run is a separate call the wearer has to opt into, so no
|
||||
* ordinary caller can accidentally block the app for minutes.
|
||||
*/
|
||||
export function fractionalDigits(constant: MathConstantName, count: number): string | null {
|
||||
const requested = Math.min(Math.max(count, 1), MAXIMUM_COMPUTED_DIGITS);
|
||||
const cached = cachedDigits(constant, requested);
|
||||
if (cached) return cached.slice(0, requested);
|
||||
// The shallow path runs synchronously; at 10k digits it is fast enough that
|
||||
// an async hop would only complicate every caller.
|
||||
const digits = computeSync(constant, requested);
|
||||
if (digits === null) return null;
|
||||
storeDigits(constant, digits);
|
||||
return digits.slice(0, requested);
|
||||
}
|
||||
|
||||
/**
|
||||
* The opted-in deep run. Cancellable, and reports progress, because a
|
||||
* multi-minute grind with no way out and nothing on screen is
|
||||
* indistinguishable from a hang.
|
||||
*/
|
||||
export async function deepFractionalDigits(
|
||||
constant: MathConstantName,
|
||||
count: number,
|
||||
cancellation: ComputeCancellation = NEVER_CANCELLED,
|
||||
progress: ComputeProgress | null = null,
|
||||
): Promise<string | null> {
|
||||
const requested = Math.min(Math.max(count, 1), MAXIMUM_FEASIBLE_DIGITS);
|
||||
const cached = cachedDigits(constant, requested);
|
||||
if (cached) return cached.slice(0, requested);
|
||||
const digits = await compute(constant, requested, cancellation, progress, new Yielder());
|
||||
if (digits === null) return null;
|
||||
storeDigits(constant, digits);
|
||||
return digits.slice(0, requested);
|
||||
}
|
||||
|
||||
function computeSync(constant: MathConstantName, requested: number): string | null {
|
||||
// A resolved-synchronously wrapper would still be async; instead run the
|
||||
// shared generator with a yielder that never fires (50ms slices cannot
|
||||
// elapse mid-loop when the whole computation is sub-second) by executing
|
||||
// the promise eagerly. The generators only await through Yielder.maybeYield,
|
||||
// which resolves immediately below the time threshold, so the promise
|
||||
// settles synchronously enough — but rather than rely on that, the shallow
|
||||
// path uses the dedicated synchronous variants.
|
||||
return computeCore(constant, requested, NEVER_CANCELLED, null, null);
|
||||
}
|
||||
|
||||
async function compute(
|
||||
constant: MathConstantName,
|
||||
requested: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
yielder: Yielder,
|
||||
): Promise<string | null> {
|
||||
return computeCoreAsync(constant, requested, cancellation, progress, yielder);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core generators
|
||||
//
|
||||
// Each exists in a synchronous and an asynchronous variant sharing the same
|
||||
// arithmetic. The async ones interleave event-loop yields; the sync ones are
|
||||
// used only under MAXIMUM_COMPUTED_DIGITS where the total cost is small.
|
||||
|
||||
/**
|
||||
* Guard digits absorb the truncation error the series accumulates in its
|
||||
* least significant places; without them the last handful of reported digits
|
||||
* are wrong, which for a position-reporting feature is the whole ballgame.
|
||||
*/
|
||||
const GUARD_DIGITS = 20;
|
||||
|
||||
function computeCore(
|
||||
constant: MathConstantName,
|
||||
requested: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
_yielder: null,
|
||||
): string | null {
|
||||
const scale = requested + GUARD_DIGITS;
|
||||
let scaled: bigint;
|
||||
switch (constant) {
|
||||
case "pi":
|
||||
scaled = machinPiSync(scale, cancellation, progress);
|
||||
break;
|
||||
case "e":
|
||||
scaled = eulerESync(scale, cancellation, progress);
|
||||
break;
|
||||
case "phi": {
|
||||
// φ = (1 + √5) / 2
|
||||
const root = integerSquareRootSync(5, scale, cancellation, progress);
|
||||
scaled = (root + powerOfTen(scale)) / 2n;
|
||||
break;
|
||||
}
|
||||
case "sqrt2":
|
||||
scaled = integerSquareRootSync(2, scale, cancellation, progress);
|
||||
break;
|
||||
case "sqrt3":
|
||||
scaled = integerSquareRootSync(3, scale, cancellation, progress);
|
||||
break;
|
||||
case "sqrt5":
|
||||
scaled = integerSquareRootSync(5, scale, cancellation, progress);
|
||||
break;
|
||||
}
|
||||
return finishDigits(scaled, requested);
|
||||
}
|
||||
|
||||
async function computeCoreAsync(
|
||||
constant: MathConstantName,
|
||||
requested: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
yielder: Yielder,
|
||||
): Promise<string | null> {
|
||||
const scale = requested + GUARD_DIGITS;
|
||||
let scaled: bigint;
|
||||
switch (constant) {
|
||||
case "pi":
|
||||
scaled = await machinPi(scale, cancellation, progress, yielder);
|
||||
break;
|
||||
case "e":
|
||||
scaled = await eulerE(scale, cancellation, progress, yielder);
|
||||
break;
|
||||
case "phi": {
|
||||
const root = await integerSquareRoot(5, scale, cancellation, progress, yielder);
|
||||
scaled = (root + powerOfTen(scale)) / 2n;
|
||||
break;
|
||||
}
|
||||
case "sqrt2":
|
||||
scaled = await integerSquareRoot(2, scale, cancellation, progress, yielder);
|
||||
break;
|
||||
case "sqrt3":
|
||||
scaled = await integerSquareRoot(3, scale, cancellation, progress, yielder);
|
||||
break;
|
||||
case "sqrt5":
|
||||
scaled = await integerSquareRoot(5, scale, cancellation, progress, yielder);
|
||||
break;
|
||||
}
|
||||
return finishDigits(scaled, requested);
|
||||
}
|
||||
|
||||
/**
|
||||
* The value is the constant × 10^scale. Strip the integer part, then the
|
||||
* guard digits.
|
||||
*/
|
||||
function finishDigits(scaled: bigint, requested: number): string | null {
|
||||
const text = scaled.toString();
|
||||
if (text.length <= CONSTANT_INTEGER_PART_DIGITS) return null;
|
||||
const fractional = text.slice(CONSTANT_INTEGER_PART_DIGITS);
|
||||
if (fractional.length < requested) return null;
|
||||
const trimmed = fractional.slice(0, fractional.length - GUARD_DIGITS);
|
||||
return trimmed.slice(0, requested);
|
||||
}
|
||||
|
||||
export function powerOfTen(exponent: number): bigint {
|
||||
return 10n ** BigInt(Math.max(0, exponent));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// π — Machin's formula: π = 16·arctan(1/5) − 4·arctan(1/239).
|
||||
//
|
||||
// Chosen over Chudnovsky because it needs only scalar big-integer operations
|
||||
// — no big×big multiply, no big÷big division, no square roots of big values.
|
||||
|
||||
// arctan(1/5) needs log10(239)/log10(5) ≈ 3.4× as many terms as arctan(1/239)
|
||||
// at the same scale, and each term costs the same, so the first phase is
|
||||
// ~77% of the work. Splitting the reported fraction that way keeps the
|
||||
// progress bar honest instead of stalling at half.
|
||||
const MACHIN_FIRST_SHARE = 0.77;
|
||||
|
||||
function machinPiSync(
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
): bigint {
|
||||
const first =
|
||||
arctanReciprocalSync(5, scale, cancellation, (fraction) =>
|
||||
progress?.(fraction * MACHIN_FIRST_SHARE, "π series 1 of 2"),
|
||||
) * 16n;
|
||||
const second =
|
||||
arctanReciprocalSync(239, scale, cancellation, (fraction) =>
|
||||
progress?.(MACHIN_FIRST_SHARE + fraction * (1 - MACHIN_FIRST_SHARE), "π series 2 of 2"),
|
||||
) * 4n;
|
||||
return first - second;
|
||||
}
|
||||
|
||||
async function machinPi(
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
yielder: Yielder,
|
||||
): Promise<bigint> {
|
||||
const first =
|
||||
(await arctanReciprocal(
|
||||
5,
|
||||
scale,
|
||||
cancellation,
|
||||
(fraction) => progress?.(fraction * MACHIN_FIRST_SHARE, "π series 1 of 2"),
|
||||
yielder,
|
||||
)) * 16n;
|
||||
const second =
|
||||
(await arctanReciprocal(
|
||||
239,
|
||||
scale,
|
||||
cancellation,
|
||||
(fraction) => progress?.(MACHIN_FIRST_SHARE + fraction * (1 - MACHIN_FIRST_SHARE), "π series 2 of 2"),
|
||||
yielder,
|
||||
)) * 4n;
|
||||
return first - second;
|
||||
}
|
||||
|
||||
/** arctan(1/x), scaled by 10^scale, by its alternating Taylor series. */
|
||||
function arctanReciprocalSync(
|
||||
x: number,
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ((fraction: number) => void) | null,
|
||||
): bigint {
|
||||
const bigX = BigInt(x);
|
||||
const xSquared = bigX * bigX;
|
||||
let power = powerOfTen(scale) / bigX;
|
||||
let sum = power;
|
||||
let divisor = 3n;
|
||||
let subtractNext = true;
|
||||
|
||||
// Terms shrink by a factor of x² each step, so the series dies after about
|
||||
// scale / (2·log10 x) of them. Known up front, which is what makes a real
|
||||
// fraction possible rather than a spinner.
|
||||
const expectedTerms = Math.max(1, scale / (2 * Math.log10(x)));
|
||||
let iterations = 0;
|
||||
|
||||
while (power !== 0n) {
|
||||
// Checked every term: one closure call is nothing beside a big-integer
|
||||
// division, and the alternative is an uncancellable multi-minute loop.
|
||||
if (cancellation()) throw new ComputeCancelled();
|
||||
|
||||
power /= xSquared;
|
||||
const term = power / divisor;
|
||||
if (term === 0n) break;
|
||||
sum = subtractNext ? sum - term : sum + term;
|
||||
subtractNext = !subtractNext;
|
||||
divisor += 2n;
|
||||
|
||||
iterations += 1;
|
||||
// Throttled: a million progress callbacks would cost more than the
|
||||
// arithmetic they are reporting on.
|
||||
if (progress && iterations % 256 === 0) {
|
||||
progress(Math.min(1, iterations / expectedTerms));
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
async function arctanReciprocal(
|
||||
x: number,
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ((fraction: number) => void) | null,
|
||||
yielder: Yielder,
|
||||
): Promise<bigint> {
|
||||
const bigX = BigInt(x);
|
||||
const xSquared = bigX * bigX;
|
||||
let power = powerOfTen(scale) / bigX;
|
||||
let sum = power;
|
||||
let divisor = 3n;
|
||||
let subtractNext = true;
|
||||
|
||||
const expectedTerms = Math.max(1, scale / (2 * Math.log10(x)));
|
||||
let iterations = 0;
|
||||
|
||||
while (power !== 0n) {
|
||||
if (cancellation()) throw new ComputeCancelled();
|
||||
|
||||
power /= xSquared;
|
||||
const term = power / divisor;
|
||||
if (term === 0n) break;
|
||||
sum = subtractNext ? sum - term : sum + term;
|
||||
subtractNext = !subtractNext;
|
||||
divisor += 2n;
|
||||
|
||||
iterations += 1;
|
||||
if (progress && iterations % 256 === 0) {
|
||||
progress(Math.min(1, iterations / expectedTerms));
|
||||
}
|
||||
if (iterations % 64 === 0) await yielder.maybeYield();
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// e = Σ 1/k!, scaled. Each term is the previous divided by k, so the whole
|
||||
// series needs only scalar division.
|
||||
|
||||
function eulerESync(
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
): bigint {
|
||||
let term = powerOfTen(scale);
|
||||
let sum = term;
|
||||
let k = 1n;
|
||||
// The term loses log10(k) digits each step; the series ends when the
|
||||
// accumulated loss passes `scale`. Tracking that sum gives a fraction
|
||||
// without needing Stirling's approximation.
|
||||
let digitsShed = 0;
|
||||
let iterations = 0;
|
||||
|
||||
while (term !== 0n) {
|
||||
if (cancellation()) throw new ComputeCancelled();
|
||||
term /= k;
|
||||
sum += term;
|
||||
digitsShed += Math.log10(Number(k));
|
||||
k += 1n;
|
||||
iterations += 1;
|
||||
if (progress && iterations % 256 === 0) {
|
||||
progress(Math.min(1, digitsShed / Math.max(scale, 1)), "e series");
|
||||
}
|
||||
// 10^scale / k! underflows long before this, but a runaway loop is not an
|
||||
// acceptable failure mode.
|
||||
if (k > 1_000_000n) break;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
async function eulerE(
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
yielder: Yielder,
|
||||
): Promise<bigint> {
|
||||
let term = powerOfTen(scale);
|
||||
let sum = term;
|
||||
let k = 1n;
|
||||
let digitsShed = 0;
|
||||
let iterations = 0;
|
||||
|
||||
while (term !== 0n) {
|
||||
if (cancellation()) throw new ComputeCancelled();
|
||||
term /= k;
|
||||
sum += term;
|
||||
digitsShed += Math.log10(Number(k));
|
||||
k += 1n;
|
||||
iterations += 1;
|
||||
if (progress && iterations % 256 === 0) {
|
||||
progress(Math.min(1, digitsShed / Math.max(scale, 1)), "e series");
|
||||
}
|
||||
if (iterations % 64 === 0) await yielder.maybeYield();
|
||||
if (k > 1_000_000n) break;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Square roots — ⌊√(value · 10^(2·scale))⌋, i.e. √value scaled by 10^scale.
|
||||
//
|
||||
// Digit-by-digit ("long division") extraction, which needs only add,
|
||||
// subtract, compare, and multiply-by-scalar.
|
||||
|
||||
function integerSquareRootSync(
|
||||
value: number,
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
): bigint {
|
||||
// Work in pairs of decimal digits, which is what the algorithm consumes.
|
||||
const totalPairs = scale + 1;
|
||||
let remainder = 0n;
|
||||
let root = 0n;
|
||||
|
||||
for (let pairIndex = 0; pairIndex < totalPairs; pairIndex++) {
|
||||
if (cancellation()) throw new ComputeCancelled();
|
||||
if (progress && pairIndex % 256 === 0) {
|
||||
progress(pairIndex / totalPairs, "root extraction");
|
||||
}
|
||||
// Bring down the next two digits: the first pair is the integer part,
|
||||
// every later pair is zero because the radicand's fractional expansion is
|
||||
// all zeros.
|
||||
const bringDown = pairIndex === 0 ? BigInt(value) : 0n;
|
||||
remainder = remainder * 100n + bringDown;
|
||||
|
||||
// Find the largest d in 0...9 with (20·root + d)·d ≤ remainder.
|
||||
const twentyRoot = root * 20n;
|
||||
let chosen = 0n;
|
||||
let chosenProduct = 0n;
|
||||
for (let digit = 9n; digit >= 1n; digit--) {
|
||||
const candidate = (twentyRoot + digit) * digit;
|
||||
if (candidate <= remainder) {
|
||||
chosen = digit;
|
||||
chosenProduct = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
remainder -= chosenProduct;
|
||||
root = root * 10n + chosen;
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
async function integerSquareRoot(
|
||||
value: number,
|
||||
scale: number,
|
||||
cancellation: ComputeCancellation,
|
||||
progress: ComputeProgress | null,
|
||||
yielder: Yielder,
|
||||
): Promise<bigint> {
|
||||
const totalPairs = scale + 1;
|
||||
let remainder = 0n;
|
||||
let root = 0n;
|
||||
|
||||
for (let pairIndex = 0; pairIndex < totalPairs; pairIndex++) {
|
||||
if (cancellation()) throw new ComputeCancelled();
|
||||
if (progress && pairIndex % 256 === 0) {
|
||||
progress(pairIndex / totalPairs, "root extraction");
|
||||
}
|
||||
const bringDown = pairIndex === 0 ? BigInt(value) : 0n;
|
||||
remainder = remainder * 100n + bringDown;
|
||||
|
||||
const twentyRoot = root * 20n;
|
||||
let chosen = 0n;
|
||||
let chosenProduct = 0n;
|
||||
for (let digit = 9n; digit >= 1n; digit--) {
|
||||
const candidate = (twentyRoot + digit) * digit;
|
||||
if (candidate <= remainder) {
|
||||
chosen = digit;
|
||||
chosenProduct = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
remainder -= chosenProduct;
|
||||
root = root * 10n + chosen;
|
||||
if (pairIndex % 64 === 0) await yielder.maybeYield();
|
||||
}
|
||||
return root;
|
||||
}
|
||||
@@ -0,0 +1,602 @@
|
||||
// Owns the one live maths problem and everything derived from it. Shared
|
||||
// state rather than per-request because the whole point of "graph it" and
|
||||
// "explain it" is that they refer to the problem already on the lens; a
|
||||
// caller that re-parsed its own capture each time would make those
|
||||
// follow-ups impossible.
|
||||
//
|
||||
// Deep digit work runs on the phone as an explicit, cancellable long run
|
||||
// the wearer opts into. The
|
||||
// optional AI pass is injected as a hook so this module stays free of any
|
||||
// networking and testable in plain Node.
|
||||
|
||||
import { type MathConstantName, ComputeCancelled, constantDisplayName, constantNamed } from "./constant-digits";
|
||||
import { highlightedMatch } from "./digit-search";
|
||||
import { formatInt } from "./math-number";
|
||||
import {
|
||||
type LongRunOffer,
|
||||
type LongRunProgress,
|
||||
type MathWorkload,
|
||||
compactDuration,
|
||||
longRunOffer as makeLongRunOffer,
|
||||
runAcceptedCount,
|
||||
runAcceptedDigits,
|
||||
runAcceptedSearch,
|
||||
} from "./workload";
|
||||
import { type MathSessionResult, MathSession, parseCommand } from "./session";
|
||||
import { type Graph } from "./plotter";
|
||||
import { type SolutionStep } from "./solver";
|
||||
import { plain } from "./text-renderer";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Branch outcome
|
||||
|
||||
/**
|
||||
* A branch result. The value/failure split matters: a parse error must never
|
||||
* be posted onward as if it were the answer.
|
||||
*/
|
||||
export type MathBranchOutcome =
|
||||
| { kind: "value"; text: string }
|
||||
| { kind: "failure"; reason: string };
|
||||
|
||||
export type MathBranchMode = "solve" | "explain" | "graph" | "derivative" | "digitSearch" | "digitCount";
|
||||
|
||||
/**
|
||||
* The app's uniform gesture contract: forward/back step through the
|
||||
* working, commit
|
||||
* graphs, alternate explains, exit clears.
|
||||
*/
|
||||
export type RingAction = "forward" | "back" | "commit" | "alternate" | "exit";
|
||||
|
||||
/**
|
||||
* Ask the configured assistant to render a spoken problem as an expression
|
||||
* (the prompt text lives in app/prompts.ts with every other prompt).
|
||||
* Injected by the app; null when no provider is configured. Resolves to the
|
||||
* model's raw reply, or null on failure — the coordinator sanitises it.
|
||||
*/
|
||||
export type AICompletion = (spokenText: string) => Promise<string | null>;
|
||||
|
||||
export class MathCoordinator {
|
||||
/** The last thing produced, for the answer view. */
|
||||
lastResult: MathSessionResult | null = null;
|
||||
lastGraph: Graph | null = null;
|
||||
/** Position in the working, for the step indicator. */
|
||||
stepPosition: { index: number; total: number } | null = null;
|
||||
|
||||
/**
|
||||
* A standing offer to grind the full-depth answer out on the phone.
|
||||
* Non-null means every surface should be offering it.
|
||||
*/
|
||||
pendingLongRun: LongRunOffer | null = null;
|
||||
/** Live progress while an accepted run is going. Null when nothing is running. */
|
||||
longRunProgress: LongRunProgress | null = null;
|
||||
/**
|
||||
* The answer a deeper pass produced. One channel, because from the
|
||||
* wearer's side it is one event: the better answer arrived.
|
||||
*/
|
||||
deepAnswer: MathBranchOutcome | null = null;
|
||||
|
||||
/** Repaint hook, set by the owning layer. */
|
||||
onChanged: (() => void) | null = null;
|
||||
|
||||
private readonly session = new MathSession();
|
||||
private completeWithAI: AICompletion | null = null;
|
||||
private longRunCancelled = false;
|
||||
private longRunRunning = false;
|
||||
private longRunStartedAtMs = 0;
|
||||
|
||||
get hasProblem(): boolean {
|
||||
return this.session.hasProblem;
|
||||
}
|
||||
|
||||
get currentSolution() {
|
||||
return this.session.currentSolution;
|
||||
}
|
||||
|
||||
get explanationSteps(): SolutionStep[] {
|
||||
return this.session.currentSolution?.steps ?? [];
|
||||
}
|
||||
|
||||
get isRunningLong(): boolean {
|
||||
return this.longRunRunning;
|
||||
}
|
||||
|
||||
setUsesDegrees(value: boolean): void {
|
||||
this.session.setUsesDegrees(value);
|
||||
}
|
||||
|
||||
setAICompletion(completion: AICompletion | null): void {
|
||||
this.completeWithAI = completion;
|
||||
}
|
||||
|
||||
private changed(): void {
|
||||
this.onChanged?.();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Branch execution
|
||||
|
||||
/**
|
||||
* Run one branch. Returns the glasses text, or a failure string — the
|
||||
* caller distinguishes them.
|
||||
*/
|
||||
run(mode: MathBranchMode, input: string): MathBranchOutcome {
|
||||
const trimmed = input.trim();
|
||||
|
||||
let sessionResult: MathSessionResult;
|
||||
switch (mode) {
|
||||
case "solve":
|
||||
// "Solve it" after Graph/Explain means show the standing answer
|
||||
// again. Requiring fresh input here made the third follow-up mode
|
||||
// uniquely fail.
|
||||
if (!trimmed && this.session.hasProblem) {
|
||||
sessionResult = this.session.execute({ kind: "restate" });
|
||||
} else if (!trimmed) {
|
||||
return { kind: "failure", reason: "Nothing was captured to solve" };
|
||||
} else {
|
||||
sessionResult = this.session.solve(trimmed);
|
||||
}
|
||||
break;
|
||||
case "graph":
|
||||
// A graph branch may fire with a fresh capture ("graph x squared") or
|
||||
// as a follow-up on the standing problem ("graph it"). Both have to
|
||||
// work from the same branch.
|
||||
if (trimmed && (parseCommand(trimmed) === null || this.isProblem(trimmed))) {
|
||||
this.session.solve(trimmed);
|
||||
}
|
||||
sessionResult = this.session.graph();
|
||||
break;
|
||||
case "explain":
|
||||
if (trimmed && this.isProblem(trimmed)) this.session.solve(trimmed);
|
||||
sessionResult = this.session.explain();
|
||||
break;
|
||||
case "derivative":
|
||||
if (trimmed && this.isProblem(trimmed)) this.session.solve(trimmed);
|
||||
sessionResult = this.session.differentiate(null);
|
||||
break;
|
||||
case "digitSearch":
|
||||
case "digitCount": {
|
||||
const digits = MathCoordinator.digitRun(trimmed);
|
||||
if (!digits) return { kind: "failure", reason: "No digits were captured to look for" };
|
||||
// The constant is read from the capture when the wearer named one
|
||||
// ("…in the golden ratio") and is π otherwise.
|
||||
const constant: MathConstantName = constantNamed(trimmed) ?? "pi";
|
||||
sessionResult = this.session.execute({
|
||||
kind: "advanced",
|
||||
topic:
|
||||
mode === "digitCount"
|
||||
? { kind: "digitCount", pattern: digits, constant }
|
||||
: { kind: "digitSearch", pattern: digits, constant, occurrence: 1 },
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return this.publish(sessionResult);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every digit in the capture, in order, with the separators dropped.
|
||||
*
|
||||
* Joined rather than "the first run" because ASR punctuates a spoken date
|
||||
* freely — "07 04 84", "070484", "7/4/84" are all the same six digits —
|
||||
* and taking the first run would search π for "07", which appears within
|
||||
* the first dozen places and looks for all the world like a working
|
||||
* answer.
|
||||
*/
|
||||
static digitRun(text: string): string | null {
|
||||
const joined = text.replace(/\D/g, "");
|
||||
return joined ? joined : null;
|
||||
}
|
||||
|
||||
private isProblem(text: string): boolean {
|
||||
return parseCommand(text)?.kind === "problem";
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Spoken questions
|
||||
|
||||
/**
|
||||
* Answer a question stated in words, whatever shape it arrived in.
|
||||
*
|
||||
* Two passes, in this order and never the other way round:
|
||||
*
|
||||
* 1. The deterministic engine alone. The spoken normaliser already turns
|
||||
* "two x squared plus three x" into `2x²+3x`, and every answer it gives
|
||||
* is exact.
|
||||
* 2. Only if that fails to parse, ask the model to REWRITE the words as an
|
||||
* expression — and then solve that expression here, deterministically.
|
||||
*
|
||||
* The model never computes anything. It is used as a translator from
|
||||
* English into notation, which is the part it is genuinely good at and the
|
||||
* part where a mistake is visible: the rewritten expression is returned
|
||||
* alongside the answer, so a wrong reading shows up as a wrong-looking
|
||||
* equation rather than as a plausible wrong number.
|
||||
*/
|
||||
async answerSpoken(text: string): Promise<MathBranchOutcome> {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return { kind: "failure", reason: "Nothing was said to calculate" };
|
||||
|
||||
const command = parseCommand(trimmed);
|
||||
if (command) {
|
||||
const outcome = this.publish(this.session.execute(command));
|
||||
if (outcome.kind === "value") return outcome;
|
||||
// A FOLLOW-UP that failed keeps its own message. "Graph it" with
|
||||
// nothing on the lens fails because there is nothing to graph, and
|
||||
// handing those two words to a rewriter would replace that with "could
|
||||
// not read that as a calculation" — true of the words, and useless
|
||||
// about the actual problem.
|
||||
if (command.kind !== "problem") return outcome;
|
||||
}
|
||||
|
||||
const rewritten = await this.expressionFromAI(trimmed);
|
||||
if (!rewritten) {
|
||||
return { kind: "failure", reason: "Could not read that as a calculation" };
|
||||
}
|
||||
const outcome = this.publish(this.session.solve(rewritten));
|
||||
if (outcome.kind !== "value") return outcome;
|
||||
// Show the reading, not just the result. A wearer glancing at the lens
|
||||
// has to be able to see that the machine understood the right question
|
||||
// before they repeat the number.
|
||||
return { kind: "value", text: `${rewritten}\n${outcome.text}` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the configured assistant to render a spoken problem as an
|
||||
* expression. Returns null when no provider is configured, when it
|
||||
* declines, or when it answers with anything that is not one.
|
||||
*/
|
||||
private async expressionFromAI(text: string): Promise<string | null> {
|
||||
if (!this.completeWithAI) return null;
|
||||
let raw: string | null;
|
||||
try {
|
||||
raw = await this.completeWithAI(text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!raw) return null;
|
||||
|
||||
let candidate = raw.trim();
|
||||
// Models fence notation out of habit even when told not to.
|
||||
if (candidate.startsWith("```")) {
|
||||
candidate = candidate
|
||||
.split("\n")
|
||||
.filter((line) => !line.startsWith("```"))
|
||||
.join(" ")
|
||||
.trim();
|
||||
}
|
||||
candidate = candidate.replace(/^[ `.,]+|[ `.,]+$/g, "");
|
||||
|
||||
if (!candidate || candidate.toLowerCase() === "none") return null;
|
||||
// A refusal or an apology is prose, and prose fed to the parser fails in
|
||||
// a way that reads like the maths was wrong rather than the routing.
|
||||
if (candidate.length > 120) return null;
|
||||
if (!/[+\-*/^=\d]/.test(candidate)) return null;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Voice follow-ups
|
||||
|
||||
/**
|
||||
* Handle a spoken follow-up. Returns null when the utterance is not a
|
||||
* maths command, so ordinary conversation flows past untouched — this is
|
||||
* called from the transcript stream, which sees everything said in the
|
||||
* room in continuous mode.
|
||||
*/
|
||||
handleUtterance(utterance: string): MathBranchOutcome | null {
|
||||
// A standing offer or a run in flight owns the next yes/no said in the
|
||||
// room. Checked before the maths parser, because "keep going" is not an
|
||||
// equation and would otherwise fall straight through.
|
||||
const longRunAnswer = this.handleLongRunUtterance(utterance);
|
||||
if (longRunAnswer) return longRunAnswer;
|
||||
|
||||
const command = parseCommand(utterance);
|
||||
if (!command) return null;
|
||||
// A bare problem with no standing session is a new problem; a follow-up
|
||||
// with no session is a no-op rather than an error card in mid-air.
|
||||
if (command.kind !== "problem" && command.kind !== "advanced" && !this.session.hasProblem) {
|
||||
return null;
|
||||
}
|
||||
return this.publish(this.session.execute(command));
|
||||
}
|
||||
|
||||
/**
|
||||
* Accept or decline a long run by voice. Only fires while an offer stands
|
||||
* or a run is going, so these phrases stay out of the way of ordinary
|
||||
* conversation the rest of the time.
|
||||
*/
|
||||
handleLongRunUtterance(utterance: string): MathBranchOutcome | null {
|
||||
const text = utterance.toLowerCase();
|
||||
|
||||
if (this.longRunRunning) {
|
||||
if (!STOP_PHRASES.some((phrase) => text.includes(phrase))) return null;
|
||||
this.cancelLongRun();
|
||||
return { kind: "failure", reason: "Stopping" };
|
||||
}
|
||||
|
||||
const offer = this.pendingLongRun;
|
||||
if (!offer) return null;
|
||||
if (ACCEPT_PHRASES.some((phrase) => text.includes(phrase))) {
|
||||
void this.acceptLongRun();
|
||||
return { kind: "value", text: `Working — ${compactDuration(offer.estimatedSeconds)}` };
|
||||
}
|
||||
if (DECLINE_PHRASES.some((phrase) => text.includes(phrase))) {
|
||||
this.declineLongRun();
|
||||
return { kind: "failure", reason: "Left it there" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
get hasLongRunInteraction(): boolean {
|
||||
return this.pendingLongRun !== null || this.longRunRunning;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Uniform gesture contract
|
||||
|
||||
handleRingAction(action: RingAction): MathBranchOutcome | null {
|
||||
// The offer and the running job both pre-empt the step navigation: while
|
||||
// either is on screen, the input is answering that question and nothing
|
||||
// else. Same gestures, no new vocabulary.
|
||||
if (this.longRunRunning) {
|
||||
if (action !== "exit") return null;
|
||||
this.cancelLongRun();
|
||||
return { kind: "failure", reason: "Stopping" };
|
||||
}
|
||||
const offer = this.pendingLongRun;
|
||||
if (offer) {
|
||||
switch (action) {
|
||||
case "commit":
|
||||
void this.acceptLongRun();
|
||||
return { kind: "value", text: `Working — ${compactDuration(offer.estimatedSeconds)}` };
|
||||
case "exit":
|
||||
this.declineLongRun();
|
||||
return { kind: "failure", reason: "Left it there" };
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.session.hasProblem) return null;
|
||||
switch (action) {
|
||||
case "forward":
|
||||
return this.publish(this.session.execute({ kind: "nextStep" }));
|
||||
case "back":
|
||||
return this.publish(this.session.execute({ kind: "previousStep" }));
|
||||
case "commit":
|
||||
return this.publish(this.session.graph());
|
||||
case "alternate":
|
||||
return this.publish(this.session.explain());
|
||||
case "exit":
|
||||
return this.publish(this.session.execute({ kind: "clear" }));
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Long runs
|
||||
|
||||
/** Take the standing offer. Returns false when there was nothing to accept. */
|
||||
acceptLongRun(): boolean {
|
||||
const offer = this.pendingLongRun;
|
||||
if (!offer || this.longRunRunning) return false;
|
||||
this.pendingLongRun = null;
|
||||
this.deepAnswer = null;
|
||||
this.longRunCancelled = false;
|
||||
this.longRunRunning = true;
|
||||
this.longRunStartedAtMs = Date.now();
|
||||
this.longRunProgress = { fraction: 0, stage: "starting", elapsedSeconds: 0 };
|
||||
this.changed();
|
||||
|
||||
const cancellation = () => this.longRunCancelled;
|
||||
const progress = (fraction: number, stage: string) => {
|
||||
this.longRunProgress = {
|
||||
fraction,
|
||||
stage,
|
||||
elapsedSeconds: (Date.now() - this.longRunStartedAtMs) / 1000,
|
||||
};
|
||||
this.changed();
|
||||
};
|
||||
|
||||
void (async () => {
|
||||
let outcome: MathBranchOutcome;
|
||||
try {
|
||||
switch (offer.workload.kind) {
|
||||
case "digitSearch": {
|
||||
const found = await runAcceptedSearch(offer, cancellation, progress);
|
||||
outcome = describeDeepSearch(found, offer.workload.pattern, offer.workload.constant);
|
||||
break;
|
||||
}
|
||||
case "constantDigits": {
|
||||
const digits = await runAcceptedDigits(offer, cancellation, progress);
|
||||
outcome = digits
|
||||
? {
|
||||
kind: "value",
|
||||
text: `${constantDisplayName(offer.workload.constant)}: ${formatInt(digits.length)} digits ready`,
|
||||
}
|
||||
: { kind: "failure", reason: "Could not generate that many digits" };
|
||||
break;
|
||||
}
|
||||
case "digitCount": {
|
||||
const counted = await runAcceptedCount(offer, cancellation, progress);
|
||||
outcome = {
|
||||
kind: "value",
|
||||
text: describeCount(counted.occurrences, counted.searchedDigits, offer.workload.pattern, offer.workload.constant),
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
outcome =
|
||||
error instanceof ComputeCancelled
|
||||
? { kind: "failure", reason: "Stopped" }
|
||||
: { kind: "failure", reason: String((error as Error)?.message ?? error) };
|
||||
}
|
||||
this.longRunRunning = false;
|
||||
this.longRunProgress = null;
|
||||
this.deepAnswer = outcome;
|
||||
this.changed();
|
||||
})();
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Turn the offer down. The shallow answer already given stands. */
|
||||
declineLongRun(): void {
|
||||
this.pendingLongRun = null;
|
||||
this.changed();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop a run in flight. The digits computed so far are discarded — a
|
||||
* partial expansion cannot answer a position question.
|
||||
*/
|
||||
cancelLongRun(): void {
|
||||
this.longRunCancelled = true;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Publishing
|
||||
|
||||
private publish(sessionResult: MathSessionResult): MathBranchOutcome {
|
||||
this.lastResult = sessionResult;
|
||||
|
||||
// The shallow answer goes out immediately; anything better arrives on
|
||||
// `deepAnswer` if the wearer accepts the standing offer. Never block the
|
||||
// first response on the second — silence is worse than a partial.
|
||||
if (sessionResult.unresolvedWorkload) {
|
||||
this.pendingLongRun = makeLongRunOffer(
|
||||
sessionResult.unresolvedWorkload,
|
||||
"Needs a deeper expansion than the fast pass holds",
|
||||
shallowDescription(sessionResult.unresolvedWorkload, sessionResult),
|
||||
);
|
||||
this.deepAnswer = null;
|
||||
}
|
||||
|
||||
switch (sessionResult.payload.kind) {
|
||||
case "graph":
|
||||
this.lastGraph = sessionResult.payload.graph;
|
||||
this.stepPosition = null;
|
||||
break;
|
||||
case "step":
|
||||
this.lastGraph = null;
|
||||
this.stepPosition = { index: sessionResult.payload.index, total: sessionResult.payload.total };
|
||||
break;
|
||||
case "solution":
|
||||
case "message":
|
||||
case "topic":
|
||||
this.lastGraph = null;
|
||||
// An advanced topic answers in one shot — there is no working to step
|
||||
// through, so the step indicator must clear.
|
||||
this.stepPosition = null;
|
||||
break;
|
||||
case "failure":
|
||||
break;
|
||||
}
|
||||
this.changed();
|
||||
|
||||
if (sessionResult.payload.kind === "failure") {
|
||||
return { kind: "failure", reason: sessionResult.payload.message };
|
||||
}
|
||||
return { kind: "value", text: this.glassesText(sessionResult) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Two or three short rows. The lens has no room for the detail sentence,
|
||||
* which is why `detail` stays in the fuller answer view.
|
||||
*/
|
||||
private glassesText(sessionResult: MathSessionResult): string {
|
||||
switch (sessionResult.payload.kind) {
|
||||
case "step": {
|
||||
const payload = sessionResult.payload;
|
||||
return `${payload.index + 1}/${payload.total} ${payload.step.title}\n${plain(payload.step.expression)}`;
|
||||
}
|
||||
case "graph":
|
||||
// The bitmap carries the picture; this is the text fallback.
|
||||
return sessionResult.headline;
|
||||
case "topic": {
|
||||
// Two field rows at most: the window is short and the headline
|
||||
// already took a line.
|
||||
const payload = sessionResult.payload;
|
||||
return [payload.headline, ...payload.fields.slice(0, 2)].join("\n");
|
||||
}
|
||||
default:
|
||||
return sessionResult.headline;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Long-run phrases
|
||||
|
||||
const ACCEPT_PHRASES = [
|
||||
"keep going",
|
||||
"run it anyway",
|
||||
"run it any way",
|
||||
"take your time",
|
||||
"go deeper",
|
||||
"keep looking",
|
||||
"do it anyway",
|
||||
"yes go on",
|
||||
];
|
||||
|
||||
const DECLINE_PHRASES = [
|
||||
"never mind",
|
||||
"nevermind",
|
||||
"forget it",
|
||||
"leave it",
|
||||
"don't bother",
|
||||
"do not bother",
|
||||
"skip it",
|
||||
];
|
||||
|
||||
const STOP_PHRASES = ["stop", "cancel", "give up", "that's enough", "thats enough"];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Descriptions
|
||||
|
||||
function describeDeepSearch(
|
||||
outcome: Awaited<ReturnType<typeof runAcceptedSearch>>,
|
||||
pattern: string,
|
||||
constant: MathConstantName,
|
||||
): MathBranchOutcome {
|
||||
switch (outcome.kind) {
|
||||
case "found":
|
||||
return {
|
||||
kind: "value",
|
||||
text:
|
||||
`${constantDisplayName(constant)}: places ${outcome.match.start}–${outcome.match.end}\n` +
|
||||
`…${highlightedMatch(outcome.match)}…`,
|
||||
};
|
||||
case "notFound":
|
||||
// Now a much stronger claim than the shallow pass could make, so it is
|
||||
// worth stating the depth that was actually reached.
|
||||
return {
|
||||
kind: "failure",
|
||||
reason:
|
||||
`${pattern} is not in the first ${formatInt(outcome.searchedDigits)} digits ` +
|
||||
`of ${constantDisplayName(constant)}`,
|
||||
};
|
||||
case "invalidPattern":
|
||||
return { kind: "failure", reason: outcome.reason };
|
||||
}
|
||||
}
|
||||
|
||||
function describeCount(
|
||||
occurrences: number,
|
||||
searchedDigits: number,
|
||||
pattern: string,
|
||||
constant: MathConstantName,
|
||||
): string {
|
||||
const times = occurrences === 1 ? "time" : "times";
|
||||
return `${pattern}: ${occurrences} ${times}\nfirst ${formatInt(searchedDigits)} digits of ${constantDisplayName(constant)}`;
|
||||
}
|
||||
|
||||
function shallowDescription(workload: MathWorkload, sessionResult: MathSessionResult): string | null {
|
||||
switch (workload.kind) {
|
||||
case "digitSearch":
|
||||
return sessionResult.headline;
|
||||
case "digitCount":
|
||||
return sessionResult.headline;
|
||||
case "constantDigits":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
// Symbolic differentiation. Every rule is applied structurally and the result
|
||||
// is handed to the simplifier, so `d/dx (x^2)` comes back as `2x` rather than
|
||||
// `2 · x^(2-1) · 1`. That matters because the derivative is user-visible — it
|
||||
// is what "explain it" shows.
|
||||
|
||||
import {
|
||||
type MathExpression,
|
||||
type MathFunction,
|
||||
ONE,
|
||||
ZERO,
|
||||
addExpr,
|
||||
divideExpr,
|
||||
fn,
|
||||
int,
|
||||
multiplyExpr,
|
||||
negate,
|
||||
powerExpr,
|
||||
relation,
|
||||
subtractExpr,
|
||||
variablesOf,
|
||||
} from "./expression";
|
||||
import { simplify } from "./simplifier";
|
||||
import { evaluateValue } from "./evaluator";
|
||||
|
||||
export function differentiate(expression: MathExpression, withRespectTo: string): MathExpression {
|
||||
return simplify(raw(expression, withRespectTo));
|
||||
}
|
||||
|
||||
/** Nth derivative. */
|
||||
export function differentiateOrder(
|
||||
expression: MathExpression,
|
||||
withRespectTo: string,
|
||||
order: number,
|
||||
): MathExpression {
|
||||
let current = expression;
|
||||
for (let step = 0; step < Math.max(0, order); step++) {
|
||||
current = differentiate(current, withRespectTo);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function raw(expression: MathExpression, variable: string): MathExpression {
|
||||
switch (expression.type) {
|
||||
case "number":
|
||||
case "constant":
|
||||
return ZERO;
|
||||
|
||||
case "variable":
|
||||
return expression.name === variable ? ONE : ZERO;
|
||||
|
||||
case "negate":
|
||||
return negate(raw(expression.inner, variable));
|
||||
|
||||
case "add":
|
||||
return addExpr(expression.terms.map((term) => raw(term, variable)));
|
||||
|
||||
case "multiply": {
|
||||
// Product rule, generalised: d(abc) = a'bc + ab'c + abc'.
|
||||
const terms: MathExpression[] = [];
|
||||
for (let index = 0; index < expression.factors.length; index++) {
|
||||
const replaced = [...expression.factors];
|
||||
replaced[index] = raw(expression.factors[index]!, variable);
|
||||
terms.push(multiplyExpr(replaced));
|
||||
}
|
||||
return addExpr(terms);
|
||||
}
|
||||
|
||||
case "power":
|
||||
return differentiatePower(expression.base, expression.exponent, variable);
|
||||
|
||||
case "function": {
|
||||
const argument = expression.args[0];
|
||||
if (!argument) return ZERO;
|
||||
const outer = derivativeOfFunction(expression.fn, argument);
|
||||
// Chain rule. Skipping this when the argument is the bare variable
|
||||
// would be a needless special case — the simplifier removes the `· 1`
|
||||
// anyway.
|
||||
return multiplyExpr([outer, raw(argument, variable)]);
|
||||
}
|
||||
|
||||
case "relation":
|
||||
// Differentiating both sides of an equation is implicit
|
||||
// differentiation's first step; useful, and harmless here.
|
||||
return relation(expression.relation, raw(expression.lhs, variable), raw(expression.rhs, variable));
|
||||
}
|
||||
}
|
||||
|
||||
function differentiatePower(
|
||||
base: MathExpression,
|
||||
exponent: MathExpression,
|
||||
variable: string,
|
||||
): MathExpression {
|
||||
const baseHasVariable = variablesOf(base).includes(variable);
|
||||
const exponentHasVariable = variablesOf(exponent).includes(variable);
|
||||
|
||||
if (!baseHasVariable && !exponentHasVariable) return ZERO;
|
||||
|
||||
if (baseHasVariable && !exponentHasVariable) {
|
||||
// Power rule: d(u^n) = n · u^(n-1) · u'
|
||||
const reduced = subtractExpr(exponent, ONE);
|
||||
return multiplyExpr([exponent, powerExpr(base, reduced), raw(base, variable)]);
|
||||
}
|
||||
|
||||
if (!baseHasVariable && exponentHasVariable) {
|
||||
// Exponential rule: d(a^v) = a^v · ln(a) · v'
|
||||
return multiplyExpr([powerExpr(base, exponent), fn("ln", [base]), raw(exponent, variable)]);
|
||||
}
|
||||
|
||||
// General case via logarithmic differentiation:
|
||||
// d(u^v) = u^v · (v' · ln u + v · u'/u)
|
||||
const uv = powerExpr(base, exponent);
|
||||
const firstTerm = multiplyExpr([raw(exponent, variable), fn("ln", [base])]);
|
||||
const secondTerm = multiplyExpr([exponent, divideExpr(raw(base, variable), base)]);
|
||||
return multiplyExpr([uv, addExpr([firstTerm, secondTerm])]);
|
||||
}
|
||||
|
||||
/** The outer derivative only; the caller applies the chain rule. */
|
||||
function derivativeOfFunction(name: MathFunction, argument: MathExpression): MathExpression {
|
||||
switch (name) {
|
||||
case "sin":
|
||||
return fn("cos", [argument]);
|
||||
case "cos":
|
||||
return negate(fn("sin", [argument]));
|
||||
case "tan":
|
||||
// sec² = 1 / cos²
|
||||
return divideExpr(ONE, powerExpr(fn("cos", [argument]), int(2)));
|
||||
case "asin":
|
||||
return divideExpr(ONE, fn("sqrt", [subtractExpr(ONE, powerExpr(argument, int(2)))]));
|
||||
case "acos":
|
||||
return negate(divideExpr(ONE, fn("sqrt", [subtractExpr(ONE, powerExpr(argument, int(2)))])));
|
||||
case "atan":
|
||||
return divideExpr(ONE, addExpr([ONE, powerExpr(argument, int(2))]));
|
||||
case "sinh":
|
||||
return fn("cosh", [argument]);
|
||||
case "cosh":
|
||||
return fn("sinh", [argument]);
|
||||
case "tanh":
|
||||
return divideExpr(ONE, powerExpr(fn("cosh", [argument]), int(2)));
|
||||
case "ln":
|
||||
return divideExpr(ONE, argument);
|
||||
case "log10":
|
||||
return divideExpr(ONE, multiplyExpr([argument, fn("ln", [int(10)])]));
|
||||
case "log2":
|
||||
return divideExpr(ONE, multiplyExpr([argument, fn("ln", [int(2)])]));
|
||||
case "sqrt":
|
||||
return divideExpr(ONE, multiplyExpr([int(2), fn("sqrt", [argument])]));
|
||||
case "cbrt":
|
||||
return divideExpr(ONE, multiplyExpr([int(3), powerExpr(fn("cbrt", [argument]), int(2))]));
|
||||
case "exp":
|
||||
return fn("exp", [argument]);
|
||||
case "abs":
|
||||
// d|u|/du = u/|u| — undefined at zero, which the plotter's
|
||||
// discontinuity handling already copes with.
|
||||
return divideExpr(argument, fn("abs", [argument]));
|
||||
case "floor":
|
||||
case "ceil":
|
||||
case "round":
|
||||
// Zero almost everywhere; the jumps are measure-zero and are not
|
||||
// representable here.
|
||||
return ZERO;
|
||||
case "factorial":
|
||||
// The derivative of the gamma function needs the digamma function,
|
||||
// which this build does not carry. Returning zero would be a lie, so
|
||||
// the caller gets an unevaluated marker instead.
|
||||
return fn("factorial", [argument]);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Numeric integration
|
||||
|
||||
/**
|
||||
* Definite integral by adaptive Simpson's rule.
|
||||
*
|
||||
* Numeric rather than symbolic: symbolic integration needs a Risch-style
|
||||
* algorithm that is far out of proportion to the feature, and a definite
|
||||
* value is what a calculator user actually asks for. (The symbolic table in
|
||||
* symbolic-integral.ts is tried first where it applies.)
|
||||
*/
|
||||
export function integrateNumeric(
|
||||
expression: MathExpression,
|
||||
variable: string,
|
||||
lower: number,
|
||||
upper: number,
|
||||
tolerance = 1e-9,
|
||||
): number | null {
|
||||
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
|
||||
if (lower === upper) return 0;
|
||||
if (lower > upper) {
|
||||
const flipped = integrateNumeric(expression, variable, upper, lower, tolerance);
|
||||
return flipped === null ? null : -flipped;
|
||||
}
|
||||
|
||||
const f = (x: number): number | null => evaluateValue(expression, { [variable]: x });
|
||||
|
||||
const simpson = (a: number, b: number, fa: number, fb: number, fm: number): number =>
|
||||
((b - a) / 6) * (fa + 4 * fm + fb);
|
||||
|
||||
const adaptive = (
|
||||
a: number,
|
||||
b: number,
|
||||
fa: number,
|
||||
fb: number,
|
||||
fm: number,
|
||||
whole: number,
|
||||
depth: number,
|
||||
): number | null => {
|
||||
const m = (a + b) / 2;
|
||||
const leftMid = (a + m) / 2;
|
||||
const rightMid = (m + b) / 2;
|
||||
const flm = f(leftMid);
|
||||
const frm = f(rightMid);
|
||||
if (flm === null || frm === null) return null;
|
||||
const left = simpson(a, m, fa, fm, flm);
|
||||
const right = simpson(m, b, fm, fb, frm);
|
||||
const total = left + right;
|
||||
// Richardson: the Simpson error term is (total - whole) / 15.
|
||||
if (depth <= 0 || Math.abs(total - whole) <= 15 * tolerance) {
|
||||
return total + (total - whole) / 15;
|
||||
}
|
||||
const leftResult = adaptive(a, m, fa, fm, flm, left, depth - 1);
|
||||
if (leftResult === null) return null;
|
||||
const rightResult = adaptive(m, b, fm, fb, frm, right, depth - 1);
|
||||
if (rightResult === null) return null;
|
||||
return leftResult + rightResult;
|
||||
};
|
||||
|
||||
const fa = f(lower);
|
||||
const fb = f(upper);
|
||||
const fm = f((lower + upper) / 2);
|
||||
if (fa === null || fb === null || fm === null) return null;
|
||||
const whole = simpson(lower, upper, fa, fb, fm);
|
||||
return adaptive(lower, upper, fa, fb, fm, whole, 24);
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// Finds a digit sequence inside the expansion of a constant.
|
||||
//
|
||||
// The position convention is fixed and stated everywhere it is reported:
|
||||
// **decimal places after the point, 1-based**. So the first `11` in π is at
|
||||
// 94–95, because π = 3.(1)4159… and the 94th digit after the point is the
|
||||
// first `1` of that pair. Every off-by-one complaint about this kind of
|
||||
// feature is a convention that was never written down, so it is written down.
|
||||
|
||||
import {
|
||||
type MathConstantName,
|
||||
MAXIMUM_COMPUTED_DIGITS,
|
||||
fractionalDigits,
|
||||
} from "./constant-digits";
|
||||
|
||||
export type DigitCountResult = {
|
||||
occurrences: number;
|
||||
searchedDigits: number;
|
||||
};
|
||||
|
||||
export type DigitMatch = {
|
||||
/** 1-based decimal place of the first digit of the match. */
|
||||
start: number;
|
||||
/** 1-based decimal place of the last digit. */
|
||||
end: number;
|
||||
/** The digits either side, for the reveal. */
|
||||
context: string;
|
||||
/** Where the match sits inside `context`, as an offset. */
|
||||
contextOffset: number;
|
||||
};
|
||||
|
||||
/** `...3421 ·11· 70679...` with the match marked. */
|
||||
export function highlightedMatch(match: DigitMatch, marker = "·"): string {
|
||||
const length = match.end - match.start + 1;
|
||||
if (match.contextOffset < 0 || match.contextOffset + length > match.context.length) {
|
||||
return match.context;
|
||||
}
|
||||
const before = match.context.slice(0, match.contextOffset);
|
||||
const middle = match.context.slice(match.contextOffset, match.contextOffset + length);
|
||||
const after = match.context.slice(match.contextOffset + length);
|
||||
return `${before}${marker}${middle}${marker}${after}`;
|
||||
}
|
||||
|
||||
export type DigitSearchOutcome =
|
||||
| { kind: "found"; match: DigitMatch }
|
||||
/**
|
||||
* Searched the whole available expansion and did not find it. Carries how
|
||||
* far the search actually reached, because "not in π" and "not in the first
|
||||
* 20,000 digits of π" are very different claims and only one of them is
|
||||
* true.
|
||||
*/
|
||||
| { kind: "notFound"; searchedDigits: number }
|
||||
| { kind: "invalidPattern"; reason: string };
|
||||
|
||||
/** How many digits either side of a hit to return. */
|
||||
export const DEFAULT_CONTEXT_RADIUS = 6;
|
||||
|
||||
export type DigitSource = (constant: MathConstantName, count: number) => string | null;
|
||||
|
||||
const DEFAULT_SOURCE: DigitSource = (constant, count) => fractionalDigits(constant, count);
|
||||
|
||||
/** Find the `occurrence`-th appearance of `pattern`. */
|
||||
export function findDigits(
|
||||
pattern: string,
|
||||
constant: MathConstantName,
|
||||
occurrence = 1,
|
||||
requested: number = MAXIMUM_COMPUTED_DIGITS,
|
||||
contextRadius: number = DEFAULT_CONTEXT_RADIUS,
|
||||
digitSource: DigitSource = DEFAULT_SOURCE,
|
||||
): DigitSearchOutcome {
|
||||
const cleaned = pattern.replace(/\D/g, "");
|
||||
if (!cleaned) return { kind: "invalidPattern", reason: "Give a sequence of digits to look for" };
|
||||
if (occurrence < 1) return { kind: "invalidPattern", reason: "Occurrence must be 1 or more" };
|
||||
|
||||
const expansion = digitSource(constant, requested);
|
||||
if (!expansion) return { kind: "notFound", searchedDigits: 0 };
|
||||
|
||||
return locateDigits(cleaned, expansion, occurrence, contextRadius);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search an expansion the caller already has — the path an accepted long run
|
||||
* uses, where the digits come from the deep generator rather than the
|
||||
* shallow cache.
|
||||
*/
|
||||
export function locateDigits(
|
||||
pattern: string,
|
||||
expansion: string,
|
||||
occurrence = 1,
|
||||
contextRadius: number = DEFAULT_CONTEXT_RADIUS,
|
||||
): DigitSearchOutcome {
|
||||
if (pattern.length > expansion.length) {
|
||||
return { kind: "notFound", searchedDigits: expansion.length };
|
||||
}
|
||||
|
||||
let seen = 0;
|
||||
// Plain scan rather than a substring search: overlapping matches count
|
||||
// ("111" contains "11" twice), and indexOf-with-skip would miss the
|
||||
// overlap.
|
||||
for (let index = 0; index <= expansion.length - pattern.length; index++) {
|
||||
let matched = true;
|
||||
for (let offset = 0; offset < pattern.length; offset++) {
|
||||
if (expansion[index + offset] !== pattern[offset]) {
|
||||
matched = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) continue;
|
||||
seen += 1;
|
||||
if (seen !== occurrence) continue;
|
||||
|
||||
const contextStart = Math.max(0, index - contextRadius);
|
||||
const contextEnd = Math.min(expansion.length, index + pattern.length + contextRadius);
|
||||
return {
|
||||
kind: "found",
|
||||
match: {
|
||||
start: index + 1,
|
||||
end: index + pattern.length,
|
||||
context: expansion.slice(contextStart, contextEnd),
|
||||
contextOffset: index - contextStart,
|
||||
},
|
||||
};
|
||||
}
|
||||
return { kind: "notFound", searchedDigits: expansion.length };
|
||||
}
|
||||
|
||||
/** How many times the pattern appears in the available expansion. */
|
||||
export function countDigits(
|
||||
pattern: string,
|
||||
constant: MathConstantName,
|
||||
requested: number = MAXIMUM_COMPUTED_DIGITS,
|
||||
digitSource: DigitSource = DEFAULT_SOURCE,
|
||||
): DigitCountResult {
|
||||
const cleaned = pattern.replace(/\D/g, "");
|
||||
if (!cleaned) return { occurrences: 0, searchedDigits: 0 };
|
||||
const expansion = digitSource(constant, requested);
|
||||
if (!expansion) return { occurrences: 0, searchedDigits: 0 };
|
||||
return countInExpansion(cleaned, expansion);
|
||||
}
|
||||
|
||||
/** Count overlapping occurrences inside an expansion the caller already has. */
|
||||
export function countInExpansion(pattern: string, expansion: string): DigitCountResult {
|
||||
if (pattern.length > expansion.length) {
|
||||
return { occurrences: 0, searchedDigits: expansion.length };
|
||||
}
|
||||
let total = 0;
|
||||
for (let index = 0; index <= expansion.length - pattern.length; index++) {
|
||||
let matched = true;
|
||||
for (let offset = 0; offset < pattern.length; offset++) {
|
||||
if (expansion[index + offset] !== pattern[offset]) {
|
||||
matched = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matched) total += 1;
|
||||
}
|
||||
return { occurrences: total, searchedDigits: expansion.length };
|
||||
}
|
||||
|
||||
/** The digits at a given range of decimal places, 1-based inclusive. */
|
||||
export function digitsAt(
|
||||
constant: MathConstantName,
|
||||
start: number,
|
||||
end: number,
|
||||
digitSource: DigitSource = DEFAULT_SOURCE,
|
||||
): string | null {
|
||||
if (start < 1 || end < start) return null;
|
||||
const expansion = digitSource(constant, end);
|
||||
if (!expansion || expansion.length < end) return null;
|
||||
return expansion.slice(start - 1, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* How deep you would expect to search before a random `length`-digit string
|
||||
* turns up: 10^length places, on average.
|
||||
*
|
||||
* Reported alongside a notFound so the answer is honest about whether the
|
||||
* search was ever likely to succeed — an eight-digit birthday first appears
|
||||
* around the hundred-millionth digit, which no on-device expansion is going
|
||||
* to reach.
|
||||
*/
|
||||
export function expectedFirstPosition(patternLength: number): number | null {
|
||||
if (patternLength <= 0 || patternLength > 18) return null;
|
||||
return Math.pow(10, patternLength);
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
// Numeric evaluation of an expression tree. Two return paths on purpose:
|
||||
// `evaluateExact` keeps rationals as rationals (so a solution can be shown as
|
||||
// `3/2`), while `evaluateValue` collapses to a plain number for the plotter,
|
||||
// which samples thousands of points and has no use for exactness.
|
||||
|
||||
import {
|
||||
type MathExpression,
|
||||
constantValue,
|
||||
evaluateFunction,
|
||||
functionIsDefined,
|
||||
} from "./expression";
|
||||
import {
|
||||
type MathNumber,
|
||||
add,
|
||||
doubleValue,
|
||||
integer,
|
||||
integerValue,
|
||||
isNegative,
|
||||
isZero,
|
||||
multiply,
|
||||
negated,
|
||||
raisedTo,
|
||||
real,
|
||||
} from "./math-number";
|
||||
|
||||
export type EvaluatorContext = {
|
||||
variables: Record<string, MathNumber>;
|
||||
/**
|
||||
* Radians unless the caller says otherwise. Trig in degrees is a real user
|
||||
* expectation for a calculator, and getting it silently wrong produces
|
||||
* answers that look plausible.
|
||||
*/
|
||||
usesDegrees: boolean;
|
||||
};
|
||||
|
||||
export function radiansContext(): EvaluatorContext {
|
||||
return { variables: {}, usesDegrees: false };
|
||||
}
|
||||
|
||||
export class EvaluationError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "EvaluationError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate keeping exactness where possible. Throws on a free variable, so
|
||||
* the caller can distinguish "not a number yet" from "evaluates to NaN".
|
||||
*/
|
||||
export function evaluateExact(expression: MathExpression, context: EvaluatorContext = radiansContext()): MathNumber {
|
||||
switch (expression.type) {
|
||||
case "number":
|
||||
return expression.value;
|
||||
|
||||
case "variable": {
|
||||
const value = context.variables[expression.name];
|
||||
if (value === undefined) {
|
||||
throw new EvaluationError(`No value given for ${expression.name}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
case "constant":
|
||||
return real(constantValue(expression.constant));
|
||||
|
||||
case "add":
|
||||
return expression.terms.reduce<MathNumber>(
|
||||
(total, term) => add(total, evaluateExact(term, context)),
|
||||
integer(0),
|
||||
);
|
||||
|
||||
case "multiply":
|
||||
return expression.factors.reduce<MathNumber>(
|
||||
(total, factor) => multiply(total, evaluateExact(factor, context)),
|
||||
integer(1),
|
||||
);
|
||||
|
||||
case "negate":
|
||||
return negated(evaluateExact(expression.inner, context));
|
||||
|
||||
case "power": {
|
||||
const baseValue = evaluateExact(expression.base, context);
|
||||
const exponentValue = evaluateExact(expression.exponent, context);
|
||||
if (isZero(baseValue) && isNegative(exponentValue)) {
|
||||
throw new EvaluationError("Division by zero");
|
||||
}
|
||||
return raisedTo(baseValue, exponentValue);
|
||||
}
|
||||
|
||||
case "function": {
|
||||
const first = expression.args[0];
|
||||
if (!first) throw new EvaluationError(`${expression.fn} needs an argument`);
|
||||
const argument = evaluateExact(first, context);
|
||||
// Integer factorial stays exact; everything else goes through the
|
||||
// floating-point implementation.
|
||||
const argumentInt = integerValue(argument);
|
||||
if (expression.fn === "factorial" && argumentInt !== null && argumentInt >= 0 && argumentInt <= 20) {
|
||||
let total = 1;
|
||||
for (let value = 2; value <= argumentInt; value++) total *= value;
|
||||
return integer(total);
|
||||
}
|
||||
const raw = adjustForDegrees(expression.fn, doubleValue(argument), context);
|
||||
if (!functionIsDefined(expression.fn, raw)) {
|
||||
throw new EvaluationError(`${expression.fn} is undefined there`);
|
||||
}
|
||||
const result = evaluateFunction(expression.fn, raw);
|
||||
if (!Number.isFinite(result)) {
|
||||
throw new EvaluationError(`${expression.fn} is undefined there`);
|
||||
}
|
||||
return adjustResultForDegrees(expression.fn, result, context);
|
||||
}
|
||||
|
||||
case "relation": {
|
||||
const left = doubleValue(evaluateExact(expression.lhs, context));
|
||||
const right = doubleValue(evaluateExact(expression.rhs, context));
|
||||
let holds: boolean;
|
||||
switch (expression.relation) {
|
||||
case "=":
|
||||
holds = left === right;
|
||||
break;
|
||||
case "<":
|
||||
holds = left < right;
|
||||
break;
|
||||
case "<=":
|
||||
holds = left <= right;
|
||||
break;
|
||||
case ">":
|
||||
holds = left > right;
|
||||
break;
|
||||
case ">=":
|
||||
holds = left >= right;
|
||||
break;
|
||||
}
|
||||
return integer(holds ? 1 : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Plotting path: never throws, returns null where the function is undefined
|
||||
* so the renderer can break the curve instead of drawing a line across the
|
||||
* gap.
|
||||
*/
|
||||
export function evaluateValue(
|
||||
expression: MathExpression,
|
||||
variables: Record<string, number>,
|
||||
usesDegrees = false,
|
||||
): number | null {
|
||||
const context: EvaluatorContext = { variables: {}, usesDegrees };
|
||||
for (const name of Object.keys(variables)) {
|
||||
context.variables[name] = real(variables[name]!);
|
||||
}
|
||||
let result: MathNumber;
|
||||
try {
|
||||
result = evaluateExact(expression, context);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const value = doubleValue(result);
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Degrees
|
||||
|
||||
function adjustForDegrees(name: string, argument: number, context: EvaluatorContext): number {
|
||||
if (!context.usesDegrees) return argument;
|
||||
switch (name) {
|
||||
case "sin":
|
||||
case "cos":
|
||||
case "tan":
|
||||
return (argument * Math.PI) / 180;
|
||||
default:
|
||||
return argument;
|
||||
}
|
||||
}
|
||||
|
||||
function adjustResultForDegrees(name: string, result: number, context: EvaluatorContext): MathNumber {
|
||||
if (!context.usesDegrees) return real(result);
|
||||
switch (name) {
|
||||
case "asin":
|
||||
case "acos":
|
||||
case "atan":
|
||||
return real((result * 180) / Math.PI);
|
||||
default:
|
||||
return real(result);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
// The expression tree everything else operates on: evaluation, simplification,
|
||||
// differentiation, solving, rendering, and plotting. A plain discriminated
|
||||
// union so the whole tree is a value — simplification and differentiation can
|
||||
// rebuild subtrees freely, and structural comparison (expressionEquals) gives
|
||||
// the simplifier the like-term matching it leans on heavily.
|
||||
|
||||
import {
|
||||
type MathNumber,
|
||||
integer as makeInteger,
|
||||
fromDecimal,
|
||||
isZero as numberIsZero,
|
||||
isOne as numberIsOne,
|
||||
numberEquals as numbersEqual,
|
||||
} from "./math-number";
|
||||
|
||||
export type MathRelation = "=" | "<" | "<=" | ">" | ">=";
|
||||
|
||||
export type MathConstant = "pi" | "e" | "tau";
|
||||
|
||||
export type MathFunction =
|
||||
| "sin"
|
||||
| "cos"
|
||||
| "tan"
|
||||
| "asin"
|
||||
| "acos"
|
||||
| "atan"
|
||||
| "sinh"
|
||||
| "cosh"
|
||||
| "tanh"
|
||||
| "ln"
|
||||
| "log10"
|
||||
| "log2"
|
||||
| "sqrt"
|
||||
| "cbrt"
|
||||
| "abs"
|
||||
| "exp"
|
||||
| "floor"
|
||||
| "ceil"
|
||||
| "round"
|
||||
| "factorial";
|
||||
|
||||
export const ALL_FUNCTIONS: readonly MathFunction[] = [
|
||||
"sin",
|
||||
"cos",
|
||||
"tan",
|
||||
"asin",
|
||||
"acos",
|
||||
"atan",
|
||||
"sinh",
|
||||
"cosh",
|
||||
"tanh",
|
||||
"ln",
|
||||
"log10",
|
||||
"log2",
|
||||
"sqrt",
|
||||
"cbrt",
|
||||
"abs",
|
||||
"exp",
|
||||
"floor",
|
||||
"ceil",
|
||||
"round",
|
||||
"factorial",
|
||||
];
|
||||
|
||||
export const ALL_CONSTANTS: readonly MathConstant[] = ["pi", "e", "tau"];
|
||||
|
||||
export type MathExpression =
|
||||
| { type: "number"; value: MathNumber }
|
||||
/** A free variable — `x`, `y`, `theta`. */
|
||||
| { type: "variable"; name: string }
|
||||
/**
|
||||
* A named constant with a known value (`pi`, `e`). Kept distinct from
|
||||
* `variable` so "solve for pi" is impossible and `2π` stays exact instead of
|
||||
* becoming 6.283…
|
||||
*/
|
||||
| { type: "constant"; constant: MathConstant }
|
||||
| { type: "add"; terms: MathExpression[] }
|
||||
| { type: "multiply"; factors: MathExpression[] }
|
||||
| { type: "power"; base: MathExpression; exponent: MathExpression }
|
||||
/**
|
||||
* Unary negation. Not folded into `multiply([-1, x])` because the renderers
|
||||
* and the step explanations both want to say "negative x".
|
||||
*/
|
||||
| { type: "negate"; inner: MathExpression }
|
||||
| { type: "function"; fn: MathFunction; args: MathExpression[] }
|
||||
/** An equation or inequality — the root node of anything solvable. */
|
||||
| { type: "relation"; relation: MathRelation; lhs: MathExpression; rhs: MathExpression };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Convenience constructors
|
||||
|
||||
export function num(value: MathNumber): MathExpression {
|
||||
return { type: "number", value };
|
||||
}
|
||||
|
||||
export function int(value: number): MathExpression {
|
||||
return num(makeInteger(value));
|
||||
}
|
||||
|
||||
export function realExpr(value: number): MathExpression {
|
||||
return num(fromDecimal(value));
|
||||
}
|
||||
|
||||
export function variable(name: string): MathExpression {
|
||||
return { type: "variable", name };
|
||||
}
|
||||
|
||||
export function constant(value: MathConstant): MathExpression {
|
||||
return { type: "constant", constant: value };
|
||||
}
|
||||
|
||||
export function addExpr(terms: MathExpression[]): MathExpression {
|
||||
return { type: "add", terms };
|
||||
}
|
||||
|
||||
export function multiplyExpr(factors: MathExpression[]): MathExpression {
|
||||
return { type: "multiply", factors };
|
||||
}
|
||||
|
||||
export function powerExpr(base: MathExpression, exponent: MathExpression): MathExpression {
|
||||
return { type: "power", base, exponent };
|
||||
}
|
||||
|
||||
export function negate(inner: MathExpression): MathExpression {
|
||||
return { type: "negate", inner };
|
||||
}
|
||||
|
||||
export function fn(name: MathFunction, args: MathExpression[]): MathExpression {
|
||||
return { type: "function", fn: name, args };
|
||||
}
|
||||
|
||||
export function relation(rel: MathRelation, lhs: MathExpression, rhs: MathExpression): MathExpression {
|
||||
return { type: "relation", relation: rel, lhs, rhs };
|
||||
}
|
||||
|
||||
export function subtractExpr(lhs: MathExpression, rhs: MathExpression): MathExpression {
|
||||
return addExpr([lhs, negate(rhs)]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Division is `a * b^-1` in the tree, which is what makes the simplifier and
|
||||
* the differentiator tractable — the renderers detect the pattern and print a
|
||||
* real fraction, so nothing user-visible degrades.
|
||||
*/
|
||||
export function divideExpr(lhs: MathExpression, rhs: MathExpression): MathExpression {
|
||||
return multiplyExpr([lhs, powerExpr(rhs, int(-1))]);
|
||||
}
|
||||
|
||||
export const ZERO: MathExpression = int(0);
|
||||
export const ONE: MathExpression = int(1);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Introspection
|
||||
|
||||
export function isZeroExpr(expression: MathExpression): boolean {
|
||||
return expression.type === "number" && numberIsZero(expression.value);
|
||||
}
|
||||
|
||||
export function isOneExpr(expression: MathExpression): boolean {
|
||||
return expression.type === "number" && numberIsOne(expression.value);
|
||||
}
|
||||
|
||||
export function numberValue(expression: MathExpression): MathNumber | null {
|
||||
return expression.type === "number" ? expression.value : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every free variable in the tree, in first-appearance order. Order matters:
|
||||
* it decides which variable "solve for x" defaults to and which axis a
|
||||
* single-variable graph uses.
|
||||
*/
|
||||
export function variablesOf(expression: MathExpression): string[] {
|
||||
const seen = new Set<string>();
|
||||
const ordered: string[] = [];
|
||||
collectVariables(expression, ordered, seen);
|
||||
return ordered;
|
||||
}
|
||||
|
||||
function collectVariables(expression: MathExpression, ordered: string[], seen: Set<string>): void {
|
||||
switch (expression.type) {
|
||||
case "variable":
|
||||
if (!seen.has(expression.name)) {
|
||||
seen.add(expression.name);
|
||||
ordered.push(expression.name);
|
||||
}
|
||||
break;
|
||||
case "number":
|
||||
case "constant":
|
||||
break;
|
||||
case "add":
|
||||
for (const term of expression.terms) collectVariables(term, ordered, seen);
|
||||
break;
|
||||
case "multiply":
|
||||
for (const factor of expression.factors) collectVariables(factor, ordered, seen);
|
||||
break;
|
||||
case "power":
|
||||
collectVariables(expression.base, ordered, seen);
|
||||
collectVariables(expression.exponent, ordered, seen);
|
||||
break;
|
||||
case "negate":
|
||||
collectVariables(expression.inner, ordered, seen);
|
||||
break;
|
||||
case "function":
|
||||
for (const argument of expression.args) collectVariables(argument, ordered, seen);
|
||||
break;
|
||||
case "relation":
|
||||
collectVariables(expression.lhs, ordered, seen);
|
||||
collectVariables(expression.rhs, ordered, seen);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
export function isRelation(expression: MathExpression): boolean {
|
||||
return expression.type === "relation";
|
||||
}
|
||||
|
||||
/**
|
||||
* Node count — used to pick the simpler of two equivalent forms and to bound
|
||||
* runaway expansion in the simplifier.
|
||||
*/
|
||||
export function complexity(expression: MathExpression): number {
|
||||
switch (expression.type) {
|
||||
case "number":
|
||||
case "variable":
|
||||
case "constant":
|
||||
return 1;
|
||||
case "add":
|
||||
return 1 + expression.terms.reduce((total, term) => total + complexity(term), 0);
|
||||
case "multiply":
|
||||
return 1 + expression.factors.reduce((total, factor) => total + complexity(factor), 0);
|
||||
case "power":
|
||||
return 1 + complexity(expression.base) + complexity(expression.exponent);
|
||||
case "negate":
|
||||
return 1 + complexity(expression.inner);
|
||||
case "function":
|
||||
return 1 + expression.args.reduce((total, argument) => total + complexity(argument), 0);
|
||||
case "relation":
|
||||
return 1 + complexity(expression.lhs) + complexity(expression.rhs);
|
||||
}
|
||||
}
|
||||
|
||||
/** Replace every occurrence of a variable with an expression. */
|
||||
export function substituting(
|
||||
expression: MathExpression,
|
||||
name: string,
|
||||
replacement: MathExpression,
|
||||
): MathExpression {
|
||||
switch (expression.type) {
|
||||
case "variable":
|
||||
return expression.name === name ? replacement : expression;
|
||||
case "number":
|
||||
case "constant":
|
||||
return expression;
|
||||
case "add":
|
||||
return addExpr(expression.terms.map((term) => substituting(term, name, replacement)));
|
||||
case "multiply":
|
||||
return multiplyExpr(expression.factors.map((factor) => substituting(factor, name, replacement)));
|
||||
case "power":
|
||||
return powerExpr(
|
||||
substituting(expression.base, name, replacement),
|
||||
substituting(expression.exponent, name, replacement),
|
||||
);
|
||||
case "negate":
|
||||
return negate(substituting(expression.inner, name, replacement));
|
||||
case "function":
|
||||
return fn(
|
||||
expression.fn,
|
||||
expression.args.map((argument) => substituting(argument, name, replacement)),
|
||||
);
|
||||
case "relation":
|
||||
return relation(
|
||||
expression.relation,
|
||||
substituting(expression.lhs, name, replacement),
|
||||
substituting(expression.rhs, name, replacement),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural equality with value-equal numbers (MathNumber compares by
|
||||
* value).
|
||||
*/
|
||||
export function expressionEquals(a: MathExpression, b: MathExpression): boolean {
|
||||
if (a === b) return true;
|
||||
if (a.type !== b.type) return false;
|
||||
switch (a.type) {
|
||||
case "number": {
|
||||
const other = b as Extract<MathExpression, { type: "number" }>;
|
||||
return numbersEqual(a.value, other.value);
|
||||
}
|
||||
case "variable":
|
||||
return a.name === (b as Extract<MathExpression, { type: "variable" }>).name;
|
||||
case "constant":
|
||||
return a.constant === (b as Extract<MathExpression, { type: "constant" }>).constant;
|
||||
case "add": {
|
||||
const other = b as Extract<MathExpression, { type: "add" }>;
|
||||
return listEquals(a.terms, other.terms);
|
||||
}
|
||||
case "multiply": {
|
||||
const other = b as Extract<MathExpression, { type: "multiply" }>;
|
||||
return listEquals(a.factors, other.factors);
|
||||
}
|
||||
case "power": {
|
||||
const other = b as Extract<MathExpression, { type: "power" }>;
|
||||
return expressionEquals(a.base, other.base) && expressionEquals(a.exponent, other.exponent);
|
||||
}
|
||||
case "negate":
|
||||
return expressionEquals(a.inner, (b as Extract<MathExpression, { type: "negate" }>).inner);
|
||||
case "function": {
|
||||
const other = b as Extract<MathExpression, { type: "function" }>;
|
||||
return a.fn === other.fn && listEquals(a.args, other.args);
|
||||
}
|
||||
case "relation": {
|
||||
const other = b as Extract<MathExpression, { type: "relation" }>;
|
||||
return (
|
||||
a.relation === other.relation &&
|
||||
expressionEquals(a.lhs, other.lhs) &&
|
||||
expressionEquals(a.rhs, other.rhs)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function listEquals(a: MathExpression[], b: MathExpression[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
for (let index = 0; index < a.length; index++) {
|
||||
if (!expressionEquals(a[index]!, b[index]!)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Relations
|
||||
|
||||
/**
|
||||
* The relation you get by multiplying or dividing both sides by a negative —
|
||||
* the classic inequality mistake, handled explicitly.
|
||||
*/
|
||||
export function flippedRelation(rel: MathRelation): MathRelation {
|
||||
switch (rel) {
|
||||
case "=":
|
||||
return "=";
|
||||
case "<":
|
||||
return ">";
|
||||
case "<=":
|
||||
return ">=";
|
||||
case ">":
|
||||
return "<";
|
||||
case ">=":
|
||||
return "<=";
|
||||
}
|
||||
}
|
||||
|
||||
export function spokenRelation(rel: MathRelation): string {
|
||||
switch (rel) {
|
||||
case "=":
|
||||
return "equals";
|
||||
case "<":
|
||||
return "is less than";
|
||||
case "<=":
|
||||
return "is at most";
|
||||
case ">":
|
||||
return "is greater than";
|
||||
case ">=":
|
||||
return "is at least";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
|
||||
export function constantValue(value: MathConstant): number {
|
||||
switch (value) {
|
||||
case "pi":
|
||||
return Math.PI;
|
||||
case "e":
|
||||
return Math.E;
|
||||
case "tau":
|
||||
return 2 * Math.PI;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Functions
|
||||
|
||||
export function evaluateFunction(name: MathFunction, argument: number): number {
|
||||
switch (name) {
|
||||
case "sin":
|
||||
return Math.sin(argument);
|
||||
case "cos":
|
||||
return Math.cos(argument);
|
||||
case "tan":
|
||||
return Math.tan(argument);
|
||||
case "asin":
|
||||
return Math.asin(argument);
|
||||
case "acos":
|
||||
return Math.acos(argument);
|
||||
case "atan":
|
||||
return Math.atan(argument);
|
||||
case "sinh":
|
||||
return Math.sinh(argument);
|
||||
case "cosh":
|
||||
return Math.cosh(argument);
|
||||
case "tanh":
|
||||
return Math.tanh(argument);
|
||||
case "ln":
|
||||
return Math.log(argument);
|
||||
case "log10":
|
||||
return Math.log10(argument);
|
||||
case "log2":
|
||||
return Math.log2(argument);
|
||||
case "sqrt":
|
||||
return Math.sqrt(argument);
|
||||
case "cbrt":
|
||||
return Math.cbrt(argument);
|
||||
case "abs":
|
||||
return Math.abs(argument);
|
||||
case "exp":
|
||||
return Math.exp(argument);
|
||||
case "floor":
|
||||
return Math.floor(argument);
|
||||
case "ceil":
|
||||
return Math.ceil(argument);
|
||||
case "round":
|
||||
return Math.round(argument);
|
||||
case "factorial":
|
||||
// Real factorial via the gamma function, so `2.5!` is defined rather
|
||||
// than an error — and so the evaluator has one code path for both.
|
||||
return gamma(argument + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Domain check used by the plotter to break a curve rather than draw a line
|
||||
* through a gap (`ln` left of zero, `tan` at its asymptotes).
|
||||
*/
|
||||
export function functionIsDefined(name: MathFunction, argument: number): boolean {
|
||||
switch (name) {
|
||||
case "ln":
|
||||
case "log10":
|
||||
case "log2":
|
||||
return argument > 0;
|
||||
case "sqrt":
|
||||
return argument >= 0;
|
||||
case "asin":
|
||||
case "acos":
|
||||
return argument >= -1 && argument <= 1;
|
||||
case "factorial":
|
||||
return argument > -1 || Math.round(argument) !== argument;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lanczos approximation of the gamma function (JavaScript has no tgamma).
|
||||
* Accurate to well past the display precision used anywhere in this app.
|
||||
*/
|
||||
export function gamma(x: number): number {
|
||||
if (Number.isNaN(x)) return NaN;
|
||||
if (x === Math.round(x)) {
|
||||
if (x <= 0) return NaN; // poles at non-positive integers
|
||||
if (x <= 171) {
|
||||
let result = 1;
|
||||
for (let k = 2; k < x; k++) result *= k;
|
||||
return result;
|
||||
}
|
||||
return Infinity;
|
||||
}
|
||||
if (x < 0.5) {
|
||||
// Reflection formula.
|
||||
return Math.PI / (Math.sin(Math.PI * x) * gamma(1 - x));
|
||||
}
|
||||
const g = 7;
|
||||
const coefficients = [
|
||||
0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,
|
||||
-176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,
|
||||
1.5056327351493116e-7,
|
||||
];
|
||||
const z = x - 1;
|
||||
let sum = coefficients[0]!;
|
||||
for (let index = 1; index < g + 2; index++) {
|
||||
sum += coefficients[index]! / (z + index);
|
||||
}
|
||||
const t = z + g + 0.5;
|
||||
return Math.sqrt(2 * Math.PI) * Math.pow(t, z + 0.5) * Math.exp(-t) * sum;
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// The parts of geometry the calculator's spoken-question surface reaches:
|
||||
// shapes, triangle-from-sides, and Pythagoras.
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Triangles
|
||||
|
||||
/** A triangle solved from whatever three parts were given. */
|
||||
export type Triangle = {
|
||||
/** Side lengths, opposite the same-named angles. */
|
||||
a: number;
|
||||
b: number;
|
||||
c: number;
|
||||
/** Angles in radians. */
|
||||
angleA: number;
|
||||
angleB: number;
|
||||
angleC: number;
|
||||
};
|
||||
|
||||
export function triangleArea(triangle: Triangle): number {
|
||||
// Heron's formula, in the numerically stable ordering: the naive form
|
||||
// loses most of its precision on a needle-thin triangle.
|
||||
const sides = [triangle.a, triangle.b, triangle.c].sort((p, q) => q - p);
|
||||
const [x, y, z] = [sides[0]!, sides[1]!, sides[2]!];
|
||||
const term = (x + (y + z)) * (z - (x - y)) * (z + (x - y)) * (x + (y - z));
|
||||
return term <= 0 ? 0 : Math.sqrt(term) / 4;
|
||||
}
|
||||
|
||||
export function trianglePerimeter(triangle: Triangle): number {
|
||||
return triangle.a + triangle.b + triangle.c;
|
||||
}
|
||||
|
||||
export function triangleKind(triangle: Triangle): string {
|
||||
const tolerance = 1e-9;
|
||||
const sides = [triangle.a, triangle.b, triangle.c].sort((p, q) => p - q);
|
||||
const descriptors: string[] = [];
|
||||
if (Math.abs(sides[0]! - sides[2]!) < tolerance) {
|
||||
descriptors.push("equilateral");
|
||||
} else if (Math.abs(sides[0]! - sides[1]!) < tolerance || Math.abs(sides[1]! - sides[2]!) < tolerance) {
|
||||
descriptors.push("isosceles");
|
||||
} else {
|
||||
descriptors.push("scalene");
|
||||
}
|
||||
const largest = Math.max(triangle.angleA, triangle.angleB, triangle.angleC);
|
||||
if (Math.abs(largest - Math.PI / 2) < 1e-6) {
|
||||
descriptors.push("right");
|
||||
} else if (largest > Math.PI / 2) {
|
||||
descriptors.push("obtuse");
|
||||
} else {
|
||||
descriptors.push("acute");
|
||||
}
|
||||
return descriptors.join(", ");
|
||||
}
|
||||
|
||||
export function triangleAnglesInDegrees(triangle: Triangle): [number, number, number] {
|
||||
return [
|
||||
(triangle.angleA * 180) / Math.PI,
|
||||
(triangle.angleB * 180) / Math.PI,
|
||||
(triangle.angleC * 180) / Math.PI,
|
||||
];
|
||||
}
|
||||
|
||||
export type TriangleResult =
|
||||
| { kind: "one"; triangle: Triangle }
|
||||
/**
|
||||
* SSA can genuinely produce two valid triangles. Returning one and calling
|
||||
* it solved is the classic wrong answer in this topic.
|
||||
*/
|
||||
| { kind: "two"; first: Triangle; second: Triangle }
|
||||
| { kind: "impossible"; reason: string };
|
||||
|
||||
/**
|
||||
* Guards acos/asin against a value a hair outside [-1, 1] from rounding,
|
||||
* which returns NaN and poisons everything downstream.
|
||||
*/
|
||||
function clampedToUnit(value: number): number {
|
||||
return Math.min(1, Math.max(-1, value));
|
||||
}
|
||||
|
||||
/** Three sides. */
|
||||
export function triangleFromSides(a: number, b: number, c: number): TriangleResult {
|
||||
if (a <= 0 || b <= 0 || c <= 0) return { kind: "impossible", reason: "Sides must be positive" };
|
||||
if (a + b <= c || b + c <= a || a + c <= b) {
|
||||
return { kind: "impossible", reason: "Those sides cannot close a triangle" };
|
||||
}
|
||||
const angleA = Math.acos(clampedToUnit((b * b + c * c - a * a) / (2 * b * c)));
|
||||
const angleB = Math.acos(clampedToUnit((a * a + c * c - b * b) / (2 * a * c)));
|
||||
return {
|
||||
kind: "one",
|
||||
triangle: { a, b, c, angleA, angleB, angleC: Math.PI - angleA - angleB },
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shapes
|
||||
|
||||
export type Shape =
|
||||
| { kind: "circle"; radius: number }
|
||||
| { kind: "rectangle"; width: number; height: number }
|
||||
| { kind: "square"; side: number }
|
||||
| { kind: "triangleBaseHeight"; base: number; height: number }
|
||||
| { kind: "trapezoid"; a: number; b: number; height: number }
|
||||
| { kind: "parallelogram"; base: number; height: number }
|
||||
| { kind: "regularPolygon"; sides: number; sideLength: number }
|
||||
| { kind: "ellipse"; a: number; b: number }
|
||||
// Solids
|
||||
| { kind: "sphere"; radius: number }
|
||||
| { kind: "cylinder"; radius: number; height: number }
|
||||
| { kind: "cone"; radius: number; height: number }
|
||||
| { kind: "cube"; side: number }
|
||||
| { kind: "rectangularPrism"; length: number; width: number; height: number }
|
||||
| { kind: "pyramid"; baseArea: number; height: number };
|
||||
|
||||
export function shapeName(shape: Shape): string {
|
||||
switch (shape.kind) {
|
||||
case "circle":
|
||||
return "circle";
|
||||
case "rectangle":
|
||||
return "rectangle";
|
||||
case "square":
|
||||
return "square";
|
||||
case "triangleBaseHeight":
|
||||
return "triangle";
|
||||
case "trapezoid":
|
||||
return "trapezoid";
|
||||
case "parallelogram":
|
||||
return "parallelogram";
|
||||
case "regularPolygon":
|
||||
return `regular ${shape.sides}-gon`;
|
||||
case "ellipse":
|
||||
return "ellipse";
|
||||
case "sphere":
|
||||
return "sphere";
|
||||
case "cylinder":
|
||||
return "cylinder";
|
||||
case "cone":
|
||||
return "cone";
|
||||
case "cube":
|
||||
return "cube";
|
||||
case "rectangularPrism":
|
||||
return "rectangular prism";
|
||||
case "pyramid":
|
||||
return "pyramid";
|
||||
}
|
||||
}
|
||||
|
||||
export function shapeIsSolid(shape: Shape): boolean {
|
||||
switch (shape.kind) {
|
||||
case "sphere":
|
||||
case "cylinder":
|
||||
case "cone":
|
||||
case "cube":
|
||||
case "rectangularPrism":
|
||||
case "pyramid":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Area for a plane figure; surface area for a solid. */
|
||||
export function shapeArea(shape: Shape): number | null {
|
||||
switch (shape.kind) {
|
||||
case "circle":
|
||||
return Math.PI * shape.radius * shape.radius;
|
||||
case "rectangle":
|
||||
return shape.width * shape.height;
|
||||
case "square":
|
||||
return shape.side * shape.side;
|
||||
case "triangleBaseHeight":
|
||||
return (shape.base * shape.height) / 2;
|
||||
case "trapezoid":
|
||||
return ((shape.a + shape.b) * shape.height) / 2;
|
||||
case "parallelogram":
|
||||
return shape.base * shape.height;
|
||||
case "regularPolygon":
|
||||
return shape.sides >= 3
|
||||
? (shape.sides * shape.sideLength * shape.sideLength) / (4 * Math.tan(Math.PI / shape.sides))
|
||||
: null;
|
||||
case "ellipse":
|
||||
return Math.PI * shape.a * shape.b;
|
||||
case "sphere":
|
||||
return 4 * Math.PI * shape.radius * shape.radius;
|
||||
case "cylinder":
|
||||
return 2 * Math.PI * shape.radius * (shape.radius + shape.height);
|
||||
case "cone":
|
||||
return (
|
||||
Math.PI *
|
||||
shape.radius *
|
||||
(shape.radius + Math.sqrt(shape.radius * shape.radius + shape.height * shape.height))
|
||||
);
|
||||
case "cube":
|
||||
return 6 * shape.side * shape.side;
|
||||
case "rectangularPrism":
|
||||
return (
|
||||
2 *
|
||||
(shape.length * shape.width + shape.length * shape.height + shape.width * shape.height)
|
||||
);
|
||||
case "pyramid":
|
||||
return null; // needs the base shape, not just its area
|
||||
}
|
||||
}
|
||||
|
||||
export function shapePerimeter(shape: Shape): number | null {
|
||||
switch (shape.kind) {
|
||||
case "circle":
|
||||
return 2 * Math.PI * shape.radius;
|
||||
case "rectangle":
|
||||
return 2 * (shape.width + shape.height);
|
||||
case "square":
|
||||
return 4 * shape.side;
|
||||
case "regularPolygon":
|
||||
return shape.sides >= 3 ? shape.sides * shape.sideLength : null;
|
||||
case "ellipse": {
|
||||
// Ramanujan's second approximation — accurate to ~1e-9 for any
|
||||
// realistic eccentricity, where the naive π(a+b) is not close.
|
||||
const h = ((shape.a - shape.b) * (shape.a - shape.b)) / ((shape.a + shape.b) * (shape.a + shape.b));
|
||||
return Math.PI * (shape.a + shape.b) * (1 + (3 * h) / (10 + Math.sqrt(4 - 3 * h)));
|
||||
}
|
||||
default:
|
||||
// Trapezoid/triangle/parallelogram need every side, and the constructor
|
||||
// only took the measurements the area formula uses. Solids have none.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function shapeVolume(shape: Shape): number | null {
|
||||
switch (shape.kind) {
|
||||
case "sphere":
|
||||
return (4 / 3) * Math.PI * shape.radius * shape.radius * shape.radius;
|
||||
case "cylinder":
|
||||
return Math.PI * shape.radius * shape.radius * shape.height;
|
||||
case "cone":
|
||||
return (Math.PI * shape.radius * shape.radius * shape.height) / 3;
|
||||
case "cube":
|
||||
return shape.side * shape.side * shape.side;
|
||||
case "rectangularPrism":
|
||||
return shape.length * shape.width * shape.height;
|
||||
case "pyramid":
|
||||
return (shape.baseArea * shape.height) / 3;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Everything known, for the readback. */
|
||||
export function shapeSummary(shape: Shape): [string, number][] {
|
||||
const parts: [string, number][] = [];
|
||||
const area = shapeArea(shape);
|
||||
if (area !== null) parts.push([shapeIsSolid(shape) ? "Surface area" : "Area", area]);
|
||||
const perimeter = shapePerimeter(shape);
|
||||
if (perimeter !== null) parts.push(["Perimeter", perimeter]);
|
||||
const volume = shapeVolume(shape);
|
||||
if (volume !== null) parts.push(["Volume", volume]);
|
||||
return parts;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Coordinate geometry
|
||||
|
||||
/** Pythagoras, either direction: give any two, get the third. */
|
||||
export function pythagoras(
|
||||
a: number | null,
|
||||
b: number | null,
|
||||
hypotenuse: number | null,
|
||||
): number | null {
|
||||
if (a !== null && b !== null && hypotenuse === null) return Math.sqrt(a * a + b * b);
|
||||
if (a !== null && b === null && hypotenuse !== null) {
|
||||
return hypotenuse > a ? Math.sqrt(hypotenuse * hypotenuse - a * a) : null;
|
||||
}
|
||||
if (a === null && b !== null && hypotenuse !== null) {
|
||||
return hypotenuse > b ? Math.sqrt(hypotenuse * hypotenuse - b * b) : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
// Tokeniser for typed and normalised spoken maths expressions.
|
||||
|
||||
import { type MathNumber, fromDecimal, integer, real } from "./math-number";
|
||||
import { type MathRelation } from "./expression";
|
||||
import { isLetter } from "../../../util/unicode-class";
|
||||
|
||||
export type MathToken =
|
||||
| { type: "number"; value: MathNumber }
|
||||
| { type: "identifier"; name: string }
|
||||
| { type: "plus" }
|
||||
| { type: "minus" }
|
||||
| { type: "star" }
|
||||
| { type: "slash" }
|
||||
| { type: "percent" }
|
||||
| { type: "caret" }
|
||||
| { type: "bang" }
|
||||
| { type: "leftParen" }
|
||||
| { type: "rightParen" }
|
||||
| { type: "pipe" }
|
||||
| { type: "comma" }
|
||||
| { type: "relation"; relation: MathRelation };
|
||||
|
||||
export type MathParseErrorKind =
|
||||
| { kind: "unexpectedCharacter"; character: string }
|
||||
| { kind: "unexpectedToken"; token: string }
|
||||
| { kind: "unexpectedEnd" }
|
||||
| { kind: "unbalancedParenthesis" }
|
||||
| { kind: "unknownFunction"; name: string }
|
||||
| { kind: "emptyInput" }
|
||||
| { kind: "tooComplex" };
|
||||
|
||||
/**
|
||||
* A parse failure with the user's own words attached, so an error can quote
|
||||
* back what was actually said rather than leaving the wearer guessing which
|
||||
* word broke.
|
||||
*/
|
||||
export class MathParseError extends Error {
|
||||
readonly errorKind: MathParseErrorKind;
|
||||
/** What the user actually said. */
|
||||
readonly context: string;
|
||||
|
||||
constructor(errorKind: MathParseErrorKind, context = "") {
|
||||
super(describeParseError(errorKind, context));
|
||||
this.name = "MathParseError";
|
||||
this.errorKind = errorKind;
|
||||
this.context = context;
|
||||
}
|
||||
}
|
||||
|
||||
/** Written for someone reading a lens mid-task, not for a log. */
|
||||
export function describeParseError(kind: MathParseErrorKind, context: string): string {
|
||||
const quoted = context ? `“${context}”` : "that expression";
|
||||
switch (kind.kind) {
|
||||
case "unexpectedCharacter":
|
||||
return `Could not read "${kind.character}" in ${quoted}`;
|
||||
case "unexpectedToken":
|
||||
return `Did not expect "${kind.token}" in ${quoted}`;
|
||||
case "unexpectedEnd":
|
||||
return `${quoted} is incomplete`;
|
||||
case "unbalancedParenthesis":
|
||||
return `Brackets do not balance in ${quoted}`;
|
||||
case "unknownFunction":
|
||||
return `"${kind.name}" is not a function this build knows`;
|
||||
case "emptyInput":
|
||||
return "Nothing to solve";
|
||||
case "tooComplex":
|
||||
return "That expression is too deeply nested";
|
||||
}
|
||||
}
|
||||
|
||||
function isDigit(character: string): boolean {
|
||||
return character >= "0" && character <= "9";
|
||||
}
|
||||
|
||||
export function tokenize(input: string): MathToken[] {
|
||||
const tokens: MathToken[] = [];
|
||||
const characters = Array.from(input);
|
||||
let index = 0;
|
||||
|
||||
while (index < characters.length) {
|
||||
const character = characters[index]!;
|
||||
|
||||
if (/\s/.test(character)) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isDigit(character) || (character === "." && index + 1 < characters.length && isDigit(characters[index + 1]!))) {
|
||||
let literal = "";
|
||||
let sawDot = false;
|
||||
while (index < characters.length) {
|
||||
const next = characters[index]!;
|
||||
if (isDigit(next)) {
|
||||
literal += next;
|
||||
} else if (next === "." && !sawDot) {
|
||||
sawDot = true;
|
||||
literal += next;
|
||||
} else if (next === "," && index + 1 < characters.length && isDigit(characters[index + 1]!)) {
|
||||
// Thousands separator inside a literal: 1,000
|
||||
index += 1;
|
||||
continue;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
// Decimals become exact rationals so later arithmetic stays exact:
|
||||
// 0.25 behaves as 1/4, not as a float.
|
||||
if (sawDot) {
|
||||
tokens.push({ type: "number", value: fromDecimal(Number(literal) || 0) });
|
||||
} else {
|
||||
const parsed = Number(literal);
|
||||
// Exact for every safe literal; anything larger degrades to a real
|
||||
// instead of losing digits silently.
|
||||
tokens.push({
|
||||
type: "number",
|
||||
value: Number.isSafeInteger(parsed) ? integer(parsed) : real(parsed),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isLetter(character) || character === "_") {
|
||||
let name = "";
|
||||
while (
|
||||
index < characters.length &&
|
||||
(isLetter(characters[index]!) || isDigit(characters[index]!) || characters[index] === "_")
|
||||
) {
|
||||
name += characters[index];
|
||||
index += 1;
|
||||
}
|
||||
tokens.push({ type: "identifier", name });
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (character) {
|
||||
case "+":
|
||||
tokens.push({ type: "plus" });
|
||||
break;
|
||||
case "-":
|
||||
case "−": // ASCII and Unicode minus
|
||||
tokens.push({ type: "minus" });
|
||||
break;
|
||||
case "*":
|
||||
case "×":
|
||||
case "⋅":
|
||||
tokens.push({ type: "star" });
|
||||
break;
|
||||
case "/":
|
||||
case "÷":
|
||||
tokens.push({ type: "slash" });
|
||||
break;
|
||||
case "%":
|
||||
tokens.push({ type: "percent" });
|
||||
break;
|
||||
case "^":
|
||||
tokens.push({ type: "caret" });
|
||||
break;
|
||||
case "!":
|
||||
tokens.push({ type: "bang" });
|
||||
break;
|
||||
case "(":
|
||||
case "[":
|
||||
case "{":
|
||||
tokens.push({ type: "leftParen" });
|
||||
break;
|
||||
case ")":
|
||||
case "]":
|
||||
case "}":
|
||||
tokens.push({ type: "rightParen" });
|
||||
break;
|
||||
case "|":
|
||||
tokens.push({ type: "pipe" });
|
||||
break;
|
||||
case ",":
|
||||
tokens.push({ type: "comma" });
|
||||
break;
|
||||
case "=":
|
||||
// "==" from a typed source is still equality.
|
||||
if (index + 1 < characters.length && characters[index + 1] === "=") index += 1;
|
||||
tokens.push({ type: "relation", relation: "=" });
|
||||
break;
|
||||
case "<":
|
||||
if (index + 1 < characters.length && characters[index + 1] === "=") {
|
||||
tokens.push({ type: "relation", relation: "<=" });
|
||||
index += 1;
|
||||
} else {
|
||||
tokens.push({ type: "relation", relation: "<" });
|
||||
}
|
||||
break;
|
||||
case ">":
|
||||
if (index + 1 < characters.length && characters[index + 1] === "=") {
|
||||
tokens.push({ type: "relation", relation: ">=" });
|
||||
index += 1;
|
||||
} else {
|
||||
tokens.push({ type: "relation", relation: ">" });
|
||||
}
|
||||
break;
|
||||
case "≤":
|
||||
tokens.push({ type: "relation", relation: "<=" });
|
||||
break;
|
||||
case "≥":
|
||||
tokens.push({ type: "relation", relation: ">=" });
|
||||
break;
|
||||
case "√":
|
||||
tokens.push({ type: "identifier", name: "sqrt" });
|
||||
break;
|
||||
case "π":
|
||||
tokens.push({ type: "identifier", name: "pi" });
|
||||
break;
|
||||
default:
|
||||
throw new MathParseError({ kind: "unexpectedCharacter", character }, input);
|
||||
}
|
||||
index += 1;
|
||||
}
|
||||
|
||||
if (tokens.length === 0) throw new MathParseError({ kind: "emptyInput" }, input);
|
||||
return tokens;
|
||||
}
|
||||