Merge branch 'calculator' of https://github.com/kalanihelekunihi/faceclaw into calculator

This commit is contained in:
James Babcock committed 2026-08-26 00:49:20 -07:00
commit f3ad55ca3d
126 files changed
+22253 -58

No files matched your search

+4
View File
@@ -15,6 +15,10 @@ dependencies {
// Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for
// the Wear OS watch remote; see FaceclawWearBridge and wear/.
implementation 'com.google.android.gms:play-services-wearable:19.0.0'
// On-device language identification and translation for the Microphones
// app's captions (translation models download on demand, per language).
implementation 'com.google.mlkit:language-id:17.0.6'
implementation 'com.google.mlkit:translate:17.0.3'
}
def faceclawSherpaOnnxVersion = "1.13.0"
@@ -0,0 +1,242 @@
package com.faceclaw.app;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.util.Log;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
* container, cutting speech recordings to roughly a tenth of their WAV size.
* Runs on its own thread; the source WAV is deleted on success when requested.
*/
public class FaceclawAudioTranscoder {
private static final String TAG = "FaceclawTranscode";
private static final long CODEC_TIMEOUT_US = 10_000;
public interface Listener {
void onDone(String outputPath, long outputBytes);
void onError(String message);
}
private final String wavPath;
private final String outputPath;
private final int bitrateBps;
private final boolean deleteSourceOnSuccess;
private final Listener listener;
public FaceclawAudioTranscoder(
String wavPath,
String outputPath,
int bitrateBps,
boolean deleteSourceOnSuccess,
Listener listener) {
this.wavPath = wavPath;
this.outputPath = outputPath;
this.bitrateBps = bitrateBps > 0 ? bitrateBps : 32_000;
this.deleteSourceOnSuccess = deleteSourceOnSuccess;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawTranscode");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
private void run() {
try {
transcode();
long size = new File(outputPath).length();
if (deleteSourceOnSuccess) {
new File(wavPath).delete();
}
if (listener != null) {
listener.onDone(outputPath, size);
}
} catch (Throwable t) {
Log.w(TAG, "transcode failed for " + wavPath, t);
new File(outputPath).delete();
if (listener != null) {
listener.onError(t.getMessage() == null ? t.toString() : t.getMessage());
}
}
}
private void transcode() throws IOException {
WavInfo wav = readWavHeader(wavPath);
MediaFormat format = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC, wav.sampleRate, wav.channels);
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrateBps);
format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024);
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
MediaMuxer muxer = null;
FileInputStream input = null;
try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
codec.start();
muxer = new MediaMuxer(outputPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
input = new FileInputStream(wavPath);
long skipped = 0;
while (skipped < wav.dataOffset) {
long step = input.skip(wav.dataOffset - skipped);
if (step <= 0) {
throw new IOException("could not seek past WAV header");
}
skipped += step;
}
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
byte[] chunk = new byte[8192];
int track = -1;
boolean muxerStarted = false;
boolean inputDone = false;
boolean outputDone = false;
long remaining = wav.dataBytes;
long presentationUs = 0;
long bytesPerSecond = (long) wav.sampleRate * wav.channels * 2;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(CODEC_TIMEOUT_US);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int wanted = (int) Math.min(chunk.length, Math.min(remaining, inBuf.capacity()));
int read = wanted > 0 ? input.read(chunk, 0, wanted) : -1;
if (read <= 0) {
codec.queueInputBuffer(inIndex, 0, 0, presentationUs,
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
inBuf.clear();
inBuf.put(chunk, 0, read);
codec.queueInputBuffer(inIndex, 0, read, presentationUs, 0);
remaining -= read;
presentationUs += read * 1_000_000L / bytesPerSecond;
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, CODEC_TIMEOUT_US);
if (outIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
track = muxer.addTrack(codec.getOutputFormat());
muxer.start();
muxerStarted = true;
} else if (outIndex >= 0) {
if (info.size > 0 && (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) == 0) {
if (!muxerStarted) {
throw new IOException("encoder produced data before format");
}
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
outBuf.position(info.offset);
outBuf.limit(info.offset + info.size);
muxer.writeSampleData(track, outBuf, info);
}
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
if (input != null) {
try {
input.close();
} catch (IOException ignored) {
}
}
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
if (muxer != null) {
try {
muxer.stop();
} catch (Throwable ignored) {
}
muxer.release();
}
}
}
private static final class WavInfo {
int sampleRate;
int channels;
long dataOffset;
long dataBytes;
}
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
private static WavInfo readWavHeader(String path) throws IOException {
try (FileInputStream in = new FileInputStream(path)) {
byte[] head = new byte[12];
if (in.read(head) != 12 || head[0] != 'R' || head[1] != 'I' || head[8] != 'W') {
throw new IOException("not a WAV file");
}
WavInfo info = new WavInfo();
long offset = 12;
byte[] chunkHead = new byte[8];
while (in.read(chunkHead) == 8) {
ByteBuffer b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN);
int id = b.getInt();
long size = b.getInt() & 0xffffffffL;
offset += 8;
if (id == 0x20746d66) { // "fmt "
byte[] fmt = new byte[(int) Math.min(size, 16)];
if (in.read(fmt) != fmt.length) {
throw new IOException("truncated fmt chunk");
}
ByteBuffer f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN);
int audioFormat = f.getShort();
info.channels = f.getShort();
info.sampleRate = f.getInt();
f.getInt();
f.getShort();
int bits = f.getShort();
if (audioFormat != 1 || bits != 16) {
throw new IOException("only 16-bit PCM WAV is supported");
}
long skip = size - fmt.length;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
} else if (id == 0x61746164) { // "data"
info.dataOffset = offset;
// A zero size means the header was never patched (crash
// mid-recording); recover the length from the file size.
info.dataBytes = size > 0 ? size : new File(path).length() - offset;
break;
} else {
long skip = size;
while (skip > 0) {
long step = in.skip(skip);
if (step <= 0) {
break;
}
skip -= step;
}
offset += size;
}
}
if (info.sampleRate == 0 || info.dataOffset == 0) {
throw new IOException("missing fmt or data chunk");
}
return info;
}
}
}
@@ -60,6 +60,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawCompassListener> compassListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private final java.util.List<FaceclawMicStatusListener> micStatusListeners =
new java.util.concurrent.CopyOnWriteArrayList<>();
private volatile String lastFirmwareCapabilities = "";
private volatile Thread workerThread;
private volatile boolean running;
private volatile boolean userDisconnectRequested;
@@ -535,6 +538,98 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
}
/** The CFW capability token string from the last firmware-info read ("" before one arrives). */
public String getFirmwareCapabilities() {
return lastFirmwareCapabilities;
}
public void addMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.add(listener);
}
}
public void removeMicStatusListener(FaceclawMicStatusListener listener) {
if (listener != null) {
micStatusListeners.remove(listener);
}
}
private void emitMicStatus(byte[] body, String address) {
if (micStatusListeners.isEmpty()) {
return;
}
String arm = address.equalsIgnoreCase(leftAddress) ? "L"
: address.equalsIgnoreCase(rightAddress) ? "R" : "?";
byte[] copy = java.util.Arrays.copyOf(body, body.length);
mainHandler.post(() -> {
for (FaceclawMicStatusListener micListener : micStatusListeners) {
try {
micListener.onMicStatus(copy, arm);
} catch (Throwable t) {
Log.w(TAG, "mic status listener failed", t);
}
}
});
}
/**
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
* the firmware answers with a field-104 status notify per temple, which
* arrives through addMicStatusListener.
*/
public void sendFaceclawMicControl(byte[] record, String label, boolean rightTemple, boolean leftTemple) {
if (record == null || record.length < 4) {
return;
}
synchronized (lock) {
if (!running || !sessionReady) {
logLine("skip mic control (" + label + "); session not ready");
return;
}
if (rightTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false));
}
if (leftTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, true));
}
logLine("queue mic control " + label);
}
interruptibleSleep.interrupt();
}
/**
* Forward render-characteristic audio packets to the listener WITHOUT
* sending the stock EvenHub audio-control enable. Used for the CFW
* mic_control streaming path, where capture is armed through settings
* field 103 and the temples emit 'SM' frames on the same characteristic
* that stock mono LC3 uses. Returns false when no session is up.
*/
public boolean startG2AudioForwarding(FaceclawAudioPacketListener listener) {
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
synchronized (lock) {
if (!running || !sessionReady || shutdownRequested) {
logLine("skip G2 audio forwarding; session not ready");
return false;
}
audioPacketListener = listener;
audioCaptureActive = true;
logLine("G2 audio forwarding enabled");
}
return true;
}
public void stopG2AudioForwarding() {
synchronized (lock) {
audioPacketListener = null;
audioCaptureActive = false;
logLine("G2 audio forwarding disabled");
}
}
public void addCompassListener(FaceclawCompassListener listener) {
if (listener != null) {
compassListeners.add(listener);
@@ -1230,6 +1325,16 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
0
);
}
if (!faceclawWakeNotification
&& frame.ok
&& frame.sid == BleProtocol.SID_UI_SETTING) {
// CFW mic status (field 104) rides both standalone pushes and
// settings read acks, from each temple on its own link.
byte[] micStatus = BleProtocol.parseFaceclawMicStatus(frame.pb);
if (micStatus != null) {
emitMicStatus(micStatus, address);
}
}
if (!faceclawWakeNotification
&& decodedWearState < 0
&& frame.ok
@@ -2502,6 +2607,7 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable {
}
BleProtocol.FirmwareInfo firmwareInfo = BleProtocol.parseSettingsFirmwareInfo(message.ackPayload);
if (firmwareInfo != null) {
lastFirmwareCapabilities = firmwareInfo.capabilities == null ? "" : firmwareInfo.capabilities;
cfwCleanupSupported = hasCapability(firmwareInfo.capabilities, "cleanup11");
textureCacheSupported = hasCapability(firmwareInfo.capabilities, "texcache12")
&& hasCapability(firmwareInfo.capabilities, "texstr14");
@@ -0,0 +1,446 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FeatureConfig;
import com.k2fsa.sherpa.onnx.OfflineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizer;
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig;
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult;
import com.k2fsa.sherpa.onnx.OfflineStream;
import java.io.File;
import java.util.ArrayDeque;
import java.util.Arrays;
/**
* Continuous captioning over the decoded mic PCM: segments speech into
* utterances with an adaptive energy gate, transcribes each utterance with
* the on-device Moonshine model, and attaches a speaker voice-print per
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
* and any beam-direction gating), so this engine has no BLE dependencies.
*
* Utterance boundaries are measured on the sample clock, not the wall clock:
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
* audio heard. The TS side converts startMs/endMs to wall-clock times using
* the engine start timestamp.
*/
public class FaceclawCaptionEngine {
private static final String TAG = "FaceclawCaptions";
private static final int SAMPLE_RATE = 16000;
private static final int FEATURE_DIM = 80;
private static final int MAX_QUEUE_PACKETS = 200;
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
private static final double ONSET_FACTOR = 3.0;
private static final double RELEASE_FACTOR = 1.8;
private static final double MIN_RMS = 220.0;
private static final double NOISE_EMA_ALPHA = 0.05;
private static final int PRE_ROLL_MS = 400;
private static final int DEFAULT_SILENCE_MS = 800;
private static final int MIN_UTTERANCE_MS = 350;
private static final int MAX_UTTERANCE_MS = 15000;
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
// segments cut at the quietest window (mirrors FaceclawVoiceController).
private static final int DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8;
private static final int CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2;
private static final int CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000;
private static final float NORMALIZE_TARGET_PEAK = 0.9f;
private static final float NORMALIZE_MAX_GAIN = 30f;
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
private static final int EMBED_MAX_SAMPLES = SAMPLE_RATE * 10;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private final Object lock = new Object();
private final Object queueLock = new Object();
private final ArrayDeque<byte[]> queue = new ArrayDeque<>();
private volatile FaceclawCaptionEngineListener listener;
private volatile String asrModelDir;
private volatile String speakerModelPath;
private volatile int silenceMs = DEFAULT_SILENCE_MS;
private Thread workerThread;
private volatile boolean started;
private OfflineRecognizer recognizer;
private FaceclawSpeakerId speakerId;
// Segmentation state (worker thread only).
private long totalSamples;
private double noiseFloor;
private boolean inUtterance;
private long utteranceStartSample;
private long silenceRunSamples;
private double utterancePeakRms;
private short[] utterance = new short[SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000)];
private int utteranceLength;
private final short[] preRoll = new short[SAMPLE_RATE * PRE_ROLL_MS / 1000];
private int preRollLength;
public void setListener(FaceclawCaptionEngineListener listener) {
this.listener = listener;
}
/** Directory holding the Moonshine model files, or null to disable ASR. */
public void setAsrModelDir(String dir) {
this.asrModelDir = dir;
}
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
public void setSpeakerModelPath(String path) {
this.speakerModelPath = path;
}
public void setSilenceMs(int ms) {
this.silenceMs = Math.max(200, Math.min(3000, ms));
}
public void start() {
synchronized (lock) {
if (started) {
return;
}
started = true;
workerThread = new Thread(this::runLoop, "FaceclawCaptionEngine");
workerThread.start();
}
}
public void stop() {
Thread threadToJoin;
synchronized (lock) {
if (!started) {
return;
}
started = false;
threadToJoin = workerThread;
workerThread = null;
}
synchronized (queueLock) {
queueLock.notifyAll();
}
if (threadToJoin != null) {
threadToJoin.interrupt();
if (Thread.currentThread() != threadToJoin) {
try {
threadToJoin.join(2000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
public void acceptPcm(byte[] pcm16le) {
if (!started || pcm16le == null || pcm16le.length < 2) {
return;
}
synchronized (queueLock) {
if (queue.size() >= MAX_QUEUE_PACKETS) {
queue.removeFirst();
}
queue.addLast(pcm16le);
queueLock.notifyAll();
}
}
private void runLoop() {
try {
loadModels();
resetSegmentation();
while (started && !Thread.currentThread().isInterrupted()) {
byte[] chunk = takeChunk();
if (chunk == null) {
continue;
}
processChunk(chunk);
}
// Flush a trailing utterance so its text isn't lost on stop.
if (inUtterance && utteranceLength > 0) {
finalizeUtterance();
}
} catch (Throwable t) {
Log.e(TAG, "caption engine failed", t);
emitStatus("Captions failed: " + t.getMessage());
} finally {
releaseModels();
}
}
private void loadModels() {
String modelDir = asrModelDir;
if (modelDir != null && new File(modelDir, "tokens.txt").exists()) {
emitStatus("Loading caption model...");
recognizer = new OfflineRecognizer(OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath())
.setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath())
.build())
.setTokens(new File(modelDir, "tokens.txt").getAbsolutePath())
.setNumThreads(1)
.build())
.build());
} else {
recognizer = null;
}
String embedModel = speakerModelPath;
if (embedModel != null && new File(embedModel).exists()) {
speakerId = new FaceclawSpeakerId(embedModel);
speakerId.ensureLoaded();
} else {
speakerId = null;
}
emitStatus(recognizer != null ? "Captions listening..." : "Captions listening (no ASR model)...");
}
private void releaseModels() {
if (recognizer != null) {
recognizer.release();
recognizer = null;
}
if (speakerId != null) {
speakerId.close();
speakerId = null;
}
}
private byte[] takeChunk() {
synchronized (queueLock) {
while (started && queue.isEmpty()) {
try {
queueLock.wait(250);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
}
return queue.pollFirst();
}
}
private void resetSegmentation() {
totalSamples = 0;
noiseFloor = 0;
inUtterance = false;
utteranceLength = 0;
preRollLength = 0;
silenceRunSamples = 0;
utterancePeakRms = 0;
}
private void processChunk(byte[] chunk) {
int count = chunk.length / 2;
short[] pcm = new short[count];
double sumSquares = 0;
for (int i = 0; i < count; i++) {
short s = (short) ((chunk[i * 2] & 0xff) | (chunk[i * 2 + 1] << 8));
pcm[i] = s;
sumSquares += (double) s * s;
}
double rms = Math.sqrt(sumSquares / Math.max(1, count));
totalSamples += count;
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = noiseFloor == 0 ? rms : noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA;
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true;
utteranceLength = 0;
silenceRunSamples = 0;
utterancePeakRms = rms;
appendUtterance(preRoll, preRollLength);
utteranceStartSample = Math.max(0, totalSamples - count - preRollLength);
appendUtterance(pcm, count);
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE);
} else {
appendPreRoll(pcm, count);
}
return;
}
appendUtterance(pcm, count);
utterancePeakRms = Math.max(utterancePeakRms, rms);
double threshold = Math.max(noiseFloor, MIN_RMS);
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count;
} else {
silenceRunSamples = 0;
}
long utteranceMs = (long) utteranceLength * 1000 / SAMPLE_RATE;
boolean silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs;
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance();
inUtterance = false;
preRollLength = 0;
}
}
private void appendPreRoll(short[] pcm, int count) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.length) {
System.arraycopy(pcm, count - preRoll.length, preRoll, 0, preRoll.length);
preRollLength = preRoll.length;
return;
}
int keep = Math.min(preRollLength, preRoll.length - count);
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep);
System.arraycopy(pcm, 0, preRoll, keep, count);
preRollLength = keep + count;
}
private void appendUtterance(short[] pcm, int count) {
int room = utterance.length - utteranceLength;
int copied = Math.min(room, count);
if (copied > 0) {
System.arraycopy(pcm, 0, utterance, utteranceLength, copied);
utteranceLength += copied;
}
}
private void finalizeUtterance() {
int length = utteranceLength;
long startMs = utteranceStartSample * 1000 / SAMPLE_RATE;
long endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE;
if (endMs - startMs < MIN_UTTERANCE_MS) {
return;
}
String text = recognizeUtterance(length);
float[] embedding = embedUtterance(length);
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms);
}
private String recognizeUtterance(int length) {
if (recognizer == null || length <= 0) {
return "";
}
StringBuilder joined = new StringBuilder();
int offset = 0;
while (offset < length) {
int remaining = length - offset;
int segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES);
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = findQuietCut(offset, segment);
}
String part = recognizeRange(offset, segment);
if (part.length() > 0) {
if (joined.length() > 0 && ".,!?;:%)]}".indexOf(part.charAt(0)) < 0) {
joined.append(' ');
}
joined.append(part);
}
offset += segment;
}
return joined.toString().trim();
}
/**
* End a decode segment at the center of the quietest window near its end
* so the cut lands between words (same approach as the PTT controller).
*/
private int findQuietCut(int offset, int segment) {
int searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES);
int win = CUT_WINDOW_SAMPLES;
if (segment - searchStart <= win) {
return segment;
}
double sum = 0;
for (int i = searchStart; i < searchStart + win; i++) {
double s = utterance[offset + i];
sum += s * s;
}
double best = sum;
int bestStart = searchStart;
for (int start = searchStart + 1; start + win <= segment; start++) {
double dropped = utterance[offset + start - 1];
double added = utterance[offset + start + win - 1];
sum += added * added - dropped * dropped;
if (sum < best) {
best = sum;
bestStart = start;
}
}
return bestStart + win / 2;
}
private String recognizeRange(int offset, int count) {
float[] samples = new float[count];
float peak = 0f;
for (int i = 0; i < count; i++) {
float v = utterance[offset + i] / 32768.0f;
samples[i] = v;
float a = Math.abs(v);
if (a > peak) {
peak = a;
}
}
if (peak > 0f) {
float gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN);
if (gain > 1f) {
for (int i = 0; i < count; i++) {
samples[i] *= gain;
}
}
}
OfflineStream stream = recognizer.createStream();
try {
stream.acceptWaveform(samples, SAMPLE_RATE);
recognizer.decode(stream);
OfflineRecognizerResult result = recognizer.getResult(stream);
String raw = result == null ? "" : result.getText();
return raw == null ? "" : raw.trim();
} finally {
stream.release();
}
}
private float[] embedUtterance(int length) {
FaceclawSpeakerId currentSpeakerId = speakerId;
if (currentSpeakerId == null || length <= 0) {
return null;
}
int count = Math.min(length, EMBED_MAX_SAMPLES);
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
short s = utterance[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
return currentSpeakerId.embed(le, SAMPLE_RATE);
}
private void emitUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
float[] embeddingCopy = embedding == null ? null : Arrays.copyOf(embedding, embedding.length);
mainHandler.post(() -> currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms));
}
private void emitSpeechStart(long startMs) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeechStart(startMs));
}
private void emitStatus(String status) {
FaceclawCaptionEngineListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onStatus(status));
}
}
@@ -0,0 +1,17 @@
package com.faceclaw.app;
public interface FaceclawCaptionEngineListener {
/**
* One finished utterance. text may be empty when no ASR model is loaded
* or recognition produced nothing; embedding is the L2-normalized
* voice-print or null when the speaker model is unavailable or the
* utterance was too short. startMs/endMs are on the engine's sample
* clock (milliseconds of audio since start()).
*/
void onUtterance(String text, float[] embedding, long startMs, long endMs, double peakRms);
/** A speech onset was detected; the utterance is now being collected. */
void onSpeechStart(long startMs);
void onStatus(String status);
}
@@ -0,0 +1,778 @@
package com.faceclaw.app;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import android.util.Base64;
import android.util.Log;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
/**
* Persistent store for caption sessions, transcript segments, and speaker
* voice profiles, backing the Microphones app and its phone-side review UI.
* All complex values cross the JS bridge as JSON strings; voice-print
* embeddings are float32 LE blobs, exposed as base64.
*/
public class FaceclawConversationStore extends SQLiteOpenHelper {
private static final String TAG = "FaceclawConvStore";
private static final String DB_NAME = "faceclaw-conversations.db";
private static final int DB_VERSION = 2;
private static FaceclawConversationStore instance;
public static synchronized FaceclawConversationStore getInstance(Context context) {
if (instance == null) {
instance = new FaceclawConversationStore(context.getApplicationContext());
}
return instance;
}
private FaceclawConversationStore(Context context) {
super(context, DB_NAME, null, DB_VERSION);
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL("CREATE TABLE speakers ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "name TEXT NOT NULL,"
+ "color TEXT NOT NULL DEFAULT '#4FC3F7',"
+ "is_wearer INTEGER NOT NULL DEFAULT 0,"
+ "embedding BLOB,"
+ "embedding_count INTEGER NOT NULL DEFAULT 0,"
+ "created_at INTEGER NOT NULL,"
+ "last_heard_at INTEGER,"
+ "tag TEXT,"
+ "last_recap TEXT,"
+ "action_items TEXT,"
+ "facts TEXT,"
+ "insights_updated_at INTEGER,"
+ "insights_session_id INTEGER)");
db.execSQL("CREATE TABLE sessions ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER,"
+ "title TEXT,"
+ "audio_path TEXT,"
+ "audio_codec TEXT,"
+ "avg_sentiment REAL,"
+ "segment_count INTEGER NOT NULL DEFAULT 0)");
db.execSQL("CREATE TABLE segments ("
+ "id INTEGER PRIMARY KEY AUTOINCREMENT,"
+ "session_id INTEGER NOT NULL,"
+ "speaker_id INTEGER,"
+ "started_at INTEGER NOT NULL,"
+ "ended_at INTEGER NOT NULL,"
+ "audio_offset_ms INTEGER,"
+ "text TEXT NOT NULL,"
+ "lang TEXT,"
+ "translation TEXT,"
+ "translation_lang TEXT,"
+ "sentiment REAL,"
+ "emotion TEXT,"
+ "search_meta TEXT,"
+ "angle INTEGER,"
+ "embedding BLOB)");
db.execSQL("CREATE INDEX idx_segments_session ON segments(session_id)");
db.execSQL("CREATE INDEX idx_segments_speaker ON segments(speaker_id)");
db.execSQL("CREATE INDEX idx_segments_time ON segments(started_at)");
db.execSQL("CREATE INDEX idx_sessions_time ON sessions(started_at)");
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion < 2) {
// v2: per-speaker conversation insights (recap of the most recent
// conversation, its action items, and accumulated inferred facts).
db.execSQL("ALTER TABLE speakers ADD COLUMN last_recap TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN action_items TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN facts TEXT");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_updated_at INTEGER");
db.execSQL("ALTER TABLE speakers ADD COLUMN insights_session_id INTEGER");
}
}
// ---- sessions ----
public long startSession(long startedAtMs, String title) {
ContentValues values = new ContentValues();
values.put("started_at", startedAtMs);
values.put("title", title == null || title.isEmpty() ? null : title);
return getWritableDatabase().insert("sessions", null, values);
}
public void endSession(long sessionId, long endedAtMs, String audioPath, String audioCodec, double avgSentiment) {
ContentValues values = new ContentValues();
values.put("ended_at", endedAtMs);
if (audioPath != null && !audioPath.isEmpty()) {
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
}
values.put("avg_sentiment", avgSentiment);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public void setSessionAudio(long sessionId, String audioPath, String audioCodec) {
ContentValues values = new ContentValues();
values.put("audio_path", audioPath);
values.put("audio_codec", audioCodec);
getWritableDatabase().update("sessions", values, "id=?", new String[]{String.valueOf(sessionId)});
}
public String getSessionAudioPath(long sessionId) {
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT audio_path FROM sessions WHERE id=?", new String[]{String.valueOf(sessionId)})) {
if (c.moveToFirst()) {
return c.isNull(0) ? "" : c.getString(0);
}
}
return "";
}
/**
* Sessions, newest first, as JSON. filterJson supports: sinceMs, untilMs,
* speakerId, emotion (matches any segment emotion in the session), query
* (LIKE over title and segment text/search metadata), limit.
*/
public String querySessions(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT DISTINCT s.id, s.started_at, s.ended_at, s.title, s.audio_path, s.audio_codec,"
+ " s.avg_sentiment, s.segment_count FROM sessions s");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
boolean joinSegments = filter.has("speakerId") || filter.has("emotion") || filter.has("query");
if (joinSegments) {
sql.append(" LEFT JOIN segments g ON g.session_id = s.id");
}
if (filter.has("sinceMs")) {
where.add("s.started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("s.started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (filter.has("speakerId")) {
where.add("g.speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("emotion")) {
where.add("g.emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("query")) {
String like = "%" + filter.getString("query") + "%";
where.add("(s.title LIKE ? OR g.text LIKE ? OR g.search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY s.started_at DESC");
int limit = filter.optInt("limit", 200);
sql.append(" LIMIT ").append(Math.max(1, Math.min(limit, 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
long id = c.getLong(0);
row.put("id", id);
row.put("startedAt", c.getLong(1));
row.put("endedAt", c.isNull(2) ? JSONObject.NULL : c.getLong(2));
row.put("title", c.isNull(3) ? "" : c.getString(3));
row.put("audioPath", c.isNull(4) ? "" : c.getString(4));
row.put("audioCodec", c.isNull(5) ? "" : c.getString(5));
row.put("avgSentiment", c.isNull(6) ? 0 : c.getDouble(6));
row.put("segmentCount", c.getLong(7));
row.put("speakers", sessionSpeakersJson(id));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "querySessions failed", e);
return "[]";
}
}
private JSONArray sessionSpeakersJson(long sessionId) throws JSONException {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT DISTINCT p.id, p.name, p.color FROM segments g"
+ " JOIN speakers p ON p.id = g.speaker_id WHERE g.session_id=?",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
out.put(row);
}
}
return out;
}
// ---- segments ----
/**
* Insert one transcript segment. json fields: sessionId, speakerId?,
* startedAt, endedAt, audioOffsetMs?, text, lang?, translation?,
* translationLang?, sentiment?, emotion?, searchMeta?, angle?,
* embeddingBase64?. Returns the new row id, or -1.
*/
public long insertSegment(String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
long sessionId = s.getLong("sessionId");
values.put("session_id", sessionId);
if (s.has("speakerId")) {
values.put("speaker_id", s.getLong("speakerId"));
}
values.put("started_at", s.getLong("startedAt"));
values.put("ended_at", s.getLong("endedAt"));
if (s.has("audioOffsetMs")) {
values.put("audio_offset_ms", s.getLong("audioOffsetMs"));
}
values.put("text", s.getString("text"));
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (s.has("angle")) {
values.put("angle", s.getInt("angle"));
}
if (s.has("embeddingBase64")) {
values.put("embedding", Base64.decode(s.getString("embeddingBase64"), Base64.NO_WRAP));
}
SQLiteDatabase db = getWritableDatabase();
long id = db.insert("segments", null, values);
db.execSQL("UPDATE sessions SET segment_count = segment_count + 1 WHERE id=?",
new Object[]{sessionId});
return id;
} catch (Throwable t) {
Log.w(TAG, "insertSegment failed", t);
return -1;
}
}
private static void putOptString(ContentValues values, JSONObject s, String jsonKey, String column)
throws JSONException {
if (s.has(jsonKey) && !s.isNull(jsonKey)) {
values.put(column, s.getString(jsonKey));
}
}
/** Update mutable fields of a segment (same keys as insertSegment). */
public void updateSegment(long segmentId, String json) {
try {
JSONObject s = new JSONObject(json);
ContentValues values = new ContentValues();
if (s.has("speakerId")) {
if (s.isNull("speakerId")) {
values.putNull("speaker_id");
} else {
values.put("speaker_id", s.getLong("speakerId"));
}
}
if (s.has("text")) {
values.put("text", s.getString("text"));
}
putOptString(values, s, "lang", "lang");
putOptString(values, s, "translation", "translation");
putOptString(values, s, "translationLang", "translation_lang");
if (s.has("sentiment")) {
values.put("sentiment", s.getDouble("sentiment"));
}
putOptString(values, s, "emotion", "emotion");
putOptString(values, s, "searchMeta", "search_meta");
if (values.size() > 0) {
getWritableDatabase().update("segments", values, "id=?",
new String[]{String.valueOf(segmentId)});
}
} catch (Throwable t) {
Log.w(TAG, "updateSegment failed", t);
}
}
public String querySegments(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle"
+ " FROM segments WHERE session_id=? ORDER BY started_at ASC",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
out.put(segmentRowJson(c));
}
} catch (JSONException e) {
Log.w(TAG, "querySegments failed", e);
}
return out.toString();
}
private JSONObject segmentRowJson(Cursor c) throws JSONException {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("speakerId", c.isNull(1) ? JSONObject.NULL : c.getLong(1));
row.put("startedAt", c.getLong(2));
row.put("endedAt", c.getLong(3));
row.put("audioOffsetMs", c.isNull(4) ? JSONObject.NULL : c.getLong(4));
row.put("text", c.getString(5));
row.put("lang", c.isNull(6) ? "" : c.getString(6));
row.put("translation", c.isNull(7) ? "" : c.getString(7));
row.put("translationLang", c.isNull(8) ? "" : c.getString(8));
row.put("sentiment", c.isNull(9) ? 0 : c.getDouble(9));
row.put("emotion", c.isNull(10) ? "" : c.getString(10));
row.put("angle", c.isNull(11) ? JSONObject.NULL : c.getInt(11));
return row;
}
/**
* Search segments across sessions. filterJson: query?, emotion?,
* speakerId?, sinceMs?, untilMs?, limit?. Matches text, translation, and
* the sentiment search metadata.
*/
public String searchSegments(String filterJson) {
try {
JSONObject filter = filterJson == null || filterJson.isEmpty()
? new JSONObject() : new JSONObject(filterJson);
StringBuilder sql = new StringBuilder(
"SELECT id, speaker_id, started_at, ended_at, audio_offset_ms, text, lang,"
+ " translation, translation_lang, sentiment, emotion, angle, session_id"
+ " FROM segments");
List<String> args = new ArrayList<>();
List<String> where = new ArrayList<>();
if (filter.has("query") && !filter.getString("query").isEmpty()) {
String like = "%" + filter.getString("query") + "%";
where.add("(text LIKE ? OR translation LIKE ? OR search_meta LIKE ?)");
args.add(like);
args.add(like);
args.add(like);
}
if (filter.has("emotion")) {
where.add("emotion = ?");
args.add(filter.getString("emotion"));
}
if (filter.has("speakerId")) {
where.add("speaker_id = ?");
args.add(String.valueOf(filter.getLong("speakerId")));
}
if (filter.has("sinceMs")) {
where.add("started_at >= ?");
args.add(String.valueOf(filter.getLong("sinceMs")));
}
if (filter.has("untilMs")) {
where.add("started_at <= ?");
args.add(String.valueOf(filter.getLong("untilMs")));
}
if (!where.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", where));
}
sql.append(" ORDER BY started_at DESC LIMIT ")
.append(Math.max(1, Math.min(filter.optInt("limit", 300), 1000)));
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(sql.toString(), args.toArray(new String[0]))) {
while (c.moveToNext()) {
JSONObject row = segmentRowJson(c);
row.put("sessionId", c.getLong(12));
out.put(row);
}
}
return out.toString();
} catch (JSONException e) {
Log.w(TAG, "searchSegments failed", e);
return "[]";
}
}
/** Per-segment voice-print embeddings of a session (for re-diarization). */
public String querySegmentEmbeddings(long sessionId) {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT id, embedding FROM segments WHERE session_id=? AND embedding IS NOT NULL",
new String[]{String.valueOf(sessionId)})) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("embeddingBase64", Base64.encodeToString(c.getBlob(1), Base64.NO_WRAP));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySegmentEmbeddings failed", e);
}
return out.toString();
}
// ---- speakers ----
public long createSpeaker(String name, String color, boolean isWearer, String embeddingBase64) {
ContentValues values = new ContentValues();
values.put("name", name);
values.put("color", color == null || color.isEmpty() ? "#4FC3F7" : color);
values.put("is_wearer", isWearer ? 1 : 0);
values.put("created_at", System.currentTimeMillis());
values.put("last_heard_at", System.currentTimeMillis());
if (embeddingBase64 != null && !embeddingBase64.isEmpty()) {
values.put("embedding", Base64.decode(embeddingBase64, Base64.NO_WRAP));
values.put("embedding_count", 1);
}
return getWritableDatabase().insert("speakers", null, values);
}
public String querySpeakers() {
JSONArray out = new JSONArray();
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT p.id, p.name, p.color, p.is_wearer, p.embedding, p.embedding_count,"
+ " p.last_heard_at, p.tag,"
+ " p.last_recap, p.action_items, p.facts, p.insights_updated_at, p.insights_session_id,"
+ " (SELECT COUNT(*) FROM segments g WHERE g.speaker_id = p.id) AS segments"
+ " FROM speakers p ORDER BY p.last_heard_at DESC", null)) {
while (c.moveToNext()) {
JSONObject row = new JSONObject();
row.put("id", c.getLong(0));
row.put("name", c.getString(1));
row.put("color", c.getString(2));
row.put("isWearer", c.getInt(3) != 0);
byte[] blob = c.isNull(4) ? null : c.getBlob(4);
row.put("embeddingBase64", blob == null ? "" : Base64.encodeToString(blob, Base64.NO_WRAP));
row.put("embeddingCount", c.getInt(5));
row.put("lastHeardAt", c.isNull(6) ? 0 : c.getLong(6));
row.put("tag", c.isNull(7) ? "" : c.getString(7));
row.put("lastRecap", c.isNull(8) ? "" : c.getString(8));
row.put("actionItemsJson", c.isNull(9) ? "" : c.getString(9));
row.put("factsJson", c.isNull(10) ? "" : c.getString(10));
row.put("insightsUpdatedAt", c.isNull(11) ? 0 : c.getLong(11));
row.put("insightsSessionId", c.isNull(12) ? 0 : c.getLong(12));
row.put("segmentCount", c.getLong(13));
out.put(row);
}
} catch (JSONException e) {
Log.w(TAG, "querySpeakers failed", e);
}
return out.toString();
}
public void renameSpeaker(long speakerId, String name) {
ContentValues values = new ContentValues();
values.put("name", name);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerColor(long speakerId, String color) {
ContentValues values = new ContentValues();
values.put("color", color);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerTag(long speakerId, String tag) {
ContentValues values = new ContentValues();
values.put("tag", tag);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Store LLM-derived conversation insights for a speaker: a recap of their
* most recent conversation, its open action items (JSON string array), and
* the accumulated inferred facts about them (JSON string array).
*/
public void setSpeakerInsights(long speakerId, String recap, String actionItemsJson,
String factsJson, long sessionId) {
ContentValues values = new ContentValues();
values.put("last_recap", recap == null ? "" : recap);
values.put("action_items", actionItemsJson == null ? "" : actionItemsJson);
values.put("facts", factsJson == null ? "" : factsJson);
values.put("insights_updated_at", System.currentTimeMillis());
values.put("insights_session_id", sessionId);
getWritableDatabase().update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
public void setSpeakerWearer(long speakerId, boolean isWearer) {
SQLiteDatabase db = getWritableDatabase();
if (isWearer) {
// Exactly one wearer profile at a time.
db.execSQL("UPDATE speakers SET is_wearer = 0");
}
ContentValues values = new ContentValues();
values.put("is_wearer", isWearer ? 1 : 0);
db.update("speakers", values, "id=?", new String[]{String.valueOf(speakerId)});
}
/**
* Running-mean centroid update for a speaker's voice-print, capped so a
* long session cannot drown the enrolled voice.
*/
public void updateSpeakerEmbedding(long speakerId, String embeddingBase64, int maxCount) {
try {
byte[] incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP);
float[] update = blobToFloats(incoming);
if (update == null) {
return;
}
float[] current = null;
int count = 0;
try (Cursor c = getReadableDatabase().rawQuery(
"SELECT embedding, embedding_count FROM speakers WHERE id=?",
new String[]{String.valueOf(speakerId)})) {
if (c.moveToFirst()) {
current = c.isNull(0) ? null : blobToFloats(c.getBlob(0));
count = c.getInt(1);
}
}
float[] merged;
if (current == null || current.length != update.length) {
merged = update;
count = 1;
} else {
int effective = Math.min(count, Math.max(1, maxCount));
merged = new float[current.length];
for (int i = 0; i < merged.length; i++) {
merged[i] = (current[i] * effective + update[i]) / (effective + 1);
}
l2NormalizeInPlace(merged);
count = count + 1;
}
ContentValues values = new ContentValues();
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", count);
values.put("last_heard_at", System.currentTimeMillis());
getWritableDatabase().update("speakers", values, "id=?",
new String[]{String.valueOf(speakerId)});
} catch (Throwable t) {
Log.w(TAG, "updateSpeakerEmbedding failed", t);
}
}
/** Move every segment from one speaker onto another and drop the source. */
public void mergeSpeakers(long fromId, long intoId) {
if (fromId == intoId) {
return;
}
SQLiteDatabase db = getWritableDatabase();
db.beginTransaction();
try {
db.execSQL("UPDATE segments SET speaker_id=? WHERE speaker_id=?",
new Object[]{intoId, fromId});
// Blend the two centroids weighted by their sample counts.
float[] from = null;
float[] into = null;
int fromCount = 0;
int intoCount = 0;
boolean fromWearer = false;
try (Cursor c = db.rawQuery(
"SELECT id, embedding, embedding_count, is_wearer FROM speakers WHERE id IN (?,?)",
new String[]{String.valueOf(fromId), String.valueOf(intoId)})) {
while (c.moveToNext()) {
long id = c.getLong(0);
float[] embedding = c.isNull(1) ? null : blobToFloats(c.getBlob(1));
int count = c.getInt(2);
if (id == fromId) {
from = embedding;
fromCount = count;
fromWearer = c.getInt(3) != 0;
} else {
into = embedding;
intoCount = count;
}
}
}
float[] merged = into;
if (from != null && into != null && from.length == into.length) {
merged = new float[into.length];
int total = Math.max(1, fromCount + intoCount);
for (int i = 0; i < merged.length; i++) {
merged[i] = (into[i] * intoCount + from[i] * fromCount) / total;
}
l2NormalizeInPlace(merged);
} else if (into == null) {
merged = from;
}
ContentValues values = new ContentValues();
if (merged != null) {
values.put("embedding", floatsToBlob(merged));
values.put("embedding_count", fromCount + intoCount);
}
if (fromWearer) {
values.put("is_wearer", 1);
}
values.put("last_heard_at", System.currentTimeMillis());
// Keep the target's conversation insights; adopt the source's only
// where the target has none (e.g. merging a rich profile into a
// freshly auto-created one).
try (Cursor c = db.rawQuery(
"SELECT last_recap, action_items, facts, insights_updated_at, insights_session_id"
+ " FROM speakers WHERE id IN (?,?) ORDER BY CASE id WHEN ? THEN 0 ELSE 1 END",
new String[]{String.valueOf(fromId), String.valueOf(intoId), String.valueOf(intoId)})) {
boolean targetHasInsights = false;
while (c.moveToNext()) {
boolean hasInsights = !c.isNull(0) && !c.getString(0).isEmpty();
if (c.getPosition() == 0) {
targetHasInsights = hasInsights;
} else if (!targetHasInsights && hasInsights) {
values.put("last_recap", c.getString(0));
values.put("action_items", c.isNull(1) ? "" : c.getString(1));
values.put("facts", c.isNull(2) ? "" : c.getString(2));
values.put("insights_updated_at", c.isNull(3) ? 0 : c.getLong(3));
values.put("insights_session_id", c.isNull(4) ? 0 : c.getLong(4));
}
}
}
db.update("speakers", values, "id=?", new String[]{String.valueOf(intoId)});
db.delete("speakers", "id=?", new String[]{String.valueOf(fromId)});
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
}
public void reassignSegmentSpeaker(long segmentId, long speakerId) {
ContentValues values = new ContentValues();
values.put("speaker_id", speakerId);
getWritableDatabase().update("segments", values, "id=?", new String[]{String.valueOf(segmentId)});
}
/** Reassign every segment of a speaker within one session (split support). */
public void reassignSessionSpeaker(long sessionId, long fromSpeakerId, long toSpeakerId) {
getWritableDatabase().execSQL(
"UPDATE segments SET speaker_id=? WHERE session_id=? AND speaker_id=?",
new Object[]{toSpeakerId, sessionId, fromSpeakerId});
}
public void deleteSpeaker(long speakerId) {
SQLiteDatabase db = getWritableDatabase();
db.execSQL("UPDATE segments SET speaker_id=NULL WHERE speaker_id=?", new Object[]{speakerId});
db.delete("speakers", "id=?", new String[]{String.valueOf(speakerId)});
}
// ---- retention ----
/**
* Delete caption sessions older than the cutoff (0 disables), and strip
* recordings older than their own cutoff while keeping the transcript.
* Returns a JSON summary of what was removed.
*/
public String applyRetention(long captionCutoffMs, long recordingCutoffMs) {
SQLiteDatabase db = getWritableDatabase();
int sessionsDeleted = 0;
int recordingsDeleted = 0;
if (recordingCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT id, audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(recordingCutoffMs)})) {
while (c.moveToNext()) {
if (!c.isNull(1)) {
paths.add(c.getString(1));
}
}
}
for (String path : paths) {
if (new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("UPDATE sessions SET audio_path=NULL, audio_codec=NULL WHERE started_at < ?",
new Object[]{recordingCutoffMs});
}
if (captionCutoffMs > 0) {
List<String> paths = new ArrayList<>();
try (Cursor c = db.rawQuery(
"SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL AND started_at < ?",
new String[]{String.valueOf(captionCutoffMs)})) {
while (c.moveToNext()) {
paths.add(c.getString(0));
}
}
for (String path : paths) {
if (path != null && new File(path).delete()) {
recordingsDeleted++;
}
}
db.execSQL("DELETE FROM segments WHERE session_id IN (SELECT id FROM sessions WHERE started_at < ?)",
new Object[]{captionCutoffMs});
Cursor count = db.rawQuery("SELECT changes()", null);
count.close();
db.execSQL("DELETE FROM sessions WHERE started_at < ?", new Object[]{captionCutoffMs});
try (Cursor c = db.rawQuery("SELECT changes()", null)) {
if (c.moveToFirst()) {
sessionsDeleted = c.getInt(0);
}
}
}
try {
JSONObject summary = new JSONObject();
summary.put("sessionsDeleted", sessionsDeleted);
summary.put("recordingsDeleted", recordingsDeleted);
return summary.toString();
} catch (JSONException e) {
return "{}";
}
}
public void deleteSession(long sessionId) {
SQLiteDatabase db = getWritableDatabase();
String path = getSessionAudioPath(sessionId);
if (!path.isEmpty()) {
new File(path).delete();
}
db.delete("segments", "session_id=?", new String[]{String.valueOf(sessionId)});
db.delete("sessions", "id=?", new String[]{String.valueOf(sessionId)});
}
// ---- blob helpers ----
private static float[] blobToFloats(byte[] blob) {
if (blob == null || blob.length < 4 || blob.length % 4 != 0) {
return null;
}
ByteBuffer buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
float[] out = new float[blob.length / 4];
for (int i = 0; i < out.length; i++) {
out[i] = buffer.getFloat();
}
return out;
}
private static byte[] floatsToBlob(float[] values) {
ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN);
for (float v : values) {
buffer.putFloat(v);
}
return buffer.array();
}
private static void l2NormalizeInPlace(float[] v) {
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
}
}
@@ -37,9 +37,6 @@ import java.util.Set;
* matching are decided in TypeScript (app/g2/even-advertisement.ts,
* app/g2/pairing-candidates.ts) where they can be unit-tested; this filter
* must stay a superset of the TS classifier's admission.
*
* Ported from the discovery path of SybilSight's Bluetooth SDK (G2.didDiscover
* and R1's advertisement snapshot), reduced to what Android exposes.
*/
@SuppressLint("MissingPermission")
public class FaceclawDeviceDiscovery {
@@ -296,7 +293,7 @@ public class FaceclawDeviceDiscovery {
/**
* Android strips the 2-byte company identifier from each manufacturer
* record and keys the SparseArray by it. Put the bytes back (little-endian)
* so TypeScript sees the same "ER"+… layout the iOS SDK and HCI captures
* so TypeScript sees the same "ER"+… layout the HCI captures
* document. Prefers Even's own id; otherwise returns the first record so an
* unexpected company id still reaches the diagnostics log.
*/
@@ -0,0 +1,262 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import com.k2fsa.sherpa.onnx.FastClusteringConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarization;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerDiarizationSegment;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationModelConfig;
import com.k2fsa.sherpa.onnx.OfflineSpeakerSegmentationPyannoteModelConfig;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* Offline re-diarization of a stored conversation recording (16-bit mono
* WAV): pyannote segmentation plus the shared speaker-embedding model with
* clustering, via sherpa-onnx. Runs on its own thread; results are handed to
* TS as JSON turns for reconciliation against the transcript segments.
*/
public class FaceclawDiarizer {
private static final String TAG = "FaceclawDiarizer";
public interface Listener {
/** progress in [0, 1]. */
void onProgress(double progress);
/** turns: [{startMs, endMs, cluster}], sorted by start time. */
void onDone(String turnsJson);
void onError(String message);
}
private final String segmentationModelPath;
private final String embeddingModelPath;
private final String wavPath;
private final int numClusters;
private final float threshold;
private final Listener listener;
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private volatile boolean cancelled;
public FaceclawDiarizer(
String segmentationModelPath,
String embeddingModelPath,
String wavPath,
int numClusters,
float threshold,
Listener listener) {
this.segmentationModelPath = segmentationModelPath;
this.embeddingModelPath = embeddingModelPath;
this.wavPath = wavPath;
this.numClusters = numClusters;
this.threshold = threshold <= 0 ? 0.5f : threshold;
this.listener = listener;
}
public void start() {
Thread thread = new Thread(this::run, "FaceclawDiarizer");
thread.setPriority(Thread.MIN_PRIORITY);
thread.start();
}
public void cancel() {
cancelled = true;
}
private void run() {
OfflineSpeakerDiarization diarization = null;
try {
float[] samples = wavPath.endsWith(".wav")
? readWavMono16(wavPath)
: decodeCompressedMono(wavPath);
OfflineSpeakerDiarizationConfig config = OfflineSpeakerDiarizationConfig.builder()
.setSegmentation(OfflineSpeakerSegmentationModelConfig.builder()
.setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig.builder()
.setModel(segmentationModelPath)
.build())
.setNumThreads(1)
.build())
.setEmbedding(SpeakerEmbeddingExtractorConfig.builder()
.setModel(embeddingModelPath)
.setNumThreads(1)
.build())
.setClustering(FastClusteringConfig.builder()
.setNumClusters(numClusters > 0 ? numClusters : -1)
.setThreshold(threshold)
.build())
.setMinDurationOn(0.3f)
.setMinDurationOff(0.5f)
.build();
diarization = new OfflineSpeakerDiarization(config);
OfflineSpeakerDiarizationSegment[] segments = diarization.processWithCallback(
samples,
(numProcessedChunks, numTotalChunks, arg) -> {
emitProgress(numTotalChunks <= 0 ? 0
: (double) numProcessedChunks / numTotalChunks);
return cancelled ? 1 : 0;
});
JSONArray turns = new JSONArray();
if (segments != null) {
for (OfflineSpeakerDiarizationSegment segment : segments) {
JSONObject turn = new JSONObject();
turn.put("startMs", Math.round(segment.getStart() * 1000));
turn.put("endMs", Math.round(segment.getEnd() * 1000));
turn.put("cluster", segment.getSpeaker());
turns.put(turn);
}
}
String result = turns.toString();
mainHandler.post(() -> listener.onDone(result));
} catch (Throwable t) {
Log.w(TAG, "diarization failed for " + wavPath, t);
String message = t.getMessage() == null ? t.toString() : t.getMessage();
mainHandler.post(() -> listener.onError(message));
} finally {
if (diarization != null) {
diarization.release();
}
}
}
private void emitProgress(double progress) {
mainHandler.post(() -> listener.onProgress(Math.max(0, Math.min(1, progress))));
}
/**
* Decode a compressed recording (M4A/AAC from the transcode sweep) into
* [-1, 1] floats via MediaExtractor + MediaCodec. Recordings are written
* as 16 kHz mono, so no resampling is needed.
*/
private static float[] decodeCompressedMono(String path) throws IOException {
android.media.MediaExtractor extractor = new android.media.MediaExtractor();
android.media.MediaCodec codec = null;
java.io.ByteArrayOutputStream pcm = new java.io.ByteArrayOutputStream();
try {
extractor.setDataSource(path);
int trackIndex = -1;
android.media.MediaFormat format = null;
for (int i = 0; i < extractor.getTrackCount(); i++) {
android.media.MediaFormat candidate = extractor.getTrackFormat(i);
String mime = candidate.getString(android.media.MediaFormat.KEY_MIME);
if (mime != null && mime.startsWith("audio/")) {
trackIndex = i;
format = candidate;
break;
}
}
if (trackIndex < 0 || format == null) {
throw new IOException("no audio track in " + path);
}
extractor.selectTrack(trackIndex);
codec = android.media.MediaCodec.createDecoderByType(
format.getString(android.media.MediaFormat.KEY_MIME));
codec.configure(format, null, null, 0);
codec.start();
android.media.MediaCodec.BufferInfo info = new android.media.MediaCodec.BufferInfo();
boolean inputDone = false;
boolean outputDone = false;
while (!outputDone) {
if (!inputDone) {
int inIndex = codec.dequeueInputBuffer(10_000);
if (inIndex >= 0) {
ByteBuffer inBuf = codec.getInputBuffer(inIndex);
int size = extractor.readSampleData(inBuf, 0);
if (size < 0) {
codec.queueInputBuffer(inIndex, 0, 0, 0,
android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM);
inputDone = true;
} else {
codec.queueInputBuffer(inIndex, 0, size, extractor.getSampleTime(), 0);
extractor.advance();
}
}
}
int outIndex = codec.dequeueOutputBuffer(info, 10_000);
if (outIndex >= 0) {
ByteBuffer outBuf = codec.getOutputBuffer(outIndex);
byte[] chunk = new byte[info.size];
outBuf.position(info.offset);
outBuf.get(chunk);
pcm.write(chunk);
codec.releaseOutputBuffer(outIndex, false);
if ((info.flags & android.media.MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputDone = true;
}
}
}
} finally {
extractor.release();
if (codec != null) {
try {
codec.stop();
} catch (Throwable ignored) {
}
codec.release();
}
}
byte[] bytes = pcm.toByteArray();
int count = bytes.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = ((short) ((bytes[i * 2] & 0xff) | (bytes[i * 2 + 1] << 8))) / 32768.0f;
}
return samples;
}
/** Load a 16-bit mono PCM WAV into [-1, 1] floats. */
private static float[] readWavMono16(String path) throws IOException {
File file = new File(path);
long fileLength = file.length();
try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
byte[] header = new byte[44];
in.readFully(header);
if (header[0] != 'R' || header[1] != 'I' || header[8] != 'W') {
throw new IOException("not a WAV file");
}
int channels = (header[22] & 0xff) | (header[23] << 8);
int bits = (header[34] & 0xff) | (header[35] << 8);
if (channels != 1 || bits != 16) {
throw new IOException("only 16-bit mono WAV is supported");
}
long dataBytes = fileLength - 44;
int count = (int) Math.min(dataBytes / 2, Integer.MAX_VALUE);
float[] samples = new float[count];
byte[] chunk = new byte[65536];
int sampleIndex = 0;
int carried = 0;
byte carriedByte = 0;
int read;
while (sampleIndex < count && (read = in.read(chunk)) > 0) {
int offset = 0;
if (carried == 1) {
samples[sampleIndex++] =
((short) ((carriedByte & 0xff) | (chunk[0] << 8))) / 32768.0f;
offset = 1;
carried = 0;
}
int pairs = (read - offset) / 2;
for (int i = 0; i < pairs && sampleIndex < count; i++) {
int base = offset + i * 2;
samples[sampleIndex++] =
((short) ((chunk[base] & 0xff) | (chunk[base + 1] << 8))) / 32768.0f;
}
if (((read - offset) & 1) == 1) {
carried = 1;
carriedByte = chunk[read - 1];
}
}
return samples;
}
}
}
@@ -12,6 +12,12 @@ public class FaceclawLc3Decoder {
public static final int COUNTER_OFFSET = 204;
public static final int SAMPLES_PER_FRAME = 160;
public static final int SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME;
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
public static final int SSR_OFFSET = 200;
public static final int ANGLE_OFFSET = 202;
static {
System.loadLibrary("faceclaw_lc3");
@@ -23,6 +29,8 @@ public class FaceclawLc3Decoder {
private long duplicatePackets;
private long missingPackets;
private long decodeErrors;
private int lastSsr;
private int lastAngleDegrees;
public FaceclawLc3Decoder() {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE);
@@ -64,9 +72,21 @@ public class FaceclawLc3Decoder {
}
realPackets++;
lastCounter = counter;
lastSsr = (short) ((packet[SSR_OFFSET] & 0xff) | (packet[SSR_OFFSET + 1] << 8));
lastAngleDegrees = (short) ((packet[ANGLE_OFFSET] & 0xff) | (packet[ANGLE_OFFSET + 1] << 8));
return decoded;
}
/** Firmware signal-strength ratio from the most recent decoded packet. */
public synchronized int getLastSsr() {
return lastSsr;
}
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
public synchronized int getLastAngleDegrees() {
return lastAngleDegrees;
}
public synchronized long getRealPackets() {
return realPackets;
}
@@ -0,0 +1,10 @@
package com.faceclaw.app;
public interface FaceclawMicStatusListener {
/**
* A CFW mic_control field-104 status record from one temple. body is the
* raw 21-byte ['M','C',ver,...] record (decoded on the TS side); arm is
* "L" or "R" for the temple's own BLE link, "?" if unmatched.
*/
void onMicStatus(byte[] body, String arm);
}
@@ -0,0 +1,266 @@
package com.faceclaw.app;
import android.media.audiofx.AcousticEchoCanceler;
import android.media.audiofx.AudioEffect;
import android.media.audiofx.AutomaticGainControl;
import android.media.audiofx.NoiseSuppressor;
import java.util.ArrayList;
import java.util.Arrays;
/**
* Phone-side noise suppression for the microphone stream.
*
* Android's built-in noise cancellation (the {@link NoiseSuppressor} capture
* effect) only attaches to AudioRecord sessions on the phone's own microphones. Faceclaw's audio
* arrives as raw PCM over BLE from the glasses, which that effect chain never
* sees, so this class picks the best engine that can actually reach the
* samples:
*
* - Audio captured by the phone itself: {@link #attachPlatformEffects(int)}
* enables the device's built-in NS/AEC/AGC chain (hardware or OEM software,
* whatever the Android build ships) on the AudioRecord session.
* - Audio from the glasses (the normal case): {@link #process(byte[])} runs a
* WebRTC-NS-family spectral suppressor here on the phone — 16 ms sqrt-Hann
* frames at 50% overlap, per-bin asymmetric noise-floor tracking, and an
* over-subtracted Wiener gain with a residual floor so speech never pumps
* in and out of silence.
*/
public final class FaceclawNoiseSuppressor {
private static final int FFT_SIZE = 256;
private static final int HOP = FFT_SIZE / 2;
private static final float OVERSUBTRACTION = 1.6f;
/** About -16 dB of residual: keeps the noise bed natural instead of gated. */
private static final float GAIN_FLOOR = 0.15f;
private static final float GAIN_SMOOTHING = 0.6f;
/** Frames of pure noise-floor learning after start/reset (~100 ms). */
private static final int NOISE_WARMUP_FRAMES = 12;
private final int sampleRate;
private final float[] window = new float[FFT_SIZE];
private final float[] history = new float[FFT_SIZE];
private final float[] overlap = new float[HOP];
private final float[] noise = new float[FFT_SIZE / 2 + 1];
private final float[] gain = new float[FFT_SIZE / 2 + 1];
private final float[] re = new float[FFT_SIZE];
private final float[] im = new float[FFT_SIZE];
private final short[] pending = new short[HOP];
private int pendingCount = 0;
private int warmupFrames = 0;
public FaceclawNoiseSuppressor(int sampleRate) {
this.sampleRate = sampleRate;
// sqrt-Hann (sine) analysis+synthesis windows: exact COLA at 50% overlap.
for (int i = 0; i < FFT_SIZE; i++) {
window[i] = (float) Math.sin(Math.PI * (i + 0.5) / FFT_SIZE);
}
reset();
}
public int getSampleRate() {
return sampleRate;
}
/** Forget the learned noise floor (e.g. after the suppressor was toggled off). */
public void reset() {
Arrays.fill(history, 0f);
Arrays.fill(overlap, 0f);
Arrays.fill(noise, 0f);
Arrays.fill(gain, 1f);
pendingCount = 0;
warmupFrames = 0;
}
/**
* Suppress noise in 16-bit little-endian mono PCM. Streaming: input not
* filling a whole 8 ms hop is carried into the next call, so the output
* may be up to one hop shorter than the input (and longer on a later
* call); total latency is one hop.
*/
public byte[] process(byte[] pcm16le) {
if (pcm16le == null || pcm16le.length < 2) {
return pcm16le;
}
int sampleCount = pcm16le.length / 2;
int hops = (pendingCount + sampleCount) / HOP;
byte[] out = new byte[hops * HOP * 2];
int outPos = 0;
int inPos = 0;
while (inPos < sampleCount) {
int take = Math.min(HOP - pendingCount, sampleCount - inPos);
for (int i = 0; i < take; i++) {
int lo = pcm16le[(inPos + i) * 2] & 0xff;
int hi = pcm16le[(inPos + i) * 2 + 1];
pending[pendingCount + i] = (short) ((hi << 8) | lo);
}
pendingCount += take;
inPos += take;
if (pendingCount == HOP) {
outPos = processHop(out, outPos);
pendingCount = 0;
}
}
return out;
}
private int processHop(byte[] out, int outPos) {
System.arraycopy(history, HOP, history, 0, FFT_SIZE - HOP);
for (int i = 0; i < HOP; i++) {
history[FFT_SIZE - HOP + i] = pending[i] / 32768f;
}
for (int i = 0; i < FFT_SIZE; i++) {
re[i] = history[i] * window[i];
im[i] = 0f;
}
fft(re, im, false);
boolean warming = warmupFrames < NOISE_WARMUP_FRAMES;
if (warming) {
warmupFrames++;
}
for (int b = 0; b <= FFT_SIZE / 2; b++) {
float power = re[b] * re[b] + im[b] * im[b];
if (warming) {
noise[b] = warmupFrames == 1 ? power : noise[b] + (power - noise[b]) / warmupFrames;
} else if (power < noise[b]) {
// Track the floor down quickly; drift up slowly so speech
// energy never gets learned as noise.
noise[b] += (power - noise[b]) * 0.2f;
} else {
noise[b] = Math.min(power, noise[b] * 1.006f + 1e-10f);
}
float clean = power - OVERSUBTRACTION * noise[b];
float g = clean > 0f ? clean / power : 0f;
if (g < GAIN_FLOOR) {
g = GAIN_FLOOR;
}
g = GAIN_SMOOTHING * gain[b] + (1f - GAIN_SMOOTHING) * g;
gain[b] = g;
re[b] *= g;
im[b] *= g;
if (b > 0 && b < FFT_SIZE / 2) {
re[FFT_SIZE - b] *= g;
im[FFT_SIZE - b] *= g;
}
}
fft(re, im, true);
for (int i = 0; i < HOP; i++) {
float value = (overlap[i] + re[i] * window[i]) * 32767f;
int sample = Math.round(value);
if (sample > 32767) {
sample = 32767;
}
if (sample < -32768) {
sample = -32768;
}
out[outPos + i * 2] = (byte) (sample & 0xff);
out[outPos + i * 2 + 1] = (byte) ((sample >> 8) & 0xff);
}
for (int i = 0; i < HOP; i++) {
overlap[i] = re[HOP + i] * window[HOP + i];
}
return outPos + HOP * 2;
}
/** In-place iterative radix-2 FFT (inverse includes the 1/N scale). */
private static void fft(float[] re, float[] im, boolean inverse) {
int n = re.length;
for (int i = 1, j = 0; i < n; i++) {
int bit = n >> 1;
for (; (j & bit) != 0; bit >>= 1) {
j ^= bit;
}
j ^= bit;
if (i < j) {
float t = re[i];
re[i] = re[j];
re[j] = t;
t = im[i];
im[i] = im[j];
im[j] = t;
}
}
for (int len = 2; len <= n; len <<= 1) {
double ang = 2 * Math.PI / len * (inverse ? 1 : -1);
float wRe = (float) Math.cos(ang);
float wIm = (float) Math.sin(ang);
for (int i = 0; i < n; i += len) {
float curRe = 1f;
float curIm = 0f;
for (int k = 0; k < len / 2; k++) {
int a = i + k;
int b = a + len / 2;
float xr = re[b] * curRe - im[b] * curIm;
float xi = re[b] * curIm + im[b] * curRe;
re[b] = re[a] - xr;
im[b] = im[a] - xi;
re[a] += xr;
im[a] += xi;
float nextRe = curRe * wRe - curIm * wIm;
curIm = curRe * wIm + curIm * wRe;
curRe = nextRe;
}
}
}
if (inverse) {
for (int i = 0; i < n; i++) {
re[i] /= n;
im[i] /= n;
}
}
}
/** Whether this Android build ships the built-in capture noise suppressor. */
public static boolean platformSuppressorAvailable() {
try {
return NoiseSuppressor.isAvailable();
} catch (Throwable t) {
return false;
}
}
/** Short engine name for the UI. */
public static String engineDescription() {
return "spectral (phone DSP)";
}
/**
* Attach the platform's built-in capture-effect chain (NoiseSuppressor,
* AcousticEchoCanceler, AutomaticGainControl — whichever this Android
* build provides) to a phone AudioRecord session. Only applicable when the
* phone itself captures the audio; the BLE glasses stream must go through
* {@link #process(byte[])} instead. The caller keeps the returned effects
* for the life of the recording and release()s them with it.
*/
public static AudioEffect[] attachPlatformEffects(int audioSessionId) {
ArrayList<AudioEffect> effects = new ArrayList<>();
try {
if (NoiseSuppressor.isAvailable()) {
NoiseSuppressor ns = NoiseSuppressor.create(audioSessionId);
if (ns != null) {
ns.setEnabled(true);
effects.add(ns);
}
}
if (AcousticEchoCanceler.isAvailable()) {
AcousticEchoCanceler aec = AcousticEchoCanceler.create(audioSessionId);
if (aec != null) {
aec.setEnabled(true);
effects.add(aec);
}
}
if (AutomaticGainControl.isAvailable()) {
AutomaticGainControl agc = AutomaticGainControl.create(audioSessionId);
if (agc != null) {
agc.setEnabled(true);
effects.add(agc);
}
}
} catch (Throwable t) {
// A missing effect just means this tier is unavailable.
}
return effects.toArray(new AudioEffect[0]);
}
}
@@ -0,0 +1,111 @@
package com.faceclaw.app;
import android.util.Log;
import com.k2fsa.sherpa.onnx.OnlineStream;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor;
import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig;
/**
* Speaker voice-print extraction over sherpa-onnx. One embedding per
* utterance; matching/enrollment happens on the TS side (cosine similarity
* against the stored speaker profiles).
*/
public class FaceclawSpeakerId {
private static final String TAG = "FaceclawSpeakerId";
private SpeakerEmbeddingExtractor extractor;
private final String modelPath;
public FaceclawSpeakerId(String modelPath) {
this.modelPath = modelPath;
}
/** Lazily loads the model; returns false when it cannot be loaded. */
public synchronized boolean ensureLoaded() {
if (extractor != null) {
return true;
}
try {
SpeakerEmbeddingExtractorConfig config = SpeakerEmbeddingExtractorConfig.builder()
.setModel(modelPath)
.setNumThreads(1)
.setDebug(false)
.setProvider("cpu")
.build();
extractor = new SpeakerEmbeddingExtractor(config);
return true;
} catch (Throwable t) {
Log.w(TAG, "could not load speaker model at " + modelPath, t);
extractor = null;
return false;
}
}
public synchronized int embeddingDim() {
return extractor == null ? 0 : extractor.getDim();
}
/**
* Compute an L2-normalized voice-print for one utterance of 16 kHz mono
* S16LE PCM. Returns null when the model is unavailable or the utterance
* is too short to embed (under ~0.5 s).
*/
public synchronized float[] embed(byte[] pcm16le, int sampleRate) {
if (pcm16le == null || pcm16le.length < sampleRate) {
return null;
}
if (!ensureLoaded()) {
return null;
}
int count = pcm16le.length / 2;
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
short s = (short) ((pcm16le[i * 2] & 0xff) | (pcm16le[i * 2 + 1] << 8));
samples[i] = s / 32768.0f;
}
OnlineStream stream = null;
try {
stream = extractor.createStream();
stream.acceptWaveform(samples, sampleRate);
stream.inputFinished();
if (!extractor.isReady(stream)) {
return null;
}
float[] embedding = extractor.compute(stream);
return l2Normalize(embedding);
} catch (Throwable t) {
Log.w(TAG, "embedding failed", t);
return null;
} finally {
if (stream != null) {
stream.release();
}
}
}
private static float[] l2Normalize(float[] v) {
if (v == null) {
return null;
}
double sum = 0;
for (float x : v) {
sum += (double) x * x;
}
double norm = Math.sqrt(sum);
if (norm <= 0) {
return v;
}
for (int i = 0; i < v.length; i++) {
v[i] = (float) (v[i] / norm);
}
return v;
}
public synchronized void close() {
if (extractor != null) {
extractor.release();
extractor = null;
}
}
}
@@ -90,6 +90,26 @@ public class FaceclawVoiceController {
private volatile boolean endpointing;
private final EndpointDetector endpointDetector = new EndpointDetector();
private java.io.ByteArrayOutputStream recordingPcm;
// Speaker verification against the enrolled wearer voice-print ("my voice
// only" command gating). Configured before start(); the utterance PCM is
// buffered (capped) and verified once at session end.
private static final int VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10;
private static final int VERIFY_MIN_SAMPLES = SAMPLE_RATE;
private volatile String verifySpeakerModelPath;
private volatile float[] verifyWearerEmbedding;
private volatile float verifyThreshold = 0.8f;
private short[] verifyBuffer;
private int verifyCount;
// Global mic processing (Microphones app config): spectral noise
// suppression and firmware-DoA beam gating, applied to every capture
// session that opts in (assistant push-to-talk, Transcribe, hands-free).
// The raw tap opts out — raw means raw, and the Microphones session does
// its own beam-compensated processing on that path.
private volatile boolean suppressionEnabled;
private volatile boolean beamFilterEnabled;
private volatile int beamCenterDeg;
private volatile int beamHalfWidthDeg = 180;
private FaceclawNoiseSuppressor suppressor;
private long queuedPackets;
private long queueDroppedPackets;
private long decodedSamples;
@@ -125,6 +145,68 @@ public class FaceclawVoiceController {
this.endpointing = endpointing;
}
/**
* Verify this session's speaker against the enrolled wearer voice-print
* and report the result via onSpeakerVerified just before the final
* transcript. Must be set before {@link #start}; pass a null model path
* to disable.
*/
public void setSpeakerVerification(String speakerModelPath, float[] wearerEmbedding, float threshold) {
this.verifySpeakerModelPath = speakerModelPath;
this.verifyWearerEmbedding = wearerEmbedding;
this.verifyThreshold = threshold > 0 ? threshold : 0.8f;
}
public void clearSpeakerVerification() {
this.verifySpeakerModelPath = null;
this.verifyWearerEmbedding = null;
}
/** Spectral noise suppression on the decoded stream. Safe to flip mid-run. */
public void setNoiseSuppression(boolean enabled) {
this.suppressionEnabled = enabled;
}
/**
* Direction gating from the Sonic Radar beam: packets whose firmware
* direction-of-arrival falls outside centerDeg ± halfWidthDeg (device
* frame, 0 = straight ahead, positive right) are dropped before any
* consumer sees them. Safe to update mid-run.
*/
public void setBeamFilter(boolean enabled, int centerDeg, int halfWidthDeg) {
this.beamFilterEnabled = enabled;
this.beamCenterDeg = centerDeg;
this.beamHalfWidthDeg = Math.max(5, Math.min(180, halfWidthDeg));
}
private boolean withinBeam(int angleDegrees) {
int delta = angleDegrees - beamCenterDeg;
while (delta > 180) delta -= 360;
while (delta < -180) delta += 360;
return Math.abs(delta) <= beamHalfWidthDeg;
}
private void applySuppression(short[] pcm, int count) {
try {
if (suppressor == null) {
suppressor = new FaceclawNoiseSuppressor(SAMPLE_RATE);
}
byte[] le = new byte[count * 2];
for (int i = 0; i < count; i++) {
le[i * 2] = (byte) (pcm[i] & 0xff);
le[i * 2 + 1] = (byte) ((pcm[i] >> 8) & 0xff);
}
byte[] cleaned = suppressor.process(le);
int cleanedCount = Math.min(count, cleaned.length / 2);
for (int i = 0; i < cleanedCount; i++) {
pcm[i] = (short) ((cleaned[i * 2] & 0xff) | (cleaned[i * 2 + 1] << 8));
}
} catch (Throwable t) {
Log.w(TAG, "noise suppression failed; passing audio through", t);
suppressionEnabled = false;
}
}
public void start(String requestedMode) {
synchronized (lock) {
if (started) {
@@ -195,7 +277,13 @@ public class FaceclawVoiceController {
}
lc3Decoder = new FaceclawLc3Decoder();
endpointDetector.reset();
if (suppressor != null) {
suppressor.reset();
}
recordingPcm = saveRecordings ? new java.io.ByteArrayOutputStream(SAMPLE_RATE * 2 * 4) : null;
boolean verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null;
verifyBuffer = verifying ? new short[VERIFY_MAX_SAMPLES] : null;
verifyCount = 0;
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.");
return;
@@ -205,6 +293,10 @@ public class FaceclawVoiceController {
? "Listening (cloud)..."
: "Listening...");
processG2Audio();
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
// on (the callbacks are posted in order to the main handler).
runSpeakerVerification();
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
if (currentMode == VoiceInputMode.ONBOARD) {
@@ -374,15 +466,38 @@ public class FaceclawVoiceController {
continue;
}
decodedSamples += count;
// Global mic processing from the Microphones app config: the
// beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker
// for every consumer, recognition included), and the spectral
// noise suppressor cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
int angleDegrees = currentDecoder.getLastAngleDegrees();
int ssr = currentDecoder.getLastSsr();
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr);
maybeEmitAudioStats(false);
continue;
}
if (suppressionEnabled) {
applySuppression(pcm, count);
}
if (recordingPcm != null) {
appendRecording(pcm, count);
}
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
int copied = Math.min(count, VERIFY_MAX_SAMPLES - verifyCount);
System.arraycopy(pcm, 0, verifyBuffer, verifyCount, copied);
verifyCount += copied;
}
if (endpointing && endpointDetector.accept(pcm, count)) {
emitSpeechEnd();
}
if (mode == VoiceInputMode.CLOUD) {
emitPcm(pcm, count);
} else {
// PCM and frame metadata flow in every mode so levels, recording,
// and the Microphones radar keep working alongside onboard ASR.
emitPcm(pcm, count);
emitFrameMeta(angleDegrees, ssr);
if (mode != VoiceInputMode.CLOUD) {
float[] samples = new float[count];
for (int i = 0; i < count; i++) {
samples[i] = pcm[i] / 32768.0f;
@@ -582,6 +697,83 @@ public class FaceclawVoiceController {
mainHandler.post(() -> currentListener.onPcm(le));
}
/**
* Embed the session's buffered utterance and compare it to the enrolled
* wearer voice-print. Fails open: a session too short to verify, or a
* model that will not load, counts as the wearer rather than silencing
* every command.
*/
private void runSpeakerVerification() {
short[] buffer = verifyBuffer;
float[] wearer = verifyWearerEmbedding;
String modelPath = verifySpeakerModelPath;
verifyBuffer = null;
if (buffer == null || wearer == null || modelPath == null) {
return;
}
if (verifyCount < VERIFY_MIN_SAMPLES) {
emitSpeakerVerified(true, 0f);
return;
}
FaceclawSpeakerId speakerId = cachedSpeakerId(modelPath);
try {
byte[] le = new byte[verifyCount * 2];
for (int i = 0; i < verifyCount; i++) {
short s = buffer[i];
le[i * 2] = (byte) (s & 0xff);
le[i * 2 + 1] = (byte) ((s >> 8) & 0xff);
}
float[] embedding = speakerId.embed(le, SAMPLE_RATE);
if (embedding == null || embedding.length != wearer.length) {
emitSpeakerVerified(true, 0f);
return;
}
double dot = 0;
for (int i = 0; i < embedding.length; i++) {
dot += (double) embedding[i] * wearer[i];
}
boolean isWearer = dot >= verifyThreshold;
Log.i(TAG, "speaker verification similarity=" + String.format(java.util.Locale.US, "%.3f", dot)
+ " threshold=" + verifyThreshold + " isWearer=" + isWearer);
emitSpeakerVerified(isWearer, (float) dot);
} catch (Throwable t) {
Log.w(TAG, "speaker verification failed", t);
emitSpeakerVerified(true, 0f);
}
}
// The 28 MB embedding model takes seconds to load; keep one instance
// across capture sessions so verification adds only the embed time.
private static FaceclawSpeakerId sharedSpeakerId;
private static String sharedSpeakerIdPath;
private static synchronized FaceclawSpeakerId cachedSpeakerId(String modelPath) {
if (sharedSpeakerId == null || !modelPath.equals(sharedSpeakerIdPath)) {
if (sharedSpeakerId != null) {
sharedSpeakerId.close();
}
sharedSpeakerId = new FaceclawSpeakerId(modelPath);
sharedSpeakerIdPath = modelPath;
}
return sharedSpeakerId;
}
private void emitSpeakerVerified(boolean isWearer, float similarity) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onSpeakerVerified(isWearer, similarity));
}
private void emitFrameMeta(int angleDegrees, int ssr) {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
return;
}
mainHandler.post(() -> currentListener.onFrameMeta(angleDegrees, ssr));
}
private void emitSpeechEnd() {
FaceclawVoiceControllerListener currentListener = listener;
if (currentListener == null) {
@@ -11,15 +11,34 @@ public interface FaceclawVoiceControllerListener {
void onTranscript(String text, boolean isFinal);
/**
* Decoded microphone audio for CLOUD mode: 16 kHz mono signed 16-bit
* little-endian PCM. Empty/absent in onboard mode.
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
* keep working while the onboard recognizer owns the transcript.
*/
void onPcm(byte[] pcm16le);
/**
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
* packet, computed on the raw stereo capture before its mono downmix: the
* direction-of-arrival in signed degrees (0 = straight ahead, positive to
* the wearer's right) and a signal-strength ratio that serves as a
* confidence/energy proxy for the angle.
*/
void onFrameMeta(int angleDegrees, int ssr);
/**
* The speaker stopped, in a hands-free session that has no button release
* to end it. Only fires when endpointing was enabled for the session; see
* FaceclawVoiceController.setEndpointing.
*/
void onSpeechEnd();
/**
* Result of speaker verification against the enrolled wearer voice-print,
* emitted once per capture session just before the final transcript.
* Only fires when verification was configured for the session; fails
* open (isWearer=true) when the utterance was too short to verify or the
* speaker model could not be loaded.
*/
void onSpeakerVerified(boolean isWearer, float similarity);
}
@@ -0,0 +1,100 @@
package com.faceclaw.app;
import android.util.Log;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
/**
* Streaming WAV writer for conversation recordings: the header is written up
* front with zero sizes and patched on finish, so a crash mid-session leaves
* a recoverable file (data length can be derived from the file size).
*/
public class FaceclawWavRecorder {
private static final String TAG = "FaceclawWavRecorder";
private final String path;
private final int sampleRate;
private final int channels;
private RandomAccessFile file;
private long dataBytes;
public FaceclawWavRecorder(String path, int sampleRate, int channels) throws IOException {
this.path = path;
this.sampleRate = sampleRate;
this.channels = channels;
File parent = new File(path).getParentFile();
if (parent != null && !parent.exists() && !parent.mkdirs()) {
throw new IOException("could not create " + parent);
}
file = new RandomAccessFile(path, "rw");
file.setLength(0);
file.write(buildHeader(0));
}
public synchronized void append(byte[] pcm16le) {
if (file == null || pcm16le == null || pcm16le.length == 0) {
return;
}
try {
file.write(pcm16le);
dataBytes += pcm16le.length;
} catch (IOException e) {
Log.w(TAG, "append failed", e);
}
}
public synchronized long getDurationMs() {
return dataBytes * 1000L / (sampleRate * channels * 2L);
}
public String getPath() {
return path;
}
/** Patch the header and close. Returns the PCM byte count (0 = empty). */
public synchronized long finish() {
RandomAccessFile current = file;
file = null;
if (current == null) {
return dataBytes;
}
try {
current.seek(0);
current.write(buildHeader((int) Math.min(dataBytes, Integer.MAX_VALUE)));
current.close();
} catch (IOException e) {
Log.w(TAG, "finish failed", e);
}
return dataBytes;
}
/** Delete the (finished or abandoned) recording. */
public synchronized void discard() {
finish();
new File(path).delete();
}
private byte[] buildHeader(int pcmBytes) {
int byteRate = sampleRate * channels * 2;
ByteBuffer b = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN);
b.put("RIFF".getBytes(StandardCharsets.US_ASCII));
b.putInt(36 + pcmBytes);
b.put("WAVE".getBytes(StandardCharsets.US_ASCII));
b.put("fmt ".getBytes(StandardCharsets.US_ASCII));
b.putInt(16);
b.putShort((short) 1);
b.putShort((short) channels);
b.putInt(sampleRate);
b.putInt(byteRate);
b.putShort((short) (channels * 2));
b.putShort((short) 16);
b.put("data".getBytes(StandardCharsets.US_ASCII));
b.putInt(pcmBytes);
return b.array();
}
}
@@ -49,6 +49,14 @@ public class BleProtocol {
// protobuf fields to stock implementations, so they are safely ignored.
public static final int FACECLAW_WAKE_CONTROL_FIELD = 101;
public static final int FACECLAW_WAKE_EVENT_FIELD = 102;
/**
* CFW mic_control (EVENCFW/16, caps tokens micctl/micmc/micraw): host
* writes an ['M','C',ver,op,...] record as settings field 103; each temple
* reports a 21-byte ['M','C',ver,...] status as field 104, both appended
* to settings read responses and as standalone commandId=3 pushes.
*/
public static final int FACECLAW_MIC_CONTROL_FIELD = 103;
public static final int FACECLAW_MIC_STATUS_FIELD = 104;
public static final int FACECLAW_WAKE_OP_ACQUIRE = 1;
public static final int FACECLAW_WAKE_OP_RELEASE = 2;
public static final int FACECLAW_WAKE_OP_CLAIM = 3;
@@ -405,6 +413,40 @@ public class BleProtocol {
));
}
/**
* Wrap an already-encoded ['M','C',ver,op,...] mic-control record as a
* settings write carrying CFW field 103. Magic zero makes it
* fire-and-forget like the wake lease: the firmware consumes the field
* before its stock nanopb decoder discards the unknown tag, and the
* "ack" is the field-104 status notify that follows.
*/
public static byte[] buildFaceclawMicControl(byte[] record) {
return concat(CollectionUtils.listOf(
encodeVarintField(1, 1),
encodeVarintField(2, 0),
encodeBytesField(FACECLAW_MIC_CONTROL_FIELD, record)
));
}
/**
* Extract the CFW mic status record (field 104) from a settings frame, or
* null when absent. The 21-byte body layout is decoded on the TS side;
* here we only validate the 'M','C',version prelude.
*/
public static byte[] parseFaceclawMicStatus(byte[] pb) {
if (pb == null) {
return null;
}
byte[] body = readFieldBytes(stripTrailingCrc(pb), FACECLAW_MIC_STATUS_FIELD);
if (body == null
|| body.length < 3
|| body[0] != 'M'
|| body[1] != 'C') {
return null;
}
return body;
}
/**
* Return the uint16 wake nonce from a CFW field-102 notification, or -1
* when this is an ordinary settings frame.
@@ -267,6 +267,21 @@ public class MessageBuilder {
);
}
/** CFW mic_control write (settings field 103) targeted at one temple. */
public OutboundMessage faceclawMicControl(byte[] record, String label, boolean leftArm) {
return new OutboundMessage(
"mic-control",
"mic control " + label + (leftArm ? " L" : " R"),
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
0,
BleProtocol.buildFaceclawMicControl(record),
ACK_TIMEOUT_MS,
-1,
leftArm
);
}
public OutboundMessage faceclawWearQuery(boolean leftArm) {
return new OutboundMessage(
"wear-query-control",
@@ -0,0 +1,44 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class FastClusteringConfig {
private final int numClusters;
private final float threshold;
private FastClusteringConfig(Builder builder) {
this.numClusters = builder.numClusters;
this.threshold = builder.threshold;
}
public static Builder builder() {
return new Builder();
}
public int getNumClusters() {
return numClusters;
}
public float getThreshold() {
return threshold;
}
public static class Builder {
private int numClusters = -1;
private float threshold = 0.5f;
public FastClusteringConfig build() {
return new FastClusteringConfig(this);
}
public Builder setNumClusters(int numClusters) {
this.numClusters = numClusters;
return this;
}
public Builder setThreshold(float threshold) {
this.threshold = threshold;
return this;
}
}
}
@@ -0,0 +1,62 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarization {
private long ptr = 0;
public OfflineSpeakerDiarization(OfflineSpeakerDiarizationConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid OfflineSpeakerDiarizationConfig: failed to create native OfflineSpeakerDiarization");
}
}
public int getSampleRate() {
return getSampleRate(ptr);
}
// Only config.clustering is used. All other fields are ignored
public void setConfig(OfflineSpeakerDiarizationConfig config) {
setConfig(ptr, config);
}
public OfflineSpeakerDiarizationSegment[] process(float[] samples) {
return process(ptr, samples);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback) {
return processWithCallback(ptr, samples, callback, 0);
}
public OfflineSpeakerDiarizationSegment[] processWithCallback(float[] samples, OfflineSpeakerDiarizationCallback callback, long arg) {
return processWithCallback(ptr, samples, callback, arg);
}
protected void finalize() throws Throwable {
release();
}
// You'd better call it manually if it is not used anymore
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
private native int getSampleRate(long ptr);
private native void delete(long ptr);
private native long newFromFile(OfflineSpeakerDiarizationConfig config);
private native void setConfig(long ptr, OfflineSpeakerDiarizationConfig config);
private native OfflineSpeakerDiarizationSegment[] process(long ptr, float[] samples);
private native OfflineSpeakerDiarizationSegment[] processWithCallback(long ptr, float[] samples, OfflineSpeakerDiarizationCallback callback, long arg);
}
@@ -0,0 +1,8 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
@FunctionalInterface
public interface OfflineSpeakerDiarizationCallback {
Integer invoke(int numProcessedChunks, int numTotalCunks, long arg);
}
@@ -0,0 +1,79 @@
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationConfig {
private final OfflineSpeakerSegmentationModelConfig segmentation;
private final SpeakerEmbeddingExtractorConfig embedding;
private final FastClusteringConfig clustering;
private final float minDurationOn;
private final float minDurationOff;
private OfflineSpeakerDiarizationConfig(Builder builder) {
this.segmentation = builder.segmentation;
this.embedding = builder.embedding;
this.clustering = builder.clustering;
this.minDurationOff = builder.minDurationOff;
this.minDurationOn = builder.minDurationOn;
}
public static Builder builder() {
return new Builder();
}
public OfflineSpeakerSegmentationModelConfig getSegmentation() {
return segmentation;
}
public SpeakerEmbeddingExtractorConfig getEmbedding() {
return embedding;
}
public FastClusteringConfig getClustering() {
return clustering;
}
public float getMinDurationOff() {
return minDurationOff;
}
public float getMinDurationOn() {
return minDurationOn;
}
public static class Builder {
private OfflineSpeakerSegmentationModelConfig segmentation = OfflineSpeakerSegmentationModelConfig.builder().build();
private SpeakerEmbeddingExtractorConfig embedding = SpeakerEmbeddingExtractorConfig.builder().build();
private FastClusteringConfig clustering = FastClusteringConfig.builder().build();
private float minDurationOn = 0.2f;
private float minDurationOff = 0.5f;
public OfflineSpeakerDiarizationConfig build() {
return new OfflineSpeakerDiarizationConfig(this);
}
public Builder setSegmentation(OfflineSpeakerSegmentationModelConfig segmentation) {
this.segmentation = segmentation;
return this;
}
public Builder setEmbedding(SpeakerEmbeddingExtractorConfig embedding) {
this.embedding = embedding;
return this;
}
public Builder setClustering(FastClusteringConfig clustering) {
this.clustering = clustering;
return this;
}
public Builder setMinDurationOff(float minDurationOff) {
this.minDurationOff = minDurationOff;
return this;
}
public Builder setMinDurationOn(float minDurationOn) {
this.minDurationOn = minDurationOn;
return this;
}
}
}
@@ -0,0 +1,27 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerDiarizationSegment {
private final float start;
private final float end;
private final int speaker;
public OfflineSpeakerDiarizationSegment(float start, float end, int speaker) {
this.start = start;
this.end = end;
this.speaker = speaker;
}
public float getStart() {
return start;
}
public float getEnd() {
return end;
}
public int getSpeaker() {
return speaker;
}
}
@@ -0,0 +1,52 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationModelConfig {
private final OfflineSpeakerSegmentationPyannoteModelConfig pyannote;
private final int numThreads;
private final boolean debug;
private final String provider;
private OfflineSpeakerSegmentationModelConfig(Builder builder) {
this.pyannote = builder.pyannote;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private OfflineSpeakerSegmentationPyannoteModelConfig pyannote = OfflineSpeakerSegmentationPyannoteModelConfig.builder().build();
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public OfflineSpeakerSegmentationModelConfig build() {
return new OfflineSpeakerSegmentationModelConfig(this);
}
public Builder setPyannote(OfflineSpeakerSegmentationPyannoteModelConfig pyannote) {
this.pyannote = pyannote;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,32 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineSpeakerSegmentationPyannoteModelConfig {
private final String model;
private OfflineSpeakerSegmentationPyannoteModelConfig(Builder builder) {
this.model = builder.model;
}
public static Builder builder() {
return new Builder();
}
public String getModel() {
return model;
}
public static class Builder {
private String model = "";
public OfflineSpeakerSegmentationPyannoteModelConfig build() {
return new OfflineSpeakerSegmentationPyannoteModelConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
}
}
@@ -0,0 +1,77 @@
// Copyright 2022-2023 by zhaoming
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OnlineStream {
private long ptr = 0;
public OnlineStream() {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
this.ptr = 0;
}
public OnlineStream(long ptr) {
this.ptr = ptr;
}
public long getPtr() {
return ptr;
}
public void setPtr(long ptr) {
this.ptr = ptr;
}
public void acceptWaveform(float[] samples, int sampleRate) {
acceptWaveform(this.ptr, samples, sampleRate);
}
public void inputFinished() {
inputFinished(this.ptr);
}
public void setOption(String key, String value) {
setOption(this.ptr, key, value);
}
public String getOption(String key) {
return getOption(this.ptr, key);
}
public boolean hasOption(String key) {
return hasOption(this.ptr, key);
}
public void release() {
close();
}
public void close() {
// stream object must be release after used
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
@Override
protected void finalize() throws Throwable {
close();
super.finalize();
}
private native void acceptWaveform(long ptr, float[] samples, int sampleRate);
private native void inputFinished(long ptr);
private native void setOption(long ptr, String key, String value);
private native String getOption(long ptr, String key);
private native boolean hasOption(long ptr, String key);
private native void delete(long ptr);
}
@@ -0,0 +1,58 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractor {
private long ptr = 0;
public SpeakerEmbeddingExtractor(SpeakerEmbeddingExtractorConfig config) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = newFromFile(config);
if (ptr == 0) {
throw new IllegalArgumentException("Invalid SpeakerEmbeddingExtractorConfig: failed to create native SpeakerEmbeddingExtractor");
}
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public OnlineStream createStream() {
long p = createStream(ptr);
return new OnlineStream(p);
}
public boolean isReady(OnlineStream s) {
return isReady(ptr, s.getPtr());
}
public float[] compute(OnlineStream s) {
return compute(ptr, s.getPtr());
}
public int getDim() {
return dim(ptr);
}
private native void delete(long ptr);
private native long newFromFile(SpeakerEmbeddingExtractorConfig config);
private native long createStream(long ptr);
private native boolean isReady(long ptr, long streamPtr);
private native float[] compute(long ptr, long streamPtr);
private native int dim(long ptr);
}
@@ -0,0 +1,53 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingExtractorConfig {
private final String model;
private final int numThreads;
private final boolean debug;
private final String provider;
private SpeakerEmbeddingExtractorConfig(Builder builder) {
this.model = builder.model;
this.numThreads = builder.numThreads;
this.debug = builder.debug;
this.provider = builder.provider;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String model = "";
private int numThreads = 1;
private boolean debug = true;
private String provider = "cpu";
public SpeakerEmbeddingExtractorConfig build() {
return new SpeakerEmbeddingExtractorConfig(this);
}
public Builder setModel(String model) {
this.model = model;
return this;
}
public Builder setNumThreads(int numThreads) {
this.numThreads = numThreads;
return this;
}
public Builder setDebug(boolean debug) {
this.debug = debug;
return this;
}
public Builder setProvider(String provider) {
this.provider = provider;
return this;
}
}
}
@@ -0,0 +1,78 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class SpeakerEmbeddingManager {
private long ptr = 0;
public SpeakerEmbeddingManager(int dim) {
System.loadLibrary("onnxruntime");
System.loadLibrary("sherpa-onnx-jni");
ptr = create(dim);
}
@Override
protected void finalize() throws Throwable {
release();
}
public void release() {
if (this.ptr == 0) {
return;
}
delete(this.ptr);
this.ptr = 0;
}
public boolean add(String name, float[] embedding) {
return add(ptr, name, embedding);
}
public boolean add(String name, float[][] embedding) {
return addList(ptr, name, embedding);
}
public boolean remove(String name) {
return remove(ptr, name);
}
public String search(float[] embedding, float threshold) {
return search(ptr, embedding, threshold);
}
public boolean verify(String name, float[] embedding, float threshold) {
return verify(ptr, name, embedding, threshold);
}
public boolean contains(String name) {
return contains(ptr, name);
}
public int getNumSpeakers() {
return numSpeakers(ptr);
}
public String[] getAllSpeakerNames() {
return allSpeakerNames(ptr);
}
private native long create(int dim);
private native void delete(long ptr);
private native boolean add(long ptr, String name, float[] embedding);
private native boolean addList(long ptr, String name, float[][] embedding);
private native boolean remove(long ptr, String name);
private native String search(long ptr, float[] embedding, float threshold);
private native boolean verify(long ptr, String name, float[] embedding, float threshold);
private native boolean contains(long ptr, String name);
private native int numSpeakers(long ptr);
private native String[] allSpeakerNames(long ptr);
}
+163
View File
@@ -502,3 +502,166 @@ Button.permissions-primary-dimmed {
.ns-dark .permissions-check {
color: #7bc47f;
}
/*
* Conversation review (phone-ui/conversations-page, conversation-page,
* speakers-page). Cards follow the pairing-row look; speaker/emotion chips
* get their background colour inline (per-speaker / per-emotion), so the
* classes only carry shape and dark text that reads on the pastel palette.
*/
.conversation-row {
background-color: #f4f4f4;
border-color: #dddddd;
border-width: 1;
border-radius: 10;
padding: 10;
margin-bottom: 8;
}
.conversation-title {
font-size: 16;
font-weight: bold;
color: #222222;
}
.conversation-meta {
font-size: 12;
color: #666666;
}
.conversation-empty {
color: #777777;
font-size: 14;
padding: 8 0;
}
.ask-answer {
font-size: 15;
color: #222222;
padding: 4 0;
}
.ns-dark .ask-answer {
color: #e6e6e6;
}
.conversation-status {
font-size: 13;
color: #2a5bc4;
}
.speaker-chip {
font-size: 12;
color: #222222;
border-radius: 10;
padding: 2 8;
margin: 0 6 4 0;
}
.speaker-chip-selected {
border-width: 2;
border-color: #222222;
font-weight: bold;
}
.speaker-tag {
font-size: 11;
color: #555555;
background-color: #e6e6e6;
border-radius: 8;
padding: 1 6;
}
.speaker-wearer-badge {
font-size: 11;
font-weight: bold;
color: #2a5bc4;
background-color: #dbe6fb;
border-radius: 8;
padding: 1 6;
}
.transcript-row {
border-radius: 8;
padding: 8;
margin-bottom: 4;
}
.transcript-row-playing {
background-color: #e6f0ff;
}
.transcript-translation {
font-size: 13;
font-style: italic;
color: #777777;
}
.transcript-time {
font-size: 11;
color: #888888;
}
.transcript-emotion {
font-size: 11;
color: #222222;
border-radius: 8;
padding: 1 6;
}
.transcript-play {
font-size: 16;
color: #2a5bc4;
margin-left: 6;
}
.ns-dark .conversation-row {
background-color: #2e2e2e;
border-color: #4a4a4a;
}
.ns-dark .conversation-title {
color: #f0f0f0;
}
.ns-dark .conversation-meta {
color: #a0a0a0;
}
.ns-dark .conversation-empty {
color: #999999;
}
.ns-dark .conversation-status {
color: #7da7f5;
}
.ns-dark .speaker-chip-selected {
border-color: #f0f0f0;
}
.ns-dark .speaker-tag {
color: #cccccc;
background-color: #3c3c3c;
}
.ns-dark .speaker-wearer-badge {
color: #a8c4f5;
background-color: #2a3b5e;
}
.ns-dark .transcript-row-playing {
background-color: #333c4c;
}
.ns-dark .transcript-translation {
color: #999999;
}
.ns-dark .transcript-time {
color: #909090;
}
.ns-dark .transcript-play {
color: #7da7f5;
}
+4
View File
@@ -1,11 +1,13 @@
import { type AppDefinition } from "./app-definition";
import launcherApp from "./launcher";
import timerApp from "./timer";
import calculatorApp from "./calculator";
import terminalApp from "./terminal";
import filesApp from "./files";
import musicApp from "./music";
import nightscoutApp from "./nightscout";
import transcribeApp from "./transcribe";
import microphonesApp from "./microphones";
import notificationsApp from "./notifications";
import calendarApp from "./calendar";
import weatherApp from "./weather";
@@ -28,11 +30,13 @@ import settingsApp from "./settings";
export const ALL_APPS: readonly AppDefinition[] = [
launcherApp,
timerApp,
calculatorApp,
terminalApp,
filesApp,
musicApp,
nightscoutApp,
transcribeApp,
microphonesApp,
notificationsApp,
calendarApp,
weatherApp,
+150
View File
@@ -0,0 +1,150 @@
/**
* Calculator window wiring: the in-process window, its long-press menu (mode
* follow-ups, clear, settings), the assistant tools, and the voice/text
* plumbing into the layer.
*
* The coordinator is a module singleton on purpose: a
* problem stated through the assistant's calculate tool is still standing
* when the wearer opens the window, and "graph it" there refers to it. The
* same session also survives closing and reopening the window.
*/
import {
createInProcessWindow,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { type MenuItem } from "../../ui/menu";
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { shell } from "../../ui/shell/shell";
import { toolRegistry } from "../../assistant/tool-registry";
import { type ToolSpec } from "../../assistant/tool-registry";
import { MathCoordinator } from "./math/coordinator";
import { rewriteSpokenProblem } from "./calculator-llm";
import {
CalculatorLayer,
calculatorDegreesSetting,
calculatorListeningSetting,
calculatorModeSetting,
} from "./calculator";
import { CALCULATOR_MODES, modeLabel } from "./calculator-commands";
export const CALCULATOR_WINDOW_ID = "calculator";
export const CALCULATOR_SURFACE_ID = "window:calculator";
/** One standing maths session, shared by the window and the assistant tools. */
const coordinator = new MathCoordinator();
coordinator.setAICompletion(rewriteSpokenProblem);
const CALCULATOR_TOOLS: ToolSpec[] = [
{
name: "calculate",
description:
"Solve a maths problem stated in words or notation ('2x + 7 = 19', 'what is 15 percent of 240', " +
"'is 91 prime', 'where does 11 first appear in pi', 'area of a circle with radius 3'). " +
"Do NOT work it out yourself first — hand the words over verbatim; the calculator answers " +
"exactly and shows the result on the glasses. Follow-ups like 'graph it' and 'explain it' " +
"act on the problem already standing.",
inputSchema: {
type: "object",
properties: {
problem: { type: "string", description: "The problem or follow-up, exactly as asked." },
},
required: ["problem"],
additionalProperties: false,
},
availability: "open",
},
];
export function createCalculatorAppWindow(options: InProcessAppOptions): InProcessWindow {
const layer = new CalculatorLayer(coordinator, options.actions);
const menuItems = (): MenuItem[] => [
...CALCULATOR_MODES.map(
(mode): MenuItem => ({
label: `${modeLabel(mode)} it`,
onSelect: (ctx) => {
ctx.stack.pop();
layer.performFollowUp(mode);
},
}),
),
{
label: "Clear",
onSelect: (ctx) => {
ctx.stack.pop();
layer.clear();
},
},
{
label: "Calculator settings",
onSelect: (ctx) => {
ctx.stack.pop();
openSettingsSubMenu(ctx, "Calculator settings", [
enumSettingMenuItem(calculatorModeSetting, {
onChange: () => layer.applySettings(),
}),
enumSettingMenuItem(calculatorListeningSetting, {
onChange: () => layer.applySettings(),
}),
toggleSettingMenuItem(calculatorDegreesSetting, {
onChange: () => layer.applySettings(),
}),
]);
},
},
];
let app: InProcessWindow;
app = createInProcessWindow({
appId: "calculator",
windowId: CALCULATOR_WINDOW_ID,
title: "Calculator",
iconLetter: "=",
icon: "calculator",
closeable: true,
menuItems,
actions: options.actions,
receiveTextInput: (text) => {
if (app.stack.receiveTextInput(text)) app.requestRender();
},
baseLayer: layer,
submitFrame: options.submitFrame,
setSurfaceVisible: (visible) => {
layer.setForeground(visible);
options.setSurfaceVisible(visible);
},
removeSurface: options.removeSurface,
onClosed: () => {
toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID);
options.onClosed();
},
});
layer.requestRender = app.requestRender;
layer.setForeground(shell.isWindowVisible(CALCULATOR_WINDOW_ID));
// Apply the persisted settings now the window exists: degrees onto the
// session, and continuous listening if the wearer left it on.
layer.applySettings();
toolRegistry.setAppTools({
windowId: CALCULATOR_WINDOW_ID,
appId: "calculator",
specs: CALCULATOR_TOOLS,
isForeground: () => shell.isWindowFocused(CALCULATOR_WINDOW_ID),
invoke: async (toolName, args) => {
if (toolName !== "calculate") {
return { ok: false, error: `Unknown calculator tool: ${toolName}` };
}
const problem = String((args as { problem?: unknown })?.problem ?? "").trim();
if (!problem) return { ok: false, error: "calculate requires a problem" };
const outcome = await coordinator.answerSpoken(problem);
app.requestRender();
if (outcome.kind === "failure") return { ok: false, error: outcome.reason };
const detail = coordinator.lastResult?.detail ?? "";
return { ok: true, content: detail || outcome.text };
},
});
return app;
}
+117
View File
@@ -0,0 +1,117 @@
// The pure classification half of the calculator's command handling. Kept
// free of any UI import so the tests can exercise exactly the
// decisions that matter: which utterances are commands, which unprompted room
// speech is worth answering, and how the modes map onto the shared session.
import { type MathBranchMode } from "./math/coordinator";
import { parseCommand } from "./math/session";
/**
* What a freshly captured problem does the moment it lands.
*
* Three, not more, because these are the three the wearer is TOLD about —
* "Solve", "Explain", "Graph" are the spoken commands, so making them the
* selectable modes means the touchpad and the voice are steering the same
* thing rather than two parallel vocabularies. Differentiation stays
* reachable by saying it ("the derivative of…"), which is how it is actually
* asked for.
*/
export type CalculatorMode = "solve" | "explain" | "graph";
export const CALCULATOR_MODES: readonly CalculatorMode[] = ["solve", "explain", "graph"];
export function modeLabel(mode: CalculatorMode): string {
switch (mode) {
case "solve":
return "Solve";
case "explain":
return "Explain";
case "graph":
return "Graph";
}
}
export function modeDetail(mode: CalculatorMode): string {
switch (mode) {
case "solve":
return "Answer it, exactly. 3/2 stays 3/2.";
case "explain":
return "Show the working step by step; scroll walks the rest.";
case "graph":
return "Plot it.";
}
}
export function modeBranch(mode: CalculatorMode): MathBranchMode {
return mode;
}
/**
* The spoken commands that select this mode. Both the bare verb and the
* "it" form, because someone shown "Solve" says both.
*/
export function modeSpokenCommands(mode: CalculatorMode): string[] {
switch (mode) {
case "solve":
return ["solve", "solve it"];
case "explain":
return ["explain", "explain it"];
case "graph":
return ["graph", "graph it"];
}
}
/**
* How the Calculator listens.
*
* "Tap to talk" arms exactly one finalized utterance per tap, so the app is
* not standing in the room claiming every sentence; "continuous" listens the
* whole time the app is foreground, gated by the looks-like-maths test
* below. The reason there are two input modes: a wearer in a
* quiet room must be able to take the microphone away from the calculator.
*/
export type CalculatorListening = "tap" | "continuous";
export function listeningLabel(listening: CalculatorListening): string {
switch (listening) {
case "tap":
return "Tap to talk";
case "continuous":
return "Continuous";
}
}
/**
* Recognise a bare mode command in an utterance. Exact match on the whole
* utterance, not a substring: "I need to solve this problem with the tax
* people" is conversation, and a substring test turns it into a command.
*/
export function modeCommand(text: string): CalculatorMode | null {
const normalized = text.toLowerCase().replace(/^[ .,!?"']+|[ .,!?"']+$/g, "");
for (const mode of CALCULATOR_MODES) {
if (modeSpokenCommands(mode).includes(normalized)) return mode;
}
return null;
}
export function isDictationCancellation(text: string): boolean {
const normalized = text.toLowerCase().replace(/^[ .,!?"']+|[ .,!?"']+$/g, "");
return ["cancel", "cancel listening", "stop listening", "never mind", "nevermind"].includes(normalized);
}
/**
* Whether unprompted room speech is worth handing to the parser. In
* continuous mode the app sees every finalized utterance in the room;
* without this gate, every sentence becomes a failed calculation on the
* lens.
*/
export function isWorthAnswering(text: string): boolean {
if (modeCommand(text) !== null) return true;
return parseCommand(text) !== null;
}
export function cycleMode(mode: CalculatorMode, forward: boolean): CalculatorMode {
const index = CALCULATOR_MODES.indexOf(mode);
const step = forward ? 1 : CALCULATOR_MODES.length - 1;
return CALCULATOR_MODES[(index + step) % CALCULATOR_MODES.length]!;
}
+96
View File
@@ -0,0 +1,96 @@
// Draws the maths engine's already-sampled graph into the app's GrayImage viewport.
// The renderer never re-evaluates the expression: segment breaks, viewport
// clipping, roots, and extrema therefore stay exactly what the plotter
// decided, and an asymptote correctly stays an open gap rather than a
// vertical line — the single most recognisable "bad graphing app" artefact.
import { GrayImage } from "../../graphics/image";
import { type Graph, type GraphPoint, viewportContainsY } from "./math/plotter";
// Brightness levels tuned for the G2's grayscale panel: the curve stays
// visually dominant over the axes, and the axes over the tick grid.
const GRID_VALUE = 38;
const AXIS_VALUE = 150;
const BORDER_VALUE = 105;
const CURVE_VALUES = [245, 200, 160];
const MARKER_VALUE = 255;
/**
* Draw the graph into `rect` (x, y, width, height) of the image. The caller
* owns the header and footer rows.
*/
export function drawGraph(
image: GrayImage,
graph: Graph,
rect: { x: number; y: number; width: number; height: number },
): void {
if (rect.width <= 1 || rect.height <= 1) return;
const viewport = graph.viewport;
const spanX = Math.max(viewport.xMax - viewport.xMin, Number.EPSILON);
const spanY = Math.max(viewport.yMax - viewport.yMin, Number.EPSILON);
const px = (x: number): number => rect.x + ((x - viewport.xMin) / spanX) * rect.width;
const py = (y: number): number => rect.y + rect.height - ((y - viewport.yMin) / spanY) * rect.height;
const clampX = (x: number): number => Math.max(rect.x, Math.min(rect.x + rect.width - 1, x));
const clampY = (y: number): number => Math.max(rect.y, Math.min(rect.y + rect.height - 1, y));
// Tick grid first so the curve and axes draw over it.
for (const tick of graph.xAxis.ticks) {
if (tick < viewport.xMin || tick > viewport.xMax) continue;
const x = Math.round(px(tick));
image.drawLine(x, rect.y, x, rect.y + rect.height - 1, GRID_VALUE);
}
for (const tick of graph.yAxis.ticks) {
if (tick < viewport.yMin || tick > viewport.yMax) continue;
const y = Math.round(py(tick));
image.drawLine(rect.x, y, rect.x + rect.width - 1, y, GRID_VALUE);
}
// Axes.
if (viewport.yMin <= 0 && viewport.yMax >= 0) {
const y = Math.round(py(0));
image.drawLine(rect.x, y, rect.x + rect.width - 1, y, AXIS_VALUE);
}
if (viewport.xMin <= 0 && viewport.xMax >= 0) {
const x = Math.round(px(0));
image.drawLine(x, rect.y, x, rect.y + rect.height - 1, AXIS_VALUE);
}
// Curves, brightest first plot.
for (let plotIndex = 0; plotIndex < graph.plots.length; plotIndex++) {
const plot = graph.plots[plotIndex]!;
const value = CURVE_VALUES[plotIndex % CURVE_VALUES.length]!;
for (const segment of plot.segments) {
let previous: GraphPoint | null = null;
for (const point of segment.points) {
if (!Number.isFinite(point.x) || !Number.isFinite(point.y)) {
previous = null;
continue;
}
if (previous) {
image.drawLine(
Math.round(clampX(px(previous.x))),
Math.round(clampY(py(previous.y))),
Math.round(clampX(px(point.x))),
Math.round(clampY(py(point.y))),
value,
);
}
previous = point;
}
}
// Markers: roots and intercepts as small filled dots.
for (const marker of plot.markers) {
if (!viewportContainsY(viewport, marker.point.y)) continue;
if (marker.point.x < viewport.xMin || marker.point.x > viewport.xMax) continue;
const cx = Math.round(px(marker.point.x));
const cy = Math.round(py(marker.point.y));
image.fillRoundedRect(cx - 3, cy - 3, 6, 6, MARKER_VALUE, 3);
}
}
// Frame the plot area.
image.drawRect(rect.x, rect.y, rect.width, rect.height, BORDER_VALUE);
}
+51
View File
@@ -0,0 +1,51 @@
/**
* One-shot completion against the assistant's configured model, used by the
* Calculator's optional rewrite fallback (see MathCoordinator.answerSpoken:
* the deterministic parser runs first; the model only ever translates words
* into notation). Resolves to null when no provider is configured or the
* call fails — the calculator then reports "could not read that" instead of
* surfacing a transport error.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel } from "../../assistant/models";
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
import {
anthropicApiKeySetting,
assistantModelSetting,
openAiApiKeySetting,
} from "../../ui/dashboard-settings";
import { CALCULATOR_REWRITE_SYSTEM_PROMPT, buildCalculatorRewriteUserMessage } from "../../prompts";
export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
const resolved = resolveAssistantModel(assistantModelSetting.get(), {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
if (!resolved) return Promise.resolve(null);
return new Promise((resolve) => {
const options: LlmStreamOptions = {
apiKey: resolved.apiKey,
model: resolved.model,
system: CALCULATOR_REWRITE_SYSTEM_PROMPT,
effort: "low",
maxTokens: 200,
messages: [{ role: "user", content: buildCalculatorRewriteUserMessage(spoken) }],
onDone: (result) => resolve(result.text ?? null),
onError: () => resolve(null),
};
switch (resolved.provider) {
case "openai":
streamOpenAiResponse(options);
break;
case "local":
streamLocalQwen(options);
break;
default:
streamAnthropicMessage(options);
break;
}
});
}
+606
View File
@@ -0,0 +1,606 @@
// The calculator app layer for FaceClaw: a spoken (or typed) problem in, an
// exact answer on the lens.
//
// Deliberately thin. Everything mathematical belongs to math/ and everything
// about the standing problem, follow-ups, and long runs belongs to
// MathCoordinator. This layer owns only what is genuinely the app's: which
// mode a fresh capture lands in, what the viewport shows, and how the
// touchpad and the microphone are shared.
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultLargeFont, getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { wrapText, truncateText } from "../../graphics/textwrap";
import { lineStep } from "../../ui/metrics";
import { gestureHints, GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../../ui/gestures";
import { type DashboardInputEvent, type Layer, type LayerActions, type LayerContext } from "../../ui/layers";
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
import { shell } from "../../ui/shell/shell";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator";
import {
type LongRunOffer,
type LongRunProgress,
offerGlassesPrompt,
progressText,
} from "./math/workload";
import { plain } from "./math/text-renderer";
import { drawGraph } from "./calculator-graph";
import {
type CalculatorListening,
type CalculatorMode,
CALCULATOR_MODES,
cycleMode,
isDictationCancellation,
isWorthAnswering,
listeningLabel,
modeCommand,
modeLabel,
} from "./calculator-commands";
// ---------------------------------------------------------------------------
// Settings
/** The mode a fresh capture lands in. Also what scrolling cycles. */
export const calculatorModeSetting = new ConfigSettingEnum<CalculatorMode>({
id: "calculator-mode",
label: "Default mode",
storageKey: "calculator.defaultMode",
defaultValue: "solve",
values: [...CALCULATOR_MODES],
formatValue: modeLabel,
description:
"What a freshly stated problem does. On the glasses, scroll applies another mode — or walks Explain steps — and tap toggles the mic.",
});
export const calculatorListeningSetting = new ConfigSettingEnum<CalculatorListening>({
id: "calculator-listening",
label: "Listening",
storageKey: "calculator.listening",
defaultValue: "tap",
values: ["tap", "continuous"],
formatValue: listeningLabel,
description:
"Tap to talk captures one problem per tap. Continuous listens the whole time the calculator is in front, answering anything that sounds like maths.",
});
export const calculatorDegreesSetting = new ConfigSettingBoolean({
id: "calculator-degrees",
label: "Trigonometry in degrees",
storageKey: "calculator.usesDegrees",
defaultValue: false,
description: "Evaluate sin/cos/tan in degrees instead of radians.",
});
// ---------------------------------------------------------------------------
// Layer
export class CalculatorLayer implements Layer {
/** Rebound by the window factory once the window exists. */
requestRender: () => void = () => {};
/**
* The last thing said to the calculator, so the view can show what it
* heard next to what it made of it — the only way a wearer can tell a
* mis-heard problem from a wrong answer.
*/
private lastInput = "";
private lastAnswer = "";
private lastFailure: string | null = null;
private working = false;
/** True while one finalized utterance is armed for capture. */
private dictating = false;
private foreground = true;
private continuousActive = false;
private unsubscribeTranscript: (() => void) | null = null;
private lastDeepAnswer: MathBranchOutcome | null = null;
constructor(
private readonly coordinator: MathCoordinator,
private readonly actions: LayerActions,
) {
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
}
private get mode(): CalculatorMode {
return calculatorModeSetting.get();
}
private get listening(): CalculatorListening {
return calculatorListeningSetting.get();
}
// -------------------------------------------------------------------------
// Lifecycle
onRemoved(): void {
this.stopListening();
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.coordinator.onChanged = null;
}
/** Foreground/background transitions, forwarded by the window factory. */
setForeground(foreground: boolean): void {
if (this.foreground === foreground) return;
this.foreground = foreground;
if (!foreground) {
this.stopListening();
} else {
this.refreshListening();
}
}
/** Apply a settings change: degrees and the listening mode both act immediately. */
applySettings(): void {
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
this.refreshListening();
this.requestRender();
}
private refreshListening(): void {
if (!this.foreground) return;
if (this.listening === "continuous") {
if (!this.continuousActive) {
this.continuousActive = true;
void this.actions.startContinuousVoiceCapture();
}
} else if (this.continuousActive) {
this.continuousActive = false;
void this.actions.stopContinuousVoiceCapture();
}
}
private stopListening(): void {
if (this.dictating) {
this.dictating = false;
void this.actions.stopVoiceCapture();
}
if (this.continuousActive) {
this.continuousActive = false;
void this.actions.stopContinuousVoiceCapture();
}
}
// -------------------------------------------------------------------------
// Asking
/**
* Put a problem (or a follow-up command) to the calculator.
*
* Command first, problem second. "graph it" contains no operator and would
* fail as an expression, and a wearer who says it expects the plot of what
* is already on the lens rather than an error about the word "it".
*/
submit(text: string): void {
const trimmed = text.trim();
if (!trimmed) return;
this.lastInput = trimmed;
this.lastFailure = null;
const selected = modeCommand(trimmed);
if (selected) {
this.performFollowUp(selected);
return;
}
const target = this.mode;
this.working = true;
this.requestRender();
void this.coordinator.answerSpoken(trimmed).then((outcome) => {
// A fresh problem lands in the app's mode: solving is the default, but
// a wearer whose default is "graph" said the problem in order to see it
// plotted, not to read its root.
if (outcome.kind === "value" && target !== "solve") {
outcome = this.coordinator.run(target, "");
}
this.working = false;
this.applyOutcome(outcome);
});
}
/**
* Re-run the standing problem through a different presentation branch.
* This is the one action used by scroll cycling, spoken "… it" commands,
* and the window menu, so changing mode can never mean "change a setting
* but leave the old answer up".
*/
performFollowUp(selected: CalculatorMode): void {
calculatorModeSetting.set(selected);
if (!this.coordinator.hasProblem) {
this.lastFailure = "No problem yet";
this.requestRender();
return;
}
this.working = false;
this.lastFailure = null;
this.applyOutcome(this.coordinator.run(selected, ""));
}
clear(): void {
this.working = false;
this.lastInput = "";
this.lastAnswer = "";
this.lastFailure = null;
this.coordinator.handleRingAction("exit");
this.requestRender();
}
private applyOutcome(outcome: MathBranchOutcome | null): void {
if (outcome?.kind === "value") {
this.lastAnswer = outcome.text;
this.lastFailure = null;
} else if (outcome?.kind === "failure") {
this.lastFailure = outcome.reason;
}
this.requestRender();
}
private handleCoordinatorChanged(): void {
// The answer a deeper pass produced arrives on its own channel; fold it
// into the standing answer the moment it lands.
const deep = this.coordinator.deepAnswer;
if (deep && deep !== this.lastDeepAnswer) {
this.lastDeepAnswer = deep;
this.applyOutcome(deep);
return;
}
this.requestRender();
}
// -------------------------------------------------------------------------
// Dictation
/**
* Arm exactly one finalized utterance. One, not a listening window: the
* indicator on the lens is what makes this honest, and a calculator that
* quietly held the microphone open would be indistinguishable from one
* that did not.
*/
beginDictation(): void {
if (this.dictating) return;
this.dictating = true;
void this.actions.startVoiceCapture(true);
this.requestRender();
}
cancelDictation(): void {
if (!this.dictating) return;
this.dictating = false;
void this.actions.stopVoiceCapture();
this.requestRender();
}
/**
* A finalized utterance reached the app.
*
* In tap mode this only ever arrives because the wearer armed it, so it is
* always taken. In continuous mode it also arrives from ordinary room
* speech, and there it has to clear the looks-like-maths gate first —
* otherwise every sentence in the room becomes a failed calculation on the
* lens.
*/
private onTranscript(event: VoiceTranscriptEvent): void {
if (!event.isFinal) return;
const text = event.text.trim();
if (!text) return;
const armed = this.dictating;
const continuous = this.continuousActive && this.foreground;
if (!armed && !continuous) return;
if (armed) {
this.dictating = false;
void this.actions.stopVoiceCapture();
}
// A standing offer or a run in flight owns the next yes/no said in the
// room ("keep going", "never mind", "stop").
const longRunAnswer = this.coordinator.handleLongRunUtterance(text);
if (longRunAnswer) {
this.applyOutcome(longRunAnswer);
return;
}
if (armed && isDictationCancellation(text)) {
this.requestRender();
return;
}
if (!armed && !isWorthAnswering(text)) return;
this.submit(text);
}
// -------------------------------------------------------------------------
// Input
receiveTextInput(text: string): void {
this.submit(text);
}
handleInput(event: DashboardInputEvent, _ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
case "scroll-down": {
const forward = event.type === "scroll-down";
// While Explain is showing a step sequence the scroll walks it — the
// coordinator owns that, and asking it first is what keeps one
// gesture from meaning two things at once. Otherwise scroll cycles
// the MODE, because the mode is what the wearer changes between
// problems and the steps only exist inside one of the three modes.
if (this.mode === "explain" && this.coordinator.stepPosition !== null) {
this.applyOutcome(this.coordinator.handleRingAction(forward ? "forward" : "back"));
return;
}
const next = cycleMode(this.mode, forward);
if (this.coordinator.hasProblem) {
this.performFollowUp(next);
} else {
calculatorModeSetting.set(next);
this.requestRender();
}
return;
}
case "click": {
// A standing offer owns the tap: accept and start the long run.
if (this.coordinator.hasLongRunInteraction) {
this.applyOutcome(this.coordinator.handleRingAction("commit"));
return;
}
if (this.dictating) {
this.cancelDictation();
} else {
this.beginDictation();
}
return;
}
case "double-click": {
if (this.dictating) {
this.cancelDictation();
return;
}
if (this.coordinator.isRunningLong || this.coordinator.pendingLongRun) {
// Stop the run, or turn the offer down; the shallow answer stands.
this.applyOutcome(this.coordinator.handleRingAction("exit"));
return;
}
// The standard leave-the-app gesture.
shell.yieldFocusToSidebar();
return;
}
default:
return;
}
}
// -------------------------------------------------------------------------
// Paint
paint(ctx: LayerContext): GrayImage {
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const small = getDefaultSmallFont();
this.paintTabs(image, small, width);
const footerY = height - small.lineHeight - 6;
const bodyTop = 46;
let bodyBottom = footerY - 4;
// A standing offer or a running deep pass takes the rows just above the
// footer, whatever view is behind it.
const offer = this.coordinator.pendingLongRun;
const progress = this.coordinator.longRunProgress;
if (offer || progress) {
bodyBottom = this.paintLongRunPanel(image, small, width, bodyBottom, offer, progress);
}
if (this.working) {
this.drawCentered(image, small, Math.round((bodyTop + bodyBottom) / 2) - 8, "Working…", 180);
} else if (this.lastFailure !== null) {
this.paintFailure(image, small, width, bodyTop, bodyBottom);
} else if (this.mode === "explain" && this.coordinator.stepPosition !== null) {
this.paintExplain(image, width, bodyTop, bodyBottom);
} else if (this.mode === "graph" && this.coordinator.lastGraph) {
this.paintGraph(image, small, width, bodyTop, bodyBottom);
} else if (this.lastAnswer) {
this.paintAnswer(image, width, bodyTop, bodyBottom);
} else {
this.paintEmpty(image, small, width, bodyTop, bodyBottom);
}
image.drawText(small, 24, footerY, this.footerHints(), 110);
return image;
}
/** The three modes as tabs, plus the deg/listening status chips. */
private paintTabs(image: GrayImage, font: UiFont, width: number): void {
let x = 24;
for (const mode of CALCULATOR_MODES) {
const label = modeLabel(mode);
const active = mode === this.mode;
image.drawText(font, x, 15, label, active ? 245 : 105);
image.drawLine(x, 35, x + font.measureText(label), 35, active ? 200 : 30);
x += font.measureText(label) + 18;
}
const chips: string[] = [];
if (calculatorDegreesSetting.get()) chips.push("deg");
if (this.dictating) chips.push("listening…");
else if (this.continuousActive && this.foreground) chips.push("mic on");
if (chips.length > 0) {
const text = chips.join(" · ");
image.drawText(font, width - 24 - font.measureText(text), 15, text, this.dictating ? 220 : 110);
}
}
private paintFailure(
image: GrayImage,
font: UiFont,
width: number,
bodyTop: number,
bodyBottom: number,
): void {
this.paintHeardLine(image, font, width, bodyTop);
const top = this.lastInput ? bodyTop + lineStep(font) + 6 : bodyTop + 8;
const lines = wrapText(font, this.lastFailure ?? "", width - 64);
const step = lineStep(font);
const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step));
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
image.drawText(font, 32, top + index * step, lines[index]!, 200);
}
}
private paintExplain(image: GrayImage, width: number, bodyTop: number, bodyBottom: number): void {
const small = getDefaultSmallFont();
const medium = getDefaultMediumFont();
const position = this.coordinator.stepPosition;
const steps = this.coordinator.explanationSteps;
const step = position ? steps[position.index] : undefined;
if (!position || !step) return;
let y = bodyTop + 2;
image.drawText(small, 24, y, `Step ${position.index + 1} of ${position.total}`, 130);
y += lineStep(small) + 2;
image.drawText(medium, 24, y, truncateText(medium, step.title, width - 48), 245);
y += lineStep(medium) + 2;
image.drawText(medium, 24, y, truncateText(medium, plain(step.expression), width - 48), 220);
y += lineStep(medium) + 6;
const detailLines = wrapText(small, step.detail, width - 64);
const stepPitch = lineStep(small);
const maxLines = Math.max(0, Math.floor((bodyBottom - y) / stepPitch));
for (let index = 0; index < Math.min(detailLines.length, maxLines); index++) {
image.drawText(small, 32, y + index * stepPitch, detailLines[index]!, 150);
}
}
private paintGraph(
image: GrayImage,
font: UiFont,
width: number,
bodyTop: number,
bodyBottom: number,
): void {
const graph = this.coordinator.lastGraph;
if (!graph) return;
const headline = this.lastAnswer.split("\n")[0] ?? "";
image.drawText(font, 24, bodyTop, truncateText(font, headline, width - 48), 200);
const plotTop = bodyTop + lineStep(font) + 4;
drawGraph(image, graph, {
x: 18,
y: plotTop,
width: width - 36,
height: Math.max(2, bodyBottom - plotTop),
});
}
private paintAnswer(image: GrayImage, width: number, bodyTop: number, bodyBottom: number): void {
const small = getDefaultSmallFont();
this.paintHeardLine(image, small, width, bodyTop);
const top = this.lastInput ? bodyTop + lineStep(small) + 4 : bodyTop;
// A short single-line answer earns the large font; anything longer drops
// to the medium font and wraps.
const lines = this.lastAnswer.split("\n");
const large = getDefaultLargeFont();
if (lines.length === 1 && large.measureText(lines[0]!) <= width - 64) {
const y = Math.round((top + bodyBottom - large.lineHeight) / 2);
this.drawCentered(image, large, y, lines[0]!, 245);
return;
}
const medium = getDefaultMediumFont();
const wrapped = lines.flatMap((line) => wrapText(medium, line, width - 64));
const step = lineStep(medium);
const maxLines = Math.max(1, Math.floor((bodyBottom - top) / step));
const shown = wrapped.slice(0, maxLines);
let y = Math.max(top, Math.round((top + bodyBottom - shown.length * step) / 2));
for (const line of shown) {
this.drawCentered(image, medium, y, line, 235);
y += step;
}
}
private paintEmpty(
image: GrayImage,
font: UiFont,
width: number,
bodyTop: number,
bodyBottom: number,
): void {
const prompt =
this.listening === "continuous"
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
: "Tap to say a problem — or use Voice input to type one";
const lines = wrapText(font, prompt, width - 96);
const step = lineStep(font);
let y = Math.round((bodyTop + bodyBottom - lines.length * step) / 2);
for (const line of lines) {
this.drawCentered(image, font, y, line, 150);
y += step;
}
}
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
if (!this.lastInput) return;
const label = "Heard";
image.drawText(font, 24, y, label, 110);
const x = 24 + font.measureText(label) + 10;
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
}
/**
* The standing offer, or the live progress of an accepted run, in the two
* rows above the footer. Returns the new body bottom.
*/
private paintLongRunPanel(
image: GrayImage,
font: UiFont,
width: number,
bodyBottom: number,
offer: LongRunOffer | null,
progress: LongRunProgress | null,
): number {
const step = lineStep(font);
const lines = progress
? [`${progressText(progress)} · ${Math.round(progress.elapsedSeconds)}s`]
: offer
? offerGlassesPrompt(offer).split("\n")
: [];
const panelTop = bodyBottom - lines.length * step - 6;
image.drawLine(16, panelTop, width - 16, panelTop, 40);
for (let index = 0; index < lines.length; index++) {
image.drawText(font, 24, panelTop + 4 + index * step, truncateText(font, lines[index]!, width - 48), 190);
}
return panelTop - 2;
}
private footerHints(): string {
if (this.coordinator.longRunProgress) {
return gestureHints([[GESTURE_DOUBLE_CLICK, "stop"]]);
}
if (this.coordinator.pendingLongRun) {
return gestureHints([
[GESTURE_CLICK, "keep going"],
[GESTURE_DOUBLE_CLICK, "leave it"],
]);
}
if (this.dictating) {
return gestureHints([
[GESTURE_CLICK, "cancel mic"],
[GESTURE_DOUBLE_CLICK, "cancel"],
]);
}
const scrollTarget =
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
return gestureHints([
[GESTURE_SCROLL, scrollTarget],
[GESTURE_CLICK, "mic"],
[GESTURE_DOUBLE_CLICK, "back"],
]);
}
private drawCentered(image: GrayImage, font: UiFont, y: number, text: string, value: number): void {
const x = Math.max(0, Math.round((image.width - font.measureText(text)) / 2));
image.drawText(font, x, y, text, value);
}
}
+20
View File
@@ -0,0 +1,20 @@
import { type AppDefinition } from "../app-definition";
import {
CALCULATOR_SURFACE_ID,
CALCULATOR_WINDOW_ID,
createCalculatorAppWindow,
} from "./calculator-app";
/**
* A spoken (or typed) problem in, an exact answer on the lens — with
* step-by-step explanations, graphs, and the advanced spoken-question
* topics (primality, integrals, digit searches in π, geometry).
*/
const calculatorApp: AppDefinition = {
appId: "calculator",
title: "Calculator",
icon: "calculator",
launch: (ctx) => ctx.launchInProcessApp(CALCULATOR_WINDOW_ID, CALCULATOR_SURFACE_ID, createCalculatorAppWindow),
};
export default calculatorApp;
+796
View File
@@ -0,0 +1,796 @@
// Routes a spoken question to the right advanced-maths topic. Separate from
// the command parser because these questions have a different shape: a
// command is about an EXPRESSION and what to do with it, whereas "is 91
// prime", "factor 360", "area of a circle with radius 3" name a topic and
// carry their own arguments. Trying to force both through one parser made
// the expression grammar swallow phrases it had no business touching.
import { type MathExpression, variablesOf } from "./expression";
import { parseSpoken } from "./parser";
import { applyNumberWords } from "./normalizer";
import { plain } from "./text-renderer";
import { describeNumber, formatInt, fromDecimal } from "./math-number";
import {
type MathConstantName,
MAXIMUM_COMPUTED_DIGITS,
constantDisplayName,
constantNamed,
constantSpokenName,
} from "./constant-digits";
import {
countDigits as countDigitsIn,
digitsAt,
expectedFirstPosition,
findDigits,
highlightedMatch,
} from "./digit-search";
import { type MathWorkload, longRunOffer as makeLongRunOffer, type LongRunOffer } from "./workload";
import {
binomial,
factorizationText,
fibonacci,
greatestCommonDivisor,
integerReport,
isPrime,
leastCommonMultiple,
modularInverse,
} from "./number-theory";
import {
type Shape,
pythagoras,
shapeName,
shapeSummary,
triangleAnglesInDegrees,
triangleArea,
triangleFromSides,
triangleKind,
trianglePerimeter,
} from "./geometry";
import { TRIG_IDENTITIES, familyText, identityMatching, solveTrig } from "./trigonometry";
import { criticalPointText, criticalPoints, limit, limitText, taylor } from "./calculus";
import { definite, integrateWithConstant } from "./symbolic-integral";
// ---------------------------------------------------------------------------
// Topics
export type AdvancedTopic =
| { kind: "integerReport"; value: number }
| { kind: "primality"; value: number }
| { kind: "factorize"; value: number }
| { kind: "gcd"; a: number; b: number }
| { kind: "lcm"; a: number; b: number }
| { kind: "modularInverse"; value: number; modulus: number }
| { kind: "fibonacci"; n: number }
| { kind: "binomial"; n: number; k: number }
| { kind: "symbolicIntegral"; expression: MathExpression; variable: string }
| { kind: "definiteIntegral"; expression: MathExpression; variable: string; from: number; to: number }
| { kind: "limit"; expression: MathExpression; variable: string; at: number }
| { kind: "taylor"; expression: MathExpression; variable: string; order: number }
| { kind: "criticalPoints"; expression: MathExpression; variable: string }
| { kind: "trigSolve"; fn: "sin" | "cos" | "tan"; value: number }
| { kind: "trigIdentity"; query: string }
/** "where does 11 first appear in pi" */
| { kind: "digitSearch"; pattern: string; constant: MathConstantName; occurrence: number }
/** "how many times does 42 appear in pi" */
| { kind: "digitCount"; pattern: string; constant: MathConstantName }
/** "what are digits 100 to 110 of pi" */
| { kind: "digitsAt"; constant: MathConstantName; from: number; to: number }
| { kind: "shapeArea"; shape: Shape }
| { kind: "triangleFromSides"; a: number; b: number; c: number }
| { kind: "pythagoras"; a: number | null; b: number | null; c: number | null };
export type AdvancedAnswer = {
headline: string;
detail: string;
/** Extra rows for the answer view, and for a longer readback. */
fields: [string, string][];
/**
* Set when this answer is only as good as the fast expansion allowed, and
* a deeper opted-in run would do better. The router itself never starts
* one: it is pure and synchronous, and the wait is a decision belonging to
* the caller.
*/
unresolvedWorkload: MathWorkload | null;
};
// ---------------------------------------------------------------------------
// Parsing
/**
* null when the utterance is not an advanced-maths question, so the caller
* can fall through to the ordinary expression path.
*/
export function parseAdvanced(utterance: string): AdvancedTopic | null {
const text = applyNumberWords(utterance.toLowerCase()).replace(/^[ .,!?]+|[ .,!?]+$/g, "");
// Constants first: "the first 11 in pi" contains digits and the word "in",
// which the geometry and number-theory matchers would both take a swing at.
const constantTopic = parseConstantDigits(text);
if (constantTopic) return constantTopic;
const numberTopic = parseNumberTheory(text);
if (numberTopic) return numberTopic;
const calculusTopic = parseCalculus(text, utterance);
if (calculusTopic) return calculusTopic;
const trigTopic = parseTrigonometry(text);
if (trigTopic) return trigTopic;
return parseGeometry(text);
}
// Constant digits ------------------------------------------------------------
function parseConstantDigits(text: string): AdvancedTopic | null {
// Must name a constant AND talk about its expansion, or "area of a circle"
// (which contains pi implicitly) would land here.
const constant = constantNamed(text);
if (!constant) return null;
const mentionsDigits =
text.includes("digit") ||
text.includes("appear") ||
text.includes("occur") ||
text.includes("within") ||
text.includes("inside") ||
text.includes("expansion") ||
text.includes("decimal place") ||
text.includes("position") ||
text.includes("show up") ||
text.includes("found in");
if (!mentionsDigits) return null;
const numbers = allInts(text);
// "digits 100 to 110 of pi"
if (
text.includes("digit") &&
text.includes(" to ") &&
numbers.length >= 2 &&
!text.includes("appear") &&
!text.includes("occur")
) {
const sorted = [numbers[0]!, numbers[1]!].sort((a, b) => a - b);
return { kind: "digitsAt", constant, from: sorted[0]!, to: sorted[1]! };
}
// "how many times does 42 appear"
if (text.includes("how many") || text.includes("count")) {
const pattern = numbers[0];
if (pattern === undefined) return null;
return { kind: "digitCount", pattern: String(pattern), constant };
}
const pattern = numbers[0];
if (pattern === undefined) return null;
// "the third time 11 appears"
const occurrence = ordinalIn(text) ?? 1;
return { kind: "digitSearch", pattern: String(pattern), constant, occurrence };
}
/**
* "3rd", not "3th". Cheap to get right and jarring to get wrong on a line
* someone reads aloud.
*/
export function ordinalWord(value: number): string {
switch (value) {
case 1:
return "first";
case 2:
return "second";
case 3:
return "third";
default:
break;
}
let suffix: string;
const mod100 = value % 100;
const mod10 = value % 10;
if (mod100 === 11 || mod100 === 12 || mod100 === 13) suffix = "th";
else if (mod10 === 1) suffix = "st";
else if (mod10 === 2) suffix = "nd";
else if (mod10 === 3) suffix = "rd";
else suffix = "th";
return `${value}${suffix}`;
}
function ordinalIn(text: string): number | null {
const words: [string, number][] = [
["first", 1],
["second", 2],
["third", 3],
["fourth", 4],
["fifth", 5],
["sixth", 6],
["seventh", 7],
["eighth", 8],
["ninth", 9],
["tenth", 10],
];
for (const [word, value] of words) {
if (text.includes(word)) return value;
}
return null;
}
// Number theory --------------------------------------------------------------
function parseNumberTheory(text: string): AdvancedTopic | null {
if (text.includes("prime") && !text.includes("factor")) {
const value = firstInt(text);
if (value !== null) return { kind: "primality", value };
}
if (text.includes("factor")) {
const value = firstInt(text);
if (value !== null) return { kind: "factorize", value };
}
if (text.includes("gcd") || text.includes("greatest common") || text.includes("highest common")) {
const values = allInts(text);
if (values.length >= 2) return { kind: "gcd", a: values[0]!, b: values[1]! };
}
if (
text.includes("lcm") ||
text.includes("lowest common multiple") ||
text.includes("least common multiple")
) {
const values = allInts(text);
if (values.length >= 2) return { kind: "lcm", a: values[0]!, b: values[1]! };
}
if (text.includes("inverse") && text.includes("mod")) {
const values = allInts(text);
if (values.length >= 2) return { kind: "modularInverse", value: values[0]!, modulus: values[1]! };
}
if (text.includes("fibonacci")) {
const value = firstInt(text);
if (value !== null) return { kind: "fibonacci", n: value };
}
if (text.includes("choose") || text.includes("binomial") || text.includes("combinations")) {
const values = allInts(text);
if (values.length >= 2) return { kind: "binomial", n: values[0]!, k: values[1]! };
}
if (text.includes("tell me about") || text.includes("facts about")) {
const value = firstInt(text);
if (value !== null) return { kind: "integerReport", value };
}
return null;
}
// Calculus -------------------------------------------------------------------
function parseCalculus(text: string, original: string): AdvancedTopic | null {
if (text.includes("integral") || text.includes("integrate") || text.includes("antiderivative")) {
const expression = expressionTail(original, ["integral of", "integrate", "antiderivative of", "integral"]);
if (!expression) return null;
const variable = variablesOf(expression)[0] ?? "x";
const bounds = boundsIn(text);
if (bounds) {
return { kind: "definiteIntegral", expression, variable, from: bounds[0], to: bounds[1] };
}
return { kind: "symbolicIntegral", expression, variable };
}
if (text.includes("limit")) {
const expression = expressionTail(original, ["limit of", "the limit of", "limit"]);
if (!expression) return null;
// "as x approaches 0"
const point = firstDoubleAfter(["approaches", "approaching", "tends to", "goes to"], text) ?? 0;
return { kind: "limit", expression, variable: variablesOf(expression)[0] ?? "x", at: point };
}
if (text.includes("taylor") || text.includes("maclaurin") || text.includes("series")) {
const expression = expressionTail(original, [
"series of",
"series for",
"expansion of",
"taylor",
"maclaurin",
]);
if (!expression) return null;
const orderValue = firstDoubleAfter(["order", "to order", "terms"], text);
const order = orderValue !== null ? Math.trunc(orderValue) : 5;
return { kind: "taylor", expression, variable: variablesOf(expression)[0] ?? "x", order };
}
if (
text.includes("critical point") ||
text.includes("turning point") ||
text.includes("maximum") ||
text.includes("minimum") ||
text.includes("extrema")
) {
const expression = expressionTail(original, [
"critical points of",
"turning points of",
"extrema of",
"maximum of",
"minimum of",
"of",
]);
if (!expression) return null;
return { kind: "criticalPoints", expression, variable: variablesOf(expression)[0] ?? "x" };
}
return null;
}
// Trigonometry ---------------------------------------------------------------
function parseTrigonometry(text: string): AdvancedTopic | null {
if (text.includes("identity") || text.includes("law of")) {
return { kind: "trigIdentity", query: text };
}
// "solve sin x = 0.5"
for (const name of ["sin", "cos", "tan"] as const) {
if (!text.includes(name)) continue;
if (!text.includes("=") && !text.includes("equals")) continue;
const value = lastDouble(text);
if (value === null) continue;
return { kind: "trigSolve", fn: name, value };
}
return null;
}
// Geometry -------------------------------------------------------------------
function parseGeometry(text: string): AdvancedTopic | null {
const numbers = allDoubles(text);
if (text.includes("pythagoras") || text.includes("hypotenuse")) {
if (numbers.length < 2) return null;
return text.includes("hypotenuse") && text.includes("find")
? { kind: "pythagoras", a: numbers[0]!, b: numbers[1]!, c: null }
: { kind: "pythagoras", a: numbers[0]!, b: null, c: numbers[1]! };
}
if (text.includes("triangle") && numbers.length >= 3 && text.includes("side")) {
return { kind: "triangleFromSides", a: numbers[0]!, b: numbers[1]!, c: numbers[2]! };
}
// Shapes, longest name first so "rectangular prism" is not read as
// "rectangle".
const shapes: [string, (values: number[]) => Shape | null][] = [
[
"rectangular prism",
(values) =>
values.length >= 3
? { kind: "rectangularPrism", length: values[0]!, width: values[1]!, height: values[2]! }
: null,
],
["sphere", (values) => (values.length >= 1 ? { kind: "sphere", radius: values[0]! } : null)],
[
"cylinder",
(values) => (values.length >= 2 ? { kind: "cylinder", radius: values[0]!, height: values[1]! } : null),
],
["cone", (values) => (values.length >= 2 ? { kind: "cone", radius: values[0]!, height: values[1]! } : null)],
["cube", (values) => (values.length >= 1 ? { kind: "cube", side: values[0]! } : null)],
["circle", (values) => (values.length >= 1 ? { kind: "circle", radius: values[0]! } : null)],
[
"rectangle",
(values) => (values.length >= 2 ? { kind: "rectangle", width: values[0]!, height: values[1]! } : null),
],
["square", (values) => (values.length >= 1 ? { kind: "square", side: values[0]! } : null)],
[
"trapezoid",
(values) =>
values.length >= 3 ? { kind: "trapezoid", a: values[0]!, b: values[1]!, height: values[2]! } : null,
],
["ellipse", (values) => (values.length >= 2 ? { kind: "ellipse", a: values[0]!, b: values[1]! } : null)],
];
const wantsMeasurement =
text.includes("area") ||
text.includes("volume") ||
text.includes("perimeter") ||
text.includes("circumference") ||
text.includes("surface");
if (!wantsMeasurement) return null;
for (const [name, build] of shapes) {
if (!text.includes(name)) continue;
const shape = build(numbers);
if (shape) return { kind: "shapeArea", shape };
}
return null;
}
// ---------------------------------------------------------------------------
// Answering
export function answerAdvanced(topic: AdvancedTopic): AdvancedAnswer {
switch (topic.kind) {
case "integerReport": {
const report = integerReport(topic.value);
const fields: [string, string][] = [
["Prime", report.isPrime ? "yes" : "no"],
["Factorisation", factorizationText(report)],
["Divisors", String(report.divisorCount)],
["Divisor sum", String(report.divisorSum)],
["Totient", String(report.totient)],
["Digit sum", String(report.digitSum)],
["Digital root", String(report.digitalRoot)],
];
if (report.isPerfect) fields.push(["Perfect", "yes"]);
return answer(
`${topic.value} = ${factorizationText(report)}`,
`${topic.value} is ${report.isPrime ? "prime" : "composite"}, ` +
`with ${report.divisorCount} divisors summing to ${report.divisorSum}.`,
fields,
);
}
case "primality": {
const prime = isPrime(topic.value);
const report = prime ? "" : factorizationText(integerReport(topic.value));
return answer(
`${topic.value} is ${prime ? "prime" : "not prime"}`,
prime
? `${topic.value} has no divisors other than 1 and itself.`
: `${topic.value} = ${report}.`,
);
}
case "factorize": {
const report = integerReport(topic.value);
return answer(
`${topic.value} = ${factorizationText(report)}`,
`${topic.value} factors into ${factorizationText(report)}.`,
);
}
case "gcd": {
const gcd = greatestCommonDivisor(topic.a, topic.b);
return answer(
`gcd(${topic.a}, ${topic.b}) = ${gcd}`,
`The greatest common divisor of ${topic.a} and ${topic.b} is ${gcd}.`,
);
}
case "lcm": {
const lcm = leastCommonMultiple(topic.a, topic.b);
return answer(
`lcm(${topic.a}, ${topic.b}) = ${lcm}`,
`The least common multiple of ${topic.a} and ${topic.b} is ${lcm}.`,
);
}
case "modularInverse": {
const inverse = modularInverse(topic.value, topic.modulus);
if (inverse === null) {
return answer(
"No inverse",
`${topic.value} and ${topic.modulus} share a factor, so no modular inverse exists.`,
);
}
return answer(
`${topic.value}⁻¹ ≡ ${inverse} (mod ${topic.modulus})`,
`${topic.value} × ${inverse} ≡ 1 (mod ${topic.modulus}).`,
);
}
case "fibonacci": {
const value = fibonacci(topic.n);
if (value === null) {
return answer("Too large", `F(${topic.n}) does not fit in an exact integer.`);
}
return answer(`F(${topic.n}) = ${value}`, `The ${topic.n}th Fibonacci number is ${value}.`);
}
case "binomial": {
const value = binomial(topic.n, topic.k);
if (value === null) {
return answer("Undefined", `${topic.n} choose ${topic.k} is not defined.`);
}
return answer(
`C(${topic.n}, ${topic.k}) = ${value}`,
`There are ${value} ways to choose ${topic.k} from ${topic.n}.`,
);
}
case "symbolicIntegral": {
const antiderivative = integrateWithConstant(topic.expression, topic.variable);
if (!antiderivative) {
return answer(
"No closed form",
"This has no elementary antiderivative that this build can find. " +
"Ask for a definite integral and it will be computed numerically.",
);
}
return answer(
`∫ = ${plain(antiderivative)}`,
`∫ ${plain(topic.expression)} d${topic.variable} = ${plain(antiderivative)}`,
);
}
case "definiteIntegral": {
const result = definite(topic.expression, topic.variable, topic.from, topic.to);
switch (result.kind) {
case "exact":
return answer(
`∫ = ${describeNumber(result.value)}`,
`Using F = ${plain(result.antiderivative)}, the definite integral is ${describeNumber(result.value)}.`,
);
case "numeric":
return answer(
`∫ ≈ ${describeNumber(fromDecimal(result.value))}`,
"No closed form was found, so this was integrated numerically.",
);
case "failed":
return answer("Could not integrate", result.reason);
}
break;
}
case "limit": {
const result = limit(topic.expression, topic.variable, topic.at);
let detail =
`As ${topic.variable} → ${format(topic.at)}, ` +
`${plain(topic.expression)} → ${limitText(result)}.`;
if (result.kind === "doesNotExist") {
detail =
`The one-sided limits disagree ` +
`(${format(result.left ?? 0)} from the left, ${format(result.right ?? 0)} from the right), ` +
`so the limit does not exist.`;
}
return answer(`limit = ${limitText(result)}`, detail);
}
case "taylor": {
const series = taylor(topic.expression, topic.variable, topic.order);
if (!series) {
return answer("No series", "Could not expand that about 0.");
}
return answer(
plain(series),
`${plain(topic.expression)} ≈ ${plain(series)} to order ${topic.order}.`,
);
}
case "criticalPoints": {
const points = criticalPoints(topic.expression, topic.variable);
if (points.length === 0) {
return answer("No critical points", "The derivative never vanishes in the search window.");
}
return answer(
points
.slice(0, 2)
.map(criticalPointText)
.join(", "),
points.map(criticalPointText).join("; "),
points.map((point) => [
point.kind.charAt(0).toUpperCase() + point.kind.slice(1),
`(${format(point.x)}, ${format(point.y)})`,
]),
);
}
case "trigSolve": {
const solution = solveTrig(topic.fn, topic.value);
switch (solution.kind) {
case "none":
return answer("No solution", `${topic.fn} never reaches ${format(topic.value)}.`);
case "all":
return answer("True for every angle", "Any value works.");
case "families": {
const text = solution.families.map(familyText).join(", ");
return answer(
`x = ${text}`,
`${topic.fn} x = ${format(topic.value)} for x = ${text}, where n is any integer.`,
solution.families.map((family, index) => [`Family ${index + 1}`, familyText(family)]),
);
}
}
break;
}
case "trigIdentity": {
const identity = identityMatching(topic.query);
if (!identity) {
return answer(
"Not in the sheet",
"Known identities: " + TRIG_IDENTITIES.map((entry) => entry.name).join(", "),
);
}
return answer(identity.plain, `${identity.name}: ${identity.plain}`);
}
case "digitSearch": {
const outcome = findDigits(topic.pattern, topic.constant, topic.occurrence);
switch (outcome.kind) {
case "invalidPattern":
return answer("Cannot search", outcome.reason);
case "notFound": {
// "Not in pi" and "not in the first 10,000 digits of pi" are very
// different claims, and only the second one is true.
let detail =
`${topic.pattern} does not appear in the first ` +
`${formatInt(outcome.searchedDigits)} digits of ${constantSpokenName(topic.constant)}.`;
const expected = expectedFirstPosition(topic.pattern.length);
if (expected !== null && expected > outcome.searchedDigits) {
const article = [8, 11, 18].includes(topic.pattern.length) ? "An" : "A";
detail +=
` ${article} ${topic.pattern.length}-digit run turns up around the ` +
`${formatInt(expected)}th place on average, so this needs a deeper expansion.`;
}
return {
headline: `Not in the first ${formatInt(outcome.searchedDigits)} digits`,
detail,
fields: [],
unresolvedWorkload: {
kind: "digitSearch",
constant: topic.constant,
pattern: topic.pattern,
occurrence: topic.occurrence,
},
};
}
case "found": {
const match = outcome.match;
const ordinalText = ordinalWord(topic.occurrence);
const places =
match.start === match.end ? `place ${match.start}` : `places ${match.start}–${match.end}`;
return answer(
`${constantDisplayName(topic.constant)}: ${places}`,
`The ${ordinalText} appearance of ${topic.pattern} in ${constantSpokenName(topic.constant)} ` +
`is at decimal ${places}, in …${highlightedMatch(match)}…`,
[
["Start", String(match.start)],
["End", String(match.end)],
["Context", highlightedMatch(match)],
],
);
}
}
break;
}
case "digitCount": {
const counted = countDigitsIn(topic.pattern, topic.constant);
return {
headline: `${counted.occurrences} time${counted.occurrences === 1 ? "" : "s"}`,
detail:
`${topic.pattern} appears ${counted.occurrences} time${counted.occurrences === 1 ? "" : "s"} ` +
`in the first ${formatInt(counted.searchedDigits)} digits of ${constantSpokenName(topic.constant)}.`,
fields: [["Searched", `${formatInt(counted.searchedDigits)} digits`]],
unresolvedWorkload: { kind: "digitCount", constant: topic.constant, pattern: topic.pattern },
};
}
case "digitsAt": {
if (topic.to - topic.from >= 200) {
return answer("Too many digits", "Ask for a span under 200 digits.");
}
const digits = digitsAt(topic.constant, topic.from, topic.to);
if (!digits) {
return {
headline: "Out of range",
detail:
`That is past the ${formatInt(MAXIMUM_COMPUTED_DIGITS)} ` +
`digits computed on device.`,
fields: [],
unresolvedWorkload: { kind: "constantDigits", constant: topic.constant, count: topic.to },
};
}
return answer(
digits,
`Decimal places ${topic.from}–${topic.to} of ${constantSpokenName(topic.constant)} are ${digits}.`,
[["Places", `${topic.from}–${topic.to}`]],
);
}
case "shapeArea": {
const parts = shapeSummary(topic.shape);
if (parts.length === 0) {
return answer("Not enough measurements", "That shape needs more values.");
}
return answer(
parts.map(([label, value]) => `${label} ${format(value)}`).join(", "),
`For that ${shapeName(topic.shape)}: ` +
parts.map(([label, value]) => `${label.toLowerCase()} ${format(value)}`).join(", ") +
".",
parts.map(([label, value]) => [label, format(value)]),
);
}
case "triangleFromSides": {
const result = triangleFromSides(topic.a, topic.b, topic.c);
switch (result.kind) {
case "impossible":
return answer("No such triangle", result.reason);
case "one":
case "two": {
const triangle = result.kind === "one" ? result.triangle : result.first;
const [angleA, angleB, angleC] = triangleAnglesInDegrees(triangle);
return answer(
`Area ${format(triangleArea(triangle))}, ${triangleKind(triangle)}`,
`Angles ${format(angleA)}°, ${format(angleB)}°, ${format(angleC)}°; ` +
`area ${format(triangleArea(triangle))}; perimeter ${format(trianglePerimeter(triangle))}.`,
[
["Angle A", `${format(angleA)}°`],
["Angle B", `${format(angleB)}°`],
["Angle C", `${format(angleC)}°`],
["Area", format(triangleArea(triangle))],
["Perimeter", format(trianglePerimeter(triangle))],
["Type", triangleKind(triangle)],
],
);
}
}
break;
}
case "pythagoras": {
const missing = pythagoras(topic.a, topic.b, topic.c);
if (missing === null) {
return answer(
"Cannot solve",
"Give two of the three sides, and a leg must be shorter than the hypotenuse.",
);
}
return answer(format(missing), `The missing side is ${format(missing)}.`);
}
}
}
function answer(headline: string, detail: string, fields: [string, string][] = []): AdvancedAnswer {
return { headline, detail, fields, unresolvedWorkload: null };
}
// ---------------------------------------------------------------------------
// Extraction helpers
export function firstInt(text: string): number | null {
return allInts(text)[0] ?? null;
}
export function allInts(text: string): number[] {
return (text.match(/-?\d+/g) ?? []).map(Number).filter(Number.isSafeInteger);
}
export function allDoubles(text: string): number[] {
return (text.match(/-?\d+(?:\.\d+)?/g) ?? []).map(Number).filter(Number.isFinite);
}
export function lastDouble(text: string): number | null {
const values = allDoubles(text);
return values.length > 0 ? values[values.length - 1]! : null;
}
function firstDoubleAfter(keywords: string[], text: string): number | null {
for (const keyword of keywords) {
const index = text.indexOf(keyword);
if (index < 0) continue;
const tail = text.slice(index + keyword.length);
const value = allDoubles(tail)[0];
if (value !== undefined) return value;
}
return null;
}
/** "from 0 to 5" */
export function boundsIn(text: string): [number, number] | null {
const match = text.match(/from\s+(-?[0-9.]+)\s+to\s+(-?[0-9.]+)/);
if (!match) return null;
const lower = Number(match[1]);
const upper = Number(match[2]);
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
return [lower, upper];
}
/** Take everything after the topic keyword and parse it as an expression. */
function expressionTail(original: string, keywords: string[]): MathExpression | null {
const lowered = original.toLowerCase();
for (const keyword of [...keywords].sort((a, b) => b.length - a.length)) {
const index = lowered.indexOf(keyword);
if (index < 0) continue;
let tail = original.slice(index + keyword.length);
// Drop trailing clauses that belong to the command, not the maths.
for (const terminator of [" from ", " as ", " with respect to ", " to order ", " dx"]) {
const cut = tail.toLowerCase().indexOf(terminator);
if (cut >= 0) tail = tail.slice(0, cut);
}
const trimmed = tail.replace(/^[ .,?]+|[ .,?]+$/g, "");
if (!trimmed) continue;
try {
return parseSpoken(trimmed);
} catch {
continue;
}
}
return null;
}
function format(value: number): string {
return describeNumber(fromDecimal(value));
}
+359
View File
@@ -0,0 +1,359 @@
// Limits, Taylor series, and curve analysis.
import {
type MathExpression,
addExpr,
divideExpr,
multiplyExpr,
num,
numberValue,
powerExpr,
subtractExpr,
variable as variableExpr,
ONE,
ZERO,
} from "./expression";
import { describeNumber, divide as divideNumber, fromDecimal, integer, isNegative, isOne, negated } from "./math-number";
import { evaluateValue } from "./evaluator";
import { differentiate } from "./derivative";
import { simplify } from "./simplifier";
import { Polynomial } from "./polynomial";
import { numericRoots as polynomialNumericRoots } from "./solver";
// ---------------------------------------------------------------------------
// Limits
export type LimitApproach = "both" | "fromLeft" | "fromRight";
export type LimitResult =
| { kind: "value"; value: number }
| { kind: "positiveInfinity" }
| { kind: "negativeInfinity" }
/** One-sided limits disagree, so the two-sided limit does not exist. */
| { kind: "doesNotExist"; left: number | null; right: number | null }
| { kind: "indeterminate" };
export function limitText(result: LimitResult): string {
switch (result.kind) {
case "value":
return describeNumber(fromDecimal(result.value));
case "positiveInfinity":
return "+∞";
case "negativeInfinity":
return "−∞";
case "doesNotExist":
return "does not exist";
case "indeterminate":
return "indeterminate";
}
}
/**
* Limit of `expression` as `variable` → `point`.
*
* Direct substitution first, then L'Hôpital for 0/0 and ∞/∞, then a numeric
* squeeze. The order matters: substitution is exact when it works, L'Hôpital
* is exact when it applies, and only the leftovers get the approximate
* treatment.
*/
export function limit(
expression: MathExpression,
variable: string,
point: number,
approach: LimitApproach = "both",
): LimitResult {
if (approach === "both" && Number.isFinite(point)) {
const direct = evaluateValue(expression, { [variable]: point });
if (direct !== null && Number.isFinite(direct)) {
return { kind: "value", value: direct };
}
}
const hospital = lHopital(expression, variable, point, 0);
if (hospital) return hospital;
return numericLimit(expression, variable, point, approach);
}
/** Detect an indeterminate quotient and differentiate top and bottom. */
function lHopital(
expression: MathExpression,
variable: string,
point: number,
depth: number,
): LimitResult | null {
// Bounded: a quotient that stays indeterminate forever (sin(1/x)/x) must
// give up rather than recurse without end.
if (depth >= 8 || !Number.isFinite(point)) return null;
const quotient = asQuotient(expression);
if (!quotient) return null;
const [numerator, denominator] = quotient;
const top = evaluateValue(numerator, { [variable]: point });
const bottom = evaluateValue(denominator, { [variable]: point });
if (top === null || bottom === null) return null;
const isZeroOverZero = Math.abs(top) < 1e-12 && Math.abs(bottom) < 1e-12;
const isInfinityOverInfinity = !Number.isFinite(top) && !Number.isFinite(bottom);
if (!isZeroOverZero && !isInfinityOverInfinity) return null;
const dTop = differentiate(numerator, variable);
const dBottom = differentiate(denominator, variable);
// A vanishing derivative of the denominator means the rule does not apply
// again; fall through to the numeric path.
const bottomValue = evaluateValue(dBottom, { [variable]: point });
if (bottomValue === null || !Number.isFinite(bottomValue)) return null;
if (Math.abs(bottomValue) > 1e-12) {
const topValue = evaluateValue(dTop, { [variable]: point });
if (topValue !== null && Number.isFinite(topValue)) {
return { kind: "value", value: topValue / bottomValue };
}
}
return lHopital(divideExpr(dTop, dBottom), variable, point, depth + 1);
}
/** Recognise `a / b`, which the tree stores as `a · b^-1`. */
export function asQuotient(expression: MathExpression): [MathExpression, MathExpression] | null {
if (expression.type !== "multiply") return null;
const numerator: MathExpression[] = [];
const denominator: MathExpression[] = [];
for (const factor of expression.factors) {
if (factor.type === "power") {
const power = numberValue(factor.exponent);
if (power && isNegative(power)) {
const positive = negated(power);
denominator.push(isOne(positive) ? factor.base : powerExpr(factor.base, num(positive)));
continue;
}
}
numerator.push(factor);
}
if (denominator.length === 0) return null;
return [
numerator.length === 0 ? ONE : numerator.length === 1 ? numerator[0]! : multiplyExpr(numerator),
denominator.length === 1 ? denominator[0]! : multiplyExpr(denominator),
];
}
function numericLimit(
expression: MathExpression,
variable: string,
point: number,
approach: LimitApproach,
): LimitResult {
const sampleSide = (side: LimitApproach): number | null => {
if (!Number.isFinite(point)) {
// At infinity, march outward instead of shrinking a step.
const sign = point > 0 ? 1 : -1;
let last: number | null = null;
for (const magnitude of [1e3, 1e4, 1e5, 1e6, 1e7]) {
const value = evaluateValue(expression, { [variable]: sign * magnitude });
if (value === null || !Number.isFinite(value)) return last;
last = value;
}
return last;
}
const samples: number[] = [];
for (const step of [1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7]) {
const offset = side === "fromLeft" ? -step : step;
const value = evaluateValue(expression, { [variable]: point + offset });
if (value === null || !Number.isFinite(value)) continue;
samples.push(value);
}
const last = samples[samples.length - 1];
if (last === undefined) return null;
// Divergence is a TREND, not a threshold. 1/x reaches only 1e7 at the
// smallest step sampled here, so a fixed "> 1e12" cutoff calls it finite;
// magnitudes growing by an order per step do not.
if (samples.length >= 3) {
const tail = samples.slice(-3);
const magnitudes = tail.map((value) => Math.abs(value));
const growing = magnitudes.every(
(value, index) => index === 0 || value > magnitudes[index - 1]! * 5,
);
if (growing && (magnitudes[magnitudes.length - 1] ?? 0) > 1e4) {
return last > 0 ? Infinity : -Infinity;
}
}
return last;
};
switch (approach) {
case "fromLeft": {
const value = sampleSide("fromLeft");
return value === null ? { kind: "indeterminate" } : classify(value);
}
case "fromRight": {
const value = sampleSide("fromRight");
return value === null ? { kind: "indeterminate" } : classify(value);
}
case "both": {
const left = sampleSide("fromLeft");
const right = sampleSide("fromRight");
if (left === null || right === null) return { kind: "indeterminate" };
// Scale-relative comparison: two values near 1e6 differing by 1 are the
// same limit; two near 0 differing by 1 are not.
const scale = Math.max(Math.abs(left), Math.abs(right), 1);
if (Math.abs(left - right) >= 1e-4 * scale) {
return { kind: "doesNotExist", left, right };
}
return classify((left + right) / 2);
}
}
}
function classify(value: number): LimitResult {
if (!Number.isFinite(value)) {
return value > 0 ? { kind: "positiveInfinity" } : { kind: "negativeInfinity" };
}
if (value > 1e12) return { kind: "positiveInfinity" };
if (value < -1e12) return { kind: "negativeInfinity" };
return { kind: "value", value };
}
// ---------------------------------------------------------------------------
// Series
/** Taylor expansion about `centre`, to `order` terms. */
export function taylor(
expression: MathExpression,
variable: string,
order = 5,
centre = 0,
): MathExpression | null {
if (order < 0 || order > 12) return null;
const terms: MathExpression[] = [];
let derivative = expression;
let factorial = 1;
for (let degree = 0; degree <= order; degree++) {
if (degree > 0) {
derivative = differentiate(derivative, variable);
factorial *= degree;
}
const coefficient = evaluateValue(derivative, { [variable]: centre });
if (coefficient === null || !Number.isFinite(coefficient)) return null;
if (Math.abs(coefficient) <= 1e-12) continue;
const scaled = divideNumber(fromDecimal(coefficient), integer(factorial));
const shifted: MathExpression =
centre === 0
? variableExpr(variable)
: subtractExpr(variableExpr(variable), num(fromDecimal(centre)));
const power: MathExpression =
degree === 0 ? ONE : degree === 1 ? shifted : powerExpr(shifted, num(integer(degree)));
terms.push(degree === 0 ? num(scaled) : multiplyExpr([num(scaled), power]));
}
if (terms.length === 0) return ZERO;
return simplify(addExpr(terms));
}
// ---------------------------------------------------------------------------
// Curve analysis
export type CriticalPointKind = "minimum" | "maximum" | "inflection" | "undetermined";
export type CriticalPoint = {
x: number;
y: number;
kind: CriticalPointKind;
};
export function criticalPointText(point: CriticalPoint): string {
const xText = describeNumber(fromDecimal(point.x));
const yText = describeNumber(fromDecimal(point.y));
return `${point.kind} at (${xText}, ${yText})`;
}
/**
* Critical points classified by the second derivative, falling back to a
* sign test where the second derivative vanishes.
*/
export function criticalPoints(
expression: MathExpression,
variable: string,
range: [number, number] = [-50, 50],
): CriticalPoint[] {
const first = differentiate(expression, variable);
const second = differentiate(first, variable);
let roots: number[];
const polynomial = Polynomial.extract(first, variable);
if (polynomial) {
roots = polynomialNumericRoots(polynomial, range, 20_000);
} else {
roots = numericRootsOfExpression(first, variable, range);
}
const points: CriticalPoint[] = [];
for (const x of roots) {
const y = evaluateValue(expression, { [variable]: x });
if (y === null || !Number.isFinite(y)) continue;
const curvature = evaluateValue(second, { [variable]: x }) ?? 0;
let kind: CriticalPointKind;
if (curvature > 1e-9) {
kind = "minimum";
} else if (curvature < -1e-9) {
kind = "maximum";
} else {
// Second-derivative test is inconclusive; look at how the first
// derivative behaves either side.
const left = evaluateValue(first, { [variable]: x - 1e-4 }) ?? 0;
const right = evaluateValue(first, { [variable]: x + 1e-4 }) ?? 0;
if (left < 0 && right > 0) kind = "minimum";
else if (left > 0 && right < 0) kind = "maximum";
else kind = "inflection";
}
points.push({ x, y, kind });
}
return points;
}
export function numericRootsOfExpression(
expression: MathExpression,
variable: string,
range: [number, number],
samples = 20_000,
): number[] {
const [lower, upper] = range;
const roots: number[] = [];
const step = (upper - lower) / samples;
let previousX = lower;
let previousY = evaluateValue(expression, { [variable]: previousX });
for (let index = 1; index <= samples; index++) {
const x = lower + index * step;
const y = evaluateValue(expression, { [variable]: x });
const lastX = previousX;
const lastY = previousY;
previousX = x;
previousY = y;
if (y === null || lastY === null) continue;
if (Math.sign(lastY) === Math.sign(y)) continue;
if (Math.abs(y) >= 1e6 || Math.abs(lastY) >= 1e6) continue;
let low = lastX;
let high = x;
for (let iteration = 0; iteration < 60; iteration++) {
const mid = (low + high) / 2;
const value = evaluateValue(expression, { [variable]: mid });
const lowValue = evaluateValue(expression, { [variable]: low });
if (value === null || lowValue === null) break;
if (value === 0) {
low = mid;
high = mid;
break;
}
if (Math.sign(value) === Math.sign(lowValue)) {
low = mid;
} else {
high = mid;
}
}
const root = (low + high) / 2;
// Same pole guard as the main solver: verify the residual.
const residual = evaluateValue(expression, { [variable]: root });
if (residual === null || Math.abs(residual) >= 1e-6) continue;
if (!roots.some((existing) => Math.abs(existing - root) < 1e-6)) roots.push(root);
}
return roots;
}
+591
View File
@@ -0,0 +1,591 @@
// Digit expansions of the well-known constants, computed exactly. Computed
// rather than tabulated: a hardcoded string of π is a thing nobody can check,
// and a single transposed digit would silently shift every position this
// feature reports. Everything here comes out of integer arithmetic that can
// be re-derived and tested.
//
// JavaScript's native BigInt supplies the operation set the generators need
// (add, subtract, compare, multiply/divide by a small scalar).
// The deep paths are async and yield to the event loop periodically — a
// multi-minute synchronous grind would freeze the app's whole UI thread.
export type MathConstantName = "pi" | "e" | "phi" | "sqrt2" | "sqrt3" | "sqrt5";
export const ALL_CONSTANT_NAMES: readonly MathConstantName[] = ["pi", "e", "phi", "sqrt2", "sqrt3", "sqrt5"];
export function constantDisplayName(constant: MathConstantName): string {
switch (constant) {
case "pi":
return "π";
case "e":
return "e";
case "phi":
return "φ";
case "sqrt2":
return "√2";
case "sqrt3":
return "√3";
case "sqrt5":
return "√5";
}
}
export function constantSpokenName(constant: MathConstantName): string {
switch (constant) {
case "pi":
return "pi";
case "e":
return "e";
case "phi":
return "the golden ratio";
case "sqrt2":
return "the square root of 2";
case "sqrt3":
return "the square root of 3";
case "sqrt5":
return "the square root of 5";
}
}
/** Digits before the decimal point — 1 for all of these. */
export const CONSTANT_INTEGER_PART_DIGITS = 1;
export function constantNamed(text: string): MathConstantName | null {
const lowered = text.toLowerCase();
if (lowered.includes("golden") || lowered.includes("phi")) return "phi";
if (lowered.includes("root of 2") || lowered.includes("root two") || lowered.includes("sqrt2")) {
return "sqrt2";
}
if (lowered.includes("root of 3") || lowered.includes("root three")) return "sqrt3";
if (lowered.includes("root of 5") || lowered.includes("root five")) return "sqrt5";
// Word-boundary matched, not substring: `includes("pi")` also fires on
// "pie", "spin", and "copying", and `=== "e"` only ever matched a
// single-character utterance, so "the digits of e" never resolved.
if (lowered.includes("π") || hasWord("pi", lowered)) return "pi";
if (hasWord("e", lowered) || lowered.includes("euler")) return "e";
return null;
}
function hasWord(word: string, text: string): boolean {
return new RegExp(`\\b${word}\\b`).test(text);
}
// ---------------------------------------------------------------------------
// Cancellation and progress
/** Cooperative cancellation for the long paths. */
export type ComputeCancellation = () => boolean;
export const NEVER_CANCELLED: ComputeCancellation = () => false;
/** Raised when a long run is cancelled partway. */
export class ComputeCancelled extends Error {
constructor() {
super("Cancelled");
this.name = "ComputeCancelled";
}
}
export type ComputeProgress = (fraction: number, stage: string) => void;
// setTimeout via globalThis so this module needs no platform lib: it runs
// under NativeScript's globals in the app and under Node in the tests.
const scheduleMacrotask: (callback: () => void) => void = (callback) =>
(globalThis as unknown as { setTimeout: (fn: () => void, ms: number) => unknown }).setTimeout(callback, 0);
/**
* Yield to the event loop when the current slice has run long enough. The
* generators call this between series terms so a deep run stays cancellable
* and the rest of the app keeps painting.
*/
class Yielder {
private lastYieldAt = Date.now();
async maybeYield(): Promise<void> {
if (Date.now() - this.lastYieldAt < 50) return;
await new Promise<void>((resolve) => scheduleMacrotask(resolve));
this.lastYieldAt = Date.now();
}
}
// ---------------------------------------------------------------------------
// Limits
/**
* How far the on-device generator will go for an ordinary question.
*
* Machin's formula is O(n²) in the digit count with scalar-only big
* arithmetic. 10,000 digits is well under a second and finds any two- or
* three-digit run plus most four-digit ones; past this, the wearer opts into
* a long run explicitly.
*/
export const MAXIMUM_COMPUTED_DIGITS = 10_000;
/**
* The furthest an opted-in long run will go, however much time the wearer
* offers. Not a time limit — the caller has already accepted the wait — but
* a memory-and-honesty one: the O(n²) cost puts this at hours on a phone,
* past which nobody is still waiting.
*/
export const MAXIMUM_FEASIBLE_DIGITS = 2_000_000;
// ---------------------------------------------------------------------------
// Cache
/**
* The same constant is asked for repeatedly within one session (search, then
* context, then a second search), and recomputing 10,000 digits each time
* would be pointlessly slow. Grow-only.
*/
const cache = new Map<MathConstantName, string>();
function cachedDigits(constant: MathConstantName, atLeast: number): string | null {
const cached = cache.get(constant);
if (!cached || cached.length < atLeast) return null;
return cached;
}
function storeDigits(constant: MathConstantName, digits: string): void {
const existing = cache.get(constant);
if (existing && existing.length >= digits.length) return;
cache.set(constant, digits);
}
// ---------------------------------------------------------------------------
// Public entry points
/**
* Fractional digits only — the part after the decimal point, which is what
* "position N of π" universally means. Clamped to MAXIMUM_COMPUTED_DIGITS;
* the deeper run is a separate call the wearer has to opt into, so no
* ordinary caller can accidentally block the app for minutes.
*/
export function fractionalDigits(constant: MathConstantName, count: number): string | null {
const requested = Math.min(Math.max(count, 1), MAXIMUM_COMPUTED_DIGITS);
const cached = cachedDigits(constant, requested);
if (cached) return cached.slice(0, requested);
// The shallow path runs synchronously; at 10k digits it is fast enough that
// an async hop would only complicate every caller.
const digits = computeSync(constant, requested);
if (digits === null) return null;
storeDigits(constant, digits);
return digits.slice(0, requested);
}
/**
* The opted-in deep run. Cancellable, and reports progress, because a
* multi-minute grind with no way out and nothing on screen is
* indistinguishable from a hang.
*/
export async function deepFractionalDigits(
constant: MathConstantName,
count: number,
cancellation: ComputeCancellation = NEVER_CANCELLED,
progress: ComputeProgress | null = null,
): Promise<string | null> {
const requested = Math.min(Math.max(count, 1), MAXIMUM_FEASIBLE_DIGITS);
const cached = cachedDigits(constant, requested);
if (cached) return cached.slice(0, requested);
const digits = await compute(constant, requested, cancellation, progress, new Yielder());
if (digits === null) return null;
storeDigits(constant, digits);
return digits.slice(0, requested);
}
function computeSync(constant: MathConstantName, requested: number): string | null {
// A resolved-synchronously wrapper would still be async; instead run the
// shared generator with a yielder that never fires (50ms slices cannot
// elapse mid-loop when the whole computation is sub-second) by executing
// the promise eagerly. The generators only await through Yielder.maybeYield,
// which resolves immediately below the time threshold, so the promise
// settles synchronously enough — but rather than rely on that, the shallow
// path uses the dedicated synchronous variants.
return computeCore(constant, requested, NEVER_CANCELLED, null, null);
}
async function compute(
constant: MathConstantName,
requested: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<string | null> {
return computeCoreAsync(constant, requested, cancellation, progress, yielder);
}
// ---------------------------------------------------------------------------
// Core generators
//
// Each exists in a synchronous and an asynchronous variant sharing the same
// arithmetic. The async ones interleave event-loop yields; the sync ones are
// used only under MAXIMUM_COMPUTED_DIGITS where the total cost is small.
/**
* Guard digits absorb the truncation error the series accumulates in its
* least significant places; without them the last handful of reported digits
* are wrong, which for a position-reporting feature is the whole ballgame.
*/
const GUARD_DIGITS = 20;
function computeCore(
constant: MathConstantName,
requested: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
_yielder: null,
): string | null {
const scale = requested + GUARD_DIGITS;
let scaled: bigint;
switch (constant) {
case "pi":
scaled = machinPiSync(scale, cancellation, progress);
break;
case "e":
scaled = eulerESync(scale, cancellation, progress);
break;
case "phi": {
// φ = (1 + √5) / 2
const root = integerSquareRootSync(5, scale, cancellation, progress);
scaled = (root + powerOfTen(scale)) / 2n;
break;
}
case "sqrt2":
scaled = integerSquareRootSync(2, scale, cancellation, progress);
break;
case "sqrt3":
scaled = integerSquareRootSync(3, scale, cancellation, progress);
break;
case "sqrt5":
scaled = integerSquareRootSync(5, scale, cancellation, progress);
break;
}
return finishDigits(scaled, requested);
}
async function computeCoreAsync(
constant: MathConstantName,
requested: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<string | null> {
const scale = requested + GUARD_DIGITS;
let scaled: bigint;
switch (constant) {
case "pi":
scaled = await machinPi(scale, cancellation, progress, yielder);
break;
case "e":
scaled = await eulerE(scale, cancellation, progress, yielder);
break;
case "phi": {
const root = await integerSquareRoot(5, scale, cancellation, progress, yielder);
scaled = (root + powerOfTen(scale)) / 2n;
break;
}
case "sqrt2":
scaled = await integerSquareRoot(2, scale, cancellation, progress, yielder);
break;
case "sqrt3":
scaled = await integerSquareRoot(3, scale, cancellation, progress, yielder);
break;
case "sqrt5":
scaled = await integerSquareRoot(5, scale, cancellation, progress, yielder);
break;
}
return finishDigits(scaled, requested);
}
/**
* The value is the constant × 10^scale. Strip the integer part, then the
* guard digits.
*/
function finishDigits(scaled: bigint, requested: number): string | null {
const text = scaled.toString();
if (text.length <= CONSTANT_INTEGER_PART_DIGITS) return null;
const fractional = text.slice(CONSTANT_INTEGER_PART_DIGITS);
if (fractional.length < requested) return null;
const trimmed = fractional.slice(0, fractional.length - GUARD_DIGITS);
return trimmed.slice(0, requested);
}
export function powerOfTen(exponent: number): bigint {
return 10n ** BigInt(Math.max(0, exponent));
}
// ---------------------------------------------------------------------------
// π — Machin's formula: π = 16·arctan(1/5) − 4·arctan(1/239).
//
// Chosen over Chudnovsky because it needs only scalar big-integer operations
// — no big×big multiply, no big÷big division, no square roots of big values.
// arctan(1/5) needs log10(239)/log10(5) ≈ 3.4× as many terms as arctan(1/239)
// at the same scale, and each term costs the same, so the first phase is
// ~77% of the work. Splitting the reported fraction that way keeps the
// progress bar honest instead of stalling at half.
const MACHIN_FIRST_SHARE = 0.77;
function machinPiSync(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
): bigint {
const first =
arctanReciprocalSync(5, scale, cancellation, (fraction) =>
progress?.(fraction * MACHIN_FIRST_SHARE, "π series 1 of 2"),
) * 16n;
const second =
arctanReciprocalSync(239, scale, cancellation, (fraction) =>
progress?.(MACHIN_FIRST_SHARE + fraction * (1 - MACHIN_FIRST_SHARE), "π series 2 of 2"),
) * 4n;
return first - second;
}
async function machinPi(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<bigint> {
const first =
(await arctanReciprocal(
5,
scale,
cancellation,
(fraction) => progress?.(fraction * MACHIN_FIRST_SHARE, "π series 1 of 2"),
yielder,
)) * 16n;
const second =
(await arctanReciprocal(
239,
scale,
cancellation,
(fraction) => progress?.(MACHIN_FIRST_SHARE + fraction * (1 - MACHIN_FIRST_SHARE), "π series 2 of 2"),
yielder,
)) * 4n;
return first - second;
}
/** arctan(1/x), scaled by 10^scale, by its alternating Taylor series. */
function arctanReciprocalSync(
x: number,
scale: number,
cancellation: ComputeCancellation,
progress: ((fraction: number) => void) | null,
): bigint {
const bigX = BigInt(x);
const xSquared = bigX * bigX;
let power = powerOfTen(scale) / bigX;
let sum = power;
let divisor = 3n;
let subtractNext = true;
// Terms shrink by a factor of x² each step, so the series dies after about
// scale / (2·log10 x) of them. Known up front, which is what makes a real
// fraction possible rather than a spinner.
const expectedTerms = Math.max(1, scale / (2 * Math.log10(x)));
let iterations = 0;
while (power !== 0n) {
// Checked every term: one closure call is nothing beside a big-integer
// division, and the alternative is an uncancellable multi-minute loop.
if (cancellation()) throw new ComputeCancelled();
power /= xSquared;
const term = power / divisor;
if (term === 0n) break;
sum = subtractNext ? sum - term : sum + term;
subtractNext = !subtractNext;
divisor += 2n;
iterations += 1;
// Throttled: a million progress callbacks would cost more than the
// arithmetic they are reporting on.
if (progress && iterations % 256 === 0) {
progress(Math.min(1, iterations / expectedTerms));
}
}
return sum;
}
async function arctanReciprocal(
x: number,
scale: number,
cancellation: ComputeCancellation,
progress: ((fraction: number) => void) | null,
yielder: Yielder,
): Promise<bigint> {
const bigX = BigInt(x);
const xSquared = bigX * bigX;
let power = powerOfTen(scale) / bigX;
let sum = power;
let divisor = 3n;
let subtractNext = true;
const expectedTerms = Math.max(1, scale / (2 * Math.log10(x)));
let iterations = 0;
while (power !== 0n) {
if (cancellation()) throw new ComputeCancelled();
power /= xSquared;
const term = power / divisor;
if (term === 0n) break;
sum = subtractNext ? sum - term : sum + term;
subtractNext = !subtractNext;
divisor += 2n;
iterations += 1;
if (progress && iterations % 256 === 0) {
progress(Math.min(1, iterations / expectedTerms));
}
if (iterations % 64 === 0) await yielder.maybeYield();
}
return sum;
}
// ---------------------------------------------------------------------------
// e = Σ 1/k!, scaled. Each term is the previous divided by k, so the whole
// series needs only scalar division.
function eulerESync(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
): bigint {
let term = powerOfTen(scale);
let sum = term;
let k = 1n;
// The term loses log10(k) digits each step; the series ends when the
// accumulated loss passes `scale`. Tracking that sum gives a fraction
// without needing Stirling's approximation.
let digitsShed = 0;
let iterations = 0;
while (term !== 0n) {
if (cancellation()) throw new ComputeCancelled();
term /= k;
sum += term;
digitsShed += Math.log10(Number(k));
k += 1n;
iterations += 1;
if (progress && iterations % 256 === 0) {
progress(Math.min(1, digitsShed / Math.max(scale, 1)), "e series");
}
// 10^scale / k! underflows long before this, but a runaway loop is not an
// acceptable failure mode.
if (k > 1_000_000n) break;
}
return sum;
}
async function eulerE(
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<bigint> {
let term = powerOfTen(scale);
let sum = term;
let k = 1n;
let digitsShed = 0;
let iterations = 0;
while (term !== 0n) {
if (cancellation()) throw new ComputeCancelled();
term /= k;
sum += term;
digitsShed += Math.log10(Number(k));
k += 1n;
iterations += 1;
if (progress && iterations % 256 === 0) {
progress(Math.min(1, digitsShed / Math.max(scale, 1)), "e series");
}
if (iterations % 64 === 0) await yielder.maybeYield();
if (k > 1_000_000n) break;
}
return sum;
}
// ---------------------------------------------------------------------------
// Square roots — ⌊√(value · 10^(2·scale))⌋, i.e. √value scaled by 10^scale.
//
// Digit-by-digit ("long division") extraction, which needs only add,
// subtract, compare, and multiply-by-scalar.
function integerSquareRootSync(
value: number,
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
): bigint {
// Work in pairs of decimal digits, which is what the algorithm consumes.
const totalPairs = scale + 1;
let remainder = 0n;
let root = 0n;
for (let pairIndex = 0; pairIndex < totalPairs; pairIndex++) {
if (cancellation()) throw new ComputeCancelled();
if (progress && pairIndex % 256 === 0) {
progress(pairIndex / totalPairs, "root extraction");
}
// Bring down the next two digits: the first pair is the integer part,
// every later pair is zero because the radicand's fractional expansion is
// all zeros.
const bringDown = pairIndex === 0 ? BigInt(value) : 0n;
remainder = remainder * 100n + bringDown;
// Find the largest d in 0...9 with (20·root + d)·d ≤ remainder.
const twentyRoot = root * 20n;
let chosen = 0n;
let chosenProduct = 0n;
for (let digit = 9n; digit >= 1n; digit--) {
const candidate = (twentyRoot + digit) * digit;
if (candidate <= remainder) {
chosen = digit;
chosenProduct = candidate;
break;
}
}
remainder -= chosenProduct;
root = root * 10n + chosen;
}
return root;
}
async function integerSquareRoot(
value: number,
scale: number,
cancellation: ComputeCancellation,
progress: ComputeProgress | null,
yielder: Yielder,
): Promise<bigint> {
const totalPairs = scale + 1;
let remainder = 0n;
let root = 0n;
for (let pairIndex = 0; pairIndex < totalPairs; pairIndex++) {
if (cancellation()) throw new ComputeCancelled();
if (progress && pairIndex % 256 === 0) {
progress(pairIndex / totalPairs, "root extraction");
}
const bringDown = pairIndex === 0 ? BigInt(value) : 0n;
remainder = remainder * 100n + bringDown;
const twentyRoot = root * 20n;
let chosen = 0n;
let chosenProduct = 0n;
for (let digit = 9n; digit >= 1n; digit--) {
const candidate = (twentyRoot + digit) * digit;
if (candidate <= remainder) {
chosen = digit;
chosenProduct = candidate;
break;
}
}
remainder -= chosenProduct;
root = root * 10n + chosen;
if (pairIndex % 64 === 0) await yielder.maybeYield();
}
return root;
}
+602
View File
@@ -0,0 +1,602 @@
// Owns the one live maths problem and everything derived from it. Shared
// state rather than per-request because the whole point of "graph it" and
// "explain it" is that they refer to the problem already on the lens; a
// caller that re-parsed its own capture each time would make those
// follow-ups impossible.
//
// Deep digit work runs on the phone as an explicit, cancellable long run
// the wearer opts into. The
// optional AI pass is injected as a hook so this module stays free of any
// networking and testable in plain Node.
import { type MathConstantName, ComputeCancelled, constantDisplayName, constantNamed } from "./constant-digits";
import { highlightedMatch } from "./digit-search";
import { formatInt } from "./math-number";
import {
type LongRunOffer,
type LongRunProgress,
type MathWorkload,
compactDuration,
longRunOffer as makeLongRunOffer,
runAcceptedCount,
runAcceptedDigits,
runAcceptedSearch,
} from "./workload";
import { type MathSessionResult, MathSession, parseCommand } from "./session";
import { type Graph } from "./plotter";
import { type SolutionStep } from "./solver";
import { plain } from "./text-renderer";
// ---------------------------------------------------------------------------
// Branch outcome
/**
* A branch result. The value/failure split matters: a parse error must never
* be posted onward as if it were the answer.
*/
export type MathBranchOutcome =
| { kind: "value"; text: string }
| { kind: "failure"; reason: string };
export type MathBranchMode = "solve" | "explain" | "graph" | "derivative" | "digitSearch" | "digitCount";
/**
* The app's uniform gesture contract: forward/back step through the
* working, commit
* graphs, alternate explains, exit clears.
*/
export type RingAction = "forward" | "back" | "commit" | "alternate" | "exit";
/**
* Ask the configured assistant to render a spoken problem as an expression
* (the prompt text lives in app/prompts.ts with every other prompt).
* Injected by the app; null when no provider is configured. Resolves to the
* model's raw reply, or null on failure — the coordinator sanitises it.
*/
export type AICompletion = (spokenText: string) => Promise<string | null>;
export class MathCoordinator {
/** The last thing produced, for the answer view. */
lastResult: MathSessionResult | null = null;
lastGraph: Graph | null = null;
/** Position in the working, for the step indicator. */
stepPosition: { index: number; total: number } | null = null;
/**
* A standing offer to grind the full-depth answer out on the phone.
* Non-null means every surface should be offering it.
*/
pendingLongRun: LongRunOffer | null = null;
/** Live progress while an accepted run is going. Null when nothing is running. */
longRunProgress: LongRunProgress | null = null;
/**
* The answer a deeper pass produced. One channel, because from the
* wearer's side it is one event: the better answer arrived.
*/
deepAnswer: MathBranchOutcome | null = null;
/** Repaint hook, set by the owning layer. */
onChanged: (() => void) | null = null;
private readonly session = new MathSession();
private completeWithAI: AICompletion | null = null;
private longRunCancelled = false;
private longRunRunning = false;
private longRunStartedAtMs = 0;
get hasProblem(): boolean {
return this.session.hasProblem;
}
get currentSolution() {
return this.session.currentSolution;
}
get explanationSteps(): SolutionStep[] {
return this.session.currentSolution?.steps ?? [];
}
get isRunningLong(): boolean {
return this.longRunRunning;
}
setUsesDegrees(value: boolean): void {
this.session.setUsesDegrees(value);
}
setAICompletion(completion: AICompletion | null): void {
this.completeWithAI = completion;
}
private changed(): void {
this.onChanged?.();
}
// -------------------------------------------------------------------------
// Branch execution
/**
* Run one branch. Returns the glasses text, or a failure string — the
* caller distinguishes them.
*/
run(mode: MathBranchMode, input: string): MathBranchOutcome {
const trimmed = input.trim();
let sessionResult: MathSessionResult;
switch (mode) {
case "solve":
// "Solve it" after Graph/Explain means show the standing answer
// again. Requiring fresh input here made the third follow-up mode
// uniquely fail.
if (!trimmed && this.session.hasProblem) {
sessionResult = this.session.execute({ kind: "restate" });
} else if (!trimmed) {
return { kind: "failure", reason: "Nothing was captured to solve" };
} else {
sessionResult = this.session.solve(trimmed);
}
break;
case "graph":
// A graph branch may fire with a fresh capture ("graph x squared") or
// as a follow-up on the standing problem ("graph it"). Both have to
// work from the same branch.
if (trimmed && (parseCommand(trimmed) === null || this.isProblem(trimmed))) {
this.session.solve(trimmed);
}
sessionResult = this.session.graph();
break;
case "explain":
if (trimmed && this.isProblem(trimmed)) this.session.solve(trimmed);
sessionResult = this.session.explain();
break;
case "derivative":
if (trimmed && this.isProblem(trimmed)) this.session.solve(trimmed);
sessionResult = this.session.differentiate(null);
break;
case "digitSearch":
case "digitCount": {
const digits = MathCoordinator.digitRun(trimmed);
if (!digits) return { kind: "failure", reason: "No digits were captured to look for" };
// The constant is read from the capture when the wearer named one
// ("…in the golden ratio") and is π otherwise.
const constant: MathConstantName = constantNamed(trimmed) ?? "pi";
sessionResult = this.session.execute({
kind: "advanced",
topic:
mode === "digitCount"
? { kind: "digitCount", pattern: digits, constant }
: { kind: "digitSearch", pattern: digits, constant, occurrence: 1 },
});
break;
}
}
return this.publish(sessionResult);
}
/**
* Every digit in the capture, in order, with the separators dropped.
*
* Joined rather than "the first run" because ASR punctuates a spoken date
* freely — "07 04 84", "070484", "7/4/84" are all the same six digits —
* and taking the first run would search π for "07", which appears within
* the first dozen places and looks for all the world like a working
* answer.
*/
static digitRun(text: string): string | null {
const joined = text.replace(/\D/g, "");
return joined ? joined : null;
}
private isProblem(text: string): boolean {
return parseCommand(text)?.kind === "problem";
}
// -------------------------------------------------------------------------
// Spoken questions
/**
* Answer a question stated in words, whatever shape it arrived in.
*
* Two passes, in this order and never the other way round:
*
* 1. The deterministic engine alone. The spoken normaliser already turns
* "two x squared plus three x" into `2x²+3x`, and every answer it gives
* is exact.
* 2. Only if that fails to parse, ask the model to REWRITE the words as an
* expression — and then solve that expression here, deterministically.
*
* The model never computes anything. It is used as a translator from
* English into notation, which is the part it is genuinely good at and the
* part where a mistake is visible: the rewritten expression is returned
* alongside the answer, so a wrong reading shows up as a wrong-looking
* equation rather than as a plausible wrong number.
*/
async answerSpoken(text: string): Promise<MathBranchOutcome> {
const trimmed = text.trim();
if (!trimmed) return { kind: "failure", reason: "Nothing was said to calculate" };
const command = parseCommand(trimmed);
if (command) {
const outcome = this.publish(this.session.execute(command));
if (outcome.kind === "value") return outcome;
// A FOLLOW-UP that failed keeps its own message. "Graph it" with
// nothing on the lens fails because there is nothing to graph, and
// handing those two words to a rewriter would replace that with "could
// not read that as a calculation" — true of the words, and useless
// about the actual problem.
if (command.kind !== "problem") return outcome;
}
const rewritten = await this.expressionFromAI(trimmed);
if (!rewritten) {
return { kind: "failure", reason: "Could not read that as a calculation" };
}
const outcome = this.publish(this.session.solve(rewritten));
if (outcome.kind !== "value") return outcome;
// Show the reading, not just the result. A wearer glancing at the lens
// has to be able to see that the machine understood the right question
// before they repeat the number.
return { kind: "value", text: `${rewritten}\n${outcome.text}` };
}
/**
* Ask the configured assistant to render a spoken problem as an
* expression. Returns null when no provider is configured, when it
* declines, or when it answers with anything that is not one.
*/
private async expressionFromAI(text: string): Promise<string | null> {
if (!this.completeWithAI) return null;
let raw: string | null;
try {
raw = await this.completeWithAI(text);
} catch {
return null;
}
if (!raw) return null;
let candidate = raw.trim();
// Models fence notation out of habit even when told not to.
if (candidate.startsWith("```")) {
candidate = candidate
.split("\n")
.filter((line) => !line.startsWith("```"))
.join(" ")
.trim();
}
candidate = candidate.replace(/^[ `.,]+|[ `.,]+$/g, "");
if (!candidate || candidate.toLowerCase() === "none") return null;
// A refusal or an apology is prose, and prose fed to the parser fails in
// a way that reads like the maths was wrong rather than the routing.
if (candidate.length > 120) return null;
if (!/[+\-*/^=\d]/.test(candidate)) return null;
return candidate;
}
// -------------------------------------------------------------------------
// Voice follow-ups
/**
* Handle a spoken follow-up. Returns null when the utterance is not a
* maths command, so ordinary conversation flows past untouched — this is
* called from the transcript stream, which sees everything said in the
* room in continuous mode.
*/
handleUtterance(utterance: string): MathBranchOutcome | null {
// A standing offer or a run in flight owns the next yes/no said in the
// room. Checked before the maths parser, because "keep going" is not an
// equation and would otherwise fall straight through.
const longRunAnswer = this.handleLongRunUtterance(utterance);
if (longRunAnswer) return longRunAnswer;
const command = parseCommand(utterance);
if (!command) return null;
// A bare problem with no standing session is a new problem; a follow-up
// with no session is a no-op rather than an error card in mid-air.
if (command.kind !== "problem" && command.kind !== "advanced" && !this.session.hasProblem) {
return null;
}
return this.publish(this.session.execute(command));
}
/**
* Accept or decline a long run by voice. Only fires while an offer stands
* or a run is going, so these phrases stay out of the way of ordinary
* conversation the rest of the time.
*/
handleLongRunUtterance(utterance: string): MathBranchOutcome | null {
const text = utterance.toLowerCase();
if (this.longRunRunning) {
if (!STOP_PHRASES.some((phrase) => text.includes(phrase))) return null;
this.cancelLongRun();
return { kind: "failure", reason: "Stopping" };
}
const offer = this.pendingLongRun;
if (!offer) return null;
if (ACCEPT_PHRASES.some((phrase) => text.includes(phrase))) {
void this.acceptLongRun();
return { kind: "value", text: `Working — ${compactDuration(offer.estimatedSeconds)}` };
}
if (DECLINE_PHRASES.some((phrase) => text.includes(phrase))) {
this.declineLongRun();
return { kind: "failure", reason: "Left it there" };
}
return null;
}
get hasLongRunInteraction(): boolean {
return this.pendingLongRun !== null || this.longRunRunning;
}
// -------------------------------------------------------------------------
// Uniform gesture contract
handleRingAction(action: RingAction): MathBranchOutcome | null {
// The offer and the running job both pre-empt the step navigation: while
// either is on screen, the input is answering that question and nothing
// else. Same gestures, no new vocabulary.
if (this.longRunRunning) {
if (action !== "exit") return null;
this.cancelLongRun();
return { kind: "failure", reason: "Stopping" };
}
const offer = this.pendingLongRun;
if (offer) {
switch (action) {
case "commit":
void this.acceptLongRun();
return { kind: "value", text: `Working — ${compactDuration(offer.estimatedSeconds)}` };
case "exit":
this.declineLongRun();
return { kind: "failure", reason: "Left it there" };
default:
return null;
}
}
if (!this.session.hasProblem) return null;
switch (action) {
case "forward":
return this.publish(this.session.execute({ kind: "nextStep" }));
case "back":
return this.publish(this.session.execute({ kind: "previousStep" }));
case "commit":
return this.publish(this.session.graph());
case "alternate":
return this.publish(this.session.explain());
case "exit":
return this.publish(this.session.execute({ kind: "clear" }));
}
}
// -------------------------------------------------------------------------
// Long runs
/** Take the standing offer. Returns false when there was nothing to accept. */
acceptLongRun(): boolean {
const offer = this.pendingLongRun;
if (!offer || this.longRunRunning) return false;
this.pendingLongRun = null;
this.deepAnswer = null;
this.longRunCancelled = false;
this.longRunRunning = true;
this.longRunStartedAtMs = Date.now();
this.longRunProgress = { fraction: 0, stage: "starting", elapsedSeconds: 0 };
this.changed();
const cancellation = () => this.longRunCancelled;
const progress = (fraction: number, stage: string) => {
this.longRunProgress = {
fraction,
stage,
elapsedSeconds: (Date.now() - this.longRunStartedAtMs) / 1000,
};
this.changed();
};
void (async () => {
let outcome: MathBranchOutcome;
try {
switch (offer.workload.kind) {
case "digitSearch": {
const found = await runAcceptedSearch(offer, cancellation, progress);
outcome = describeDeepSearch(found, offer.workload.pattern, offer.workload.constant);
break;
}
case "constantDigits": {
const digits = await runAcceptedDigits(offer, cancellation, progress);
outcome = digits
? {
kind: "value",
text: `${constantDisplayName(offer.workload.constant)}: ${formatInt(digits.length)} digits ready`,
}
: { kind: "failure", reason: "Could not generate that many digits" };
break;
}
case "digitCount": {
const counted = await runAcceptedCount(offer, cancellation, progress);
outcome = {
kind: "value",
text: describeCount(counted.occurrences, counted.searchedDigits, offer.workload.pattern, offer.workload.constant),
};
break;
}
}
} catch (error) {
outcome =
error instanceof ComputeCancelled
? { kind: "failure", reason: "Stopped" }
: { kind: "failure", reason: String((error as Error)?.message ?? error) };
}
this.longRunRunning = false;
this.longRunProgress = null;
this.deepAnswer = outcome;
this.changed();
})();
return true;
}
/** Turn the offer down. The shallow answer already given stands. */
declineLongRun(): void {
this.pendingLongRun = null;
this.changed();
}
/**
* Stop a run in flight. The digits computed so far are discarded — a
* partial expansion cannot answer a position question.
*/
cancelLongRun(): void {
this.longRunCancelled = true;
}
// -------------------------------------------------------------------------
// Publishing
private publish(sessionResult: MathSessionResult): MathBranchOutcome {
this.lastResult = sessionResult;
// The shallow answer goes out immediately; anything better arrives on
// `deepAnswer` if the wearer accepts the standing offer. Never block the
// first response on the second — silence is worse than a partial.
if (sessionResult.unresolvedWorkload) {
this.pendingLongRun = makeLongRunOffer(
sessionResult.unresolvedWorkload,
"Needs a deeper expansion than the fast pass holds",
shallowDescription(sessionResult.unresolvedWorkload, sessionResult),
);
this.deepAnswer = null;
}
switch (sessionResult.payload.kind) {
case "graph":
this.lastGraph = sessionResult.payload.graph;
this.stepPosition = null;
break;
case "step":
this.lastGraph = null;
this.stepPosition = { index: sessionResult.payload.index, total: sessionResult.payload.total };
break;
case "solution":
case "message":
case "topic":
this.lastGraph = null;
// An advanced topic answers in one shot — there is no working to step
// through, so the step indicator must clear.
this.stepPosition = null;
break;
case "failure":
break;
}
this.changed();
if (sessionResult.payload.kind === "failure") {
return { kind: "failure", reason: sessionResult.payload.message };
}
return { kind: "value", text: this.glassesText(sessionResult) };
}
/**
* Two or three short rows. The lens has no room for the detail sentence,
* which is why `detail` stays in the fuller answer view.
*/
private glassesText(sessionResult: MathSessionResult): string {
switch (sessionResult.payload.kind) {
case "step": {
const payload = sessionResult.payload;
return `${payload.index + 1}/${payload.total} ${payload.step.title}\n${plain(payload.step.expression)}`;
}
case "graph":
// The bitmap carries the picture; this is the text fallback.
return sessionResult.headline;
case "topic": {
// Two field rows at most: the window is short and the headline
// already took a line.
const payload = sessionResult.payload;
return [payload.headline, ...payload.fields.slice(0, 2)].join("\n");
}
default:
return sessionResult.headline;
}
}
}
// ---------------------------------------------------------------------------
// Long-run phrases
const ACCEPT_PHRASES = [
"keep going",
"run it anyway",
"run it any way",
"take your time",
"go deeper",
"keep looking",
"do it anyway",
"yes go on",
];
const DECLINE_PHRASES = [
"never mind",
"nevermind",
"forget it",
"leave it",
"don't bother",
"do not bother",
"skip it",
];
const STOP_PHRASES = ["stop", "cancel", "give up", "that's enough", "thats enough"];
// ---------------------------------------------------------------------------
// Descriptions
function describeDeepSearch(
outcome: Awaited<ReturnType<typeof runAcceptedSearch>>,
pattern: string,
constant: MathConstantName,
): MathBranchOutcome {
switch (outcome.kind) {
case "found":
return {
kind: "value",
text:
`${constantDisplayName(constant)}: places ${outcome.match.start}–${outcome.match.end}\n` +
`…${highlightedMatch(outcome.match)}…`,
};
case "notFound":
// Now a much stronger claim than the shallow pass could make, so it is
// worth stating the depth that was actually reached.
return {
kind: "failure",
reason:
`${pattern} is not in the first ${formatInt(outcome.searchedDigits)} digits ` +
`of ${constantDisplayName(constant)}`,
};
case "invalidPattern":
return { kind: "failure", reason: outcome.reason };
}
}
function describeCount(
occurrences: number,
searchedDigits: number,
pattern: string,
constant: MathConstantName,
): string {
const times = occurrences === 1 ? "time" : "times";
return `${pattern}: ${occurrences} ${times}\nfirst ${formatInt(searchedDigits)} digits of ${constantDisplayName(constant)}`;
}
function shallowDescription(workload: MathWorkload, sessionResult: MathSessionResult): string | null {
switch (workload.kind) {
case "digitSearch":
return sessionResult.headline;
case "digitCount":
return sessionResult.headline;
case "constantDigits":
return null;
}
}
+234
View File
@@ -0,0 +1,234 @@
// Symbolic differentiation. Every rule is applied structurally and the result
// is handed to the simplifier, so `d/dx (x^2)` comes back as `2x` rather than
// `2 · x^(2-1) · 1`. That matters because the derivative is user-visible — it
// is what "explain it" shows.
import {
type MathExpression,
type MathFunction,
ONE,
ZERO,
addExpr,
divideExpr,
fn,
int,
multiplyExpr,
negate,
powerExpr,
relation,
subtractExpr,
variablesOf,
} from "./expression";
import { simplify } from "./simplifier";
import { evaluateValue } from "./evaluator";
export function differentiate(expression: MathExpression, withRespectTo: string): MathExpression {
return simplify(raw(expression, withRespectTo));
}
/** Nth derivative. */
export function differentiateOrder(
expression: MathExpression,
withRespectTo: string,
order: number,
): MathExpression {
let current = expression;
for (let step = 0; step < Math.max(0, order); step++) {
current = differentiate(current, withRespectTo);
}
return current;
}
function raw(expression: MathExpression, variable: string): MathExpression {
switch (expression.type) {
case "number":
case "constant":
return ZERO;
case "variable":
return expression.name === variable ? ONE : ZERO;
case "negate":
return negate(raw(expression.inner, variable));
case "add":
return addExpr(expression.terms.map((term) => raw(term, variable)));
case "multiply": {
// Product rule, generalised: d(abc) = a'bc + ab'c + abc'.
const terms: MathExpression[] = [];
for (let index = 0; index < expression.factors.length; index++) {
const replaced = [...expression.factors];
replaced[index] = raw(expression.factors[index]!, variable);
terms.push(multiplyExpr(replaced));
}
return addExpr(terms);
}
case "power":
return differentiatePower(expression.base, expression.exponent, variable);
case "function": {
const argument = expression.args[0];
if (!argument) return ZERO;
const outer = derivativeOfFunction(expression.fn, argument);
// Chain rule. Skipping this when the argument is the bare variable
// would be a needless special case — the simplifier removes the `· 1`
// anyway.
return multiplyExpr([outer, raw(argument, variable)]);
}
case "relation":
// Differentiating both sides of an equation is implicit
// differentiation's first step; useful, and harmless here.
return relation(expression.relation, raw(expression.lhs, variable), raw(expression.rhs, variable));
}
}
function differentiatePower(
base: MathExpression,
exponent: MathExpression,
variable: string,
): MathExpression {
const baseHasVariable = variablesOf(base).includes(variable);
const exponentHasVariable = variablesOf(exponent).includes(variable);
if (!baseHasVariable && !exponentHasVariable) return ZERO;
if (baseHasVariable && !exponentHasVariable) {
// Power rule: d(u^n) = n · u^(n-1) · u'
const reduced = subtractExpr(exponent, ONE);
return multiplyExpr([exponent, powerExpr(base, reduced), raw(base, variable)]);
}
if (!baseHasVariable && exponentHasVariable) {
// Exponential rule: d(a^v) = a^v · ln(a) · v'
return multiplyExpr([powerExpr(base, exponent), fn("ln", [base]), raw(exponent, variable)]);
}
// General case via logarithmic differentiation:
// d(u^v) = u^v · (v' · ln u + v · u'/u)
const uv = powerExpr(base, exponent);
const firstTerm = multiplyExpr([raw(exponent, variable), fn("ln", [base])]);
const secondTerm = multiplyExpr([exponent, divideExpr(raw(base, variable), base)]);
return multiplyExpr([uv, addExpr([firstTerm, secondTerm])]);
}
/** The outer derivative only; the caller applies the chain rule. */
function derivativeOfFunction(name: MathFunction, argument: MathExpression): MathExpression {
switch (name) {
case "sin":
return fn("cos", [argument]);
case "cos":
return negate(fn("sin", [argument]));
case "tan":
// sec² = 1 / cos²
return divideExpr(ONE, powerExpr(fn("cos", [argument]), int(2)));
case "asin":
return divideExpr(ONE, fn("sqrt", [subtractExpr(ONE, powerExpr(argument, int(2)))]));
case "acos":
return negate(divideExpr(ONE, fn("sqrt", [subtractExpr(ONE, powerExpr(argument, int(2)))])));
case "atan":
return divideExpr(ONE, addExpr([ONE, powerExpr(argument, int(2))]));
case "sinh":
return fn("cosh", [argument]);
case "cosh":
return fn("sinh", [argument]);
case "tanh":
return divideExpr(ONE, powerExpr(fn("cosh", [argument]), int(2)));
case "ln":
return divideExpr(ONE, argument);
case "log10":
return divideExpr(ONE, multiplyExpr([argument, fn("ln", [int(10)])]));
case "log2":
return divideExpr(ONE, multiplyExpr([argument, fn("ln", [int(2)])]));
case "sqrt":
return divideExpr(ONE, multiplyExpr([int(2), fn("sqrt", [argument])]));
case "cbrt":
return divideExpr(ONE, multiplyExpr([int(3), powerExpr(fn("cbrt", [argument]), int(2))]));
case "exp":
return fn("exp", [argument]);
case "abs":
// d|u|/du = u/|u| — undefined at zero, which the plotter's
// discontinuity handling already copes with.
return divideExpr(argument, fn("abs", [argument]));
case "floor":
case "ceil":
case "round":
// Zero almost everywhere; the jumps are measure-zero and are not
// representable here.
return ZERO;
case "factorial":
// The derivative of the gamma function needs the digamma function,
// which this build does not carry. Returning zero would be a lie, so
// the caller gets an unevaluated marker instead.
return fn("factorial", [argument]);
}
}
// ---------------------------------------------------------------------------
// Numeric integration
/**
* Definite integral by adaptive Simpson's rule.
*
* Numeric rather than symbolic: symbolic integration needs a Risch-style
* algorithm that is far out of proportion to the feature, and a definite
* value is what a calculator user actually asks for. (The symbolic table in
* symbolic-integral.ts is tried first where it applies.)
*/
export function integrateNumeric(
expression: MathExpression,
variable: string,
lower: number,
upper: number,
tolerance = 1e-9,
): number | null {
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
if (lower === upper) return 0;
if (lower > upper) {
const flipped = integrateNumeric(expression, variable, upper, lower, tolerance);
return flipped === null ? null : -flipped;
}
const f = (x: number): number | null => evaluateValue(expression, { [variable]: x });
const simpson = (a: number, b: number, fa: number, fb: number, fm: number): number =>
((b - a) / 6) * (fa + 4 * fm + fb);
const adaptive = (
a: number,
b: number,
fa: number,
fb: number,
fm: number,
whole: number,
depth: number,
): number | null => {
const m = (a + b) / 2;
const leftMid = (a + m) / 2;
const rightMid = (m + b) / 2;
const flm = f(leftMid);
const frm = f(rightMid);
if (flm === null || frm === null) return null;
const left = simpson(a, m, fa, fm, flm);
const right = simpson(m, b, fm, fb, frm);
const total = left + right;
// Richardson: the Simpson error term is (total - whole) / 15.
if (depth <= 0 || Math.abs(total - whole) <= 15 * tolerance) {
return total + (total - whole) / 15;
}
const leftResult = adaptive(a, m, fa, fm, flm, left, depth - 1);
if (leftResult === null) return null;
const rightResult = adaptive(m, b, fm, fb, frm, right, depth - 1);
if (rightResult === null) return null;
return leftResult + rightResult;
};
const fa = f(lower);
const fb = f(upper);
const fm = f((lower + upper) / 2);
if (fa === null || fb === null || fm === null) return null;
const whole = simpson(lower, upper, fa, fb, fm);
return adaptive(lower, upper, fa, fb, fm, whole, 24);
}
+184
View File
@@ -0,0 +1,184 @@
// Finds a digit sequence inside the expansion of a constant.
//
// The position convention is fixed and stated everywhere it is reported:
// **decimal places after the point, 1-based**. So the first `11` in π is at
// 94–95, because π = 3.(1)4159… and the 94th digit after the point is the
// first `1` of that pair. Every off-by-one complaint about this kind of
// feature is a convention that was never written down, so it is written down.
import {
type MathConstantName,
MAXIMUM_COMPUTED_DIGITS,
fractionalDigits,
} from "./constant-digits";
export type DigitCountResult = {
occurrences: number;
searchedDigits: number;
};
export type DigitMatch = {
/** 1-based decimal place of the first digit of the match. */
start: number;
/** 1-based decimal place of the last digit. */
end: number;
/** The digits either side, for the reveal. */
context: string;
/** Where the match sits inside `context`, as an offset. */
contextOffset: number;
};
/** `...3421 ·11· 70679...` with the match marked. */
export function highlightedMatch(match: DigitMatch, marker = "·"): string {
const length = match.end - match.start + 1;
if (match.contextOffset < 0 || match.contextOffset + length > match.context.length) {
return match.context;
}
const before = match.context.slice(0, match.contextOffset);
const middle = match.context.slice(match.contextOffset, match.contextOffset + length);
const after = match.context.slice(match.contextOffset + length);
return `${before}${marker}${middle}${marker}${after}`;
}
export type DigitSearchOutcome =
| { kind: "found"; match: DigitMatch }
/**
* Searched the whole available expansion and did not find it. Carries how
* far the search actually reached, because "not in π" and "not in the first
* 20,000 digits of π" are very different claims and only one of them is
* true.
*/
| { kind: "notFound"; searchedDigits: number }
| { kind: "invalidPattern"; reason: string };
/** How many digits either side of a hit to return. */
export const DEFAULT_CONTEXT_RADIUS = 6;
export type DigitSource = (constant: MathConstantName, count: number) => string | null;
const DEFAULT_SOURCE: DigitSource = (constant, count) => fractionalDigits(constant, count);
/** Find the `occurrence`-th appearance of `pattern`. */
export function findDigits(
pattern: string,
constant: MathConstantName,
occurrence = 1,
requested: number = MAXIMUM_COMPUTED_DIGITS,
contextRadius: number = DEFAULT_CONTEXT_RADIUS,
digitSource: DigitSource = DEFAULT_SOURCE,
): DigitSearchOutcome {
const cleaned = pattern.replace(/\D/g, "");
if (!cleaned) return { kind: "invalidPattern", reason: "Give a sequence of digits to look for" };
if (occurrence < 1) return { kind: "invalidPattern", reason: "Occurrence must be 1 or more" };
const expansion = digitSource(constant, requested);
if (!expansion) return { kind: "notFound", searchedDigits: 0 };
return locateDigits(cleaned, expansion, occurrence, contextRadius);
}
/**
* Search an expansion the caller already has — the path an accepted long run
* uses, where the digits come from the deep generator rather than the
* shallow cache.
*/
export function locateDigits(
pattern: string,
expansion: string,
occurrence = 1,
contextRadius: number = DEFAULT_CONTEXT_RADIUS,
): DigitSearchOutcome {
if (pattern.length > expansion.length) {
return { kind: "notFound", searchedDigits: expansion.length };
}
let seen = 0;
// Plain scan rather than a substring search: overlapping matches count
// ("111" contains "11" twice), and indexOf-with-skip would miss the
// overlap.
for (let index = 0; index <= expansion.length - pattern.length; index++) {
let matched = true;
for (let offset = 0; offset < pattern.length; offset++) {
if (expansion[index + offset] !== pattern[offset]) {
matched = false;
break;
}
}
if (!matched) continue;
seen += 1;
if (seen !== occurrence) continue;
const contextStart = Math.max(0, index - contextRadius);
const contextEnd = Math.min(expansion.length, index + pattern.length + contextRadius);
return {
kind: "found",
match: {
start: index + 1,
end: index + pattern.length,
context: expansion.slice(contextStart, contextEnd),
contextOffset: index - contextStart,
},
};
}
return { kind: "notFound", searchedDigits: expansion.length };
}
/** How many times the pattern appears in the available expansion. */
export function countDigits(
pattern: string,
constant: MathConstantName,
requested: number = MAXIMUM_COMPUTED_DIGITS,
digitSource: DigitSource = DEFAULT_SOURCE,
): DigitCountResult {
const cleaned = pattern.replace(/\D/g, "");
if (!cleaned) return { occurrences: 0, searchedDigits: 0 };
const expansion = digitSource(constant, requested);
if (!expansion) return { occurrences: 0, searchedDigits: 0 };
return countInExpansion(cleaned, expansion);
}
/** Count overlapping occurrences inside an expansion the caller already has. */
export function countInExpansion(pattern: string, expansion: string): DigitCountResult {
if (pattern.length > expansion.length) {
return { occurrences: 0, searchedDigits: expansion.length };
}
let total = 0;
for (let index = 0; index <= expansion.length - pattern.length; index++) {
let matched = true;
for (let offset = 0; offset < pattern.length; offset++) {
if (expansion[index + offset] !== pattern[offset]) {
matched = false;
break;
}
}
if (matched) total += 1;
}
return { occurrences: total, searchedDigits: expansion.length };
}
/** The digits at a given range of decimal places, 1-based inclusive. */
export function digitsAt(
constant: MathConstantName,
start: number,
end: number,
digitSource: DigitSource = DEFAULT_SOURCE,
): string | null {
if (start < 1 || end < start) return null;
const expansion = digitSource(constant, end);
if (!expansion || expansion.length < end) return null;
return expansion.slice(start - 1, end);
}
/**
* How deep you would expect to search before a random `length`-digit string
* turns up: 10^length places, on average.
*
* Reported alongside a notFound so the answer is honest about whether the
* search was ever likely to succeed — an eight-digit birthday first appears
* around the hundred-millionth digit, which no on-device expansion is going
* to reach.
*/
export function expectedFirstPosition(patternLength: number): number | null {
if (patternLength <= 0 || patternLength > 18) return null;
return Math.pow(10, patternLength);
}
+189
View File
@@ -0,0 +1,189 @@
// Numeric evaluation of an expression tree. Two return paths on purpose:
// `evaluateExact` keeps rationals as rationals (so a solution can be shown as
// `3/2`), while `evaluateValue` collapses to a plain number for the plotter,
// which samples thousands of points and has no use for exactness.
import {
type MathExpression,
constantValue,
evaluateFunction,
functionIsDefined,
} from "./expression";
import {
type MathNumber,
add,
doubleValue,
integer,
integerValue,
isNegative,
isZero,
multiply,
negated,
raisedTo,
real,
} from "./math-number";
export type EvaluatorContext = {
variables: Record<string, MathNumber>;
/**
* Radians unless the caller says otherwise. Trig in degrees is a real user
* expectation for a calculator, and getting it silently wrong produces
* answers that look plausible.
*/
usesDegrees: boolean;
};
export function radiansContext(): EvaluatorContext {
return { variables: {}, usesDegrees: false };
}
export class EvaluationError extends Error {
constructor(message: string) {
super(message);
this.name = "EvaluationError";
}
}
/**
* Evaluate keeping exactness where possible. Throws on a free variable, so
* the caller can distinguish "not a number yet" from "evaluates to NaN".
*/
export function evaluateExact(expression: MathExpression, context: EvaluatorContext = radiansContext()): MathNumber {
switch (expression.type) {
case "number":
return expression.value;
case "variable": {
const value = context.variables[expression.name];
if (value === undefined) {
throw new EvaluationError(`No value given for ${expression.name}`);
}
return value;
}
case "constant":
return real(constantValue(expression.constant));
case "add":
return expression.terms.reduce<MathNumber>(
(total, term) => add(total, evaluateExact(term, context)),
integer(0),
);
case "multiply":
return expression.factors.reduce<MathNumber>(
(total, factor) => multiply(total, evaluateExact(factor, context)),
integer(1),
);
case "negate":
return negated(evaluateExact(expression.inner, context));
case "power": {
const baseValue = evaluateExact(expression.base, context);
const exponentValue = evaluateExact(expression.exponent, context);
if (isZero(baseValue) && isNegative(exponentValue)) {
throw new EvaluationError("Division by zero");
}
return raisedTo(baseValue, exponentValue);
}
case "function": {
const first = expression.args[0];
if (!first) throw new EvaluationError(`${expression.fn} needs an argument`);
const argument = evaluateExact(first, context);
// Integer factorial stays exact; everything else goes through the
// floating-point implementation.
const argumentInt = integerValue(argument);
if (expression.fn === "factorial" && argumentInt !== null && argumentInt >= 0 && argumentInt <= 20) {
let total = 1;
for (let value = 2; value <= argumentInt; value++) total *= value;
return integer(total);
}
const raw = adjustForDegrees(expression.fn, doubleValue(argument), context);
if (!functionIsDefined(expression.fn, raw)) {
throw new EvaluationError(`${expression.fn} is undefined there`);
}
const result = evaluateFunction(expression.fn, raw);
if (!Number.isFinite(result)) {
throw new EvaluationError(`${expression.fn} is undefined there`);
}
return adjustResultForDegrees(expression.fn, result, context);
}
case "relation": {
const left = doubleValue(evaluateExact(expression.lhs, context));
const right = doubleValue(evaluateExact(expression.rhs, context));
let holds: boolean;
switch (expression.relation) {
case "=":
holds = left === right;
break;
case "<":
holds = left < right;
break;
case "<=":
holds = left <= right;
break;
case ">":
holds = left > right;
break;
case ">=":
holds = left >= right;
break;
}
return integer(holds ? 1 : 0);
}
}
}
/**
* Plotting path: never throws, returns null where the function is undefined
* so the renderer can break the curve instead of drawing a line across the
* gap.
*/
export function evaluateValue(
expression: MathExpression,
variables: Record<string, number>,
usesDegrees = false,
): number | null {
const context: EvaluatorContext = { variables: {}, usesDegrees };
for (const name of Object.keys(variables)) {
context.variables[name] = real(variables[name]!);
}
let result: MathNumber;
try {
result = evaluateExact(expression, context);
} catch {
return null;
}
const value = doubleValue(result);
return Number.isFinite(value) ? value : null;
}
// ---------------------------------------------------------------------------
// Degrees
function adjustForDegrees(name: string, argument: number, context: EvaluatorContext): number {
if (!context.usesDegrees) return argument;
switch (name) {
case "sin":
case "cos":
case "tan":
return (argument * Math.PI) / 180;
default:
return argument;
}
}
function adjustResultForDegrees(name: string, result: number, context: EvaluatorContext): MathNumber {
if (!context.usesDegrees) return real(result);
switch (name) {
case "asin":
case "acos":
case "atan":
return real((result * 180) / Math.PI);
default:
return real(result);
}
}
+487
View File
@@ -0,0 +1,487 @@
// The expression tree everything else operates on: evaluation, simplification,
// differentiation, solving, rendering, and plotting. A plain discriminated
// union so the whole tree is a value — simplification and differentiation can
// rebuild subtrees freely, and structural comparison (expressionEquals) gives
// the simplifier the like-term matching it leans on heavily.
import {
type MathNumber,
integer as makeInteger,
fromDecimal,
isZero as numberIsZero,
isOne as numberIsOne,
numberEquals as numbersEqual,
} from "./math-number";
export type MathRelation = "=" | "<" | "<=" | ">" | ">=";
export type MathConstant = "pi" | "e" | "tau";
export type MathFunction =
| "sin"
| "cos"
| "tan"
| "asin"
| "acos"
| "atan"
| "sinh"
| "cosh"
| "tanh"
| "ln"
| "log10"
| "log2"
| "sqrt"
| "cbrt"
| "abs"
| "exp"
| "floor"
| "ceil"
| "round"
| "factorial";
export const ALL_FUNCTIONS: readonly MathFunction[] = [
"sin",
"cos",
"tan",
"asin",
"acos",
"atan",
"sinh",
"cosh",
"tanh",
"ln",
"log10",
"log2",
"sqrt",
"cbrt",
"abs",
"exp",
"floor",
"ceil",
"round",
"factorial",
];
export const ALL_CONSTANTS: readonly MathConstant[] = ["pi", "e", "tau"];
export type MathExpression =
| { type: "number"; value: MathNumber }
/** A free variable — `x`, `y`, `theta`. */
| { type: "variable"; name: string }
/**
* A named constant with a known value (`pi`, `e`). Kept distinct from
* `variable` so "solve for pi" is impossible and `2π` stays exact instead of
* becoming 6.283…
*/
| { type: "constant"; constant: MathConstant }
| { type: "add"; terms: MathExpression[] }
| { type: "multiply"; factors: MathExpression[] }
| { type: "power"; base: MathExpression; exponent: MathExpression }
/**
* Unary negation. Not folded into `multiply([-1, x])` because the renderers
* and the step explanations both want to say "negative x".
*/
| { type: "negate"; inner: MathExpression }
| { type: "function"; fn: MathFunction; args: MathExpression[] }
/** An equation or inequality — the root node of anything solvable. */
| { type: "relation"; relation: MathRelation; lhs: MathExpression; rhs: MathExpression };
// ---------------------------------------------------------------------------
// Convenience constructors
export function num(value: MathNumber): MathExpression {
return { type: "number", value };
}
export function int(value: number): MathExpression {
return num(makeInteger(value));
}
export function realExpr(value: number): MathExpression {
return num(fromDecimal(value));
}
export function variable(name: string): MathExpression {
return { type: "variable", name };
}
export function constant(value: MathConstant): MathExpression {
return { type: "constant", constant: value };
}
export function addExpr(terms: MathExpression[]): MathExpression {
return { type: "add", terms };
}
export function multiplyExpr(factors: MathExpression[]): MathExpression {
return { type: "multiply", factors };
}
export function powerExpr(base: MathExpression, exponent: MathExpression): MathExpression {
return { type: "power", base, exponent };
}
export function negate(inner: MathExpression): MathExpression {
return { type: "negate", inner };
}
export function fn(name: MathFunction, args: MathExpression[]): MathExpression {
return { type: "function", fn: name, args };
}
export function relation(rel: MathRelation, lhs: MathExpression, rhs: MathExpression): MathExpression {
return { type: "relation", relation: rel, lhs, rhs };
}
export function subtractExpr(lhs: MathExpression, rhs: MathExpression): MathExpression {
return addExpr([lhs, negate(rhs)]);
}
/**
* Division is `a * b^-1` in the tree, which is what makes the simplifier and
* the differentiator tractable — the renderers detect the pattern and print a
* real fraction, so nothing user-visible degrades.
*/
export function divideExpr(lhs: MathExpression, rhs: MathExpression): MathExpression {
return multiplyExpr([lhs, powerExpr(rhs, int(-1))]);
}
export const ZERO: MathExpression = int(0);
export const ONE: MathExpression = int(1);
// ---------------------------------------------------------------------------
// Introspection
export function isZeroExpr(expression: MathExpression): boolean {
return expression.type === "number" && numberIsZero(expression.value);
}
export function isOneExpr(expression: MathExpression): boolean {
return expression.type === "number" && numberIsOne(expression.value);
}
export function numberValue(expression: MathExpression): MathNumber | null {
return expression.type === "number" ? expression.value : null;
}
/**
* Every free variable in the tree, in first-appearance order. Order matters:
* it decides which variable "solve for x" defaults to and which axis a
* single-variable graph uses.
*/
export function variablesOf(expression: MathExpression): string[] {
const seen = new Set<string>();
const ordered: string[] = [];
collectVariables(expression, ordered, seen);
return ordered;
}
function collectVariables(expression: MathExpression, ordered: string[], seen: Set<string>): void {
switch (expression.type) {
case "variable":
if (!seen.has(expression.name)) {
seen.add(expression.name);
ordered.push(expression.name);
}
break;
case "number":
case "constant":
break;
case "add":
for (const term of expression.terms) collectVariables(term, ordered, seen);
break;
case "multiply":
for (const factor of expression.factors) collectVariables(factor, ordered, seen);
break;
case "power":
collectVariables(expression.base, ordered, seen);
collectVariables(expression.exponent, ordered, seen);
break;
case "negate":
collectVariables(expression.inner, ordered, seen);
break;
case "function":
for (const argument of expression.args) collectVariables(argument, ordered, seen);
break;
case "relation":
collectVariables(expression.lhs, ordered, seen);
collectVariables(expression.rhs, ordered, seen);
break;
}
}
export function isRelation(expression: MathExpression): boolean {
return expression.type === "relation";
}
/**
* Node count — used to pick the simpler of two equivalent forms and to bound
* runaway expansion in the simplifier.
*/
export function complexity(expression: MathExpression): number {
switch (expression.type) {
case "number":
case "variable":
case "constant":
return 1;
case "add":
return 1 + expression.terms.reduce((total, term) => total + complexity(term), 0);
case "multiply":
return 1 + expression.factors.reduce((total, factor) => total + complexity(factor), 0);
case "power":
return 1 + complexity(expression.base) + complexity(expression.exponent);
case "negate":
return 1 + complexity(expression.inner);
case "function":
return 1 + expression.args.reduce((total, argument) => total + complexity(argument), 0);
case "relation":
return 1 + complexity(expression.lhs) + complexity(expression.rhs);
}
}
/** Replace every occurrence of a variable with an expression. */
export function substituting(
expression: MathExpression,
name: string,
replacement: MathExpression,
): MathExpression {
switch (expression.type) {
case "variable":
return expression.name === name ? replacement : expression;
case "number":
case "constant":
return expression;
case "add":
return addExpr(expression.terms.map((term) => substituting(term, name, replacement)));
case "multiply":
return multiplyExpr(expression.factors.map((factor) => substituting(factor, name, replacement)));
case "power":
return powerExpr(
substituting(expression.base, name, replacement),
substituting(expression.exponent, name, replacement),
);
case "negate":
return negate(substituting(expression.inner, name, replacement));
case "function":
return fn(
expression.fn,
expression.args.map((argument) => substituting(argument, name, replacement)),
);
case "relation":
return relation(
expression.relation,
substituting(expression.lhs, name, replacement),
substituting(expression.rhs, name, replacement),
);
}
}
/**
* Structural equality with value-equal numbers (MathNumber compares by
* value).
*/
export function expressionEquals(a: MathExpression, b: MathExpression): boolean {
if (a === b) return true;
if (a.type !== b.type) return false;
switch (a.type) {
case "number": {
const other = b as Extract<MathExpression, { type: "number" }>;
return numbersEqual(a.value, other.value);
}
case "variable":
return a.name === (b as Extract<MathExpression, { type: "variable" }>).name;
case "constant":
return a.constant === (b as Extract<MathExpression, { type: "constant" }>).constant;
case "add": {
const other = b as Extract<MathExpression, { type: "add" }>;
return listEquals(a.terms, other.terms);
}
case "multiply": {
const other = b as Extract<MathExpression, { type: "multiply" }>;
return listEquals(a.factors, other.factors);
}
case "power": {
const other = b as Extract<MathExpression, { type: "power" }>;
return expressionEquals(a.base, other.base) && expressionEquals(a.exponent, other.exponent);
}
case "negate":
return expressionEquals(a.inner, (b as Extract<MathExpression, { type: "negate" }>).inner);
case "function": {
const other = b as Extract<MathExpression, { type: "function" }>;
return a.fn === other.fn && listEquals(a.args, other.args);
}
case "relation": {
const other = b as Extract<MathExpression, { type: "relation" }>;
return (
a.relation === other.relation &&
expressionEquals(a.lhs, other.lhs) &&
expressionEquals(a.rhs, other.rhs)
);
}
}
}
function listEquals(a: MathExpression[], b: MathExpression[]): boolean {
if (a.length !== b.length) return false;
for (let index = 0; index < a.length; index++) {
if (!expressionEquals(a[index]!, b[index]!)) return false;
}
return true;
}
// ---------------------------------------------------------------------------
// Relations
/**
* The relation you get by multiplying or dividing both sides by a negative —
* the classic inequality mistake, handled explicitly.
*/
export function flippedRelation(rel: MathRelation): MathRelation {
switch (rel) {
case "=":
return "=";
case "<":
return ">";
case "<=":
return ">=";
case ">":
return "<";
case ">=":
return "<=";
}
}
export function spokenRelation(rel: MathRelation): string {
switch (rel) {
case "=":
return "equals";
case "<":
return "is less than";
case "<=":
return "is at most";
case ">":
return "is greater than";
case ">=":
return "is at least";
}
}
// ---------------------------------------------------------------------------
// Constants
export function constantValue(value: MathConstant): number {
switch (value) {
case "pi":
return Math.PI;
case "e":
return Math.E;
case "tau":
return 2 * Math.PI;
}
}
// ---------------------------------------------------------------------------
// Functions
export function evaluateFunction(name: MathFunction, argument: number): number {
switch (name) {
case "sin":
return Math.sin(argument);
case "cos":
return Math.cos(argument);
case "tan":
return Math.tan(argument);
case "asin":
return Math.asin(argument);
case "acos":
return Math.acos(argument);
case "atan":
return Math.atan(argument);
case "sinh":
return Math.sinh(argument);
case "cosh":
return Math.cosh(argument);
case "tanh":
return Math.tanh(argument);
case "ln":
return Math.log(argument);
case "log10":
return Math.log10(argument);
case "log2":
return Math.log2(argument);
case "sqrt":
return Math.sqrt(argument);
case "cbrt":
return Math.cbrt(argument);
case "abs":
return Math.abs(argument);
case "exp":
return Math.exp(argument);
case "floor":
return Math.floor(argument);
case "ceil":
return Math.ceil(argument);
case "round":
return Math.round(argument);
case "factorial":
// Real factorial via the gamma function, so `2.5!` is defined rather
// than an error — and so the evaluator has one code path for both.
return gamma(argument + 1);
}
}
/**
* Domain check used by the plotter to break a curve rather than draw a line
* through a gap (`ln` left of zero, `tan` at its asymptotes).
*/
export function functionIsDefined(name: MathFunction, argument: number): boolean {
switch (name) {
case "ln":
case "log10":
case "log2":
return argument > 0;
case "sqrt":
return argument >= 0;
case "asin":
case "acos":
return argument >= -1 && argument <= 1;
case "factorial":
return argument > -1 || Math.round(argument) !== argument;
default:
return true;
}
}
/**
* Lanczos approximation of the gamma function (JavaScript has no tgamma).
* Accurate to well past the display precision used anywhere in this app.
*/
export function gamma(x: number): number {
if (Number.isNaN(x)) return NaN;
if (x === Math.round(x)) {
if (x <= 0) return NaN; // poles at non-positive integers
if (x <= 171) {
let result = 1;
for (let k = 2; k < x; k++) result *= k;
return result;
}
return Infinity;
}
if (x < 0.5) {
// Reflection formula.
return Math.PI / (Math.sin(Math.PI * x) * gamma(1 - x));
}
const g = 7;
const coefficients = [
0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,
-176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,
1.5056327351493116e-7,
];
const z = x - 1;
let sum = coefficients[0]!;
for (let index = 1; index < g + 2; index++) {
sum += coefficients[index]! / (z + index);
}
const t = z + g + 0.5;
return Math.sqrt(2 * Math.PI) * Math.pow(t, z + 0.5) * Math.exp(-t) * sum;
}
+274
View File
@@ -0,0 +1,274 @@
// The parts of geometry the calculator's spoken-question surface reaches:
// shapes, triangle-from-sides, and Pythagoras.
// ---------------------------------------------------------------------------
// Triangles
/** A triangle solved from whatever three parts were given. */
export type Triangle = {
/** Side lengths, opposite the same-named angles. */
a: number;
b: number;
c: number;
/** Angles in radians. */
angleA: number;
angleB: number;
angleC: number;
};
export function triangleArea(triangle: Triangle): number {
// Heron's formula, in the numerically stable ordering: the naive form
// loses most of its precision on a needle-thin triangle.
const sides = [triangle.a, triangle.b, triangle.c].sort((p, q) => q - p);
const [x, y, z] = [sides[0]!, sides[1]!, sides[2]!];
const term = (x + (y + z)) * (z - (x - y)) * (z + (x - y)) * (x + (y - z));
return term <= 0 ? 0 : Math.sqrt(term) / 4;
}
export function trianglePerimeter(triangle: Triangle): number {
return triangle.a + triangle.b + triangle.c;
}
export function triangleKind(triangle: Triangle): string {
const tolerance = 1e-9;
const sides = [triangle.a, triangle.b, triangle.c].sort((p, q) => p - q);
const descriptors: string[] = [];
if (Math.abs(sides[0]! - sides[2]!) < tolerance) {
descriptors.push("equilateral");
} else if (Math.abs(sides[0]! - sides[1]!) < tolerance || Math.abs(sides[1]! - sides[2]!) < tolerance) {
descriptors.push("isosceles");
} else {
descriptors.push("scalene");
}
const largest = Math.max(triangle.angleA, triangle.angleB, triangle.angleC);
if (Math.abs(largest - Math.PI / 2) < 1e-6) {
descriptors.push("right");
} else if (largest > Math.PI / 2) {
descriptors.push("obtuse");
} else {
descriptors.push("acute");
}
return descriptors.join(", ");
}
export function triangleAnglesInDegrees(triangle: Triangle): [number, number, number] {
return [
(triangle.angleA * 180) / Math.PI,
(triangle.angleB * 180) / Math.PI,
(triangle.angleC * 180) / Math.PI,
];
}
export type TriangleResult =
| { kind: "one"; triangle: Triangle }
/**
* SSA can genuinely produce two valid triangles. Returning one and calling
* it solved is the classic wrong answer in this topic.
*/
| { kind: "two"; first: Triangle; second: Triangle }
| { kind: "impossible"; reason: string };
/**
* Guards acos/asin against a value a hair outside [-1, 1] from rounding,
* which returns NaN and poisons everything downstream.
*/
function clampedToUnit(value: number): number {
return Math.min(1, Math.max(-1, value));
}
/** Three sides. */
export function triangleFromSides(a: number, b: number, c: number): TriangleResult {
if (a <= 0 || b <= 0 || c <= 0) return { kind: "impossible", reason: "Sides must be positive" };
if (a + b <= c || b + c <= a || a + c <= b) {
return { kind: "impossible", reason: "Those sides cannot close a triangle" };
}
const angleA = Math.acos(clampedToUnit((b * b + c * c - a * a) / (2 * b * c)));
const angleB = Math.acos(clampedToUnit((a * a + c * c - b * b) / (2 * a * c)));
return {
kind: "one",
triangle: { a, b, c, angleA, angleB, angleC: Math.PI - angleA - angleB },
};
}
// ---------------------------------------------------------------------------
// Shapes
export type Shape =
| { kind: "circle"; radius: number }
| { kind: "rectangle"; width: number; height: number }
| { kind: "square"; side: number }
| { kind: "triangleBaseHeight"; base: number; height: number }
| { kind: "trapezoid"; a: number; b: number; height: number }
| { kind: "parallelogram"; base: number; height: number }
| { kind: "regularPolygon"; sides: number; sideLength: number }
| { kind: "ellipse"; a: number; b: number }
// Solids
| { kind: "sphere"; radius: number }
| { kind: "cylinder"; radius: number; height: number }
| { kind: "cone"; radius: number; height: number }
| { kind: "cube"; side: number }
| { kind: "rectangularPrism"; length: number; width: number; height: number }
| { kind: "pyramid"; baseArea: number; height: number };
export function shapeName(shape: Shape): string {
switch (shape.kind) {
case "circle":
return "circle";
case "rectangle":
return "rectangle";
case "square":
return "square";
case "triangleBaseHeight":
return "triangle";
case "trapezoid":
return "trapezoid";
case "parallelogram":
return "parallelogram";
case "regularPolygon":
return `regular ${shape.sides}-gon`;
case "ellipse":
return "ellipse";
case "sphere":
return "sphere";
case "cylinder":
return "cylinder";
case "cone":
return "cone";
case "cube":
return "cube";
case "rectangularPrism":
return "rectangular prism";
case "pyramid":
return "pyramid";
}
}
export function shapeIsSolid(shape: Shape): boolean {
switch (shape.kind) {
case "sphere":
case "cylinder":
case "cone":
case "cube":
case "rectangularPrism":
case "pyramid":
return true;
default:
return false;
}
}
/** Area for a plane figure; surface area for a solid. */
export function shapeArea(shape: Shape): number | null {
switch (shape.kind) {
case "circle":
return Math.PI * shape.radius * shape.radius;
case "rectangle":
return shape.width * shape.height;
case "square":
return shape.side * shape.side;
case "triangleBaseHeight":
return (shape.base * shape.height) / 2;
case "trapezoid":
return ((shape.a + shape.b) * shape.height) / 2;
case "parallelogram":
return shape.base * shape.height;
case "regularPolygon":
return shape.sides >= 3
? (shape.sides * shape.sideLength * shape.sideLength) / (4 * Math.tan(Math.PI / shape.sides))
: null;
case "ellipse":
return Math.PI * shape.a * shape.b;
case "sphere":
return 4 * Math.PI * shape.radius * shape.radius;
case "cylinder":
return 2 * Math.PI * shape.radius * (shape.radius + shape.height);
case "cone":
return (
Math.PI *
shape.radius *
(shape.radius + Math.sqrt(shape.radius * shape.radius + shape.height * shape.height))
);
case "cube":
return 6 * shape.side * shape.side;
case "rectangularPrism":
return (
2 *
(shape.length * shape.width + shape.length * shape.height + shape.width * shape.height)
);
case "pyramid":
return null; // needs the base shape, not just its area
}
}
export function shapePerimeter(shape: Shape): number | null {
switch (shape.kind) {
case "circle":
return 2 * Math.PI * shape.radius;
case "rectangle":
return 2 * (shape.width + shape.height);
case "square":
return 4 * shape.side;
case "regularPolygon":
return shape.sides >= 3 ? shape.sides * shape.sideLength : null;
case "ellipse": {
// Ramanujan's second approximation — accurate to ~1e-9 for any
// realistic eccentricity, where the naive π(a+b) is not close.
const h = ((shape.a - shape.b) * (shape.a - shape.b)) / ((shape.a + shape.b) * (shape.a + shape.b));
return Math.PI * (shape.a + shape.b) * (1 + (3 * h) / (10 + Math.sqrt(4 - 3 * h)));
}
default:
// Trapezoid/triangle/parallelogram need every side, and the constructor
// only took the measurements the area formula uses. Solids have none.
return null;
}
}
export function shapeVolume(shape: Shape): number | null {
switch (shape.kind) {
case "sphere":
return (4 / 3) * Math.PI * shape.radius * shape.radius * shape.radius;
case "cylinder":
return Math.PI * shape.radius * shape.radius * shape.height;
case "cone":
return (Math.PI * shape.radius * shape.radius * shape.height) / 3;
case "cube":
return shape.side * shape.side * shape.side;
case "rectangularPrism":
return shape.length * shape.width * shape.height;
case "pyramid":
return (shape.baseArea * shape.height) / 3;
default:
return null;
}
}
/** Everything known, for the readback. */
export function shapeSummary(shape: Shape): [string, number][] {
const parts: [string, number][] = [];
const area = shapeArea(shape);
if (area !== null) parts.push([shapeIsSolid(shape) ? "Surface area" : "Area", area]);
const perimeter = shapePerimeter(shape);
if (perimeter !== null) parts.push(["Perimeter", perimeter]);
const volume = shapeVolume(shape);
if (volume !== null) parts.push(["Volume", volume]);
return parts;
}
// ---------------------------------------------------------------------------
// Coordinate geometry
/** Pythagoras, either direction: give any two, get the third. */
export function pythagoras(
a: number | null,
b: number | null,
hypotenuse: number | null,
): number | null {
if (a !== null && b !== null && hypotenuse === null) return Math.sqrt(a * a + b * b);
if (a !== null && b === null && hypotenuse !== null) {
return hypotenuse > a ? Math.sqrt(hypotenuse * hypotenuse - a * a) : null;
}
if (a === null && b !== null && hypotenuse !== null) {
return hypotenuse > b ? Math.sqrt(hypotenuse * hypotenuse - b * b) : null;
}
return null;
}
+219
View File
@@ -0,0 +1,219 @@
// Tokeniser for typed and normalised spoken maths expressions.
import { type MathNumber, fromDecimal, integer, real } from "./math-number";
import { type MathRelation } from "./expression";
import { isLetter } from "../../../util/unicode-class";
export type MathToken =
| { type: "number"; value: MathNumber }
| { type: "identifier"; name: string }
| { type: "plus" }
| { type: "minus" }
| { type: "star" }
| { type: "slash" }
| { type: "percent" }
| { type: "caret" }
| { type: "bang" }
| { type: "leftParen" }
| { type: "rightParen" }
| { type: "pipe" }
| { type: "comma" }
| { type: "relation"; relation: MathRelation };
export type MathParseErrorKind =
| { kind: "unexpectedCharacter"; character: string }
| { kind: "unexpectedToken"; token: string }
| { kind: "unexpectedEnd" }
| { kind: "unbalancedParenthesis" }
| { kind: "unknownFunction"; name: string }
| { kind: "emptyInput" }
| { kind: "tooComplex" };
/**
* A parse failure with the user's own words attached, so an error can quote
* back what was actually said rather than leaving the wearer guessing which
* word broke.
*/
export class MathParseError extends Error {
readonly errorKind: MathParseErrorKind;
/** What the user actually said. */
readonly context: string;
constructor(errorKind: MathParseErrorKind, context = "") {
super(describeParseError(errorKind, context));
this.name = "MathParseError";
this.errorKind = errorKind;
this.context = context;
}
}
/** Written for someone reading a lens mid-task, not for a log. */
export function describeParseError(kind: MathParseErrorKind, context: string): string {
const quoted = context ? `“${context}”` : "that expression";
switch (kind.kind) {
case "unexpectedCharacter":
return `Could not read "${kind.character}" in ${quoted}`;
case "unexpectedToken":
return `Did not expect "${kind.token}" in ${quoted}`;
case "unexpectedEnd":
return `${quoted} is incomplete`;
case "unbalancedParenthesis":
return `Brackets do not balance in ${quoted}`;
case "unknownFunction":
return `"${kind.name}" is not a function this build knows`;
case "emptyInput":
return "Nothing to solve";
case "tooComplex":
return "That expression is too deeply nested";
}
}
function isDigit(character: string): boolean {
return character >= "0" && character <= "9";
}
export function tokenize(input: string): MathToken[] {
const tokens: MathToken[] = [];
const characters = Array.from(input);
let index = 0;
while (index < characters.length) {
const character = characters[index]!;
if (/\s/.test(character)) {
index += 1;
continue;
}
if (isDigit(character) || (character === "." && index + 1 < characters.length && isDigit(characters[index + 1]!))) {
let literal = "";
let sawDot = false;
while (index < characters.length) {
const next = characters[index]!;
if (isDigit(next)) {
literal += next;
} else if (next === "." && !sawDot) {
sawDot = true;
literal += next;
} else if (next === "," && index + 1 < characters.length && isDigit(characters[index + 1]!)) {
// Thousands separator inside a literal: 1,000
index += 1;
continue;
} else {
break;
}
index += 1;
}
// Decimals become exact rationals so later arithmetic stays exact:
// 0.25 behaves as 1/4, not as a float.
if (sawDot) {
tokens.push({ type: "number", value: fromDecimal(Number(literal) || 0) });
} else {
const parsed = Number(literal);
// Exact for every safe literal; anything larger degrades to a real
// instead of losing digits silently.
tokens.push({
type: "number",
value: Number.isSafeInteger(parsed) ? integer(parsed) : real(parsed),
});
}
continue;
}
if (isLetter(character) || character === "_") {
let name = "";
while (
index < characters.length &&
(isLetter(characters[index]!) || isDigit(characters[index]!) || characters[index] === "_")
) {
name += characters[index];
index += 1;
}
tokens.push({ type: "identifier", name });
continue;
}
switch (character) {
case "+":
tokens.push({ type: "plus" });
break;
case "-":
case "−": // ASCII and Unicode minus
tokens.push({ type: "minus" });
break;
case "*":
case "×":
case "⋅":
tokens.push({ type: "star" });
break;
case "/":
case "÷":
tokens.push({ type: "slash" });
break;
case "%":
tokens.push({ type: "percent" });
break;
case "^":
tokens.push({ type: "caret" });
break;
case "!":
tokens.push({ type: "bang" });
break;
case "(":
case "[":
case "{":
tokens.push({ type: "leftParen" });
break;
case ")":
case "]":
case "}":
tokens.push({ type: "rightParen" });
break;
case "|":
tokens.push({ type: "pipe" });
break;
case ",":
tokens.push({ type: "comma" });
break;
case "=":
// "==" from a typed source is still equality.
if (index + 1 < characters.length && characters[index + 1] === "=") index += 1;
tokens.push({ type: "relation", relation: "=" });
break;
case "<":
if (index + 1 < characters.length && characters[index + 1] === "=") {
tokens.push({ type: "relation", relation: "<=" });
index += 1;
} else {
tokens.push({ type: "relation", relation: "<" });
}
break;
case ">":
if (index + 1 < characters.length && characters[index + 1] === "=") {
tokens.push({ type: "relation", relation: ">=" });
index += 1;
} else {
tokens.push({ type: "relation", relation: ">" });
}
break;
case "≤":
tokens.push({ type: "relation", relation: "<=" });
break;
case "≥":
tokens.push({ type: "relation", relation: ">=" });
break;
case "√":
tokens.push({ type: "identifier", name: "sqrt" });
break;
case "π":
tokens.push({ type: "identifier", name: "pi" });
break;
default:
throw new MathParseError({ kind: "unexpectedCharacter", character }, input);
}
index += 1;
}
if (tokens.length === 0) throw new MathParseError({ kind: "emptyInput" }, input);
return tokens;
}
+362
View File
@@ -0,0 +1,362 @@
// A numeric literal that stays exact for as long as it can. Rationals rather
// than floats everywhere is the difference between a step-by-step solution
// that reads `x = 3/2` and one that reads `x = 1.5`, and between
// `1/3 + 1/3 + 1/3 = 1` and `0.9999999999999999`. Anything genuinely
// irrational (a square root, a sine) degrades to `real` and stays there.
//
// Number.isSafeInteger is the overflow guard: an operation whose exact
// result leaves the safe-integer range falls back to a `real` value rather
// than silently losing precision.
export type MathNumber =
| { kind: "integer"; value: number }
| { kind: "rational"; n: number; d: number }
| { kind: "real"; value: number };
export function integer(value: number): MathNumber {
return { kind: "integer", value };
}
export function real(value: number): MathNumber {
return { kind: "real", value };
}
/**
* Always stored normalised: `d > 1`, sign on the numerator, gcd(n, d) === 1.
* Construct rationals only through here.
*/
export function rational(numerator: number, denominator: number): MathNumber {
if (denominator === 0) return real(NaN);
if (!Number.isSafeInteger(numerator) || !Number.isSafeInteger(denominator)) {
return real(numerator / denominator);
}
let n = numerator;
let d = denominator;
if (d < 0) {
n = -n;
d = -d;
}
const divisor = greatestCommonDivisor(Math.abs(n), d);
if (divisor > 1) {
n /= divisor;
d /= divisor;
}
return d === 1 ? integer(n) : { kind: "rational", n, d };
}
/**
* Recover an exact rational from a decimal literal so `0.25` behaves like
* `1/4` in later arithmetic. Bounded denominator: beyond that the value is
* almost certainly a measurement rather than a fraction, and pretending
* otherwise produces absurd numerators. (Stern-Brocot search.)
*/
export function fromDecimal(value: number, maximumDenominator = 1_000_000): MathNumber {
if (!Number.isFinite(value)) return real(value);
if (value === Math.round(value) && Number.isSafeInteger(value)) {
return integer(value);
}
let lowerN = 0;
let lowerD = 1;
let upperN = 1;
let upperD = 0;
const negative = value < 0;
const target = Math.abs(value);
for (let iterations = 0; iterations < 64; iterations++) {
const mediantN = lowerN + upperN;
const mediantD = lowerD + upperD;
if (mediantD > maximumDenominator) break;
const mediant = mediantN / mediantD;
if (Math.abs(mediant - target) < 1e-12) {
return rational(negative ? -mediantN : mediantN, mediantD);
}
if (mediant < target) {
lowerN = mediantN;
lowerD = mediantD;
} else {
upperN = mediantN;
upperD = mediantD;
}
}
return real(value);
}
// ---------------------------------------------------------------------------
// Inspection
export function doubleValue(value: MathNumber): number {
switch (value.kind) {
case "integer":
return value.value;
case "rational":
return value.n / value.d;
case "real":
return value.value;
}
}
export function isZero(value: MathNumber): boolean {
switch (value.kind) {
case "integer":
return value.value === 0;
case "rational":
return value.n === 0;
case "real":
return value.value === 0;
}
}
export function isOne(value: MathNumber): boolean {
switch (value.kind) {
case "integer":
return value.value === 1;
case "rational":
return false; // normalised, so never 1
case "real":
return value.value === 1;
}
}
export function isNegative(value: MathNumber): boolean {
switch (value.kind) {
case "integer":
return value.value < 0;
case "rational":
return value.n < 0;
case "real":
return value.value < 0;
}
}
export function isExact(value: MathNumber): boolean {
return value.kind !== "real";
}
export function isInteger(value: MathNumber): boolean {
return value.kind === "integer";
}
export function integerValue(value: MathNumber): number | null {
return value.kind === "integer" ? value.value : null;
}
function asFraction(value: MathNumber): [number, number] | null {
switch (value.kind) {
case "integer":
return [value.value, 1];
case "rational":
return [value.n, value.d];
case "real":
return null;
}
}
// ---------------------------------------------------------------------------
// Arithmetic
//
// Exactness is preserved whenever both operands are exact; a single `real`
// anywhere makes the whole result `real`. Overflow falls back to `real`
// rather than trapping — a solver exploring a bad branch must not crash the
// app on the wearer's face.
function safeProducts(...values: number[]): boolean {
return values.every((value) => Number.isSafeInteger(value));
}
export function add(lhs: MathNumber, rhs: MathNumber): MathNumber {
const left = asFraction(lhs);
const right = asFraction(rhs);
if (!left || !right) return real(doubleValue(lhs) + doubleValue(rhs));
const [ln, ld] = left;
const [rn, rd] = right;
const crossLeft = ln * rd;
const crossRight = rn * ld;
const numerator = crossLeft + crossRight;
const denominator = ld * rd;
if (!safeProducts(crossLeft, crossRight, numerator, denominator)) {
return real(doubleValue(lhs) + doubleValue(rhs));
}
return rational(numerator, denominator);
}
export function subtract(lhs: MathNumber, rhs: MathNumber): MathNumber {
return add(lhs, negated(rhs));
}
export function multiply(lhs: MathNumber, rhs: MathNumber): MathNumber {
const left = asFraction(lhs);
const right = asFraction(rhs);
if (!left || !right) return real(doubleValue(lhs) * doubleValue(rhs));
const [ln, ld] = left;
const [rn, rd] = right;
const numerator = ln * rn;
const denominator = ld * rd;
if (!safeProducts(numerator, denominator)) {
return real(doubleValue(lhs) * doubleValue(rhs));
}
return rational(numerator, denominator);
}
export function divide(lhs: MathNumber, rhs: MathNumber): MathNumber {
if (isZero(rhs)) return real(doubleValue(lhs) / doubleValue(rhs));
const left = asFraction(lhs);
const right = asFraction(rhs);
if (!left || !right) return real(doubleValue(lhs) / doubleValue(rhs));
const [ln, ld] = left;
const [rn, rd] = right;
const numerator = ln * rd;
const denominator = ld * rn;
if (!safeProducts(numerator, denominator)) {
return real(doubleValue(lhs) / doubleValue(rhs));
}
return rational(numerator, denominator);
}
export function negated(value: MathNumber): MathNumber {
switch (value.kind) {
case "integer":
return integer(-value.value);
case "rational":
return { kind: "rational", n: -value.n, d: value.d };
case "real":
return real(-value.value);
}
}
export function reciprocal(value: MathNumber): MathNumber {
switch (value.kind) {
case "integer":
return rational(1, value.value);
case "rational":
return rational(value.d, value.n);
case "real":
return real(1 / value.value);
}
}
/**
* Exact only when the result stays rational — an integer exponent, or a
* perfect root. `2^0.5` is irrational, so it becomes `real`.
*/
export function raisedTo(base: MathNumber, exponent: MathNumber): MathNumber {
const fraction = asFraction(base);
const power = integerValue(exponent);
if (power !== null && fraction) {
const [n, d] = fraction;
if (power >= 0 && power < 64) {
let numerator = 1;
let denominator = 1;
let overflow = false;
for (let step = 0; step < power; step++) {
numerator *= n;
denominator *= d;
if (!safeProducts(numerator, denominator)) {
overflow = true;
break;
}
}
if (!overflow) return rational(numerator, denominator);
} else if (power < 0 && power > -64 && !isZero(base)) {
return raisedTo(reciprocal(base), integer(-power));
}
}
// An exact root of a perfect power stays exact: sqrt(9/4) = 3/2.
if (exponent.kind === "rational" && exponent.d === 2 && exponent.n === 1 && fraction) {
const [n, d] = fraction;
if (n >= 0) {
const rootN = Math.round(Math.sqrt(n));
const rootD = Math.round(Math.sqrt(d));
if (rootN * rootN === n && rootD * rootD === d) return rational(rootN, rootD);
}
}
return real(Math.pow(doubleValue(base), doubleValue(exponent)));
}
export function greatestCommonDivisor(a: number, b: number): number {
let x = Math.abs(a);
let y = Math.abs(b);
while (y !== 0) {
const next = x % y;
x = y;
y = next;
}
return x === 0 ? 1 : x;
}
// ---------------------------------------------------------------------------
// Comparison
export function lessThan(lhs: MathNumber, rhs: MathNumber): boolean {
return doubleValue(lhs) < doubleValue(rhs);
}
/**
* Structural equality would make `integer(1)` differ from `real(1)`, which
* breaks simplification against numbers that came back from a numeric path.
* Compare by value instead, with a tolerance for reals.
*/
export function numberEquals(lhs: MathNumber, rhs: MathNumber): boolean {
const left = asFraction(lhs);
const right = asFraction(rhs);
if (left && right) return left[0] === right[0] && left[1] === right[1];
const a = doubleValue(lhs);
const b = doubleValue(rhs);
if (a === b) return true;
const scale = Math.max(Math.abs(a), Math.abs(b), 1);
return Math.abs(a - b) <= 1e-12 * scale;
}
// ---------------------------------------------------------------------------
// Display
/** Plain-text form for the glasses and for spoken readback. */
export function describeNumber(value: MathNumber): string {
switch (value.kind) {
case "integer":
return String(value.value);
case "rational":
return `${value.n}/${value.d}`;
case "real": {
const raw = value.value;
if (Number.isNaN(raw)) return "NaN";
if (raw === Math.round(raw) && Math.abs(raw) < 1e15) {
return String(Math.round(raw));
}
// Trim to something a person can read aloud, without exponent notation
// for ordinary magnitudes.
if (Math.abs(raw) >= 1e-4 && Math.abs(raw) < 1e9) {
let text = raw.toFixed(6);
while (text.endsWith("0")) text = text.slice(0, -1);
if (text.endsWith(".")) text = text.slice(0, -1);
return text;
}
return formatG(raw);
}
}
}
/** Approximates C's %g: shortest of decimal/exponent form at 6 significant digits. */
function formatG(value: number): string {
if (value === 0) return "0";
const exponent = Math.floor(Math.log10(Math.abs(value)));
if (exponent >= -4 && exponent < 6) {
const text = value.toPrecision(6);
return trimTrailingZeros(text);
}
const mantissa = trimTrailingZeros((value / Math.pow(10, exponent)).toPrecision(6));
const sign = exponent >= 0 ? "+" : "-";
const padded = String(Math.abs(exponent)).padStart(2, "0");
return `${mantissa}e${sign}${padded}`;
}
function trimTrailingZeros(text: string): string {
if (!text.includes(".")) return text;
let result = text;
while (result.endsWith("0")) result = result.slice(0, -1);
if (result.endsWith(".")) result = result.slice(0, -1);
return result;
}
/** Thousands-separated integer. */
export function formatInt(value: number): string {
return Math.round(value).toLocaleString("en-US");
}
+509
View File
@@ -0,0 +1,509 @@
// Turns a spoken utterance into something the parser can read. ASR does not
// emit `2x^2 + 3x - 5 = 0`; it emits "two x squared plus three x minus five
// equals zero", and on a bad pass "2 x squared plus 3x minus 5 = 0". Every
// rule here exists because one of those shapes has to land on the same tree.
//
// Deliberately NOT an LLM pass: a misheard equation should fail visibly at
// the parser rather than be silently "corrected" into a different problem.
import { ALL_CONSTANTS, ALL_FUNCTIONS } from "./expression";
export function normalizeSpoken(raw: string): string {
let text = raw
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase()
.replace(/’/g, "'");
text = stripTrailingDiscourse(text);
text = stripPreamble(text);
text = applyPhrases(text, FUNCTION_PHRASES);
text = applyPhrases(text, RELATION_PHRASES);
text = applyPhrases(text, OPERATOR_PHRASES);
// Number words BEFORE powers: "two to the power of five" must become
// "2 to the power of 5" first, or the power rule collapses it to
// "two^five" and the parser sees two variables. Ordinals ("fifth") are
// untouched by the number pass, so "to the fifth" still resolves after.
text = applyNumberWords(text);
text = applyPowerPhrases(text);
text = applyFractionPhrases(text);
text = applyPhrases(text, CONSTANT_PHRASES);
text = collapseDigitGroups(text);
text = insertImplicitMultiplication(text);
return text.replace(/\s+/g, " ").trim();
}
// ---------------------------------------------------------------------------
// Preamble
const TRAILING_PUNCTUATION = /^[ .,!?;:]+|[ .,!?;:]+$/g;
/**
* Remove terminal speech-recognition filler that is not part of the problem.
*
* Always-on transcription commonly finalizes a valid equation together with
* the next conversational beat (`3x plus 1. Thank you.`). Leaving that suffix
* attached makes the deterministic parser reject otherwise complete
* arithmetic and sends the calculator through its optional model-rewrite
* fallback — turning an offline feature into an API-key error for users
* without a cloud provider configured.
*/
export function stripTrailingDiscourse(text: string): string {
const phrases = ["thank you", "thanks", "please", "okay", "ok"].sort((a, b) => b.length - a.length);
let result = text.replace(TRAILING_PUNCTUATION, "");
let changed = true;
while (changed) {
changed = false;
for (const phrase of phrases) {
if (result === phrase) {
result = "";
changed = true;
break;
}
const suffix = " " + phrase;
if (result.endsWith(suffix)) {
result = result.slice(0, -suffix.length).replace(TRAILING_PUNCTUATION, "");
changed = true;
break;
}
}
}
return result;
}
/**
* "solve two x plus one equals five" → "two x plus one equals five". The
* command word is consumed by the caller (it chooses the mode); what reaches
* the parser must be only the mathematics.
*/
const PREAMBLES = [
"can you solve",
"please solve",
"solve for me",
"work out",
"figure out",
"calculate",
"compute",
"evaluate",
"simplify",
"solve",
"what is",
"whats",
"what's",
"how much is",
"tell me",
"the answer to",
"the equation",
"the expression",
"graph",
"plot",
].sort((a, b) => b.length - a.length);
export function stripPreamble(text: string): string {
let result = text.trim();
let changed = true;
// Loop: "can you solve what is 2 plus 2" has two stacked preambles.
while (changed) {
changed = false;
for (const preamble of PREAMBLES) {
if (result.startsWith(preamble + " ")) {
result = result.slice(preamble.length + 1);
changed = true;
break;
}
}
}
return result;
}
// ---------------------------------------------------------------------------
// Phrase tables
function escapeRegExp(text: string): string {
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
/**
* Longest-first replacement. "less than or equal to" must be tried before
* "less than", or the tail becomes a stray "or equal to".
*/
function applyPhrases(text: string, phrases: readonly [string, string][]): string {
let result = text;
const sorted = [...phrases].sort((a, b) => b[0].length - a[0].length);
for (const [phrase, replacement] of sorted) {
result = result.replace(new RegExp(`\\b${escapeRegExp(phrase)}\\b`, "g"), replacement);
}
return result;
}
const RELATION_PHRASES: readonly [string, string][] = [
["is less than or equal to", "<="],
["less than or equal to", "<="],
["is greater than or equal to", ">="],
["greater than or equal to", ">="],
["is at most", "<="],
["at most", "<="],
["is at least", ">="],
["at least", ">="],
["is less than", "<"],
["less than", "<"],
["is greater than", ">"],
["greater than", ">"],
["is equal to", "="],
["equal to", "="],
["equals", "="],
["is the same as", "="],
];
const OPERATOR_PHRASES: readonly [string, string][] = [
["multiplied by", "*"],
["times", "*"],
["product of", "*"],
["divided by", "/"],
["over", "/"],
["split by", "/"],
["plus", "+"],
["added to", "+"],
["and then add", "+"],
["minus", "-"],
["subtract", "-"],
["take away", "-"],
["less", "-"],
["modulo", "%"],
["mod", "%"],
["open paren", "("],
["open parenthesis", "("],
["open bracket", "("],
["close paren", ")"],
["close parenthesis", ")"],
["close bracket", ")"],
["negative", "-"],
["minus sign", "-"],
// "15 percent of 240" is 15/100 × 240. Mapping `percent` to the modulo
// operator (which shares the glyph) turned every percentage question into a
// remainder question — 15 % 240 = 15, silently wrong.
["percent of", "/100 *"],
["percent", "/100"],
];
const FUNCTION_PHRASES: readonly [string, string][] = [
["square root of", "sqrt"],
["the square root of", "sqrt"],
["root of", "sqrt"],
["cube root of", "cbrt"],
["natural log of", "ln"],
["natural log", "ln"],
["log base ten of", "log10"],
["log base 10 of", "log10"],
["log base two of", "log2"],
["log base 2 of", "log2"],
["log of", "log10"],
["absolute value of", "abs"],
["the absolute value of", "abs"],
["sine of", "sin"],
["cosine of", "cos"],
["tangent of", "tan"],
["arc sine of", "asin"],
["arc cosine of", "acos"],
["arc tangent of", "atan"],
["inverse sine of", "asin"],
["inverse cosine of", "acos"],
["inverse tangent of", "atan"],
["factorial of", "factorial"],
["e to the power of", "exp"],
];
const CONSTANT_PHRASES: readonly [string, string][] = [
["pi", "pi"],
["tau", "tau"],
["euler's number", "e"],
["eulers number", "e"],
];
// ---------------------------------------------------------------------------
// Powers
/** "x squared", "x to the power of three", "x cubed", "two to the fifth". */
export function applyPowerPhrases(text: string): string {
let result = text;
const ordinalPowers: readonly [string, string][] = [
["squared", "^2"],
["cubed", "^3"],
];
for (const [phrase, replacement] of ordinalPowers) {
result = result.replace(new RegExp(`\\s*\\b${phrase}\\b`, "g"), replacement);
}
// "to the power of N" / "to the N power" / "to the N"
for (const pattern of [
/\s*to the power of\s+/g,
/\s*raised to the power of\s+/g,
/\s*raised to\s+/g,
/\s*to the\s+(?=[0-9])/g,
]) {
result = result.replace(pattern, "^");
}
// Ordinal exponents: "to the fifth" was normalised to "^fifth" above.
const ordinals: readonly [string, string][] = [
["first", "1"],
["second", "2"],
["third", "3"],
["fourth", "4"],
["fifth", "5"],
["sixth", "6"],
["seventh", "7"],
["eighth", "8"],
["ninth", "9"],
["tenth", "10"],
];
for (const [word, digit] of ordinals) {
result = result.replace(new RegExp(`\\^\\s*${word}(\\s+power)?\\b`, "g"), `^${digit}`);
result = result.replace(new RegExp(`\\s*to the ${word}(\\s+power)?\\b`, "g"), `^${digit}`);
}
// Postfix "seven factorial" — the function table only covers the prefix
// form ("factorial of seven").
result = result.replace(/(\d+|\))\s*\bfactorial\b/g, "$1!");
return result.replace(/\^\s+/g, "^");
}
// ---------------------------------------------------------------------------
// Fractions
/** "three over four", "two thirds", "one half". */
export function applyFractionPhrases(text: string): string {
let result = text;
const denominators: readonly [string, number][] = [
["halves", 2],
["half", 2],
["thirds", 3],
["third", 3],
["quarters", 4],
["quarter", 4],
["fourths", 4],
["fourth", 4],
["fifths", 5],
["fifth", 5],
["sixths", 6],
["sixth", 6],
["eighths", 8],
["eighth", 8],
["tenths", 10],
["tenth", 10],
];
for (const [word, denominator] of denominators) {
// Only when a number precedes it — "a third of the guests" is not
// arithmetic, but "two thirds" is.
result = result.replace(new RegExp(`\\b(\\d+)\\s+${word}\\b`, "g"), `($1/${denominator})`);
}
return result;
}
// ---------------------------------------------------------------------------
// Number words
const UNITS: readonly [string, number][] = [
["zero", 0],
["one", 1],
["two", 2],
["three", 3],
["four", 4],
["five", 5],
["six", 6],
["seven", 7],
["eight", 8],
["nine", 9],
["ten", 10],
["eleven", 11],
["twelve", 12],
["thirteen", 13],
["fourteen", 14],
["fifteen", 15],
["sixteen", 16],
["seventeen", 17],
["eighteen", 18],
["nineteen", 19],
];
const TENS: readonly [string, number][] = [
["twenty", 20],
["thirty", 30],
["forty", 40],
["fifty", 50],
["sixty", 60],
["seventy", 70],
["eighty", 80],
["ninety", 90],
];
/**
* Fold English number words into digits, including compounds
* ("twenty five" → 25, "three hundred" → 300, "two thousand ten" → 2010).
*/
export function applyNumberWords(text: string): string {
const tokens = text.split(/\s+/).filter((token) => token.length > 0);
const output: string[] = [];
let index = 0;
while (index < tokens.length) {
const [value, consumed] = parseNumberPhrase(tokens, index);
if (value !== null && consumed > 0) {
output.push(String(value));
index += consumed;
} else {
output.push(tokens[index]!);
index += 1;
}
}
return output.join(" ");
}
/**
* What the previous token was, which is what decides whether the next one
* continues the same number or starts a new one. "twenty five" is 25;
* "five five" is two numbers; "one hundred forty four" is 144.
*/
type NumberTokenKind = "none" | "unit" | "ten" | "hundred" | "scale";
/**
* Greedy left-to-right accumulation with scale words. Returns [null, 0] when
* the token at `start` does not begin a number phrase.
*/
function parseNumberPhrase(tokens: string[], start: number): [number | null, number] {
let total = 0;
let current = 0;
let consumed = 0;
let matchedAnything = false;
let previous: NumberTokenKind = "none";
let index = start;
while (index < tokens.length) {
const token = tokens[index]!.replace(/^[,-]+|[,-]+$/g, "");
const unit = UNITS.find(([word]) => word === token)?.[1];
const ten = TENS.find(([word]) => word === token)?.[1];
if (unit !== undefined) {
// A unit may follow a ten ("twenty five"), a hundred ("a hundred
// four"), or a scale ("two thousand ten") — but not another unit, which
// is two separate numbers.
if (previous === "unit") break;
current += unit;
previous = "unit";
} else if (ten !== undefined) {
// After "hundred", `current` is non-zero; bailing there turned
// "one hundred forty four" into "100 44".
if (previous !== "none" && previous !== "hundred" && previous !== "scale") break;
current += ten;
previous = "ten";
} else if (token === "hundred") {
if (previous !== "unit" && previous !== "ten" && previous !== "none") break;
if (current === 0) current = 1;
current *= 100;
previous = "hundred";
} else if (token === "thousand" || token === "million") {
const scale = token === "thousand" ? 1_000 : 1_000_000;
if (current === 0 && total === 0) current = 1;
total += current * scale;
current = 0;
previous = "scale";
} else if (token === "and" && matchedAnything && index + 1 < tokens.length && isNumberWord(tokens[index + 1]!)) {
// "three hundred and five" — but a bare "and" elsewhere is not part of
// the number.
index += 1;
consumed += 1;
continue;
} else {
break;
}
matchedAnything = true;
index += 1;
consumed += 1;
}
if (!matchedAnything) return [null, 0];
return [total + current, consumed];
}
function isNumberWord(token: string): boolean {
const cleaned = token.replace(/^[,-]+|[,-]+$/g, "");
return (
UNITS.some(([word]) => word === cleaned) ||
TENS.some(([word]) => word === cleaned) ||
["hundred", "thousand", "million"].includes(cleaned)
);
}
// ---------------------------------------------------------------------------
// Digit groups
/**
* ASR renders a dictated "four hundred and five" that it already digitised as
* "4 05" or "400 5". The first is a spacing artefact and joins; the second is
* genuinely ambiguous and is left alone.
*/
export function collapseDigitGroups(text: string): string {
return text.replace(/\b(\d+)\s+(0\d+)\b/g, "$1$2");
}
// ---------------------------------------------------------------------------
// Implicit multiplication
/**
* Names the implicit-multiplication repair must not split.
*/
export const RESERVED_WORDS: readonly string[] = [...ALL_FUNCTIONS, ...ALL_CONSTANTS, "log", "sqrt"];
/**
* The subset that is CALLED, so a star between the name and its opening paren
* is an artefact of the implicit-multiplication rules rather than a real
* operator. Constants are excluded deliberately: `pi*(r+1)` must keep its
* multiplication.
*/
export const RESERVED_FUNCTION_NAMES: readonly string[] = [...ALL_FUNCTIONS, "log", "sqrt"];
/**
* "2x" and "3(x+1)" and "2 x" all mean multiplication. Inserting the `*` here
* rather than in the parser keeps the grammar unambiguous, and makes the
* normalised string something a wearer can read back and verify.
*/
export function insertImplicitMultiplication(text: string): string {
let result = text;
const rules: readonly [RegExp, string][] = [
// number followed by a letter or an opening paren: 2x, 2(
[/(\d)\s*([a-z(])/g, "$1*$2"],
// closing paren followed by a number, letter, or paren: )x, )2, )(
[/(\))\s*([a-z0-9(])/g, "$1*$2"],
// a single-letter variable immediately followed by a paren
[/(?<![a-z])([a-z])\s*(\()/g, "$1*$2"],
];
for (const [pattern, replacement] of rules) {
result = result.replace(pattern, replacement);
}
// The rules above will happily turn `sqrt(x)` into `sqr*t*(x)` and `2*x`
// inside a known function name; repair the known names.
//
// The optional star belongs strictly BETWEEN the characters of a word.
// Appending one after the final character too (`p\*?i\*?` for `pi`) made
// the repair swallow a legitimate operator that merely FOLLOWED the word:
// `2*pi*r` became `2*pir`, which then parsed as three unbound variables and
// silently produced a different answer.
//
// A function name immediately before its opening paren is the one
// exception: the rules above rewrite `sqrt(x)` to `sqr*t*(x)`, so that
// trailing star really is spurious. A CONSTANT is never called, so a star
// after `pi` or `e` is always a real multiplication — `pi*(r+1)` has to
// keep it. Run the narrower function rule first, then the general
// between-characters repair over everything.
for (const name of RESERVED_FUNCTION_NAMES) {
if (name.length <= 1) continue;
const pattern = name.split("").join("\\*?") + "\\*?(?=\\()";
result = result.replace(new RegExp(pattern, "g"), name);
}
for (const name of RESERVED_WORDS) {
// A single-character constant such as `e` has nothing to repair between
// characters, so it is skipped entirely.
if (name.length <= 1) continue;
const pattern = name.split("").join("\\*?");
result = result.replace(new RegExp(pattern, "g"), name);
}
return result;
}
+369
View File
@@ -0,0 +1,369 @@
// Integer arithmetic: primality, factorisation, modular arithmetic, and the
// classic sequences. Modular products use BigInt internally — `(a*b) % m`
// in floats loses exactness long before the moduli a cryptography-flavoured
// question uses.
// ---------------------------------------------------------------------------
// Primality
const SMALL_PRIMES = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37];
/**
* Deterministic Miller-Rabin. The witness set below is *proven* to decide
* primality for every 64-bit integer, so this is exact rather than
* probabilistic.
*/
export function isPrime(value: number): boolean {
if (!Number.isSafeInteger(value) || value < 2) return false;
for (const small of SMALL_PRIMES) {
if (value === small) return true;
if (value % small === 0) return false;
}
let d = BigInt(value - 1);
let r = 0;
while (d % 2n === 0n) {
d /= 2n;
r += 1;
}
const n = BigInt(value);
for (const witness of SMALL_PRIMES) {
let x = modularPowerBig(BigInt(witness), d, n);
if (x === 1n || x === n - 1n) continue;
let composite = true;
for (let step = 0; step < r - 1; step++) {
x = (x * x) % n;
if (x === n - 1n) {
composite = false;
break;
}
}
if (composite) return false;
}
return true;
}
/** The next prime at or above `value`, or null when the search is unreasonable. */
export function nextPrime(after: number): number | null {
let candidate = Math.max(2, after + 1);
// Bounded: a caller asking near the integer ceiling should get null, not a hang.
while (Number.isSafeInteger(candidate) && candidate - after < 1_000_000) {
if (isPrime(candidate)) return candidate;
candidate += 1;
}
return null;
}
export function primesUpTo(limit: number): number[] {
if (limit < 2) return [];
const sieve = new Uint8Array(limit + 1).fill(1);
sieve[0] = 0;
sieve[1] = 0;
for (let candidate = 2; candidate * candidate <= limit; candidate++) {
if (!sieve[candidate]) continue;
for (let multiple = candidate * candidate; multiple <= limit; multiple += candidate) {
sieve[multiple] = 0;
}
}
const result: number[] = [];
for (let value = 2; value <= limit; value++) {
if (sieve[value]) result.push(value);
}
return result;
}
// ---------------------------------------------------------------------------
// Factorisation
export type PrimeFactor = { prime: number; exponent: number };
/**
* Trial division by small primes, then Pollard's rho for the rest.
*
* Trial division alone is fine to ~10⁷ and hopeless beyond; rho handles
* semiprimes with large factors, which is exactly the shape that makes a
* naive factoriser appear to hang.
*/
export function factorize(value: number): PrimeFactor[] {
if (value === 0 || !Number.isSafeInteger(value)) return [];
let remaining = Math.abs(value);
const counts = new Map<number, number>();
for (const prime of primesUpTo(100_000)) {
if (prime * prime > remaining) break;
while (remaining % prime === 0) {
counts.set(prime, (counts.get(prime) ?? 0) + 1);
remaining /= prime;
}
}
if (remaining > 1) {
for (const factor of pollardFactorize(remaining)) {
counts.set(factor, (counts.get(factor) ?? 0) + 1);
}
}
return [...counts.keys()].sort((a, b) => a - b).map((prime) => ({ prime, exponent: counts.get(prime)! }));
}
function pollardFactorize(value: number): number[] {
if (value === 1) return [];
if (isPrime(value)) return [value];
const divisor = pollardRho(value);
if (divisor === null) return [value];
return [...pollardFactorize(divisor), ...pollardFactorize(value / divisor)];
}
function pollardRho(n: number): number | null {
if (n % 2 === 0) return 2;
const big = BigInt(n);
// Deterministic sequence of offsets rather than a random one: the same
// input must factor the same way every run, or a test is a coin flip.
for (let offset = 1n; offset <= 40n; offset++) {
let x = 2n;
let y = 2n;
let divisor = 1n;
while (divisor === 1n) {
x = (x * x + offset) % big;
y = (y * y + offset) % big;
y = (y * y + offset) % big;
divisor = gcdBig(absBig(x - y), big);
}
if (divisor !== big) return Number(divisor);
}
return null;
}
function absBig(value: bigint): bigint {
return value < 0n ? -value : value;
}
function gcdBig(a: bigint, b: bigint): bigint {
let x = a;
let y = b;
while (y !== 0n) {
const next = x % y;
x = y;
y = next;
}
return x;
}
export function divisorsOf(value: number): number[] {
const magnitude = Math.abs(value);
if (magnitude <= 0) return [];
let result: number[] = [1];
for (const factor of factorize(magnitude)) {
const expanded: number[] = [];
let power = 1;
for (let step = 0; step <= factor.exponent; step++) {
expanded.push(...result.map((existing) => existing * power));
power *= factor.prime;
}
result = expanded;
}
return result.sort((a, b) => a - b);
}
/**
* σ(n) and τ(n) come free from the factorisation and are what "perfect
* number" and "abundant" questions actually need.
*/
export function divisorSum(value: number): number {
return factorize(Math.abs(value)).reduce((total, factor) => {
let term = 1;
let power = 1;
for (let step = 0; step < factor.exponent; step++) {
power *= factor.prime;
term += power;
}
return total * term;
}, 1);
}
export function divisorCount(value: number): number {
return factorize(Math.abs(value)).reduce((total, factor) => total * (factor.exponent + 1), 1);
}
/** Euler's totient. */
export function totient(value: number): number {
if (value <= 0) return 0;
let result = value;
for (const factor of factorize(value)) {
result = (result / factor.prime) * (factor.prime - 1);
}
return result;
}
// ---------------------------------------------------------------------------
// GCD / LCM
export function greatestCommonDivisor(a: number, b: number): number {
let x = Math.abs(a);
let y = Math.abs(b);
while (y !== 0) {
const next = x % y;
x = y;
y = next;
}
return x;
}
export function leastCommonMultiple(a: number, b: number): number {
if (a === 0 || b === 0) return 0;
return Math.abs((a / greatestCommonDivisor(a, b)) * b);
}
/** Extended Euclid: returns `[gcd, x, y]` with `ax + by = gcd`. */
export function extendedGCD(a: number, b: number): [number, number, number] {
if (b === 0) return [Math.abs(a), a < 0 ? -1 : 1, 0];
const [gcd, x, y] = extendedGCD(b, a % b);
return [gcd, y, x - Math.floor(a / b) * y];
}
// ---------------------------------------------------------------------------
// Modular arithmetic
/**
* Always non-negative, unlike `%`. JavaScript's `%` on a negative left
* operand returns a negative remainder, which is wrong for every modular use
* here.
*/
export function mod(value: number, modulus: number): number {
if (modulus === 0) return 0;
const remainder = value % modulus;
return remainder < 0 ? remainder + Math.abs(modulus) : remainder;
}
function modularPowerBig(base: bigint, exponent: bigint, modulus: bigint): bigint {
let result = 1n;
let b = base % modulus;
if (b < 0n) b += modulus;
let e = exponent;
while (e > 0n) {
if (e & 1n) result = (result * b) % modulus;
b = (b * b) % modulus;
e >>= 1n;
}
return result;
}
export function modularPower(base: number, exponent: number, modulus: number): number {
if (modulus <= 1) return 0;
if (exponent < 0) {
const inverse = modularInverse(base, modulus);
if (inverse === null) return 0;
return modularPower(inverse, -exponent, modulus);
}
return Number(modularPowerBig(BigInt(base), BigInt(exponent), BigInt(modulus)));
}
/**
* null when `value` and `modulus` are not coprime — an inverse genuinely
* does not exist, and returning 0 would look like an answer.
*/
export function modularInverse(value: number, modulus: number): number | null {
if (modulus <= 1) return null;
const [gcd, x] = extendedGCD(mod(value, modulus), modulus);
if (gcd !== 1) return null;
return mod(x, modulus);
}
// ---------------------------------------------------------------------------
// Sequences
/** null past fib(78) — the largest that stays an exact double. */
export function fibonacci(n: number): number | null {
if (n < 0) return null;
if (n === 0) return 0;
let a = 0;
let b = 1; // fib(0), fib(1)
// Stop one short and return `b`, so a representable answer is never
// reported as overflow by computing fib(n+1) as a side effect.
for (let step = 1; step < n; step++) {
const next = a + b;
if (!Number.isSafeInteger(next)) return null;
a = b;
b = next;
}
return b;
}
export function binomial(n: number, k: number): number | null {
if (n < 0 || k < 0 || k > n) return null;
const kk = Math.min(k, n - k);
let result = 1;
for (let step = 0; step < kk; step++) {
// Multiply then divide keeps every intermediate an integer, and the
// division is always exact at this point.
const product = result * (n - step);
if (!Number.isSafeInteger(product)) return null;
result = product / (step + 1);
}
return result;
}
/**
* Digit sum, digital root, and palindrome are the recreational-maths staples
* a spoken question actually reaches for.
*/
export function digitSum(value: number): number {
let remaining = Math.abs(value);
let total = 0;
while (remaining > 0) {
total += remaining % 10;
remaining = Math.floor(remaining / 10);
}
return total;
}
export function digitalRoot(value: number): number {
const magnitude = Math.abs(value);
return magnitude === 0 ? 0 : 1 + ((magnitude - 1) % 9);
}
export function isPalindrome(value: number): boolean {
const text = String(Math.abs(value));
return text === [...text].reverse().join("");
}
export function isPerfect(value: number): boolean {
return value > 1 && divisorSum(value) - value === value;
}
// ---------------------------------------------------------------------------
// Reporting
/** A full dossier on one integer, which is what "tell me about 5,040" means. */
export type IntegerReport = {
value: number;
isPrime: boolean;
factorization: PrimeFactor[];
divisorCount: number;
divisorSum: number;
totient: number;
digitSum: number;
digitalRoot: number;
isPerfect: boolean;
isPalindrome: boolean;
};
export function factorizationText(report: IntegerReport): string {
if (report.factorization.length === 0) return String(report.value);
return report.factorization
.map((factor) => (factor.exponent === 1 ? `${factor.prime}` : `${factor.prime}^${factor.exponent}`))
.join(" × ");
}
export function integerReport(value: number): IntegerReport {
return {
value,
isPrime: isPrime(value),
factorization: factorize(value),
divisorCount: divisorCount(value),
divisorSum: divisorSum(value),
totient: totient(Math.abs(value)),
digitSum: digitSum(value),
digitalRoot: digitalRoot(value),
isPerfect: isPerfect(value),
isPalindrome: isPalindrome(value),
};
}
+320
View File
@@ -0,0 +1,320 @@
// Recursive-descent parser over the token stream.
//
// Grammar, loosest to tightest:
//
// relation := sum (relop sum)?
// sum := product (('+' | '-') product)*
// product := unary (('*' | '/' | '%') unary)*
// unary := ('-' | '+') unary | power
// power := postfix ('^' unary)? -- right associative
// postfix := primary '!'*
// primary := number | constant | variable | call | '(' relation ')' | '|' sum '|'
//
// `^` binds tighter than unary minus on its left, so `-x^2` is `-(x^2)` — the
// convention every textbook and every other calculator uses. Getting that
// backwards silently changes the answer, so it has a test.
import { type MathToken, MathParseError, tokenize } from "./lexer";
import {
type MathExpression,
type MathFunction,
type MathConstant,
ALL_CONSTANTS,
ALL_FUNCTIONS,
addExpr,
constant,
divideExpr,
fn,
int,
multiplyExpr,
negate,
num,
powerExpr,
relation,
subtractExpr,
variable,
} from "./expression";
import { describeNumber } from "./math-number";
import { normalizeSpoken } from "./normalizer";
/**
* Guards against a pathological input recursing until the stack dies. A crash
* on the wearer's face is the worst possible failure mode.
*/
const MAXIMUM_DEPTH = 64;
export function parse(input: string): MathExpression {
const parser = new Parser(tokenize(input), input);
const expression = parser.parseRelation();
parser.expectEnd();
return expression;
}
/** Parse text that came from speech: normalise first, then parse. */
export function parseSpoken(spoken: string): MathExpression {
const normalized = normalizeSpoken(spoken);
if (!normalized) throw new MathParseError({ kind: "emptyInput" }, spoken);
try {
return parse(normalized);
} catch (error) {
// Report against what the user SAID, not against the normalised form they
// never typed — otherwise the error quotes machine output.
if (error instanceof MathParseError) {
throw new MathParseError(error.errorKind, spoken);
}
throw error;
}
}
class Parser {
private index = 0;
private depth = 0;
constructor(
private readonly tokens: MathToken[],
private readonly source: string,
) {}
// -------------------------------------------------------------------------
// Grammar
parseRelation(): MathExpression {
const lhs = this.parseSum();
const token = this.peek();
if (!token || token.type !== "relation") return lhs;
this.advance();
const rhs = this.parseSum();
// A chained relation ("1 < x < 5") is a real thing people say, but it is
// not a single relation node; rejecting it beats silently keeping only
// half of the constraint.
if (this.peek()?.type === "relation") {
throw new MathParseError({ kind: "unexpectedToken", token: "chained comparison" }, this.source);
}
return relation(token.relation, lhs, rhs);
}
private parseSum(): MathExpression {
const terms = [this.parseProduct()];
for (;;) {
const token = this.peek();
if (token?.type === "plus") {
this.advance();
terms.push(this.parseProduct());
} else if (token?.type === "minus") {
this.advance();
terms.push(negate(this.parseProduct()));
} else {
return terms.length === 1 ? terms[0]! : addExpr(terms);
}
}
}
private parseProduct(): MathExpression {
let factors = [this.parseUnary()];
for (;;) {
const token = this.peek();
if (token?.type === "star") {
this.advance();
factors.push(this.parseUnary());
} else if (token?.type === "slash") {
this.advance();
factors.push(powerExpr(this.parseUnary(), int(-1)));
} else if (token?.type === "percent") {
this.advance();
const divisor = this.parseUnary();
const dividend = factors.length === 1 ? factors[0]! : multiplyExpr(factors);
// a mod b == a - b * floor(a / b). Expressed in terms the rest of the
// engine already handles, so modulo needs no special case in the
// evaluator, simplifier, or differentiator.
const quotient = fn("floor", [divideExpr(dividend, divisor)]);
factors = [subtractExpr(dividend, multiplyExpr([divisor, quotient]))];
} else {
return factors.length === 1 ? factors[0]! : multiplyExpr(factors);
}
}
}
private parseUnary(): MathExpression {
const token = this.peek();
if (token?.type === "minus") {
this.advance();
return negate(this.parseUnary());
}
if (token?.type === "plus") {
this.advance();
return this.parseUnary();
}
return this.parsePower();
}
private parsePower(): MathExpression {
const base = this.parsePostfix();
if (this.peek()?.type !== "caret") return base;
this.advance();
// Right-associative AND allows a unary exponent: 2^-1, 2^3^2 = 2^(3^2).
const exponent = this.parseUnary();
return powerExpr(base, exponent);
}
private parsePostfix(): MathExpression {
let expression = this.parsePrimary();
while (this.peek()?.type === "bang") {
this.advance();
expression = fn("factorial", [expression]);
}
return expression;
}
private parsePrimary(): MathExpression {
this.depth += 1;
try {
if (this.depth >= MAXIMUM_DEPTH) {
throw new MathParseError({ kind: "tooComplex" }, this.source);
}
const token = this.peek();
if (!token) throw new MathParseError({ kind: "unexpectedEnd" }, this.source);
switch (token.type) {
case "number":
this.advance();
return num(token.value);
case "identifier":
this.advance();
return this.parseIdentifier(token.name);
case "leftParen": {
this.advance();
const inner = this.parseRelation();
this.expect("rightParen");
return inner;
}
case "pipe": {
// |x| — absolute value. The closing pipe is unambiguous because a
// bare `|` is not an operator anywhere else in the grammar.
this.advance();
const inner = this.parseSum();
this.expect("pipe");
return fn("abs", [inner]);
}
case "minus":
case "plus":
return this.parseUnary();
default:
throw new MathParseError({ kind: "unexpectedToken", token: describeToken(token) }, this.source);
}
} finally {
this.depth -= 1;
}
}
private parseIdentifier(name: string): MathExpression {
const lowered = name.toLowerCase();
if ((ALL_CONSTANTS as readonly string[]).includes(lowered)) {
return constant(lowered as MathConstant);
}
// `log` with no base means base 10 in ordinary speech, not natural log.
const resolved = lowered === "log" ? "log10" : lowered;
if ((ALL_FUNCTIONS as readonly string[]).includes(resolved)) {
const functionName = resolved as MathFunction;
// Both `sin(x)` and the spoken-normalised `sin x` must parse.
if (this.peek()?.type === "leftParen") {
this.advance();
const args = [this.parseSum()];
while (this.peek()?.type === "comma") {
this.advance();
args.push(this.parseSum());
}
this.expect("rightParen");
return fn(functionName, args);
}
const argument = this.parsePower();
return fn(functionName, [argument]);
}
// A run of single letters from the normaliser ("xy") is a product of
// variables, not one variable named "xy". Multi-letter names are only
// honoured when the caller typed them with an underscore or a digit.
if (lowered.length > 1 && !lowered.includes("_") && !/\d/.test(lowered)) {
const factors = lowered.split("").map((letter) => variable(letter));
return multiplyExpr(factors);
}
return variable(lowered);
}
// -------------------------------------------------------------------------
// Token plumbing
private peek(): MathToken | null {
return this.index < this.tokens.length ? this.tokens[this.index]! : null;
}
private advance(): void {
this.index += 1;
}
private expect(type: MathToken["type"]): void {
const actual = this.peek();
if (!actual) {
throw new MathParseError(
type === "rightParen" ? { kind: "unbalancedParenthesis" } : { kind: "unexpectedEnd" },
this.source,
);
}
if (actual.type !== type) {
throw new MathParseError(
type === "rightParen"
? { kind: "unbalancedParenthesis" }
: { kind: "unexpectedToken", token: describeToken(actual) },
this.source,
);
}
this.advance();
}
expectEnd(): void {
if (this.index < this.tokens.length) {
throw new MathParseError(
{ kind: "unexpectedToken", token: describeToken(this.tokens[this.index]!) },
this.source,
);
}
}
}
function describeToken(token: MathToken): string {
switch (token.type) {
case "number":
return describeNumber(token.value);
case "identifier":
return token.name;
case "plus":
return "+";
case "minus":
return "-";
case "star":
return "*";
case "slash":
return "/";
case "percent":
return "%";
case "caret":
return "^";
case "bang":
return "!";
case "leftParen":
return "(";
case "rightParen":
return ")";
case "pipe":
return "|";
case "comma":
return ",";
case "relation":
return token.relation;
}
}
+305
View File
@@ -0,0 +1,305 @@
// Samples an expression into drawable graph data: segments, viewport,
// roots, and extrema.
import { type MathExpression, subtractExpr, variablesOf } from "./expression";
import { evaluateValue } from "./evaluator";
import { simplify } from "./simplifier";
import { Polynomial } from "./polynomial";
import { numericRoots } from "./solver";
import { fromDecimal, describeNumber } from "./math-number";
import { plain } from "./text-renderer";
// ---------------------------------------------------------------------------
// Plot model
export type GraphViewport = {
xMin: number;
xMax: number;
yMin: number;
yMax: number;
};
export const STANDARD_VIEWPORT: GraphViewport = { xMin: -10, xMax: 10, yMin: -10, yMax: 10 };
export function viewportContainsY(viewport: GraphViewport, y: number): boolean {
return y >= viewport.yMin && y <= viewport.yMax;
}
export type GraphPoint = { x: number; y: number };
/**
* One continuous run of the curve. A function with a pole produces several,
* so the renderer can lift the pen instead of drawing a vertical line through
* the asymptote — which is the single most recognisable "bad graphing app"
* artefact.
*/
export type GraphSegment = { points: GraphPoint[] };
export type GraphMarkerKind = "root" | "yIntercept" | "extremum" | "intersection";
export type GraphMarker = {
kind: GraphMarkerKind;
point: GraphPoint;
label: string;
};
export type GraphPlot = {
expression: MathExpression;
variable: string;
segments: GraphSegment[];
markers: GraphMarker[];
label: string;
};
export type GraphAxis = { ticks: number[]; step: number };
export type Graph = {
plots: GraphPlot[];
viewport: GraphViewport;
xAxis: GraphAxis;
yAxis: GraphAxis;
};
// ---------------------------------------------------------------------------
// Plotter
/**
* Sample count. Matched to the glasses viewport width times a small factor:
* enough that a curve looks smooth after projection, few enough that plotting
* stays imperceptible on a phone.
*/
export const DEFAULT_SAMPLE_COUNT = 1_200;
/** Values beyond this are treated as a pole rather than a steep slope. */
const POLE_THRESHOLD = 1e7;
export function plot(
expressions: MathExpression[],
options: {
variable?: string;
viewport?: GraphViewport;
usesDegrees?: boolean;
sampleCount?: number;
} = {},
): Graph {
const usesDegrees = options.usesDegrees ?? false;
const sampleCount = options.sampleCount ?? DEFAULT_SAMPLE_COUNT;
const variable =
options.variable ??
expressions.map((expression) => variablesOf(expression)[0]).find((name) => name !== undefined) ??
"x";
// Autoscale y from the data when the caller did not pin a viewport, but
// keep x at the requested (or standard) window — rescaling both makes a
// graph that never matches what the user asked to see.
const xWindow = options.viewport ?? STANDARD_VIEWPORT;
const plots: GraphPlot[] = [];
const allY: number[] = [];
for (const expression of expressions) {
const body = functionBody(expression);
const sampled = sample(body, variable, xWindow.xMin, xWindow.xMax, sampleCount, usesDegrees);
for (const segment of sampled) {
for (const point of segment.points) allY.push(point.y);
}
plots.push({
expression: body,
variable,
segments: sampled,
markers: markers(body, variable, xWindow.xMin, xWindow.xMax),
label: plain(body),
});
}
const viewport = options.viewport ?? autoscale(xWindow, allY);
const clipped = plots.map((entry) => ({
expression: entry.expression,
variable: entry.variable,
segments: clip(entry.segments, viewport),
markers: entry.markers.filter((marker) => viewportContainsY(viewport, marker.point.y)),
label: entry.label,
}));
return {
plots: clipped,
viewport,
xAxis: axis(viewport.xMin, viewport.xMax),
yAxis: axis(viewport.yMin, viewport.yMax),
};
}
/**
* `y = x^2 + 1` and `x^2 + 1` should plot the same curve, and `x^2 + y = 1`
* should plot `y = 1 - x^2` where that is easy to see.
*/
export function functionBody(expression: MathExpression): MathExpression {
if (expression.type !== "relation") return expression;
if (expression.lhs.type === "variable" && expression.lhs.name === "y") return expression.rhs;
if (expression.rhs.type === "variable" && expression.rhs.name === "y") return expression.lhs;
// Otherwise plot the difference, whose zero set is the relation — the
// honest generalisation, and it makes an implicit curve's roots visible on
// the x-axis.
return simplify(subtractExpr(expression.lhs, expression.rhs));
}
// ---------------------------------------------------------------------------
// Sampling
export function sample(
expression: MathExpression,
variable: string,
xMin: number,
xMax: number,
count: number,
usesDegrees: boolean,
): GraphSegment[] {
if (count <= 1 || xMax <= xMin) return [];
const segments: GraphSegment[] = [];
let current: GraphPoint[] = [];
const step = (xMax - xMin) / (count - 1);
let previousY: number | null = null;
for (let index = 0; index < count; index++) {
const x = xMin + index * step;
const y = evaluateValue(expression, { [variable]: x }, usesDegrees);
if (y === null || !Number.isFinite(y) || Math.abs(y) >= POLE_THRESHOLD) {
// Undefined or blown up: close the run.
if (current.length > 1) segments.push({ points: current });
current = [];
previousY = null;
continue;
}
// A jump this large between adjacent samples is a pole, not a slope.
// tan(x) is the canonical case: without this the curve gets a vertical
// line at every asymptote.
if (previousY !== null && Math.abs(y - previousY) > (xMax - xMin) * 50) {
if (current.length > 1) segments.push({ points: current });
current = [];
}
current.push({ x, y });
previousY = y;
}
if (current.length > 1) segments.push({ points: current });
return segments;
}
// ---------------------------------------------------------------------------
// Autoscale
/**
* Percentile-based rather than min/max: one spike near a pole would
* otherwise compress the interesting part of the curve into a flat line.
*/
export function autoscale(xWindow: GraphViewport, yValues: number[]): GraphViewport {
const finite = yValues.filter((y) => Number.isFinite(y) && Math.abs(y) < POLE_THRESHOLD).sort((a, b) => a - b);
if (finite.length <= 2) {
return { xMin: xWindow.xMin, xMax: xWindow.xMax, yMin: -10, yMax: 10 };
}
const lowIndex = Math.floor(finite.length * 0.02);
const highIndex = Math.floor(finite.length * 0.98);
let low = finite[Math.min(lowIndex, finite.length - 1)]!;
let high = finite[Math.min(highIndex, finite.length - 1)]!;
// Always show the x-axis when the curve comes near it — a graph that omits
// y = 0 reads as if the function never crosses.
if (low > 0 && low < high * 0.5) low = 0;
if (high < 0 && high > low * 0.5) high = 0;
if (high - low < 1e-9) {
const centre = (high + low) / 2;
low = centre - 1;
high = centre + 1;
}
const padding = (high - low) * 0.1;
return {
xMin: xWindow.xMin,
xMax: xWindow.xMax,
yMin: low - padding,
yMax: high + padding,
};
}
export function clip(segments: GraphSegment[], viewport: GraphViewport): GraphSegment[] {
const result: GraphSegment[] = [];
for (const segment of segments) {
let current: GraphPoint[] = [];
for (const point of segment.points) {
if (viewportContainsY(viewport, point.y)) {
current.push(point);
} else if (current.length > 0) {
// Keep one out-of-range point so the line exits the frame rather than
// stopping short of the edge.
current.push(point);
result.push({ points: current });
current = [];
}
}
if (current.length > 1) result.push({ points: current });
}
return result;
}
// ---------------------------------------------------------------------------
// Axes
/**
* "Nice" tick spacing — 1, 2, 5, 10 × 10^n — so labels are round numbers a
* person can read at a glance on the lens.
*/
export function axis(lower: number, upper: number, targetTicks = 10): GraphAxis {
const span = upper - lower;
if (span <= 0 || !Number.isFinite(span)) return { ticks: [], step: 1 };
const rough = span / targetTicks;
const magnitude = Math.pow(10, Math.floor(Math.log10(rough)));
const normalized = rough / magnitude;
let step: number;
if (normalized <= 1) step = magnitude;
else if (normalized <= 2) step = 2 * magnitude;
else if (normalized <= 5) step = 5 * magnitude;
else step = 10 * magnitude;
const ticks: number[] = [];
let value = Math.ceil(lower / step) * step;
while (value <= upper && ticks.length < 64) {
// Snap a near-zero tick to exactly zero so the axis line and its label
// agree.
ticks.push(Math.abs(value) < step * 1e-9 ? 0 : value);
value += step;
}
return { ticks, step };
}
// ---------------------------------------------------------------------------
// Markers
export function markers(
expression: MathExpression,
variable: string,
xMin: number,
xMax: number,
): GraphMarker[] {
const result: GraphMarker[] = [];
const yIntercept = evaluateValue(expression, { [variable]: 0 });
if (yIntercept !== null && Number.isFinite(yIntercept) && xMin <= 0 && xMax >= 0) {
result.push({
kind: "yIntercept",
point: { x: 0, y: yIntercept },
label: `(0, ${describeNumber(fromDecimal(yIntercept))})`,
});
}
// Roots: exact when the curve is a polynomial, sampled otherwise.
const polynomial = Polynomial.extract(expression, variable);
if (polynomial) {
for (const root of numericRoots(polynomial, [xMin, xMax], 4_000)) {
result.push({
kind: "root",
point: { x: root, y: 0 },
label: `(${describeNumber(fromDecimal(root))}, 0)`,
});
}
}
return result.slice(0, 12);
}
+247
View File
@@ -0,0 +1,247 @@
// Dense coefficient vector in one variable, `coefficients[i]` multiplying
// `x^i`. Extracting one of these is how the solver decides what technique
// applies — degree 1 is a rearrangement, degree 2 is the quadratic formula,
// higher degrees go to rational-root plus a numeric fallback.
import {
type MathExpression,
addExpr,
constantValue,
int,
multiplyExpr,
negate,
num,
powerExpr,
variable as variableExpr,
} from "./expression";
import {
type MathNumber,
add,
doubleValue,
greatestCommonDivisor,
integer,
integerValue,
isOne,
isZero,
multiply,
negated,
numberEquals,
real,
} from "./math-number";
export class Polynomial {
readonly coefficients: MathNumber[];
readonly variable: string;
constructor(coefficients: MathNumber[], variable: string) {
this.coefficients = [...coefficients];
this.variable = variable;
while (this.coefficients.length > 1 && isZero(this.coefficients[this.coefficients.length - 1]!)) {
this.coefficients.pop();
}
if (this.coefficients.length === 0) this.coefficients.push(integer(0));
}
get degree(): number {
return this.coefficients.length - 1;
}
get isZero(): boolean {
return this.coefficients.every(isZero);
}
at(power: number): MathNumber {
return power < this.coefficients.length ? this.coefficients[power]! : integer(0);
}
/**
* Build from an expression, or null when it is not a polynomial in
* `variable` (a variable inside `sin`, a symbolic exponent, `1/x`).
*/
static extract(expression: MathExpression, variable: string): Polynomial | null {
const coefficients: MathNumber[] = [];
if (!accumulate(expression, variable, integer(1), coefficients)) return null;
return new Polynomial(coefficients, variable);
}
// -------------------------------------------------------------------------
// Evaluation
/**
* Horner's method — fewer operations and less rounding drift than summing
* `a_i * x^i` term by term, which matters for the numeric root refinement.
*/
value(x: number): number {
let result = 0;
for (let index = this.coefficients.length - 1; index >= 0; index--) {
result = result * x + doubleValue(this.coefficients[index]!);
}
return result;
}
get derivative(): Polynomial {
if (this.degree <= 0) return new Polynomial([integer(0)], this.variable);
const derived: MathNumber[] = [];
for (let power = 1; power <= this.degree; power++) {
derived.push(multiply(this.at(power), integer(power)));
}
return new Polynomial(derived, this.variable);
}
expression(): MathExpression {
const terms: MathExpression[] = [];
for (let power = this.degree; power >= 0; power--) {
const coefficient = this.at(power);
if (isZero(coefficient)) continue;
let variablePart: MathExpression | null;
if (power === 0) {
variablePart = null;
} else if (power === 1) {
variablePart = variableExpr(this.variable);
} else {
variablePart = powerExpr(variableExpr(this.variable), int(power));
}
if (!variablePart) {
terms.push(num(coefficient));
continue;
}
if (isOne(coefficient)) {
terms.push(variablePart);
} else if (numberEquals(coefficient, integer(-1))) {
terms.push(negate(variablePart));
} else {
terms.push(multiplyExpr([num(coefficient), variablePart]));
}
}
if (terms.length === 0) return int(0);
return terms.length === 1 ? terms[0]! : addExpr(terms);
}
/**
* Divide out a known root, returning the depressed polynomial. Used after a
* rational root is found so the remainder can be solved in closed form.
*/
dividingByRoot(root: MathNumber): Polynomial {
if (this.degree < 1) return this;
const quotient: MathNumber[] = new Array(this.degree).fill(integer(0));
let carry: MathNumber = integer(0);
for (let power = this.degree; power >= 1; power--) {
const coefficient = add(this.at(power), carry);
quotient[power - 1] = coefficient;
carry = multiply(coefficient, root);
}
return new Polynomial(quotient, this.variable);
}
/**
* Integer coefficients after clearing denominators, for the rational root
* theorem. Only meaningful when every coefficient is exact.
*/
get integerCoefficients(): number[] | null {
let multiplier = 1;
for (const coefficient of this.coefficients) {
if (coefficient.kind === "integer") continue;
if (coefficient.kind === "real") return null;
multiplier = (multiplier / greatestCommonDivisor(multiplier, coefficient.d)) * coefficient.d;
if (multiplier >= 1_000_000) return null;
}
const scaled: number[] = [];
for (const coefficient of this.coefficients) {
const value = integerValue(multiply(coefficient, integer(multiplier)));
if (value === null) return null;
scaled.push(value);
}
return scaled;
}
}
function accumulate(
expression: MathExpression,
variable: string,
scale: MathNumber,
coefficients: MathNumber[],
): boolean {
switch (expression.type) {
case "number":
addAt(multiply(scale, expression.value), 0, coefficients);
return true;
case "constant":
addAt(multiply(scale, real(constantValue(expression.constant))), 0, coefficients);
return true;
case "variable":
if (expression.name === variable) {
addAt(scale, 1, coefficients);
return true;
}
// Another free variable makes this not a single-variable polynomial —
// refuse rather than silently treating it as a constant, which would
// produce a confidently wrong root.
return false;
case "negate":
return accumulate(expression.inner, variable, negated(scale), coefficients);
case "add":
for (const term of expression.terms) {
if (!accumulate(term, variable, scale, coefficients)) return false;
}
return true;
case "multiply": {
// Multiply out: each factor must itself be a polynomial, and the
// product of their coefficient vectors is the result.
let product: MathNumber[] = [integer(1)];
for (const factor of expression.factors) {
const factorCoefficients: MathNumber[] = [];
if (!accumulate(factor, variable, integer(1), factorCoefficients)) return false;
product = convolve(product, factorCoefficients);
if (product.length > 64) return false;
}
for (let power = 0; power < product.length; power++) {
addAt(multiply(scale, product[power]!), power, coefficients);
}
return true;
}
case "power": {
const exponentNumber = expression.exponent.type === "number" ? expression.exponent.value : null;
const power = exponentNumber ? integerValue(exponentNumber) : null;
if (power === null || power < 0 || power > 32) return false;
const baseCoefficients: MathNumber[] = [];
if (!accumulate(expression.base, variable, integer(1), baseCoefficients)) return false;
let product: MathNumber[] = [integer(1)];
for (let step = 0; step < power; step++) {
product = convolve(product, baseCoefficients);
if (product.length > 64) return false;
}
for (let index = 0; index < product.length; index++) {
addAt(multiply(scale, product[index]!), index, coefficients);
}
return true;
}
case "function":
case "relation":
return false;
}
}
function addAt(value: MathNumber, power: number, coefficients: MathNumber[]): void {
while (coefficients.length <= power) coefficients.push(integer(0));
coefficients[power] = add(coefficients[power]!, value);
}
function convolve(lhs: MathNumber[], rhs: MathNumber[]): MathNumber[] {
if (lhs.length === 0 || rhs.length === 0) return [integer(0)];
const result: MathNumber[] = new Array(lhs.length + rhs.length - 1).fill(integer(0));
for (let i = 0; i < lhs.length; i++) {
if (isZero(lhs[i]!)) continue;
for (let j = 0; j < rhs.length; j++) {
if (isZero(rhs[j]!)) continue;
result[i + j] = add(result[i + j]!, multiply(lhs[i]!, rhs[j]!));
}
}
return result;
}
+509
View File
@@ -0,0 +1,509 @@
// The command parser and the stateful session. Stateful on purpose: "graph
// it" is only meaningful with respect to the problem already on screen, and
// requiring the wearer to restate the equation to see its graph would defeat
// the point.
import {
type MathExpression,
variablesOf,
} from "./expression";
import { describeNumber, fromDecimal } from "./math-number";
import { type AdvancedTopic, answerAdvanced, parseAdvanced } from "./advanced";
import { type EvaluatorContext, radiansContext } from "./evaluator";
import { parseSpoken } from "./parser";
import { MathParseError } from "./lexer";
import {
type MathSolution,
type SolutionStep,
solutionHeadline,
solve as solveExpression,
} from "./solver";
import { type Graph, type GraphViewport, functionBody, plot } from "./plotter";
import { differentiate, integrateNumeric } from "./derivative";
import { plain } from "./text-renderer";
import { type MathWorkload } from "./workload";
// ---------------------------------------------------------------------------
// Commands
/**
* What the wearer asked for. Parsed from speech so "graph it" and "explain
* it" work as follow-ups to a problem already on the glasses, without
* restating it.
*/
export type MathCommand =
/** A new problem to solve or evaluate. */
| { kind: "problem"; text: string }
| { kind: "graph" }
| { kind: "explain" }
| { kind: "nextStep" }
| { kind: "previousStep" }
| { kind: "derivative"; variable: string | null }
| { kind: "integrate"; variable: string | null; from: number; to: number }
| { kind: "restate" }
| { kind: "clear" }
/**
* A named advanced topic — primality, an integral, a limit, a shape's
* area. These carry their own arguments rather than acting on a standing
* expression, which is why they are a separate case.
*/
| { kind: "advanced"; topic: AdvancedTopic };
/**
* Recognise a follow-up. Returns null when the utterance is not a math
* command at all, so the caller can leave unrelated speech alone.
*/
export function parseCommand(utterance: string): MathCommand | null {
const text = utterance.toLowerCase().replace(/^[ .,!?]+|[ .,!?]+$/g, "");
// Follow-ups first: they are short and would otherwise be parsed as
// (nonsense) expressions.
//
// The bare verb is listed alongside every "it"/"that"/"this" form on
// purpose: these are the three commands a wearer is TOLD they can say, and
// someone who has been shown "Graph" will say "graph" as often as
// "graph it". Leaving the bare form out makes the feature look broken in
// exactly the case the user was taught.
switch (text) {
case "graph":
case "graph it":
case "graph that":
case "graph this":
case "plot":
case "plot it":
case "plot that":
case "plot this":
case "show the graph":
case "draw it":
case "show me the graph":
return { kind: "graph" };
case "explain":
case "explain it":
case "explain that":
case "explain this":
case "show the steps":
case "show your work":
case "how did you get that":
case "walk me through it":
case "step by step":
return { kind: "explain" };
// "Solve it" on a problem already on the lens is a request for the answer
// again, not a re-parse — `restate` is exactly that, and routing it here
// means the standing problem is never silently re-solved with a fresh
// (and possibly worse) reading of the same words.
case "solve":
case "solve it":
case "solve that":
case "solve this":
case "work it out":
case "what is the answer":
case "what's the answer":
case "whats the answer":
case "answer it":
return { kind: "restate" };
case "next":
case "next step":
case "continue":
case "go on":
case "and then":
return { kind: "nextStep" };
case "back":
case "previous":
case "previous step":
case "go back":
case "last step":
return { kind: "previousStep" };
case "say it again":
case "repeat that":
case "what was it":
case "restate":
case "again":
return { kind: "restate" };
case "clear":
case "reset":
case "start over":
case "never mind":
case "forget it":
return { kind: "clear" };
default:
break;
}
const derivative = matchDerivative(text);
if (derivative !== null) return { kind: "derivative", variable: derivative };
// Advanced topics BEFORE the plain-integral shortcut and the
// looks-like-maths gate: "the integral of x squared from 0 to 3" and "is 91
// prime" both contain digits and operator words, and the expression parser
// would make nonsense of either.
const topic = parseAdvanced(utterance);
if (topic) return { kind: "advanced", topic };
const integral = matchIntegral(text);
if (integral) return integral;
// Anything with a digit, an operator word, or a relation is a problem.
if (!looksLikeMath(text)) return null;
return { kind: "problem", text: utterance };
}
/** Distinguishes "no derivative request" (null) from "derivative of the default variable". */
function matchDerivative(text: string): string | null {
const triggers = [
"differentiate",
"take the derivative",
"the derivative",
"derivative of",
"d by d",
"find the derivative",
];
if (!triggers.some((trigger) => text.includes(trigger))) return null;
// "derivative with respect to t"
const marker = "with respect to ";
const index = text.indexOf(marker);
if (index >= 0) {
const tail = text.slice(index + marker.length).trim();
const first = tail.split(/\s+/)[0];
if (first && first.length <= 12) return first;
}
return "";
}
function matchIntegral(text: string): MathCommand | null {
if (!text.includes("integral") && !text.includes("integrate")) return null;
// "integrate from 0 to 5"
const match = text.match(/from\s+(-?[0-9.]+)\s+to\s+(-?[0-9.]+)/);
if (!match) return null;
const lower = Number(match[1]);
const upper = Number(match[2]);
if (!Number.isFinite(lower) || !Number.isFinite(upper)) return null;
return { kind: "integrate", variable: null, from: lower, to: upper };
}
/**
* A cheap gate so ordinary conversation is not parsed as arithmetic. The
* continuous-listening path hands this every utterance in the room; without
* the gate, "I'll see you at four" becomes a math problem.
*/
export function looksLikeMath(text: string): boolean {
const operatorWords = [
"plus",
"minus",
"times",
"divided",
"equals",
"squared",
"cubed",
"square root",
"solve",
"what is",
"calculate",
"percent",
"factorial",
"to the power",
"log",
"sine",
"cosine",
"tangent",
];
const hasOperatorWord = operatorWords.some((word) => text.includes(word));
const hasSymbol = /[+\-*/^=<>]/.test(text);
const hasDigit = /\d/.test(text);
// Spelled-out numbers count too. "what is two plus two" has an operator
// word and no digit at all, and requiring a digit made the single most
// obvious test phrase fall straight through the gate.
const numberWords = [
"zero",
"one",
"two",
"three",
"four",
"five",
"six",
"seven",
"eight",
"nine",
"ten",
"eleven",
"twelve",
"twenty",
"thirty",
"forty",
"fifty",
"hundred",
"thousand",
"million",
];
const hasNumberWord = numberWords.some((word) => new RegExp(`\\b${word}\\b`).test(text));
// A bare number is not a problem; a number with an operation is.
return (hasDigit || hasSymbol || hasNumberWord) && (hasOperatorWord || hasSymbol);
}
// ---------------------------------------------------------------------------
// Result
export type MathSessionPayload =
| { kind: "solution"; solution: MathSolution }
| { kind: "graph"; graph: Graph }
| { kind: "step"; step: SolutionStep; index: number; total: number }
| { kind: "message"; message: string }
| { kind: "failure"; message: string }
/**
* An advanced-topic answer, which has its own field list rather than a
* solution's step sequence.
*/
| { kind: "topic"; headline: string; detail: string; fields: string[] };
export type MathSessionResult = {
payload: MathSessionPayload;
/** One short line for the glasses. */
headline: string;
/** Fuller text for the answer view and for spoken readback. */
detail: string;
/**
* Work this answer could not finish at the fast depth. The caller decides
* whether to offer the wearer a longer run, or let the shallow answer
* stand.
*/
unresolvedWorkload: MathWorkload | null;
};
function result(
payload: MathSessionPayload,
headline: string,
detail: string,
unresolvedWorkload: MathWorkload | null = null,
): MathSessionResult {
return { payload, headline, detail, unresolvedWorkload };
}
// ---------------------------------------------------------------------------
// Session
/** Holds one problem and everything derived from it, so follow-ups work. */
export class MathSession {
private expression: MathExpression | null = null;
private solution: MathSolution | null = null;
private stepIndex = 0;
private usesDegrees: boolean;
constructor(usesDegrees = false) {
this.usesDegrees = usesDegrees;
}
get hasProblem(): boolean {
return this.expression !== null;
}
get currentSolution(): MathSolution | null {
return this.solution;
}
setUsesDegrees(value: boolean): void {
this.usesDegrees = value;
}
private get context(): EvaluatorContext {
const context = radiansContext();
context.usesDegrees = this.usesDegrees;
return context;
}
// Entry point --------------------------------------------------------------
handle(utterance: string): MathSessionResult | null {
const command = parseCommand(utterance);
if (!command) return null;
return this.execute(command);
}
execute(command: MathCommand): MathSessionResult {
switch (command.kind) {
case "problem":
return this.solve(command.text);
case "graph":
return this.graph();
case "explain":
return this.explain();
case "nextStep":
return this.moveStep(1);
case "previousStep":
return this.moveStep(-1);
case "derivative":
return this.differentiate(command.variable === "" ? null : command.variable);
case "integrate":
return this.integrate(command.variable, command.from, command.to);
case "restate":
return this.restate();
case "advanced": {
const answer = answerAdvanced(command.topic);
return result(
{
kind: "topic",
headline: answer.headline,
detail: answer.detail,
fields: answer.fields.map(([label, value]) => `${label}: ${value}`),
},
answer.headline,
answer.detail,
answer.unresolvedWorkload,
);
}
case "clear":
this.expression = null;
this.solution = null;
this.stepIndex = 0;
return result({ kind: "message", message: "Cleared" }, "Cleared", "Ready for a new problem.");
}
}
// Commands -----------------------------------------------------------------
solve(text: string): MathSessionResult {
let parsed: MathExpression;
try {
parsed = parseSpoken(text);
} catch (error) {
const message =
error instanceof MathParseError ? error.message : String((error as Error)?.message ?? error);
return result({ kind: "failure", message }, "Could not read that", message);
}
const solved = solveExpression(parsed, null, this.context);
this.expression = parsed;
this.solution = solved;
this.stepIndex = 0;
return result(
{ kind: "solution", solution: solved },
solutionHeadline(solved),
`${plain(solved.restatement)} → ${solutionHeadline(solved)}`,
);
}
graph(viewport: GraphViewport | null = null): MathSessionResult {
if (!this.expression) return this.noProblem();
const plotted = plot([this.expression], {
viewport: viewport ?? undefined,
usesDegrees: this.usesDegrees,
});
const pointCount = plotted.plots.reduce(
(total, entry) => total + entry.segments.reduce((count, segment) => count + segment.points.length, 0),
0,
);
if (pointCount <= 1) {
return result(
{ kind: "failure", message: "Nothing to plot" },
"Nothing to plot",
"That expression has no real values in the visible window.",
);
}
const label = plotted.plots[0]?.label ?? "";
return result(
{ kind: "graph", graph: plotted },
`y = ${label}`,
`x from ${format(plotted.viewport.xMin)} to ${format(plotted.viewport.xMax)}, ` +
`y from ${format(plotted.viewport.yMin)} to ${format(plotted.viewport.yMax)}`,
);
}
explain(): MathSessionResult {
const solution = this.solution;
if (!solution || solution.steps.length === 0) return this.noProblem();
this.stepIndex = 0;
return this.stepResult(0, solution);
}
private moveStep(delta: number): MathSessionResult {
const solution = this.solution;
if (!solution || solution.steps.length === 0) return this.noProblem();
// Clamp rather than wrap: a wearer scrolling past the end should land on
// the answer and stay there, not loop back to the start.
this.stepIndex = Math.max(0, Math.min(solution.steps.length - 1, this.stepIndex + delta));
return this.stepResult(this.stepIndex, solution);
}
private stepResult(index: number, solution: MathSolution): MathSessionResult {
const step = solution.steps[index]!;
return result(
{ kind: "step", step, index, total: solution.steps.length },
`${index + 1}/${solution.steps.length} ${step.title}`,
`${plain(step.expression)}\n${step.detail}`,
);
}
differentiate(requestedVariable: string | null): MathSessionResult {
if (!this.expression) return this.noProblem();
const body = functionBody(this.expression);
const variable = requestedVariable ?? variablesOf(body)[0] ?? "x";
const derived = differentiate(body, variable);
const derivedSolution: MathSolution = {
kind: { kind: "simplified", value: derived },
variable,
steps: [
{
id: 0,
title: "Original",
expression: body,
detail: `Differentiate with respect to ${variable}.`,
},
{
id: 1,
title: "Derivative",
expression: derived,
detail: "Apply the power, product, and chain rules, then simplify.",
},
],
restatement: body,
};
this.expression = derived;
this.solution = derivedSolution;
this.stepIndex = 0;
return result(
{ kind: "solution", solution: derivedSolution },
`d/d${variable} = ${plain(derived)}`,
`The derivative of ${plain(body)} with respect to ${variable} is ${plain(derived)}.`,
);
}
integrate(requestedVariable: string | null, lower: number, upper: number): MathSessionResult {
if (!this.expression) return this.noProblem();
const body = functionBody(this.expression);
const variable = requestedVariable ?? variablesOf(body)[0] ?? "x";
const area = integrateNumeric(body, variable, lower, upper);
if (area === null) {
return result(
{ kind: "failure", message: "Could not integrate over that range" },
"Could not integrate",
`The function is undefined somewhere between ${format(lower)} and ${format(upper)}.`,
);
}
const value = fromDecimal(area);
return result(
{ kind: "message", message: describeNumber(value) },
`∫ = ${describeNumber(value)}`,
`The definite integral of ${plain(body)} d${variable} ` +
`from ${format(lower)} to ${format(upper)} is ${describeNumber(value)}.`,
);
}
restate(): MathSessionResult {
const solution = this.solution;
if (!solution) return this.noProblem();
return result(
{ kind: "solution", solution },
solutionHeadline(solution),
`${plain(solution.restatement)} → ${solutionHeadline(solution)}`,
);
}
// Helpers ------------------------------------------------------------------
private noProblem(): MathSessionResult {
return result(
{ kind: "failure", message: "No problem yet" },
"No problem yet",
"Say an equation first, then ask to graph or explain it.",
);
}
}
function format(value: number): string {
return describeNumber(fromDecimal(value));
}
+361
View File
@@ -0,0 +1,361 @@
// Canonicalises an expression tree. The goal is not a full CAS — it is a
// *stable canonical form*, because three downstream features depend on one:
//
// * the solver compares subtrees for equality to collect like terms,
// * the step explanations need `2x + 3x` to visibly become `5x`,
// * the renderers should not print `1 * x + 0`.
//
// Rewriting runs to a fixed point with an iteration cap, so a rule pair that
// oscillates degrades to "stopped early" rather than hanging the app.
import {
type MathExpression,
type MathFunction,
ONE,
ZERO,
addExpr,
expressionEquals,
fn,
functionIsDefined,
evaluateFunction,
isOneExpr,
isZeroExpr,
multiplyExpr,
negate,
num,
numberValue,
powerExpr,
relation,
variablesOf,
} from "./expression";
import {
type MathNumber,
add,
doubleValue,
fromDecimal,
integer,
integerValue,
isExact,
isOne,
isZero,
isNegative,
multiply,
negated,
numberEquals,
raisedTo,
rational,
} from "./math-number";
const MAXIMUM_PASSES = 24;
export function simplify(expression: MathExpression): MathExpression {
let current = expression;
for (let iteration = 0; iteration < MAXIMUM_PASSES; iteration++) {
const next = pass(current);
if (expressionEquals(next, current)) return next;
current = next;
}
return current;
}
// ---------------------------------------------------------------------------
// One rewrite pass
function pass(expression: MathExpression): MathExpression {
switch (expression.type) {
case "number":
case "variable":
case "constant":
return expression;
case "negate": {
const simplified = pass(expression.inner);
const value = numberValue(simplified);
if (value) return num(negated(value));
// Double negation.
if (simplified.type === "negate") return simplified.inner;
return negate(simplified);
}
case "add":
return simplifySum(expression.terms.map(pass));
case "multiply":
return simplifyProduct(expression.factors.map(pass));
case "power":
return simplifyPower(pass(expression.base), pass(expression.exponent));
case "function": {
const simplified = expression.args.map(pass);
// Fold a constant argument, but only when the result stays clean —
// `sqrt(2)` must remain symbolic, not become 1.4142135623730951 in the
// middle of an exact solution.
const first = simplified[0];
if (first) {
const value = numberValue(first);
if (value) {
const folded = foldFunction(expression.fn, value);
if (folded) return num(folded);
}
}
return fn(expression.fn, simplified);
}
case "relation":
return relation(expression.relation, pass(expression.lhs), pass(expression.rhs));
}
}
// ---------------------------------------------------------------------------
// Sums
function simplifySum(rawTerms: MathExpression[]): MathExpression {
const flattened: MathExpression[] = [];
for (const term of rawTerms) {
if (term.type === "add") {
flattened.push(...term.terms);
} else {
flattened.push(term);
}
}
// Collect like terms by their non-numeric part: 2x + 3x → 5x.
let constantTotal: MathNumber = integer(0);
const collected: { key: MathExpression; coefficient: MathNumber }[] = [];
for (const term of flattened) {
const value = numberValue(term);
if (value) {
constantTotal = add(constantTotal, value);
continue;
}
const [coefficient, key] = splitCoefficient(term);
const existing = collected.find((entry) => expressionEquals(entry.key, key));
if (existing) {
existing.coefficient = add(existing.coefficient, coefficient);
} else {
collected.push({ key, coefficient });
}
}
const terms: MathExpression[] = [];
for (const entry of collected) {
if (isZero(entry.coefficient)) continue;
terms.push(rebuild(entry.coefficient, entry.key));
}
// Stable ordering: highest polynomial degree first, which is how a person
// writes it and how the quadratic solver expects to read it.
terms.sort((a, b) => degree(b) - degree(a));
if (!isZero(constantTotal)) terms.push(num(constantTotal));
if (terms.length === 0) return ZERO;
return terms.length === 1 ? terms[0]! : addExpr(terms);
}
/**
* Split a term into its numeric coefficient and the rest, so like terms can
* be matched: `3 * x * y` → (3, x*y), `-x` → (-1, x).
*/
export function splitCoefficient(term: MathExpression): [MathNumber, MathExpression] {
switch (term.type) {
case "negate": {
const [coefficient, key] = splitCoefficient(term.inner);
return [negated(coefficient), key];
}
case "multiply": {
let coefficient: MathNumber = integer(1);
const rest: MathExpression[] = [];
for (const factor of term.factors) {
const value = numberValue(factor);
if (value) {
coefficient = multiply(coefficient, value);
} else {
rest.push(factor);
}
}
if (rest.length === 0) return [coefficient, ONE];
return [coefficient, rest.length === 1 ? rest[0]! : multiplyExpr(rest)];
}
default:
return [integer(1), term];
}
}
function rebuild(coefficient: MathNumber, key: MathExpression): MathExpression {
if (isOneExpr(key)) return num(coefficient);
if (isOne(coefficient)) return key;
if (numberEquals(coefficient, integer(-1))) return negate(key);
return multiplyExpr([num(coefficient), key]);
}
// ---------------------------------------------------------------------------
// Products
function simplifyProduct(rawFactors: MathExpression[]): MathExpression {
const flattened: MathExpression[] = [];
let sign = false;
for (const factor of rawFactors) {
if (factor.type === "multiply") {
flattened.push(...factor.factors);
} else if (factor.type === "negate") {
// Pull negation out to the front rather than leaving it buried, so
// `(-x) * (-y)` cancels to `x * y`.
sign = !sign;
flattened.push(factor.inner);
} else {
flattened.push(factor);
}
}
let coefficient: MathNumber = integer(1);
// Collect powers with matching bases: x * x^2 → x^3.
const bases: { base: MathExpression; exponents: MathExpression[] }[] = [];
for (const factor of flattened) {
const value = numberValue(factor);
if (value) {
coefficient = multiply(coefficient, value);
continue;
}
const [base, exponent] = splitPower(factor);
const existing = bases.find((entry) => expressionEquals(entry.base, base));
if (existing) {
existing.exponents.push(exponent);
} else {
bases.push({ base, exponents: [exponent] });
}
}
if (isZero(coefficient)) return ZERO;
const factors: MathExpression[] = [];
for (const entry of bases) {
const exponent = entry.exponents.length === 1 ? entry.exponents[0]! : simplify(addExpr(entry.exponents));
if (isZeroExpr(exponent)) continue; // x^0 = 1
if (isOneExpr(exponent)) {
factors.push(entry.base);
continue;
}
factors.push(powerExpr(entry.base, exponent));
}
if (sign) coefficient = negated(coefficient);
if (factors.length === 0) return num(coefficient);
if (!isOne(coefficient)) {
if (numberEquals(coefficient, integer(-1))) {
return negate(factors.length === 1 ? factors[0]! : multiplyExpr(factors));
}
factors.unshift(num(coefficient));
}
return factors.length === 1 ? factors[0]! : multiplyExpr(factors);
}
export function splitPower(expression: MathExpression): [MathExpression, MathExpression] {
if (expression.type === "power") return [expression.base, expression.exponent];
return [expression, ONE];
}
// ---------------------------------------------------------------------------
// Powers
function simplifyPower(base: MathExpression, exponent: MathExpression): MathExpression {
if (isZeroExpr(exponent)) return ONE; // x^0 = 1
if (isOneExpr(exponent)) return base; // x^1 = x
if (isOneExpr(base)) return ONE; // 1^n = 1
const exponentValue = numberValue(exponent);
if (isZeroExpr(base) && exponentValue && !isNegative(exponentValue)) return ZERO;
// Fold a numeric power only when it stays exact. `2^0.5` must stay
// symbolic so an exact answer does not silently become a float.
const baseValue = numberValue(base);
if (baseValue && exponentValue) {
const result = raisedTo(baseValue, exponentValue);
if (isExact(result) || !isExact(baseValue) || !isExact(exponentValue)) {
return num(result);
}
}
// (x^a)^b = x^(a*b)
if (base.type === "power") {
return powerExpr(base.base, simplify(multiplyExpr([base.exponent, exponent])));
}
return powerExpr(base, exponent);
}
// ---------------------------------------------------------------------------
// Function folding
/**
* Fold only where the result is exact or the input was already inexact. This
* is what keeps `sqrt(2)` symbolic while `sqrt(9)` becomes 3.
*/
function foldFunction(name: MathFunction, value: MathNumber): MathNumber | null {
if (!functionIsDefined(name, doubleValue(value))) return null;
const asInt = integerValue(value);
if (name === "factorial" && asInt !== null && asInt >= 0 && asInt <= 20) {
let total = 1;
for (let step = 2; step <= asInt; step++) total *= step;
return integer(total);
}
if (name === "sqrt" && isExact(value)) {
const exact = raisedTo(value, rational(1, 2));
return isExact(exact) ? exact : null;
}
if (name === "floor" || name === "ceil" || name === "round" || name === "abs") {
const result = evaluateFunction(name, doubleValue(value));
return Number.isFinite(result) ? fromDecimal(result) : null;
}
// Inexact input already lost exactness, so folding costs nothing.
if (isExact(value)) return null;
const result = evaluateFunction(name, doubleValue(value));
return Number.isFinite(result) ? { kind: "real", value: result } : null;
}
// ---------------------------------------------------------------------------
// Degree
const NON_POLYNOMIAL = Math.floor(Number.MAX_SAFE_INTEGER / 4);
/**
* Polynomial degree in the given variable (or the highest across all
* variables when none is named). Drives term ordering and lets the solver
* pick a strategy.
*/
export function degree(expression: MathExpression, inVariable: string | null = null): number {
switch (expression.type) {
case "number":
case "constant":
return 0;
case "variable":
return inVariable === null || inVariable === expression.name ? 1 : 0;
case "negate":
return degree(expression.inner, inVariable);
case "add":
return expression.terms.reduce((highest, term) => Math.max(highest, degree(term, inVariable)), 0);
case "multiply":
return expression.factors.reduce((total, factor) => total + degree(factor, inVariable), 0);
case "power": {
const value = numberValue(expression.exponent);
const power = value ? integerValue(value) : null;
if (power === null) {
// A symbolic exponent is not a polynomial degree; report 0 for a
// constant base and a large sentinel otherwise so the solver does not
// mistake it for something it can factor.
return variablesOf(expression.base).length === 0 ? 0 : NON_POLYNOMIAL;
}
return degree(expression.base, inVariable) * power;
}
case "function":
// Any variable inside a transcendental function makes this
// non-polynomial.
return expression.args.some((argument) => variablesOf(argument).length > 0) ? NON_POLYNOMIAL : 0;
case "relation":
return Math.max(degree(expression.lhs, inVariable), degree(expression.rhs, inVariable));
}
}
/** True when the expression is a genuine polynomial in `variable`. */
export function isPolynomial(expression: MathExpression, inVariable: string): boolean {
return degree(expression, inVariable) < NON_POLYNOMIAL / 2;
}
+781
View File
@@ -0,0 +1,781 @@
// Solves what a person actually asks a calculator, in the order of
// likelihood: evaluate an arithmetic expression, solve a linear or quadratic
// equation, find polynomial roots, or fall back to a numeric search.
//
// Every branch records SolutionSteps as it goes, because "explain it" is a
// first-class feature here rather than a reconstruction after the fact — a
// post-hoc explanation of a numeric root search would be fiction.
import {
type MathExpression,
type MathRelation,
ZERO,
addExpr,
divideExpr,
expressionEquals,
flippedRelation,
fn,
int,
multiplyExpr,
negate,
num,
numberValue,
relation as relationExpr,
spokenRelation,
subtractExpr,
variable as variableExpr,
variablesOf,
} from "./expression";
import {
type MathNumber,
add,
describeNumber,
doubleValue,
fromDecimal,
integer,
isExact,
isNegative,
isZero,
multiply,
negated,
divide as divideNumber,
raisedTo,
rational,
real,
subtract,
} from "./math-number";
import { type EvaluatorContext, evaluateExact, evaluateValue, radiansContext } from "./evaluator";
import { simplify } from "./simplifier";
import { Polynomial } from "./polynomial";
import { plain, renderNumber } from "./text-renderer";
// ---------------------------------------------------------------------------
// Steps
/**
* One line of "explain it". Recorded by the solver as it works rather than
* reconstructed afterwards.
*/
export type SolutionStep = {
id: number;
/** Short imperative title — what was done ("Divide by 2"). */
title: string;
/** The expression AFTER this step. */
expression: MathExpression;
/** Plain-language reason, shown under the title. */
detail: string;
};
export function stepPlain(step: SolutionStep): string {
return plain(step.expression);
}
export class SolutionStepRecorder {
readonly steps: SolutionStep[] = [];
add(title: string, expression: MathExpression, detail: string): void {
// Collapse a step that changed nothing — "Simplify: 2x + 1" following
// "Start: 2x + 1" is noise that makes a short solve look laborious.
const last = this.steps[this.steps.length - 1];
if (last && expressionEquals(last.expression, expression)) return;
this.steps.push({ id: this.steps.length, title, expression, detail });
}
}
// ---------------------------------------------------------------------------
// Solution
export type MathSolutionKind =
/** A finite set of values for the unknown. */
| { kind: "roots"; roots: MathExpression[] }
/** The relation holds for every value (`x = x`). */
| { kind: "identity" }
/** It holds for none (`0 = 1`). */
| { kind: "contradiction" }
/** A single evaluated value — the answer to "what is 2+2". */
| { kind: "value"; value: MathExpression }
/** Simplified but not solved (no unknown to solve for). */
| { kind: "simplified"; value: MathExpression }
/** An inequality's solution interval. */
| {
kind: "interval";
lower: MathExpression | null;
upper: MathExpression | null;
relation: MathRelation;
};
export type MathSolution = {
kind: MathSolutionKind;
variable: string | null;
steps: SolutionStep[];
/**
* The input as the engine understood it, so a wearer can confirm the
* machine heard the same problem they said.
*/
restatement: MathExpression;
};
/** The short line the glasses show. */
export function solutionHeadline(solution: MathSolution): string {
const kind = solution.kind;
switch (kind.kind) {
case "roots":
if (kind.roots.length === 0) return "No real solution";
return kind.roots.map((root) => `${solution.variable ?? "x"} = ${plain(root)}`).join(", ");
case "identity":
return "True for every value";
case "contradiction":
return "No solution";
case "value":
return plain(kind.value);
case "simplified":
return plain(kind.value);
case "interval":
return intervalText(kind.lower, kind.upper, kind.relation, solution.variable);
}
}
function intervalText(
lower: MathExpression | null,
upper: MathExpression | null,
relation: MathRelation,
variable: string | null,
): string {
const name = variable ?? "x";
if (lower && !upper) return `${name} ${relation} ${plain(lower)}`;
if (upper && !lower) return `${name} ${relation} ${plain(upper)}`;
if (lower && upper) return `${plain(lower)} < ${name} < ${plain(upper)}`;
return `all ${name}`;
}
// ---------------------------------------------------------------------------
// Solver
export function solve(
expression: MathExpression,
solveFor: string | null = null,
context: EvaluatorContext = radiansContext(),
): MathSolution {
const steps = new SolutionStepRecorder();
const simplified = simplify(expression);
if (simplified.type !== "relation") {
return evaluateExpression(simplified, expression, context, steps);
}
const relation = simplified.relation;
const lhs = simplified.lhs;
const rhs = simplified.rhs;
const variable = solveFor ?? variablesOf(simplified)[0] ?? null;
if (!variable) {
// No unknown: this is a claim to check, not an equation to solve.
return checkClaim(relation, lhs, rhs, expression, context);
}
steps.add("Start", simplified, `Solve for ${variable}.`);
// Move everything to the left: lhs - rhs (relation) 0.
const difference = simplify(subtractExpr(lhs, rhs));
if (!(rhs.type === "number" && isZero(rhs.value))) {
steps.add(
"Move every term to one side",
relationExpr(relation, difference, ZERO),
`Subtract ${plain(rhs)} from both sides.`,
);
}
const polynomial = Polynomial.extract(difference, variable);
if (!polynomial) {
return solveNumerically(difference, relation, variable, expression, context, steps);
}
if (relation !== "=") {
return solveInequality(polynomial, relation, variable, expression, steps);
}
switch (polynomial.degree) {
case 0: {
// No unknown survived: either always true or never.
const constant = polynomial.at(0);
const kind: MathSolutionKind = isZero(constant) ? { kind: "identity" } : { kind: "contradiction" };
steps.add(
isZero(constant) ? "Every value works" : "No value works",
relationExpr("=", num(constant), ZERO),
isZero(constant)
? `${variable} cancelled out and the statement is true.`
: `${variable} cancelled out and ${describeNumber(constant)} ≠ 0.`,
);
return { kind, variable, steps: steps.steps, restatement: simplified };
}
case 1:
return solveLinear(polynomial, variable, simplified, steps);
case 2:
return solveQuadratic(polynomial, variable, simplified, steps);
default:
return solvePolynomial(polynomial, variable, simplified, steps);
}
}
// ---------------------------------------------------------------------------
// Plain expressions
function evaluateExpression(
simplified: MathExpression,
original: MathExpression,
context: EvaluatorContext,
steps: SolutionStepRecorder,
): MathSolution {
steps.add("Start", original, "Evaluate the expression.");
if (!expressionEquals(simplified, original)) {
steps.add("Simplify", simplified, "Combine like terms and constants.");
}
let value: MathNumber | null = null;
try {
value = evaluateExact(simplified, context);
} catch {
value = null;
}
if (value !== null) {
const result = num(value);
if (!expressionEquals(result, simplified)) {
steps.add("Result", result, "Evaluate what is left.");
}
return { kind: { kind: "value", value: result }, variable: null, steps: steps.steps, restatement: original };
}
// Free variables remain — the best answer is the simplified form.
return {
kind: { kind: "simplified", value: simplified },
variable: null,
steps: steps.steps,
restatement: original,
};
}
function checkClaim(
relation: MathRelation,
lhs: MathExpression,
rhs: MathExpression,
original: MathExpression,
context: EvaluatorContext,
): MathSolution {
const steps = new SolutionStepRecorder();
steps.add("Start", original, "Check whether this is true.");
let left: MathNumber;
let right: MathNumber;
try {
left = evaluateExact(lhs, context);
right = evaluateExact(rhs, context);
} catch {
return {
kind: { kind: "simplified", value: original },
variable: null,
steps: steps.steps,
restatement: original,
};
}
steps.add(
"Evaluate both sides",
relationExpr(relation, num(left), num(right)),
`${describeNumber(left)} ${spokenRelation(relation)} ${describeNumber(right)}?`,
);
let holds: boolean;
switch (relation) {
case "=":
holds = doubleValue(left) === doubleValue(right);
break;
case "<":
holds = doubleValue(left) < doubleValue(right);
break;
case "<=":
holds = doubleValue(left) <= doubleValue(right);
break;
case ">":
holds = doubleValue(left) > doubleValue(right);
break;
case ">=":
holds = doubleValue(left) >= doubleValue(right);
break;
}
return {
kind: holds ? { kind: "identity" } : { kind: "contradiction" },
variable: null,
steps: steps.steps,
restatement: original,
};
}
// ---------------------------------------------------------------------------
// Linear
function solveLinear(
polynomial: Polynomial,
variable: string,
original: MathExpression,
steps: SolutionStepRecorder,
): MathSolution {
const a = polynomial.at(1);
const b = polynomial.at(0);
steps.add(
`Isolate the ${variable} term`,
relationExpr("=", multiplyExpr([num(a), variableExpr(variable)]), num(negated(b))),
`Move the constant across: ${renderNumber(a)}${variable} = ${renderNumber(negated(b))}.`,
);
const root = divideNumber(negated(b), a);
const rootExpression = num(root);
steps.add(
`Divide by ${renderNumber(a)}`,
relationExpr("=", variableExpr(variable), rootExpression),
`${variable} = ${renderNumber(negated(b))} ÷ ${renderNumber(a)} = ${renderNumber(root)}.`,
);
return {
kind: { kind: "roots", roots: [rootExpression] },
variable,
steps: steps.steps,
restatement: original,
};
}
// ---------------------------------------------------------------------------
// Quadratic
function solveQuadratic(
polynomial: Polynomial,
variable: string,
original: MathExpression,
steps: SolutionStepRecorder,
): MathSolution {
const a = polynomial.at(2);
const b = polynomial.at(1);
const c = polynomial.at(0);
steps.add(
"Read off the coefficients",
polynomial.expression(),
`a = ${renderNumber(a)}, b = ${renderNumber(b)}, c = ${renderNumber(c)}.`,
);
const discriminant = subtract(multiply(b, b), multiply(multiply(integer(4), a), c));
steps.add(
"Discriminant",
num(discriminant),
`b² − 4ac = ${renderNumber(b)}² − 4·${renderNumber(a)}·${renderNumber(c)} = ${renderNumber(discriminant)}.`,
);
if (isNegative(discriminant)) {
steps.add(
"No real roots",
num(discriminant),
"The discriminant is negative, so the parabola never crosses the axis.",
);
return { kind: { kind: "roots", roots: [] }, variable, steps: steps.steps, restatement: original };
}
// Prefer an exact square root — it reads far better in a step-by-step than
// the formula does, and it is what a person would do.
const squareRoot = raisedTo(discriminant, rational(1, 2));
const exactRoot = isExact(squareRoot);
if (isZero(discriminant)) {
const root = divideNumber(negated(b), multiply(integer(2), a));
steps.add(
"One repeated root",
relationExpr("=", variableExpr(variable), num(root)),
`With a zero discriminant, ${variable} = −b / 2a = ${renderNumber(root)}.`,
);
return {
kind: { kind: "roots", roots: [num(root)] },
variable,
steps: steps.steps,
restatement: original,
};
}
steps.add(
"Quadratic formula",
relationExpr(
"=",
variableExpr(variable),
divideExpr(addExpr([negate(num(b)), fn("sqrt", [num(discriminant)])]), multiplyExpr([int(2), num(a)])),
),
"x = (−b ± √(b² − 4ac)) / 2a.",
);
const denominator = multiply(integer(2), a);
let rootExpressions: MathExpression[];
if (exactRoot) {
const first = divideNumber(add(negated(b), squareRoot), denominator);
const second = divideNumber(subtract(negated(b), squareRoot), denominator);
const ordered = [first, second].sort((x, y) => doubleValue(x) - doubleValue(y));
rootExpressions = ordered.map((value) => num(value));
steps.add(
"Exact roots",
addExpr(rootExpressions),
`√${renderNumber(discriminant)} = ${renderNumber(squareRoot)}, so the roots are exact.`,
);
} else {
// Keep the surd symbolic: (−b ± √D) / 2a is the exact answer, and
// collapsing it to decimals throws away information the wearer may want
// to see.
const surd = fn("sqrt", [num(discriminant)]);
const plus = simplify(divideExpr(addExpr([num(negated(b)), surd]), num(denominator)));
const minus = simplify(divideExpr(subtractExpr(num(negated(b)), surd), num(denominator)));
rootExpressions = [plus, minus];
const approximate: number[] = [];
for (const root of rootExpressions) {
try {
approximate.push(doubleValue(evaluateExact(root)));
} catch {
// Leave it out of the approximation line.
}
}
steps.add(
"Roots",
addExpr(rootExpressions),
approximate.length === 2
? `≈ ${formatApprox(approximate[0]!)} and ${formatApprox(approximate[1]!)}.`
: "The discriminant is not a perfect square, so the roots keep their radical.",
);
}
return {
kind: { kind: "roots", roots: rootExpressions },
variable,
steps: steps.steps,
restatement: original,
};
}
// ---------------------------------------------------------------------------
// Higher degree
function solvePolynomial(
polynomial: Polynomial,
variable: string,
original: MathExpression,
steps: SolutionStepRecorder,
): MathSolution {
steps.add(
`Degree ${polynomial.degree} polynomial`,
polynomial.expression(),
"Look for rational roots, then solve what is left.",
);
let remaining = polynomial;
const found: MathNumber[] = [];
// Rational root theorem: p/q where p divides the constant term and q
// divides the leading coefficient. Cheap, exact, and it usually cracks the
// textbook cases outright.
for (;;) {
if (remaining.degree <= 2) break;
const root = rationalRoot(remaining);
if (!root) break;
found.push(root);
remaining = remaining.dividingByRoot(root);
steps.add(
`Factor out (${variable} − ${renderNumber(root)})`,
remaining.expression(),
`${variable} = ${renderNumber(root)} is a root; divide it out.`,
);
}
switch (remaining.degree) {
case 1:
found.push(divideNumber(negated(remaining.at(0)), remaining.at(1)));
break;
case 2: {
const a = remaining.at(2);
const b = remaining.at(1);
const c = remaining.at(0);
const discriminant = subtract(multiply(b, b), multiply(multiply(integer(4), a), c));
if (!isNegative(discriminant)) {
const squareRoot = raisedTo(discriminant, rational(1, 2));
found.push(divideNumber(add(negated(b), squareRoot), multiply(integer(2), a)));
if (!isZero(discriminant)) {
found.push(divideNumber(subtract(negated(b), squareRoot), multiply(integer(2), a)));
}
}
steps.add(
"Solve the quadratic factor",
remaining.expression(),
"What is left is a quadratic; apply the formula.",
);
break;
}
default: {
// No closed form available — find the real roots numerically.
const numeric = numericRoots(remaining);
found.push(...numeric.map((root) => fromDecimal(root)));
steps.add(
"Numeric roots",
remaining.expression(),
numeric.length === 0
? "No further real roots were found."
: "No exact factorisation remains, so the rest are found numerically.",
);
break;
}
}
const sorted = [...found].sort((x, y) => doubleValue(x) - doubleValue(y));
return {
kind: { kind: "roots", roots: sorted.map((value) => num(value)) },
variable,
steps: steps.steps,
restatement: original,
};
}
export function rationalRoot(polynomial: Polynomial): MathNumber | null {
const integers = polynomial.integerCoefficients;
if (!integers) return null;
const constant = integers[0];
const leading = integers[integers.length - 1];
if (!constant || !leading) return null;
const constantDivisors = divisorsOf(Math.abs(constant));
const leadingDivisors = divisorsOf(Math.abs(leading));
for (const p of constantDivisors) {
for (const q of leadingDivisors) {
for (const sign of [1, -1]) {
const candidate = rational(sign * p, q);
if (Math.abs(polynomial.value(doubleValue(candidate))) < 1e-9) {
return candidate;
}
}
}
}
return null;
}
function divisorsOf(value: number): number[] {
if (value <= 0 || value >= 1_000_000) return [1];
const result: number[] = [];
for (let candidate = 1; candidate * candidate <= value; candidate++) {
if (value % candidate === 0) {
result.push(candidate);
if (candidate !== value / candidate) result.push(value / candidate);
}
}
return result.sort((a, b) => a - b);
}
/**
* Bracket-and-bisect over a sampled range. Bisection rather than bare Newton
* because Newton alone diverges on a bad seed, and a diverging solver on the
* lens produces nonsense rather than nothing.
*/
export function numericRoots(
polynomial: Polynomial,
searchRange: [number, number] = [-1000, 1000],
samples = 20_000,
): number[] {
if (polynomial.degree < 1) return [];
const [lower, upper] = searchRange;
const roots: number[] = [];
const step = (upper - lower) / samples;
let previousX = lower;
let previousY = polynomial.value(previousX);
for (let index = 1; index <= samples; index++) {
const x = lower + index * step;
const y = polynomial.value(x);
if (y === 0) {
roots.push(x);
previousX = x;
previousY = y;
continue;
}
if (Math.sign(previousY) === Math.sign(y) || previousY === 0) {
previousX = x;
previousY = y;
continue;
}
let low = previousX;
let high = x;
for (let iteration = 0; iteration < 80; iteration++) {
const mid = (low + high) / 2;
const value = polynomial.value(mid);
if (value === 0) {
low = mid;
high = mid;
break;
}
if (Math.sign(value) === Math.sign(polynomial.value(low))) {
low = mid;
} else {
high = mid;
}
}
roots.push((low + high) / 2);
previousX = x;
previousY = y;
}
// Collapse near-duplicates from adjacent brackets around a tangency.
const unique: number[] = [];
for (const root of roots.sort((a, b) => a - b)) {
if (!unique.some((existing) => Math.abs(existing - root) < 1e-7)) {
unique.push(root);
}
}
return unique;
}
// ---------------------------------------------------------------------------
// Inequalities
function solveInequality(
polynomial: Polynomial,
relation: MathRelation,
variable: string,
original: MathExpression,
steps: SolutionStepRecorder,
): MathSolution {
if (polynomial.degree !== 1) {
// Higher-degree inequalities need sign analysis across intervals; report
// the roots rather than pretending to a full solution.
const solution = solvePolynomial(polynomial, variable, original, steps);
steps.add(
"Critical points",
polynomial.expression(),
"The sign can only change at these values; test a point in each interval.",
);
return { kind: solution.kind, variable, steps: steps.steps, restatement: original };
}
const a = polynomial.at(1);
const b = polynomial.at(0);
const bound = divideNumber(negated(b), a);
// Dividing by a negative flips the inequality — the single most common
// mistake in this whole topic, so it gets its own visible step.
const finalRelation = isNegative(a) ? flippedRelation(relation) : relation;
steps.add(
`Divide by ${renderNumber(a)}`,
relationExpr(finalRelation, variableExpr(variable), num(bound)),
isNegative(a)
? `Dividing by a negative flips the inequality to ${finalRelation}.`
: `${variable} ${finalRelation} ${renderNumber(bound)}.`,
);
const isUpperBound = finalRelation === "<" || finalRelation === "<=";
return {
kind: {
kind: "interval",
lower: isUpperBound ? null : num(bound),
upper: isUpperBound ? num(bound) : null,
relation: finalRelation,
},
variable,
steps: steps.steps,
restatement: original,
};
}
// ---------------------------------------------------------------------------
// Non-polynomial
function solveNumerically(
difference: MathExpression,
relation: MathRelation,
variable: string,
original: MathExpression,
context: EvaluatorContext,
steps: SolutionStepRecorder,
): MathSolution {
steps.add(
"Not a polynomial",
difference,
`${variable} appears inside a function, so the roots are found numerically.`,
);
let roots: number[] = [];
const lower = -100;
const upper = 100;
const samples = 40_000;
const step = (upper - lower) / samples;
const evaluate = (x: number): number | null =>
evaluateValue(difference, { [variable]: x }, context.usesDegrees);
let previousX = lower;
let previousY = evaluate(previousX);
for (let index = 1; index <= samples; index++) {
const x = lower + index * step;
const y = evaluate(x);
const lastX = previousX;
const lastY = previousY;
previousX = x;
previousY = y;
if (y === null || lastY === null) continue;
// A sign change across a pole is not a root. Reject brackets where the
// function magnitude explodes — that is `tan`'s asymptote, not a
// crossing.
if (Math.sign(lastY) === Math.sign(y)) continue;
if (Math.abs(lastY) >= 1e6 || Math.abs(y) >= 1e6) continue;
let low = lastX;
let high = x;
for (let iteration = 0; iteration < 60; iteration++) {
const mid = (low + high) / 2;
const value = evaluate(mid);
if (value === null) break;
if (value === 0) {
low = mid;
high = mid;
break;
}
const lowValue = evaluate(low);
if (lowValue !== null && Math.sign(value) === Math.sign(lowValue)) {
low = mid;
} else {
high = mid;
}
}
const root = (low + high) / 2;
// Verify the root rather than trusting the sign change.
//
// A sign change also happens across a pole — tan(x) flips from +∞ to −∞
// at π/2 — and bisection happily converges onto the asymptote. The
// magnitude guard above does not catch it, because a sample 0.0025 away
// from the pole is only ~400, nowhere near the cutoff. At a real root the
// residual is ~1e-15; at a pole it is enormous.
const residual = evaluate(root);
if (residual === null || Math.abs(residual) >= 1e-6) continue;
if (!roots.some((existing) => Math.abs(existing - root) < 1e-6)) roots.push(root);
}
// Keep the roots NEAREST ZERO, not the first twelve found.
//
// sin(x) = 0 has 64 roots in this window; scanning left to right and
// stopping at twelve returned everything between −97 and −63 and nothing
// near the origin, which is the opposite of what anyone asking
// "solve sin x = 0" wants to see.
const maximumRoots = 12;
if (roots.length > maximumRoots) {
roots = roots.sort((a, b) => Math.abs(a) - Math.abs(b)).slice(0, maximumRoots);
}
roots.sort((a, b) => a - b);
steps.add(
"Roots found",
difference,
roots.length === 0
? "No sign change was found between −100 and 100."
: "Searched between −100 and 100 for sign changes.",
);
return {
kind: { kind: "roots", roots: roots.map((root) => num(fromDecimal(root))) },
variable,
steps: steps.steps,
restatement: original,
};
}
function formatApprox(value: number): string {
return describeNumber(real(value));
}
@@ -0,0 +1,323 @@
// Symbolic antiderivatives. Not a Risch implementation — that is a
// research-grade algorithm and wildly out of proportion here. This is the
// standard table plus three structural rules (linearity, linear substitution,
// integration by parts), which is what actually covers a calculus course.
// Anything outside that returns null rather than a wrong answer, and the
// caller falls back to the numeric integrator, which always works.
import {
type MathExpression,
addExpr,
divideExpr,
fn,
int,
multiplyExpr,
negate,
num,
numberValue,
powerExpr,
subtractExpr,
variable as variableExpr,
variablesOf,
} from "./expression";
import {
type MathNumber,
add,
fromDecimal,
integer,
multiply,
numberEquals,
rational,
reciprocal,
} from "./math-number";
import { simplify } from "./simplifier";
import { differentiate, integrateNumeric } from "./derivative";
import { evaluateValue } from "./evaluator";
import { Polynomial } from "./polynomial";
import { isZero } from "./math-number";
/** Indefinite integral with respect to `variable`, without the constant. */
export function integrate(expression: MathExpression, variable: string): MathExpression | null {
const raw = attempt(simplify(expression), variable, 0);
if (!raw) return null;
return simplify(raw);
}
/** With `+ C`, for display. */
export function integrateWithConstant(expression: MathExpression, variable: string): MathExpression | null {
const result = integrate(expression, variable);
return result ? addExpr([result, variableExpr("C")]) : null;
}
function attempt(expression: MathExpression, variable: string, depth: number): MathExpression | null {
// Integration by parts can recurse; bound it so a bad pairing cannot loop
// forever.
if (depth >= 6) return null;
// Constant with respect to the variable: ∫c dx = cx
if (!variablesOf(expression).includes(variable)) {
return multiplyExpr([expression, variableExpr(variable)]);
}
switch (expression.type) {
case "variable":
if (expression.name === variable) {
return divideExpr(powerExpr(variableExpr(variable), int(2)), int(2));
}
// A variable other than the integration variable is already handled by
// the constant guard above; nothing reaches here.
return null;
case "negate": {
const inner = attempt(expression.inner, variable, depth);
return inner ? negate(inner) : null;
}
case "add": {
// Linearity. If any term is not integrable the whole thing is not.
const integrated: MathExpression[] = [];
for (const term of expression.terms) {
const part = attempt(term, variable, depth);
if (!part) return null;
integrated.push(part);
}
return addExpr(integrated);
}
case "multiply":
return integrateProduct(expression.factors, variable, depth);
case "power":
return integratePower(expression.base, expression.exponent, variable);
case "function": {
const argument = expression.args[0];
if (!argument) return null;
return integrateFunction(expression.fn, argument, variable);
}
case "number":
case "constant":
case "relation":
return null;
}
}
// ---------------------------------------------------------------------------
// Products
function integrateProduct(
factors: MathExpression[],
variable: string,
depth: number,
): MathExpression | null {
// Pull constants out front: ∫ c·f dx = c·∫f dx
const constants: MathExpression[] = [];
const variableFactors: MathExpression[] = [];
for (const factor of factors) {
if (variablesOf(factor).includes(variable)) {
variableFactors.push(factor);
} else {
constants.push(factor);
}
}
if (variableFactors.length === 0) {
return multiplyExpr([...factors, variableExpr(variable)]);
}
if (constants.length > 0) {
const inner = variableFactors.length === 1 ? variableFactors[0]! : multiplyExpr(variableFactors);
const integrated = attempt(inner, variable, depth);
if (!integrated) return null;
return multiplyExpr([...constants, integrated]);
}
if (variableFactors.length !== 2) return null;
return integrateByParts(variableFactors[0]!, variableFactors[1]!, variable, depth);
}
/**
* ∫u dv = uv − ∫v du.
*
* Picks `u` by the LIATE heuristic: whichever factor differentiates toward
* nothing goes in `u`. Getting this backwards turns a one-step integral into
* an infinite regress, which is why the depth bound above exists.
*/
function integrateByParts(
first: MathExpression,
second: MathExpression,
variable: string,
depth: number,
): MathExpression | null {
const priority = (expression: MathExpression): number => {
if (expression.type === "function") {
switch (expression.fn) {
case "ln":
case "log10":
case "log2":
return 5;
case "asin":
case "acos":
case "atan":
return 4;
case "sin":
case "cos":
case "tan":
return 2;
case "exp":
return 1;
default:
return 0;
}
}
if (expression.type === "variable" || expression.type === "power") return 3;
return 0;
};
const [u, dv] = priority(first) >= priority(second) ? [first, second] : [second, first];
const v = attempt(dv, variable, depth + 1);
if (!v) return null;
const du = differentiate(u, variable);
const remainder = simplify(multiplyExpr([v, du]));
const integratedRemainder = attempt(remainder, variable, depth + 1);
if (!integratedRemainder) return null;
return subtractExpr(multiplyExpr([u, v]), integratedRemainder);
}
// ---------------------------------------------------------------------------
// Powers
function integratePower(
base: MathExpression,
exponent: MathExpression,
variable: string,
): MathExpression | null {
const baseHasVariable = variablesOf(base).includes(variable);
const exponentHasVariable = variablesOf(exponent).includes(variable);
// Exponential: ∫a^x dx = a^x / ln a
if (!baseHasVariable && exponentHasVariable) {
const linear = linearForm(exponent, variable);
if (!linear) return null;
const antiderivative = divideExpr(powerExpr(base, exponent), fn("ln", [base]));
return divideExpr(antiderivative, num(linear.slope));
}
if (!baseHasVariable || exponentHasVariable) return null;
const power = numberValue(exponent);
if (!power) return null;
// Chain rule in reverse, but only for a LINEAR inner function. For a
// nonlinear inner function the substitution needs its own du, which this
// level of machinery cannot supply — returning null there is the
// difference between "no symbolic form" and a wrong one.
const linear = linearForm(base, variable);
if (!linear) return null;
// ∫(ax+b)^-1 dx = ln|ax+b| / a
if (numberEquals(power, integer(-1))) {
return divideExpr(fn("ln", [fn("abs", [base])]), num(linear.slope));
}
const raised = add(power, integer(1));
return divideExpr(powerExpr(base, num(raised)), num(multiply(raised, linear.slope)));
}
// ---------------------------------------------------------------------------
// Functions
function integrateFunction(
functionName: string,
argument: MathExpression,
variable: string,
): MathExpression | null {
// Same restriction as powers: the inner function must be linear, or the
// substitution is not this simple.
const linear = linearForm(argument, variable);
if (!linear) return null;
const scale = num(reciprocal(linear.slope));
let antiderivative: MathExpression | null;
switch (functionName) {
case "sin":
antiderivative = negate(fn("cos", [argument]));
break;
case "cos":
antiderivative = fn("sin", [argument]);
break;
case "tan":
antiderivative = negate(fn("ln", [fn("abs", [fn("cos", [argument])])]));
break;
case "exp":
antiderivative = fn("exp", [argument]);
break;
case "sinh":
antiderivative = fn("cosh", [argument]);
break;
case "cosh":
antiderivative = fn("sinh", [argument]);
break;
case "ln":
// ∫ln(u) du = u·ln(u) − u
antiderivative = subtractExpr(multiplyExpr([argument, fn("ln", [argument])]), argument);
break;
case "sqrt":
// ∫√u du = (2/3)·u^(3/2)
antiderivative = multiplyExpr([num(rational(2, 3)), powerExpr(argument, num(rational(3, 2)))]);
break;
default:
antiderivative = null;
break;
}
if (!antiderivative) return null;
return multiplyExpr([scale, antiderivative]);
}
// ---------------------------------------------------------------------------
// Definite
export type DefiniteResult =
/** Evaluated from an exact antiderivative — the fundamental theorem. */
| { kind: "exact"; value: MathNumber; antiderivative: MathExpression }
/** No symbolic form; the value came from adaptive Simpson. */
| { kind: "numeric"; value: number }
| { kind: "failed"; reason: string };
/**
* Prefer the symbolic route so the answer can show its working, and fall
* back to numerics so the question always gets answered.
*/
export function definite(
expression: MathExpression,
variable: string,
lower: number,
upper: number,
): DefiniteResult {
const antiderivative = integrate(expression, variable);
if (antiderivative) {
const atUpper = evaluateValue(antiderivative, { [variable]: upper });
const atLower = evaluateValue(antiderivative, { [variable]: lower });
if (atUpper !== null && atLower !== null && Number.isFinite(atUpper) && Number.isFinite(atLower)) {
return { kind: "exact", value: fromDecimal(atUpper - atLower), antiderivative };
}
}
const value = integrateNumeric(expression, variable, lower, upper);
if (value === null) {
return { kind: "failed", reason: "The function is undefined somewhere in that range" };
}
return { kind: "numeric", value };
}
// ---------------------------------------------------------------------------
// Linear form
/**
* Recognises `a·x + b`, which is the only inner function the substitution
* rules above can undo.
*/
export function linearForm(
expression: MathExpression,
variable: string,
): { slope: MathNumber; intercept: MathNumber } | null {
const polynomial = Polynomial.extract(expression, variable);
if (!polynomial || polynomial.degree !== 1 || isZero(polynomial.at(1))) return null;
return { slope: polynomial.at(1), intercept: polynomial.at(0) };
}
+129
View File
@@ -0,0 +1,129 @@
// The glasses have no formula renderer and a 576-pixel-wide window, so every
// expression needs a compact ASCII form. Two things make this more than a
// tree walk. First, the internal representation is deliberately uniform —
// division is `a · b^-1`, subtraction is `a + (−b)` — and printing that
// literally would produce `2*x^-1` where a reader expects `2/x`, so the
// renderer detects those patterns and un-normalises them. Second, precedence
// decides bracketing: too few brackets is wrong, too many is unreadable.
import { type MathExpression, numberValue } from "./expression";
import { type MathNumber, describeNumber, isNegative, isOne, negated } from "./math-number";
const PRECEDENCE = {
relation: 1,
sum: 2,
product: 3,
power: 4,
} as const;
export function plain(expression: MathExpression): string {
return render(expression, 0);
}
export function renderNumber(value: MathNumber): string {
return describeNumber(value);
}
function render(expression: MathExpression, parentPrecedence: number): string {
switch (expression.type) {
case "number": {
const text = describeNumber(expression.value);
return wrap(text, isNegative(expression.value) && parentPrecedence >= PRECEDENCE.product);
}
case "variable":
return expression.name;
case "constant":
return expression.constant;
case "negate":
return wrap("-" + render(expression.inner, PRECEDENCE.product), parentPrecedence > PRECEDENCE.sum);
case "add": {
let output = "";
for (let index = 0; index < expression.terms.length; index++) {
const term = expression.terms[index]!;
if (term.type === "negate") {
output += index === 0 ? "-" : " - ";
output += render(term.inner, PRECEDENCE.sum);
continue;
}
const value = numberValue(term);
if (value && isNegative(value) && index > 0) {
output += " - " + describeNumber(negated(value));
continue;
}
if (index > 0) output += " + ";
output += render(term, PRECEDENCE.sum);
}
return wrap(output, parentPrecedence > PRECEDENCE.sum);
}
case "multiply": {
const numerator: MathExpression[] = [];
const denominator: MathExpression[] = [];
for (const factor of expression.factors) {
if (factor.type === "power") {
const power = numberValue(factor.exponent);
if (power && isNegative(power)) {
const positive = negated(power);
denominator.push(
isOne(positive) ? factor.base : { type: "power", base: factor.base, exponent: { type: "number", value: positive } },
);
continue;
}
}
numerator.push(factor);
}
const top =
numerator.length === 0
? "1"
: numerator.map((factor) => render(factor, PRECEDENCE.product)).join("*");
if (denominator.length === 0) {
return wrap(top, parentPrecedence > PRECEDENCE.product);
}
const bottom = denominator.map((factor) => render(factor, PRECEDENCE.power)).join("*");
return wrap(`${top}/${bottom}`, parentPrecedence > PRECEDENCE.product);
}
case "power": {
const exponentValue = numberValue(expression.exponent);
if (exponentValue && exponentValue.kind === "rational" && exponentValue.n === 1 && exponentValue.d === 2) {
return `sqrt(${render(expression.base, 0)})`;
}
// A bare negative power is a reciprocal. Printing `x^(-1)` where a
// reader expects `1/x` is the plain-text twin of the fraction
// un-normalisation above.
if (exponentValue && isNegative(exponentValue)) {
const positive = negated(exponentValue);
const inner = isOne(positive)
? render(expression.base, PRECEDENCE.power)
: `${render(expression.base, PRECEDENCE.power + 1)}^${describeNumber(positive)}`;
return wrap(`1/${inner}`, parentPrecedence > PRECEDENCE.product);
}
const body =
`${render(expression.base, PRECEDENCE.power + 1)}` +
`^${render(expression.exponent, PRECEDENCE.power + 1)}`;
return wrap(body, parentPrecedence > PRECEDENCE.power);
}
case "function": {
const inner = expression.args.map((argument) => render(argument, 0)).join(", ");
if (expression.fn === "factorial") return `(${inner})!`;
if (expression.fn === "abs") return `|${inner}|`;
return `${expression.fn}(${inner})`;
}
case "relation":
return (
`${render(expression.lhs, PRECEDENCE.relation)} ` +
`${expression.relation} ` +
`${render(expression.rhs, PRECEDENCE.relation)}`
);
}
}
function wrap(body: string, condition: boolean): string {
return condition ? `(${body})` : body;
}
+172
View File
@@ -0,0 +1,172 @@
// Trig equation solving and the identity sheet. The distinguishing feature
// versus the generic numeric root finder: a trig equation has infinitely many
// solutions, and the right answer is the FAMILY (`x = π/6 + 2πn`), not the
// twelve roots that happen to fall in the search window. Reporting a finite
// list for `sin x = 1/2` is the standard wrong answer in this topic.
import {
type MathExpression,
ZERO,
constant,
divideExpr,
int,
multiplyExpr,
} from "./expression";
import { describeNumber, fromDecimal } from "./math-number";
import { plain } from "./text-renderer";
// ---------------------------------------------------------------------------
// Solutions
/** One family of solutions: `base + period·n`. */
export type SolutionFamily = {
base: number;
period: number;
/** Exact form when the base is a recognisable multiple of π. */
baseExpression: MathExpression | null;
};
export function familyValues(family: SolutionFamily, lower: number, upper: number): number[] {
if (family.period <= 0) {
return family.base >= lower && family.base <= upper ? [family.base] : [];
}
const results: number[] = [];
let n = Math.floor((lower - family.base) / family.period);
for (;;) {
const value = family.base + n * family.period;
if (value > upper) break;
if (value >= lower) results.push(value);
n += 1;
if (results.length > 1000) break;
}
return results;
}
export function familyText(family: SolutionFamily): string {
const baseText = family.baseExpression
? plain(family.baseExpression)
: describeNumber(fromDecimal(family.base));
// The separator is load-bearing: "2*pi" run straight into the index reads
// as "2*pin", a different and nonsensical expression to anyone scanning it
// on the lens.
const periodText = describePi(family.period);
return `${baseText} + ${periodText}·n`;
}
export type SolutionSet =
| { kind: "families"; families: SolutionFamily[] }
| { kind: "all" }
| { kind: "none" };
/** Solve `f(x) = value` for sin, cos, or tan. */
export function solveTrig(functionName: "sin" | "cos" | "tan", value: number): SolutionSet {
switch (functionName) {
case "sin": {
if (Math.abs(value) > 1) return { kind: "none" };
const principal = Math.asin(value);
// sin has two families per period, which collapse to one when the value
// is ±1 (the turning points).
if (Math.abs(Math.abs(value) - 1) < 1e-12) {
return { kind: "families", families: [family(principal, 2 * Math.PI)] };
}
return {
kind: "families",
families: [family(principal, 2 * Math.PI), family(Math.PI - principal, 2 * Math.PI)],
};
}
case "cos": {
if (Math.abs(value) > 1) return { kind: "none" };
const principal = Math.acos(value);
if (Math.abs(principal) < 1e-12 || Math.abs(principal - Math.PI) < 1e-12) {
return { kind: "families", families: [family(principal, 2 * Math.PI)] };
}
return {
kind: "families",
families: [family(principal, 2 * Math.PI), family(-principal, 2 * Math.PI)],
};
}
case "tan":
// tan repeats every π, not 2π — one family covers everything.
return { kind: "families", families: [family(Math.atan(value), Math.PI)] };
}
}
function family(base: number, period: number): SolutionFamily {
return { base, period, baseExpression: piMultiple(base) };
}
/**
* Recognise a value as an exact rational multiple of π, so a solution reads
* `π/6` instead of `0.5235987755982988`.
*/
export function piMultiple(value: number): MathExpression | null {
if (!Number.isFinite(value)) return null;
if (Math.abs(value) < 1e-12) return ZERO;
const ratio = value / Math.PI;
for (const denominator of [1, 2, 3, 4, 6, 8, 12]) {
const scaled = ratio * denominator;
const rounded = Math.round(scaled);
if (Math.abs(scaled - rounded) >= 1e-9 || Math.abs(rounded) >= 1e6) continue;
const numerator = rounded;
if (numerator === 0) return ZERO;
if (numerator === 1 && denominator === 1) return constant("pi");
// pi/6, not (1/6)*pi — a unit numerator belongs in the fraction, which is
// how every textbook writes these solution families.
if (numerator === 1) return divideExpr(constant("pi"), int(denominator));
if (denominator === 1) return multiplyExpr([int(numerator), constant("pi")]);
return divideExpr(multiplyExpr([int(numerator), constant("pi")]), int(denominator));
}
return null;
}
function describePi(value: number): string {
const multiple = piMultiple(value);
return multiple ? plain(multiple) : describeNumber(fromDecimal(value));
}
// ---------------------------------------------------------------------------
// Identities
export type TrigIdentity = {
name: string;
plain: string;
};
/**
* The identity sheet, for the "explain it" surface and for a wearer who
* wants the reference on the lens.
*/
export const TRIG_IDENTITIES: readonly TrigIdentity[] = [
{ name: "Pythagorean", plain: "sin²θ + cos²θ = 1" },
{ name: "Pythagorean (tan)", plain: "1 + tan²θ = sec²θ" },
{ name: "Sine sum", plain: "sin(a ± b) = sin a cos b ± cos a sin b" },
{ name: "Cosine sum", plain: "cos(a ± b) = cos a cos b ∓ sin a sin b" },
{ name: "Tangent sum", plain: "tan(a ± b) = (tan a ± tan b) / (1 ∓ tan a tan b)" },
{ name: "Double angle (sine)", plain: "sin 2θ = 2 sin θ cos θ" },
{ name: "Double angle (cosine)", plain: "cos 2θ = cos²θ − sin²θ = 2cos²θ − 1" },
{ name: "Half angle", plain: "sin²θ = (1 − cos 2θ)/2" },
{ name: "Law of sines", plain: "a/sin A = b/sin B = c/sin C" },
{ name: "Law of cosines", plain: "c² = a² + b² − 2ab cos C" },
];
export function identityMatching(query: string): TrigIdentity | null {
const needle = query.toLowerCase();
return (
TRIG_IDENTITIES.find((identity) => identity.name.toLowerCase().includes(needle)) ??
TRIG_IDENTITIES.find((identity) => needle.includes(identity.name.toLowerCase())) ??
null
);
}
// ---------------------------------------------------------------------------
// Conversion
export function degreesFromRadians(radians: number): number {
return (radians * 180) / Math.PI;
}
export function radiansFromDegrees(degrees: number): number {
return (degrees * Math.PI) / 180;
}
+292
View File
@@ -0,0 +1,292 @@
// Workload estimation and routing for heavy maths requests.
//
// The router collapses to two honest options: the fast on-device answer, and an
// explicit, cancellable long run on the phone that the wearer opts into.
// The estimate is the whole point — every workload carries a closed-form
// cost model so the offer can price the wait before doing any work.
import {
type ComputeCancellation,
type ComputeProgress,
type MathConstantName,
MAXIMUM_COMPUTED_DIGITS,
MAXIMUM_FEASIBLE_DIGITS,
NEVER_CANCELLED,
constantSpokenName,
deepFractionalDigits,
} from "./constant-digits";
import {
type DigitCountResult,
type DigitSearchOutcome,
countInExpansion,
locateDigits,
} from "./digit-search";
import { formatInt } from "./math-number";
// ---------------------------------------------------------------------------
// Durations
/**
* Turns a seconds estimate into something worth reading out. Deliberately
* coarse: the cost model behind these numbers is accurate to an order of
* magnitude, not to the second, so "about 247 seconds" would be a precision
* the estimate does not have.
*/
export function describeDuration(seconds: number): string {
if (!Number.isFinite(seconds) || seconds <= 0) return "no time at all";
if (seconds < 1) return "under a second";
if (seconds < 10) return `about ${Math.round(seconds)} seconds`;
if (seconds < 60) return `about ${roundedTo(seconds, 5)} seconds`;
const minutes = seconds / 60;
if (minutes < 2) return "about a minute and a half";
if (minutes < 60) return `about ${Math.round(minutes)} minutes`;
const hours = minutes / 60;
if (hours < 2) return "about an hour and a half";
if (hours < 24) return `about ${Math.round(hours)} hours`;
const days = hours / 24;
if (days < 365) return `about ${Math.round(days)} days`;
const years = days / 365;
return years < 2 ? "well over a year" : `about ${Math.round(years)} years`;
}
/** Short form for the lens, where there is no room for a sentence. */
export function compactDuration(seconds: number): string {
if (!Number.isFinite(seconds) || seconds <= 0) return "0s";
if (seconds < 60) return `${Math.round(seconds)}s`;
if (seconds < 3600) return `${Math.round(seconds / 60)}m`;
if (seconds < 86_400) return `${Math.round(seconds / 3600)}h`;
return `${Math.round(seconds / 86_400)}d`;
}
function roundedTo(value: number, step: number): number {
return Math.max(step, Math.round(value / step) * step);
}
// ---------------------------------------------------------------------------
// Workloads
export type MathWorkload =
/** Generate the decimal expansion of a constant. */
| { kind: "constantDigits"; constant: MathConstantName; count: number }
/** Find a digit run inside a constant's expansion. */
| { kind: "digitSearch"; constant: MathConstantName; pattern: string; occurrence: number }
/** Count a digit run across the deepest available expansion. */
| { kind: "digitCount"; constant: MathConstantName; pattern: string };
/**
* Rough seconds on the phone. Machin with scalar big arithmetic is O(n²);
* measured with V8 BigInt: 10,000 digits ≈ 0.06 s on an M-class core, so
* ~0.35 s per (n/10k)² on a device-class core. It does not need to be
* precise, only right about the ORDER OF MAGNITUDE, because the routing
* threshold and the offers are separated by factors of thousands — and at
* this value an hour reaches about a million digits, which is where any
* five-digit run and most six-digit ones live.
*/
const SECONDS_PER_TEN_K_SQUARED = 0.35;
export function estimatedLocalSeconds(workload: MathWorkload): number {
switch (workload.kind) {
case "constantDigits": {
const n = Math.max(workload.count, 1);
return SECONDS_PER_TEN_K_SQUARED * (n / 10_000) * (n / 10_000);
}
case "digitSearch":
// Dominated by generating enough digits to have a hope of finding the
// run: a k-digit pattern first appears around 10^k places.
return estimatedLocalSeconds({
kind: "constantDigits",
constant: workload.constant,
count: expectedDepth(workload.pattern.length),
});
case "digitCount":
return estimatedLocalSeconds({
kind: "constantDigits",
constant: workload.constant,
count: expectedDepth(workload.pattern.length),
});
}
}
/**
* How many decimal places the workload would have to reach to answer
* properly, as opposed to however few it settled for.
*/
export function requiredDepth(workload: MathWorkload): number {
switch (workload.kind) {
case "constantDigits":
return Math.max(workload.count, 1);
case "digitSearch":
// The n-th occurrence sits roughly n× as deep as the first.
return expectedDepth(workload.pattern.length) * Math.max(workload.occurrence, 1);
case "digitCount":
return expectedDepth(workload.pattern.length);
}
}
/** Roughly where a random run of `length` digits first appears. */
export function expectedDepth(patternLength: number): number {
if (patternLength <= 0) return 1_000;
// Cap so the estimator does not overflow on a silly pattern.
const bounded = Math.min(patternLength, 12);
return Math.max(Math.pow(10, bounded), 1_000);
}
export function describeWorkload(workload: MathWorkload): string {
switch (workload.kind) {
case "constantDigits":
return `${formatInt(workload.count)} digits of ${constantSpokenName(workload.constant)}`;
case "digitSearch":
return `searching ${constantSpokenName(workload.constant)} for ${workload.pattern}`;
case "digitCount":
return `counting ${workload.pattern} in ${constantSpokenName(workload.constant)}`;
}
}
// ---------------------------------------------------------------------------
// Offers
/**
* An offer to grind the full-depth answer out on the phone. Produced when a
* question needs a deeper expansion than the fast path holds. It is a value,
* not an action: nothing runs until the caller accepts it, because the whole
* point is that the wearer decides whether the wait is affordable right now.
*/
export type LongRunOffer = {
/** What would be run. */
workload: MathWorkload;
/** The phone's own estimate for the full-depth run. */
estimatedSeconds: number;
/** Why the fast path could not settle it. */
reason: string;
/** How deep the run would go — digits, for the expansion workloads. */
depth: number;
/**
* What was already answered without waiting, if anything. A shallow search
* still ran, so the offer is "look further", not "look at all".
*/
shallowAnswerDescription: string | null;
};
/** The sentence the answer view shows. */
export function offerPrompt(offer: LongRunOffer): string {
return (
`${offer.reason}. This phone can keep going to ${formatInt(offer.depth)} digits — ` +
`${describeDuration(offer.estimatedSeconds)}. Run it anyway?`
);
}
/** Two short rows for the lens. */
export function offerGlassesPrompt(offer: LongRunOffer): string {
return `Keep going? ${compactDuration(offer.estimatedSeconds)}\n${formatInt(offer.depth)} digits on phone`;
}
/** Progress from a long local run, so a four-minute grind is not a dead screen. */
export type LongRunProgress = {
/** 0…1, or null when the phase cannot estimate itself. */
fraction: number | null;
/** What it is doing right now. */
stage: string;
/** Seconds elapsed so far. */
elapsedSeconds: number;
};
export function progressText(progress: LongRunProgress): string {
if (progress.fraction === null) return progress.stage;
return `${Math.round(progress.fraction * 100)}% ${progress.stage}`;
}
/**
* The longest run the phone will even OFFER to attempt once the wearer has
* said they will wait. An hour reaches most five- and six-digit runs under
* the cost model. Past it the offer is withheld rather than shown: proposing
* a multi-hour grind that still might not answer the question is worse than
* plainly saying this device cannot settle it.
*/
export const LONG_RUN_CEILING_SECONDS = 3600;
/**
* Build the offer, or decline to make one. Withheld in two cases, both
* because the wait would not buy an answer: the depth needed exceeds what
* the phone can hold, or the run would take longer than the ceiling.
*/
export function longRunOffer(
workload: MathWorkload,
reason: string,
shallow: string | null = null,
): LongRunOffer | null {
const depth = requiredDepth(workload);
if (depth <= MAXIMUM_COMPUTED_DIGITS) return null;
if (depth > MAXIMUM_FEASIBLE_DIGITS) return null;
const estimate = estimatedLocalSeconds({
kind: "constantDigits",
constant: workload.constant,
count: depth,
});
if (estimate > LONG_RUN_CEILING_SECONDS) return null;
return {
workload,
estimatedSeconds: estimate,
reason,
depth,
shallowAnswerDescription: shallow,
};
}
// ---------------------------------------------------------------------------
// Accepted runs
/**
* Run the search at full depth on the phone. Only called once the wearer has
* accepted the offer's stated wait. Async and chunked — the generators yield
* to the event loop, and the cancellation closure is checked every term.
*/
export async function runAcceptedSearch(
offer: LongRunOffer,
cancellation: ComputeCancellation = NEVER_CANCELLED,
progress: ComputeProgress | null = null,
): Promise<DigitSearchOutcome> {
if (offer.workload.kind !== "digitSearch") {
return { kind: "notFound", searchedDigits: 0 };
}
const expansion = await deepFractionalDigits(
offer.workload.constant,
offer.depth,
cancellation,
progress,
);
if (!expansion) return { kind: "notFound", searchedDigits: 0 };
return locateDigits(offer.workload.pattern, expansion, offer.workload.occurrence);
}
/** Generate the accepted depth and count overlapping occurrences there. */
export async function runAcceptedCount(
offer: LongRunOffer,
cancellation: ComputeCancellation = NEVER_CANCELLED,
progress: ComputeProgress | null = null,
): Promise<DigitCountResult> {
if (offer.workload.kind !== "digitCount") {
return { occurrences: 0, searchedDigits: 0 };
}
const expansion = await deepFractionalDigits(
offer.workload.constant,
offer.depth,
cancellation,
progress,
);
if (!expansion) return { occurrences: 0, searchedDigits: 0 };
return countInExpansion(offer.workload.pattern, expansion);
}
/** Generate the digits at full depth, once the offer has been accepted. */
export async function runAcceptedDigits(
offer: LongRunOffer,
cancellation: ComputeCancellation = NEVER_CANCELLED,
progress: ComputeProgress | null = null,
): Promise<string | null> {
return deepFractionalDigits(offer.workload.constant, offer.depth, cancellation, progress);
}
+137
View File
@@ -0,0 +1,137 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../../ui/gestures";
import { Layer, type DashboardInputEvent, type LayerContext } from "../../ui/layers";
import { micSession, type CaptionLine, type MicSessionState } from "./mic-session";
/**
* On-glasses captions: each line is prefixed with the speaker's name (their
* id until tagged); foreign-language lines show the translation directly
* beneath the recognized original. Scroll reviews recent lines; the view
* snaps back to live on new speech.
*/
export class CaptionsLayer implements Layer {
private state: MicSessionState = micSession.getState();
private unsubscribe: (() => void) | null = null;
private scrollback = 0;
private lastLineCount = 0;
start(requestRender: () => void): void {
this.unsubscribe = micSession.onState((state) => {
if (state.captionLines.length !== this.lastLineCount) {
this.lastLineCount = state.captionLines.length;
this.scrollback = 0;
}
this.state = state;
requestRender();
});
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const state = this.state;
image.drawText(font, 12, 4, "Captions", 220);
if (!state.captionsActive) {
image.drawText(font, 12, 26, "Captions are off (enable in the Microphones menu).", 130);
}
const rows = this.buildRows(font, width - 36, state.captionLines);
const step = lineStep(font) + 1;
const bodyTop = 24;
const footerY = height - font.lineHeight - 4;
const visibleRows = Math.max(1, Math.floor((footerY - bodyTop) / step));
const maxScrollback = Math.max(0, rows.length - visibleRows);
this.scrollback = Math.min(this.scrollback, maxScrollback);
const firstRow = Math.max(0, rows.length - visibleRows - this.scrollback);
for (let index = firstRow; index < Math.min(rows.length, firstRow + visibleRows); index++) {
const row = rows[index]!;
const y = bodyTop + (index - firstRow) * step;
if (row.prefix) {
image.drawText(font, 16, y, row.prefix, row.prefixValue);
image.drawText(font, 16 + font.measureText(row.prefix), y, row.text, row.value);
} else {
image.drawText(font, 16 + row.indent, y, row.text, row.value);
}
}
image.drawText(
font,
12,
footerY,
`${GESTURE_SCROLL} history ${GESTURE_DOUBLE_CLICK} back` +
(this.scrollback > 0 ? ` (${this.scrollback} back)` : ""),
110,
);
return image;
}
private buildRows(
font: UiFont,
maxWidth: number,
lines: CaptionLine[],
): Array<{ prefix?: string; prefixValue: number; text: string; value: number; indent: number }> {
const rows: Array<{ prefix?: string; prefixValue: number; text: string; value: number; indent: number }> = [];
for (const line of lines) {
const prefix = `${line.speakerName}: `;
const prefixWidth = font.measureText(prefix);
const wrapped = wrapWords(font, line.text, maxWidth - prefixWidth);
wrapped.forEach((text, index) => {
rows.push({
prefix: index === 0 ? prefix : undefined,
prefixValue: line.isWearer ? 255 : 170,
text,
value: 230,
indent: index === 0 ? 0 : prefixWidth,
});
});
if (line.translation) {
for (const text of wrapWords(font, `→ ${line.translation}`, maxWidth - prefixWidth)) {
rows.push({ prefixValue: 0, text, value: 140, indent: prefixWidth });
}
}
}
return rows;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
this.scrollback += 1;
return;
case "scroll-down":
this.scrollback = Math.max(0, this.scrollback - 1);
return;
case "double-click":
ctx.stack.pop();
return;
default:
return;
}
}
onRemoved(): void {
this.unsubscribe?.();
this.unsubscribe = null;
}
}
function wrapWords(font: UiFont, text: string, maxWidth: number): string[] {
const words = text.split(/\s+/).filter(Boolean);
const lines: string[] = [];
let line = "";
for (const word of words) {
const candidate = line ? `${line} ${word}` : word;
if (line && font.measureText(candidate) > maxWidth) {
lines.push(line);
line = word;
} else {
line = candidate;
}
}
if (line) lines.push(line);
return lines.length ? lines : [""];
}
+240
View File
@@ -0,0 +1,240 @@
import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
import { parseConversationQuery } from "../../phone-ui/conversation-search";
declare const com: any;
/**
* Question answering over saved conversations with the on-phone LLM (the
* assistant's Qwen model). Retrieval-then-read: the question is parsed for
* date/emotion filters and keywords, matching transcript segments and
* session/speaker metadata are pulled from the conversation store, and the
* model answers strictly from that context. Everything stays on-device.
*/
// Qwen runs with a 4096-token context; leave room for the answer.
const MAX_CONTEXT_CHARS = 7000;
const MAX_KEYWORDS = 6;
const SEGMENTS_PER_KEYWORD = 30;
const RECENT_SEGMENT_LIMIT = 120;
const SESSION_LIMIT = 20;
const KEYWORD_STOPWORDS = new Set(
("the a an and or but of in on at to from with about for was were is are am be been did do does have has had " +
"what when where who whom whose which why how tell show me my i we us our you your they them their it its " +
"that this these those there anyone someone something anything talk talked talking say said saying speak spoke " +
"conversation conversations chat chats discussion discussions transcript transcripts recording recordings").split(" "),
);
export type ConversationAnswerCallbacks = {
onDelta?: (textSoFar: string) => void;
onDone: (answer: string, contextNote: string) => void;
onError: (message: string) => void;
};
export function conversationQaAvailable(): boolean {
return global.isAndroid && isLocalModelReady();
}
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
type SegmentRow = {
id: number;
sessionId: number;
speakerId: number | null;
startedAt: number;
text: string;
translation: string;
emotion: string;
sentiment: number;
};
function parseSegments(json: string): SegmentRow[] {
return (JSON.parse(json) as Array<Record<string, unknown>>).map((row) => ({
id: Number(row.id),
sessionId: Number(row.sessionId ?? 0),
speakerId: row.speakerId === null ? null : Number(row.speakerId),
startedAt: Number(row.startedAt),
text: String(row.text ?? ""),
translation: String(row.translation ?? ""),
emotion: String(row.emotion ?? ""),
sentiment: Number(row.sentiment ?? 0),
}));
}
function keywords(text: string): string[] {
const seen = new Set<string>();
for (const raw of text.toLowerCase().split(/[^a-z0-9']+/)) {
const word = raw.trim();
if (word.length < 3 || KEYWORD_STOPWORDS.has(word)) continue;
seen.add(word);
if (seen.size >= MAX_KEYWORDS) break;
}
return [...seen];
}
function formatWhen(ms: number): string {
const date = new Date(ms);
const pad = (value: number) => String(value).padStart(2, "0");
return (
`${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ` +
`${pad(date.getHours())}:${pad(date.getMinutes())}`
);
}
/**
* Assemble the retrieval context: speaker roster (names, tags, wearer flag),
* session metadata, and the transcript lines most relevant to the question.
* Returns the context text plus a short human-readable note about coverage.
*/
function buildContext(question: string): { context: string; note: string } {
const filters = parseConversationQuery(question);
const filterJson: Record<string, unknown> = {};
if (filters.sinceMs !== null) filterJson.sinceMs = filters.sinceMs;
if (filters.untilMs !== null) filterJson.untilMs = filters.untilMs;
if (filters.emotion !== null) filterJson.emotion = filters.emotion;
const speakerNames = new Map<number, string>();
const speakerLines: string[] = [];
for (const row of JSON.parse(String(store().querySpeakers())) as Array<Record<string, unknown>>) {
const id = Number(row.id);
const name = String(row.name ?? "");
speakerNames.set(id, name);
const parts = [name];
if (row.isWearer) parts.push("(the wearer/user)");
if (row.tag) parts.push(`tag: ${String(row.tag)}`);
parts.push(`${Number(row.segmentCount ?? 0)} lines heard`);
speakerLines.push(`- ${parts.join(", ")}`);
}
const sessions = JSON.parse(
String(store().querySessions(JSON.stringify({ ...filterJson, limit: SESSION_LIMIT }))),
) as Array<Record<string, unknown>>;
const sessionTitles = new Map<number, string>();
const sessionLines: string[] = [];
for (const row of sessions) {
const id = Number(row.id);
const title = String(row.title ?? "") || `Conversation ${formatWhen(Number(row.startedAt))}`;
sessionTitles.set(id, title);
const who = ((row.speakers as Array<{ name?: unknown }>) ?? [])
.map((speaker) => String(speaker.name ?? ""))
.filter(Boolean)
.join(", ");
sessionLines.push(
`- [#${id}] ${title} — ${formatWhen(Number(row.startedAt))}, ${Number(row.segmentCount ?? 0)} lines` +
(who ? `, speakers: ${who}` : "") +
`, overall sentiment ${Number(row.avgSentiment ?? 0).toFixed(2)}`,
);
}
const terms = keywords(filters.text);
const segmentMap = new Map<number, SegmentRow>();
if (terms.length > 0) {
for (const term of terms) {
const rows = parseSegments(
String(
store().searchSegments(
JSON.stringify({ ...filterJson, query: term, limit: SEGMENTS_PER_KEYWORD }),
),
),
);
for (const row of rows) segmentMap.set(row.id, row);
}
}
if (segmentMap.size === 0) {
// No keyword hits (or no keywords): fall back to the filtered recents so
// summary-style questions still have material to read.
for (const row of parseSegments(
String(store().searchSegments(JSON.stringify({ ...filterJson, limit: RECENT_SEGMENT_LIMIT }))),
)) {
segmentMap.set(row.id, row);
}
}
const segments = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
const excerptLines: string[] = [];
let currentSession = -1;
for (const segment of segments) {
if (segment.sessionId !== currentSession) {
currentSession = segment.sessionId;
excerptLines.push(`\n## ${sessionTitles.get(currentSession) ?? `Conversation #${currentSession}`}`);
}
const speaker =
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
excerptLines.push(`[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`);
if (segment.translation) excerptLines.push(` (translated: ${segment.translation})`);
}
let context =
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
`${sessionLines.join("\n") || "(none in range)"}\n\n` +
`# Transcript excerpts\n${excerptLines.join("\n") || "(no matching lines)"}`;
let truncated = false;
if (context.length > MAX_CONTEXT_CHARS) {
context = context.slice(0, MAX_CONTEXT_CHARS);
truncated = true;
}
const note =
`${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` +
`${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` +
(terms.length ? ` (keywords: ${terms.join(", ")})` : "") +
(truncated ? "; context truncated to fit the on-device model" : "");
return { context, note };
}
/**
* Ask the on-device model a question about saved conversations. Returns a
* cancel handle; the answer streams via onDelta and lands in onDone with a
* note describing what data was consulted.
*/
export function askConversations(
question: string,
callbacks: ConversationAnswerCallbacks,
): LlmStreamHandle {
if (!conversationQaAvailable()) {
setTimeout(
() =>
callbacks.onError(
"The on-phone model isn't downloaded yet — enable it under Settings > Assistant on the glasses.",
),
0,
);
return { cancel: () => {} };
}
let context: { context: string; note: string };
try {
context = buildContext(question);
} catch (error) {
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
return { cancel: () => {} };
}
return streamLocalQwen({
apiKey: "",
model: "local",
system:
"You answer questions about the user's saved conversations, captured by their smart glasses. " +
"Use ONLY the provided speaker roster, conversation list, and transcript excerpts. " +
"Cite conversations by their title and date when referencing them. " +
"If the provided data does not contain the answer, say so plainly rather than guessing. " +
"Transcript sentiment scores range -1 (very negative) to 1 (very positive); " +
"bracketed words after a speaker's name are detected emotional tones. " +
`Today's date is ${formatWhen(Date.now())}.`,
messages: [
{
role: "user",
content: `${context.context}\n\n# Question\n${question.trim()}`,
},
],
maxTokens: 400,
onTextDelta: (_delta, textSoFar) => callbacks.onDelta?.(textSoFar),
onDone: (result) => callbacks.onDone(result.text.trim(), context.note),
onError: (message) => callbacks.onError(message),
});
}
+312
View File
@@ -0,0 +1,312 @@
// Reference DSP for the G2 4-mic head array: two 2-mic temples, front mic
// near each hinge and rear mic at each tip. Beamform WITHIN each temple (shared clock), then FUSE
// the two per-temple bearings — never raw cross-temple delay-and-sum, because
// the temples sample on independent clocks.
//
// One substitution from classic GCC-PHAT: the FFT cross-spectrum is
// replaced by a direct time-domain normalized cross-correlation over
// -maxLag..+maxLag. For the lags this array needs (|lag| <= 16 at 16 kHz) the
// direct form is cheaper than an FFT round-trip and carries no FFT dependency;
// the parabolic-interpolated peak still yields sub-sample delay.
//
// Pure TypeScript; no platform imports. Hot paths are Float32Array in and out.
export type MicSide = "left" | "right";
export type MicPosition = "front" | "rear";
// ---------------------------------------------------------------------------
// Array geometry (metres, head-centred; x=right, y=forward, z=up)
export type Vec3 = { x: number; y: number; z: number };
export type MicGeometry = {
positions: Record<MicSide, Record<MicPosition, Vec3>>;
speedOfSound: number;
};
export const G2_DEFAULT_GEOMETRY: MicGeometry = {
positions: {
left: {
front: { x: -0.07, y: 0.05, z: 0 },
rear: { x: -0.08, y: -0.06, z: 0 },
},
right: {
front: { x: 0.07, y: 0.05, z: 0 },
rear: { x: 0.08, y: -0.06, z: 0 },
},
},
speedOfSound: 343,
};
// ---------------------------------------------------------------------------
// Bearing helpers
/** Azimuth in radians, 0 = straight ahead (device +y), positive to the right. */
export type Bearing = { azimuth: number };
export function bearingDegrees(bearing: Bearing): number {
return (bearing.azimuth * 180) / Math.PI;
}
export function bearingUnitVector(bearing: Bearing): Vec3 {
return { x: Math.sin(bearing.azimuth), y: Math.cos(bearing.azimuth), z: 0 };
}
function dot(a: Vec3, b: Vec3): number {
return a.x * b.x + a.y * b.y + a.z * b.z;
}
// ---------------------------------------------------------------------------
// Fractional delay (shared by the beamformer and by tests)
/** Linear-interpolating fractional delay (positive = later). */
export function fractionalDelay(x: Float32Array, samples: number): Float32Array {
const n = x.length;
if (Math.abs(samples) < 1e-4) return x.slice();
const out = new Float32Array(n);
const i0 = Math.floor(samples);
const frac = samples - i0;
for (let i = 0; i < n; i++) {
const j = i - i0;
const a = j >= 0 && j < n ? x[j]! : 0;
const b = j - 1 >= 0 && j - 1 < n ? x[j - 1]! : 0;
out[i] = a * (1 - frac) + b * frac;
}
return out;
}
// ---------------------------------------------------------------------------
// Delay-and-sum beamformer (per temple, 2 mics, shared clock)
export class DelaySumBeamformer {
constructor(
public readonly sampleRate: number,
public readonly geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
) {}
/**
* Steer this temple's front/rear pair toward `target` (device-frame bearing)
* and return the summed beam, halved to preserve amplitude.
*/
beam(side: MicSide, front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
const n = Math.min(front.length, rear.length);
if (n === 0) return new Float32Array(0);
const d = bearingUnitVector(target);
const pF = this.geometry.positions[side].front;
const pR = this.geometry.positions[side].rear;
// Delay (samples) that aligns each mic's copy of a wavefront from `d`.
const tauF = (dot(pF, d) / this.geometry.speedOfSound) * this.sampleRate;
const tauR = (dot(pR, d) / this.geometry.speedOfSound) * this.sampleRate;
const ref = Math.min(tauF, tauR);
const sF = fractionalDelay(front, tauF - ref);
const sR = fractionalDelay(rear, tauR - ref);
const out = new Float32Array(n);
for (let i = 0; i < n; i++) out[i] = (sF[i]! + sR[i]!) * 0.5;
return out;
}
}
// ---------------------------------------------------------------------------
// Two-mic adaptive noise canceller (NLMS) for inter-mic ANC
/**
* Griffiths–Jim style: `primary` carries the steered speech, `reference` is a
* null-steered noise estimate (front−rear after alignment). An NLMS filter
* subtracts the correlated noise from the primary.
*/
export class InterMicNoiseCanceller {
private readonly weights: Float32Array;
private readonly history: Float32Array;
constructor(
private readonly taps: number = 32,
private readonly stepSize: number = 0.05,
) {
this.weights = new Float32Array(taps);
this.history = new Float32Array(taps);
}
process(primary: Float32Array, reference: Float32Array): Float32Array {
const n = Math.min(primary.length, reference.length);
const out = new Float32Array(n);
const { weights, history, taps } = this;
const eps = 1e-6;
for (let i = 0; i < n; i++) {
for (let k = taps - 1; k > 0; k--) history[k] = history[k - 1]!;
history[0] = reference[i]!;
let yhat = 0;
let energy = 0;
for (let k = 0; k < taps; k++) {
yhat += weights[k]! * history[k]!;
energy += history[k]! * history[k]!;
}
const e = primary[i]! - yhat;
out[i] = e;
const step = (this.stepSize * e) / (energy + eps);
for (let k = 0; k < taps; k++) weights[k] += step * history[k]!;
}
return out;
}
}
// ---------------------------------------------------------------------------
// TDOA via direct normalized cross-correlation (GCC-PHAT substitution)
/**
* Estimate the delay of `b` relative to `a` in samples: if `b` is a copy of
* `a` delayed by k samples, this returns approximately +k.
*
* Substitution note: classic GCC-PHAT whitens an FFT cross-spectrum. Here
* the lag search is tiny (|lag| <= maxLag, typically <= 16),
* so a direct time-domain normalized cross-correlation over the lag range is
* cheaper and dependency-free. The per-lag normalization by the overlapped
* segments' energies plays the whitening role of making the peak
* amplitude-invariant. The peak is refined by parabolic interpolation for
* sub-sample precision.
*/
export function gccPhatDelaySamples(a: Float32Array, b: Float32Array, maxLag: number): number {
const n = Math.min(a.length, b.length);
if (n === 0 || maxLag < 0) return 0;
const lagCount = 2 * maxLag + 1;
const corr = new Float32Array(lagCount);
let bestLag = 0;
let best = -Infinity;
for (let lag = -maxLag; lag <= maxLag; lag++) {
// Correlate a[i] against b[i + lag] over the overlapping span.
const start = Math.max(0, -lag);
const end = Math.min(n, n - lag);
let sum = 0;
let ea = 0;
let eb = 0;
for (let i = start; i < end; i++) {
const av = a[i]!;
const bv = b[i + lag]!;
sum += av * bv;
ea += av * av;
eb += bv * bv;
}
const r = sum / (Math.sqrt(ea * eb) + 1e-12);
corr[lag + maxLag] = r;
if (r > best) {
best = r;
bestLag = lag;
}
}
// Parabolic interpolation around the integer peak for sub-sample precision.
const idx = bestLag + maxLag;
if (idx > 0 && idx < lagCount - 1) {
const y0 = corr[idx - 1]!;
const y1 = corr[idx]!;
const y2 = corr[idx + 1]!;
const denom = y0 - 2 * y1 + y2;
if (Math.abs(denom) > 1e-12) {
const offset = (0.5 * (y0 - y2)) / denom;
if (Math.abs(offset) <= 1) return bestLag + offset;
}
}
return bestLag;
}
// ---------------------------------------------------------------------------
// Compass / IMU fusion: world-frame arc <-> device-frame steering
/**
* `compassHeading` is the world heading of the device +y axis (radians,
* 0 = north); `imuYaw` is a supplementary IMU trim (radians).
*/
export type HeadOrientation = { compassHeading: number; imuYaw: number };
/** Combined world→device rotation about vertical (compass anchored, IMU trims). */
function worldToDevice(head: HeadOrientation): number {
return -(head.compassHeading + head.imuYaw);
}
/** Rotate a world-frame target bearing into the device frame for steering. */
export function steerWorldToDevice(worldAzimuth: number, head: HeadOrientation): Bearing {
return { azimuth: worldAzimuth + worldToDevice(head) };
}
/** Convert a device-frame estimate back to world frame for display/fusion. */
export function deviceToWorld(deviceAzimuth: number, head: HeadOrientation): number {
return deviceAzimuth - worldToDevice(head);
}
/**
* Triangulate the two temples' world-frame bearings into a single estimate:
* a circular-mean-safe blend, weighted by each temple's confidence.
*/
export function fuseBearings(
left: number,
leftWeight: number,
right: number,
rightWeight: number,
): number {
const w = Math.max(1e-3, leftWeight + rightWeight);
const x = (Math.sin(left) * leftWeight + Math.sin(right) * rightWeight) / w;
const y = (Math.cos(left) * leftWeight + Math.cos(right) * rightWeight) / w;
return Math.atan2(x, y);
}
// ---------------------------------------------------------------------------
// Faceclaw integration utilities
/**
* Azimuth from an inter-mic delay, mirroring the firmware's approach:
* asin of the delay normalized by the acoustic travel time across the pair,
* clamped to ±90°.
*/
export function bearingFromTdoa(
delaySamples: number,
sampleRate: number,
micSpacingMeters: number,
): number {
const maxDelay = (micSpacingMeters / 343) * sampleRate;
if (maxDelay <= 0) return 0;
const normalized = Math.max(-1, Math.min(1, delaySamples / maxDelay));
return Math.asin(normalized);
}
/** Wrap an angle to (-π, π]. */
function wrapAngle(angle: number): number {
const twoPi = 2 * Math.PI;
let a = angle % twoPi;
if (a <= -Math.PI) a += twoPi;
else if (a > Math.PI) a -= twoPi;
return a;
}
/** Whether `azimuth` falls inside the arc, handling wraparound at ±π. */
export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: number): boolean {
return Math.abs(wrapAngle(azimuth - arcCenter)) <= arcHalfWidth;
}
// ---------------------------------------------------------------------------
// Top-level per-temple pipeline
export class TempleAudioPipeline {
private readonly beamformer: DelaySumBeamformer;
private readonly anc = new InterMicNoiseCanceller();
enableANC = true;
constructor(
public readonly side: MicSide,
sampleRate: number,
geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
) {
this.beamformer = new DelaySumBeamformer(sampleRate, geometry);
}
/**
* Process one frame's deinterleaved front/rear channels toward a
* device-frame target, returning the enhanced mono beam for this temple.
*/
process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
const beam = this.beamformer.beam(this.side, front, rear, target);
if (!this.enableANC) return beam;
// Null-steer reference: the difference channel carries off-axis noise.
const n = Math.min(front.length, rear.length);
const reference = new Float32Array(n);
for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!;
return this.anc.process(beam, reference);
}
}
+30
View File
@@ -0,0 +1,30 @@
import { type AppDefinition } from "../app-definition";
import {
createMicrophonesAppWindow,
MICROPHONES_SURFACE_ID,
MICROPHONES_WINDOW_ID,
} from "./microphones-app";
import { micSession } from "./mic-session";
/**
* The Microphones app: per-mic control and levels, the Sonic Radar
* (direction-of-arrival + beam-filtered listening), and live captions with
* speaker recognition, translation, and tone analysis. Saved conversations
* are reviewed in the phone UI.
*/
const microphonesApp: AppDefinition = {
appId: "microphones",
title: "Microphones",
icon: "mic",
boot: () => {
// Prune expired captions/recordings per the retention settings.
void micSession.applyRetentionSweep();
},
launch: async (ctx) => {
await ctx.launchInProcessApp(MICROPHONES_WINDOW_ID, MICROPHONES_SURFACE_ID, (options) =>
createMicrophonesAppWindow(options),
);
},
};
export default microphonesApp;
+80
View File
@@ -0,0 +1,80 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GESTURE_DOUBLE_CLICK } from "../../ui/gestures";
import { Layer, type DashboardInputEvent, type LayerContext } from "../../ui/layers";
import { micSession, type MicSessionState } from "./mic-session";
/**
* Live level meters, one per active microphone (a single mixed channel on
* stock firmware; up to four mics with the mic-control firmware), plus the
* firmware's signal-strength/DoA readout.
*/
export class LevelsLayer implements Layer {
private state: MicSessionState = micSession.getState();
private unsubscribe: (() => void) | null = null;
start(requestRender: () => void): void {
this.unsubscribe = micSession.onState((state) => {
this.state = state;
requestRender();
});
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const state = this.state;
image.drawText(font, 12, 6, "Microphone Levels", 220);
image.drawText(font, 12, 24, state.statusText, 110);
const barX = 130;
const barWidth = width - barX - 40;
const rowHeight = 26;
let y = 48;
const labels = state.levelLabels.length ? state.levelLabels : ["Glasses mic"];
labels.forEach((label, index) => {
const level = state.levels[index] ?? 0;
const enabled = state.levelsEnabled[index] ?? true;
image.drawText(font, 12, y + 3, enabled ? label : `${label} (off)`, enabled ? 190 : 90);
image.drawRect(barX, y, barWidth, 16, enabled ? 70 : 40);
const fill = Math.round((barWidth - 4) * level);
if (fill > 0) {
// A disabled mic still meters (dimly) so its signal can be checked
// before switching it back on.
image.fillRect(barX + 2, y + 2, fill, 12, !enabled ? 70 : level > 0.85 ? 255 : 190);
}
// Tick marks at -20 dB and -40 dB of the 60 dB meter range.
for (const fraction of [1 / 3, 2 / 3]) {
const tickX = barX + Math.round(barWidth * fraction);
image.drawLine(tickX, y + 16, tickX, y + 19, 60);
}
y += rowHeight;
});
y += 6;
if (state.doaDeviceDeg !== null) {
image.drawText(font, 12, y, `Direction of arrival: ${Math.round(state.doaDeviceDeg)}°`, 160);
y += 18;
}
image.drawText(font, 12, y, `Signal-strength ratio: ${state.ssr}`, 130);
y += 18;
const ancText = state.ancOn && state.ancEngine ? `on — ${state.ancEngine}` : "off";
image.drawText(font, 12, y, `Noise cancellation: ${ancText}`, 130);
image.drawText(font, 12, height - font.lineHeight - 4, `${GESTURE_DOUBLE_CLICK} back`, 110);
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
if (event.type === "double-click") {
ctx.stack.pop();
}
}
onRemoved(): void {
this.unsubscribe?.();
this.unsubscribe = null;
}
}
+195
View File
@@ -0,0 +1,195 @@
import { getStringSetting, setStringSetting } from "../../native/settings-store";
import { toJavaBytes } from "../../native/cloud-stt";
import { toUint8Array } from "../../util/array-util";
import {
MIC_LEASE_MS,
decodeMicStatus,
defaultMicConfig,
encodeMicControl,
templeActive,
type MicConfig,
type MicOp,
type MicSide,
type MicStatus,
} from "./mic-protocol";
declare const com: any;
/**
* Host-side controller for the CFW mic_control array: sends the same config
* to both temples (each temple is its own endpoint and its own 2-mic array),
* tracks per-temple field-104 status, and keeps the 90 s streaming lease
* renewed while capture is armed. Degrades cleanly on firmware without the
* feature: micControlSupported() is false and apply() refuses to arm.
*/
const CONFIG_SETTING_KEY = "microphones.array-config";
const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3);
export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null };
function activeCommunicator(): any {
if (!global.isAndroid) return null;
try {
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
} catch {
return null;
}
}
/** True when the connected glasses advertise the CFW mic_control feature. */
export function micControlSupported(): boolean {
const communicator = activeCommunicator();
if (!communicator) return false;
try {
const caps = String(communicator.getFirmwareCapabilities() ?? "");
return caps.trim().split(/\s+/).includes("micctl");
} catch {
return false;
}
}
export function loadMicConfig(): MicConfig {
const raw = getStringSetting(CONFIG_SETTING_KEY, "");
if (!raw) return defaultMicConfig();
try {
const parsed = JSON.parse(raw) as Partial<MicConfig>;
const defaults = defaultMicConfig();
return {
...defaults,
...parsed,
hostMics: { ...defaults.hostMics, ...(parsed.hostMics ?? {}) },
};
} catch {
return defaultMicConfig();
}
}
export function saveMicConfig(config: MicConfig): void {
setStringSetting(CONFIG_SETTING_KEY, JSON.stringify(config));
}
class MicArrayController {
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
private statuses: MicArrayStatuses = { left: null, right: null };
// The Java listener proxy must stay referenced or it gets GC'd.
private statusProxy: any | null = null;
private proxyCommunicator: any | null = null;
private leaseTimer: ReturnType<typeof setInterval> | null = null;
private armed = false;
getStatuses(): MicArrayStatuses {
return this.statuses;
}
onStatus(listener: (statuses: MicArrayStatuses) => void): () => void {
this.statusListeners.add(listener);
listener(this.statuses);
return () => this.statusListeners.delete(listener);
}
/**
* True when both temples report the same active configuration; only then
* should cross-temple bearing fusion be trusted.
*/
arrayConsistent(): boolean {
const { left, right } = this.statuses;
if (!left || !right) return false;
return (
left.active &&
right.active &&
left.effectiveRateHz === right.effectiveRateHz &&
left.codec === right.codec &&
left.format === right.format
);
}
/**
* Push the config to the temples. Each active temple streams its full
* front+rear pair (per-mic selection happens host-side after the split);
* a temple whose mics are all toggled off gets STOP instead so its stream
* doesn't spend BLE bandwidth.
*/
apply(config: MicConfig): boolean {
const leftActive = templeActive(config, "left");
const rightActive = templeActive(config, "right");
if (leftActive || rightActive) {
if (!this.send("configure", config, "configure", rightActive, leftActive)) return false;
}
if (!leftActive || !rightActive) {
this.send("stop", undefined, "stop-disabled-temple", !rightActive, !leftActive);
}
saveMicConfig(config);
this.armed = config.armHardware && (leftActive || rightActive);
this.syncLeaseTimer();
return true;
}
query(): boolean {
return this.send("query", undefined, "query");
}
stopStreaming(): void {
this.armed = false;
this.syncLeaseTimer();
this.send("stop", undefined, "stop");
}
private renewLease(): void {
if (!this.armed) return;
this.send("renew", undefined, "renew");
}
private send(
op: MicOp,
config: MicConfig | undefined,
label: string,
rightTemple = true,
leftTemple = true,
): boolean {
const communicator = activeCommunicator();
if (!communicator) return false;
this.ensureStatusProxy(communicator);
try {
const record = encodeMicControl(op, config);
communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple);
return true;
} catch (error) {
console.warn(`mic control ${label} failed: ${error}`);
return false;
}
}
private ensureStatusProxy(communicator: any): void {
if (this.statusProxy && this.proxyCommunicator === communicator) return;
if (this.statusProxy && this.proxyCommunicator) {
try {
this.proxyCommunicator.removeMicStatusListener(this.statusProxy);
} catch {
// The old communicator instance may be gone after a reconnect.
}
}
this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
onMicStatus: (body: any, arm: string) => {
const status = decodeMicStatus(toUint8Array(body));
if (!status) return;
const side: MicSide = String(arm) === "L" ? "left" : "right";
this.statuses = { ...this.statuses, [side]: { ...status, side } };
this.statusListeners.forEach((listener) => listener(this.statuses));
},
});
this.proxyCommunicator = communicator;
communicator.addMicStatusListener(this.statusProxy);
}
private syncLeaseTimer(): void {
if (this.armed && !this.leaseTimer) {
this.leaseTimer = setInterval(() => this.renewLease(), LEASE_RENEW_MS);
} else if (!this.armed && this.leaseTimer) {
clearInterval(this.leaseTimer);
this.leaseTimer = null;
}
}
}
export const micArrayController = new MicArrayController();
+206
View File
@@ -0,0 +1,206 @@
import { Utils } from "@nativescript/core";
declare const com: any;
declare const java: any;
/**
* Download management for the Microphones app's on-device models, mirroring
* the Moonshine flow in app/native/asr-model.ts (FaceclawModelDownloader,
* resume + pinned sha256 per file):
*
* - Speaker embedding (WeSpeaker CAM++ English, 512-dim): voice-prints for
* caption attribution, wearer recognition, and re-diarization.
* - Pyannote segmentation 3.0: offline re-diarization of saved recordings.
*
* The caption ASR model itself is the shared Moonshine download managed by
* asr-model.ts; this module only adds the speaker models.
*/
export type MicModelId = "speaker-embedding" | "diarization-segmentation";
type MicModelFile = {
/** Local file name under the model directory. */
name: string;
/** Source file name in the mirror repo (may differ from the local name). */
remoteName: string;
sha256: string;
sizeBytes: number;
};
type MicModel = {
id: MicModelId;
label: string;
baseUrl: string;
files: MicModelFile[];
totalBytes: number;
};
// Hugging Face mirrors of the sherpa-onnx release assets (served per-file,
// sha256 pinned from the repos' LFS pointers).
export const MIC_MODELS: MicModel[] = [
{
id: "speaker-embedding",
label: "Speaker voices (WeSpeaker English)",
baseUrl: "https://huggingface.co/csukuangfj/speaker-embedding-models/resolve/main/",
files: [
{
name: "speaker-embedding.onnx",
remoteName: "wespeaker_en_voxceleb_CAM%2B%2B.onnx",
sha256: "c46fad10b5f81e1aa4a60c162714208577093655076c5450f8c469e522ec54ef",
sizeBytes: 29292684,
},
],
totalBytes: 29292684,
},
{
id: "diarization-segmentation",
label: "Speaker turns (pyannote segmentation)",
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-pyannote-segmentation-3-0/resolve/main/",
files: [
{
name: "segmentation.onnx",
remoteName: "model.onnx",
sha256: "220ad67ca923bef2fa91f2390c786097bf305bceb5e261d4af67b38e938e1079",
sizeBytes: 5992913,
},
],
totalBytes: 5992913,
},
];
export type MicModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
const downloaders = new Map<MicModelId, any>();
const progress = new Map<MicModelId, number>();
const stateListeners = new Set<(id: MicModelId, state: MicModelState) => void>();
function model(id: MicModelId): MicModel {
const found = MIC_MODELS.find((entry) => entry.id === id);
if (!found) throw new Error(`unknown mic model ${id}`);
return found;
}
function modelDirPath(id: MicModelId): string {
const context = Utils.android.getApplicationContext();
return `${context.getFilesDir().getAbsolutePath()}/faceclaw-mic-models/${id}`;
}
function filePath(id: MicModelId, file: MicModelFile): string {
return `${modelDirPath(id)}/${file.name}`;
}
function isFilePresent(id: MicModelId, file: MicModelFile): boolean {
try {
const javaFile = new java.io.File(filePath(id, file));
return javaFile.exists() && javaFile.length() > 0;
} catch {
return false;
}
}
export function isMicModelReady(id: MicModelId): boolean {
if (!global.isAndroid) return false;
return model(id).files.every((file) => isFilePresent(id, file));
}
/** Absolute path of a ready model's primary file, or null when absent. */
export function micModelPath(id: MicModelId): string | null {
if (!isMicModelReady(id)) return null;
return filePath(id, model(id).files[0]!);
}
export function micModelState(id: MicModelId): MicModelState {
if (downloaders.has(id)) {
return {
status: "downloading",
bytesDownloaded: progress.get(id) ?? 0,
totalBytes: model(id).totalBytes,
};
}
return {
status: isMicModelReady(id) ? "ready" : "absent",
bytesDownloaded: 0,
totalBytes: model(id).totalBytes,
};
}
export function onMicModelStateChanged(
listener: (id: MicModelId, state: MicModelState) => void,
): () => void {
stateListeners.add(listener);
return () => stateListeners.delete(listener);
}
function notifyStateChanged(id: MicModelId): void {
const state = micModelState(id);
stateListeners.forEach((listener) => listener(id, state));
}
export function startMicModelDownload(id: MicModelId): void {
if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return;
downloadNextFile(id);
notifyStateChanged(id);
}
function downloadNextFile(id: MicModelId): void {
const definition = model(id);
const nextFile = definition.files.find((file) => !isFilePresent(id, file));
if (!nextFile) {
downloaders.delete(id);
notifyStateChanged(id);
return;
}
const completedBytes = definition.files
.filter((file) => isFilePresent(id, file))
.reduce((sum, file) => sum + file.sizeBytes, 0);
const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({
onProgress: (bytes: number, _total: number) => {
progress.set(id, completedBytes + Number(bytes));
notifyStateChanged(id);
},
onDone: () => {
downloadNextFile(id);
},
onError: (message: string) => {
console.error(`Mic model download failed (${id}/${nextFile.name}): ${message}`);
downloaders.delete(id);
notifyStateChanged(id);
},
});
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
`${definition.baseUrl}${nextFile.remoteName}`,
filePath(id, nextFile),
nextFile.sha256,
nextFile.sizeBytes,
listener,
);
downloaders.set(id, downloader);
downloader.start();
}
export function cancelMicModelDownload(id: MicModelId): void {
const downloader = downloaders.get(id);
if (!downloader) return;
downloader.cancel();
downloaders.delete(id);
notifyStateChanged(id);
}
export function deleteMicModel(id: MicModelId): void {
if (!global.isAndroid) return;
cancelMicModelDownload(id);
try {
for (const file of model(id).files) {
new java.io.File(filePath(id, file)).delete();
new java.io.File(`${filePath(id, file)}.part`).delete();
}
new java.io.File(modelDirPath(id)).delete();
} catch (error) {
console.error(`Mic model delete failed: ${String(error)}`);
}
notifyStateChanged(id);
}
+297
View File
@@ -0,0 +1,297 @@
/**
* Wire codec for the CFW mic_control feature (g2flash `microphone-configurations`
* branch, EVENCFW/16, caps tokens micctl/micmc/micraw). Mirrors the contract
* header in g2flash patches/mic_control.c:
*
* field 103 (write) ['M','C', ver, op, <9 config bytes for CONFIGURE>]
* field 104 (read) 21-byte ['M','C', ver, active, src, chanMask, codec,
* fmt, rateLo, rateHi, brLo, brHi, flags, hwArmed,
* sideId, frames u32 LE, effRateLo, effRateHi]
* 'SM' stream frame 21-byte header ['S','M', ver, flags, seq16, tick32,
* nCh, rateDiv16, fmt, codec, angleS16, ssrS16,
* payLen16] + payload
*
* Per-temple: each arm is its own endpoint and its own 2-mic array (front
* mic near the hinge = chanMask bit0, rear mic near the touchpad = bit1);
* the same config goes to both temples. Stereo payloads arrive as two
* CONCATENATED channel blocks (front then rear), not interleaved. Sessions
* hold a 90 s lease; renew with op RENEW while streaming.
*
* Pure module: byte codecs only, no platform imports (the BLE plumbing lives
* in mic-control.ts).
*/
export const MIC_PROTOCOL_VERSION = 1;
export const MIC_LEASE_MS = 90_000;
export type MicOp = "configure" | "query" | "stop" | "renew";
const OP_CODES: Record<MicOp, number> = { configure: 1, query: 2, stop: 3, renew: 4 };
export type MicSource = "codec" | "pdm";
export type MicWireCodec = "lc3" | "raw";
export type MicSampleFormat = "pcm16" | "pcm24" | "pcm32";
export type MicSide = "left" | "right";
export type MicPosition = "front" | "rear";
export const MIC_FLAG_BEAMFORM = 0x01;
/**
* Gated: actually brings up the mic front end and streaming on the glasses.
* The firmware validation checklist calls for confirming the tap ABI on
* sacrificial hardware first, so the UI defaults this OFF and warns.
*/
export const MIC_FLAG_ARM_HW = 0x02;
export const SM_FLAG_TRUNCATED = 0x80;
export type MicConfig = {
source: MicSource;
/** bit0 = front mic, bit1 = rear mic (per temple). */
channelMask: number;
codec: MicWireCodec;
format: MicSampleFormat;
sampleRateHz: number;
lc3BitrateBps: number;
beamform: boolean;
armHardware: boolean;
/**
* Host-side per-microphone selection. Each temple streams its full
* front+rear stereo pair over BLE (wire channelMask stays 0b11); the phone
* splits the concatenated channels and this mask decides which of the four
* mics feed levels-for-use, beamforming, captions, and recording. A temple
* with both mics off is sent STOP so its stream doesn't waste bandwidth.
*/
hostMics: Record<MicChannelKey, boolean>;
};
/** One of the four physical microphones, as split on the host. */
export type MicChannelKey = "leftFront" | "leftRear" | "rightFront" | "rightRear";
export const MIC_CHANNEL_KEYS: readonly MicChannelKey[] = [
"leftFront",
"leftRear",
"rightFront",
"rightRear",
];
export function micChannelKey(side: MicSide, position: MicPosition): MicChannelKey {
if (side === "left") return position === "front" ? "leftFront" : "leftRear";
return position === "front" ? "rightFront" : "rightRear";
}
export function micChannelLabel(key: MicChannelKey): string {
switch (key) {
case "leftFront":
return "L front";
case "leftRear":
return "L rear";
case "rightFront":
return "R front";
case "rightRear":
return "R rear";
}
}
/** Whether a temple has any host-enabled microphone. */
export function templeActive(config: MicConfig, side: MicSide): boolean {
return side === "left"
? config.hostMics.leftFront || config.hostMics.leftRear
: config.hostMics.rightFront || config.hostMics.rightRear;
}
export function defaultMicConfig(): MicConfig {
return {
source: "pdm",
channelMask: 0b11,
codec: "raw",
format: "pcm16",
sampleRateHz: 16_000,
lc3BitrateBps: 0,
beamform: true,
armHardware: false,
hostMics: { leftFront: true, leftRear: true, rightFront: true, rightRear: true },
};
}
export function micEnabled(config: MicConfig, position: MicPosition): boolean {
return (config.channelMask & (position === "front" ? 0b01 : 0b10)) !== 0;
}
export function setMicEnabled(config: MicConfig, position: MicPosition, on: boolean): MicConfig {
const bit = position === "front" ? 0b01 : 0b10;
const channelMask = on ? config.channelMask | bit : config.channelMask & ~bit;
return { ...config, channelMask };
}
const SOURCE_CODES: Record<MicSource, number> = { codec: 0, pdm: 1 };
const CODEC_CODES: Record<MicWireCodec, number> = { lc3: 0, raw: 1 };
const FORMAT_CODES: Record<MicSampleFormat, number> = { pcm16: 0, pcm24: 1, pcm32: 2 };
export function bytesPerSample(format: MicSampleFormat): number {
return format === "pcm32" ? 4 : format === "pcm24" ? 3 : 2;
}
/** Encode the field-103 record body: ['M','C', ver, op, <payload>]. */
export function encodeMicControl(op: MicOp, config?: MicConfig): Uint8Array {
const header = [0x4d, 0x43, MIC_PROTOCOL_VERSION, OP_CODES[op]];
if (op !== "configure" || !config) {
return Uint8Array.from(header);
}
const rate = Math.round(Math.max(8_000, Math.min(48_000, config.sampleRateHz)) / 100);
const bitrate = Math.round(Math.max(0, Math.min(500_000, config.lc3BitrateBps)) / 100);
const flags =
(config.beamform ? MIC_FLAG_BEAMFORM : 0) | (config.armHardware ? MIC_FLAG_ARM_HW : 0);
return Uint8Array.from([
...header,
SOURCE_CODES[config.source],
config.channelMask & 0b11,
CODEC_CODES[config.codec],
FORMAT_CODES[config.format],
rate & 0xff,
(rate >> 8) & 0xff,
bitrate & 0xff,
(bitrate >> 8) & 0xff,
flags,
]);
}
export type MicStatus = {
active: boolean;
source: MicSource;
channelMask: number;
codec: MicWireCodec;
format: MicSampleFormat;
/** Requested rate, echoed back; the firmware always captures at 16 kHz. */
sampleRateHz: number;
lc3BitrateBps: number;
beamform: boolean;
armHardware: boolean;
hardwareArmed: boolean;
side: MicSide;
framesEmitted: number;
/** The rate actually in effect (16 kHz on current firmware). */
effectiveRateHz: number;
};
function le16(data: Uint8Array, offset: number): number {
return data[offset]! | (data[offset + 1]! << 8);
}
function sle16(data: Uint8Array, offset: number): number {
const value = le16(data, offset);
return value >= 0x8000 ? value - 0x10000 : value;
}
function le32(data: Uint8Array, offset: number): number {
return (
(data[offset]! | (data[offset + 1]! << 8) | (data[offset + 2]! << 16)) +
data[offset + 3]! * 0x1000000
);
}
const SOURCE_NAMES: MicSource[] = ["codec", "pdm"];
const CODEC_NAMES: MicWireCodec[] = ["lc3", "raw"];
const FORMAT_NAMES: MicSampleFormat[] = ["pcm16", "pcm24", "pcm32"];
/** Decode a field-104 status record body, or null when it isn't one. */
export function decodeMicStatus(body: Uint8Array): MicStatus | null {
if (body.length < 21 || body[0] !== 0x4d || body[1] !== 0x43 || body[2] !== MIC_PROTOCOL_VERSION) {
return null;
}
const flags = body[12]!;
return {
active: body[3] !== 0,
source: SOURCE_NAMES[body[4]!] ?? "pdm",
channelMask: body[5]! & 0b11,
codec: CODEC_NAMES[body[6]!] ?? "lc3",
format: FORMAT_NAMES[body[7]!] ?? "pcm16",
sampleRateHz: le16(body, 8) * 100,
lc3BitrateBps: le16(body, 10) * 100,
beamform: (flags & MIC_FLAG_BEAMFORM) !== 0,
armHardware: (flags & MIC_FLAG_ARM_HW) !== 0,
hardwareArmed: body[13] !== 0,
side: body[14] === 2 ? "left" : "right",
framesEmitted: le32(body, 15),
effectiveRateHz: le16(body, 19) * 100,
};
}
export type MicStreamFrame = {
/** Wraps at 2^16 (~13.6 min at ~80 fps); unwrap on the receiving side. */
sequence: number;
/** This temple's 1 ms OS tick — coarse L/R alignment only. */
deviceTickMs: number;
channelCount: number;
sampleRateHz: number;
format: MicSampleFormat;
codec: MicWireCodec;
/** On-temple TDOA bearing in signed degrees (0 unless beamform + stereo). */
angleDegrees: number;
/** Signal-strength ratio paired with the angle (confidence proxy). */
ssr: number;
truncated: boolean;
payload: Uint8Array;
};
export const SM_HEADER_BYTES = 21;
/** Decode an 'SM' multi-channel stream frame, or null when it isn't one. */
export function decodeMicStreamFrame(data: Uint8Array): MicStreamFrame | null {
if (
data.length < SM_HEADER_BYTES ||
data[0] !== 0x53 ||
data[1] !== 0x4d ||
data[2] !== MIC_PROTOCOL_VERSION
) {
return null;
}
const payLen = le16(data, 19);
const payload = data.subarray(SM_HEADER_BYTES, SM_HEADER_BYTES + payLen);
return {
sequence: le16(data, 4),
deviceTickMs: le32(data, 6),
channelCount: data[10]!,
sampleRateHz: le16(data, 11) * 100,
format: FORMAT_NAMES[data[13]!] ?? "pcm16",
codec: CODEC_NAMES[data[14]!] ?? "raw",
angleDegrees: sle16(data, 15),
ssr: sle16(data, 17),
truncated: (data[3]! & SM_FLAG_TRUNCATED) !== 0,
payload,
};
}
/**
* Split a raw PCM16 SM payload into per-channel Float32Arrays in [-1, 1].
* Channels arrive as CONCATENATED blocks (front then rear), matching the
* firmware's stereo callback, which dispatches two back-to-back extraction
* results rather than interleaving.
*/
export function splitConcatenatedPcm16(frame: MicStreamFrame): Float32Array[] | null {
if (frame.codec !== "raw" || frame.format !== "pcm16" || frame.channelCount < 1) {
return null;
}
const channels = frame.channelCount;
const bytesPerChannel = Math.floor(frame.payload.length / channels) & ~1;
const samplesPerChannel = bytesPerChannel / 2;
if (samplesPerChannel <= 0) return null;
const out: Float32Array[] = [];
for (let channel = 0; channel < channels; channel++) {
const base = channel * bytesPerChannel;
const samples = new Float32Array(samplesPerChannel);
for (let index = 0; index < samplesPerChannel; index++) {
const lo = frame.payload[base + index * 2]!;
const hi = frame.payload[base + index * 2 + 1]!;
let value = lo | (hi << 8);
if (value >= 0x8000) value -= 0x10000;
samples[index] = value / 32768;
}
out.push(samples);
}
return out;
}
/** The stock (non-CFW) 205-byte LC3 packet, for stream sniffing. */
export const STOCK_PACKET_BYTES = 205;
export function isStockAudioPacket(data: Uint8Array): boolean {
return data.length === STOCK_PACKET_BYTES;
}
File diff suppressed because it is too large. Load diff
+130
View File
@@ -0,0 +1,130 @@
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
/**
* Microphones app settings. All stored through the shared settings store so
* the phone UI, glasses menus, and the session engine see the same values.
*/
export type RetentionPeriod = "none" | "1d" | "1w" | "1m" | "1q" | "1y" | "forever";
const RETENTION_VALUES: readonly RetentionPeriod[] = ["none", "1d", "1w", "1m", "1q", "1y", "forever"];
const RETENTION_LABELS: Record<RetentionPeriod, string> = {
none: "None",
"1d": "1 day",
"1w": "1 week",
"1m": "1 month",
"1q": "1 quarter",
"1y": "1 year",
forever: "Forever",
};
export function retentionLabel(period: RetentionPeriod): string {
return RETENTION_LABELS[period];
}
/**
* The deletion cutoff timestamp for a retention period as of now, or 0 when
* nothing should be deleted. "None" deletes everything already stored (and
* the session engine also skips persisting new data).
*/
export function retentionCutoffMs(period: RetentionPeriod, nowMs: number): number {
const day = 24 * 60 * 60 * 1000;
switch (period) {
case "none":
return nowMs;
case "1d":
return nowMs - day;
case "1w":
return nowMs - 7 * day;
case "1m":
return nowMs - 30 * day;
case "1q":
return nowMs - 91 * day;
case "1y":
return nowMs - 365 * day;
case "forever":
return 0;
}
}
export const captionsEnabledSetting = new ConfigSettingBoolean({
id: "microphones-captions",
label: "Captions",
description: "Live speech-to-text captions with speaker names, from the glasses microphones.",
storageKey: "microphones.captions-enabled",
defaultValue: false,
});
export const translateEnabledSetting = new ConfigSettingBoolean({
id: "microphones-translate",
label: "Translate",
description:
"Detect foreign languages in captions and show a translation to the phone's language beneath the original line.",
storageKey: "microphones.translate-enabled",
defaultValue: false,
});
export const saveCaptionsSetting = new ConfigSettingBoolean({
id: "microphones-save-captions",
label: "Save captions",
description: "Keep caption transcripts for later review in the phone app.",
storageKey: "microphones.save-captions",
defaultValue: true,
});
export const saveRecordingsSetting = new ConfigSettingBoolean({
id: "microphones-save-recordings",
label: "Save recordings",
description:
"Keep the audio of caption sessions (compressed to AAC) so transcripts can be replayed and re-analyzed.",
storageKey: "microphones.save-recordings",
defaultValue: false,
});
export const captionsRetentionSetting = new ConfigSettingEnum<RetentionPeriod>({
id: "microphones-captions-retention",
label: "Keep captions",
description: "How long saved captions are retained before automatic deletion.",
storageKey: "microphones.captions-retention",
defaultValue: "1m",
values: RETENTION_VALUES,
formatValue: retentionLabel,
});
export const recordingsRetentionSetting = new ConfigSettingEnum<RetentionPeriod>({
id: "microphones-recordings-retention",
label: "Keep recordings",
description: "How long saved recordings are retained before automatic deletion.",
storageKey: "microphones.recordings-retention",
defaultValue: "1w",
values: RETENTION_VALUES,
formatValue: retentionLabel,
});
export const beamFilterSetting = new ConfigSettingBoolean({
id: "microphones-beam-filter",
label: "Beam filter",
description:
"Only listen to sounds from the shaded direction on the Sonic Radar; audio from other directions is muted.",
storageKey: "microphones.beam-filter",
defaultValue: false,
});
export const ancEnabledSetting = new ConfigSettingBoolean({
id: "microphones-anc",
label: "Noise cancellation",
description:
"Cancel background noise in captured audio: spectral noise suppression on the phone, plus adaptive inter-mic cancellation when the 4-mic array is streaming.",
storageKey: "microphones.anc-enabled",
defaultValue: true,
});
export const wearerCommandsOnlySetting = new ConfigSettingBoolean({
id: "microphones-wearer-commands",
label: "My voice only",
description:
"Restrict voice command recognition to your own enrolled voice; other speakers are captioned but cannot trigger commands.",
storageKey: "microphones.wearer-commands-only",
defaultValue: false,
});
+512
View File
@@ -0,0 +1,512 @@
import { imageFromAsciiArt } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { appViewportSize } from "../../ui/shell/geometry";
import { LIST_ROW_TEXT_INSET } from "../../ui/metrics";
import {
MenuLayer,
TextPageLayer,
drawRightValueMenuItem,
drawSubmenuIndicator,
drawToggleMenuItem,
openModalMenu,
type MenuItem,
} from "../../ui/menu";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { shell } from "../../ui/shell/shell";
import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../native/asr-model";
import { CaptionsLayer } from "./captions-layer";
import { LevelsLayer } from "./levels-layer";
import { RadarLayer } from "./radar-layer";
import { loadMicConfig, micArrayController, micControlSupported, saveMicConfig } from "./mic-control";
import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models";
import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol";
import { micSession } from "./mic-session";
import { speakerRegistry, type SpeakerProfile } from "./speakers";
import { formatRelativeTime } from "../../util/date-util";
import {
ancEnabledSetting,
beamFilterSetting,
captionsEnabledSetting,
captionsRetentionSetting,
recordingsRetentionSetting,
saveCaptionsSetting,
saveRecordingsSetting,
translateEnabledSetting,
wearerCommandsOnlySetting,
} from "./mic-settings";
export const MICROPHONES_WINDOW_ID = "microphones";
export const MICROPHONES_SURFACE_ID = "window:microphones";
const TRAY_ICON_ID = "microphones";
// 14x15 dual-mic glyph for the top-bar tray.
const TRAY_ICON = imageFromAsciiArt(
[
" ## ## ",
" # # # # ",
" # # # # ",
" # # # # ",
" # # # # ",
"# ## # # ## #",
" # # # # ",
" ## ## ",
" # # ",
" ### ### ",
],
220,
);
const MENU_LAYOUT = {
x: 8,
y: 8,
width: 292,
showBorder: false,
minHeight: 0,
maxHeight: appViewportSize("min").height - 16,
opaque: true,
};
function submenuItem(label: string, onSelect: MenuItem["onSelect"]): MenuItem {
return {
label,
onSelect,
render: ({ image, x, y, width, height, selected, disabled, text }) => {
const font = getDefaultSmallFont();
const value = disabled ? 70 : selected ? 255 : 200;
image.drawText(font, x, y + LIST_ROW_TEXT_INSET, text, value);
drawSubmenuIndicator(image, font, x, y, width, height, value);
},
};
}
/** Persist a config change and re-apply it live when the array is streaming. */
function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void {
const updated = mutate(loadMicConfig());
saveMicConfig(updated);
if (micSession.getState().mode === "extended") {
micArrayController.apply(updated);
}
}
function micChannelToggleItem(
key: MicChannelKey,
description: string,
disabled: () => boolean,
): MenuItem {
return configToggleItem(
`${micChannelLabel(key)} mic`,
() => loadMicConfig().hostMics[key],
() => micSession.setMicChannelEnabled(key, !loadMicConfig().hostMics[key]),
disabled,
description,
);
}
function configToggleItem(
label: string,
isOn: () => boolean,
toggle: () => void,
disabled?: () => boolean,
description?: string,
): MenuItem {
return {
label,
description,
disabled,
onSelect: () => toggle(),
render: ({ image, x, y, width, selected }) => {
drawToggleMenuItem(image, getDefaultSmallFont(), x, y, width, label, isOn(), selected);
},
};
}
function configPickerItem<T extends string | number>(
label: string,
values: readonly T[],
format: (value: T) => string,
current: () => T,
apply: (value: T) => void,
disabled?: () => boolean,
description?: string,
): MenuItem {
return {
label,
description,
disabled,
onSelect: (ctx) => {
const items = values.map(
(value): MenuItem => ({
label: format(value) + (current() === value ? " *" : ""),
onSelect: (innerCtx) => {
apply(value);
innerCtx.stack.pop();
},
}),
);
openModalMenu(ctx, label, items, Math.max(0, values.indexOf(current())));
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, format(current()));
},
};
}
function micSetupMenu(): MenuLayer {
const unsupported = () => !micControlSupported();
return new MenuLayer(
"Microphone setup",
[
{
label: "Array status",
description:
"Per-temple mic-control status. Requires the mic-control custom firmware (caps token micctl).",
onSelect: (ctx) => {
micArrayController.query();
const { left, right } = micArrayController.getStatuses();
const describe = (side: string, status: typeof left) =>
status
? `${side}: ${status.active ? "active" : "idle"}, mics ${status.channelMask & 1 ? "front" : ""}${
status.channelMask & 2 ? "+rear" : ""
}, ${status.effectiveRateHz / 1000} kHz, ${status.hardwareArmed ? "armed" : "not armed"}, ${
status.framesEmitted
} frames`
: `${side}: no status yet`;
const consistency = micArrayController.arrayConsistent()
? "Array consistent (L/R matched)."
: "Array not yet consistent.";
ctx.stack.push(
new TextPageLayer(
"Array status",
micControlSupported()
? `${describe("Left", left)}\n${describe("Right", right)}\n${consistency}`
: "The connected firmware does not advertise mic-control. Flash the microphone-configurations CFW build to control the four mics individually; on stock firmware the glasses stream one mixed channel with a firmware-computed direction of arrival.",
),
);
},
},
// Each temple streams its full front+rear pair; these four toggles are
// host-side selection over the split channels, so any combination of
// the four microphones can be used independently.
micChannelToggleItem("leftFront", "Enable the left temple's front microphone (near the hinge).", unsupported),
micChannelToggleItem("leftRear", "Enable the left temple's rear microphone (by the touchpad).", unsupported),
micChannelToggleItem("rightFront", "Enable the right temple's front microphone (near the hinge).", unsupported),
micChannelToggleItem("rightRear", "Enable the right temple's rear microphone (by the touchpad).", unsupported),
configPickerItem(
"Source",
["pdm", "codec"] as const,
(value) => (value === "pdm" ? "PDM mics" : "Codec DMIC"),
() => loadMicConfig().source,
(value) => updateMicConfig((config) => ({ ...config, source: value })),
unsupported,
"PDM streams one channel; the codec front end streams both mics of a temple.",
),
configPickerItem(
"Codec",
["raw", "lc3"] as const,
(value) => (value === "raw" ? "Raw PCM" : "LC3"),
() => loadMicConfig().codec,
(value) => updateMicConfig((config) => ({ ...config, codec: value })),
unsupported,
"Raw PCM preserves inter-mic phase for beamforming. (Current firmware always streams raw.)",
),
configPickerItem(
"Sample rate",
[8000, 16000, 24000, 32000, 48000] as const,
(value) => `${value / 1000} kHz`,
() => loadMicConfig().sampleRateHz,
(value) => updateMicConfig((config) => ({ ...config, sampleRateHz: value })),
unsupported,
"Requested capture rate per mic pair. Current firmware pins the effective rate at 16 kHz and echoes the request in the status.",
),
configPickerItem(
"LC3 bitrate",
[0, 16000, 32000, 48000, 64000] as const,
(value) => (value === 0 ? "Default" : `${value / 1000} kbps`),
() => loadMicConfig().lc3BitrateBps,
(value) => updateMicConfig((config) => ({ ...config, lc3BitrateBps: value })),
unsupported,
"Requested LC3 bitrate per mic pair (takes effect once the firmware's LC3 path lands).",
),
configToggleItem(
"Direction detection",
() => loadMicConfig().beamform,
() => updateMicConfig((config) => ({ ...config, beamform: !config.beamform })),
unsupported,
"Have each temple compute and stream its own sound direction (TDOA angle + signal strength).",
),
{
label: "4-mic capture (advanced)",
description:
"Arms the microphone hardware and multi-channel streaming on the glasses. Leave off until the firmware validation checklist has passed on this hardware.",
disabled: unsupported,
onSelect: (ctx) => {
const config = loadMicConfig();
if (config.armHardware) {
updateMicConfig((current) => ({ ...current, armHardware: false }));
micArrayController.stopStreaming();
return;
}
openModalMenu(ctx, "Arm mic hardware?", [
{
label: "Cancel",
onSelect: (innerCtx) => innerCtx.stack.pop(),
},
{
label: "Enable (validated hardware only)",
onSelect: (innerCtx) => {
updateMicConfig((current) => ({ ...current, armHardware: true }));
innerCtx.stack.pop();
},
},
]);
},
render: ({ image, x, y, width, selected }) => {
drawToggleMenuItem(
image,
getDefaultSmallFont(),
x,
y,
width,
"4-mic capture (advanced)",
loadMicConfig().armHardware,
selected,
);
},
},
],
MENU_LAYOUT,
);
}
function modelItem(
label: string,
ready: () => boolean,
stateText: () => string,
download: () => void,
): MenuItem {
return {
label,
onSelect: () => {
if (!ready()) download();
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, label, stateText());
},
};
}
function voiceSpeakersMenu(): MenuLayer {
return new MenuLayer(
"Voice & speakers",
[
{
label: "Enroll my voice",
description:
"Speak a sentence after selecting; the next utterance becomes (or refreshes) your voice-print, used to label you in captions and to restrict commands to your voice.",
disabled: () => !micSession.getState().captionsActive,
onSelect: () => micSession.armWearerEnrollment(),
},
toggleSettingMenuItem(wearerCommandsOnlySetting),
modelItem(
"Speaker voice model",
() => isMicModelReady("speaker-embedding"),
() => {
const state = micModelState("speaker-embedding");
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (28 MB)";
},
() => startMicModelDownload("speaker-embedding"),
),
modelItem(
"Caption ASR model",
isAsrModelReady,
() => {
const state = asrModelState();
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (141 MB)";
},
startAsrModelDownload,
),
modelItem(
"Re-diarization model",
() => isMicModelReady("diarization-segmentation"),
() => {
const state = micModelState("diarization-segmentation");
if (state.status === "ready") return "ready";
if (state.status === "downloading") {
return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`;
}
return "download (6 MB)";
},
() => startMicModelDownload("diarization-segmentation"),
),
],
MENU_LAYOUT,
);
}
/** The full contact card behind a People row (and behind the encounter popup). */
function speakerDetailBody(profile: SpeakerProfile): string {
const parts: string[] = [
`Last heard ${profile.lastHeardAt ? `${formatRelativeTime(profile.lastHeardAt)} ago` : "never"}.`,
];
if (profile.tag) parts.push(`Note: ${profile.tag}`);
if (profile.lastRecap) parts.push(`Last conversation: ${profile.lastRecap}`);
if (profile.actionItems.length) {
parts.push(`To do:\n${profile.actionItems.map((item) => `- ${item}`).join("\n")}`);
}
if (profile.facts.length) {
parts.push(`Facts:\n${profile.facts.map((fact) => `- ${fact}`).join("\n")}`);
}
if (parts.length === 1) {
parts.push(
"No conversation insights yet. A recap, action items, and remembered facts are " +
"generated when a captioned conversation with this person ends (requires the " +
"on-phone assistant model).",
);
}
return parts.join("\n\n");
}
/**
* People: the speaker-recognition contacts, most recently heard first — the
* review surface behind the encounter popup. Selecting a person shows when
* they were last heard, their note, the recap and action items of the last
* conversation, and the facts remembered about them.
*/
function peopleMenu(): MenuLayer {
const profiles = speakerRegistry.all().filter((profile) => !profile.isWearer);
const items: MenuItem[] = profiles.length
? profiles.map((profile) => ({
label: profile.name,
onSelect: (ctx) => {
ctx.stack.push(new TextPageLayer(profile.name, speakerDetailBody(profile)));
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(
image,
getDefaultSmallFont(),
x,
y,
width,
profile.name,
profile.lastHeardAt ? `${formatRelativeTime(profile.lastHeardAt)} ago` : "",
);
},
}))
: [
{
label: "No people heard yet",
disabled: true,
onSelect: () => {},
},
];
return new MenuLayer("People", items, MENU_LAYOUT);
}
function storageMenu(): MenuLayer {
return new MenuLayer(
"Storage",
[
toggleSettingMenuItem(saveCaptionsSetting),
toggleSettingMenuItem(saveRecordingsSetting),
enumSettingMenuItem(captionsRetentionSetting),
enumSettingMenuItem(recordingsRetentionSetting),
{
label: "Review on phone",
description:
"Saved conversations, transcripts, speakers, and recordings are reviewed in the Faceclaw phone app: Conversations in the top-right menu.",
onSelect: (ctx) => {
ctx.stack.push(
new TextPageLayer(
"Review on phone",
"Open the Faceclaw app on your phone and choose Conversations from the top-right menu to search captions, filter by speaker, play linked recordings, rename/merge speakers, re-diarize sessions, and see the tone/sentiment graph of each conversation.",
),
);
},
},
],
MENU_LAYOUT,
);
}
export function createMicrophonesAppWindow(options: InProcessAppOptions): InProcessWindow {
const menu = new MenuLayer(
"Microphones",
[
submenuItem("Sonic Radar", (ctx) => {
const layer = new RadarLayer();
ctx.stack.push(layer);
layer.start(ctx.actions.requestRender);
}),
submenuItem("Levels", (ctx) => {
const layer = new LevelsLayer();
ctx.stack.push(layer);
layer.start(ctx.actions.requestRender);
}),
submenuItem("Captions view", (ctx) => {
const layer = new CaptionsLayer();
ctx.stack.push(layer);
layer.start(ctx.actions.requestRender);
}),
submenuItem("People", (ctx) => {
speakerRegistry.reload();
ctx.stack.push(peopleMenu());
}),
configToggleItem(
"Captions",
() => captionsEnabledSetting.get(),
() => micSession.setCaptionsEnabled(!captionsEnabledSetting.get()),
undefined,
captionsEnabledSetting.description,
),
toggleSettingMenuItem(translateEnabledSetting),
toggleSettingMenuItem(beamFilterSetting),
toggleSettingMenuItem(ancEnabledSetting),
submenuItem("Microphone setup", (ctx) => {
ctx.stack.push(micSetupMenu());
}),
submenuItem("Voice & speakers", (ctx) => {
ctx.stack.push(voiceSpeakersMenu());
}),
submenuItem("Storage", (ctx) => {
ctx.stack.push(storageMenu());
}),
],
MENU_LAYOUT,
);
const app = createInProcessWindow({
appId: "microphones",
windowId: MICROPHONES_WINDOW_ID,
title: "Microphones",
iconLetter: "Mi",
icon: "mic",
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(menu),
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
micSession.stop();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
},
});
micSession.start();
shell.setTrayIcon(TRAY_ICON_ID, TRAY_ICON);
return app;
}
+688
View File
@@ -0,0 +1,688 @@
/**
* Known-first-names dataset: the 5,000 most common US first
* names per sex from the Social Security Administration baby-names data
* (births 1935-2024 summed), plus the British/Irish supplement the US list
* under-represents (Niamh, Callum, Isobel...). Deduplicated and normalized
* to 9,260 entries: lowercase, diacritics folded, punctuation dropped - the
* same key normalization lookups apply, so "Siobhán" and "Siobhan" reach
* the same entry.
*
* This is a POSITIVE signal only: membership means a word is almost
* certainly usable as a given name; absence means nothing (real names the
* SSA list misses must still pass). Callers use it to skip more expensive
* or more error-prone checks, never to reject a candidate.
*/
const RAW =
"aaden,aadhya,aadi,aadya,aaiden,aakash,aaliyah,aamir,aanya,aarav,aaren,aariv,aariz,aarna,aaron,aarron," +
"aarush,aarya,aaryan,aayan,aayden,aayush,abagail,abbas,abbey,abbi,abbie,abbigail,abbigale,abbott,abby," +
"abbygail,abdallah,abdiel,abdirahman,abdoulaye,abdul,abdulaziz,abdullah,abdullahi,abdulrahman,abdurrahman," +
"abe,abel,abelardo,abhinav,abhiram,abiel,abigael,abigail,abigale,abigayle,abimael,abisai,able,abner," +
"abraham,abrahan,abram,abran,abriana,abrianna,abriella,abrielle,abril,abubakar,abygail,acacia,ace," +
"achilles,acie,ada,adah,adair,adalberto,adalee,adaleigh,adalia,adalie,adalina,adaline,adalyn,adalynn," +
"adam,adamari,adamaris,adams,adan,adara,adarius,addalyn,addalynn,addelyn,addie,addilyn,addilynn,addison," +
"addisyn,addyson,adel,adela,adelaida,adelaide,adele,adelia,adelina,adeline,adelita,adell,adella,adelle," +
"adelyn,adelynn,adem,aden,adena,adham,adia,adian,adiel,adil,adilene,adilyn,adilynn,adin,adina,adira," +
"adison,aditi,aditya,adler,adley,adnan,adolfo,adolph,adolphus,adon,adonai,adonis,adora,adrain,adria," +
"adrian,adriana,adriane,adrianna,adrianne,adriano,adriel,adrien,adriene,adrienne,adrina,adryan,advaith," +
"advik,adyn,adyson,aedan,aeden,aerial,aeris,afton,agapito,agastya,agatha,agnes,agustin,ahaan,aharon," +
"ahmad,ahmari,ahmed,ahmir,ahren,ahron,ahuva,aida,aidan,aide,aiden,aidyn,aila,ailani,ailany,aileen," +
"ailyn,aime,aimee,aine,ainhoa,ainsley,aisha,aisling,aislinn,aislyn,aislynn,aitana,aiyana,aiyanna,aiyden," +
"aiza,aizen,aj,aja,ajani,ajay,akai,akash,akeem,akhil,akil,akilah,akira,akiva,akram,aksel,akshara,akshay," +
"al,alaia,alain,alaina,alaiya,alaiyah,alan,alana,alanah,alane,alani,alanis,alanna,alannah,alara,alaric," +
"alasdair,alasia,alastair,alaura,alaya,alayah,alayla,alayna,alaysha,alaysia,alba,albert,alberta,albertha," +
"alberto,albin,albino,aldair,alden,alder,aldo,aldon,alea,aleah,alec,alecia,aleczander,aleen,aleena," +
"aleesha,aleia,aleida,aleigha,aleisha,alejandra,alejandro,alek,aleksandar,aleksander,aleksandr,aleksandra," +
"alekzander,alen,alena,alene,alesha,aleshia,alesia,alessa,alessandra,alessandro,alessia,alessio,aleta," +
"aletha,alethea,alethia,alex,alexa,alexander,alexandra,alexandre,alexandrea,alexandria,alexandro,alexandros," +
"alexavier,alexei,alexi,alexia,alexie,alexis,alexsander,alexsandra,alexus,alexys,alexzander,alexzandria," +
"aleya,aleyah,aleyda,aleyna,alfie,alfonso,alfonza,alfonzo,alford,alfred,alfreda,alfredo,ali,alia,aliah," +
"aliana,alianna,alias,alice,alicia,alida,alijah,alina,aline,alisa,alise,alisha,alishia,alisia,alison," +
"alissa,alisson,alistair,alister,alita,alivia,alix,aliya,aliyah,aliyana,aliza,alizabeth,alize,allan," +
"allana,allegra,allen,allena,allene,alli,allie,allison,allissa,allisson,allister,alliyah,ally,allyn," +
"allysa,allyson,allyssa,alma,alon,alona,alondra,alonna,alonso,alonza,alonzo,alora,aloysius,alpha,alphonse," +
"alphonso,alphonzo,alston,alta,althea,alton,alva,alvaro,alvie,alvin,alvina,alvino,alvis,alya,alyana," +
"alyanna,alyce,alycia,alyna,alysa,alyse,alysha,alysia,alyson,alyssa,alyssia,alyvia,alyza,amaan,amada," +
"amadeo,amadeus,amado,amador,amadou,amaia,amaiya,amal,amalia,aman,amanda,amando,amani,amar,amara,amarah," +
"amare,amaree,amari,amaria,amariah,amarie,amarion,amaris,amaru,amauri,amaury,amaya,amayah,ambar,amber," +
"amberlee,amberly,amberlyn,ambria,ambrose,ambrosia,amee,ameen,ameena,ameer,ameera,ameerah,amelia,amelie," +
"amen,amer,amere,america,americo,amerie,ames,amethyst,ami,amia,amiah,amias,amie,amiee,amiel,amilcar," +
"amilia,amin,amina,aminah,aminata,amir,amira,amiracle,amirah,amiri,amit,amity,amiya,amiyah,ammar,ammie," +
"ammon,amna,amon,amor,amora,amos,amoura,amparo,amr,amy,amya,amyah,amyra,an,ana,anabel,anabella,anabelle," +
"anahi,anai,anaiah,anais,anaiya,anakin,analee,analeigh,anali,analia,analicia,analiese,analisa,analise," +
"analy,anamaria,anand,ananda,ananya,anas,anastacia,anastacio,anastasia,anastasios,anay,anaya,anayah," +
"anayeli,ander,anders,andersen,anderson,andi,andie,andon,andra,andrae,andre,andrea,andreana,andreas," +
"andree,andrei,andres,andrew,andrey,andrez,andria,andriana,andru,andy,aneesh,anel,anela,anessa,anette," +
"anfernee,angel,angela,angelena,angelene,angeles,angelia,angelic,angelica,angelie,angelika,angelina," +
"angeline,angelique,angelita,angella,angelle,angelo,angely,angelyn,angie,angus,ani,ania,aniah,anibal," +
"anika,anil,anirudh,anisa,anish,anisha,anissa,aniston,anita,anitra,aniya,aniyah,anja,anjali,anjanette," +
"anjel,anjelica,ann,anna,annabel,annabell,annabella,annabelle,annabeth,annah,annalee,annaleigh,annaliese," +
"annalisa,annalise,annalyse,annamae,annamaria,annamarie,annastasia,anne,anneliese,annelise,annemarie," +
"annetta,annette,annie,annika,anniston,annmarie,ansel,anselmo,ansh,ansley,anson,anthoney,anthoni,anthony," +
"antione,antionette,antoine,antoinette,anton,antone,antonella,antonette,antoni,antonia,antonina,antonino," +
"antonio,antony,antron,antuan,antwain,antwan,antwaun,antwoine,antwon,antwone,anuj,anushka,anwar,anya," +
"anyah,anyla,anylah,anyssa,aoife,apolinar,apollo,apolonio,april,apryl,ara,arabella,arabelle,araceli," +
"aracely,aram,aramis,aran,aranza,arash,arath,arav,araya,arcadio,arcelia,archer,archibald,archie,ardell," +
"arden,ardis,ardith,areli,arely,aren,ares,aretha,argenis,arham,ari,aria,ariadna,ariadne,ariah,arian," +
"ariana,ariane,arianna,arianne,arianny,aric,arica,arick,arie,ariel,ariela,ariella,arielle,aries,arik," +
"arika,arin,arion,arionna,aris,arisbeth,arissa,aristotle,arius,ariya,ariyah,ariyana,ariyanna,arizona," +
"arjun,arla,arlan,arland,arleen,arlen,arlena,arlene,arlet,arleth,arlette,arley,arlie,arlin,arline," +
"arlis,arlo,arlon,arlyn,armaan,arman,armand,armando,armani,armen,armida,armin,armon,armond,armondo," +
"armoni,arnaldo,arnav,arne,arnell,arnetta,arnie,arnita,arno,arnold,arnoldo,arnulfo,aron,arron,arrow," +
"arsenio,art,artem,artemio,artemis,arther,arthur,artie,artis,arturo,arun,arvid,arvil,arvin,arwen,arya," +
"aryah,aryan,aryana,aryanna,aryeh,aryn,asa,asad,asael,asaiah,asante,ash,asha,ashanti,ashby,ashely," +
"asher,ashia,ashlea,ashlee,ashlei,ashleigh,ashley,ashli,ashlie,ashlin,ashly,ashlyn,ashlynn,ashlynne," +
"ashraf,ashten,ashtin,ashton,ashtyn,ashwin,asia,asiah,asim,asiya,asiyah,aslan,asma,aspen,aspyn,aston," +
"astrid,asya,athan,athanasios,atharv,atharva,athena,atlas,atlee,atom,atreus,atreyu,atticus,aubree," +
"aubrey,aubri,aubriana,aubrianna,aubrie,aubriella,aubrielle,aubry,auden,audie,audra,audrea,audree," +
"audrey,audriana,audrianna,audrie,audrina,audry,august,augusta,augustin,augustine,augusto,augustus," +
"aundre,aundrea,aura,aurelia,aurelio,aurelius,aurora,austen,austin,auston,austyn,auther,author,authur," +
"autry,autum,autumn,ava,avah,avalon,avalyn,avalynn,avan,avani,avarie,avary,avaya,avayah,aven,averi," +
"averie,avery,avi,avia,avian,aviana,avianna,aviel,avigail,avin,avinash,avion,avis,aviva,avon,avraham," +
"avram,avril,avrohom,avyaan,avyan,awilda,axel,axell,axl,axle,axton,aya,ayaan,ayah,ayan,ayana,ayanna," +
"ayansh,aydan,ayden,aydin,ayesha,ayla,aylah,aylani,ayleen,aylin,ayman,ayoub,ayra,aysha,aysia,ayub," +
"ayush,ayva,ayvah,ayven,azaan,azael,azai,azaiah,azalea,azalia,azari,azaria,azariah,aziah,aziel,aziyah," +
"aziz,aziza,azlan,azrael,azriel,azucena,azul,azure,babette,baby,babyboy,babygirl,baila,bailee,baileigh," +
"bailey,bailie,bakari,baker,baldemar,baleigh,baltazar,bambi,bane,banks,barak,barb,barbara,barbie,barbra," +
"bari,barney,baron,barret,barrett,barrie,barrington,barron,barry,bart,bartholomew,bartley,barton,baruch," +
"bashir,basil,basilio,bastian,batsheva,baxter,baylee,bayleigh,baylen,bayley,baylie,baylor,bayron,bear," +
"beatrice,beatrix,beatriz,beau,beaux,becca,beck,beckam,beckett,beckham,becki,beckie,becky,belen,belinda," +
"bella,bellamy,belle,belva,ben,benaiah,benard,benedict,benicio,benigno,benita,benito,benjaman,benjamen," +
"benjamin,benjamine,benji,benjie,benjiman,bennet,bennett,bennie,benny,benson,bentlee,bentley,bently," +
"benton,benuel,benzion,berenice,berkeley,berkley,berlin,bernabe,bernadette,bernadine,bernard,bernardino," +
"bernardo,bernell,bernetta,bernice,bernie,berniece,bernita,berry,bert,berta,bertha,bertie,berton,bertram," +
"bertrand,beryl,bess,bessie,beth,bethanie,bethann,bethany,bethel,betsey,betsy,bette,bettie,bettina," +
"betty,bettye,beulah,beverlee,beverley,beverly,bexley,beyonce,bianca,bianka,bijan,bilal,bill,billi," +
"billie,billy,binyamin,binyomin,birdie,bishop,bjorn,blade,bladimir,blain,blaine,blair,blaire,blaise," +
"blaize,blake,blakely,blanca,blanche,blane,blas,blayke,blayne,blayze,blaze,blessing,bliss,blossom," +
"blythe,bo,boaz,bob,bobbi,bobbie,bobby,bobbye,bode,boden,bodhi,bodie,boe,bonifacio,bonita,bonnie,bonny," +
"booker,boone,boris,boruch,boston,bowen,bowie,boyce,boyd,bracha,brad,braden,bradford,bradlee,bradley," +
"bradly,bradon,brady,bradyn,braedan,braeden,braedon,braedyn,braelyn,braelynn,braiden,brain,bram,branch," +
"brandan,brandee,branden,brandi,brandie,brandin,brando,brandon,brandt,brandy,brandyn,brannon,branson," +
"brant,brantlee,brantley,branton,braulio,braven,braxten,braxton,braxtyn,brayan,braydan,brayden,braydin," +
"braydon,braylan,braylee,brayleigh,braylen,braylin,braylon,brayson,brayton,brea,breana,breann,breanna," +
"breanne,breck,brecken,breckin,bree,breeana,breeanna,brenda,brendan,brenden,brendon,brendyn,brenna," +
"brennan,brennen,brenner,brennon,brent,brentley,brently,brenton,breon,breona,breonna,bret,brett,bretton," +
"brevin,brewer,brexton,breyden,bria,brian,briana,brianda,briann,brianna,brianne,briant,briar,brice," +
"bricen,bridger,bridget,bridgett,bridgette,brie,brieanna,briella,brielle,brien,brienna,brigette,briggs," +
"brigham,brighton,brigid,brigitte,briley,brinda,brinlee,brinley,brion,briona,brionna,brisa,briseida," +
"briseyda,brison,brissa,bristol,britany,britnee,britney,britni,britny,britt,britta,brittaney,brittani," +
"brittanie,brittany,brittnay,brittnee,brittney,brittni,brittnie,brittny,britton,brixton,broc,brock," +
"broden,broderick,brodey,brodie,brodrick,brody,brogan,bronson,bronwen,bronwyn,bronx,brook,brooke,brookelyn," +
"brookelynn,brooklin,brooklyn,brooklynn,brooklynne,brooks,bruce,bruno,bryan,bryana,bryanna,bryant," +
"bryar,bryce,brycen,bryden,bryer,brylan,brylee,bryleigh,brylie,bryn,brynlee,brynleigh,brynley,brynn," +
"brynna,brynne,brynnlee,bryon,brysen,bryson,bryston,bryton,buck,buckley,bucky,bud,buddy,buffy,buford," +
"burke,burl,burley,burnell,burt,burton,buster,butch,byran,byron,cache,cade,caden,cadence,cady,cadyn," +
"caeden,caedmon,cael,caelan,caelyn,caesar,cage,cai,caiden,cailee,cailey,cailin,cailyn,cain,cainan," +
"caine,cairo,caison,caitlin,caitlyn,caitlynn,caius,cal,calan,calandra,calder,cale,caleb,caleigh,calen," +
"caley,cali,calin,calista,caliyah,calla,callahan,callan,calleigh,callen,calli,callie,calliope,callista," +
"callum,cally,calum,calvert,calvin,cam,camari,cambree,cambria,cambrie,camden,camdon,camdyn,camelia," +
"camellia,cameran,cameren,cameron,cameryn,cami,camie,camila,camilla,camille,camilo,cammie,campbell," +
"camren,camrin,camron,camryn,canaan,candace,candance,candelaria,candelario,candi,candice,candida,candido," +
"candie,candis,candra,candy,candyce,cannon,canon,canyon,capri,caprice,cara,cardell,caren,carey,cari," +
"caridad,carie,carin,carina,carisa,carissa,carl,carla,carlee,carleen,carleigh,carlene,carleton,carletta," +
"carley,carli,carlie,carlin,carlisle,carlito,carlo,carlos,carlotta,carlton,carly,carlyle,carlyn,carma," +
"carman,carmel,carmela,carmelita,carmella,carmello,carmelo,carmen,carmine,carnell,carol,carolann,carole," +
"carolee,carolina,caroline,carolyn,carolyne,carolynn,caron,carri,carrie,carrington,carrol,carroll," +
"carsen,carson,carsten,carsyn,carter,cartier,carver,cary,caryl,caryn,carys,casandra,case,casen,casey," +
"cash,cashton,casie,casimir,cason,casper,caspian,cass,cassandra,cassaundra,cassey,cassi,cassia,cassian," +
"cassidy,cassie,cassius,cassondra,cassy,castiel,cataleya,catalina,catarina,catarino,cate,catelyn,caterina," +
"cathal,catharine,catherine,cathey,cathi,cathie,cathleen,cathrine,cathryn,cathy,catina,catrina,catriona," +
"cattleya,cavan,cavin,cayde,cayden,caydence,cayla,cayleb,caylee,cayleigh,caylie,caylin,cayman,caysen," +
"cayson,ceasar,cecelia,cecil,cecile,cecilia,cecilio,cecily,cedar,cedric,cedrick,celena,celene,celeste," +
"celestina,celestine,celestino,celia,celina,celine,celso,cesar,chace,chad,chadd,chadrick,chadwick," +
"chaim,chaise,champ,chan,chana,chance,chancellor,chancey,chanda,chandler,chandra,chanel,chanell,chanelle," +
"channing,chanse,chantal,chante,chantel,chantell,chantelle,chantz,charis,charise,charisma,charissa," +
"charisse,charity,charla,charlee,charleen,charleigh,charlene,charles,charleston,charlette,charley," +
"charli,charlie,charline,charlize,charlotte,charlton,charly,charlyn,charmaine,charolette,chas,chase," +
"chasen,chasidy,chasity,chason,chassidy,chassity,chastity,chauncey,chauncy,chava,chaya,chayce,chayse," +
"chayton,chaz,chazz,che,chelsea,chelsee,chelsey,chelsi,chelsie,chelsy,chenoa,cher,cheree,cherelle," +
"cheri,cherie,cherilyn,cherise,cherish,cherlyn,cherokee,cherri,cherrie,cherry,cherryl,cheryl,cheryle," +
"cheryll,chesley,chesney,chester,chet,chevelle,chevy,cheyann,cheyanne,cheyenne,cheyne,chiara,chico," +
"china,chip,chiquita,chirstopher,chistopher,chloe,chloee,chole,chosen,chozen,chris,chrissy,christ," +
"christa,christain,christal,christan,christel,christen,christena,christene,christi,christian,christiana," +
"christiane,christianna,christie,christien,christin,christina,christine,christion,christofer,christoher," +
"christon,christop,christoper,christoph,christophe,christopher,christos,christpher,christy,chrystal," +
"chuck,chyanne,chyna,chynna,cian,ciana,cianna,ciara,ciaran,ciarra,cicely,cielo,cienna,ciera,cierra," +
"cillian,cinda,cindi,cindy,cinnamon,cinthia,cinthya,cipriano,cirilo,ciro,citlali,citlalli,citlaly," +
"cj,clair,claira,claire,clancy,clara,clarance,clare,clarence,claribel,clarice,clarisa,clarissa,clark," +
"clarke,claud,claude,claudette,claudia,claudie,claudine,claudio,clay,clayton,clem,clement,clemente," +
"clementine,cleo,cleon,cleopatra,cleophus,cleta,cletus,cleve,cleveland,cliff,clifford,cliffton,clifton," +
"clint,clinton,clive,cloe,cloey,clover,clovis,cloyd,clyde,coby,coco,coda,codey,codi,codie,cody,coen," +
"cohen,colbie,colby,colden,cole,coleen,coleman,coleson,coleton,colette,coley,colin,colleen,collette," +
"collier,collin,collins,colsen,colson,colston,colt,colten,colter,coltin,colton,coltyn,columbus,conan," +
"concepcion,concetta,conlan,conley,connell,conner,connie,connor,conor,conrad,conrado,constance,constantine," +
"constantino,consuela,consuelo,contessa,conway,cooper,copeland,cora,coral,coralee,coralie,coraline," +
"corban,corben,corbett,corbin,corby,corbyn,cord,cordarius,cordelia,cordell,cordero,coreen,coretta," +
"corey,cori,corie,corin,corina,corine,corinna,corinne,corissa,corliss,cormac,cornel,cornelia,cornelio," +
"cornelious,cornelius,cornell,correy,corrie,corrin,corrina,corrine,corry,cort,cortez,cortland,cortney," +
"corwin,cory,cosette,cosme,cosmo,coty,courtland,courtnee,courtney,courtnie,cove,coy,craig,crawford," +
"cree,creed,creighton,crew,cris,criselda,crispin,crissy,crista,cristal,cristen,cristhian,cristi,cristian," +
"cristiano,cristie,cristin,cristina,cristine,cristo,cristobal,cristofer,cristopher,cristy,croix,crosby," +
"cross,cru,crue,cruz,crysta,crystal,cuauhtemoc,cullen,curley,curran,curry,curt,curtis,curtiss,cutler," +
"cutter,cy,cydney,cylas,cyle,cyndi,cynthia,cypress,cyril,cyrus,dacia,dade,daelyn,daemon,daevon,dafne," +
"dafydd,dagan,dagoberto,dahlia,daija,daijah,dailyn,dain,daina,daisey,daisha,daisy,daivd,daja,dajah," +
"dajon,dajuan,dakarai,dakari,dakoda,dakota,dakotah,dalary,dale,dalen,daleyza,dalia,dalila,dalilah," +
"dallas,dallen,dallin,dalton,dalvin,dalyn,damani,damar,damarcus,damari,damario,damarion,damaris,damarius," +
"damein,damen,dameon,damian,damien,damion,damir,damon,damond,damone,damoni,damonte,dan,dana,danae," +
"dandre,dane,daneil,danelle,danette,dangelo,dani,dania,danial,danica,daniel,daniela,daniele,daniell," +
"daniella,danielle,danika,danilo,danisha,danita,daniyah,dann,danna,dannette,danni,dannie,danniel,dannielle," +
"danny,dante,danthony,dany,danya,danyel,danyell,danyelle,daphne,daquan,dara,darby,darcey,darci,darcie," +
"darcy,darek,darel,darell,daren,daria,darian,dariana,darick,dariel,darien,darin,dario,darion,darious," +
"daris,darius,darl,darla,darleen,darlene,darline,darlyn,darnel,darnell,darold,daron,darrel,darrell," +
"darren,darrian,darrick,darrien,darrin,darrion,darris,darrius,darron,darry,darryl,darryle,darryll," +
"darryn,darsh,dartagnan,darvin,darwin,daryl,daryle,daryll,daryn,dasean,dash,dasha,dashaun,dashawn," +
"dashiell,dashon,dasia,dathan,daulton,daunte,davante,dave,daven,daveon,davey,davi,davian,david,davida," +
"davidson,davie,davien,davin,davina,davion,davis,davon,davonta,davontae,davonte,davy,dawayne,dawn," +
"dawna,dawne,dawson,dawud,dax,daxon,daxton,dayana,dayanara,dayanna,dayla,daylan,dayle,daylen,daylin," +
"daylon,daymon,dayna,dayne,daysha,dayshawn,dayson,dayton,dayvon,dea,deacon,deagan,dean,deana,deandra," +
"deandre,deandrea,deane,deangelo,deann,deanna,deanne,deante,deanthony,deasia,deaundre,debbi,debbie," +
"debbra,debby,debi,debora,deborah,debra,debrah,decarlos,decker,declan,dede,dedra,dedric,dedrick,dee," +
"deeann,deeanna,deedee,deedra,deegan,deen,deena,deidra,deidre,deion,deirdre,deisy,deja,dejah,dejon," +
"dejuan,deklan,del,delaine,delainey,delana,delaney,delani,delanie,delano,delbert,delena,delfina,delfino," +
"delia,delicia,delila,delilah,delina,delinda,delisa,dell,della,delma,delmar,delmas,delmer,delois,deloise," +
"delon,delonte,delora,delores,deloris,delphine,delroy,delta,delton,delvin,delvon,delwin,demar,demarco," +
"demarcus,demari,demario,demarion,demarius,demarkus,demetra,demetri,demetria,demetric,demetrice,demetrio," +
"demetrios,demetrious,demetris,demetrius,demi,demian,demir,demitri,demitrius,demond,demonte,dempsey," +
"dena,denae,denali,deneen,denice,denim,denis,denise,denisha,denisse,denita,deniz,denna,dennie,dennis," +
"dennise,denny,denton,denver,denzel,denzell,denzil,deon,deondre,deonna,deonta,deontae,deontay,deonte," +
"dequan,derald,dereck,derek,dereon,derian,deric,derick,derik,derion,deron,derreck,derrek,derrel,derrell," +
"derric,derrick,derrik,derrion,derron,derry,derwin,deryl,desarae,desean,deshaun,deshawn,deshon,deshun," +
"desi,desirae,desiray,desire,desirea,desiree,desmon,desmond,dessie,destanie,destany,desteny,destin," +
"destinee,destiney,destini,destinie,destiny,destry,detra,detrick,dev,devan,devansh,devante,devaughn," +
"deven,devin,devion,devlin,devon,devona,devonna,devonta,devontae,devonte,devora,devorah,devyn,dewayne," +
"dewey,dewitt,dexter,deyanira,deysi,dezirae,dezmond,dhruv,dia,diamond,dian,diana,diandra,diane,diann," +
"dianna,dianne,diante,dick,dickie,didier,diedra,diedre,diego,diesel,dieter,dietrich,dijon,dilan,dillan," +
"dillard,dillion,dillon,dimas,dimitri,dimitrios,dimitrius,dina,dinah,dino,dion,diondre,dione,dionicio," +
"dionisio,dionna,dionne,dionte,dior,dirk,divine,divya,dixie,dixon,diya,djuan,dmitri,dock,dolan,dollie," +
"dolly,dolores,domanic,domenic,domenica,domenick,domenico,dominga,domingo,dominic,dominica,dominick," +
"dominik,dominique,dominque,domonic,domonick,domonique,don,dona,donal,donald,donato,donavan,donaven," +
"donavin,donavon,dondre,donell,donetta,donielle,donita,donivan,donn,donna,donnell,donnie,donny,donovan," +
"donta,dontae,dontavious,dontavius,dontay,donte,dontez,dontrell,donya,donzell,dora,doran,dorcas,doreen," +
"dorene,doretha,doretta,dori,dorian,dorie,dorien,dorinda,dorine,doris,dorman,dorothea,dorothy,dorris," +
"dorsey,dortha,dorthy,dottie,doug,douglas,douglass,dov,dovid,dovie,doyle,draco,drake,draven,drayden," +
"drayton,dream,dreama,drema,dresden,drew,drexel,dru,drucilla,drue,drusilla,duane,dudley,dujuan,duke," +
"dulce,duncan,duran,durand,durell,durrell,durward,durwood,dustan,dusten,dusti,dustin,duston,dusty," +
"dustyn,dutch,dutton,duwayne,dwain,dwaine,dwan,dwane,dwayne,dwight,dwyane,dyan,dyana,dyanna,dylan," +
"dyland,dylen,dyllan,dyllon,dylon,dymond,dynasty,dyson,eamon,eamonn,ean,earl,earle,earlene,earlie," +
"earline,early,earnest,earnestine,eartha,earvin,eason,easter,easton,eastyn,eathan,eben,ebenezer,eber," +
"ebonee,eboni,ebonie,ebony,echo,ed,eda,edan,edd,eddie,eddy,eden,eder,edgar,edgard,edgardo,edie,edin," +
"edison,edith,edmond,edmund,edmundo,edna,edric,edrick,edsel,edson,eduard,eduardo,edvin,edward,edwardo," +
"edwin,edwina,edythe,effie,efraim,efrain,efrem,efren,egan,egypt,ehren,eian,eiden,eileen,eilidh,eisley," +
"eitan,eithan,ela,eladio,elaina,elaine,elam,elan,elana,elani,elayna,elayne,elba,elbert,elda,elden," +
"elder,eldon,eldred,eldridge,eleana,eleanor,eleanora,eleanore,eleazar,elena,eleni,elexis,elgin,eli," +
"elia,eliah,eliam,elian,eliana,elianna,elias,eliazar,elicia,elida,elie,eliel,eliezer,eligio,elija," +
"elijah,elijiah,elimelech,elin,elina,elinor,elio,eliot,eliott,elisa,elisabeth,elise,eliseo,elisha," +
"elisheva,elisia,elissa,eliud,eliyahu,eliza,elizabet,elizabeth,elizebeth,ella,elle,ellen,ellena,ellery," +
"elli,ellia,elliana,ellianna,ellie,elliette,elliot,elliott,ellis,ellison,ellsworth,elly,ellyn,elma," +
"elmer,elmo,elmore,elnora,elodie,eloisa,eloise,elon,elora,elouise,elowen,elowyn,eloy,elroy,elsa,elsie," +
"elson,elspeth,elsy,elton,elva,elvia,elvin,elvira,elvis,elwin,elwood,elwyn,ely,elyana,elyas,elyjah," +
"elyse,elysia,elyssa,elzie,ema,emalee,emalyn,eman,emani,emanuel,emari,ember,emberlee,emberly,emberlyn," +
"emberlynn,emelia,emelie,emeline,emely,emelyn,emerald,emeri,emerie,emerson,emersyn,emery,emi,emil," +
"emile,emilee,emiley,emili,emilia,emiliana,emiliano,emilie,emilio,emily,emir,emma,emmalee,emmaleigh," +
"emmaline,emmalyn,emmalynn,emmanuel,emmarie,emme,emmeline,emmerson,emmet,emmett,emmi,emmie,emmit,emmitt," +
"emmy,emoni,emory,empress,emre,emrie,emry,emrys,ena,ender,enedina,enid,enmanuel,ennis,enoc,enoch,enos," +
"enrico,enrique,ensley,enzo,eoin,eowyn,ephraim,epifanio,era,erasmo,eren,eriberto,eric,erica,erich," +
"erick,ericka,erickson,erie,erik,erika,erin,erinn,eris,erlinda,erma,ermias,erna,ernest,ernestina,ernestine," +
"ernesto,ernie,ernst,eron,eros,errick,errol,erskine,ervin,erwin,eryk,eryn,erynn,esai,esau,esdras,esequiel," +
"esha,eshaan,eshan,esme,esmeralda,esperanza,essence,essie,esteban,estefani,estefania,estefany,estela," +
"estella,estelle,estephanie,ester,estevan,esther,eston,estrella,eternity,ethan,ethaniel,ethel,ethen," +
"ethyn,etienne,etta,eugene,eugenia,eugenie,eugenio,eula,eulalia,eulalio,eunice,eusebio,eva,evalina," +
"evalyn,evalynn,evan,evander,evangelina,evangeline,evangelos,evans,evaristo,eve,evelin,evelina,eveline," +
"evelyn,evelyne,evelynn,ever,everardo,everest,everett,everette,everlee,everleigh,everley,everly,evert," +
"evette,evie,evin,evon,evonne,evren,ewan,exavier,eyad,eyan,ezekial,ezekiel,ezell,ezequiel,ezio,ezra," +
"ezrah,ezriel,fabian,fabiana,fabio,fabiola,fabricio,fabrizio,fahad,faigy,faisal,faith,fallon,falon," +
"fannie,fanny,fantasia,farah,fares,farhan,farid,faris,faron,farrah,farrell,farris,fatima,fatimah,fatoumata," +
"faustino,fausto,favian,faviola,fawn,fay,faye,fayth,federico,felecia,felica,felice,felicia,feliciano," +
"felicity,felipe,felisa,felisha,felix,felton,fenix,fenton,ferdinand,fergus,fermin,fern,fernanda,fernando," +
"ferrell,ferris,ffion,fidel,fidencio,filiberto,filip,filomena,finlay,finley,finn,finneas,finnegan," +
"finnian,finnick,finnigan,finnley,fiona,fionn,fischer,fisher,fitzgerald,flavio,fletcher,flint,flor," +
"flora,florence,florencio,florentino,florian,florine,flossie,floyd,flynn,fonda,ford,forest,forrest," +
"foster,fox,foy,fran,francene,frances,francesca,francesco,franchesca,francheska,francie,francine,francis," +
"francisca,francisco,franco,francois,frank,frankie,franklin,franklyn,franky,fransisco,frantz,franz," +
"frazier,fred,freda,freddie,freddrick,freddy,frederic,frederica,frederick,frederico,fredia,fredric," +
"fredrick,fredy,freeman,freida,freya,freyja,frida,frieda,fritz,fulton,furman,fynn,gabe,gabino,gabrial," +
"gabriel,gabriela,gabriele,gabriella,gabrielle,gadiel,gael,gaetano,gage,gaia,gaige,gail,gaines,gala," +
"gale,galen,galilea,gamaliel,gannon,gardner,garen,garet,gareth,garett,garey,garfield,garin,garland," +
"garner,garnet,garnett,garold,garon,garren,garret,garrett,garrick,garrison,garry,garth,garvin,gary," +
"gaspar,gaston,gatlin,gauge,gavan,gaven,gavin,gavino,gavriel,gavyn,gay,gaye,gayla,gayle,gaylen,gaylene," +
"gaylon,gaylord,gaynell,gearld,gearldine,geary,gema,gemma,gena,genaro,gene,general,genesis,geneva," +
"genevieve,genie,genna,gennaro,geno,genoveva,gentry,geoff,geoffery,geoffrey,george,georgeann,georgene," +
"georgetta,georgette,georgia,georgiana,georgiann,georgianna,georgie,georgina,georgine,georgios,geovani," +
"geovanni,geovanny,geovany,gerald,geraldine,geraldo,geralyn,gerard,gerardo,geremy,gerhard,geri,germaine," +
"german,gerold,gerome,geronimo,gerri,gerrit,gerry,gerson,gertie,gertrude,gethin,gia,giacomo,giada," +
"gian,giana,giancarlo,gianfranco,gianluca,gianna,gianni,giannis,giavanna,gibran,gibson,gideon,gigi," +
"gil,gilbert,gilberto,gilda,gildardo,giles,gill,gillian,gina,ginger,ginny,gino,gio,gionni,giorgio," +
"giovani,giovanna,giovanni,giovanny,giovany,giovonni,girard,gisel,gisela,gisele,gisell,giselle,gissel," +
"gisselle,gitty,giulia,giuliana,giuliano,giuseppe,gizelle,gladis,gladys,glen,glenda,glendon,glenn," +
"glenna,glenwood,glinda,gloria,glory,glynda,glynis,glynn,godfrey,golda,golden,goldie,gonzalo,gorden," +
"gordon,gorge,grace,gracelyn,gracelynn,gracen,gracey,gracie,graciela,gracyn,grady,graeme,graham,graig," +
"granger,grant,granville,gray,grayce,grayden,graydon,graysen,grayson,grecia,greer,greg,gregg,greggory," +
"gregor,gregorio,gregory,greta,gretchen,gretta,grey,greysen,greyson,griffen,griffin,griffith,grisel," +
"griselda,grover,gruffudd,gryffin,guadalupe,guido,guillermina,guillermo,guinevere,gunnar,gunner,gunther," +
"gus,gussie,gustav,gustave,gustavo,guy,gwen,gwenda,gwendolyn,gwenyth,gwyn,gwyneth,habib,hadassah,haddie," +
"haden,hadi,hadlee,hadleigh,hadley,hafsa,hagan,hagen,haiden,haider,haidyn,hailee,haileigh,hailey,hailie," +
"haily,haisley,hakeem,hakim,hal,hala,halee,haleigh,halen,haley,hali,halie,halima,halle,halley,hallie," +
"halo,hamid,hamilton,hamish,hampton,hamza,hamzah,hana,hanah,hanan,hank,hanna,hannah,hans,hansel,hanson," +
"hardy,haris,harlan,harland,harlee,harleigh,harlem,harlen,harley,harlie,harlin,harlon,harlow,harlyn," +
"harmon,harmoni,harmonie,harmony,harold,haroon,harper,harrell,harriet,harriett,harriette,harris,harrison," +
"harry,hart,hartley,harun,harvey,hasan,hasani,hashim,haskell,hassan,hattie,haven,hawa,hawk,haydee," +
"hayden,haydn,haydon,hayes,haylee,hayleigh,hayley,hayli,haylie,hayward,haywood,haze,hazel,hazen,haziel," +
"heath,heather,heaven,heavenly,heber,hector,hedy,heide,heidi,heidy,helen,helena,helene,hellen,henderson," +
"hendrick,hendrik,hendrix,henley,henri,henrietta,henrik,henry,hensley,herb,herbert,herbie,hercules," +
"heriberto,herlinda,herman,hermelinda,herminia,herminio,hermione,hermon,hernan,herschel,hershel,hershell," +
"hershy,hester,heston,heyward,hezekiah,hiba,hilario,hilary,hilda,hillard,hillary,hillel,hilton,hindy," +
"hipolito,hiram,hobart,hobert,hogan,holden,holland,holley,holli,hollie,hollis,holly,hollyn,holt,holton," +
"homer,homero,honesty,honey,honor,hope,horace,horacio,horatio,hortencia,hosanna,hosea,houston,howard," +
"howell,hoyt,hubert,huck,huda,hudson,huey,hugh,hugo,humberto,humza,hung,hunter,huntley,hurley,hussain," +
"hussein,huston,huxley,huxton,huy,hyrum,iain,ian,ibn,ibraheem,ibrahim,ibrahima,ida,idalia,idella,idris," +
"iesha,ieshia,ignacio,ignatius,ikaika,ike,iker,ila,ilan,ilana,ileana,ilene,iliana,ilias,illiana,ilona," +
"ilse,ilya,ilyas,ima,iman,imani,imanol,imari,imelda,immanuel,imogen,imogene,imran,ina,inaaya,inara," +
"inaya,india,indiana,indie,indigo,indira,indy,indya,ines,inez,infant,inga,ingrid,ioannis,iola,iona," +
"ione,iqra,ira,iran,ireland,irelynn,irene,irie,irina,iris,irma,irvin,irving,irwin,isa,isaac,isaak," +
"isabel,isabela,isabell,isabella,isabelle,isac,isacc,isadora,isadore,isael,isai,isaiah,isaias,isaih," +
"isak,isamar,isela,isha,ishaan,ishan,ishmael,isiah,isidoro,isidro,isis,isla,ismael,ismail,isobel,isra," +
"israel,isreal,issa,issabella,issac,issiah,italia,italy,itzayana,itzel,iva,ivaan,ivan,ivana,ivanna," +
"ivar,ivelisse,iver,iverson,ivette,ivey,ivie,ivonne,ivory,ivy,iwan,iyana,iyanna,iyla,iyonna,izaac," +
"izaak,izabel,izabela,izabella,izabelle,izael,izaiah,izak,izan,izaya,izayah,iziah,izrael,izzabella," +
"jabari,jabez,jabriel,jabril,jacalyn,jacari,jace,jacee,jacelyn,jacen,jaceon,jacey,jaci,jacie,jaciel," +
"jacinda,jacinta,jacinto,jack,jackeline,jackelyn,jacki,jackie,jacklyn,jacklynn,jackson,jacky,jaclyn," +
"jaclynn,jacob,jacobe,jacobi,jacobo,jacoby,jacolby,jacorey,jacori,jacory,jacqualine,jacque,jacquelin," +
"jacqueline,jacquelyn,jacquelyne,jacquelynn,jacques,jacquez,jacquline,jacqulyn,jacy,jad,jada,jadah," +
"jadan,jadarius,jade,jadelyn,jaden,jadiel,jadin,jadon,jadyn,jae,jaeda,jaeden,jaedon,jaedyn,jael,jaela," +
"jaelen,jaelin,jaelyn,jaelynn,jafet,jagger,jahari,jahaziel,jaheem,jaheim,jahiem,jahir,jahleel,jahlil," +
"jahmal,jahmari,jahmir,jahsiah,jahzara,jahziel,jai,jaida,jaidan,jaide,jaiden,jaidon,jaidyn,jaila,jailen," +
"jailene,jailyn,jailynn,jaime,jaimee,jaimie,jaina,jair,jaire,jairo,jairus,jaison,jajuan,jakai,jakari," +
"jakayla,jake,jakeb,jakeem,jakhari,jakhi,jakob,jakobe,jakobi,jakoby,jakub,jala,jalaya,jalayah,jaleah," +
"jaleel,jaleesa,jalen,jalia,jalil,jalin,jalisa,jalissa,jaliyah,jalon,jalyn,jalynn,jalyssa,jamaal,jamaine," +
"jamal,jamall,jamar,jamarcus,jamari,jamarian,jamario,jamarion,jamarius,jamarr,jamauri,jamaya,jame," +
"jamee,jameel,jameer,jameka,jamel,jamell,jamere,james,jamesha,jameson,jamey,jami,jamia,jamiah,jamichael," +
"jamie,jamier,jamieson,jamil,jamila,jamilah,jamin,jamir,jamison,jamiya,jamiyah,jammie,jamon,jamya," +
"jan,jana,janae,janai,janay,janaya,jancarlos,jane,janea,janee,janeen,janel,janell,janella,janelle," +
"janelly,janely,janene,janessa,janet,janeth,janett,janetta,janette,janey,jani,jania,janiah,janice," +
"janie,janiece,janina,janine,janis,janise,janita,janiya,janiyah,jann,janna,jannah,jannet,jannette," +
"jannie,jansen,janson,january,janya,janyce,janyla,japheth,jaquan,jaquelin,jaqueline,jaquelyn,jaquez," +
"jarad,jared,jarek,jarel,jarell,jaren,jaret,jareth,jarett,jarid,jariel,jarin,jarius,jarod,jarom,jaron," +
"jarrad,jarred,jarrel,jarrell,jarren,jarret,jarrett,jarrid,jarrod,jarron,jarvis,jasai,jase,jasean," +
"jasen,jashaun,jashawn,jashon,jashua,jasiah,jasiel,jasir,jasleen,jaslene,jaslyn,jaslynn,jasmin,jasmine," +
"jasmyn,jasmyne,jason,jasper,jasson,jathan,jaunita,javan,javari,javaris,javaughn,javen,javeon,javian," +
"javien,javier,javin,javion,javon,javontae,javonte,jawad,jawan,jax,jaxen,jaxin,jaxon,jaxsen,jaxson," +
"jaxsyn,jaxton,jaxtyn,jaxx,jaxxon,jaxyn,jay,jaya,jayce,jaycee,jaycen,jayceon,jayci,jaycie,jaycob,jayda," +
"jaydan,jayde,jayden,jaydin,jaydon,jaydyn,jaye,jaykob,jayla,jaylah,jaylan,jaylani,jaylee,jayleen,jaylen," +
"jaylene,jaylin,jaylon,jaylyn,jaylynn,jayme,jaymee,jaymes,jaymie,jayna,jayne,jayon,jayquan,jayse,jaysen," +
"jayshawn,jayson,jaythan,jayton,jayven,jayveon,jayvian,jayvin,jayvion,jayvon,jazelle,jaziah,jaziel," +
"jazlyn,jazlynn,jazmin,jazmine,jazmyn,jazmyne,jazzlyn,jazzlynn,jazzmin,jazzmine,jc,jd,jean,jeana,jeancarlo," +
"jeancarlos,jeane,jeanelle,jeanene,jeanetta,jeanette,jeanie,jeanine,jeanluc,jeanmarie,jeanna,jeanne," +
"jeannette,jeannie,jeannine,jeanpaul,jeanpierre,jeb,jed,jedediah,jedidiah,jeff,jefferey,jefferson," +
"jeffery,jeffory,jeffrey,jeffrie,jeffry,jehu,jelani,jelena,jemma,jena,jenae,jencarlos,jenee,jenell," +
"jenelle,jenesis,jenessa,jenette,jenevieve,jeni,jenica,jenice,jenifer,jeniffer,jenine,jenise,jenna," +
"jennah,jennette,jenni,jennica,jennie,jennifer,jenniffer,jennings,jenny,jennyfer,jens,jensen,jenson," +
"jerad,jerald,jeraldine,jeramiah,jeramie,jeramy,jerard,jerardo,jere,jered,jerel,jerell,jereme,jeremey," +
"jeremi,jeremiah,jeremias,jeremie,jeremih,jeremy,jeremyah,jeri,jeriah,jerica,jericho,jeriel,jerilyn," +
"jerimiah,jermain,jermaine,jermel,jermey,jermiah,jerod,jerold,jerome,jeromy,jeron,jerone,jeronimo," +
"jerrad,jerred,jerrel,jerrell,jerrett,jerri,jerrica,jerrick,jerrid,jerrie,jerrod,jerrold,jerron,jerry," +
"jersey,jerson,jesenia,jeshua,jesiah,jesica,jess,jessa,jessalyn,jesse,jessee,jessenia,jessi,jessiah," +
"jessica,jessie,jessika,jesslyn,jessy,jestin,jesus,jet,jethro,jett,jetta,jettson,jevin,jevon,jewel," +
"jewell,jhene,jhett,jhon,jhonatan,jhonathan,jhonny,jia,jianna,jibreel,jibril,jihad,jill,jillian,jim," +
"jimena,jimi,jimmie,jimmy,jin,jina,jionni,jiovanni,jiraiya,jireh,jiselle,jiya,jo,joab,joachim,joah," +
"joan,joana,joanie,joann,joanna,joanne,joao,joaquin,job,jobe,joby,jocelin,jocelyn,jocelyne,jocelynn," +
"jock,jodee,jodi,jodie,jody,joe,joel,joell,joella,joelle,joellen,joely,joeseph,joesph,joetta,joette," +
"joey,johan,johana,johann,johanna,johannah,johannes,johathan,john,johna,johnathan,johnathen,johnathon," +
"johndavid,johnell,johnetta,johnette,johnie,johnmichael,johnna,johnnie,johnny,johnpaul,johnson,johny," +
"joi,joie,jolee,joleen,jolene,jolie,joline,jolyn,jolynn,jomar,jon,jona,jonah,jonas,jonatan,jonathan," +
"jonathen,jonathon,jonell,jonelle,jones,joni,jonmichael,jonna,jonnathan,jonnie,jonny,jonpaul,jonte," +
"jonthan,jordan,jordana,jorden,jordi,jordin,jordon,jordy,jordyn,jordynn,jorel,jorge,jori,jorja,jorje," +
"jory,josalyn,joscelyn,jose,joseangel,josef,josefa,josefina,joselin,joseline,joseluis,joselyn,josemanuel," +
"joseph,josephina,josephine,josette,josey,josh,joshua,joshuah,joshue,josiah,josias,josie,joslyn,joslynn," +
"jospeh,josselyn,josslyn,josue,jourdan,journee,journei,journey,journi,journie,jovan,jovana,jovani," +
"jovanna,jovanni,jovanny,jovany,jovi,jovie,jovita,jovon,joy,joya,joyce,joycelyn,joye,jozef,joziah," +
"jr,jream,jrue,juan,juana,juancarlos,juanita,juanito,juanjose,juanpablo,juaquin,jubilee,judah,judd," +
"jude,judge,judi,judie,judith,judson,judy,juelz,julee,julene,jules,juli,julia,julian,juliana,juliane," +
"juliann,julianna,julianne,juliano,julie,julieann,julieanne,julien,julienne,juliet,julieta,julietta," +
"juliette,julio,julious,julisa,julissa,julius,jullian,jun,june,junior,juniper,junius,jurnee,justen," +
"justice,justin,justina,justine,justino,justis,justo,juston,justus,justyce,justyn,juvenal,juventino," +
"juwan,kabir,kace,kacee,kacen,kacey,kaci,kacie,kacper,kacy,kadarius,kade,kadeem,kaden,kadence,kadie," +
"kadin,kady,kadyn,kaeden,kaedyn,kael,kaela,kaelan,kaeli,kaelin,kaelyn,kaelynn,kaeson,kage,kahlani," +
"kahlil,kai,kaia,kaidan,kaiden,kaidence,kaidyn,kaila,kailah,kailani,kailee,kaileigh,kailen,kailey," +
"kaili,kaily,kailyn,kailynn,kaimana,kain,kaine,kainen,kainoa,kaira,kairi,kairo,kaisen,kaiser,kaisley," +
"kaison,kaitlin,kaitlyn,kaitlynn,kaito,kaius,kaiya,kaizen,kaizer,kal,kala,kalan,kalani,kale,kalea," +
"kaleah,kaleb,kalee,kaleena,kalei,kaleigh,kalel,kalem,kalen,kalena,kaleo,kaley,kali,kalia,kaliah,kalib," +
"kalie,kalil,kalin,kalina,kalista,kaliyah,kallen,kalli,kallie,kalman,kalob,kalon,kalvin,kalyn,kalynn," +
"kam,kamal,kamar,kamara,kamari,kamaria,kamarion,kamau,kamaya,kambree,kambria,kamden,kamdyn,kameron," +
"kameryn,kami,kamia,kamiah,kamil,kamila,kamilah,kamilla,kamille,kamiya,kamiyah,kamora,kamran,kamren," +
"kamrin,kamron,kamryn,kamya,kanaan,kanan,kandace,kandi,kandice,kandis,kandy,kane,kanesha,kanisha,kaniya," +
"kaniyah,kannon,kanon,kanye,kanyon,kara,karah,karalyn,karam,karan,kareem,karely,karen,karena,karey," +
"kari,karie,karim,karime,karin,karina,karis,karisa,karisma,karissa,karl,karla,karlee,karleen,karleigh," +
"karlene,karley,karli,karlie,karlo,karlos,karlton,karly,karlyn,karma,karmelo,karmen,karmyn,karol,karolina," +
"karoline,karolyn,karon,karren,karri,karrie,karsen,karson,karsten,karsyn,karter,karthik,kartier,kary," +
"karyl,karyme,karyn,kasandra,kase,kasen,kasey,kash,kashawn,kashif,kashmir,kashton,kasi,kasia,kasie," +
"kason,kasper,kassandra,kassi,kassidy,kassie,kassius,kasyn,kataleya,katalina,katana,katarina,kate," +
"katelin,katelyn,katelynn,kateri,katerina,katey,kathaleen,katharina,katharine,katherin,katherine,kathern," +
"katheryn,katheryne,kathey,kathi,kathie,kathleen,kathlene,kathlyn,kathrine,kathryn,kathryne,kathy," +
"kathyrn,kati,katia,katie,katilyn,katina,katlin,katlyn,katlynn,katrice,katrina,kattie,katy,katya,kaven," +
"kavin,kavion,kavon,kavya,kawika,kay,kaya,kayce,kaycee,kaycen,kayci,kaycie,kaydance,kaydee,kayden," +
"kaydence,kaydin,kaydon,kaye,kayla,kaylah,kaylan,kaylani,kayle,kaylea,kayleb,kaylee,kayleen,kaylei," +
"kayleigh,kaylen,kaylene,kayley,kayli,kaylie,kaylin,kaylon,kaylyn,kaylynn,kayne,kaysen,kayson,kaytlin," +
"kaytlyn,kayvon,kc,keagan,keagen,keaira,kean,keana,keandre,keane,keanna,keanu,keara,keasia,keaton," +
"kecia,kedric,kedrick,keegan,keelan,keeley,keely,keena,keenan,keenen,keesha,keeton,kegan,kehlani,keifer," +
"keigan,keila,keilan,keilani,keily,keion,keir,keira,keiran,keiry,keisha,keith,kejuan,kekoa,kelan,kelby," +
"kelcey,kelci,kelcie,keli,kelis,kellan,kelle,kellee,kellen,keller,kelley,kelli,kellie,kellin,kelly," +
"kellyann,kellye,kellyn,kelsea,kelsee,kelsey,kelsi,kelsie,kelson,kelsy,kelton,kelvin,kemari,kemarion," +
"kemp,kemper,ken,kena,kenadee,kenadie,kenai,kenan,kenda,kendal,kendale,kendall,kendarius,kendell,kendon," +
"kendra,kendric,kendrick,kendrix,kendyl,keneth,kenia,kenisha,kenith,kenji,kenlee,kenleigh,kenley,kenna," +
"kennadi,kennady,kennan,kennard,kennedi,kennedy,kenneth,kenney,kennie,kennith,kennon,kennth,kenny," +
"kenric,kenrick,kensington,kensley,kent,kenton,kentrell,kenya,kenyan,kenyatta,kenyetta,kenyon,kenzi," +
"kenzie,kenzley,kenzo,keon,keona,keondre,keoni,keonna,keontae,keonte,keosha,kera,keren,keri,kermit," +
"keron,kerri,kerrie,kerrigan,kerry,kerstin,kervin,kerwin,kesean,kesha,keshaun,keshav,keshawn,keshia," +
"keshon,keturah,kevan,keven,kevin,kevion,kevon,kevyn,keya,keyan,keyana,keyanna,keyara,keyla,keylan," +
"keyon,keyona,keyonna,keysha,keyshawn,kezia,keziah,khadija,khadijah,khai,khalani,khaleb,khaled,khaleel," +
"khaleesi,khalia,khalid,khalif,khalil,khalilah,khamari,khang,khari,khary,khaza,khiry,khloe,khristian," +
"khristopher,khyree,kia,kiaan,kiah,kian,kiana,kianna,kiara,kiarra,kiefer,kiel,kiera,kieran,kiernan," +
"kierra,kiersten,kierstin,kierstyn,kiesha,kieth,kile,kilee,kiley,kilian,killian,kim,kimani,kimball," +
"kimber,kimberely,kimberlee,kimberley,kimberli,kimberlie,kimberly,kimberlyn,kimi,kimora,kindra,king," +
"kingsley,kingston,kinlee,kinleigh,kinley,kinsey,kinslee,kinsleigh,kinsley,kinzie,kinzley,kion,kiona," +
"kip,kipp,kipton,kira,kiran,kirby,kirk,kirkland,kirra,kirsten,kirstie,kirstin,kirstyn,kirt,kisha,kishan," +
"kit,kitana,kitty,kiya,kiyah,kiyan,kiyomi,kizzy,klaire,klara,klarissa,klaus,klay,klayton,kloe,kloey," +
"knowledge,knox,koa,koah,kobe,kobi,kobie,koby,koda,kodi,kodie,kody,koen,kofi,kohen,kohl,kolbe,kolby," +
"kole,kolin,kollin,kolson,kolt,kolten,kolter,kolton,koltyn,kong,konner,konnor,konrad,konstantinos," +
"kooper,kora,korben,korbin,korbyn,kordell,korey,kori,korie,korina,korra,kortney,kory,kota,kourtney," +
"koy,kraig,kreed,kreg,krew,kris,krish,krisha,krishiv,krishna,krissy,krista,kristal,kristan,kristel," +
"kristen,kristi,kristian,kristiana,kristie,kristin,kristina,kristine,kristofer,kristoffer,kristopher," +
"kristy,kristyn,kross,krue,kruz,krysta,krystal,krysten,krystian,krystin,krystina,krystle,krystyna," +
"kunal,kurt,kurtis,kush,kwame,ky,kya,kyah,kyaire,kyan,kyana,kyanna,kyara,kyden,kye,kyla,kylah,kylan," +
"kylar,kyle,kylee,kyleigh,kylen,kylene,kyler,kyley,kyli,kylian,kylie,kylin,kylo,kylon,kymani,kymberly," +
"kyndal,kyndall,kyndra,kyng,kyngston,kynlee,kynleigh,kynslee,kyra,kyrah,kyran,kyree,kyren,kyrie,kyrin," +
"kyro,kyron,kyrsten,kysen,kyson,kyzer,lacee,lacey,lachelle,lachlan,laci,lacie,lacy,ladarius,ladarrius," +
"ladd,ladell,ladon,ladonna,lael,laela,lafayette,laiken,laikyn,laila,lailah,lailani,laina,laine,lainey," +
"lainie,laird,laisha,laith,lajuan,lajuana,lake,lakeisha,lakelyn,lakelynn,laken,lakendra,lakenya,lakesha," +
"lakeshia,lakia,lakiesha,lakin,lakisha,lakita,lakota,lakyn,lamar,lamarcus,lamarion,lamarr,lambert," +
"lamon,lamonica,lamont,lamonte,lamya,lana,lanae,lance,landan,landen,lander,landin,landis,landon,landry," +
"landyn,lane,lanell,lanette,laney,langdon,langston,lani,lanie,lanita,laniya,laniyah,lanna,lannie,lanny," +
"lanora,laquan,laquisha,laquita,lara,larae,laraine,laramie,larenzo,larhonda,larisa,larissa,lark,larkin," +
"laron,laronda,laroy,larry,lars,larson,larue,lary,lashae,lashanda,lashawn,lashawna,lashawnda,lashay," +
"lashonda,lashunda,lasonya,laszlo,latanya,latarsha,latasha,latashia,latavia,latesha,lathan,latia,laticia," +
"latina,latisha,latonia,latonya,latoria,latosha,latoya,latrell,latrice,latricia,latrina,latrisha,launa," +
"laura,lauralee,laurance,laure,laureen,laurel,lauren,laurence,laurene,laurent,lauretta,laurette,lauri," +
"laurie,laurin,lauro,lauryn,lavada,lavar,lavell,lavelle,lavender,lavern,laverne,lavina,lavinia,lavon," +
"lavonda,lavonna,lavonne,lavonte,lawana,lawanda,lawanna,lawerence,lawrance,lawrence,lawson,lawton," +
"laya,layan,layden,layla,laylah,laylani,layna,layne,laynee,laynie,layton,lazaro,lazarus,lea,leah,leamon," +
"leana,leander,leandra,leandre,leandro,leann,leanna,leanne,leatha,leatrice,lebron,leda,ledger,lee," +
"leeann,leeanna,leeanne,leela,leeland,leen,leena,leeroy,leesa,legacy,legend,leia,leif,leigh,leigha," +
"leighann,leighton,leila,leilah,leilani,leilany,leisa,leisha,lela,leland,lelia,lemar,lemuel,len,lena," +
"lenard,leni,lenin,lenna,lennard,lennie,lennon,lennox,lenny,lenora,lenore,lenox,lenwood,leo,leobardo," +
"leola,leon,leona,leonard,leonardo,leonel,leonidas,leonor,leonora,leopold,leopoldo,leora,leota,leron," +
"leroy,les,lesa,lesia,leslee,lesley,lesli,leslie,lesly,leslye,lessie,lester,leta,letha,leticia,letisha," +
"letitia,lettie,letty,lev,levar,levern,levi,levin,leviticus,levon,levy,lew,lewis,lex,lexi,lexie,lexington," +
"lexis,lexus,lexy,leya,leyla,leylani,leyna,leyton,lezlie,lia,liah,liam,lian,liana,liane,lianna,lianne," +
"libby,liberty,librado,lida,lidia,liesl,lila,lilah,lili,lilia,lilian,liliana,lilianna,lilibeth,lilith," +
"lilla,lilli,lillia,lillian,lilliana,lillianna,lillie,lillith,lilly,lillyan,lillyana,lillyann,lillyanna," +
"lily,lilyan,lilyana,lilyann,lilyanna,lilyanne,lina,lincoln,linda,lindell,linden,lindon,lindsay,lindsey," +
"lindsy,lindy,linette,link,linkin,linley,linn,linnea,linnie,lino,linsey,linton,linus,linwood,lionel," +
"lionell,lipa,lisa,lisamarie,lisandra,lisandro,lisbeth,lise,lisette,lisha,lissa,lissette,lita,little," +
"litzy,liv,livia,liya,liyah,liz,liza,lizabeth,lizandro,lizbeth,lizet,lizeth,lizette,lizzette,lizzie," +
"llewellyn,lloyd,lluvia,lochlan,locke,logan,logen,lois,loki,lola,lolita,lon,lona,londa,london,londyn," +
"londynn,long,loni,lonna,lonnell,lonnie,lonny,lonzo,lora,loraine,loralee,loralei,loran,lorcan,loreal," +
"loree,loreen,lorelai,lorelei,loren,lorena,lorene,lorenz,lorenza,lorenzo,loretta,lori,loriann,lorianne," +
"lorie,lorin,lorinda,lorine,lorna,lorne,lorraine,lorretta,lorri,lorrie,lory,loryn,lottie,lotus,lou," +
"louann,louanne,louella,louie,louis,louisa,louise,lourdes,louvenia,love,lovell,lovely,lovie,lowell," +
"loy,loyal,loyalty,loyce,loyd,lu,luan,luana,luann,luanna,luanne,luc,luca,lucas,lucca,lucero,luci,lucia," +
"lucian,luciana,lucianna,luciano,lucie,lucien,lucila,lucile,lucille,lucinda,lucio,lucious,lucius,lucky," +
"lucretia,lucus,lucy,ludwig,lue,luella,luigi,luis,luisa,luisangel,luiz,luka,lukas,luke,lukus,lula," +
"lulu,luna,lupe,lupita,luqman,lura,luther,lux,luz,lyam,lyanna,lydell,lydia,lyla,lylah,lyle,lyman,lyn," +
"lynda,lyndell,lynden,lyndi,lyndon,lyndsay,lyndsey,lynell,lynelle,lynette,lynlee,lynn,lynna,lynne," +
"lynnea,lynnette,lynsey,lynwood,lyon,lyra,lyric,mabel,mable,mac,macario,macarthur,macayla,mace,macee," +
"macen,maceo,macey,machelle,maci,macie,mack,mackenna,mackenzi,mackenzie,macklin,macon,macy,madaline," +
"madalyn,madalynn,madden,maddex,maddie,maddison,maddix,maddox,maddux,madelaine,madeleine,madelene," +
"madelin,madeline,madelyn,madelyne,madelynn,madelynne,madge,madilyn,madilynn,madisen,madison,madisson," +
"madisyn,madonna,madysen,madyson,mae,maegan,mael,maelyn,maelynn,maeve,magali,magaly,magan,magda,magdalena," +
"magdalene,magen,maggie,magnolia,magnus,maha,mahala,mahalia,mahdi,maher,mahir,mahki,mahlon,mahmoud," +
"mahogany,mai,maia,maida,maile,maira,maisha,maisie,maison,maisy,maisyn,maite,maiya,maizie,maja,majesty," +
"major,makaela,makai,makaila,makaio,makala,makari,makayla,makaylah,makeda,makena,makenna,makenzi,makenzie," +
"makhai,makhi,makiah,makinley,makiya,makiyah,makoa,maksim,makyla,malachai,malachi,malaika,malak,malakai," +
"malakhi,malaki,malani,malaya,malayah,malaysia,malcolm,malcom,male,malea,maleah,maleek,malek,malena," +
"malia,maliah,maliek,malik,malika,malikai,maliki,malina,malinda,malique,malisa,malissa,maliya,maliyah," +
"malka,malky,mallorie,mallory,malorie,malvin,mamadou,mamie,manda,mandi,mandie,mandy,manfred,manley," +
"manning,manny,manolo,manuel,manuela,maple,mara,marah,maranda,marbella,marc,marcanthony,marcel,marcela," +
"marcelina,marceline,marcelino,marcell,marcella,marcelle,marcello,marcellous,marcellus,marcelo,marchelle," +
"marci,marcia,marcial,marciano,marcie,marco,marcoantonio,marcos,marcus,marcy,marek,marely,maren,margaret," +
"margarett,margarette,margarita,margarito,margaux,marge,margery,margie,margo,margot,margret,marguerite," +
"mari,maria,mariaelena,mariah,mariajose,mariam,mariama,marian,mariana,mariann,marianna,marianne,mariano," +
"maribel,maribeth,maricela,maricella,maricruz,marie,mariel,mariela,mariella,marielle,marietta,marigold," +
"marika,marilee,marilou,marilu,marilyn,marilynn,marin,marina,marino,mario,marion,maris,marisa,marisela," +
"marisol,marissa,marita,maritza,marius,marivel,mariya,mariyah,marjorie,marjory,mark,markanthony,markayla," +
"markeith,markel,markell,markese,markia,markis,markita,marko,markos,markus,marla,marlan,marlana,marlee," +
"marleen,marleigh,marlen,marlena,marlene,marley,marli,marlie,marlin,marlo,marlon,marlow,marlowe,marly," +
"marlyn,marlys,marna,marni,marnie,marquan,marque,marquel,marquell,marques,marquese,marquetta,marquette," +
"marquez,marquis,marquise,marquita,marqus,mars,marsha,marshal,marshall,marshaun,marshawn,marta,martavious," +
"martel,martell,martez,martha,marti,martin,martina,martine,martinez,marty,marva,marvel,marvell,marvin," +
"marvis,marwa,marwan,mary,marya,maryah,maryalice,maryam,maryann,maryanna,maryanne,marybeth,maryelizabeth," +
"maryellen,marygrace,maryjane,maryjo,marykate,marylee,marylin,marylou,marylyn,marylynn,maryn,maryrose," +
"maryssa,masen,masiah,mason,massiah,massimo,masyn,mateo,mateus,mateusz,matheo,matheus,mathew,mathias," +
"mathieu,mathis,matias,matilda,matilde,matt,mattea,matteo,mattew,matthew,matthews,matthias,matthieu," +
"mattias,mattie,mattison,maude,maudie,maura,maureen,maurice,mauricio,maurine,mauro,maury,maverick," +
"maverik,mavis,mavrick,max,maxie,maxim,maxime,maximilian,maximiliano,maximillian,maximino,maximo,maximus," +
"maxine,maxon,maxson,maxton,maxwell,maxx,may,maya,mayah,maycee,mayer,maylee,maylin,maynard,maynor," +
"mayra,mayson,mayte,mazen,mazi,mazie,mcarthur,mccoy,mckay,mckayla,mckenna,mckenzi,mckenzie,mckinley," +
"mea,meadow,meagan,meaghan,meah,mecca,mechelle,medina,meera,meg,megan,meggan,meghan,meghann,mehdi," +
"mehki,mei,meilani,meir,mekayla,mekhi,mel,melaine,melani,melania,melanie,melannie,melany,melba,melia," +
"melina,melinda,melisa,melissa,melissia,melita,mellisa,mellissa,melodee,melodi,melodie,melody,melonie," +
"melony,melton,melva,melvin,melvina,melvyn,melynda,melyssa,memphis,menachem,mendel,mendy,menno,meranda," +
"mercedes,mercedez,mercer,mercy,meredith,meri,merideth,meridith,merilyn,merissa,merl,merle,merlene," +
"merlin,merlyn,merri,merrick,merrie,merrilee,merrill,merritt,merry,merton,mervin,meryl,messiah,meta," +
"meyer,mia,miabella,miah,mica,micaela,micah,micahel,micaiah,micayla,micha,michael,michaela,michaelangelo," +
"michaele,michaella,michal,michale,micheal,michel,michela,michelangelo,michele,michell,michelle,michial," +
"michiel,mick,mickael,mickayla,mickeal,mickel,mickey,micki,mickie,micky,miesha,migdalia,migel,miguel," +
"miguelangel,mihir,mika,mikael,mikaela,mikah,mikail,mikaila,mikal,mikala,mikayla,mike,mikeal,mikel," +
"mikell,mikey,mikhael,mikhail,miki,mikki,miko,mila,milagro,milagros,milah,milan,milana,milani,milania," +
"mildred,mileena,milena,miles,miley,milford,milissa,milla,millard,miller,millicent,millie,milly,milo," +
"milton,mimi,mina,mindi,mindy,minerva,minh,minnie,miquel,mira,miracle,miranda,mirella,mireya,miriah," +
"miriam,mirian,mirna,misael,mischa,misha,missy,misti,mistie,misty,mitch,mitchel,mitchell,mitzi,miya," +
"miyah,modesto,mohamad,mohamed,mohammad,mohammed,moira,moises,moishe,molli,mollie,molly,mona,monae," +
"monet,monica,monika,monique,monroe,monserrat,montana,montavious,monte,montel,montez,montgomery,montie," +
"montreal,montrel,montrell,montserrat,monty,morag,mordecai,mordechai,morgan,morgen,moriah,morris,morrison," +
"morton,mose,moses,moshe,moussa,muhammad,muhammed,murad,muriel,murphy,murray,murry,musa,mustafa,mya," +
"myah,myasia,mychael,mychal,myesha,myia,myka,mykah,mykal,mykayla,mykel,myla,mylah,mylan,mylee,myles," +
"mylie,mylo,myra,myranda,myriah,myriam,myrna,myron,myrtis,myrtle,nabeel,nabil,nada,nader,nadia,nadine," +
"nadir,nadya,naeem,nael,naftali,nahla,nahom,nahomi,nahum,naia,naila,nailah,naim,naima,naimah,naisha," +
"naiya,najah,najee,nakayla,nakia,nakisha,nakita,nakiya,nakoa,nala,nalani,nallely,nam,nan,nana,nanci," +
"nancie,nancy,nanette,nannette,nannie,naomi,naomie,naomy,napoleon,naquan,narciso,nariah,nariyah,naseem," +
"nash,nashawn,nasir,nasser,nassir,natalee,natali,natalia,natalie,nataly,natalya,natan,natanael,natasha," +
"natashia,nate,nathalia,nathalie,nathaly,nathan,nathanael,nathaneal,nathanial,nathaniel,nathen,nathon," +
"natisha,natividad,natosha,nautica,navaeh,naveah,naveen,navy,navya,naya,nayeli,nayely,nayla,nazareth," +
"nazario,nazir,neal,nechama,ned,neda,nedra,neel,neely,neena,neftali,neha,nehemiah,nehemias,neida,neil," +
"neill,neko,nelda,nell,nella,nellie,nelly,nels,nelson,nena,neo,nereida,neriah,nerissa,nery,nestor," +
"nettie,neva,nevada,nevaeh,nevan,neveah,neville,nevin,newell,newton,neymar,nia,niah,niall,niam,niamh," +
"nicco,niccolo,nichelle,nichol,nicholai,nicholas,nicholaus,nichole,nicholle,nick,nickey,nicki,nickie," +
"nicklas,nicklaus,nickolas,nickolaus,nickole,nicky,nico,nicodemus,nicol,nicola,nicolai,nicolas,nicolaus," +
"nicole,nicolette,nicolle,nicolo,nidia,niel,niels,niesha,nigel,nika,nikhil,niki,nikia,nikita,nikki," +
"nikko,niklas,niklaus,niko,nikola,nikolai,nikolaos,nikolas,nikolaus,nikole,nila,nilda,nile,niles,nils," +
"nilsa,nima,nina,ninfa,nino,nisa,nisha,nita,nixon,niya,niyah,noa,noah,noam,noble,noe,noel,noelani," +
"noelia,noella,noelle,noemi,noemy,nohemi,nola,nolan,noland,nolen,nona,noor,nora,norah,norbert,norberto," +
"noreen,norene,nori,norine,norma,norman,normand,norris,norwood,nour,nova,novah,novalee,novella,nuala," +
"nubia,nya,nyah,nyasia,nydia,nyla,nylah,nyle,nyomi,nyra,nyree,nyssa,oakland,oaklee,oakleigh,oakley," +
"oaklyn,oaklynn,obadiah,obed,obie,ocean,ocie,octavia,octavian,octavio,octavious,octavius,odalis,odalys," +
"odelia,odell,oden,odessa,odette,odie,odin,odis,ofelia,oisin,ola,olaf,olan,ole,olen,olga,olin,olive," +
"oliver,olivia,olivier,ollie,olympia,olyvia,om,oma,omar,omari,omarion,omer,ona,oneal,oneil,onyx,oona," +
"opal,ophelia,ora,oral,oralia,oran,oren,ori,oriana,orin,orion,orla,orlando,orlin,orrin,orson,orval," +
"orville,oryan,osama,osbaldo,oscar,oshea,osiel,osiris,oskar,osman,osmar,osvaldo,oswald,oswaldo,otha," +
"otis,otoniel,ottis,otto,ouida,ovidio,owen,ozell,ozias,oziel,ozzie,ozzy,pablo,paden,padraic,padraig," +
"page,paige,paislee,paisleigh,paisley,paityn,palmer,paloma,pam,pamala,pamela,pamelia,pamella,panagiotis," +
"pandora,pansy,paola,paolo,paris,park,parker,parks,parnell,parris,parrish,parth,pascal,pascual,pasquale," +
"passion,pat,patience,patric,patrica,patrice,patricia,patricio,patrick,patrina,patryk,patsy,patti," +
"pattie,patton,patty,paul,paula,pauletta,paulette,paulina,pauline,paulino,paulo,pavel,pax,paxton,payden," +
"payne,payson,payten,payton,pearce,pearl,pearlie,pearline,pearson,pedro,peggie,peggy,peighton,penelope," +
"penn,penney,penni,pennie,penny,pepper,percival,percy,peri,perla,pernell,perri,perrin,perry,persephone," +
"perseus,pete,peter,petra,petrina,peyton,phaedra,pharaoh,phebe,phil,philip,philipp,philippa,philippe," +
"phillip,phillips,phillis,philomena,phineas,phoebe,phoenix,phong,phylicia,phylis,phyliss,phyllis,pia," +
"pierce,pierre,pierson,pieter,pietro,pilar,pinchas,piper,pippa,polly,pooja,poppy,porfirio,porsche," +
"porsha,porter,portia,powell,pranav,precious,prentice,prentis,prentiss,prescott,preslee,presleigh," +
"presley,preslie,preston,price,pricilla,prince,princess,princeton,priscila,priscilla,prisha,priya," +
"priyanka,promise,prudence,pryce,pyper,qasim,qiana,quade,quadir,quan,queen,queenie,quentin,quenton," +
"quest,quiana,quin,quincey,quincy,quinlan,quinn,quinten,quintin,quinton,quran,race,rachael,racheal," +
"rachel,rachele,rachell,rachelle,racquel,radley,rae,raeann,raeanna,raeanne,raechel,raegan,raekwon," +
"raelee,raeleigh,raelene,raelyn,raelynn,raena,rafael,rafaela,rafe,rafeal,raffaele,ragan,raghav,raheem," +
"rahim,rahman,rahsaan,rahul,raiden,rain,raina,raine,rainer,rainey,raizel,raizy,raj,rajan,rakeem,rakim," +
"raleigh,ralph,ram,ramel,rami,ramir,ramiro,ramon,ramona,ramond,ramone,ramses,ramsey,ramy,rana,ranae," +
"rance,rand,randa,randal,randall,randee,randel,randell,randi,randle,randolf,randolph,randon,randy," +
"ranger,rani,rania,raniyah,ransom,raoul,raphael,rapheal,raquan,raquel,rashaad,rashaan,rashad,rashan," +
"rashard,rashaud,rashaun,rashawn,rasheed,rasheeda,rasheem,rasheen,rashid,rashida,rashod,rashon,raudel," +
"raul,raven,ravi,ravon,ravyn,ray,raya,rayaan,rayan,rayann,rayanna,rayburn,rayce,rayden,raye,rayfield," +
"rayford,raylan,raylee,rayleigh,raylen,raylene,raylon,raylynn,raymon,raymond,raymundo,rayna,raynaldo," +
"raynard,rayne,rayquan,rayshaun,rayshawn,rayven,rayvon,rayyan,raziel,reagan,reanna,reba,rebbecca,rebeca," +
"rebecca,rebecka,rebeka,rebekah,rebekka,rebel,reco,reece,reed,reef,reem,reema,reena,rees,reese,refugio," +
"regan,regena,regenia,reggie,regina,reginal,reginald,regis,rehan,reid,reign,reilly,reina,reinaldo," +
"remi,remington,remy,ren,rena,renae,renaldo,renard,renata,renato,renay,rene,renea,renee,renesmee,renita," +
"reno,renzo,reta,retha,reuben,reuven,reva,revan,rex,rexford,rey,reya,reyansh,reyes,reymundo,reyna," +
"reynaldo,reynold,reynolds,reza,rhea,rheanna,rhett,rhianna,rhiannon,rhoda,rhodes,rhonda,rhyan,rhyder," +
"rhylee,rhys,ria,riaan,rian,riana,rianna,ric,ricardo,riccardo,ricco,rich,richard,richardo,richelle," +
"richie,richmond,rick,rickey,ricki,rickie,ricky,rico,rider,ridge,ridley,riggs,rigo,rigoberto,rihanna," +
"riker,rikki,rilee,rileigh,riley,rilynn,rima,rina,rio,rion,riot,ripley,risa,rishaan,rishabh,rishi," +
"rita,ritchie,rito,river,rivers,rivka,rivky,riya,riyan,roan,rob,robb,robbie,robbin,robby,robert,roberta," +
"roberto,robie,robin,robinson,roby,robyn,rocco,rochel,rochell,rochelle,rocio,rock,rockey,rockie,rockwell," +
"rocky,rod,roddy,roderic,roderick,rodger,rodney,rodolfo,rodrick,rodrigo,rodriguez,rodriquez,roel,roen," +
"rogan,rogelio,roger,rogers,rohan,rohit,roisin,rojelio,roland,rolanda,rolando,rolf,rolland,rollie," +
"rollin,roma,roman,romario,rome,romel,romell,romello,romelo,romeo,romero,romie,romina,rommel,romona," +
"romy,ron,rona,ronak,ronal,ronald,ronaldo,ronan,ronda,rondal,rondell,ronell,ronen,roni,ronin,ronna," +
"ronnell,ronney,ronni,ronnie,ronny,rony,roosevelt,roque,rori,rory,rosa,rosalba,rosalee,rosalia,rosalie," +
"rosalina,rosalind,rosalinda,rosaline,rosalio,rosalva,rosalyn,rosalynn,rosamaria,rosana,rosann,rosanna," +
"rosanne,rosario,rosaura,roscoe,rose,roseann,roseanna,roseanne,rosella,roselyn,roselynn,rosemarie," +
"rosemary,rosendo,rosetta,rosevelt,roshan,roshawn,rosie,rosina,rosio,rosita,roslyn,ross,rosy,rowan," +
"rowdy,rowen,rowena,rowland,rowyn,roxana,roxane,roxann,roxanna,roxanne,roxie,roxy,roy,royal,royalty," +
"royce,ruairi,ruben,rubi,rubie,rubin,ruby,rubye,rudolf,rudolfo,rudolph,rudra,rudy,rueben,ruel,rufino," +
"rufus,ruger,rupert,rush,russ,russel,russell,rustin,ruston,rusty,ruth,ruthann,ruthanne,ruthie,rya," +
"ryan,ryann,ryanne,ryatt,ryden,ryder,rye,ryen,ryker,rylan,ryland,rylee,ryleigh,rylen,ryley,rylie,rylin," +
"rylynn,ryne,ryon,ryu,saad,saanvi,sabastian,sabian,sabina,sabine,sabino,sable,sabra,sabrena,sabrina," +
"sacha,sachin,sade,sadie,saeed,safa,safiya,sagar,sage,sahana,sahar,sahara,sahasra,sahil,sai,said,saif," +
"saige,sailor,saint,saira,sakura,sal,salah,saleem,saleh,salem,salena,salim,salina,sallie,sally,salma," +
"salman,salome,salomon,salvador,salvatore,sam,sama,samanta,samantha,samar,samara,samarah,samaria,samarth," +
"samatha,samaya,sameer,samer,sami,samia,samir,samira,samirah,samiya,samiyah,sammantha,sammi,sammie," +
"sammuel,sammy,samone,sampson,samson,samual,samuel,samya,san,sana,sanaa,sanai,sanaya,sanchez,sandee," +
"sander,sanders,sandi,sandie,sandra,sandro,sandy,sanford,sania,saniya,saniyah,sanjana,sanjay,sanjuana," +
"sanjuanita,santa,santana,santiago,santina,santino,santo,santonio,santos,sanya,saoirse,saphira,sapphire," +
"sara,sarah,sarahi,sarai,saray,saraya,sarena,sari,sariah,sarina,sarita,sariya,sariyah,sasha,saul,saundra," +
"savana,savanah,savanna,savannah,saverio,savion,savon,sawyer,saxon,saylor,scarlet,scarlett,scarlette," +
"schuyler,scot,scott,scottie,scotty,scout,seamus,sean,seana,seanna,sebastian,sebastien,sebrina,sedona," +
"sedric,sedrick,seferino,sekani,sekou,selah,selena,selene,selim,selina,selma,selvin,selwyn,semaj,sena," +
"seneca,september,sequoia,sera,serafin,serafina,seraphina,seren,serena,serene,serenity,serge,sergio," +
"serina,serjio,servando,seth,seven,severo,sevyn,seymour,shaan,shad,shade,shae,shaelyn,shaelynn,shaheed," +
"shai,shaila,shain,shaina,shaindy,shaine,shakayla,shakia,shakir,shakira,shakur,shala,shalanda,shalom," +
"shalonda,shalyn,shamar,shamara,shameka,shamel,shamika,shamir,shamus,shamya,shan,shana,shanae,shanaya," +
"shanda,shandi,shandra,shane,shaneka,shanel,shanell,shanelle,shanequa,shani,shania,shaniah,shanice," +
"shaniece,shanika,shaniqua,shanique,shanita,shaniya,shaniyah,shanna,shannan,shannen,shannon,shanon," +
"shanta,shantal,shante,shantel,shantell,shantelle,shanti,shanya,shaquan,shaquana,shaquille,shaquita," +
"shara,sharee,shareef,sharen,shari,sharie,sharif,sharilyn,sharita,sharla,sharleen,sharlene,sharmaine," +
"sharod,sharon,sharonda,sharron,sharyl,sharyn,shasta,shaun,shauna,shaunda,shaunna,shaunte,shaurya," +
"shavon,shavonne,shaw,shawana,shawanda,shawn,shawna,shawnda,shawnee,shawnna,shawnta,shay,shaya,shayan," +
"shayden,shaye,shayla,shaylee,shaylin,shaylyn,shaylynn,shayna,shayne,shea,shedrick,sheena,sheila,sheilah," +
"shekinah,shelba,shelbi,shelbie,shelby,sheldon,shelia,shelley,shelli,shellie,shelly,shelton,shemar," +
"shena,shenika,shepard,shepherd,sheree,sheri,sheridan,sherie,sherika,sherilyn,sherita,sherlyn,sherman," +
"sheron,sherrell,sherri,sherrie,sherril,sherrill,sherrod,sherron,sherry,sherryl,sherwin,sherwood,sheryl," +
"sheyla,shia,shiann,shianne,shiela,shifra,shilo,shiloh,shimon,shira,shirlee,shirleen,shirlene,shirley," +
"shiv,shivam,shivani,shivansh,shlok,shlomo,shmuel,sholom,shomari,shon,shona,shonda,shonna,shonta,shoshana," +
"shreya,shreyas,shriya,shulem,shyann,shyanna,shyanne,shyheim,shyla,shylah,sia,sian,sianna,siara,sid," +
"siddharth,sidney,sidra,siena,sienna,siera,sierra,sigmund,signe,sigrid,silas,silvana,silvano,silver," +
"silverio,silvestre,silvia,silvio,sima,simcha,simeon,simon,simona,simone,simran,sinai,sincere,sinclair," +
"sindy,sinead,siobhan,sione,sir,sire,sirena,siri,sixto,siya,skip,sky,skye,skyla,skylah,skylar,skylee," +
"skyler,skylor,skylynn,skyy,slade,slater,sloan,sloane,smith,socorro,sofia,sofie,soham,sol,solana,soledad," +
"soleil,soloman,solomon,somer,sommer,sondra,sonia,sonja,sonny,sonya,sophia,sophie,sora,soraya,soren," +
"sparkle,spence,spencer,spenser,spring,stacee,stacey,staci,stacia,stacie,stacy,stafford,stan,stanford," +
"stanley,stanton,star,starla,starr,stavros,steele,stefan,stefani,stefania,stefanie,stefano,stefany," +
"steffan,steffanie,steffany,steffen,stefon,stella,stellan,stephaine,stephan,stephani,stephania,stephanie," +
"stephany,stephen,stephenie,stephine,stephon,sterling,stetson,stevan,steve,steven,stevens,stevenson," +
"stevi,stevie,stewart,stockton,stone,stoney,storm,stormi,stormie,stormy,story,stratton,struan,stryker," +
"stuart,sue,sueann,suellen,sulaiman,suleiman,sulema,sullivan,sultan,sumaya,summer,summit,sumner,sunday," +
"sunil,sunni,sunnie,sunny,sunshine,suraj,suri,surya,susan,susana,susann,susanna,susannah,susanne,susie," +
"sutton,suzan,suzann,suzanna,suzannah,suzanne,suzette,suzie,suzy,sven,sy,sybil,syble,sydnee,sydney," +
"sydni,sydnie,syed,syeda,sylas,sylis,sylus,sylvan,sylvester,sylvia,sylvie,symone,symphony,syncere," +
"synthia,syreeta,syrus,tab,tabatha,tabetha,tabitha,tad,tadd,tadeo,tadhg,taelor,taelyn,taelynn,tafari," +
"taft,taha,tahir,tahj,tahlia,tai,taina,taisha,tait,taj,taja,takia,takisha,takiyah,takoda,tal,tala," +
"talan,taleah,talen,talha,talia,taliah,talisa,talisha,talitha,taliyah,tallon,tallulah,talmadge,talmage," +
"talon,talya,tam,tamala,tamar,tamara,tamatha,tambra,tameka,tamekia,tamela,tamera,tamesha,tami,tamia," +
"tamica,tamie,tamika,tamiko,tamir,tamira,tamisha,tamiya,tammara,tammi,tammie,tammy,tamra,tamya,tamyra," +
"tana,tanaya,taneisha,tanesha,tangela,tania,tanika,tanisha,taniya,taniyah,tanja,tanna,tanner,tanvi," +
"tanya,taquan,tara,tarah,taraji,taran,tarek,tari,tarik,tariq,taron,tarra,tarsha,taryn,tasha,tashawn," +
"tashia,tashina,tasia,tasneem,tate,tatiana,tatianna,tatiyana,tatum,tatyana,tatyanna,taurean,taurus," +
"tavares,tavaris,tavarus,taven,tavia,tavian,tavin,tavion,tavis,tavon,tawana,tawanda,tawanna,tawni," +
"tawny,tawnya,taya,tayden,tayla,taylar,taylee,taylen,tayler,taylin,taylon,taylor,tayshaun,tayshawn," +
"tayson,tayton,taytum,tayvion,tayvon,tea,teagan,teague,teal,teanna,ted,tedd,teddi,teddie,teddy,teegan," +
"teena,tegan,tehya,tejas,telly,temperance,tena,tenisha,tenley,tennille,tennyson,tenzin,teo,teodoro," +
"teon,tequila,tera,terance,terell,terence,teresa,terese,teresita,teressa,teri,teron,terra,terran,terrance," +
"terrel,terrell,terrence,terri,terrie,terrill,terrion,terron,terry,tesha,tesla,tess,tessa,tessie,tevin," +
"tevon,tex,thad,thaddaeus,thaddeus,thai,thais,thalia,thane,thanh,thatcher,thayer,thayne,thea,theadore," +
"theda,thelma,theo,theodis,theodor,theodora,theodore,theophilus,theresa,therese,therman,theron,thiago," +
"thien,thomas,thomasina,thomasine,thompson,thor,thorin,thornton,thurman,thurston,tia,tiago,tiana,tianna," +
"tiara,tiarra,tiberius,tiera,tiernan,tierney,tierra,tiesha,tiffaney,tiffani,tiffanie,tiffany,tiffiny," +
"tiger,tilden,tillie,tillman,tilly,tim,timmie,timmothy,timmy,timoteo,timothy,timur,tina,tinley,tino," +
"tinsley,tionna,tirzah,tisha,titan,tito,titus,tj,tobey,tobi,tobias,tobin,toby,tod,todd,tom,tomas,tomeka," +
"tomi,tomika,tommi,tommie,tommy,tonda,toney,toni,tonia,tonie,tonisha,tonja,tony,tonya,toren,torey," +
"tori,torian,toriano,torie,torin,torrance,torrence,torrey,torri,torrie,torrin,torry,tory,tosha,tou," +
"tova,towanda,townes,toya,trace,tracee,tracey,traci,tracie,tracy,trae,tramaine,trapper,traveon,travion," +
"travis,travon,tray,trayvon,tre,treasa,treasure,tremain,tremaine,tremayne,trena,trent,trenten,trentin," +
"trenton,tresa,treshawn,tressa,tressie,treston,treva,trevan,treven,treveon,trever,trevin,trevion,trevis," +
"trevon,trevor,trey,treyson,treyton,treyvon,tricia,trina,trinidad,triniti,trinity,tripp,trish,trisha," +
"trista,tristan,tristen,tristian,tristin,triston,tristyn,triton,troy,tru,trudi,trudie,trudy,true,truett," +
"truitt,truman,truth,trystan,trysten,tryston,tuan,tucker,tuesday,turner,twana,twila,twyla,ty,tyana," +
"tyanna,tyasia,tyce,tye,tyesha,tyisha,tyjuan,tyla,tylan,tylar,tylen,tyler,tylin,tylon,tylor,tymir," +
"tynisha,tyquan,tyra,tyran,tyre,tyree,tyreek,tyreese,tyrek,tyreke,tyrel,tyrell,tyren,tyrese,tyrice," +
"tyrik,tyrin,tyriq,tyrique,tyrome,tyron,tyrone,tyronne,tyrus,tysean,tysen,tyshaun,tyshawn,tyshon,tyson," +
"tytan,tytus,tyus,tywan,tzvi,ubaldo,ulices,ulises,ulisses,ulyses,ulysses,uma,umar,una,unique,unknown," +
"urban,uri,uriah,uriel,urijah,ursula,usher,usman,uziel,uzziah,vada,val,valarie,valen,valencia,valente," +
"valentin,valentina,valentine,valentino,valeri,valeria,valerie,valery,valor,valorie,van,vance,vander," +
"vanesa,vanessa,vania,vanita,vanity,vann,vanna,vannessa,varun,vasilios,vaughan,vaughn,vayda,ved,veda," +
"vedant,veer,velda,velia,velma,velvet,venessa,venice,venita,ventura,venus,vera,verda,verdell,verenice," +
"verity,verl,verla,verlin,verlon,vern,verna,vernal,vernard,verne,vernell,vernetta,vernice,vernie,vernita," +
"vernon,verona,veronica,veronika,vesta,vester,viaan,vianca,vianey,vianna,vianney,vic,vicente,vickey," +
"vicki,vickie,vicky,victor,victoria,victoriano,vida,vidal,vienna,viet,viggo,vihaan,vijay,viki,vikki," +
"vikram,viktor,viktoria,vilma,vina,vinay,vince,vincent,vincente,vincenzo,vinh,vinny,vinson,viola,violet," +
"violeta,violetta,violette,viraj,virgie,virgil,virgilio,virginia,viridiana,vishal,vishnu,vita,vito," +
"vittorio,vivaan,vivek,vivian,viviana,vivianna,vivianne,vivien,vivienne,vladimir,von,vonda,vonnie," +
"wade,walden,waldo,waleed,wali,walid,walker,wallace,wally,walt,walter,walton,wanda,ward,wardell,warner," +
"warren,washington,watson,waverly,wayde,waylan,wayland,waylen,waylon,wayman,waymon,waymond,wayne,webb," +
"webster,wednesday,weldon,wellington,wells,welton,wendall,wendel,wendell,wendi,wendie,wendy,werner," +
"wes,weslee,wesley,wesly,wesson,west,westen,westin,westley,weston,westyn,wheeler,whit,whitley,whitney," +
"whittney,wil,wilber,wilbert,wilbur,wilburn,wilda,wilder,wiley,wilford,wilfred,wilfredo,wilhelm,wilhelmina," +
"wiliam,will,willa,willaim,willam,willard,willem,willette,william,williams,willian,willie,willis,willow," +
"willy,wilma,wilmer,wilson,wilton,windell,windy,winfield,winford,winfred,winifred,winnie,winona,winslow," +
"winston,winter,winton,wisdom,witten,wolf,wolfgang,wonda,woodie,woodrow,woody,worth,wren,wrenlee,wrenley," +
"wyatt,wylder,wylie,wyman,wynn,wynona,wynter,xaiden,xaiver,xander,xavi,xavian,xavien,xavier,xavion," +
"xavior,xayden,xena,xenia,ximena,xiomara,xion,xitlali,xitlaly,xochitl,xyla,xzavier,xzavion,yaakov," +
"yadiel,yadira,yael,yahaira,yahir,yahya,yair,yajaira,yakov,yale,yamil,yamilet,yamileth,yan,yana,yancey," +
"yancy,yandel,yaneli,yanely,yanet,yanira,yannick,yara,yareli,yarely,yaretzi,yaretzy,yariel,yaritza," +
"yaseen,yash,yasin,yasir,yasmeen,yasmin,yasmine,yasser,yassin,yazan,yazmin,yazmine,yechiel,yehoshua," +
"yehuda,yehudis,yerik,yesenia,yeshua,yesica,yessenia,yessica,yisrael,yisroel,yitty,yitzchak,yitzchok," +
"yocheved,yoel,yohan,yolanda,yolonda,yonatan,yonathan,yosef,yoselin,yoselyn,yoseph,young,yousef,yousif," +
"youssef,yousuf,yovani,ysabella,ysidro,yuliana,yulisa,yulissa,yuna,yunus,yurem,yuri,yuridia,yusef," +
"yusra,yusuf,yuvaan,yuvan,yuvraj,yves,yvette,yvonne,zabdiel,zac,zacarias,zacary,zacchaeus,zach,zacharia," +
"zachariah,zacharias,zacharie,zachary,zachery,zachry,zack,zackariah,zackary,zackery,zada,zade,zaden," +
"zadie,zahara,zahid,zahir,zahra,zaid,zaida,zaiden,zaidyn,zain,zaina,zainab,zaine,zaira,zaire,zak,zakai," +
"zakari,zakaria,zakariya,zakary,zakery,zaki,zakia,zakiya,zakiyah,zakkary,zamari,zamarion,zamir,zana," +
"zander,zandra,zane,zaniah,zaniya,zaniyah,zara,zarah,zarek,zaria,zariah,zariya,zariyah,zavian,zavien," +
"zavier,zavion,zaya,zayan,zayd,zayda,zayden,zayla,zaylee,zaylen,zayn,zayna,zaynab,zayne,zayra,zayvion," +
"zeb,zebadiah,zebedee,zebulon,zechariah,zeke,zelda,zella,zelma,zen,zena,zenaida,zendaya,zeno,zenobia," +
"zenon,zephaniah,zephyr,zeppelin,zeth,zeus,zev,zhane,zhuri,zia,ziad,ziggy,zina,zinnia,zion,zipporah," +
"ziva,ziya,ziyad,zoe,zoee,zoey,zofia,zoie,zoila,zola,zona,zora,zoraida,zoya,zulema,zulma,zuri,zuriel," +
"zyair,zyaire,zyan,zyion,zyla,zyler,zymir,zyon";
let known: Set<string> | null = null;
/** Name-key normalization: diacritics folded, lowercased, non-alphanumerics dropped. */
function nameKey(value: string): string {
return value
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase()
.replace(/[^a-z0-9]/g, "");
}
/** Whether a single word is in the known-first-names dataset. Lazy-parses the blob on first call. */
export function isKnownFirstName(word: string): boolean {
if (!known) known = new Set(RAW.split(","));
const key = nameKey(word);
return key.length >= 2 && known.has(key);
}
/** How many names the dataset holds (for diagnostics/UI). */
export function knownFirstNameCount(): number {
if (!known) known = new Set(RAW.split(","));
return known.size;
}
+153
View File
@@ -0,0 +1,153 @@
/**
* Introduction-based speaker naming: when a caption line
* contains a spoken introduction, the auto-numbered
* "Speaker N" profile it belongs to can be renamed without the user typing
* anything. Two shapes are recognized:
*
* - Self-introduction ("I'm Alice", "my name is Alice") — names the speaker
* of the line itself.
* - Third-party introduction ("this is Bob", "meet Bob") — pends the name
* and applies it to the next DIFFERENT still-generic speaker heard.
*
* Names inferred from speech are suggestions applied only to still-generic
* profiles; a name the user set (or a prior inference) is never overwritten.
* Pure module: no platform imports.
*/
import { isKnownFirstName } from "./name-data";
export type NameInference = { name: string; kind: "self" | "third-party" };
// Words that commonly follow "I'm ..." / "this is ..." without being names.
// Lowercase; candidates are checked case-insensitively against this list (ASR
// capitalization is unreliable, so "I'm Hungry" must fail like "I'm hungry").
// The list is deliberately broad — a rejected real introduction costs one
// manual rename, while a false positive mislabels a person in every caption.
const NOT_NAMES = new Set([
// Function words, intensifiers, hedges.
..."not no yes so just really very too quite pretty almost about kind sort still always never now then here there back home away the a an in on at all both only even also again once maybe probably definitely honestly actually literally basically totally"
.split(" "),
// States and feelings ("I'm hungry", "I'm exhausted").
..."sure sorry afraid good fine okay ok great well better worse bad happy sad glad mad angry upset annoyed frustrated tired exhausted sleepy awake hungry thirsty starving full sick ill healthy cold hot warm freezing boiling excited nervous anxious worried scared terrified confused surprised shocked impressed disappointed proud grateful thankful jealous embarrassed ashamed bored curious serious certain positive confident comfortable uncomfortable ready busy free late early lost stuck broke rich famous lucky unlucky alone single married engaged divorced retired unemployed pregnant alive dead"
.split(" "),
// Progressive verbs ("I'm wanting", "I'm heading out").
..."gonna going wanting hoping waiting wondering thinking trying getting feeling speaking calling recording sending listening watching looking asking telling saying talking hearing working walking running driving flying riding leaving arriving staying coming moving heading starting stopping finishing planning meeting eating drinking cooking cleaning shopping packing visiting traveling studying learning teaching reading writing playing singing dancing sleeping dreaming waking standing sitting laughing crying kidding joking guessing counting checking testing building making taking giving putting buying selling paying saving spending losing winning helping"
.split(" "),
// Descriptors that read as predicates, not names.
..."new old young tall short big small huge tiny first last next early done finished set open closed right wrong left in out off on up down over under behind ahead around through outside inside upstairs downstairs downtown"
.split(" "),
// Exclamations and judgments ("this is Ridiculous", "this is Awesome").
..."awesome amazing ridiculous terrible horrible incredible impossible important interesting boring crazy insane weird strange funny hilarious perfect beautiful gorgeous wonderful fantastic excellent brilliant stupid dumb silly annoying confusing exciting embarrassing frustrating disappointing surprising shocking unbelievable outrageous obvious delicious gross disgusting nasty cool nice sweet cute lovely fair unfair true false real fake normal typical classic official final original actual exact simple easy hard difficult complicated expensive cheap heavy light loud quiet fast slow safe dangerous risky enough everything nothing something anything everyone nobody someone anyone mine yours hers theirs ours it that this what how why where who when"
.split(" "),
]);
// Suffixes that mark ordinary adjectives, not given names ("Ridiculous",
// "Wonderful", "Believable"). Kept narrow: common name endings like "-ly"
// (Emily, Kelly) and "-ing" (Ming, Qing) are deliberately NOT here — those
// rely on the word list and the LLM tier instead.
const NON_NAME_SUFFIXES = ["ous", "ful", "less", "able", "ible"];
// Trigger phrases match either sentence case; the captured NAME must be
// capitalized (ASR renders proper nouns capitalized, and this rejects
// ordinary words after "I'm ...").
const NAME = "([A-Z][a-zA-Z'-]+(?: [A-Z][a-zA-Z'-]+)?)";
const SELF_PATTERNS = [
new RegExp(`\\b[Mm]y name is ${NAME}`),
new RegExp(`\\b[Ii](?:'| a)m ${NAME}`),
new RegExp(`\\b[Cc]all me ${NAME}`),
new RegExp(`\\b[Tt]his is ${NAME} speaking\\b`),
];
const THIRD_PARTY_PATTERNS = [
new RegExp(`\\b[Tt]his is ${NAME}`),
new RegExp(`\\b(?:[Mm]eet|[Ss]ay hi to|[Ss]ay hello to) ${NAME}`),
new RegExp(`\\b[Hh]is name is ${NAME}`),
new RegExp(`\\b[Hh]er name is ${NAME}`),
new RegExp(`\\b[Tt]heir name is ${NAME}`),
];
function plausibleName(candidate: string): string | null {
const words = candidate.trim().split(/\s+/);
if (words.length === 0 || words.length > 2) return null;
for (const word of words) {
if (word.length < 2 || word.length > 20) return null;
const lower = word.toLowerCase();
// NOT_NAMES wins even over the known-names dataset: words like "Rich"
// and "Young" ARE real given names, but "I'm rich" is far more often a
// predicate, and a false positive mislabels a person in every caption.
if (NOT_NAMES.has(lower)) return null;
// The suffix rule, by contrast, defers to the dataset — it exists to
// catch adjectives ("Ridiculous"), and it wrongly rejects real names
// like Precious, Demetrious, and Octavious without this carve-out.
if (
!isKnownFirstName(word) &&
NON_NAME_SUFFIXES.some((suffix) => lower.length > suffix.length + 2 && lower.endsWith(suffix))
) {
return null;
}
if (!/^[A-Z]/.test(word)) return null;
}
return words.join(" ");
}
/**
* Whether an inferred name's given name (its first word — a second word is
* a surname the first-name dataset can't know) appears in the known-names
* dataset. Positive signal only: known names can skip the slower LLM
* verification, but absence never rejects — real names the dataset misses
* (non-Anglophone names especially) still go through the normal path.
*/
export function isKnownName(candidate: string): boolean {
const first = candidate.trim().split(/\s+/)[0];
return first ? isKnownFirstName(first) : false;
}
/**
* Detect a spoken introduction in one caption line. Self-introductions win
* over third-party shapes ("this is Alice speaking" is a self-intro even
* though "this is Alice" also matches the third-party pattern).
*/
export function inferIntroducedName(text: string): NameInference | null {
if (!text) return null;
for (const pattern of SELF_PATTERNS) {
const match = pattern.exec(text);
if (match) {
const name = plausibleName(match[1]!);
if (name) return { name, kind: "self" };
}
}
for (const pattern of THIRD_PARTY_PATTERNS) {
const match = pattern.exec(text);
if (match) {
const name = plausibleName(match[1]!);
if (name) return { name, kind: "third-party" };
}
}
return null;
}
/** True for the auto-assigned "Speaker N" names (safe to overwrite). */
export function isGenericSpeakerName(name: string): boolean {
return /^Speaker \d+$/.test(name.trim());
}
export type PendingIntroduction = { name: string; bySpeakerId: number; atMs: number };
/** How long a third-party introduction waits for its subject to speak. */
export const PENDING_INTRODUCTION_TTL_MS = 2 * 60 * 1000;
/**
* Whether a pending third-party introduction applies to a newly heard
* speaker: someone OTHER than the introducer, still generically named,
* speaking within the window.
*/
export function pendingApplies(
pending: PendingIntroduction,
speakerId: number,
speakerName: string,
nowMs: number,
): boolean {
if (nowMs - pending.atMs > PENDING_INTRODUCTION_TTL_MS) return false;
if (speakerId === pending.bySpeakerId) return false;
return isGenericSpeakerName(speakerName);
}
+79
View File
@@ -0,0 +1,79 @@
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import type { NameInference } from "./name-guess";
/**
* Second-tier verification of heuristic name detections using the on-phone
* LLM (the assistant's Qwen model, when downloaded). The regex tier is
* deliberately permissive about unusual real names; the model is asked the
* one question the regex can't answer — "is this actually a person's name
* being introduced here?" — before a speaker profile gets renamed.
*
* Verdicts: true (confirmed), false (rejected), null (model unavailable,
* busy, timed out, or gave a non-answer). Callers treat null as "trust the
* heuristic": renames are retroactively correctable, so a missing model
* must not disable introduction naming entirely.
*/
const VERDICT_TIMEOUT_MS = 25_000;
// One verification at a time: the runner is shared with the assistant, and
// caption lines can arrive faster than CPU inference finishes.
let inFlight = false;
export function verifyIntroducedName(
inference: NameInference,
sentence: string,
): Promise<boolean | null> {
if (!global.isAndroid || !isLocalModelReady() || inFlight) {
return Promise.resolve(null);
}
inFlight = true;
return new Promise((resolve) => {
let settled = false;
const finish = (verdict: boolean | null) => {
if (settled) return;
settled = true;
inFlight = false;
resolve(verdict);
};
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
try {
streamLocalQwen({
apiKey: "",
model: "local",
system:
"You verify speaker introductions for a live-captioning app. " +
"Given a transcript line and a candidate name, decide whether the candidate " +
"is genuinely a person's name being introduced (their own or someone else's), " +
"rather than an ordinary word, feeling, or phrase. " +
'Answer with exactly one word: "YES" or "NO".',
messages: [
{
role: "user",
content:
`Transcript line: "${sentence.trim()}"\n` +
`Candidate name: "${inference.name}" ` +
`(detected as a ${inference.kind === "self" ? "self-introduction" : "third-party introduction"}).\n` +
`Is "${inference.name}" a person's name being introduced in this line?`,
},
],
maxTokens: 8,
onDone: (result) => {
clearTimeout(timer);
const text = result.text.trim().toUpperCase();
if (/\bYES\b/.test(text)) finish(true);
else if (/\bNO\b/.test(text)) finish(false);
else finish(null);
},
onError: () => {
clearTimeout(timer);
finish(null);
},
});
} catch (error) {
clearTimeout(timer);
console.warn(`name verification failed: ${error}`);
finish(null);
}
});
}
+137
View File
@@ -0,0 +1,137 @@
// Speaker-dot tracking for the Sonic Radar. Dots live in the world frame
// (compass-anchored) so they stay put while the wearer's head turns; the
// people they mark can wander, so each confirmed utterance re-fixes its
// speaker's dot, and while a voice is actively being heard the nearest dot
// glides with the live direction-of-arrival. Pure TypeScript; the session
// engine owns the wall clock and passes `nowMs` in.
export type SpeakerDot = {
speakerId: number;
name: string;
color: string;
worldAngleDeg: number;
/** Wall time this speaker was last heard (confirmed or live). */
lastSpokenMs: number;
/** True while this speaker's voice is currently being heard. */
speaking: boolean;
};
/** How long a silent speaker stays on the radar (fading all the while). */
export const SPEAKER_DOT_TTL_MS = 90_000;
/** A confirmed utterance within this arc of a dot refines it; beyond, they moved. */
const SNAP_DEG = 45;
/** Live DOA within this arc of a dot is attributed to that speaker. */
const MATCH_DEG = 30;
/** Smoothing for utterance-confirmed position updates. */
const MOVE_SMOOTHING = 0.5;
/** Smoothing for packet-rate live tracking while they talk (and walk). */
const LIVE_SMOOTHING = 0.25;
const IDLE_BRIGHTNESS_MAX = 220;
const IDLE_BRIGHTNESS_MIN = 80;
/** Wrap to (-180, 180]. */
export function normalizeDeg(deg: number): number {
let value = deg % 360;
if (value > 180) value -= 360;
if (value <= -180) value += 360;
return value;
}
/** Angle EMA that follows the shorter way around the circle. */
export function smoothAngleDeg(previous: number | null, next: number, alpha: number): number {
if (previous === null) return normalizeDeg(next);
const delta = normalizeDeg(next - previous);
return normalizeDeg(previous + delta * alpha);
}
export type SpeakerIdentity = { speakerId: number; name: string; color: string };
/**
* A captioned utterance was attributed to a speaker: fix their dot at the
* utterance's bearing. Small changes smooth (jittery DOA); large ones snap
* (the person actually moved). Returns the updated dot list.
*/
export function placeDotForUtterance(
dots: SpeakerDot[],
who: SpeakerIdentity,
worldAngleDeg: number,
nowMs: number,
): SpeakerDot[] {
const existing = dots.find((dot) => dot.speakerId === who.speakerId);
if (!existing) {
return [
...dots,
{
...who,
worldAngleDeg: normalizeDeg(worldAngleDeg),
lastSpokenMs: nowMs,
speaking: false,
},
];
}
const delta = normalizeDeg(worldAngleDeg - existing.worldAngleDeg);
existing.worldAngleDeg =
Math.abs(delta) > SNAP_DEG
? normalizeDeg(worldAngleDeg)
: smoothAngleDeg(existing.worldAngleDeg, worldAngleDeg, MOVE_SMOOTHING);
existing.name = who.name;
existing.color = who.color;
existing.lastSpokenMs = nowMs;
existing.speaking = false;
return dots;
}
/**
* Speech is being heard right now from `worldAngleDeg`: mark the nearest dot
* within MATCH_DEG as actively speaking and glide it toward the live bearing,
* so a talker walking around drags their dot with them. Everyone else stops
* "speaking". Returns whether a dot matched.
*/
export function trackSpeakingDot(dots: SpeakerDot[], worldAngleDeg: number, nowMs: number): boolean {
let nearest: SpeakerDot | null = null;
let nearestDelta = Infinity;
for (const dot of dots) {
const delta = Math.abs(normalizeDeg(worldAngleDeg - dot.worldAngleDeg));
if (delta < nearestDelta) {
nearestDelta = delta;
nearest = dot;
}
}
for (const dot of dots) dot.speaking = false;
if (!nearest || nearestDelta > MATCH_DEG) return false;
nearest.speaking = true;
nearest.lastSpokenMs = nowMs;
nearest.worldAngleDeg = smoothAngleDeg(nearest.worldAngleDeg, worldAngleDeg, LIVE_SMOOTHING);
return true;
}
/** No voice is being heard: nobody is actively speaking. */
export function clearSpeaking(dots: SpeakerDot[]): void {
for (const dot of dots) dot.speaking = false;
}
/** Drop dots whose speakers have been silent past the TTL. */
export function pruneDots(
dots: SpeakerDot[],
nowMs: number,
ttlMs: number = SPEAKER_DOT_TTL_MS,
): SpeakerDot[] {
return dots.filter((dot) => nowMs - dot.lastSpokenMs < ttlMs);
}
/**
* Dot brightness: full while actively speaking, then fading with silence age
* from IDLE_BRIGHTNESS_MAX down to IDLE_BRIGHTNESS_MIN at the TTL.
*/
export function dotBrightness(
dot: SpeakerDot,
nowMs: number,
ttlMs: number = SPEAKER_DOT_TTL_MS,
): number {
if (dot.speaking) return 255;
const age = Math.max(0, nowMs - dot.lastSpokenMs);
const fraction = Math.min(1, age / ttlMs);
return Math.round(IDLE_BRIGHTNESS_MAX - (IDLE_BRIGHTNESS_MAX - IDLE_BRIGHTNESS_MIN) * fraction);
}
+223
View File
@@ -0,0 +1,223 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../../ui/gestures";
import { Layer, type DashboardInputEvent, type LayerContext } from "../../ui/layers";
import { micSession, type MicSessionState } from "./mic-session";
import { dotBrightness } from "./radar-dots";
/**
* The Sonic Radar: a top-down view around the wearer's head. The shaded
* wedge is the listening beam (scroll rotates it, click locks it onto the
* detected talker); the bright tick is the live direction-of-arrival from
* the glasses DSP; speaker dots mark who was last heard where, labeled with
* the speaker id or their name once tagged. With a compass heading the whole
* scene is world-anchored (an N tick marks north and dots stay put as the
* head turns); dots glow full-bright while their speaker is talking and fade
* with the time since they last spoke.
*/
export class RadarLayer implements Layer {
private state: MicSessionState = micSession.getState();
private unsubscribe: (() => void) | null = null;
start(requestRender: () => void): void {
this.unsubscribe = micSession.onState((state) => {
this.state = state;
requestRender();
});
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultSmallFont();
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const state = this.state;
const centerX = (width / 2) | 0;
const centerY = ((height - 24) / 2 + 6) | 0;
const radius = Math.min(centerX - 90, ((height - 40) / 2) | 0);
image.drawText(font, 12, 6, "Sonic Radar", 220);
const modeText = state.mode === "extended" ? "4-mic array" : "firmware DoA";
image.drawText(font, width - font.measureText(modeText) - 12, 6, modeText, 110);
// Beam wedge (device frame; the head is the reference for display).
const beamDeviceDeg = this.deviceBeamDeg(state);
const halfWidth = state.beamWidthDeg / 2;
if (state.beamFilterOn || state.beamLocked) {
for (let offset = -halfWidth; offset <= halfWidth; offset += 1.5) {
const angle = ((beamDeviceDeg + offset - 90) * Math.PI) / 180;
image.drawLine(
centerX,
centerY,
centerX + Math.round(Math.cos(angle) * radius),
centerY + Math.round(Math.sin(angle) * radius),
26,
);
}
}
// Beam edges stay visible even when the fill is off.
for (const edge of [-halfWidth, halfWidth]) {
const angle = ((beamDeviceDeg + edge - 90) * Math.PI) / 180;
image.drawLine(
centerX,
centerY,
centerX + Math.round(Math.cos(angle) * radius),
centerY + Math.round(Math.sin(angle) * radius),
70,
);
}
// Radar rings.
drawCircle(image, centerX, centerY, radius, 60);
drawCircle(image, centerX, centerY, (radius / 2) | 0, 30);
image.drawText(font, centerX - (font.measureText("front") / 2) | 0, centerY - radius - 14, "front", 90);
// Compass north tick: anchors the world frame visually as the head turns.
if (state.headingDeg !== null) {
const northAngle = ((this.worldToDeviceDeg(0, state) - 90) * Math.PI) / 180;
const cos = Math.cos(northAngle);
const sin = Math.sin(northAngle);
image.drawLine(
centerX + Math.round(cos * (radius - 6)),
centerY + Math.round(sin * (radius - 6)),
centerX + Math.round(cos * (radius + 2)),
centerY + Math.round(sin * (radius + 2)),
200,
);
image.drawText(
font,
centerX + Math.round(cos * (radius + 10)) - ((font.measureText("N") / 2) | 0),
centerY + Math.round(sin * (radius + 10)) - ((font.lineHeight / 2) | 0),
"N",
200,
);
}
// Head marker.
image.fillRoundedRect(centerX - 4, centerY - 5, 8, 10, 160, 3);
// Live direction-of-arrival tick.
if (state.doaDeviceDeg !== null && state.ssr > 0) {
const angle = ((state.doaDeviceDeg - 90) * Math.PI) / 180;
const tipX = centerX + Math.cos(angle) * radius;
const tipY = centerY + Math.sin(angle) * radius;
image.drawLine(
centerX + Math.round(Math.cos(angle) * (radius * 0.55)),
centerY + Math.round(Math.sin(angle) * (radius * 0.55)),
Math.round(tipX),
Math.round(tipY),
255,
);
}
// Speaker dots with id/name labels: full-bright (and ringed) while that
// person is actively speaking, fading with silence age otherwise.
const now = Date.now();
for (const dot of state.speakerDots) {
const deviceDeg = this.worldToDeviceDeg(dot.worldAngleDeg, state);
const angle = ((deviceDeg - 90) * Math.PI) / 180;
const dotX = centerX + Math.round(Math.cos(angle) * radius * 0.8);
const dotY = centerY + Math.round(Math.sin(angle) * radius * 0.8);
const value = dotBrightness(dot, now);
if (dot.speaking) {
image.fillRoundedRect(dotX - 4, dotY - 4, 9, 9, value, 4);
drawCircle(image, dotX, dotY, 7, 140);
} else {
image.fillRoundedRect(dotX - 3, dotY - 3, 7, 7, value, 3);
}
const label = dot.name;
const labelValue = Math.max(90, value - 50);
const labelX = dotX + 8 + font.measureText(label) > width ? dotX - font.measureText(label) - 8 : dotX + 8;
image.drawText(font, labelX, dotY - 6, label, labelValue);
}
// Side panel: beam + level readouts.
const panelX = width - 116;
let panelY = 26;
image.drawText(
font,
panelX,
panelY,
state.headingDeg !== null ? `Heading ${Math.round(state.headingDeg)}°` : "No compass",
state.headingDeg !== null ? 150 : 90,
);
panelY += 16;
image.drawText(font, panelX, panelY, `Beam ${Math.round(state.beamWorldDeg)}°`, 170);
panelY += 16;
image.drawText(font, panelX, panelY, `Width ${Math.round(state.beamWidthDeg)}°`, 130);
panelY += 16;
image.drawText(font, panelX, panelY, state.beamLocked ? "Locked on talker" : "Manual aim", 130);
panelY += 16;
if (state.doaDeviceDeg !== null) {
image.drawText(font, panelX, panelY, `Heard ${Math.round(state.doaDeviceDeg)}°`, 200);
panelY += 16;
}
image.drawText(font, panelX, panelY, state.beamFilterOn ? "Filter: beam only" : "Filter: off", 130);
const footerY = height - font.lineHeight - 4;
image.drawText(
font,
12,
footerY,
`${GESTURE_SCROLL} aim ${GESTURE_CLICK} lock ${GESTURE_DOUBLE_CLICK} back`,
110,
);
return image;
}
private deviceBeamDeg(state: MicSessionState): number {
if (state.headingDeg === null) return state.beamWorldDeg;
return normalize(state.beamWorldDeg - state.headingDeg);
}
private worldToDeviceDeg(worldDeg: number, state: MicSessionState): number {
if (state.headingDeg === null) return normalize(worldDeg);
return normalize(worldDeg - state.headingDeg);
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-up":
micSession.setBeamWorldDeg(this.state.beamWorldDeg - 15);
return;
case "scroll-down":
micSession.setBeamWorldDeg(this.state.beamWorldDeg + 15);
return;
case "click":
micSession.toggleBeamLock();
return;
case "long-press":
micSession.adjustBeamWidth(30 * (this.state.beamWidthDeg >= 180 ? -5 : 1));
return;
case "double-click":
ctx.stack.pop();
return;
default:
return;
}
}
onRemoved(): void {
this.unsubscribe?.();
this.unsubscribe = null;
}
}
function normalize(deg: number): number {
let value = deg % 360;
if (value > 180) value -= 360;
if (value < -180) value += 360;
return value;
}
function drawCircle(image: GrayImage, centerX: number, centerY: number, radius: number, value: number): void {
const steps = Math.max(48, radius * 4);
for (let step = 0; step < steps; step++) {
const angle = (step / steps) * 2 * Math.PI;
image.setPixel(
centerX + Math.round(Math.cos(angle) * radius),
centerY + Math.round(Math.sin(angle) * radius),
value,
);
}
}
+199
View File
@@ -0,0 +1,199 @@
import { Utils } from "@nativescript/core";
import { toUint8Array } from "../../util/array-util";
import { micModelPath } from "./mic-models";
import { SPEAKER_KNOWN_THRESHOLD, encodeEmbedding, speakerRegistry } from "./speakers";
declare const com: any;
declare const android: any;
/**
* Re-diarize a saved conversation: run offline speaker diarization (pyannote
* segmentation + speaker embeddings + clustering) over the session's
* recording, then reconcile the resulting speaker turns against the stored
* transcript segments — matching each cluster back to an enrolled speaker by
* voice-print, or minting a new speaker when no one matches.
*/
export type RediarizeResult = {
turns: number;
clusters: number;
reassignedSegments: number;
newSpeakers: number;
};
export type RediarizeCallbacks = {
onProgress: (fraction: number) => void;
onDone: (result: RediarizeResult) => void;
onError: (message: string) => void;
};
type Turn = { startMs: number; endMs: number; cluster: number };
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
export function rediarizeSession(
sessionId: number,
options: { numSpeakers?: number; threshold?: number },
callbacks: RediarizeCallbacks,
): void {
const audioPath = String(store().getSessionAudioPath(sessionId));
if (!audioPath) {
callbacks.onError("This conversation has no saved recording to re-analyze.");
return;
}
const segmentationModel = micModelPath("diarization-segmentation");
const embeddingModel = micModelPath("speaker-embedding");
if (!segmentationModel || !embeddingModel) {
callbacks.onError("Download the re-diarization and speaker voice models first (Microphones > Voice & speakers).");
return;
}
const listener = new com.faceclaw.app.FaceclawDiarizer.Listener({
onProgress: (progress: number) => callbacks.onProgress(Number(progress)),
onDone: (turnsJson: string) => {
try {
const turns = JSON.parse(String(turnsJson)) as Turn[];
callbacks.onDone(applyTurns(sessionId, turns));
} catch (error) {
callbacks.onError(`Applying diarization failed: ${String(error)}`);
}
},
onError: (message: string) => callbacks.onError(String(message)),
});
const diarizer = new com.faceclaw.app.FaceclawDiarizer(
segmentationModel,
embeddingModel,
audioPath,
options.numSpeakers ?? -1,
options.threshold ?? 0.5,
listener,
);
diarizer.start();
}
type StoredSegment = {
id: number;
speakerId: number | null;
startedAt: number;
endedAt: number;
audioOffsetMs: number | null;
};
function applyTurns(sessionId: number, turns: Turn[]): RediarizeResult {
const segments = (JSON.parse(String(store().querySegments(sessionId))) as Array<Record<string, unknown>>).map(
(row): StoredSegment => ({
id: Number(row.id),
speakerId: row.speakerId === null ? null : Number(row.speakerId),
startedAt: Number(row.startedAt),
endedAt: Number(row.endedAt),
audioOffsetMs: row.audioOffsetMs === null ? null : Number(row.audioOffsetMs),
}),
);
const embeddings = new Map<number, Float32Array>();
for (const row of JSON.parse(String(store().querySegmentEmbeddings(sessionId))) as Array<
Record<string, unknown>
>) {
const decoded = decodeBase64Floats(String(row.embeddingBase64 ?? ""));
if (decoded) embeddings.set(Number(row.id), decoded);
}
// Assign each transcript segment to the diarization cluster it overlaps most.
const segmentClusters = new Map<number, number>();
for (const segment of segments) {
if (segment.audioOffsetMs === null) continue;
const startMs = segment.audioOffsetMs;
const endMs = startMs + (segment.endedAt - segment.startedAt);
let bestCluster = -1;
let bestOverlap = 0;
for (const turn of turns) {
const overlap = Math.min(endMs, turn.endMs) - Math.max(startMs, turn.startMs);
if (overlap > bestOverlap) {
bestOverlap = overlap;
bestCluster = turn.cluster;
}
}
if (bestCluster >= 0) segmentClusters.set(segment.id, bestCluster);
}
// Centroid per cluster from the stored per-segment voice-prints.
const clusterEmbeddings = new Map<number, Float32Array[]>();
for (const [segmentId, cluster] of segmentClusters) {
const embedding = embeddings.get(segmentId);
if (!embedding) continue;
const list = clusterEmbeddings.get(cluster) ?? [];
list.push(embedding);
clusterEmbeddings.set(cluster, list);
}
speakerRegistry.reload();
const clusterSpeaker = new Map<number, number>();
let newSpeakers = 0;
for (const [cluster, prints] of clusterEmbeddings) {
const centroid = meanEmbedding(prints);
if (!centroid) continue;
const best = speakerRegistry.bestMatch(centroid);
if (best && best.similarity >= SPEAKER_KNOWN_THRESHOLD) {
clusterSpeaker.set(cluster, best.profile.id);
} else {
const id = Number(store().createSpeaker(nextAutoName(), "#90A4AE", false, encodeEmbedding(centroid)));
speakerRegistry.reload();
clusterSpeaker.set(cluster, id);
newSpeakers += 1;
}
}
let reassigned = 0;
for (const segment of segments) {
const cluster = segmentClusters.get(segment.id);
if (cluster === undefined) continue;
const speakerId = clusterSpeaker.get(cluster);
if (speakerId === undefined || speakerId === segment.speakerId) continue;
store().reassignSegmentSpeaker(segment.id, speakerId);
reassigned += 1;
}
speakerRegistry.reload();
return {
turns: turns.length,
clusters: clusterEmbeddings.size,
reassignedSegments: reassigned,
newSpeakers,
};
}
function nextAutoName(): string {
const taken = new Set(speakerRegistry.all().map((profile) => profile.name));
for (let index = 1; ; index++) {
const candidate = `Speaker ${index}`;
if (!taken.has(candidate)) return candidate;
}
}
function meanEmbedding(prints: Float32Array[]): Float32Array | null {
if (prints.length === 0) return null;
const mean = new Float32Array(prints[0]!.length);
for (const print of prints) {
if (print.length !== mean.length) continue;
for (let i = 0; i < mean.length; i++) mean[i] += print[i]!;
}
let norm = 0;
for (let i = 0; i < mean.length; i++) norm += mean[i]! * mean[i]!;
norm = Math.sqrt(norm);
if (norm > 0) {
for (let i = 0; i < mean.length; i++) mean[i] /= norm;
}
return mean;
}
function decodeBase64Floats(base64: string): Float32Array | null {
if (!base64) return null;
try {
const bytes = toUint8Array(android.util.Base64.decode(base64, android.util.Base64.NO_WRAP));
if (bytes.length < 4 || bytes.length % 4 !== 0) return null;
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.length));
} catch {
return null;
}
}
+405
View File
@@ -0,0 +1,405 @@
// Deterministic sentiment analysis: a keyword emotion classifier (with
// negation scoping and confidence calibration) plus hand-lexicon polarity
// scoring. No platform NLP dependency — the deterministic tiers carry the
// whole signal.
import { NON_WORD_RUN } from "../../util/unicode-class";
export type EmotionLabel =
| "happy"
| "sad"
| "angry"
| "frustrated"
| "excited"
| "anxious"
| "surprised"
| "confident"
| "confused"
| "disgusted"
| "embarrassed"
| "relieved"
| "bored"
| "calm"
| "neutral";
export type SentimentBucket = "veryPositive" | "positive" | "neutral" | "negative" | "veryNegative";
export type EmotionDetection = {
emotion: EmotionLabel;
confidence: number;
source: "keyword" | "lexical" | "none";
};
// Declaration order matters twice: EMOTION_ORDER breaks ranking ties, and
// EMOTION_PATTERNS is evaluated in table order.
const EMOTION_ORDER: EmotionLabel[] = [
"happy", "sad", "angry", "frustrated", "excited", "anxious", "surprised",
"confident", "confused", "disgusted", "embarrassed", "relieved", "bored",
"calm", "neutral",
];
// Terms users commonly use when searching for each emotion. The classifier
// matches these too, so storage and retrieval speak the same vocabulary.
const SEARCH_ALIASES: Record<EmotionLabel, string[]> = {
happy: ["happy", "happiness", "glad", "pleased", "joy", "joyful", "cheerful", "delighted"],
sad: ["sad", "sadness", "unhappy", "heartbroken", "depressed", "grief", "somber", "upset"],
angry: ["angry", "anger", "mad", "furious", "livid", "irate", "outraged", "rage", "hostile"],
frustrated: ["frustrated", "frustration", "annoyed", "annoying", "irritated", "irritating", "fed up"],
excited: ["excited", "excitement", "enthusiastic", "thrilled", "pumped", "stoked"],
anxious: ["anxious", "anxiety", "worried", "worry", "nervous", "afraid", "scared", "stressed", "panic"],
surprised: ["surprised", "surprise", "shocked", "unexpected", "astonished", "amazed"],
confident: ["confident", "confidence", "certain", "assured", "sure", "determined"],
confused: ["confused", "confusion", "unclear", "uncertain", "puzzled", "lost"],
disgusted: ["disgusted", "disgust", "repulsed", "revolted", "sickened", "revolting"],
embarrassed: ["embarrassed", "embarrassment", "ashamed", "humiliated", "mortified", "self-conscious", "awkward"],
relieved: ["relieved", "relief", "reassured", "thank goodness", "weight off my shoulders"],
bored: ["bored", "boredom", "boring", "uninterested", "tedious", "zoned out"],
calm: ["calm", "calmness", "relaxed", "peaceful", "at ease", "settled"],
neutral: ["neutral"],
};
// High-precision subset suitable for silently turning natural language into
// an exact filter. Ambiguous terms such as "sure", "lost", "miss", and "mad"
// remain indexed search aliases but never hijack ordinary searches.
const INFERRED_FILTER_ALIASES: Record<EmotionLabel, string[]> = {
happy: ["happy", "happiness", "glad", "joy", "joyful", "cheerful", "delighted"],
sad: ["sad", "sadness", "unhappy", "heartbroken", "depressed", "grief", "somber"],
angry: ["angry", "anger", "furious", "livid", "irate", "outraged", "rage", "hostile"],
frustrated: ["frustrated", "frustration", "annoyed", "irritated", "fed up"],
excited: ["excited", "excitement", "enthusiastic", "thrilled", "pumped", "stoked"],
anxious: ["anxious", "anxiety", "worried", "nervous", "afraid", "scared", "panic"],
surprised: ["surprised", "surprise", "shocked", "astonished"],
confident: ["confident", "confidence", "assured"],
confused: ["confused", "confusion", "puzzled"],
disgusted: ["disgusted", "disgust", "repulsed", "revolted", "sickened"],
embarrassed: ["embarrassed", "embarrassment", "ashamed", "humiliated", "mortified"],
relieved: ["relieved", "relief", "reassured"],
bored: ["bored", "boredom", "uninterested"],
calm: ["calm", "calmness", "relaxed"],
neutral: [],
};
const EMOTION_PATTERNS: Array<[EmotionLabel, string[]]> = [
["happy", ["happy", "happier", "glad", "pleased", "wonderful", "great", "awesome", "fantastic", "love", "enjoy", "thankful", "grateful", "delighted", "joyful", "cheerful"]],
["excited", ["excited", "enthusiastic", "amazing", "incredible", "can't wait", "thrilled", "pumped", "stoked", "wow", "unbelievable", "blown away"]],
["sad", ["sad", "unhappy", "unfortunately", "miss", "sorry", "regret", "disappointing", "heartbreaking", "crying", "depressed", "grief", "mourn"]],
["angry", ["angry", "mad", "furious", "livid", "irate", "outraged", "rage", "hostile", "unacceptable", "ridiculous", "hate", "damn", "pissed", "how dare you", "enough is enough", "what the hell", "this is bullshit", "you never listen", "i have had enough", "stop it right now"]],
["frustrated", ["frustrated", "annoyed", "annoying", "irritated", "irritating", "stuck", "struggling", "can't figure", "doesn't work", "broken", "keeps failing", "fed up", "tired of", "sick of", "waste of time", "again and again", "why does this keep"]],
["anxious", ["worried", "anxious", "nervous", "concerned", "afraid", "scared", "uneasy", "stressed", "stress", "panic", "overwhelmed", "overwhelm", "dread"]],
["surprised", ["surprised", "astonished", "unexpected", "didn't expect", "shocking", "no way", "whoa"]],
["confident", ["confident", "certain", "sure", "guaranteed", "no doubt", "convinced"]],
["confused", ["confused", "don't understand", "unclear", "what do you mean", "lost", "lost me", "makes no sense", "wait what", "i'm not sure", "puzzled"]],
["disgusted", ["disgusted", "repulsed", "revolted", "sickened", "that's disgusting", "that is disgusting", "makes me sick", "turns my stomach"]],
["embarrassed", ["embarrassed", "ashamed", "humiliated", "mortified", "self-conscious", "i could die of embarrassment"]],
["relieved", ["relieved", "what a relief", "thank goodness", "thank god", "weight off my shoulders", "can breathe again"]],
["bored", ["bored", "uninterested", "zoned out", "lost interest", "couldn't care less"]],
["calm", ["calm", "relaxed", "peaceful", "at ease", "settled", "untroubled"]],
];
const EMOTION_SEARCH_PATTERNS: Array<[EmotionLabel, string[]]> = EMOTION_PATTERNS.map(
([emotion, patterns]) => [emotion, dedupe(patterns.concat(SEARCH_ALIASES[emotion]))],
);
function dedupe(values: string[]): string[] {
return Array.from(new Set(values));
}
// These lexicons carry the whole polarity signal, so both sets are extended
// (see below).
const POSITIVE_LEXICON = new Set<string>([
"amazing", "awesome", "best", "excellent", "fantastic", "glad", "great", "happy",
"love", "outstanding", "perfect", "pleased", "positive", "relieved", "thankful", "wonderful",
// Extension: common polarity words added so the lexicons carry the whole
// polarity signal on their own.
"beautiful", "better", "brilliant", "delighted", "enjoy", "enjoyed", "excited",
"fun", "good", "grateful", "helpful", "hopeful", "impressive", "joy", "lovely",
"nice", "proud", "success", "successful", "thrilled",
]);
const NEGATIVE_LEXICON = new Set<string>([
"angry", "ashamed", "awful", "bad", "bored", "disappointed", "disappointing",
"disgusted", "embarrassed", "frustrated", "hate", "horrible", "negative", "sad",
"terrible", "unacceptable", "unhappy", "upset", "worried", "worse",
// Extension: see the note on POSITIVE_LEXICON.
"annoyed", "annoying", "anxious", "broken", "crying", "depressed", "dreadful",
"failed", "failing", "furious", "hurt", "miserable", "nasty", "painful", "poor",
"scared", "stressed", "useless", "worst", "wrong",
]);
const NEGATION_TOKENS = new Set<string>([
"not", "never", "no", "neither", "nor", "hardly", "without", "cannot", "cant",
]);
// Contrastive conjunctions end a negation scope: "not happy, but angry"
// expresses anger. Coordinating "and" is intentionally excluded because
// "not angry and frustrated" often negates both states.
const NEGATION_SCOPE_BOUNDARY_TOKENS = new Set<string>([
"but", "however", "though", "yet",
]);
function emotionTokens(text: string): string[] {
// Preserve the semantic negation before punctuation/apostrophes are stripped.
const normalized = text
.toLowerCase()
.replace(/’/g, "'")
.replace(/can't/g, "cannot")
.replace(/i'm/g, "i am")
.replace(/n't/g, " not");
return normalized.split(NON_WORD_RUN).filter((token) => token.length > 0);
}
function emotionCueIsNegated(start: number, tokens: string[]): boolean {
if (start <= 0) return false;
const windowStart = Math.max(0, start - 3);
for (let i = start - 1; i >= windowStart; i--) {
if (NEGATION_SCOPE_BOUNDARY_TOKENS.has(tokens[i])) {
// Only a negation between the boundary and the cue still applies.
for (let j = i + 1; j < start; j++) {
if (NEGATION_TOKENS.has(tokens[j])) return true;
}
return false;
}
}
for (let i = windowStart; i < start; i++) {
if (NEGATION_TOKENS.has(tokens[i])) return true;
}
return false;
}
function matchesAt(patternTokens: string[], start: number, tokens: string[]): boolean {
for (let i = 0; i < patternTokens.length; i++) {
if (tokens[start + i] !== patternTokens[i]) return false;
}
return true;
}
// Match whole words/phrases, not substrings ("mad" must not match "made"),
// and ignore explicitly negated cues ("I am not angry").
function nonNegatedMatchCount(pattern: string, tokens: string[]): number {
const patternTokens = emotionTokens(pattern);
if (patternTokens.length === 0 || patternTokens.length > tokens.length) return 0;
let count = 0;
for (let start = 0; start <= tokens.length - patternTokens.length; start++) {
if (!matchesAt(patternTokens, start, tokens)) continue;
if (emotionCueIsNegated(start, tokens)) continue;
count++;
}
return count;
}
function containsNonNegatedPattern(pattern: string, tokens: string[]): boolean {
const patternTokens = emotionTokens(pattern);
if (patternTokens.length === 0 || patternTokens.length > tokens.length) return false;
for (let start = 0; start <= tokens.length - patternTokens.length; start++) {
if (!matchesAt(patternTokens, start, tokens)) continue;
if (emotionCueIsNegated(start, tokens)) continue;
return true;
}
return false;
}
function detectEmotionWithScore(text: string, sentimentScore: number): EmotionDetection {
const tokens = emotionTokens(text);
// Stable ordering makes runner-up selection reproducible, while equal
// evidence is handled as ambiguity below rather than guessed.
const ranked: Array<[EmotionLabel, number]> = [];
for (const [emotion, patterns] of EMOTION_SEARCH_PATTERNS) {
let score = 0;
for (const pattern of patterns) {
score += nonNegatedMatchCount(pattern, tokens);
}
if (score > 0) ranked.push([emotion, score]);
}
ranked.sort((a, b) => {
if (a[1] !== b[1]) return b[1] - a[1];
return EMOTION_ORDER.indexOf(a[0]) - EMOTION_ORDER.indexOf(b[0]);
});
if (ranked.length > 0) {
const [bestEmotion, bestScore] = ranked[0];
const runnerUpScore = ranked.length > 1 ? ranked[1][1] : 0;
if (bestScore - runnerUpScore === 0) {
// Equally strong cues for different emotions are genuinely ambiguous;
// picking one would make mixed statements depend on declaration order.
return { emotion: "neutral", confidence: 0.40, source: "none" };
}
const confidence =
bestScore >= 2
? Math.min(0.98, 0.88 + (bestScore - 2) * 0.03)
: // A single cue can be quoted, topical, idiomatic, or sarcastic.
0.70;
return { emotion: bestEmotion, confidence, source: "keyword" };
}
// Polarity is not an emotion taxonomy: a negative score cannot distinguish
// anger, sadness, fear, or frustration. Keep the categorical label neutral
// and expose the independent score for search.
if (Math.abs(sentimentScore) < 0.10) {
return { emotion: "neutral", confidence: 0.76, source: "none" };
}
return { emotion: "neutral", confidence: 0.46, source: "lexical" };
}
export function detectEmotion(text: string): EmotionDetection {
return detectEmotionWithScore(text, scoreSentiment(text));
}
// Score a single text string in [-1, 1]. Sentences are scored independently
// and averaged; the lexicon delta per sentence is normalized by sentence
// length.
export function scoreSentiment(text: string): number {
const sentences = text
.split(/[.!?]+/)
.map((sentence) => sentence.trim())
.filter((sentence) => sentence.length > 0);
if (sentences.length === 0) return 0;
const scores: number[] = [];
for (const sentence of sentences) {
const tokens = emotionTokens(sentence);
if (tokens.length === 0) continue;
let delta = 0;
for (let i = 0; i < tokens.length; i++) {
let polarity = 0;
if (POSITIVE_LEXICON.has(tokens[i])) polarity = 1;
else if (NEGATIVE_LEXICON.has(tokens[i])) polarity = -1;
if (polarity === 0) continue;
if (emotionCueIsNegated(i, tokens)) polarity = -polarity;
delta += polarity;
}
if (delta === 0) {
scores.push(0);
continue;
}
const normalized = delta / Math.max(Math.floor(tokens.length / 3), 1);
scores.push(Math.max(-1, Math.min(1, normalized)));
}
if (scores.length === 0) return 0;
const average = scores.reduce((total, score) => total + score, 0) / scores.length;
return Math.max(-1, Math.min(1, average));
}
export function sentimentBucket(score: number): SentimentBucket {
if (score > 0.3) return "veryPositive";
if (score > 0.1) return "positive";
if (score < -0.3) return "veryNegative";
if (score < -0.1) return "negative";
return "neutral";
}
export type AnalyzedLine = {
score: number;
emotion: EmotionLabel;
emotionConfidence: number;
};
export function analyzeLine(text: string): AnalyzedLine {
const score = scoreSentiment(text);
const detection = detectEmotionWithScore(text, score);
return { score, emotion: detection.emotion, emotionConfidence: detection.confidence };
}
export type SentimentPoint = {
index: number;
score: number;
emotion: EmotionLabel;
};
export type SentimentSummary = {
averageScore: number;
bucket: SentimentBucket;
bucketCounts: Record<SentimentBucket, number>;
emotionDistribution: Partial<Record<EmotionLabel, number>>;
trend: SentimentPoint[];
};
const MAX_TREND_POINTS = 60;
export function summarize(lines: AnalyzedLine[]): SentimentSummary {
const bucketCounts: Record<SentimentBucket, number> = {
veryPositive: 0,
positive: 0,
neutral: 0,
negative: 0,
veryNegative: 0,
};
const emotionDistribution: Partial<Record<EmotionLabel, number>> = {};
if (lines.length === 0) {
return { averageScore: 0, bucket: "neutral", bucketCounts, emotionDistribution, trend: [] };
}
let total = 0;
for (const line of lines) {
total += line.score;
bucketCounts[sentimentBucket(line.score)]++;
emotionDistribution[line.emotion] = (emotionDistribution[line.emotion] ?? 0) + 1;
}
const averageScore = total / lines.length;
// Downsample by averaging within evenly-spaced chunks, keeping the most
// extreme emotion per chunk so spikes stay visible on the chart. The chunk
// width rounds up so the trend never exceeds MAX_TREND_POINTS.
const trend: SentimentPoint[] = [];
const step = Math.max(1, Math.ceil(lines.length / MAX_TREND_POINTS));
for (let i = 0; i < lines.length; i += step) {
const chunk = lines.slice(i, i + step);
const chunkTotal = chunk.reduce((sum, line) => sum + line.score, 0);
let extreme = chunk[0];
for (const line of chunk) {
if (Math.abs(line.score) > Math.abs(extreme.score)) extreme = line;
}
trend.push({ index: i, score: chunkTotal / chunk.length, emotion: extreme.emotion });
}
return {
averageScore,
bucket: sentimentBucket(averageScore),
bucketCounts,
emotionDistribution,
trend,
};
}
// Human-readable metadata for the transcript search index. Keeping the alias
// mapping beside the classifier prevents an analysis label and its retrieval
// aliases from drifting apart.
export function searchableMetadata(emotion: EmotionLabel, score: number): string {
const emotionTerms = [emotion as string].concat(SEARCH_ALIASES[emotion]);
let sentimentTerms: string[];
switch (sentimentBucket(score)) {
case "veryPositive":
sentimentTerms = ["very positive", "positive sentiment"];
break;
case "positive":
sentimentTerms = ["positive", "positive sentiment"];
break;
case "neutral":
sentimentTerms = ["neutral", "neutral sentiment"];
break;
case "negative":
sentimentTerms = ["negative", "negative sentiment"];
break;
case "veryNegative":
sentimentTerms = ["very negative", "negative sentiment"];
break;
}
return emotionTerms.concat(sentimentTerms).join(" ");
}
// Infer an exact emotion filter from ordinary search language. Returns null
// for no emotion or multiple distinct emotions so compound requests can fall
// back to normal ranked retrieval.
export function inferredEmotionFilter(query: string): EmotionLabel | null {
const tokens = emotionTokens(query);
const matches: EmotionLabel[] = [];
for (const emotion of EMOTION_ORDER) {
if (emotion === "neutral") continue;
const patterns = dedupe([emotion as string].concat(INFERRED_FILTER_ALIASES[emotion]));
if (patterns.some((pattern) => containsNonNegatedPattern(pattern, tokens))) {
matches.push(emotion);
}
}
return matches.length === 1 ? matches[0] : null;
}
+183
View File
@@ -0,0 +1,183 @@
import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { speakerRegistry } from "./speakers";
declare const com: any;
/**
* Per-speaker conversation insights, generated with the on-phone LLM when a
* caption session ends: a short recap of the conversation, the action items
* it produced, and a consolidated list of inferred facts about the person
* (fed by the transcript, the contact's tag/notes, and previously inferred
* facts). Stored on the speaker row in the conversation store, where the
* encounter popup and the People view read them back.
*
* Best-effort by design: no model, a busy runner, or an unparseable reply
* simply leaves the speaker's existing insights in place. Everything runs
* sequentially — the runner is shared with the assistant and caption name
* verification.
*/
/** Speakers with fewer transcript lines than this in the session are skipped. */
const MIN_SPEAKER_LINES = 2;
/** Transcript excerpt budget; Qwen runs with a 4096-token context. */
const MAX_TRANSCRIPT_CHARS = 4500;
const MAX_FACTS = 8;
const MAX_ACTION_ITEMS = 6;
const GENERATION_TIMEOUT_MS = 90_000;
type SegmentLine = { speakerId: number | null; text: string };
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
/** One generation at a time across sessions; a second session-end call queues behind the first. */
let chain: Promise<void> = Promise.resolve();
/**
* Generate and store insights for every non-wearer speaker heard in the
* session. Fire-and-forget: call after the session row is finished; errors
* only log.
*/
export function generateSessionInsights(sessionId: number): void {
if (!global.isAndroid || sessionId < 0 || !isLocalModelReady()) return;
chain = chain.then(() => generateForSession(sessionId)).catch((error) => {
console.warn(`speaker insights generation failed: ${error}`);
});
}
async function generateForSession(sessionId: number): Promise<void> {
const segments = (JSON.parse(String(store().querySegments(sessionId))) as Array<Record<string, unknown>>)
.map((row): SegmentLine => ({
speakerId: row.speakerId === null ? null : Number(row.speakerId),
text: String(row.text ?? "").trim(),
}))
.filter((line) => line.text);
if (!segments.length) return;
speakerRegistry.reload();
const lineCounts = new Map<number, number>();
for (const line of segments) {
if (line.speakerId !== null) {
lineCounts.set(line.speakerId, (lineCounts.get(line.speakerId) ?? 0) + 1);
}
}
const transcript = buildTranscript(segments);
for (const [speakerId, lines] of lineCounts) {
if (lines < MIN_SPEAKER_LINES) continue;
const profile = speakerRegistry.byId(speakerId);
if (!profile || profile.isWearer) continue;
try {
const insights = await generateForSpeaker(profile.name, profile.tag, profile.facts, transcript);
if (insights) {
speakerRegistry.setInsights(
speakerId,
insights.recap,
insights.actionItems.slice(0, MAX_ACTION_ITEMS),
insights.facts.slice(0, MAX_FACTS),
sessionId,
);
}
} catch (error) {
console.warn(`speaker insights failed for ${profile.name}: ${error}`);
}
}
}
function buildTranscript(segments: SegmentLine[]): string {
const names = new Map<number, string>();
for (const profile of speakerRegistry.all()) {
names.set(profile.id, profile.isWearer ? `${profile.name} (the wearer)` : profile.name);
}
const lines = segments.map((line) => {
const who = line.speakerId !== null ? (names.get(line.speakerId) ?? "Speaker") : "Speaker";
return `${who}: ${line.text}`;
});
let text = lines.join("\n");
if (text.length > MAX_TRANSCRIPT_CHARS) {
// Keep the end of the conversation: goodbyes and plans (the action items)
// cluster there, and the recap should reflect where things landed.
text = "..." + text.slice(text.length - MAX_TRANSCRIPT_CHARS);
}
return text;
}
type SpeakerInsights = { recap: string; actionItems: string[]; facts: string[] };
function generateForSpeaker(
name: string,
notes: string,
existingFacts: string[],
transcript: string,
): Promise<SpeakerInsights | null> {
return new Promise((resolve) => {
let settled = false;
const finish = (value: SpeakerInsights | null) => {
if (!settled) {
settled = true;
resolve(value);
}
};
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
try {
streamLocalQwen({
apiKey: "",
model: "local",
system:
"You maintain a contact's conversation memory for a live-captioning app. " +
"From the transcript, produce for the named person: a recap of this conversation " +
"(1-2 sentences), the concrete action items agreed or requested in it (things to do, " +
"send, or follow up on — empty if none), and an updated list of short facts about the " +
"person (job, family, interests, plans, preferences). Merge the previously known facts " +
"with anything new from the transcript; drop duplicates and anything now outdated. " +
"Never invent details that are not supported by the notes, known facts, or transcript. " +
'Reply with ONLY a JSON object: {"recap": "...", "actionItems": ["..."], "facts": ["..."]}.',
messages: [
{
role: "user",
content:
`Person: ${name}\n` +
(notes ? `Contact notes: ${notes}\n` : "") +
`Previously known facts: ${existingFacts.length ? JSON.stringify(existingFacts) : "(none)"}\n\n` +
`# Transcript\n${transcript}`,
},
],
maxTokens: 350,
onDone: (result) => {
clearTimeout(timer);
finish(parseInsights(result.text));
},
onError: (message) => {
clearTimeout(timer);
console.warn(`speaker insights model error: ${message}`);
finish(null);
},
});
} catch (error) {
clearTimeout(timer);
console.warn(`speaker insights request failed: ${error}`);
finish(null);
}
});
}
/** Parse the model's reply leniently: the first {...} block wins. */
export function parseInsights(reply: string): SpeakerInsights | null {
const start = reply.indexOf("{");
const end = reply.lastIndexOf("}");
if (start < 0 || end <= start) return null;
try {
const parsed = JSON.parse(reply.slice(start, end + 1)) as Record<string, unknown>;
const recap = String(parsed.recap ?? "").trim();
if (!recap) return null;
const list = (value: unknown): string[] =>
Array.isArray(value) ? value.map((item) => String(item).trim()).filter(Boolean) : [];
return { recap, actionItems: list(parsed.actionItems), facts: list(parsed.facts) };
} catch {
return null;
}
}
+311
View File
@@ -0,0 +1,311 @@
import { Utils } from "@nativescript/core";
import { encodeBase64 } from "../../native/cloud-stt";
import { toUint8Array } from "../../util/array-util";
import { micModelPath } from "./mic-models";
import { wearerCommandsOnlySetting } from "./mic-settings";
declare const com: any;
declare const android: any;
/**
* Speaker voice-print registry over FaceclawConversationStore. Matching is
* cosine similarity on L2-normalized embeddings: >= 0.80 auto-assign ("known"), 0.50-0.80 "uncertain" (assigned
* but flagged for review), below that a new speaker is enrolled. Centroids
* update as a capped running mean so one long session cannot drown an
* enrolled voice.
*/
export const SPEAKER_KNOWN_THRESHOLD = 0.8;
export const SPEAKER_UNCERTAIN_THRESHOLD = 0.5;
/** Cap on the running-mean weight of an existing centroid. */
export const CENTROID_UPDATE_CAP = 24;
const SPEAKER_COLORS = [
"#4FC3F7",
"#FFB74D",
"#81C784",
"#E57373",
"#BA68C8",
"#FFD54F",
"#4DB6AC",
"#F06292",
"#A1887F",
"#90A4AE",
];
export type SpeakerProfile = {
id: number;
name: string;
color: string;
isWearer: boolean;
embedding: Float32Array | null;
embeddingCount: number;
lastHeardAt: number;
tag: string;
segmentCount: number;
/** LLM recap of this speaker's most recent conversation ("" until one is generated). */
lastRecap: string;
/** Open action items from that conversation. */
actionItems: string[];
/** Accumulated inferred facts about this person. */
facts: string[];
insightsUpdatedAt: number;
insightsSessionId: number;
};
export type SpeakerMatch = {
profile: SpeakerProfile;
similarity: number;
classification: "known" | "uncertain" | "new";
/**
* The profile's last_heard_at BEFORE this utterance updated it — i.e. when
* this person's voice was previously heard. 0 for a newly enrolled speaker.
*/
previousLastHeardAt: number;
};
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
/** Parse a stored JSON string array leniently; anything malformed reads as empty. */
function parseStringArray(json: unknown): string[] {
try {
const parsed = JSON.parse(String(json || "[]"));
return Array.isArray(parsed) ? parsed.map((item) => String(item)).filter(Boolean) : [];
} catch {
return [];
}
}
function decodeEmbedding(base64: string): Float32Array | null {
if (!base64) return null;
try {
const bytes = toUint8Array(android.util.Base64.decode(base64, android.util.Base64.NO_WRAP));
if (bytes.length < 4 || bytes.length % 4 !== 0) return null;
return new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.length));
} catch {
return null;
}
}
export function encodeEmbedding(embedding: Float32Array): string {
return encodeBase64(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength));
}
export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
if (a.length !== b.length || a.length === 0) return 0;
let dot = 0;
for (let i = 0; i < a.length; i++) dot += a[i]! * b[i]!;
// Embeddings are L2-normalized at the source, so the dot product is the
// cosine; renormalizing here would only amplify float noise.
return dot;
}
export class SpeakerRegistry {
private profiles: SpeakerProfile[] = [];
private loaded = false;
private readonly changeListeners = new Set<() => void>();
onChanged(listener: () => void): () => void {
this.changeListeners.add(listener);
return () => this.changeListeners.delete(listener);
}
private notifyChanged(): void {
this.changeListeners.forEach((listener) => listener());
}
reload(): void {
try {
const rows = JSON.parse(String(store().querySpeakers())) as Array<Record<string, unknown>>;
this.profiles = rows.map((row) => ({
id: Number(row.id),
name: String(row.name ?? ""),
color: String(row.color ?? SPEAKER_COLORS[0]),
isWearer: Boolean(row.isWearer),
embedding: decodeEmbedding(String(row.embeddingBase64 ?? "")),
embeddingCount: Number(row.embeddingCount ?? 0),
lastHeardAt: Number(row.lastHeardAt ?? 0),
tag: String(row.tag ?? ""),
segmentCount: Number(row.segmentCount ?? 0),
lastRecap: String(row.lastRecap ?? ""),
actionItems: parseStringArray(row.actionItemsJson),
facts: parseStringArray(row.factsJson),
insightsUpdatedAt: Number(row.insightsUpdatedAt ?? 0),
insightsSessionId: Number(row.insightsSessionId ?? 0),
}));
this.loaded = true;
} catch (error) {
console.error(`speaker reload failed: ${String(error)}`);
this.profiles = [];
}
}
all(): SpeakerProfile[] {
if (!this.loaded) this.reload();
return this.profiles;
}
byId(id: number): SpeakerProfile | null {
return this.all().find((profile) => profile.id === id) ?? null;
}
wearer(): SpeakerProfile | null {
return this.all().find((profile) => profile.isWearer) ?? null;
}
/** Best cosine match against every enrolled profile, or null when none have prints. */
bestMatch(embedding: Float32Array): { profile: SpeakerProfile; similarity: number } | null {
let best: { profile: SpeakerProfile; similarity: number } | null = null;
for (const profile of this.all()) {
if (!profile.embedding || profile.embedding.length !== embedding.length) continue;
const similarity = cosineSimilarity(embedding, profile.embedding);
if (!best || similarity > best.similarity) {
best = { profile, similarity };
}
}
return best;
}
/**
* Match an utterance's voice-print to a profile, enrolling a new
* "Speaker N" when nothing crosses the uncertain threshold. Known and
* uncertain matches also fold the print into the centroid.
*/
assign(embedding: Float32Array): SpeakerMatch {
const best = this.bestMatch(embedding);
if (best && best.similarity >= SPEAKER_UNCERTAIN_THRESHOLD) {
const previousLastHeardAt = best.profile.lastHeardAt;
store().updateSpeakerEmbedding(best.profile.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
this.reload();
const profile = this.byId(best.profile.id) ?? best.profile;
return {
profile,
similarity: best.similarity,
classification: best.similarity >= SPEAKER_KNOWN_THRESHOLD ? "known" : "uncertain",
previousLastHeardAt,
};
}
const name = this.nextAutoName();
const color = SPEAKER_COLORS[this.all().length % SPEAKER_COLORS.length]!;
const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding)));
this.reload();
this.notifyChanged();
const profile = this.byId(id)!;
return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 };
}
private nextAutoName(): string {
const taken = new Set(this.all().map((profile) => profile.name));
for (let index = 1; ; index++) {
const candidate = `Speaker ${index}`;
if (!taken.has(candidate)) return candidate;
}
}
/** Enroll (or refresh) the wearer's own voice from a training utterance. */
enrollWearer(embedding: Float32Array): SpeakerProfile {
const existing = this.wearer();
if (existing) {
store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
this.reload();
this.notifyChanged();
return this.byId(existing.id)!;
}
const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding)));
this.reload();
this.notifyChanged();
return this.byId(id)!;
}
/** Is this utterance the wearer's own voice? Used to gate command recognition. */
isWearerVoice(embedding: Float32Array): boolean {
const wearer = this.wearer();
if (!wearer?.embedding || wearer.embedding.length !== embedding.length) return false;
return cosineSimilarity(embedding, wearer.embedding) >= SPEAKER_KNOWN_THRESHOLD;
}
rename(id: number, name: string): void {
store().renameSpeaker(id, name);
this.reload();
this.notifyChanged();
}
setColor(id: number, color: string): void {
store().setSpeakerColor(id, color);
this.reload();
this.notifyChanged();
}
setTag(id: number, tag: string): void {
store().setSpeakerTag(id, tag);
this.reload();
this.notifyChanged();
}
/** Store LLM-derived insights: recap + action items of `sessionId`, merged fact list. */
setInsights(id: number, recap: string, actionItems: string[], facts: string[], sessionId: number): void {
store().setSpeakerInsights(id, recap, JSON.stringify(actionItems), JSON.stringify(facts), sessionId);
this.reload();
this.notifyChanged();
}
setWearer(id: number, isWearer: boolean): void {
store().setSpeakerWearer(id, isWearer);
this.reload();
this.notifyChanged();
}
merge(fromId: number, intoId: number): void {
store().mergeSpeakers(fromId, intoId);
this.reload();
this.notifyChanged();
}
/**
* Split support: give one session's segments of a speaker their own new
* profile (e.g. when two people were fused into one). Returns the new id.
*/
splitFromSession(sessionId: number, speakerId: number): number {
const source = this.byId(speakerId);
const name = this.nextAutoName();
const color = SPEAKER_COLORS[this.all().length % SPEAKER_COLORS.length]!;
const newId = Number(store().createSpeaker(name, color, false, ""));
store().reassignSessionSpeaker(sessionId, speakerId, newId);
if (source) {
this.reload();
}
this.notifyChanged();
return newId;
}
remove(id: number): void {
store().deleteSpeaker(id);
this.reload();
this.notifyChanged();
}
}
export const speakerRegistry = new SpeakerRegistry();
/**
* The speaker-verification config for "my voice only" command gating, or
* null when the setting is off, no wearer voice is enrolled, or the speaker
* model isn't downloaded. Consumed by the voice-capture path so assistant /
* dictation commands from other speakers are discarded.
*/
export function wearerVerificationOptions(): {
speakerModelPath: string;
wearerEmbedding: Float32Array;
threshold: number;
} | null {
if (!wearerCommandsOnlySetting.get()) return null;
const wearer = speakerRegistry.wearer();
if (!wearer?.embedding) return null;
const speakerModelPath = micModelPath("speaker-embedding");
if (!speakerModelPath) return null;
return { speakerModelPath, wearerEmbedding: wearer.embedding, threshold: SPEAKER_KNOWN_THRESHOLD };
}
+124
View File
@@ -0,0 +1,124 @@
declare const com: any;
declare const java: any;
/**
* On-device language identification and translation for captions, over ML
* Kit (bundled language-id, per-pair translation models downloaded on
* demand). Foreign-language captions show the original line with the
* translation to the phone's default language beneath it.
*/
let languageIdentifier: any | null = null;
// Translators are expensive to build (model download on first use); cache by
// "src>dst" and keep the proxies referenced so they aren't GC'd mid-flight.
const translators = new Map<string, any>();
const downloadedPairs = new Set<string>();
/** The phone's default language as a two-letter code (the translation target). */
export function deviceLanguage(): string {
if (!global.isAndroid) return "en";
try {
return String(java.util.Locale.getDefault().getLanguage() || "en");
} catch {
return "en";
}
}
function successListener(resolve: (value: any) => void): any {
return new com.google.android.gms.tasks.OnSuccessListener({
onSuccess: (result: any) => resolve(result),
});
}
function failureListener(reject: (reason: Error) => void): any {
return new com.google.android.gms.tasks.OnFailureListener({
onFailure: (error: any) => reject(new Error(String(error?.getMessage?.() ?? error))),
});
}
/**
* Identify the language of a caption line. Resolves to a BCP-47 code, or
* "und" when ML Kit isn't confident. Short lines are noisy; callers should
* apply hysteresis across consecutive finals before switching languages.
*/
export function identifyLanguage(text: string): Promise<string> {
if (!global.isAndroid || !text.trim()) return Promise.resolve("und");
return new Promise((resolve) => {
try {
if (!languageIdentifier) {
languageIdentifier = com.google.mlkit.nl.languageid.LanguageIdentification.getClient();
}
languageIdentifier
.identifyLanguage(text)
.addOnSuccessListener(successListener((code: any) => resolve(String(code))))
.addOnFailureListener(failureListener(() => resolve("und")));
} catch (error) {
console.warn(`language id failed: ${error}`);
resolve("und");
}
});
}
function translatorFor(sourceLang: string, targetLang: string): any | null {
const key = `${sourceLang}>${targetLang}`;
const cached = translators.get(key);
if (cached) return cached;
try {
const TranslateLanguage = com.google.mlkit.nl.translate.TranslateLanguage;
const source = TranslateLanguage.fromLanguageTag(sourceLang);
const target = TranslateLanguage.fromLanguageTag(targetLang);
if (source == null || target == null) return null;
const options = new com.google.mlkit.nl.translate.TranslatorOptions.Builder()
.setSourceLanguage(source)
.setTargetLanguage(target)
.build();
const translator = com.google.mlkit.nl.translate.Translation.getClient(options);
translators.set(key, translator);
return translator;
} catch (error) {
console.warn(`translator setup failed (${key}): ${error}`);
return null;
}
}
/**
* Translate one caption line. Resolves to the translated text, or "" when the
* pair is unsupported or the model download hasn't finished (translation of
* later lines picks up once it has; downloads are WiFi-unrestricted since
* caption sessions are live).
*/
export function translateText(text: string, sourceLang: string, targetLang: string): Promise<string> {
if (!global.isAndroid || !text.trim() || sourceLang === targetLang) {
return Promise.resolve("");
}
const translator = translatorFor(sourceLang, targetLang);
if (!translator) return Promise.resolve("");
const key = `${sourceLang}>${targetLang}`;
return new Promise((resolve) => {
const runTranslate = () => {
translator
.translate(text)
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
.addOnFailureListener(failureListener(() => resolve("")));
};
if (downloadedPairs.has(key)) {
runTranslate();
return;
}
try {
const conditions = new com.google.mlkit.common.model.DownloadConditions.Builder().build();
translator
.downloadModelIfNeeded(conditions)
.addOnSuccessListener(
successListener(() => {
downloadedPairs.add(key);
runTranslate();
}),
)
.addOnFailureListener(failureListener(() => resolve("")));
} catch (error) {
console.warn(`translation model download failed (${key}): ${error}`);
resolve("");
}
});
}
+1 -3
View File
@@ -1,9 +1,7 @@
/**
* How far away an advertising device probably is, from its signal strength.
*
* Ported from SybilSight's `BLEProximity.swift` (itself from ProLink's
* `PeripheralInfo.basicEstimatedDistance`), with the calibration table
* narrowed to the two products this app pairs.
* The calibration table is narrowed to the two products this app pairs.
*
* ## The model
*
+9
View File
@@ -66,6 +66,8 @@ import {
uninstallEvenHubPackage,
} from "../apps/evenhub/installed-apps";
import { closeRunningPackage, launchInstalledPackage } from "../apps/evenhub/manager";
import { wearerVerificationOptions } from "../apps/microphones/speakers";
import { micSession } from "../apps/microphones/mic-session";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting";
@@ -1563,6 +1565,13 @@ class DashboardController {
sonioxApiKey: sonioxApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
endpointing,
// "My voice only" (Microphones app): verify command utterances
// against the enrolled wearer voice-print; non-matching speakers'
// finals are suppressed by the bridge.
speakerVerification: wearerVerificationOptions() ?? undefined,
// The Microphones app's processing config applies to every capture:
// spectral noise suppression and the Sonic Radar listening beam.
...micSession.captureProcessingOptions(),
};
if (kind === "ptt") {
voiceControlBridge.startPushToTalk(options);
-4
View File
@@ -1,10 +1,6 @@
/**
* Parsers for what Even Realities hardware puts in a BLE advertisement.
*
* Ported from SybilSight's SDK (`extractSN` / `extractMac` /
* `extractG2AdvertisementFlag` in `G2.swift`, and `ringMacWireBytes` /
* `ringNameAddressOctets` in `R1.swift`).
*
* ## G2 temple (confirmed by HCI capture)
*
* Manufacturer-specific AD record: `45 52` + SN(14, ASCII) + MAC(6, little-endian) + flag(1).
+11
View File
@@ -78,6 +78,17 @@ export function hasCustomFirmware(info: FirmwareInfo): boolean {
return REQUIRED_FIRMWARE_EXTENSIONS.every((extension) => tokens.includes(extension));
}
/**
* True when the firmware advertises the optional mic-control extension: the
* private field-103/104 channel for per-temple multi-microphone capture that
* the Microphones app's 4-mic array mode and beamforming need. Absent on the
* stock single-mono-stream path, which still works (firmware-computed
* direction of arrival, one mixed channel).
*/
export function hasMicControl(info: FirmwareInfo): boolean {
return info.capabilities.trim().split(/\s+/).includes("micctl");
}
/** The higher of the two arms' reported versions, or "" if none reported. */
export function reportedFirmwareVersion(info: FirmwareInfo): string {
const versions = [info.leftVersion, info.rightVersion].map((v) => v.trim()).filter(Boolean);
+20 -20
View File
@@ -19,7 +19,7 @@ export type FirmwarePatchSet = {
export const CFW_PATCH_SET: FirmwarePatchSet = {
"base": "g2_2.2.6.10.bin",
"baseSha256": "f4dfb0b49ad3de3c2daf17f8a27a157c3dc98411d6a0d3ab2cfd0918f41b9afa",
"outputSha256": "e37c8d7b5876abd68cb58dd19a91cbb515b35fe91c0032648c7d20bb02601884",
"outputSha256": "066477d7ac55682016ba178d3fa7f4db38a024997d1ab3f1e6720348bf43ca6e",
"patches": [
{
"offset": 1089903,
@@ -48,61 +48,61 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1447919,
"old": "7ef7fefd",
"new": "bcf24cfa",
"new": "bcf2a8fa",
"desc": "bl snapshot_side @ 0x4db968"
},
{
"offset": 1448931,
"old": "7ef704fc",
"new": "bcf252f8",
"new": "bcf2aef8",
"desc": "bl snapshot_side @ 0x4dbd5c"
},
{
"offset": 1165461,
"old": "45f0cefd",
"new": "01f3fdf9",
"new": "01f359fa",
"desc": "bl image_deferred (deferred consumer -> FIFO restore + worker, both lenses)"
},
{
"offset": 1186287,
"old": "d9f7d4ff",
"new": "fdf258f8",
"new": "fdf288f9",
"desc": "bl settings_send_wrapper (append caps field 100)"
},
{
"offset": 1183983,
"old": "f4f75aff",
"new": "fdf21afa",
"new": "fdf228fb",
"desc": "bl settings_decode_wrapper (Faceclaw lease field 101)"
},
{
"offset": 926945,
"old": "08f068fa",
"new": "3bf33dff",
"new": "3cf34bf8",
"desc": "bl faceclaw_display_start @ 0x45c65a (fail-open double-tap takeover)"
},
{
"offset": 927137,
"old": "08f008fa",
"new": "3bf3ddfe",
"new": "3bf3ebff",
"desc": "bl faceclaw_display_start @ 0x45c71a (fail-open double-tap takeover)"
},
{
"offset": 822553,
"old": "28f003f8",
"new": "55f34dff",
"new": "56f31dfd",
"desc": "bl evenhub_longpress (Faceclaw lease gates custom event vs stock dialog)"
},
{
"offset": 823369,
"old": "1cf09bfb",
"new": "55f3d9fd",
"new": "56f3a9fb",
"desc": "bl ring_release (forward source-qualified release-long-press)"
},
{
"offset": 1474137,
"old": "7fb50600",
"new": "b6f211be",
"new": "b6f241bf",
"desc": "even_ai_display_ctrl entry -> conditional Faceclaw lease trampoline"
},
{
@@ -120,13 +120,13 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 1198787,
"old": "dff770fb",
"new": "f9f28cfb",
"new": "f9f262fc",
"desc": "bl faceclaw_send_wear_event @ 0x49ec3c (outside onboarding)"
},
{
"offset": 1198881,
"old": "dff741fb",
"new": "f9f25dfb",
"new": "f9f233fc",
"desc": "bl faceclaw_send_wear_event @ 0x49ec9a (outside onboarding)"
},
{
@@ -138,43 +138,43 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
{
"offset": 4301227,
"old": "",
"new": "02fb01f000e000bfb0b546f27424c2f22a040146206846f23f52c5f28802b2eb500f40f2543044f281124cf67e25c2f20000c0f24802caf2101518bf25609047002804bf691c2160b0bd00bf0846ffe7014640f2543044f29f22c2f20000c0f24802104770b500284ed001684ef66762ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf70b54ff6fd03c0f2450314460d46984746f27021c2f22a010e6831466ff39601b1f1005f0ad131684ef66762ccf2ff02914218bf0026412d03d070bd0026412dfbd1002cf9d04af26951c0f2450105468847002e01462846efd00129edd196f807120029e9d02168002948bf70bd47f25d02c0f2580208469047284670bd00bf2de9f04f9db046f2702ac2f22a0adaf800b058466ff39600b0f1005f40f07d81dbf800004ef66767ccf2ff07b84240f074819bf82802002800f05d81dbf82442002c00f0588140f2b870c2f20000d0f8008004a802f08effb8f1000f00f0dd81032045f24075cff68070a4f1600108f160026ff0bf03c0f2020500bf0e6ed41826664e6ec03366668e6eab42a666ce6ee6660e6f26674e6f66678e6fa667ce6fe667d1f88060c4f88060d1f88460c4f88460d1f88860c4f88860d1f88c60c4f88c60d1f89060c4f89060d1f89460c4f89460d1f89860c4f89860d1f89c60c4f89c60d1f8a060c4f8a060d1f8a460c4f8a460d1f8a860c4f8a860d1f8ac60c4f8ac60d1f8b060c4f8b060d1f8b460c4f8b460d1f8b860c4f8b860d1f8bc60c4f8bc6051f8c06fc4f8c0604e68c4f8c4608e68c4f8c860ce68c4f8cc600e69c4f8d0604e69c4f8d4608e69c4f8d860ce69c4f8dc600e6ac4f8e0604e6ac4f8e4608e6ac4f8e860ce6ac4f8ec600e6bc4f8f0604e6bc4f8f4608e6bc4f8f860ce6bc4f8fc600e6cc4f800614e6cc4f804618e6cc4f80861ce6cc4f80c610e6dc4f810614e6dc4f814618e6dc4f81861ce6dc4f81c617ff462afdaf800603040b0f1005f40f0b2803068b84240f0ae8096f8ce00002840f0348196f8dc10002000298df8140000f0ac8105a90a5c002a00f07f810a185378002b00f03d819378002b00f03b81d378002b00f039811379002b00f037815379002b00f035819379002b00f03381d379002b00f03181137a002b00f02f81537a002b00f02d81937a002b00f02b81d37a002b00f02981137b002b00f02781537b002b00f02581937b002b00f02381d37b002b00f02181137c002b00f01f81537c002b00f01d81937c002b00f01b81d37c002b00f01981137d002b00f01781537d002b00f01581937d002b00f01381d37d002b00f01181137e002b00f00f81537e002b00f00d81937e002b00f00b81d37e002b00f00981137f002b00f00781537f002b00f00581937f002b00f00381d27f002a00f00181203060287ff47baf20e19bf82a0270b1dbf82c0240b144f63421c2f207010968401a002800f3ae8000208bf82a024cf61520c0f246001db0bde8f04f004704f001fb002800f0838000216ff017020023c518c1542f338618b3f50d7f02f12f026970a970e97029716971a971e97129726972a972e97229736973a973e97329746974a974e97429756975a97586f82f1086f830106976a976e97629776977a977e97785f8201085f8211085f8221085f8231085f8241085f8251085f8261085f8271085f8281085f8291085f82a1085f82b1085f82c1085f82d1085f82e10b7d1012180f8ce104ff0ff310760a0f8e010a0f8e210a0f8e410a0f8e610a0f8e810a0f8ea10a0f8ec10a0f8ee10a0f8f010a0f8f210a0f8f410a0f8f610a0f8f810a0f8fa10a0f8fc10a0f8fe1010e000208bf82802cbf824028bf82a0201208bf829024cf61520c0f24600804715e00020caf8000000208bf82802cbf824024ff4163045f20f11069005a8c0f24701cdf81480884701208bf82a0204a802f06efecbf8d4001db0bde8f08f01303ae0023038e0033036e0043034e0053032e0063030e007302ee008302ce009302ae00a3028e00b3026e00c3024e00d3022e00e3020e00f301ee010301ce011301ae0123018e0133016e0143014e0153012e0163010e017300ee018300ce019300ae01a3008e01b3006e01c3004e01d3002e01e3000e01f305f2821d84ff052020a541dd0014445225e284a7018d04f225d288a7014d052225c28ca7010d044225b280a710cd045225a284a7108d0522259288a7104d020225828ca7142f0168500208df87300012096f8dd10002900f0c280002105aa535c002b00f0a08057187b78002b5ed0bb78002b5dd0fb78002b5cd03b79002b5bd07b79002b5ad0bb79002b59d0fb79002b58d03b7a002b57d07b7a002b56d0bb7a002b55d0fb7a002b54d03b7b002b53d07b7b002b52d0bb7b002b51d0fb7b002b50d03b7c002b4fd07b7c002b4ed0bb7c002b4dd0fb7c002b4cd03b7d002b4bd07b7d002b4ad0bb7d002b49d0fb7d002b48d03b7e002b47d07b7e002b46d0bb7e002b45d0fb7e002b44d03b7f002b43d07b7f002b42d0bb7f002b41d0fb7f002b40d0203160297ff49aaf54e001313ae0023138e0033136e0043134e0053132e0063130e007312ee008312ce009312ae00a3128e00b3126e00c3124e00d3122e00e3120e00f311ee010311ce011311ae0123118e0133116e0143114e0153112e0163110e017310ee018310ce019310ae01a3108e01b3106e01c3104e01d3102e01e3100e01f315f2915d84ff05303535411d00a444b235e2953700cd049235d29937008d050235c29d37004d020235b29137140f0c28200218df87310013096f8de10002900f0be80002105aa535c002b00f0a08057187b78002b5ed0bb78002b5dd0fb78002b5cd03b79002b5bd07b79002b5ad0bb79002b59d0fb79002b58d03b7a002b57d07b7a002b56d0bb7a002b55d0fb7a002b54d03b7b002b53d07b7b002b52d0bb7b002b51d0fb7b002b50d03b7c002b4fd07b7c002b4ed0bb7c002b4dd0fb7c002b4cd03b7d002b4bd07b7d002b4ad0bb7d002b49d0fb7d002b48d03b7e002b47d07b7e002b46d0bb7e002b45d0fb7e002b44d03b7f002b43d07b7f002b42d0bb7f002b41d0fb7f002b40d0203160297ff49aaf50e001313ae0023138e0033136e0043134e0053132e0063130e007312ee008312ce009312ae00a3128e00b3126e00c3124e00d3122e00e3120e00f311ee010311ce011311ae0123118e0133116e0143114e0153112e0163110e017310ee018310ce019310ae01a3108e01b3106e01c3104e01d3102e01e3100e01f315f2911d84ff0440353540dd00Line truncated
"new": "02fb01f000e000bfb0b546f27424c2f22a040146206846f23f52c5f28802b2eb500f40f2543044f281124cf67e25c2f20000c0f24802caf2101518bf25609047002804bf691c2160b0bd00bf0846ffe7014640f2543044f29f22c2f20000c0f24802104770b500284ed001684ef66862ccf2ff02914218bf70bd90f8081190f8095142f6c52c8d42c0f2500c2ad205eb8501014491f80d2191f80e4191f80ae191f80b3191f80c1142ea0424012c4ff0010298bf14466e1c054680f80961b9b164294eea032028bf6421012888bf024644f62060824228bf02460cf5e2731046984706e0002180f80811bde870406047e047d5f8040138b149f29942c0f244022146bde87040104770bd00bf70b54ff6fd03c0f2450314460d46984746f27021c2f22a010e6831466ff39601b1f1005f0ad131684ef66862ccf2ff02914218bf0026412d03d070bd0026412dfbd1002cf9d04af26951c0f2450105468847002e01462846efd00129edd196f807120029e9d02168002948bf70bd47f25d02c0f2580208469047284670bd00bf2de9f04f9bb046f2702ac2f22a0adaf800b058466ff39600b0f1005f40f07f81dbf800004ef66867ccf2ff07b84240f076819bf82802002800f05f81dbf82442002c00f05a8140f2b870c2f20000d0f8008002a802f038ffb8f1000f00f01b82032045f24075cff68070a4f1600108f160026ff0bf03c0f2020500bf0e6ed41826664e6ec03366668e6eab42a666ce6ee6660e6f26674e6f66678e6fa667ce6fe667d1f88060c4f88060d1f88460c4f88460d1f88860c4f88860d1f88c60c4f88c60d1f89060c4f89060d1f89460c4f89460d1f89860c4f89860d1f89c60c4f89c60d1f8a060c4f8a060d1f8a460c4f8a460d1f8a860c4f8a860d1f8ac60c4f8ac60d1f8b060c4f8b060d1f8b460c4f8b460d1f8b860c4f8b860d1f8bc60c4f8bc6051f8c06fc4f8c0604e68c4f8c4608e68c4f8c860ce68c4f8cc600e69c4f8d0604e69c4f8d4608e69c4f8d860ce69c4f8dc600e6ac4f8e0604e6ac4f8e4608e6ac4f8e860ce6ac4f8ec600e6bc4f8f0604e6bc4f8f4608e6bc4f8f860ce6bc4f8fc600e6cc4f800614e6cc4f804618e6cc4f80861ce6cc4f80c610e6dc4f810614e6dc4f814618e6dc4f81861ce6dc4f81c617ff462afdaf800e000ea0e00b0f1005f40f0b380def80000b84240f0ae809ef8ce00002840f070819ef8dc10002000298df80c0000f0e88103a90a5c002a00f0bb810a185378002b00f079819378002b00f07781d378002b00f075811379002b00f073815379002b00f071819379002b00f06f81d379002b00f06d81137a002b00f06b81537a002b00f06981937a002b00f06781d37a002b00f06581137b002b00f06381537b002b00f06181937b002b00f05f81d37b002b00f05d81137c002b00f05b81537c002b00f05981937c002b00f05781d37c002b00f05581137d002b00f05381537d002b00f05181937d002b00f04f81d37d002b00f04d81137e002b00f04b81537e002b00f04981937e002b00f04781d37e002b00f04581137f002b00f04381537f002b00f04181937f002b00f03f81d27f002a00f03d81203060287ff47baf5ce19bf82a0270b1dbf82c0240b144f63421c2f207010968401a002800f3ea8000208bf82a024cf61520c0f246001bb0bde8f04f00474ff41b7004f099fd002800f0bd804ff69451cff6ff71002200bf43183e3183f86c2283f86d2283f86e2283f86f2283f8702283f8712283f8722283f8732283f8742283f8752283f8762283f8772283f8782283f8792283f87a2283f87b2283f87c2283f87d2283f87e2283f87f2283f8802283f8812283f8822283f8832283f8842283f8852283f8862283f8872283f8882283f8892283f88a2283f88b2283f88c2283f88d2283f88e2283f88f2283f8902283f8912283f8922283f8932283f8942283f8952283f8962283f8972283f8982283f8992283f89a2283f89b2283f89c2283f89d2283f89e2283f89f2283f8a02283f8a12283f8a22283f8a32283f8a42283f8a52283f8a62283f8a72283f8a82283f8a9227ff480af012180f8ce104ff0ff310760a0f8e010a0f8e210a0f8e410a0f8e610a0f8e810a0f8ea10a0f8ec10a0f8ee10a0f8f010a0f8f210a0f8f410a0f8f610a0f8f810a0f8fa10a0f8fc10a0f8fe1010e000208bf82802cbf824028bf82a0201208bf829024cf61520c0f24600804715e00020caf8000000208bf82802cbf824024ff4163045f20f11049003a8c0f24701cdf80c80884701208bf82a0202a802f016fecbf8d4001bb0bde8f08f01303ae0023038e0033036e0043034e0053032e0063030e007302ee008302ce009302ae00a3028e00b3026e00c3024e00d3022e00e3020e00f301ee010301ce011301ae0123018e0133016e0143014e0153012e0163010e017300ee018300ce019300ae01a3008e01b3006e01c3004e01d3002e01e3000e01f305f2821d84ff052020a541dd0014445225e284a7018d04f225d288a7014d052225c28ca7010d044225b280a710cd045225a284a7108d0522259288a7104d020225828ca7141f0b38300208df86b0001209ef8dd10002900f0c280002103aa535c002b00f0a08057187b78002b5ed0bb78002b5dd0fb78002b5cd03b79002b5bd07b79002b5ad0bb79002b59d0fb79002b58d03b7a002b57d07b7a002b56d0bb7a002b55d0fb7a002b54d03b7b002b53d07b7b002b52d0bb7b002b51d0fb7b002b50d03b7c002b4fd07b7c002b4ed0bb7c002b4dd0fb7c002b4cd03b7d002b4bd07b7d002b4ad0bb7d002b49d0fb7d002b48d03b7e002b47d07b7e002b46d0bb7e002b45d0fb7e002b44d03b7f002b43d07b7f002b42d0bb7f002b41d0fb7f002b40d0203160297ff49aaf54e001313ae0023138e0033136e0043134e0053132e0063130e007312ee008312ce009312ae00a3128e00b3126e00c3124e00d3122e00e3120e00f311ee010311ce011311ae0123118e0133116e0143114e0153112e0163110e017310ee018310ce019310ae01a3108e01b3106e01c3104e01d3102e01e3100e01f315f2915d84ff05303535411d00a444b235e2953700cd049235d29937008d050235c29d37004d020235b29137140f0c28200218df86b1001309ef8de10002900f0be80002103aa535c002b00f0a08057187b78002b5ed0bb78002b5dd0fb78002b5cd03b79002b5bd07b79002b5ad0bb79002b59d0fb79002b58d03b7a002b57d07b7a002b56d0bb7a002b55d0fb7a002b54d03b7b002b53d07b7b002b52d0bb7b002b51d0fb7b002b50d03b7c002b4fd07b7c002b4ed0bb7c002b4dd0fb7c002b4cd03b7d002b4bd07b7d002b4ad0bb7d002b49d0fb7d002b48d03b7e002b47d07b7e002b46d0bb7e002b45d0fb7e002b44d03b7f002b43d07b7f002b42d0bb7f002b41d0fb7f002b40d0203160297ff49aaf50e001313ae0023138e003313Line truncated
"desc": "append injected blobs to main-app payload"
},
{
"offset": 777711,
"old": "44c33500",
"new": "584b3600",
"new": "085b3600",
"desc": "main-app subheader payload size (ps)"
},
{
"offset": 152,
"old": "c4c33500",
"new": "d84b3600",
"new": "885b3600",
"desc": "main-app TOC entry size (ps + 128)"
},
{
"offset": 777831,
"old": "44c33504",
"new": "584b3604",
"new": "085b3604",
"desc": "main-app preamble length (low 24 bits)"
},
{
"offset": 777835,
"old": "df8110c9",
"new": "0c327522",
"new": "af452382",
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
},
{
"offset": 156,
"old": "c1be0fb1",
"new": "e3712717",
"new": "5e6eca15",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
},
{
"offset": 777715,
"old": "c1be0fb1",
"new": "e3712717",
"new": "5e6eca15",
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
}
]
+3 -4
View File
@@ -1,15 +1,14 @@
/**
* Product artwork for Even Realities hardware, keyed by the decoded serial.
*
* Ported from SybilSight's `SybilSightAssets.glassesAssetName(for:identity:)`.
* Prefer the photo of the exact SKU the serial decoded to, and fall back to
* the generic product shot. Every surface that draws "the user's glasses"
* should route through this — once the pair is known, showing brown Frame B
* for a grey Frame A pair is a small lie the wearer will notice immediately.
*
* The PNGs live under `app/images/glasses/` (copied from SybilSight's asset
* catalog; six SKU crops at 600×196 on a shared bounding box so they render at
* the same scale, plus the generic G2 and R1 product shots). NativeScript
* The PNGs live under `app/images/glasses/` (six SKU crops at 600×196 on a
* shared bounding box so they render at the same scale, plus the generic G2
* and R1 product shots). NativeScript
* resolves `~/` to the app folder for `<Image src>`.
*
* This module is pure (no NativeScript imports) so it can run under node tests.
-2
View File
@@ -2,8 +2,6 @@
* Decodes an Even Realities serial number into the product family, frame shape,
* and colourway the wearer can actually see.
*
* Ported from SybilSight's `GlassesHardwareIdentity.swift`.
*
* ## Why this exists
*
* A G2 arm advertises its 14-character serial inside the manufacturer data
+2 -3
View File
@@ -3,9 +3,8 @@
* G2 pairs (a left and a right temple that advertise the same serial) and R1
* rings, each ranked by how close it probably is.
*
* Ported from SybilSight's `PairingScanPresentation` / `PairingFlowState`
* (nearest-device margin, previously-paired match, closest-first ordering)
* and extended with the left↔right pair grouping that Faceclaw needs because
* Ranks by nearest-device margin, previously-paired match, and closest-first
* ordering, with the left↔right pair grouping that Faceclaw needs because
* it connects to each arm by address rather than through a vendor SDK.
*
* This module is pure (no NativeScript imports) so it can run under node tests.
+2
View File
@@ -22,6 +22,8 @@ export const ICON_SVGS = {
'<svg viewBox="0 0 24 24" fill="none"><rect width="7" height="7" x="3" y="3" rx="1"/><rect width="7" height="7" x="14" y="3" rx="1"/><rect width="7" height="7" x="14" y="14" rx="1"/><rect width="7" height="7" x="3" y="14" rx="1"/></svg>',
timer:
'<svg viewBox="0 0 24 24" fill="none"><line x1="10" x2="14" y1="2" y2="2"/><line x1="12" x2="15" y1="14" y2="11"/><circle cx="12" cy="14" r="8"/></svg>',
calculator:
'<svg viewBox="0 0 24 24" fill="none"><rect width="16" height="20" x="4" y="2" rx="2"/><line x1="8" x2="16" y1="6" y2="6"/><line x1="16" x2="16" y1="14" y2="18"/><path d="M16 10h.01"/><path d="M12 10h.01"/><path d="M8 10h.01"/><path d="M12 14h.01"/><path d="M8 14h.01"/><path d="M12 18h.01"/><path d="M8 18h.01"/></svg>',
// Not a Lucide icon: an L-tetromino built from four squares, for Blocks.
"l-piece":
'<svg viewBox="0 0 24 24" fill="none"><rect width="6" height="6" x="5" y="1.5" rx="1"/><rect width="6" height="6" x="5" y="9" rx="1"/><rect width="6" height="6" x="5" y="16.5" rx="1"/><rect width="6" height="6" x="12.5" y="16.5" rx="1"/></svg>',
+91
View File
@@ -36,6 +36,23 @@ export type PushToTalkOptions = {
* Ignored when the mic is already running for another holder.
*/
endpointing?: boolean;
/**
* "My voice only": verify the captured utterance against the enrolled
* wearer voice-print, and suppress the final transcript when someone else
* spoke. wearerEmbedding is the L2-normalized profile centroid.
*/
speakerVerification?: {
speakerModelPath: string;
wearerEmbedding: Float32Array;
threshold: number;
};
/** Spectral noise suppression on the decoded stream (Microphones config). */
noiseSuppression?: boolean;
/**
* Sonic Radar beam gating: only listen to the configured wedge (device
* frame). Null/absent leaves the mic omnidirectional.
*/
beamFilter?: { centerDeg: number; halfWidthDeg: number } | null;
};
/** Who currently wants the mic running. */
@@ -43,6 +60,16 @@ type CaptureHolder = "ptt" | "continuous";
export type RawPcmListener = (pcm: Uint8Array) => void;
/**
* Per-packet DSP metadata the glasses firmware appends to each 50 ms audio
* packet: direction-of-arrival in signed degrees (0 = straight ahead,
* positive to the wearer's right) and a signal-strength ratio used as a
* confidence/energy proxy. Computed on the raw stereo capture before the
* firmware's mono downmix, so it survives even though only mono PCM arrives.
*/
export type MicFrameMeta = { angleDegrees: number; ssr: number };
export type FrameMetaListener = (meta: MicFrameMeta) => void;
export class FaceclawVoiceControlBridge {
private readonly statusListeners = new Set<(state: VoiceControlState) => void>();
private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>();
@@ -63,7 +90,13 @@ export class FaceclawVoiceControlBridge {
// listeners. STT capture preempts it — a live assistant/transcribe session
// owns the single G2 mic and the raw tap is stopped for its duration.
private readonly rawPcmListeners = new Set<RawPcmListener>();
private readonly frameMetaListeners = new Set<FrameMetaListener>();
private rawActive = false;
// Set when the current capture session runs "my voice only" verification;
// a non-wearer verdict arrives just before the final transcript and makes
// the bridge swallow it.
private verificationActive = false;
private verificationRejected = false;
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
@@ -110,6 +143,12 @@ export class FaceclawVoiceControlBridge {
return () => this.rawPcmListeners.delete(listener);
}
/** Subscribe to per-packet firmware DSP metadata (DOA angle + SSR). */
onFrameMeta(listener: FrameMetaListener): () => void {
this.frameMetaListeners.add(listener);
return () => this.frameMetaListeners.delete(listener);
}
/**
* Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns
* false) if an STT capture already owns the mic; the caller re-tries when its
@@ -123,6 +162,11 @@ export class FaceclawVoiceControlBridge {
this.controller?.setCommunicator(options.communicator);
this.controller?.setSaveRecordings(false);
this.controller?.setEndpointing(false);
// Raw means raw: the Microphones session (and EvenHub apps) get the
// unprocessed stream and run their own beam/suppression processing.
this.controller?.setNoiseSuppression(false);
this.controller?.setBeamFilter(false, 0, 180);
this.controller?.clearSpeakerVerification();
this.cloudClient = null;
this.rawActive = true;
this.started = true;
@@ -155,6 +199,24 @@ export class FaceclawVoiceControlBridge {
this.controller?.setCommunicator(options.communicator);
this.controller?.setSaveRecordings(options.saveRecording);
this.controller?.setEndpointing(Boolean(options.endpointing));
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
if (options.beamFilter) {
this.controller?.setBeamFilter(true, Math.round(options.beamFilter.centerDeg), Math.round(options.beamFilter.halfWidthDeg));
} else {
this.controller?.setBeamFilter(false, 0, 180);
}
this.verificationRejected = false;
if (options.speakerVerification) {
this.verificationActive = true;
this.controller?.setSpeakerVerification(
options.speakerVerification.speakerModelPath,
toJavaFloats(options.speakerVerification.wearerEmbedding),
options.speakerVerification.threshold,
);
} else {
this.verificationActive = false;
this.controller?.clearSpeakerVerification();
}
const cloudClient = this.createCloudClient(options);
if (cloudClient) {
@@ -270,16 +332,37 @@ export class FaceclawVoiceControlBridge {
this.rawPcmListeners.forEach((listener) => listener(bytes));
}
},
onFrameMeta: (angleDegrees: number, ssr: number) => {
if (this.frameMetaListeners.size === 0) return;
const meta = { angleDegrees: Number(angleDegrees), ssr: Number(ssr) };
this.frameMetaListeners.forEach((listener) => listener(meta));
},
onSpeechEnd: () => {
for (const listener of this.speechEndListeners) {
listener();
}
},
onSpeakerVerified: (isWearer: boolean, similarity: number) => {
if (!this.verificationActive) return;
this.verificationRejected = !isWearer;
if (!isWearer) {
this.setStatus(
`Ignored — not your enrolled voice (match ${Math.round(Number(similarity) * 100)}%).`,
);
}
},
});
this.controller.setListener(this.listenerProxy);
}
private emitTranscript(text: string, isFinal: boolean): void {
// "My voice only": a rejected session's final transcript is emptied so
// nothing downstream (assistant, dictation) acts on another speaker's
// words. The verification verdict is posted before the final, and cloud
// finals arrive over the network later still, so the flag is settled.
if (isFinal && this.verificationActive && this.verificationRejected) {
text = "";
}
const event = { text, isFinal };
for (const listener of this.transcriptListeners) {
listener(event);
@@ -294,4 +377,12 @@ export class FaceclawVoiceControlBridge {
}
}
function toJavaFloats(values: Float32Array): any {
const javaFloats = Array.create("float", values.length);
for (let i = 0; i < values.length; i++) {
javaFloats[i] = values[i]!;
}
return javaFloats;
}
export const voiceControlBridge = new FaceclawVoiceControlBridge();
+15
View File
@@ -0,0 +1,15 @@
import { NavigatedData, Page } from "@nativescript/core";
import { AskViewModel } from "./ask-view-model";
export function navigatingTo(args: NavigatedData): void {
const page = args.object as Page;
if (!page.bindingContext) {
page.bindingContext = new AskViewModel();
}
}
export function unloaded(args: NavigatedData): void {
const page = args.object as Page;
(page.bindingContext as AskViewModel | undefined)?.cancel();
}
+20
View File
@@ -0,0 +1,20 @@
<Page xmlns="http://schemas.nativescript.org/tns.xsd" navigatingTo="navigatingTo" unloaded="unloaded">
<ActionBar title="Ask about conversations" />
<GridLayout rows="auto,auto,auto,*" class="p-8">
<TextField
row="0"
hint="e.g. What did Alice say about the budget last week?"
text="{{ question }}"
returnKeyType="go"
returnPress="{{ onAskTap }}" />
<Button row="1" text="{{ askLabel }}" tap="{{ onAskTap }}" isEnabled="{{ askEnabled }}" class="m-t-4" />
<Label row="2" text="{{ statusText }}" class="conversation-meta m-t-4" textWrap="true" visibility="{{ statusVisibility }}" />
<ScrollView row="3" class="m-t-8">
<StackLayout>
<Label text="{{ answer }}" class="ask-answer" textWrap="true" visibility="{{ answerVisibility }}" />
<Label text="{{ contextNote }}" class="conversation-meta m-t-8" textWrap="true" visibility="{{ noteVisibility }}" />
</StackLayout>
</ScrollView>
</GridLayout>
</Page>
+84
View File
@@ -0,0 +1,84 @@
import { Observable } from "@nativescript/core";
import {
askConversations,
conversationQaAvailable,
} from "../apps/microphones/conversation-qa";
import type { LlmStreamHandle } from "../assistant/llm-protocol";
/**
* "Ask about conversations": free-form questions over the saved transcripts,
* speakers, tags, and metadata, answered entirely on-device by the local
* assistant model. The answer streams in as it generates.
*/
export class AskViewModel extends Observable {
private stream: LlmStreamHandle | null = null;
private busy = false;
constructor() {
super();
this.set("question", "");
this.set("answer", "");
this.set("contextNote", "");
this.set("askLabel", "Ask");
this.set("askEnabled", true);
this.set("answerVisibility", "collapsed");
this.set("noteVisibility", "collapsed");
if (!conversationQaAvailable()) {
this.set(
"statusText",
"The on-phone model isn't downloaded yet. Enable it on the glasses under Settings > Assistant, then come back — questions are answered entirely on-device.",
);
this.set("statusVisibility", "visible");
} else {
this.set("statusText", "");
this.set("statusVisibility", "collapsed");
}
}
onAskTap = (): void => {
const question = String(this.get("question") ?? "").trim();
if (!question || this.busy) return;
this.cancel();
this.busy = true;
this.set("askLabel", "Thinking...");
this.set("askEnabled", false);
this.set("answer", "");
this.set("answerVisibility", "collapsed");
this.set("noteVisibility", "collapsed");
this.set("statusText", "Reading your conversations...");
this.set("statusVisibility", "visible");
this.stream = askConversations(question, {
onDelta: (textSoFar) => {
this.set("answer", textSoFar);
this.set("answerVisibility", "visible");
this.set("statusVisibility", "collapsed");
},
onDone: (answer, contextNote) => {
this.finish();
this.set("answer", answer || "(no answer)");
this.set("answerVisibility", "visible");
this.set("contextNote", contextNote);
this.set("noteVisibility", "visible");
},
onError: (message) => {
this.finish();
this.set("statusText", message);
this.set("statusVisibility", "visible");
},
});
};
cancel(): void {
this.stream?.cancel();
this.stream = null;
if (this.busy) this.finish();
}
private finish(): void {
this.busy = false;
this.stream = null;
this.set("askLabel", "Ask");
this.set("askEnabled", true);
}
}
+100
View File
@@ -0,0 +1,100 @@
import { Utils } from "@nativescript/core";
import type { EmotionLabel, SentimentBucket } from "../apps/microphones/sentiment";
declare const com: any;
/**
* Shared plumbing for the conversation review pages: access to the Java
* conversation store, the row shapes it returns, and the small formatting/
* color helpers every page needs. Callers must guard store access with
* `global.isAndroid`.
*/
export type SessionSpeakerRef = { id: number; name: string; color: string };
export type SessionRow = {
id: number;
startedAt: number;
endedAt: number;
title: string;
audioPath: string;
audioCodec: string;
avgSentiment: number;
segmentCount: number;
speakers: SessionSpeakerRef[];
};
export type SegmentRow = {
id: number;
speakerId: number | null;
startedAt: number;
endedAt: number;
audioOffsetMs: number | null;
text: string;
lang: string;
translation: string;
translationLang: string;
sentiment: number;
emotion: string;
angle: number | null;
};
export function conversationStore(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
export const NO_CONVERSATIONS_MESSAGE =
"No conversations yet — enable Captions + Save captions in the Microphones app on the glasses.";
/** "Mar 4, 2026, 2:15 PM" in the phone's locale. */
export function formatDateTime(ms: number): string {
if (!Number.isFinite(ms) || ms <= 0) return "";
const date = new Date(ms);
return `${date.toLocaleDateString()} ${date.toLocaleTimeString([], { hour: "numeric", minute: "2-digit" })}`;
}
/** Time-of-day only, e.g. "2:15 PM" — used for per-line timestamps. */
export function formatTime(ms: number): string {
if (!Number.isFinite(ms) || ms <= 0) return "";
return new Date(ms).toLocaleTimeString([], { hour: "numeric", minute: "2-digit", second: "2-digit" });
}
/** mm:ss (hours spill into minutes: 90 min -> "90:00"). */
export function formatDuration(ms: number): string {
const totalSeconds = Math.max(0, Math.round(ms / 1000));
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}:${seconds < 10 ? "0" : ""}${seconds}`;
}
export const SENTIMENT_BUCKET_COLORS: Record<SentimentBucket, string> = {
veryPositive: "#2E7D32",
positive: "#66BB6A",
neutral: "#9E9E9E",
negative: "#EF6C00",
veryNegative: "#C62828",
};
export const EMOTION_COLORS: Record<EmotionLabel, string> = {
happy: "#81C784",
excited: "#AED581",
calm: "#4DB6AC",
relieved: "#66BB6A",
confident: "#4FC3F7",
surprised: "#BA68C8",
confused: "#FFB74D",
bored: "#90A4AE",
anxious: "#FFA726",
embarrassed: "#F06292",
frustrated: "#FF8A65",
sad: "#7986CB",
angry: "#E57373",
disgusted: "#A1887F",
neutral: "#BDBDBD",
};
export function emotionColor(emotion: string): string {
return EMOTION_COLORS[emotion as EmotionLabel] ?? EMOTION_COLORS.neutral;
}
+24
View File
@@ -0,0 +1,24 @@
import { EventData, NavigatedData, Page } from "@nativescript/core";
import { ConversationViewModel } from "./conversation-view-model";
type ConversationContext = { sessionId?: number } | undefined;
export function navigatingTo(args: NavigatedData): void {
const page = args.object as Page;
let model = page.bindingContext as ConversationViewModel | undefined;
if (!model) {
const context = args.context as ConversationContext;
model = new ConversationViewModel(context?.sessionId ?? 0);
page.bindingContext = model;
}
model.reload();
}
export function unloaded(args: EventData): void {
const page = args.object as Page;
const model = page.bindingContext as ConversationViewModel | undefined;
// The page has no forward navigation, so unloaded means it's going away:
// stop playback and release the MediaPlayer.
model?.dispose();
}
+49
View File
@@ -0,0 +1,49 @@
<Page xmlns="http://schemas.nativescript.org/tns.xsd" navigatingTo="navigatingTo" unloaded="unloaded">
<ActionBar title="{{ pageTitle }}">
<ActionItem text="Re-diarize" tap="{{ onRediarizeTap }}" android.position="popup" />
<ActionItem text="Split speaker..." tap="{{ onSplitSpeakerTap }}" android.position="popup" />
<ActionItem text="Delete conversation" tap="{{ onDeleteTap }}" android.position="popup" />
</ActionBar>
<GridLayout rows="auto,auto,auto,*" class="p-8">
<StackLayout row="0">
<Label text="{{ headerDate }}" class="conversation-meta" />
<Label text="{{ headerMeta }}" class="conversation-meta" textWrap="true" />
</StackLayout>
<Image row="1" src="{{ graphSource }}" height="120" stretch="fill" class="m-t-8" visibility="{{ graphVisibility }}" />
<Label row="2" text="{{ status }}" class="conversation-status m-t-4" textWrap="true" visibility="{{ statusVisibility }}" />
<ScrollView row="3" class="m-t-8">
<StackLayout>
<Label text="{{ emptyMessage }}" class="conversation-empty" textWrap="true" visibility="{{ emptyVisibility }}" />
<Repeater items="{{ transcriptRows }}">
<Repeater.itemsLayout>
<StackLayout />
</Repeater.itemsLayout>
<Repeater.itemTemplate>
<StackLayout class="{{ rowClass }}" tap="{{ onRowTap }}" longPress="{{ onRowLongPress }}">
<GridLayout columns="*,auto">
<Label col="0" textWrap="true">
<Label.formattedText>
<FormattedString>
<Span text="{{ speakerPrefix }}" color="{{ speakerColor }}" fontWeight="bold" />
<Span text="{{ text }}" />
</FormattedString>
</Label.formattedText>
</Label>
<Label col="1" text="{{ playIndicator }}" class="transcript-play" verticalAlignment="top" />
</GridLayout>
<Label text="{{ translation }}" class="transcript-translation" textWrap="true" visibility="{{ translationVisibility }}" />
<StackLayout orientation="horizontal" class="m-t-4">
<Label text="{{ timeLabel }}" class="transcript-time" />
<Label text="{{ emotionLabel }}" class="transcript-emotion m-l-6" backgroundColor="{{ emotionColor }}" visibility="{{ emotionVisibility }}" />
</StackLayout>
</StackLayout>
</Repeater.itemTemplate>
</Repeater>
</StackLayout>
</ScrollView>
</GridLayout>
</Page>
+159
View File
@@ -0,0 +1,159 @@
import { inferredEmotionFilter, type EmotionLabel } from "../apps/microphones/sentiment";
import { NON_WORD_RUN } from "../util/unicode-class";
/**
* Natural-language search parsing for the conversations list. The grammar is
* deliberately small and lossy-safe: recognized phrases become structured
* filters and are stripped from the query; everything left over is passed on
* as a plain text query.
*
* Recognized date phrases (case-insensitive):
* - "past/last N hours/days/weeks/months" (N as digits, "one".."twelve",
* "a"/"an", "couple (of)", "few"; N omitted means 1), with optional
* leading "in/over/during/from" and "the" -> sinceMs
* - "today", "this week", "this month" -> sinceMs at the period start
* - "yesterday" -> sinceMs/untilMs bracketing
* that day (untilMs is dropped when combined with an open-ended phrase)
*
* Emotion: `inferredEmotionFilter` over the date-stripped query; when it
* names exactly one emotion, that becomes the filter and the tokens that
* triggered it (checked as unigrams and bigrams, so "fed up" works) are
* stripped. When any filter was recognized, connective filler words
* ("conversations where the speaker was ...") are stripped too, so the
* residual text query is only the meaningful content words.
*/
export type ParsedConversationQuery = {
sinceMs: number | null;
untilMs: number | null;
emotion: EmotionLabel | null;
text: string;
};
const NUMBER_WORDS: Record<string, number> = {
a: 1,
an: 1,
one: 1,
two: 2,
three: 3,
four: 4,
five: 5,
six: 6,
seven: 7,
eight: 8,
nine: 9,
ten: 10,
eleven: 11,
twelve: 12,
couple: 2,
few: 3,
};
const UNIT_MS: Record<string, number> = {
hour: 3_600_000,
day: 86_400_000,
week: 7 * 86_400_000,
month: 30 * 86_400_000,
};
const RELATIVE_RANGE_PATTERN =
/\b(?:(?:in|over|during|from)\s+)?(?:the\s+)?(?:past|last)\s+(?:(\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|couple(?:\s+of)?|few|a|an)\s+)?(hour|day|week|month)s?\b/g;
/**
* Filler words that carry no search content once the query has been
* understood as a filter request ("conversations where the speaker was
* angry"). Only stripped when an emotion or date filter matched, so ordinary
* text queries pass through untouched.
*/
const FILLER_WORDS = new Set<string>([
"conversations", "conversation", "chats", "chat", "sessions", "session",
"where", "when", "which", "who", "that", "whose",
"the", "a", "an", "was", "were", "is", "are", "be", "being", "been",
"speaker", "speakers", "someone", "anybody", "anyone", "people", "person",
"sounded", "sounds", "felt", "feels", "feeling", "seemed", "seems",
"show", "me", "find", "search", "list", "get", "all", "any",
"i", "we", "they", "he", "she", "it", "you",
"in", "of", "with", "about", "on", "at", "and", "or", "to",
"talked", "talking", "talks", "said", "saying",
]);
function parseCount(word: string | undefined): number {
if (!word) return 1;
const normalized = word.replace(/\s+of$/, "").trim();
if (/^\d+$/.test(normalized)) return Math.max(1, parseInt(normalized, 10));
return NUMBER_WORDS[normalized] ?? 1;
}
function startOfDay(nowMs: number): number {
const date = new Date(nowMs);
date.setHours(0, 0, 0, 0);
return date.getTime();
}
function tokenize(text: string): string[] {
return text.split(NON_WORD_RUN).filter((token) => token.length > 0);
}
export function parseConversationQuery(raw: string, nowMs: number = Date.now()): ParsedConversationQuery {
let working = raw.toLowerCase();
const sinceCandidates: number[] = [];
let untilMs: number | null = null;
let openEndedRange = false;
working = working.replace(RELATIVE_RANGE_PATTERN, (_match, count: string | undefined, unit: string) => {
sinceCandidates.push(nowMs - parseCount(count) * UNIT_MS[unit]!);
openEndedRange = true;
return " ";
});
if (/\byesterday\b/.test(working)) {
const todayStart = startOfDay(nowMs);
sinceCandidates.push(todayStart - UNIT_MS.day!);
untilMs = todayStart;
working = working.replace(/\byesterday\b/g, " ");
}
if (/\btoday\b/.test(working)) {
sinceCandidates.push(startOfDay(nowMs));
openEndedRange = true;
working = working.replace(/\btoday\b/g, " ");
}
if (/\bthis\s+week\b/.test(working)) {
const now = new Date(nowMs);
sinceCandidates.push(startOfDay(nowMs) - now.getDay() * UNIT_MS.day!);
openEndedRange = true;
working = working.replace(/\bthis\s+week\b/g, " ");
}
if (/\bthis\s+month\b/.test(working)) {
const now = new Date(nowMs);
sinceCandidates.push(new Date(now.getFullYear(), now.getMonth(), 1).getTime());
openEndedRange = true;
working = working.replace(/\bthis\s+month\b/g, " ");
}
// "yesterday" alone brackets that day, but combined with an open-ended
// phrase ("yesterday and the past week") the upper bound would silently
// hide results the other phrase asked for.
if (openEndedRange) untilMs = null;
const sinceMs = sinceCandidates.length > 0 ? Math.min(...sinceCandidates) : null;
const emotion = inferredEmotionFilter(working);
let tokens = tokenize(working);
if (emotion) {
const kept: string[] = [];
for (let i = 0; i < tokens.length; i++) {
if (i + 1 < tokens.length && inferredEmotionFilter(`${tokens[i]} ${tokens[i + 1]}`) === emotion) {
i++;
continue;
}
if (inferredEmotionFilter(tokens[i]!) === emotion) continue;
kept.push(tokens[i]!);
}
tokens = kept;
}
if (emotion || sinceMs !== null) {
tokens = tokens.filter((token) => !FILLER_WORDS.has(token));
}
return { sinceMs, untilMs, emotion, text: tokens.join(" ") };
}
+467
View File
@@ -0,0 +1,467 @@
import { Frame, ImageSource, Observable, Screen, Utils, action, confirm, fromObject } from "@nativescript/core";
import { rediarizeSession } from "../apps/microphones/rediarize";
import { summarize, type AnalyzedLine, type EmotionLabel } from "../apps/microphones/sentiment";
import { speakerRegistry } from "../apps/microphones/speakers";
import {
conversationStore,
emotionColor,
formatDateTime,
formatDuration,
formatTime,
type SegmentRow,
type SessionRow,
} from "./conversation-format";
declare const android: any;
/**
* One transcript line. Rows are Observables so playback highlighting can
* update just the affected rows instead of rebuilding the whole Repeater.
*/
export type TranscriptRow = Observable;
const UNKNOWN_SPEAKER_COLOR = "#90A4AE";
const GRAPH_HEIGHT_DIP = 120;
export class ConversationViewModel extends Observable {
private readonly sessionId: number;
private session: SessionRow | null = null;
private segments: SegmentRow[] = [];
private rows: TranscriptRow[] = [];
private player: any = null; // android.media.MediaPlayer
private activeRowIndex = -1;
private playing = false;
private rediarizeRunning = false;
private _pageTitle = "Conversation";
private _headerDate = "";
private _headerMeta = "";
private _status = "";
private _emptyMessage = "";
private _graphSource: ImageSource | null = null;
constructor(sessionId: number) {
super();
this.sessionId = sessionId;
}
// --- bound properties ------------------------------------------------------
get pageTitle(): string {
return this._pageTitle;
}
get headerDate(): string {
return this._headerDate;
}
get headerMeta(): string {
return this._headerMeta;
}
get status(): string {
return this._status;
}
set status(value: string) {
if (this._status !== value) {
this._status = value;
this.notifyPropertyChange("status", value);
this.notifyPropertyChange("statusVisibility", this.statusVisibility);
}
}
get statusVisibility(): "visible" | "collapse" {
return this._status ? "visible" : "collapse";
}
get emptyMessage(): string {
return this._emptyMessage;
}
get emptyVisibility(): "visible" | "collapse" {
return this._emptyMessage ? "visible" : "collapse";
}
get graphSource(): ImageSource | null {
return this._graphSource;
}
get graphVisibility(): "visible" | "collapse" {
return this._graphSource ? "visible" : "collapse";
}
get transcriptRows(): TranscriptRow[] {
return this.rows;
}
// --- lifecycle -------------------------------------------------------------
reload(): void {
if (!global.isAndroid) {
this._emptyMessage = "Conversation review is only available on Android.";
this.notifyPropertyChange("emptyMessage", this._emptyMessage);
this.notifyPropertyChange("emptyVisibility", this.emptyVisibility);
return;
}
this.stopPlaybackHighlight();
try {
const sessions = JSON.parse(String(conversationStore().querySessions("{}"))) as SessionRow[];
this.session = sessions.find((row) => row.id === this.sessionId) ?? null;
this.segments = JSON.parse(String(conversationStore().querySegments(this.sessionId))) as SegmentRow[];
} catch (error) {
console.error(`conversation load failed: ${String(error)}`);
this.session = null;
this.segments = [];
}
this.refreshHeader();
this.rebuildRows();
this.rebuildGraph();
}
/** Release the MediaPlayer when the page goes away. */
dispose(): void {
this.stopPlaybackHighlight();
if (this.player) {
try {
this.player.release();
} catch {
// already released
}
this.player = null;
}
}
// --- header / rows ---------------------------------------------------------
private refreshHeader(): void {
const session = this.session;
this._pageTitle = session?.title || (session ? formatDateTime(session.startedAt) : "Conversation");
this._headerDate = session ? formatDateTime(session.startedAt) : "";
if (session) {
const durationMs = session.endedAt > session.startedAt ? session.endedAt - session.startedAt : 0;
const speakerNames = (session.speakers ?? []).map((speaker) => speaker.name).join(", ");
const parts = [formatDuration(durationMs), `${session.segmentCount} lines`];
if (speakerNames) parts.push(speakerNames);
if (!session.audioPath) parts.push("no recording");
this._headerMeta = parts.join(" · ");
} else {
this._headerMeta = "";
}
this._emptyMessage = !session
? "This conversation is no longer in the store."
: this.segments.length === 0
? "No transcript lines were saved for this conversation."
: "";
this.notifyPropertyChange("pageTitle", this._pageTitle);
this.notifyPropertyChange("headerDate", this._headerDate);
this.notifyPropertyChange("headerMeta", this._headerMeta);
this.notifyPropertyChange("emptyMessage", this._emptyMessage);
this.notifyPropertyChange("emptyVisibility", this.emptyVisibility);
}
private rebuildRows(): void {
const speakerById = new Map<number, { name: string; color: string }>();
for (const speaker of this.session?.speakers ?? []) {
speakerById.set(speaker.id, speaker);
}
// Session speakers only cover voices heard in this session; a reassignment
// to a brand-new speaker still needs a name, so fall back to the registry.
for (const profile of speakerRegistry.all()) {
if (!speakerById.has(profile.id)) {
speakerById.set(profile.id, { name: profile.name, color: profile.color });
}
}
this.rows = this.segments.map((segment, index) => {
const speaker = segment.speakerId !== null ? speakerById.get(segment.speakerId) : undefined;
const emotion = (segment.emotion || "neutral") as string;
const row = fromObject({
speakerPrefix: `${speaker?.name ?? "Unknown"}: `,
speakerColor: speaker?.color || UNKNOWN_SPEAKER_COLOR,
text: segment.text ?? "",
translation: segment.translation ?? "",
translationVisibility: segment.translation ? "visible" : "collapse",
emotionLabel: emotion,
emotionColor: emotionColor(emotion),
emotionVisibility: emotion !== "neutral" ? "visible" : "collapse",
timeLabel: formatTime(segment.startedAt),
playIndicator: "",
rowClass: "transcript-row",
onRowTap: () => this.onRowTapped(index),
onRowLongPress: () => {
void this.onRowLongPressed(index);
},
});
return row;
});
this.notifyPropertyChange("transcriptRows", this.rows);
}
// --- sentiment graph -------------------------------------------------------
/**
* Renders the per-line sentiment trend into a Bitmap: a horizontal zero
* line, a polyline of scores over the conversation, and a dot per trend
* point colored by the dominant emotion of that stretch.
*/
private rebuildGraph(): void {
this._graphSource = null;
if (global.isAndroid && this.segments.length > 1) {
try {
this._graphSource = this.drawGraph();
} catch (error) {
console.error(`sentiment graph failed: ${String(error)}`);
}
}
this.notifyPropertyChange("graphSource", this._graphSource);
this.notifyPropertyChange("graphVisibility", this.graphVisibility);
}
private drawGraph(): ImageSource | null {
const lines: AnalyzedLine[] = this.segments.map((segment) => ({
score: segment.sentiment ?? 0,
emotion: (segment.emotion || "neutral") as EmotionLabel,
emotionConfidence: 1,
}));
const trend = summarize(lines).trend;
if (trend.length < 2) return null;
const scale = Screen.mainScreen.scale;
const width = Math.max(320, Math.round(Screen.mainScreen.widthDIPs * scale));
const height = Math.round(GRAPH_HEIGHT_DIP * scale);
const padding = Math.round(10 * scale);
const bitmap = android.graphics.Bitmap.createBitmap(width, height, android.graphics.Bitmap.Config.ARGB_8888);
const canvas = new android.graphics.Canvas(bitmap);
canvas.drawColor(android.graphics.Color.parseColor("#26909090"));
const xAt = (i: number): number => padding + (i / (trend.length - 1)) * (width - 2 * padding);
const yAt = (score: number): number => {
const clamped = Math.max(-1, Math.min(1, score));
return height / 2 - clamped * (height / 2 - padding);
};
const paint = new android.graphics.Paint(android.graphics.Paint.ANTI_ALIAS_FLAG);
// Zero line.
paint.setStyle(android.graphics.Paint.Style.STROKE);
paint.setStrokeWidth(1 * scale);
paint.setColor(android.graphics.Color.parseColor("#80909090"));
canvas.drawLine(padding, height / 2, width - padding, height / 2, paint);
// Score polyline.
const path = new android.graphics.Path();
path.moveTo(xAt(0), yAt(trend[0].score));
for (let i = 1; i < trend.length; i++) {
path.lineTo(xAt(i), yAt(trend[i].score));
}
paint.setStrokeWidth(2 * scale);
paint.setColor(android.graphics.Color.parseColor("#78909C"));
canvas.drawPath(path, paint);
// Emotion dots.
paint.setStyle(android.graphics.Paint.Style.FILL);
for (let i = 0; i < trend.length; i++) {
paint.setColor(android.graphics.Color.parseColor(emotionColor(trend[i].emotion)));
canvas.drawCircle(xAt(i), yAt(trend[i].score), 3.5 * scale, paint);
}
return new ImageSource(bitmap);
}
// --- playback --------------------------------------------------------------
private onRowTapped(index: number): void {
if (!global.isAndroid) return;
const session = this.session;
const segment = this.segments[index];
if (!session?.audioPath || !segment) return;
// Lines predating the recording (or a mid-session codec restart) may miss
// an offset; fall back to the wall-clock delta from the session start.
const offsetMs =
segment.audioOffsetMs !== null && segment.audioOffsetMs !== undefined
? segment.audioOffsetMs
: Math.max(0, segment.startedAt - session.startedAt);
if (index === this.activeRowIndex && this.player) {
if (this.playing) {
try {
this.player.pause();
} catch {
// fall through to state update; a dead player just loses the toggle
}
this.playing = false;
} else {
try {
this.player.start();
this.playing = true;
} catch {
this.playing = false;
}
}
this.updateRowPlayback();
return;
}
if (!this.ensurePlayer(session.audioPath)) return;
try {
this.player.seekTo(offsetMs);
this.player.start();
this.playing = true;
this.activeRowIndex = index;
} catch (error) {
this.status = `Playback failed: ${String(error)}`;
this.playing = false;
this.activeRowIndex = -1;
}
this.updateRowPlayback();
}
private ensurePlayer(audioPath: string): boolean {
if (this.player) return true;
try {
const player = new android.media.MediaPlayer();
player.setDataSource(audioPath);
player.prepare();
player.setOnCompletionListener(
new android.media.MediaPlayer.OnCompletionListener({
onCompletion: () => {
Utils.executeOnMainThread(() => {
this.playing = false;
this.updateRowPlayback();
});
},
}),
);
this.player = player;
return true;
} catch (error) {
this.status = `Could not open the recording: ${String(error)}`;
return false;
}
}
private stopPlaybackHighlight(): void {
if (this.player && this.playing) {
try {
this.player.pause();
} catch {
// ignore; the player may already be gone
}
}
this.playing = false;
this.activeRowIndex = -1;
}
private updateRowPlayback(): void {
for (let i = 0; i < this.rows.length; i++) {
const row = this.rows[i];
const isActive = i === this.activeRowIndex;
const rowClass = isActive ? "transcript-row transcript-row-playing" : "transcript-row";
const indicator = isActive ? (this.playing ? "⏸" : "▶") : "";
if (row.get("rowClass") !== rowClass) row.set("rowClass", rowClass);
if (row.get("playIndicator") !== indicator) row.set("playIndicator", indicator);
}
}
// --- actions ---------------------------------------------------------------
onRediarizeTap(): void {
if (!global.isAndroid || this.rediarizeRunning) return;
this.rediarizeRunning = true;
this.status = "Re-diarizing… 0%";
rediarizeSession(this.sessionId, {}, {
onProgress: (fraction) => {
Utils.executeOnMainThread(() => {
this.status = `Re-diarizing… ${Math.round(fraction * 100)}%`;
});
},
onDone: (result) => {
Utils.executeOnMainThread(() => {
this.rediarizeRunning = false;
this.status =
`Re-diarization done: ${result.clusters} voices, ` +
`${result.reassignedSegments} lines reassigned, ${result.newSpeakers} new speakers.`;
this.reload();
});
},
onError: (message) => {
Utils.executeOnMainThread(() => {
this.rediarizeRunning = false;
this.status = message;
});
},
});
}
async onSplitSpeakerTap(): Promise<void> {
if (!global.isAndroid) return;
const speakers = this.session?.speakers ?? [];
if (speakers.length === 0) {
this.status = "This conversation has no speakers to split.";
return;
}
const picked = await action({
title: "Split speaker",
message: "Move one speaker's lines in this conversation to a new profile.",
cancelButtonText: "Cancel",
actions: speakers.map((speaker) => speaker.name),
});
const chosen = speakers.find((speaker) => speaker.name === picked);
if (!chosen) return;
const confirmed = await confirm({
title: "Split speaker",
message: `Move ${chosen.name}'s lines in this conversation to a new speaker?`,
okButtonText: "Split",
cancelButtonText: "Cancel",
});
if (!confirmed) return;
speakerRegistry.splitFromSession(this.sessionId, chosen.id);
this.status = "";
this.reload();
}
async onDeleteTap(): Promise<void> {
if (!global.isAndroid) return;
const confirmed = await confirm({
title: "Delete conversation",
message: "Delete this conversation, its transcript, and its recording link? This cannot be undone.",
okButtonText: "Delete",
cancelButtonText: "Cancel",
});
if (!confirmed) return;
this.dispose();
try {
conversationStore().deleteSession(this.sessionId);
} catch (error) {
this.status = `Delete failed: ${String(error)}`;
return;
}
Frame.topmost()?.goBack();
}
private async onRowLongPressed(index: number): Promise<void> {
if (!global.isAndroid) return;
const segment = this.segments[index];
if (!segment) return;
const profiles = speakerRegistry.all();
if (profiles.length === 0) return;
const picked = await action({
title: "Reassign speaker",
message: "Whose line is this?",
cancelButtonText: "Cancel",
actions: profiles.map((profile) => profile.name),
});
const chosen = profiles.find((profile) => profile.name === picked);
if (!chosen || chosen.id === segment.speakerId) return;
try {
conversationStore().reassignSegmentSpeaker(segment.id, chosen.id);
} catch (error) {
this.status = `Reassign failed: ${String(error)}`;
return;
}
speakerRegistry.reload();
this.reload();
}
}
+19
View File
@@ -0,0 +1,19 @@
import { NavigatedData, Page } from "@nativescript/core";
import { ConversationsViewModel } from "./conversations-view-model";
type ConversationsContext = { speakerId?: number } | undefined;
export function navigatingTo(args: NavigatedData): void {
const page = args.object as Page;
// Preserve the model across back-navigation (returning from a conversation
// or the speakers page) so the search and filters aren't lost; reload every
// time so deletions, merges, and renames made elsewhere show up.
let model = page.bindingContext as ConversationsViewModel | undefined;
if (!model) {
const context = args.context as ConversationsContext;
model = new ConversationsViewModel({ speakerId: context?.speakerId });
page.bindingContext = model;
}
model.reload();
}
+56
View File
@@ -0,0 +1,56 @@
<Page xmlns="http://schemas.nativescript.org/tns.xsd" navigatingTo="navigatingTo">
<ActionBar title="Conversations">
<ActionItem text="Ask AI" tap="{{ onAskTap }}" android.position="actionBar" />
<ActionItem text="Speakers" tap="{{ onSpeakersTap }}" android.position="actionBar" />
</ActionBar>
<GridLayout rows="auto,auto,*" class="p-8">
<SearchBar
row="0"
hint="Search (e.g. lunch past 2 days, angry)"
text="{{ searchText }}"
submit="{{ onSearchSubmit }}"
clear="{{ onSearchClear }}" />
<ScrollView row="1" orientation="horizontal" class="m-t-8" visibility="{{ filterChipsVisibility }}">
<StackLayout orientation="horizontal">
<Repeater items="{{ speakerChips }}">
<Repeater.itemsLayout>
<StackLayout orientation="horizontal" />
</Repeater.itemsLayout>
<Repeater.itemTemplate>
<Label text="{{ name }}" class="{{ chipClass }}" backgroundColor="{{ chipBackground }}" tap="{{ onChipTap }}" />
</Repeater.itemTemplate>
</Repeater>
</StackLayout>
</ScrollView>
<ScrollView row="2" class="m-t-8">
<StackLayout>
<Label text="{{ emptyMessage }}" class="conversation-empty" textWrap="true" visibility="{{ emptyVisibility }}" />
<Repeater items="{{ sessionRows }}">
<Repeater.itemsLayout>
<StackLayout />
</Repeater.itemsLayout>
<Repeater.itemTemplate>
<StackLayout class="conversation-row" tap="{{ onRowTap }}">
<GridLayout columns="*,auto">
<Label col="0" text="{{ title }}" class="conversation-title" textWrap="true" />
<Label col="1" text="" width="12" height="12" borderRadius="6" backgroundColor="{{ sentimentColor }}" verticalAlignment="center" />
</GridLayout>
<Label text="{{ meta }}" class="conversation-meta" />
<Repeater items="{{ speakerChips }}" visibility="{{ chipsVisibility }}" class="m-t-4">
<Repeater.itemsLayout>
<WrapLayout />
</Repeater.itemsLayout>
<Repeater.itemTemplate>
<Label text="{{ name }}" class="speaker-chip" backgroundColor="{{ chipBackground }}" />
</Repeater.itemTemplate>
</Repeater>
</StackLayout>
</Repeater.itemTemplate>
</Repeater>
</StackLayout>
</ScrollView>
</GridLayout>
</Page>
Loaded 100 of 126 files, more files were not shown because too many files have changed in this diff. Show more