From 2d717a9cdb5eeffcacdc00fc6b56f3a99b5ace2f Mon Sep 17 00:00:00 2001 From: Chris Lee Date: Tue, 1 Sep 2026 22:27:48 -0400 Subject: [PATCH 1/5] Add on-device Whisper as a second offline transcription option sherpa-onnx's native library already reads an offline Whisper config (verified against sherpa-onnx/jni/offline-recognizer.cc at v1.13.0); only the Java wrapper was a package-private stub. This adds a public OfflineWhisperModelConfig builder and wires base.en (int8) in as a second on-device model next to Moonshine. - FaceclawVoiceController: setOnboardModelKind() selects the model before start("onboard"); defaults to Moonshine. Whisper skips the 700ms live-partial redecode (each call re-encodes the whole utterance plus ~10s of tail padding) and gates near-silent segments before decode. - asr-model.ts: ASR_MODELS registry with per-model download state. - Settings > Voice: "On-device (Moonshine)", "On-device (Whisper)", and "OpenAI (Whisper)" for the existing cloud provider (stored value unchanged); one model row per on-device model. - Microphones live captions pass "moonshine" explicitly (unchanged). Co-Authored-By: Claude Sonnet 5 Co-Authored-By: Claude Opus 5 --- .../faceclaw/app/FaceclawVoiceController.java | 124 ++++++-- .../k2fsa/sherpa/onnx/OfflineModelConfig.java | 17 +- .../onnx/OfflineWhisperModelConfig.java | 114 ++++++++ app/apps/microphones/mic-session.ts | 5 +- app/apps/microphones/microphones-app.ts | 9 +- app/native/asr-model.ts | 267 +++++++++++------- app/native/voice-control.ts | 12 +- app/ui/dashboard-settings.ts | 17 +- app/ui/dashboard/settings-menus.ts | 56 ++-- 9 files changed, 459 insertions(+), 162 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index 12680324..d6b8171d 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -13,6 +13,7 @@ import com.k2fsa.sherpa.onnx.OfflineRecognizer; import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig; import com.k2fsa.sherpa.onnx.OfflineRecognizerResult; import com.k2fsa.sherpa.onnx.OfflineStream; +import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig; import java.io.File; import java.nio.charset.StandardCharsets; @@ -61,12 +62,43 @@ public class FaceclawVoiceController { "decoder_model_merged.ort", "tokens.txt" }; + // Second on-device model: sherpa-onnx's offline Whisper backend (base.en, + // int8-quantized -- see the model-choice note in asr-model.ts). Directory + // shared with the TS-side download flow, same convention as ASR_MODEL_DIR. + // + // On every call, sherpa-onnx (offline-recognizer-whisper-impl.h, v1.13.0) + // runs Whisper's encoder over the whole buffer plus 1000 frames (~10s) of + // zero tail padding, capped at 30s. Re-decoding on Moonshine's + // TRANSCRIPT_DECODE_INTERVAL_MS live-partial cadence would repeat that + // encode, growing with the utterance, roughly 1.4x/second while the user is + // still speaking, so onboardModelKind gates that loop off for Whisper -- + // see processRecognizer(). Whisper is also known to hallucinate text on + // near-silent input; recognizeTranscriptSegment() gates that too. + private static final String ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"; + private static final String[] ASR_WHISPER_MODEL_FILES = { + "base.en-encoder.int8.onnx", + "base.en-decoder.int8.onnx", + "base.en-tokens.txt" + }; + // Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f + // buffer) a segment is treated as silence and never reaches the Whisper + // recognizer at all, rather than risking a hallucinated non-answer. Picked + // conservatively low (well under typical mic noise floor already seen in + // this pipeline's normalization target) -- UNTESTED on real hardware, tune + // against real glasses captures rather than trusting this number. + private static final float WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f; private enum VoiceInputMode { - ONBOARD, // on-phone Moonshine transcription + ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind) CLOUD // decode locally, emit PCM for a cloud recognizer on the TS side } + /** Which on-device model ONBOARD mode uses. Set via setOnboardModelKind() before start(). */ + private enum OnboardModelKind { + MOONSHINE, + WHISPER + } + private final Context appContext; private final Handler mainHandler = new Handler(Looper.getMainLooper()); private final Object lock = new Object(); @@ -86,6 +118,7 @@ public class FaceclawVoiceController { private volatile boolean usePhoneMic; private boolean activePhoneMic; private VoiceInputMode mode = VoiceInputMode.CLOUD; + private volatile OnboardModelKind onboardModelKind = OnboardModelKind.MOONSHINE; private OfflineRecognizer recognizer; private FaceclawLc3Decoder lc3Decoder; private final float[] transcriptSamples = new float[TRANSCRIPT_SEGMENT_MAX_SAMPLES]; @@ -150,6 +183,17 @@ public class FaceclawVoiceController { this.saveRecordings = saveRecordings; } + /** + * Which on-device model {@link #start}("onboard") should load: "whisper" + * selects the second on-device model (sherpa-onnx offline Whisper); any + * other value (including null/absent) keeps the existing Moonshine model, + * so callers that never call this see unchanged behavior. Must be set + * before {@link #start}; has no effect in CLOUD mode. + */ + public void setOnboardModelKind(String kind) { + this.onboardModelKind = "whisper".equals(kind) ? OnboardModelKind.WHISPER : OnboardModelKind.MOONSHINE; + } + /** * When true, watch the decoded PCM and fire {@code onSpeechEnd} once the * speaker stops. Used by hands-free ("Hey Even") capture, which has no @@ -310,14 +354,15 @@ public class FaceclawVoiceController { try { deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; + OnboardModelKind currentOnboardKind = onboardModelKind; if (currentMode == VoiceInputMode.ONBOARD) { - File modelDir = findAsrModelDir(); + File modelDir = findAsrModelDir(currentOnboardKind); if (modelDir == null) { emitStatus("Voice model not downloaded (see Settings > Voice)."); return; } emitStatus("Loading transcription model..."); - recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir)); + recognizer = new OfflineRecognizer(buildRecognizerConfig(modelDir, currentOnboardKind)); resetTranscriptState(); lastTranscript = ""; } @@ -453,32 +498,48 @@ public class FaceclawVoiceController { return b.array(); } - private OfflineRecognizerConfig buildRecognizerConfig(File modelDir) { + private OfflineRecognizerConfig buildRecognizerConfig(File modelDir, OnboardModelKind kind) { + OfflineModelConfig.Builder modelConfig = OfflineModelConfig.builder() + .setNumThreads(1); + if (kind == OnboardModelKind.WHISPER) { + modelConfig + .setWhisper(OfflineWhisperModelConfig.builder() + .setEncoder(new File(modelDir, "base.en-encoder.int8.onnx").getAbsolutePath()) + .setDecoder(new File(modelDir, "base.en-decoder.int8.onnx").getAbsolutePath()) + .setLanguage("en") + .setTask("transcribe") + .build()) + .setTokens(new File(modelDir, "base.en-tokens.txt").getAbsolutePath()); + } else { + modelConfig + .setMoonshine(OfflineMoonshineModelConfig.builder() + .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) + .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) + .build()) + .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()); + } return OfflineRecognizerConfig.builder() .setFeatureConfig(FeatureConfig.builder() .setSampleRate(SAMPLE_RATE) .setFeatureDim(FEATURE_DIM) .build()) - .setModelConfig(OfflineModelConfig.builder() - .setMoonshine(OfflineMoonshineModelConfig.builder() - .setEncoder(new File(modelDir, "encoder_model.ort").getAbsolutePath()) - .setMergedDecoder(new File(modelDir, "decoder_model_merged.ort").getAbsolutePath()) - .build()) - .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) - .setNumThreads(1) - .build()) + .setModelConfig(modelConfig.build()) .build(); } /** - * The Moonshine model directory, populated by the download flow in - * asr-model.ts (releases before 0.5.0 copied the same files out of the - * APK, so upgraded installs are already complete). Null when any file is + * The on-device model directory for the given kind, populated by the + * download flow in asr-model.ts (releases before 0.5.0 copied the + * Moonshine files out of the APK directly, so upgraded installs are + * already complete for that model). Null when any expected file is * missing, i.e. the model still needs to be downloaded. */ - private File findAsrModelDir() { - File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + ASR_MODEL_DIR); - for (String fileName : ASR_MODEL_FILES) { + private File findAsrModelDir(OnboardModelKind kind) { + boolean whisper = kind == OnboardModelKind.WHISPER; + String dirName = whisper ? ASR_WHISPER_MODEL_DIR : ASR_MODEL_DIR; + String[] fileNames = whisper ? ASR_WHISPER_MODEL_FILES : ASR_MODEL_FILES; + File modelDir = new File(appContext.getFilesDir(), ASR_ROOT + File.separator + dirName); + for (String fileName : fileNames) { File file = new File(modelDir, fileName); if (!file.exists() || file.length() == 0) { return null; @@ -658,6 +719,16 @@ public class FaceclawVoiceController { private void processRecognizer(float[] samples) { appendTranscriptSamples(samples); + // Each Whisper call re-encodes the whole buffer plus ~10s of tail + // padding (see the ASR_WHISPER_* comment above), + // so it skips the live-partial redecode Moonshine does on this + // interval and only decodes when a segment commits (8s buffer fill) + // or the utterance ends (decodeTranscript(true) in runLoop()). This + // means no live preview text while speaking in Whisper mode -- status + // stays "Listening..." until release. This is a deliberate tradeoff. + if (onboardModelKind == OnboardModelKind.WHISPER) { + return; + } long now = SystemClock.elapsedRealtime(); if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES && now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS) { @@ -767,6 +838,14 @@ public class FaceclawVoiceController { return ""; } float[] segment = Arrays.copyOf(transcriptSamples, sampleCount); + // Whisper hallucinates text on near-silent input (a known quirk of the + // model, not this pipeline); gate it on the PRE-normalization peak, since + // normalizePeak() below would otherwise amplify true silence right up to + // the target level and hide the very thing being checked for. Moonshine + // does not share this failure mode in practice, so it is left unchanged. + if (onboardModelKind == OnboardModelKind.WHISPER && peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD) { + return ""; + } normalizePeak(segment); OfflineStream offlineStream = currentRecognizer.createStream(); try { @@ -780,7 +859,7 @@ public class FaceclawVoiceController { } } - private static void normalizePeak(float[] samples) { + private static float peakAmplitude(float[] samples) { float peak = 0f; for (float s : samples) { float a = Math.abs(s); @@ -788,6 +867,11 @@ public class FaceclawVoiceController { peak = a; } } + return peak; + } + + private static void normalizePeak(float[] samples) { + float peak = peakAmplitude(samples); if (peak <= 0f) { return; } @@ -805,7 +889,7 @@ public class FaceclawVoiceController { (committedTranscriptSampleCount + transcriptSampleCount) / (double) SAMPLE_RATE; String preview = text.length() <= TRANSCRIPT_LOG_PREVIEW_CHARS ? text : text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."; - Log.i(TAG, "Moonshine decode final=" + isFinal + Log.i(TAG, (onboardModelKind == OnboardModelKind.WHISPER ? "Whisper" : "Moonshine") + " decode final=" + isFinal + " audioSec=" + String.format(java.util.Locale.US, "%.2f", totalAudioSec) + " segmentAudioSec=" + String.format(java.util.Locale.US, "%.2f", segmentSampleCount / (double) SAMPLE_RATE) + " textLen=" + text.length() + " text=\"" + preview + "\""); diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java index 0d916bf8..ee130387 100644 --- a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java @@ -162,7 +162,7 @@ public class OfflineModelConfig { public static class Builder { private OfflineTransducerModelConfig transducer = new OfflineTransducerModelConfig(); private OfflineParaformerModelConfig paraformer = new OfflineParaformerModelConfig(); - private OfflineWhisperModelConfig whisper = new OfflineWhisperModelConfig(); + private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build(); private OfflineFireRedAsrModelConfig fireRedAsr = new OfflineFireRedAsrModelConfig(); private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build(); private OfflineNemoEncDecCtcModelConfig nemo = new OfflineNemoEncDecCtcModelConfig(); @@ -191,6 +191,11 @@ public class OfflineModelConfig { return this; } + public Builder setWhisper(OfflineWhisperModelConfig whisper) { + this.whisper = whisper; + return this; + } + public Builder setTokens(String tokens) { this.tokens = tokens; return this; @@ -228,16 +233,6 @@ class OfflineParaformerModelConfig { private final QnnConfig qnnConfig = new QnnConfig(); } -class OfflineWhisperModelConfig { - private final String encoder = ""; - private final String decoder = ""; - private final String language = "en"; - private final String task = "transcribe"; - private final int tailPaddings = 1000; - private final boolean enableTokenTimestamps = false; - private final boolean enableSegmentTimestamps = false; -} - class OfflineFireRedAsrModelConfig { private final String encoder = ""; private final String decoder = ""; diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java new file mode 100644 index 00000000..51245a2b --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OfflineWhisperModelConfig.java @@ -0,0 +1,114 @@ +package com.k2fsa.sherpa.onnx; + +/** + * Config for sherpa-onnx's offline (non-streaming) Whisper backend. + * + * Field names and types here are load-bearing: the native JNI glue reads + * them by name via reflection (GetFieldID), not through the getters below, + * so they must match sherpa-onnx's own OfflineWhisperModelConfig exactly + * (verified against sherpa-onnx/jni/offline-recognizer.cc at the pinned + * release tag: it unconditionally reads encoder, decoder, language, task, + * tailPaddings, enableTokenTimestamps and enableSegmentTimestamps off the + * OfflineModelConfig's "whisper" field). This mirrors the + * OfflineMoonshineModelConfig.java pattern already used in this package. + */ +public class OfflineWhisperModelConfig { + private final String encoder; + private final String decoder; + private final String language; + private final String task; + private final int tailPaddings; + private final boolean enableTokenTimestamps; + private final boolean enableSegmentTimestamps; + + private OfflineWhisperModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + this.language = builder.language; + this.task = builder.task; + this.tailPaddings = builder.tailPaddings; + this.enableTokenTimestamps = builder.enableTokenTimestamps; + this.enableSegmentTimestamps = builder.enableSegmentTimestamps; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public String getLanguage() { + return language; + } + + public String getTask() { + return task; + } + + public int getTailPaddings() { + return tailPaddings; + } + + public boolean getEnableTokenTimestamps() { + return enableTokenTimestamps; + } + + public boolean getEnableSegmentTimestamps() { + return enableSegmentTimestamps; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + private String language = "en"; + private String task = "transcribe"; + private int tailPaddings = 1000; + private boolean enableTokenTimestamps = false; + private boolean enableSegmentTimestamps = false; + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public Builder setLanguage(String language) { + this.language = language; + return this; + } + + public Builder setTask(String task) { + this.task = task; + return this; + } + + public Builder setTailPaddings(int tailPaddings) { + this.tailPaddings = tailPaddings; + return this; + } + + public Builder setEnableTokenTimestamps(boolean enableTokenTimestamps) { + this.enableTokenTimestamps = enableTokenTimestamps; + return this; + } + + public Builder setEnableSegmentTimestamps(boolean enableSegmentTimestamps) { + this.enableSegmentTimestamps = enableSegmentTimestamps; + return this; + } + + public OfflineWhisperModelConfig build() { + return new OfflineWhisperModelConfig(this); + } + } +} diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index e2d9cf5b..5f019ab8 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -730,7 +730,10 @@ class MicSession { }, }); engine.setListener(this.captionListenerProxy); - engine.setAsrModelDir(isAsrModelReady() ? moonshineModelDir() : null); + // Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine) + // that only ever uses Moonshine; unaffected by the two-model on-device + // Whisper addition in native/voice-control.ts's push-to-talk path. + engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null); engine.setSpeakerModelPath(micModelPath("speaker-embedding")); engine.start(); this.captionEngine = engine; diff --git a/app/apps/microphones/microphones-app.ts b/app/apps/microphones/microphones-app.ts index d4a0b0ee..ef0d491e 100644 --- a/app/apps/microphones/microphones-app.ts +++ b/app/apps/microphones/microphones-app.ts @@ -328,17 +328,20 @@ function voiceSpeakersMenu(): MenuLayer { () => startMicModelDownload("speaker-embedding"), ), modelItem( + // Live captions always use Moonshine (see the matching note in + // mic-session.ts); the second on-device model (Whisper) is a + // push-to-talk dictation option only, managed in Settings > Voice. "Caption ASR model", - isAsrModelReady, + () => isAsrModelReady("moonshine"), () => { - const state = asrModelState(); + const state = asrModelState("moonshine"); if (state.status === "ready") return "ready"; if (state.status === "downloading") { return `${Math.round((state.bytesDownloaded / state.totalBytes) * 100)}%`; } return "download (141 MB)"; }, - startAsrModelDownload, + () => startAsrModelDownload("moonshine"), ), modelItem( "Re-diarization model", diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts index 3c71e31a..591bdc73 100644 --- a/app/native/asr-model.ts +++ b/app/native/asr-model.ts @@ -4,50 +4,107 @@ declare const com: any; declare const java: any; /** - * Download management for the on-device transcription model (Moonshine, via - * sherpa-onnx). The model is no longer bundled in the APK; it is fetched on - * demand into the same filesDir location that earlier releases copied the - * bundled files to, so upgraded installs that already used on-device - * transcription need no download. FaceclawVoiceController reads the files - * from this directory. + * Download management for the on-device transcription models (sherpa-onnx + * offline recognizers). Models are fetched on demand into filesDir rather + * than bundled in the APK; FaceclawVoiceController reads the files from the + * directory each model's `dirName` names. * - * Mirrors the on-phone assistant model flow in llama.ts, except the model is - * three files rather than one; they download sequentially through + * Two models, same shape: + * - "moonshine": the original on-device option (three files). The model is + * no longer bundled in the APK; it is fetched into the same filesDir + * location that earlier releases copied the bundled files to, so + * upgraded installs that already used on-device transcription need no + * download. + * - "whisper-base-en": sherpa-onnx's offline Whisper backend, base.en, + * int8-quantized. Picked over tiny.en for materially better accuracy + * (Whisper's own tiny/base WER gap is real and well documented) on the + * reasoning that FaceclawVoiceController.java no longer re-decodes a + * Whisper segment on every live-partial tick — see its ASR_WHISPER_* + * comment — so base.en's extra compute is a one-time cost per utterance, + * not a cost paid ~1.4x/second while the user is speaking. tiny.en is an + * easy swap (this file's numbers only) if base.en proves too slow on a + * real phone; nobody has measured that yet. Files are csukuangfj's own + * (the sherpa-onnx maintainer's) HF mirror of the upstream sherpa-onnx + * release asset, matching the mirror-not-the-GH-tarball approach already + * used for Moonshine below. Every hash here was computed locally from a + * freshly downloaded copy of the file, not copied from a doc page. + * + * Mirrors the on-phone assistant model flow in llama.ts, except each model + * here is multiple files rather than one; they download sequentially through * FaceclawModelDownloader (resume + pinned sha256 per file). */ +export type AsrModelId = "moonshine" | "whisper-base-en"; + type AsrModelFile = { name: string; sha256: string; sizeBytes: number; }; -export const ASR_MODEL = { - label: "Moonshine (English)", - dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27", - // Hugging Face mirror of the sherpa-onnx release asset of the same name. - // The GitHub release only offers a tar.bz2, which the phone can't unpack; - // this mirror serves the same files (sha256-verified) individually. - baseUrl: - "https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/", - files: [ - { - name: "decoder_model_merged.ort", - sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d", - sizeBytes: 109424400, - }, - { - name: "encoder_model.ort", - sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf", - sizeBytes: 31326816, - }, - { - name: "tokens.txt", - sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049", - sizeBytes: 549350, - }, - ] as AsrModelFile[], - totalBytes: 141300566, +type AsrModelDef = { + label: string; + dirName: string; + baseUrl: string; + files: AsrModelFile[]; + totalBytes: number; +}; + +export const ASR_MODELS: Record = { + moonshine: { + label: "Moonshine (English)", + dirName: "sherpa-onnx-moonshine-base-en-quantized-2026-02-27", + // Hugging Face mirror of the sherpa-onnx release asset of the same name. + // The GitHub release only offers a tar.bz2, which the phone can't unpack; + // this mirror serves the same files (sha256-verified) individually. + baseUrl: + "https://huggingface.co/csukuangfj2/sherpa-onnx-moonshine-base-en-quantized-2026-02-27/resolve/main/", + files: [ + { + name: "decoder_model_merged.ort", + sha256: "d9d7b333af34bc552580576ddcf248a1c6c839e0d3b43b09afb9376ed009899d", + sizeBytes: 109424400, + }, + { + name: "encoder_model.ort", + sha256: "7c66495948d0d08ec1af454cd4b5514862ae6511e94712a60e6d83eaec8dc8cf", + sizeBytes: 31326816, + }, + { + name: "tokens.txt", + sha256: "2870d843e14c1e187bf1913a521562a63b53933814bd7f2145120468f494a049", + sizeBytes: 549350, + }, + ], + totalBytes: 141300566, + }, + "whisper-base-en": { + label: "Whisper (English, base)", + dirName: "sherpa-onnx-whisper-base-en-int8", + // csukuangfj/sherpa-onnx-whisper-base.en on Hugging Face: the sherpa-onnx + // maintainer's own mirror of the project's whisper export, individual + // files (no tar.bz2-unpack problem to begin with, but kept as the same + // per-file-with-pinned-hash shape as Moonshine above for consistency). + baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base.en/resolve/main/", + files: [ + { + name: "base.en-encoder.int8.onnx", + sha256: "ef6b936f4c9b1d90a3b68634b60c4ed8576b26172b33c2535ec0e933c9edb823", + sizeBytes: 29120534, + }, + { + name: "base.en-decoder.int8.onnx", + sha256: "f7162ad6db2dbef16cfaeaa7f945b9d7dd9c1b8d472f6aca82f2273d185e4d41", + sizeBytes: 130669978, + }, + { + name: "base.en-tokens.txt", + sha256: "306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930", + sizeBytes: 835554, + }, + ], + totalBytes: 160626066, + }, }; export type AsrModelState = { @@ -56,121 +113,141 @@ export type AsrModelState = { totalBytes: number; }; -let downloader: any = null; -// Bytes of files already fully downloaded in this run, plus progress within -// the file currently downloading; drives the aggregate percentage. -let completedBytes = 0; -let currentFileBytes = 0; -const stateListeners = new Set<(state: AsrModelState) => void>(); +type ModelRuntime = { + downloader: any; + // Bytes of files already fully downloaded in this run, plus progress within + // the file currently downloading; drives the aggregate percentage. + completedBytes: number; + currentFileBytes: number; + stateListeners: Set<(state: AsrModelState) => void>; +}; -function modelDirPath(): string { +function freshRuntime(): ModelRuntime { + return { downloader: null, completedBytes: 0, currentFileBytes: 0, stateListeners: new Set() }; +} + +const runtimes: Record = { + moonshine: freshRuntime(), + "whisper-base-en": freshRuntime(), +}; + +function modelDirPath(id: AsrModelId): string { const context = Utils.android.getApplicationContext(); - return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODEL.dirName}`; + return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/${ASR_MODELS[id].dirName}`; } -function filePath(file: AsrModelFile): string { - return `${modelDirPath()}/${file.name}`; +function filePath(id: AsrModelId, file: AsrModelFile): string { + return `${modelDirPath(id)}/${file.name}`; } -function isFilePresent(file: AsrModelFile): boolean { +function isFilePresent(id: AsrModelId, file: AsrModelFile): boolean { try { - const javaFile = new java.io.File(filePath(file)); + const javaFile = new java.io.File(filePath(id, file)); return javaFile.exists() && javaFile.length() > 0; } catch { return false; } } -export function isAsrModelReady(): boolean { +export function isAsrModelReady(id: AsrModelId): boolean { if (!global.isAndroid) return false; - return ASR_MODEL.files.every(isFilePresent); + return ASR_MODELS[id].files.every((file) => isFilePresent(id, file)); } -export function asrModelState(): AsrModelState { - if (downloader) { +export function asrModelState(id: AsrModelId): AsrModelState { + const runtime = runtimes[id]; + const totalBytes = ASR_MODELS[id].totalBytes; + if (runtime.downloader) { return { status: "downloading", - bytesDownloaded: completedBytes + currentFileBytes, - totalBytes: ASR_MODEL.totalBytes, + bytesDownloaded: runtime.completedBytes + runtime.currentFileBytes, + totalBytes, }; } return { - status: isAsrModelReady() ? "ready" : "absent", + status: isAsrModelReady(id) ? "ready" : "absent", bytesDownloaded: 0, - totalBytes: ASR_MODEL.totalBytes, + totalBytes, }; } -export function onAsrModelStateChanged(listener: (state: AsrModelState) => void): () => void { - stateListeners.add(listener); - return () => stateListeners.delete(listener); +export function onAsrModelStateChanged(id: AsrModelId, listener: (state: AsrModelState) => void): () => void { + const runtime = runtimes[id]; + runtime.stateListeners.add(listener); + return () => runtime.stateListeners.delete(listener); } -function notifyStateChanged(): void { - const state = asrModelState(); - stateListeners.forEach((listener) => listener(state)); +function notifyStateChanged(id: AsrModelId): void { + const state = asrModelState(id); + runtimes[id].stateListeners.forEach((listener) => listener(state)); } -export function startAsrModelDownload(): void { - if (!global.isAndroid || downloader || isAsrModelReady()) return; - completedBytes = ASR_MODEL.files.filter(isFilePresent).reduce((sum, f) => sum + f.sizeBytes, 0); - downloadNextFile(); - notifyStateChanged(); +export function startAsrModelDownload(id: AsrModelId): void { + const runtime = runtimes[id]; + if (!global.isAndroid || runtime.downloader || isAsrModelReady(id)) return; + runtime.completedBytes = ASR_MODELS[id].files + .filter((file) => isFilePresent(id, file)) + .reduce((sum, f) => sum + f.sizeBytes, 0); + downloadNextFile(id); + notifyStateChanged(id); } -function downloadNextFile(): void { - const nextFile = ASR_MODEL.files.find((file) => !isFilePresent(file)); +function downloadNextFile(id: AsrModelId): void { + const runtime = runtimes[id]; + const def = ASR_MODELS[id]; + const nextFile = def.files.find((file) => !isFilePresent(id, file)); if (!nextFile) { - downloader = null; - notifyStateChanged(); + runtime.downloader = null; + notifyStateChanged(id); return; } - currentFileBytes = 0; + runtime.currentFileBytes = 0; const listener = new com.faceclaw.app.FaceclawModelDownloaderListener({ onProgress: (bytes: number, _total: number) => { - currentFileBytes = Number(bytes); - notifyStateChanged(); + runtime.currentFileBytes = Number(bytes); + notifyStateChanged(id); }, onDone: () => { - completedBytes += nextFile.sizeBytes; - currentFileBytes = 0; - downloadNextFile(); + runtime.completedBytes += nextFile.sizeBytes; + runtime.currentFileBytes = 0; + downloadNextFile(id); }, onError: (message: string) => { - console.error(`Voice model download failed (${nextFile.name}): ${message}`); - downloader = null; - notifyStateChanged(); + console.error(`Voice model download failed (${id}/${nextFile.name}): ${message}`); + runtime.downloader = null; + notifyStateChanged(id); }, }); - downloader = new com.faceclaw.app.FaceclawModelDownloader( - `${ASR_MODEL.baseUrl}${nextFile.name}`, - filePath(nextFile), + runtime.downloader = new com.faceclaw.app.FaceclawModelDownloader( + `${def.baseUrl}${nextFile.name}`, + filePath(id, nextFile), nextFile.sha256, nextFile.sizeBytes, listener, ); - downloader.start(); + runtime.downloader.start(); } /** Stops the download; already-fetched bytes are kept and resumed next time. */ -export function cancelAsrModelDownload(): void { - if (!downloader) return; - downloader.cancel(); - downloader = null; - notifyStateChanged(); +export function cancelAsrModelDownload(id: AsrModelId): void { + const runtime = runtimes[id]; + if (!runtime.downloader) return; + runtime.downloader.cancel(); + runtime.downloader = null; + notifyStateChanged(id); } -export function deleteAsrModel(): void { +export function deleteAsrModel(id: AsrModelId): void { if (!global.isAndroid) return; - cancelAsrModelDownload(); + cancelAsrModelDownload(id); try { - for (const file of ASR_MODEL.files) { - new java.io.File(filePath(file)).delete(); - new java.io.File(`${filePath(file)}.part`).delete(); + for (const file of ASR_MODELS[id].files) { + new java.io.File(filePath(id, file)).delete(); + new java.io.File(`${filePath(id, file)}.part`).delete(); } - new java.io.File(modelDirPath()).delete(); + new java.io.File(modelDirPath(id)).delete(); } catch (error) { - console.error(`Voice model delete failed: ${String(error)}`); + console.error(`Voice model delete failed (${id}): ${String(error)}`); } - notifyStateChanged(); + notifyStateChanged(id); } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 2874d5a9..acfd9f31 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -26,7 +26,12 @@ export type VoiceControlState = { detail: string; }; -export type VoiceProviderKind = "onboard" | "elevenlabs" | "whisper" | "soniox"; +// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper, +// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model +// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx +// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than +// renamed, since it's a persisted settings value on real installs already. +export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** @@ -314,6 +319,9 @@ export class FaceclawVoiceControlBridge { this.cloudClient = null; this.started = true; + // Which on-device model to load; a no-op setter for every provider except + // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). + this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); this.controller?.start("onboard"); } @@ -323,7 +331,7 @@ export class FaceclawVoiceControlBridge { * than failing the capture outright. */ private createCloudClient(options: PushToTalkOptions): CloudSttClient | null { - if (options.provider === "onboard") return null; + if (options.provider === "onboard" || options.provider === "onboard-whisper") return null; const sttOptions = { apiKey: "", onTranscript: (event: CloudSttTranscriptEvent) => diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 5ac7ba8e..aad0019e 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -523,12 +523,19 @@ export const ringConnectionModeSetting = new ConfigSettingEnum = { - onboard: "On-device", + onboard: "On-device (Moonshine)", + "onboard-whisper": "On-device (Whisper)", elevenlabs: "ElevenLabs", - whisper: "Whisper", + whisper: "OpenAI (Whisper)", soniox: "Soniox", }; @@ -537,7 +544,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ label: "Transcription Provider", storageKey: "voice.provider", defaultValue: "onboard", - values: ["onboard", "elevenlabs", "whisper", "soniox"], + values: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox"], formatValue: (value) => voiceProviderLabels[value] ?? value, isDisabled: (value) => { if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; @@ -545,7 +552,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ if (value === "soniox") return sonioxApiKeySetting.get().trim().length === 0; return false; }, - description: "Speech-to-text engine for voice input. ElevenLabs, Whisper, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. On-device transcription needs the voice model downloaded (below).", + description: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. The two On-device options need their voice model downloaded (below) and never leave the phone.", }); const wakeWordActionLabels: Record = { diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index f67a1066..2dd7a745 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -13,12 +13,13 @@ import { startLocalModelDownload, } from "../../native/llama"; import { - ASR_MODEL, + ASR_MODELS, asrModelState, cancelAsrModelDownload, deleteAsrModel, onAsrModelStateChanged, startAsrModelDownload, + type AsrModelId, } from "../../native/asr-model"; import { TextViewerLayer } from "../../apps/files/text-viewer"; import type { LayerContext } from "../layers"; @@ -119,7 +120,8 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), - asrModelMenuItem(), + asrModelMenuItem("moonshine"), + asrModelMenuItem("whisper-base-en"), ], }, { @@ -344,48 +346,52 @@ function localModelMenuItem(): MenuItem { }; } -const ASR_MODEL_MB = `${Math.round(ASR_MODEL.totalBytes / 1e6)}MB`; +function asrModelMb(id: AsrModelId): string { + return `${Math.round(ASR_MODELS[id].totalBytes / 1e6)}MB`; +} -let asrModelRenderUnsub: (() => void) | null = null; +const asrModelRenderUnsub: Partial void>> = {}; -function watchAsrModelDownload(ctx: LayerContext): void { - asrModelRenderUnsub?.(); - asrModelRenderUnsub = onAsrModelStateChanged((state) => { +function watchAsrModelDownload(id: AsrModelId, ctx: LayerContext): void { + asrModelRenderUnsub[id]?.(); + asrModelRenderUnsub[id] = onAsrModelStateChanged(id, (state) => { ctx.actions.requestRender(); if (state.status !== "downloading") { - asrModelRenderUnsub?.(); - asrModelRenderUnsub = null; + asrModelRenderUnsub[id]?.(); + asrModelRenderUnsub[id] = undefined; } }); } -function asrModelStatusText(): string { - const state = asrModelState(); +function asrModelStatusText(id: AsrModelId): string { + const state = asrModelState(id); if (state.status === "ready") return "downloaded"; if (state.status === "downloading") { const pct = state.totalBytes > 0 ? Math.floor((state.bytesDownloaded / state.totalBytes) * 100) : 0; - return `${pct}% of ${ASR_MODEL_MB}`; + return `${pct}% of ${asrModelMb(id)}`; } return "not downloaded"; } -/** Download/cancel/delete management for the on-device transcription model. */ -function asrModelMenuItem(): MenuItem { +/** Download/cancel/delete management for one on-device transcription model. */ +function asrModelMenuItem(id: AsrModelId): MenuItem { + const def = ASR_MODELS[id]; + const rowLabel = `On-device model: ${def.label}`; return { - label: "On-device voice model", + label: rowLabel, description: - `${ASR_MODEL.label} (${ASR_MODEL_MB} download). ` + + `${def.label} (${asrModelMb(id)} download). ` + "Transcribes voice input on the phone itself, with no API key or cloud service. " + - "Required for the On-device transcription provider; the cloud providers work without it. " + + "Required for its matching Transcription Provider option; the other providers work without it. " + "An interrupted download resumes where it left off.", onSelect: (ctx) => { - const state = asrModelState(); + const state = asrModelState(id); const action: MenuItem = state.status === "downloading" ? { label: "Cancel download", onSelect: (innerCtx) => { - cancelAsrModelDownload(); + cancelAsrModelDownload(id); innerCtx.stack.pop(); }, } @@ -393,22 +399,22 @@ function asrModelMenuItem(): MenuItem { ? { label: "Delete model", onSelect: (innerCtx) => { - deleteAsrModel(); + deleteAsrModel(id); innerCtx.stack.pop(); }, } : { - label: `Download (${ASR_MODEL_MB})`, + label: `Download (${asrModelMb(id)})`, onSelect: (innerCtx) => { - startAsrModelDownload(); - watchAsrModelDownload(innerCtx); + startAsrModelDownload(id); + watchAsrModelDownload(id, innerCtx); innerCtx.stack.pop(); }, }; - openModalMenu(ctx, "On-device voice model", [action], 0); + openModalMenu(ctx, rowLabel, [action], 0); }, render: ({ image, x, y, width }) => { - drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-device voice model", asrModelStatusText()); + drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, rowLabel, asrModelStatusText(id)); }, }; } From 239d0a84e1dec5253cb58bd732f69cc139bbaec7 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Thu, 17 Sep 2026 21:56:35 -0700 Subject: [PATCH 2/5] Add a CHANGELOG note for the alternate-SID change --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index c2b473ce..fd3efc74 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ * Add Wear OS watch battery indicator to the top bar and Glanceboard * Glanceboard tap setting includes a Disable option * Refinements to builtin games +* Optimize bluetooth message transport ## 0.7.0 - 2026-09-12 From 25c398d60f6b1dec5bad5da85b50f7d0782e811d Mon Sep 17 00:00:00 2001 From: James Babcock Date: Thu, 17 Sep 2026 22:12:33 -0700 Subject: [PATCH 3/5] Wait for final results from Whisper before completing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit
Prompts Used User: This checkout corresponds to https://github.com/jimrandomh/faceclaw/pull/27. Review this branch. GPT-6 Astra: > Found one issue: > > - **[P2] Wait for Whisper’s final result before completing dictation** — [FaceclawVoiceController.java:729](/Users/jbabcock/repositories/g2/faceclaw/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java:729). Skipping partial decodes leaves short utterances without text until release, but the dialog still finalizes after its existing timeout. If decoding exceeds roughly 2.7 seconds (native stop’s 1.5-second wait plus the dialog’s 1.2-second timeout), auto-send can abandon a short command or send an incomplete longer one. Continue can append the late editing instruction literally instead of refining the message. I reproduced these outcomes with delayed transcript events. These flows should await recognition completion. > > Validation: TypeScript and a clean Android build passed. All 21 test failures also occur on `main`. Actual Whisper latency was not tested on hardware.
User: Thanks, let's fix that issue. GPT-6 Astra: > Fixed. Dictation now waits for native recognition to finish before fallback timers run. Auto-send, Continue, and AI Chat preserve delayed Whisper results; cancellation ignores late transcripts. > > TypeScript and Android builds pass. All 11 new regression tests pass. The full suite still has the same 21 pre-existing failures.
--- .../faceclaw/app/FaceclawVoiceController.java | 18 +- .../app/FaceclawVoiceControllerListener.java | 3 + app/apps/ai-chat/voice-draft.ts | 17 +- app/g2/dashboard-controller.ts | 4 +- app/native/voice-control.ts | 18 +- app/ui/layers.ts | 2 +- app/ui/shell/voice-input.ts | 72 ++++++-- tests/ai-chat.test.cjs | 32 +++- tests/fixtures/TextureCacheProtocolTest.java | 58 ++++++ tests/texture-cache-protocol.test.cjs | 23 +++ tests/voice-finalization.test.cjs | 166 ++++++++++++++++++ 11 files changed, 385 insertions(+), 28 deletions(-) create mode 100644 tests/fixtures/TextureCacheProtocolTest.java create mode 100644 tests/texture-cache-protocol.test.cjs create mode 100644 tests/voice-finalization.test.cjs diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index d6b8171d..37a53d58 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -266,20 +266,26 @@ public class FaceclawVoiceController { } public void start(String requestedMode) { + start(requestedMode, 0); + } + + public void start(String requestedMode, int captureId) { synchronized (lock) { if (started) { emitStatus("Voice control is already listening."); + emitStopped(captureId); return; } if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) { emitStatus("Voice control needs an active G2 connection."); + emitStopped(captureId); return; } mode = parseMode(requestedMode); activePhoneMic = usePhoneMic; started = true; audioStarted = false; - workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); + workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController"); workerThread.start(); } } @@ -350,7 +356,7 @@ public class FaceclawVoiceController { return VoiceInputMode.ONBOARD; } - private void runLoop() { + private void runLoop(int captureId) { try { deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; @@ -432,6 +438,7 @@ public class FaceclawVoiceController { audioStarted = false; workerThread = null; } + emitStopped(captureId); } } @@ -1014,6 +1021,13 @@ public class FaceclawVoiceController { mainHandler.post(currentListener::onSpeechEnd); } + private void emitStopped(int captureId) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener != null) { + mainHandler.post(() -> currentListener.onStopped(captureId)); + } + } + /** * Decides when a hands-free utterance is over, so "Hey Even" capture can * stop without a button release. diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index c60ec655..95c5826f 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -10,6 +10,9 @@ public interface FaceclawVoiceControllerListener { */ void onTranscript(String text, boolean isFinal); + /** Capture has finished, after the final transcript or a startup/recognition error. */ + void onStopped(int captureId); + /** * Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM. * Emitted in every mode so raw-PCM consumers (levels, recording, captions) diff --git a/app/apps/ai-chat/voice-draft.ts b/app/apps/ai-chat/voice-draft.ts index 0cb6854c..f91093fc 100644 --- a/app/apps/ai-chat/voice-draft.ts +++ b/app/apps/ai-chat/voice-draft.ts @@ -89,11 +89,22 @@ export class VoiceDraft { } private stopAndFinalize(): void { - // Install fallback first: stop may synchronously deliver the final. - this.timer = setTimeout(() => this.finish(), 1500); + const generation = this.generation; + const afterStop = () => { + if (generation !== this.generation || !this.isFinishing()) return; + this.timer = setTimeout(() => this.finish(), 1500); + }; if (this.started) { this.started = false; - void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error))); + // Native recognition can take longer than the synchronous stop join. + // Its final callback may finish the draft before this promise resolves. + const stopped = this.actions.stopVoiceCapture(); + if (!stopped) { afterStop(); return; } + void stopped.then(afterStop, (error) => { + if (generation === this.generation) this.cancel(String(error)); + }); + } else { + afterStop(); } } diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 84fb9548..a70a7c55 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1912,9 +1912,9 @@ class DashboardController { return this.beginVoiceCapture("ptt", endpointing); } - private stopVoiceCapture(): void { + private stopVoiceCapture(): Promise | void { ++this.pttCaptureGeneration; - voiceControlBridge.stopPushToTalk(); + return voiceControlBridge.stopPushToTalk(); } private startContinuousVoiceCapture(): void { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index acfd9f31..f3acadab 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -108,6 +108,9 @@ export class FaceclawVoiceControlBridge { private listening = false; private detail = ""; private started = false; + private nativeCaptureFinished: Promise = Promise.resolve(); + private nextNativeCaptureId = 1; + private readonly nativeCaptureResolvers = new Map void>(); // The mic is a single shared stream; these are the reasons it is running. // The first holder starts capture (choosing the provider); the mic stops // when the last one releases. Transcript events broadcast to every @@ -189,8 +192,13 @@ export class FaceclawVoiceControlBridge { } /** End push-to-talk: for cloud, commit for a final result if it was the last holder. */ - stopPushToTalk(): void { + stopPushToTalk(): Promise | void { + // Shared continuous capture keeps running; cloud providers finalize via + // their own callbacks. Only a stopped native recognizer must be awaited. + const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1 + ? this.nativeCaptureFinished : undefined; this.releaseCapture("ptt", true); + return finished; } /** Begin continuous capture (Transcribe): the mic stays on until released. */ @@ -322,7 +330,9 @@ export class FaceclawVoiceControlBridge { // Which on-device model to load; a no-op setter for every provider except // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); - this.controller?.start("onboard"); + const captureId = this.nextNativeCaptureId++; + this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); + this.controller?.start("onboard", captureId); } /** @@ -491,6 +501,10 @@ export class FaceclawVoiceControlBridge { onTranscript: (text: string, isFinal: boolean) => { this.emitTranscript(String(text), Boolean(isFinal)); }, + onStopped: (captureId: number) => { + this.nativeCaptureResolvers.get(captureId)?.(); + this.nativeCaptureResolvers.delete(captureId); + }, onPcm: (pcm: any) => { const bytes = toUint8Array(pcm); this.cloudClient?.acceptPcm(bytes); diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 50ad0473..32315136 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -28,7 +28,7 @@ export type LayerActions = { * (hands-free); otherwise it runs until stopVoiceCapture. */ startVoiceCapture: (endpointing?: boolean) => Promise | void; - /** Stop push-to-talk; for a cloud provider this also commits for a final result. */ + /** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */ stopVoiceCapture: () => Promise | void; /** Start continuous capture (Transcribe); shares the mic with push-to-talk. */ startContinuousVoiceCapture: () => Promise | void; diff --git a/app/ui/shell/voice-input.ts b/app/ui/shell/voice-input.ts index 41498570..bb594c49 100644 --- a/app/ui/shell/voice-input.ts +++ b/app/ui/shell/voice-input.ts @@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings"; import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures"; import { Layer, type LayerActions, type LayerContext } from "../layers"; import { paintInputDialog } from "./input-dialog"; -// After the mic stops, the provider's final transcript can trail in (cloud -// commit round-trip); wait this long for it before refining with what we have. +// After native recognition finishes, a cloud final can still trail in. const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200; /** @@ -80,6 +79,10 @@ export class VoiceInputLayer implements Layer { private capturing = false; /** Continuation stopped; waiting for the trailing final transcript. */ private followupFinalizeTimer: ReturnType | null = null; + private pendingFollowup = false; + private stoppingCapture = false; + private stopGeneration = 0; + private acceptingTranscript = true; private refineHandle: AnthropicStreamHandle | null = null; private menuIndex = 0; /** Auto-send (wakeword skip-confirmation) is waiting to fire. */ @@ -155,21 +158,48 @@ export class VoiceInputLayer implements Layer { } this.capturing = false; this.phase = "menu"; - void this.actions.stopVoiceCapture(); if (this.autoSend && this.sendTargets.length) { // Skip-confirmation (wakeword): send to the default target as soon as the // transcript finalizes, or after a short wait for the trailing final. this.pendingAutoSend = true; this.status = "Sending..."; - this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS); } else if (this.status.endsWith("...")) { // A progress status ("Starting microphone...", "Listening...") gives // way to the menu prompt; an error (ending in ".") stays visible. this.status = "Send, continue, or discard?"; } + this.stopCapture(() => { + if (this.pendingAutoSend) { + this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS); + } + }); this.actions.requestRender(); } + /** Native stop can outlast its synchronous join while Whisper is decoding. */ + private stopCapture(onStopped: () => void): void { + const generation = ++this.stopGeneration; + this.stoppingCapture = true; + const afterStop = () => { + if (generation !== this.stopGeneration) return; + this.stoppingCapture = false; + onStopped(); + this.actions.requestRender(); + }; + const stopped = this.actions.stopVoiceCapture(); + if (!stopped) { + afterStop(); + return; + } + void stopped.then(afterStop, (error) => { + if (generation !== this.stopGeneration) return; + this.stoppingCapture = false; + this.pendingAutoSend = false; + this.acceptingTranscript = false; + this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error)); + }); + } + /** Fire the queued skip-confirmation send (or fall back to the menu). */ private performAutoSend(): void { if (this.autoSendTimer !== null) { @@ -193,7 +223,7 @@ export class VoiceInputLayer implements Layer { /** The menu rows: one per send target, then Continue, then Discard. */ private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> { const text = this.displayText().trim(); - const hasText = text.length > 0; + const hasText = text.length > 0 && !this.stoppingCapture; const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0; const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = []; for (const target of this.sendTargets) { @@ -201,6 +231,7 @@ export class VoiceInputLayer implements Layer { label: target.label, dim: !hasText, onSelect: () => { + if (this.stoppingCapture) return; this.dismiss(); if (hasText) target.onSend(text); }, @@ -208,9 +239,9 @@ export class VoiceInputLayer implements Layer { } rows.push({ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", - dim: !hasLlmKey, + dim: !hasLlmKey || this.stoppingCapture, onSelect: () => { - if (hasLlmKey) this.startContinuation(); + if (hasLlmKey && !this.stoppingCapture) this.startContinuation(); }, }); rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() }); @@ -222,7 +253,7 @@ export class VoiceInputLayer implements Layer { const inMenu = this.phase === "menu"; paintInputDialog(image, { title: this.capturing && this.listening ? "Voice ●" : "Voice", - status: this.status, + status: this.stoppingCapture ? "Finishing transcription..." : this.status, text: this.displayText() || this.placeholderText(), rows: inMenu ? this.menuRows() : [], selectedRow: this.menuIndex, @@ -284,6 +315,7 @@ export class VoiceInputLayer implements Layer { /** Continue selected: keep the message aside and record a follow-up. */ private startContinuation(): void { + this.acceptingTranscript = true; this.baseText = this.displayText().trim(); this.finalizedText = ""; this.liveText = ""; @@ -298,15 +330,18 @@ export class VoiceInputLayer implements Layer { private endContinuationCapture(): void { if (!this.capturing) return; this.capturing = false; - void this.actions.stopVoiceCapture(); - this.status = "Refining..."; + this.pendingFollowup = true; + this.status = "Finishing transcription..."; + this.stopCapture(() => { + if (this.pendingFollowup) { + this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS); + } + }); this.actions.requestRender(); - // The provider's committed transcript arrives shortly after stop; refine - // when it does, or after a timeout with whatever partials we have. - this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS); } private beginRefine(): void { + this.pendingFollowup = false; if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; @@ -343,9 +378,10 @@ export class VoiceInputLayer implements Layer { /** Abort a continuation (mic or LLM stage) and restore the prior message. */ private cancelContinuation(status: string): void { + this.acceptingTranscript = false; if (this.capturing) { this.capturing = false; - void this.actions.stopVoiceCapture(); + this.stopCapture(() => {}); } this.refineHandle?.cancel(); this.refineHandle = null; @@ -353,6 +389,7 @@ export class VoiceInputLayer implements Layer { } private backToMenu(text: string, status: string): void { + this.pendingFollowup = false; if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; @@ -366,6 +403,9 @@ export class VoiceInputLayer implements Layer { } onRemoved(): void { + ++this.stopGeneration; + this.acceptingTranscript = false; + this.pendingFollowup = false; this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); @@ -428,14 +468,14 @@ export class VoiceInputLayer implements Layer { private onTranscript(event: VoiceTranscriptEvent): void { // The refine stream owns the text buffers once it starts; a transcript // that trails in after that point is stale. - if (this.phase === "refining") return; + if (!this.acceptingTranscript || this.phase === "refining") return; if (event.isFinal) { const finalText = event.text.trim() || this.liveText.trim(); if (finalText) { this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; } this.liveText = ""; - if (this.phase === "continuing" && this.followupFinalizeTimer !== null) { + if (this.phase === "continuing" && this.pendingFollowup) { // The follow-up finalized; no need to keep waiting. this.beginRefine(); return; diff --git a/tests/ai-chat.test.cjs b/tests/ai-chat.test.cjs index 9423f06c..e4ded14a 100644 --- a/tests/ai-chat.test.cjs +++ b/tests/ai-chat.test.cjs @@ -125,7 +125,7 @@ test('synchronous provider errors do not leave a permanently active turn', () => assert.equal(session.status, 'failed'); }); -function draftEnv({ prepare = async () => true, start = async () => {}, initialStatus = 'Ready' } = {}) { +function draftEnv({ prepare = async () => true, start = async () => {}, stop = () => {}, initialStatus = 'Ready' } = {}) { const transcripts = new Set(), statuses = new Set(), timers = new Map(); const sent = []; let starts = 0, stops = 0, mic = false, timerId = 0; @@ -139,7 +139,7 @@ function draftEnv({ prepare = async () => true, start = async () => {}, initialS setTimeout(cb) { timers.set(++timerId, cb); return timerId; }, clearTimeout(id) { timers.delete(id); }, }); - const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; } }, prepare, () => {}, (text) => sent.push(text)); + const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; return stop(); } }, prepare, () => {}, (text) => sent.push(text)); return { draft, sent, starts: () => starts, stops: () => stops, mic: () => mic, transcript(text, isFinal = false) { for (const cb of [...transcripts]) cb({ text, isFinal }); }, status(status) { for (const cb of [...statuses]) cb({ status }); }, @@ -174,6 +174,34 @@ test('release before permission resolves never starts a microphone', async () => assert.deepEqual(env.sent, []); }); +test('chat waits past its fallback deadline for native recognition', async () => { + let complete; + const stopped = new Promise(resolve => { complete = resolve; }); + const env = draftEnv({ stop: () => stopped }); + await env.draft.start(); + env.transcript('first segment'); + env.draft.release(); env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.draft.phase, 'finishing'); + env.transcript('first segment and the rest', true); + complete(); await stopped; env.timeout(); + assert.deepEqual(env.sent, ['first segment and the rest']); +}); + +test('a cancelled chat capture cannot finish a newer capture when native stop resolves', async () => { + let complete; + const stopped = new Promise(resolve => { complete = resolve; }); + const env = draftEnv({ stop: () => stopped }); + await env.draft.start(); + env.draft.release(); env.draft.cancel(); + await env.draft.start(); + env.transcript('new speech'); + complete(); await stopped; env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.draft.phase, 'listening'); + env.draft.cancel(); +}); + test('release during async mic startup stops capture after it starts', async () => { let started; const env = draftEnv({ start: () => new Promise((resolve) => { started = resolve; }) }); diff --git a/tests/fixtures/TextureCacheProtocolTest.java b/tests/fixtures/TextureCacheProtocolTest.java new file mode 100644 index 00000000..77ecac96 --- /dev/null +++ b/tests/fixtures/TextureCacheProtocolTest.java @@ -0,0 +1,58 @@ +package com.faceclaw.app; + +import java.lang.reflect.*; +import java.util.*; + +public class TextureCacheProtocolTest { + static int u16(byte[] b, int p) { return (b[p]&255) | (b[p+1]&255)<<8; } + static int u32(byte[] b, int p) { return u16(b,p) | u16(b,p+2)<<16; } + public static void main(String[] args) throws Exception { + TextureCacheState cache = new TextureCacheState(); + byte[] pixels = new byte[255*255]; + for (int i=0;i=0; + ImageAtlas.Entry tiny = new ImageAtlas.Entry(1,1,new byte[]{15}); + int imageOffset = cache.ensureImage(100,tiny); + assert imageOffset>65535; + GlyphAtlas.Glyph glyph = new GlyphAtlas.Glyph(1,1,0,1,0,null,new byte[]{15}); + int glyphOffset = cache.ensureGlyph(42,'A',glyph); + int table = cache.fontTableOffset(42); + assert table>65535 && glyphOffset==table+96*4; + byte[] memory = new byte[262144]; + for (byte[] upload: cache.drainUploadPayloads(3600)) { + assert upload[0]==18; + int pos=1; + while (pos selected=Class.forName("com.faceclaw.app.TexturePlanner$Selected"); + Constructor ctor=selected.getDeclaredConstructors()[0]; ctor.setAccessible(true); + Object image=ctor.newInstance(SurfaceCompositor.ScreenDraw.image(100,321,123),null,tiny,321,15); + Method encode=TexturePlanner.class.getDeclaredMethod("encodeImageDraw",TextureCacheState.class,selected); + encode.setAccessible(true); + byte[] draw=(byte[])encode.invoke(null,cache,image); + assert draw.length==10 && draw[0]==19 && u32(draw,1)==imageOffset; + assert u16(draw,5)==321 && u16(draw,7)==123 && draw[9]==31; + Object letter=ctor.newInstance(SurfaceCompositor.ScreenDraw.glyph(42,'A',321,123,255),glyph,null,321,15); + Method runs=TexturePlanner.class.getDeclaredMethod("buildRuns",TextureCacheState.class,List.class,List.class); + runs.setAccessible(true); List out=new ArrayList<>(); + runs.invoke(null,cache,Collections.singletonList(letter),out); + byte[] run=out.get(0); + assert run.length==12 && run[0]==20 && u32(run,1)==table; + assert u16(run,5)==321 && u16(run,7)==123 && run[9]==31 && run[10]==1 && run[11]=='A'; + int id=200; + while (cache.ensureImage(id++,big)>=0) {} + int before=cache.usedBytes(); + assert cache.ensureImage(id,big)==-1 && cache.usedBytes()==before; + cache.reset(); assert cache.usedBytes()==0 && !cache.hasPendingUploads(); + assert cache.ensureImage(id,tiny)==0; + } +} diff --git a/tests/texture-cache-protocol.test.cjs b/tests/texture-cache-protocol.test.cjs new file mode 100644 index 00000000..dc397251 --- /dev/null +++ b/tests/texture-cache-protocol.test.cjs @@ -0,0 +1,23 @@ +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +test('256 KiB cache uploads and planner draws preserve wide image and glyph offsets', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-texture-java-')); + const javaBin = (name) => process.env.JAVA_HOME ? path.join(process.env.JAVA_HOME, 'bin', name) : name; + const root = path.join(__dirname, '../App_Resources/Android/src/main/java/com/faceclaw/app'); + try { + const clock = path.join(directory, 'SystemClock.java'); + const log = path.join(directory, 'Log.java'); + fs.writeFileSync(clock, 'package android.os; public class SystemClock { public static long elapsedRealtime() { return 0; } }'); + fs.writeFileSync(log, 'package android.util; public class Log { public static int i(String tag, String msg) { return 0; } }'); + const sources = ['GlyphAtlas.java', 'ImageAtlas.java', 'FwGlyphAtlas.java', 'SurfaceCompositor.java', + 'util/BmpUtil.java', 'util/CollectionUtils.java', 'g2protocol/BleProtocol.java', + 'g2protocol/BleImageOptimizer.java', 'g2protocol/TextureCacheState.java', 'g2protocol/TexturePlanner.java']; + execFileSync(javaBin('javac'), ['-d', directory, ...sources.map(file => path.join(root, file)), + clock, log, path.join(__dirname, 'fixtures/TextureCacheProtocolTest.java')], { stdio: 'pipe' }); + execFileSync(javaBin('java'), ['-ea', '-cp', directory, 'com.faceclaw.app.TextureCacheProtocolTest'], { stdio: 'pipe' }); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs new file mode 100644 index 00000000..2077d488 --- /dev/null +++ b/tests/voice-finalization.test.cjs @@ -0,0 +1,166 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, deps, globals = {}) { + const exports = {}; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, { exports, console, ...globals, require: name => deps[name] ?? {} }); + return exports; +} + +function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) { + const listeners = new Set(), timers = new Map(), sent = [], refinements = []; + let complete, timerId = 0, starts = 0, frame; + let completion = new Promise(resolve => { complete = resolve; }); + const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', { + '../../native/voice-control': { voiceControlBridge: { + onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); }, + onStatus() { return () => {}; }, + } }, + '../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } }, + '../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } }, + '../gestures': { gestureHints: () => '' }, + './input-dialog': { paintInputDialog(_image, options) { frame = options; } }, + }, { + setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; }, + clearTimeout(id) { timers.delete(id); }, + }); + const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); }; + const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(), + actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() { + if (synchronousFinal) transcript(synchronousFinal, true); + return deferred ? completion : undefined; + } }, + sendTargets: [{ id: 'send', label: 'Send', onSend: text => sent.push(text) }], + }); + layer.startCapture(); + return { layer, sent, refinements, transcript, starts: () => starts, + input: type => layer.handleInput({ type }, {}), + frame() { layer.paint({}, () => ({})); return frame; }, + timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } }, + async complete() { complete(); await completion; await Promise.resolve(); }, + nextCapture() { completion = new Promise(resolve => { complete = resolve; }); }, + timers: () => timers.size, + }; +} + +for (const partial of ['', 'Set a timer']) { + test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => { + const env = dialog({ autoSend: true }); + if (partial) env.transcript(partial); + env.layer.endCapture(); + env.timeout(); // well beyond the old fallback deadline + assert.deepEqual(env.sent, []); + assert.equal(env.timers(), 0); + env.transcript('Set a timer for five minutes', true); + await env.complete(); env.timeout(); + assert.deepEqual(env.sent, ['Set a timer for five minutes']); + assert.equal(env.timers(), 0); + }); +} + +async function continuation() { + const env = dialog(); + env.layer.endCapture(); + env.transcript('Set a timer for five minutes', true); + await env.complete(); + env.nextCapture(); + env.input('scroll-down'); env.input('click'); // Continue + env.input('click'); // stop follow-up + return env; +} + +test('Continue refines the complete late result instead of appending an editing instruction', async () => { + const env = await continuation(); + env.timeout(); + assert.equal(env.refinements.length, 0); + env.transcript('Change five to ten', true); + await env.complete(); env.timeout(); + assert.equal(env.refinements.length, 1); + assert.equal(env.refinements[0].original, 'Set a timer for five minutes'); + assert.equal(env.refinements[0].followup, 'Change five to ten'); +}); + +test('Send and Continue cannot consume partial text while native recognition is pending', async () => { + const env = dialog(); + env.transcript('Set a timer'); env.layer.endCapture(); + assert.equal(env.frame().status, 'Finishing transcription...'); + env.input('click'); env.input('scroll-down'); env.input('click'); + assert.deepEqual(env.sent, []); + assert.equal(env.starts(), 1); + env.transcript('Set a timer for five minutes', true); await env.complete(); + env.input('scroll-up'); env.input('click'); + assert.deepEqual(env.sent, ['Set a timer for five minutes']); +}); + +test('cancelling a pending continuation ignores its late transcript', async () => { + const env = await continuation(); + env.input('double-click'); + env.transcript('Change five to ten', true); await env.complete(); env.timeout(); + assert.equal(env.refinements.length, 0); + assert.equal(env.frame().text, 'Set a timer for five minutes'); +}); + +test('closing while native recognition is pending never schedules a fallback or sends', async () => { + const env = dialog({ autoSend: true }); + env.layer.endCapture(); env.layer.onRemoved(); + env.transcript('late', true); await env.complete(); env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.timers(), 0); +}); + +test('failed or empty native captures return to the menu without sending', async () => { + const env = dialog({ autoSend: true }); + env.layer.endCapture(); await env.complete(); env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.frame().status, 'Send, continue, or discard?'); +}); + +test('cloud fallback and a synchronous final still send exactly once', () => { + for (const synchronousFinal of [undefined, 'complete']) { + const env = dialog({ autoSend: true, deferred: false, synchronousFinal }); + env.transcript('partial'); env.layer.endCapture(); env.timeout(); + assert.deepEqual(env.sent, [synchronousFinal ?? 'partial']); + } +}); + +test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => { + let controller; + class Controller { + constructor() { controller = this; } + setListener(listener) { this.listener = listener; } + setCommunicator() {} setUsePhoneMic() {} setSaveRecordings() {} setEndpointing() {} + setNoiseSuppression() {} setBeamFilter() {} clearSpeakerVerification() {} setOnboardModelKind() {} + start(_mode, id) { this.captureId = id; this.active = true; } + stop() { this.active = false; } + isCapturing() { return this.active; } + } + const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', { + '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, + './speech-pause': { SpeechPauseDetector: class { reset() {} } }, + }, { global: { isAndroid: true }, com: { faceclaw: { app: { + FaceclawVoiceController: Controller, + FaceclawVoiceControllerListener: function (listener) { return listener; }, + } } } }); + const bridge = new FaceclawVoiceControlBridge(); + const options = { provider: 'onboard-whisper' }; + const events = []; + bridge.onTranscript(event => events.push(event.text)); + bridge.startPushToTalk(options); + const stopped = bridge.stopPushToTalk().then(() => events.push('stopped')); + controller.listener.onStopped(0); // an older raw/cloud capture's callback + await Promise.resolve(); assert.deepEqual(events, []); + controller.listener.onTranscript('complete', true); + assert.deepEqual(events, ['complete']); + controller.listener.onStopped(controller.captureId); await stopped; + assert.deepEqual(events, ['complete', 'stopped']); + bridge.startContinuousCapture(options); bridge.startPushToTalk(options); + assert.equal(bridge.stopPushToTalk(), undefined); + assert.equal(controller.active, true); + bridge.stopContinuousCapture(); controller.listener.onStopped(controller.captureId); +}); From bed21c9a5f4c030107644a23fbff5672875b34da Mon Sep 17 00:00:00 2001 From: James Babcock Date: Thu, 17 Sep 2026 22:31:50 -0700 Subject: [PATCH 4/5] Short-then-long-press from off opens the glanceboard --- CHANGELOG | 1 + app/apps/glanceboard/glanceboard-settings.ts | 2 +- app/g2/glance-host.ts | 2 +- app/g2/glance-state.ts | 8 ++++---- tests/glance-state.test.cjs | 5 +++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index fd3efc74..b9654730 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,7 @@ * Glanceboard tap setting includes a Disable option * Refinements to builtin games * Optimize bluetooth message transport +* Short-the-long gesture from screen off activates the glanceboard ## 0.7.0 - 2026-09-12 diff --git a/app/apps/glanceboard/glanceboard-settings.ts b/app/apps/glanceboard/glanceboard-settings.ts index 6036219c..d42dd129 100644 --- a/app/apps/glanceboard/glanceboard-settings.ts +++ b/app/apps/glanceboard/glanceboard-settings.ts @@ -78,7 +78,7 @@ export const glanceShowOnLongPressSetting = new ConfigSettingBoolean({ label: "Show on long press", storageKey: "glanceboard.showOnLongPress", defaultValue: true, - description: "Holding a long-press while asleep shows the Glanceboard until the press is released.", + description: "A long-press or short-then-long-press while asleep shows the Glanceboard until the press is released.", }); /** The head-tilt wake shows the board (for the tap duration) instead of the regular UI. */ diff --git a/app/g2/glance-host.ts b/app/g2/glance-host.ts index 555ee070..d58781c7 100644 --- a/app/g2/glance-host.ts +++ b/app/g2/glance-host.ts @@ -77,7 +77,7 @@ export class GlanceHost { const provider = this.options.getProvider(); if (!provider?.isEnabled()) return null; if (gesture === "click" && !provider.showOnTap()) return null; - if (gesture === "long-press" && !provider.showOnLongPress()) return null; + if ((gesture === "long-press" || gesture === "short-then-long-press") && !provider.showOnLongPress()) return null; if (gesture === "head-tilt" && !provider.showOnHeadTilt()) return null; return glanceEventForGesture(gesture, this.isVisible()); } diff --git a/app/g2/glance-state.ts b/app/g2/glance-state.ts index 8f606f36..b8369486 100644 --- a/app/g2/glance-state.ts +++ b/app/g2/glance-state.ts @@ -2,7 +2,7 @@ * The Glanceboard's visibility rules, as a pure reducer so they can be tested * without the display plumbing. The board is an alternate sleep-time display: * while the shell is asleep a press (single tap, or a head-tilt wake) shows it - * for a short while, a hold (long-press) shows it until the press is released, + * for a short while, a hold (long-press or short-then-long-press) shows it until release, * and a double-tap dismisses it in favour of the regular UI. GlanceHost feeds * these events in and applies the resulting visibility to the compositor. * @@ -49,8 +49,7 @@ export function reduceGlance( case "hold": return { visible: true, holding: true, hideAtMs: null }; case "release": - // Releases also follow presses the board was never holding for (a - // tap-then-hold, a release delivered late); those mean nothing here. + // Releases delivered late or after a disabled hold gesture mean nothing here. return state.holding ? GLANCE_HIDDEN : state; case "timeout": if (!state.visible || state.holding || state.hideAtMs === null || nowMs < state.hideAtMs) return state; @@ -67,7 +66,7 @@ export function reduceGlance( * already handles or ignores while asleep. */ export function glanceEventForGesture( - gesture: "click" | "double-click" | "long-press" | "long-press-release" | "head-tilt" | string, + gesture: "click" | "double-click" | "long-press" | "short-then-long-press" | "long-press-release" | "head-tilt" | string, boardVisible: boolean, ): GlanceEvent | null { switch (gesture) { @@ -75,6 +74,7 @@ export function glanceEventForGesture( case "head-tilt": return { type: "press" }; case "long-press": + case "short-then-long-press": return { type: "hold" }; case "long-press-release": return { type: "release" }; diff --git a/tests/glance-state.test.cjs b/tests/glance-state.test.cjs index 7e1eab49..0bc7d415 100644 --- a/tests/glance-state.test.cjs +++ b/tests/glance-state.test.cjs @@ -47,15 +47,16 @@ test("dismiss hides from any state", () => { assert.deepEqual(reduceGlance(shown, { type: "dismiss" }, T0), GLANCE_HIDDEN); }); -test("gesture mapping: taps and head-tilt press, long-press holds, double-tap dismisses only a visible board", () => { +test("gesture mapping: taps and head-tilt press, both long-press gestures hold, double-tap dismisses only a visible board", () => { assert.deepEqual(glanceEventForGesture("click", false), { type: "press" }); assert.deepEqual(glanceEventForGesture("head-tilt", false), { type: "press" }); assert.deepEqual(glanceEventForGesture("long-press", false), { type: "hold" }); + assert.deepEqual(glanceEventForGesture("short-then-long-press", false), { type: "hold" }); + assert.deepEqual(glanceEventForGesture("short-then-long-press", true), { type: "hold" }); assert.deepEqual(glanceEventForGesture("long-press-release", true), { type: "release" }); assert.equal(glanceEventForGesture("double-click", false), null); assert.deepEqual(glanceEventForGesture("double-click", true), { type: "dismiss" }); assert.equal(glanceEventForGesture("scroll-up", true), null); - assert.equal(glanceEventForGesture("short-then-long-press", true), null); }); test("the press timeout is the host's when supplied", () => { From 17274746a736b72845a350dffdeb2fea76b097a7 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Thu, 17 Sep 2026 22:50:54 -0700 Subject: [PATCH 5/5] Glanceboard has a 2x3 layout option --- CHANGELOG | 1 + app/apps/glanceboard/board.ts | 20 +++- app/apps/glanceboard/glanceboard-app.ts | 72 +++++++++----- app/apps/glanceboard/glanceboard-settings.ts | 33 +++++-- app/apps/glanceboard/index.ts | 9 +- app/apps/glanceboard/layout.ts | 14 +++ tests/glanceboard-layout.test.cjs | 22 +++++ tests/glanceboard-settings.test.cjs | 99 ++++++++++++++++++++ 8 files changed, 234 insertions(+), 36 deletions(-) create mode 100644 tests/glanceboard-settings.test.cjs diff --git a/CHANGELOG b/CHANGELOG index b9654730..2ab4edad 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,7 @@ * Refinements to builtin games * Optimize bluetooth message transport * Short-the-long gesture from screen off activates the glanceboard +* Glanceboard has a 2x3 layout option ## 0.7.0 - 2026-09-12 diff --git a/app/apps/glanceboard/board.ts b/app/apps/glanceboard/board.ts index dc815f7d..6fe473bb 100644 --- a/app/apps/glanceboard/board.ts +++ b/app/apps/glanceboard/board.ts @@ -1,7 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { onAnySettingChanged } from "../../ui/dashboard-settings"; -import { glanceShowLinesSetting, glanceSlotSettings } from "./glanceboard-settings"; -import { QUADRANT_LAYOUT, resolveGlanceRegions, slotDividers, type GlanceLayout, type GlanceRegion } from "./layout"; +import { glanceLayout, glanceShowLinesSetting, glanceSlotSettings } from "./glanceboard-settings"; +import { resolveGlanceRegions, slotDividers, type GlanceLayout, type GlanceRegion } from "./layout"; import { type GlanceWidget } from "./widget"; import { findGlanceWidget, glanceWidgetSpans } from "./widgets"; @@ -27,11 +27,18 @@ export class GlanceBoard { private regions: LiveRegion[] = []; private started = false; private unsubscribeSettings: (() => void) | null = null; + private currentLayout: GlanceLayout; constructor( private readonly requestRender: () => void, - readonly layout: GlanceLayout = QUADRANT_LAYOUT, - ) {} + private readonly fixedLayout?: GlanceLayout, + ) { + this.currentLayout = fixedLayout ?? glanceLayout(); + } + + get layout(): GlanceLayout { + return this.currentLayout; + } get width(): number { return this.layout.width; @@ -65,9 +72,12 @@ export class GlanceBoard { /** Rebuild the live regions from the settings, keeping widgets whose region is unchanged. */ private applySlotSettings(): void { if (!this.started) return; + const layout = this.fixedLayout ?? glanceLayout(); + const layoutChanged = layout !== this.currentLayout; + this.currentLayout = layout; const previous = new Map(this.regions.map((live) => [live.key, live])); const next: LiveRegion[] = []; - let changed = false; + let changed = layoutChanged; for (const region of this.currentRegions()) { const key = regionKey(region); const kept = previous.get(key); diff --git a/app/apps/glanceboard/glanceboard-app.ts b/app/apps/glanceboard/glanceboard-app.ts index f37f9a61..a0e2b236 100644 --- a/app/apps/glanceboard/glanceboard-app.ts +++ b/app/apps/glanceboard/glanceboard-app.ts @@ -1,7 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { truncateText, wrapText } from "../../graphics/textwrap"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; -import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from "../../ui/dashboard-settings"; import { type InputEvent } from "../../ui/gestures"; import { type Layer, type LayerContext, type PaintBelow } from "../../ui/layers"; import { drawSelectionHighlight, MenuLayer, openModalMenu, type MenuItem } from "../../ui/menu"; @@ -12,6 +12,8 @@ import { GlanceBoard } from "./board"; import { clearConflictingSlots, glanceboardEnabledSetting, + glanceLayout, + glanceLayoutSetting, glanceShowLinesSetting, glanceShowOnHeadTiltSetting, glanceShowOnLongPressSetting, @@ -20,7 +22,7 @@ import { glanceTapDurationSetting, type GlanceSlotChoice, } from "./glanceboard-settings"; -import { QUADRANT_LAYOUT, resolveGlanceRegions, slotDividers } from "./layout"; +import { resolveGlanceRegions, slotDividers } from "./layout"; import { glanceWidgetSpans } from "./widgets"; export const GLANCEBOARD_WINDOW_ID = "glanceboard"; @@ -49,24 +51,33 @@ function pageMenuLayout(width: number) { * and the menu (enable toggle, live preview, slot contents, settings). Right * half: a scaled preview of the board's slots with their dividers and the * name of the widget in each. "Select Contents" moves focus into that - * preview, where scroll picks a quadrant and a tap opens the picker for it; + * preview, where scroll picks a slot and a tap opens the picker for it; * a double-tap returns focus to the menu. */ class GlanceboardHomeLayer implements Layer { private menu: MenuLayer | null = null; private menuTop = 0; + private menuHeight = 0; private focus: "menu" | "preview" = "menu"; private selectedSlot = 0; + constructor(private readonly setPreviewVisible: (visible: boolean) => void) {} + private currentMenu(ctx: LayerContext, menuTop: number): MenuLayer { - if (!this.menu || this.menuTop !== menuTop) { - const { width, height } = ctx.stack.getBaseSize(); + const { width, height } = ctx.stack.getBaseSize(); + if (!this.menu || this.menuTop !== menuTop || this.menuHeight !== height) { const half = (width / 2) | 0; const items: MenuItem[] = [ toggleSettingMenuItem(glanceboardEnabledSetting), { label: "Preview", - onSelect: (menuCtx) => menuCtx.stack.push(new GlancePreviewLayer(() => menuCtx.actions.requestRender())), + onSelect: (menuCtx) => { + menuCtx.stack.push(new GlancePreviewLayer( + () => menuCtx.actions.requestRender(), + () => this.setPreviewVisible(false), + )); + this.setPreviewVisible(true); + }, }, { label: "Select Contents", @@ -80,6 +91,7 @@ class GlanceboardHomeLayer implements Layer { }, ]; this.menuTop = menuTop; + this.menuHeight = height; this.menu = new MenuLayer(null, items, { x: 8, y: menuTop, width: half - 16, showBorder: false, minHeight: 0, maxHeight: height - menuTop, @@ -92,9 +104,10 @@ class GlanceboardHomeLayer implements Layer { private previewRect(width: number, height: number): { x: number; y: number; width: number; height: number } { const half = (width / 2) | 0; const availableWidth = width - half - 2 * PREVIEW_MARGIN; - const scale = Math.min(availableWidth / QUADRANT_LAYOUT.width, (height - 2 * PREVIEW_MARGIN) / QUADRANT_LAYOUT.height); - const previewWidth = Math.round(QUADRANT_LAYOUT.width * scale); - const previewHeight = Math.round(QUADRANT_LAYOUT.height * scale); + const layout = glanceLayout(); + const scale = Math.min(availableWidth / layout.width, (height - 2 * PREVIEW_MARGIN) / layout.height); + const previewWidth = Math.round(layout.width * scale); + const previewHeight = Math.round(layout.height * scale); return { x: half + PREVIEW_MARGIN + (((availableWidth - previewWidth) / 2) | 0), y: ((height - previewHeight) / 2) | 0, @@ -123,6 +136,7 @@ class GlanceboardHomeLayer implements Layer { const menuCtx: LayerContext = { ...ctx, stack: Object.assign(Object.create(ctx.stack), { isFocused: () => menuFocused }) }; menu.paint(menuCtx, () => image); + this.selectedSlot = Math.min(this.selectedSlot, glanceLayout().slots.length - 1); const preview = this.previewRect(width, height); drawSlotPreview(image, preview, { selectedSlot: this.selectedSlot, @@ -133,7 +147,8 @@ class GlanceboardHomeLayer implements Layer { async handleInput(event: InputEvent, ctx: LayerContext): Promise { if (this.focus === "preview") { - const slotCount = QUADRANT_LAYOUT.slots.length; + const slotCount = glanceLayout().slots.length; + this.selectedSlot = Math.min(this.selectedSlot, slotCount - 1); switch (event.type) { case "scroll-up": this.selectedSlot = (this.selectedSlot + slotCount - 1) % slotCount; @@ -166,7 +181,7 @@ class GlanceboardHomeLayer implements Layer { * The board's slots scaled into `rect`: dividers (full brightness, or the * dimmest dotted line when "Show lines" is off), the widget name centred in * each region (a merged double-height region has one name and no line - * across it), and the selected quadrant highlighted. + * across it), and the selected slot highlighted. */ function drawSlotPreview( image: GrayImage, @@ -174,7 +189,7 @@ function drawSlotPreview( options: { selectedSlot: number; selectionFocused: boolean }, ): void { const font = getDefaultSmallFont(); - const layout = QUADRANT_LAYOUT; + const layout = glanceLayout(); const settings = glanceSlotSettings(layout); const choices = settings.map((setting) => setting.get()); const regions = resolveGlanceRegions(layout, choices, glanceWidgetSpans); @@ -232,14 +247,15 @@ function drawDottedLine(image: GrayImage, x0: number, y0: number, x1: number, y1 /** The picker for one slot: every widget, the current one marked; choosing applies the one-per-board rule. */ function openSlotPicker(ctx: LayerContext, slotIndex: number): void { - const setting = glanceSlotSettings(QUADRANT_LAYOUT)[slotIndex]; + const layout = glanceLayout(); + const setting = glanceSlotSettings(layout)[slotIndex]; if (!setting) return; const current = setting.get(); const items: MenuItem[] = setting.values.map((value) => ({ label: setting.displayValue(value), onSelect: (menuCtx) => { setting.set(value); - clearConflictingSlots(QUADRANT_LAYOUT, slotIndex, value); + clearConflictingSlots(layout, slotIndex, value); menuCtx.stack.pop(); }, render: ({ image, x, y }) => { @@ -249,7 +265,7 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void { openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current))); } -/** Settings: which sleep gestures show the board, for how long, and the slot lines. */ +/** Settings: layout, sleep gestures, duration, and slot lines. */ class GlanceSettingsLayer implements Layer { private menu: MenuLayer | null = null; @@ -257,6 +273,7 @@ class GlanceSettingsLayer implements Layer { if (!this.menu) { const { width } = ctx.stack.getBaseSize(); const items: MenuItem[] = [ + enumSettingMenuItem(glanceLayoutSetting), enumSettingMenuItem(glanceTapDurationSetting), toggleSettingMenuItem(glanceShowOnLongPressSetting), toggleSettingMenuItem(glanceShowOnHeadTiltSetting), @@ -280,7 +297,7 @@ class GlanceSettingsLayer implements Layer { class GlancePreviewLayer implements Layer { private readonly board: GlanceBoard; - constructor(requestRender: () => void) { + constructor(requestRender: () => void, private readonly onClose: () => void) { this.board = new GlanceBoard(requestRender); this.board.start(); } @@ -302,27 +319,40 @@ class GlancePreviewLayer implements Layer { onRemoved(): void { this.board.stop(); + this.onClose(); } } export function createGlanceboardAppWindow(options: InProcessAppOptions): InProcessWindow { - return createInProcessWindow({ + let previewVisible = false; + const heightMode = () => previewVisible && glanceLayoutSetting.get() === "2x3" ? "max" as const : "medium" as const; + let unsubscribeSettings: (() => void) | undefined; + const app = createInProcessWindow({ appId: "glanceboard", windowId: GLANCEBOARD_WINDOW_ID, title: "Glanceboard", iconLetter: "Gb", icon: "eye", closeable: true, - // 576x288: the board's own size, so the preview is pixel-exact. - heightMode: "medium", + // Settings and the contents picker stay in the default 576x288 viewport; + // only the live six-slot preview needs the taller window. + heightMode: heightMode(), actions: options.actions, // Not wrapped in YieldAtRootLayer: the home page routes double-click // itself (preview focus -> menu focus, menu focus -> sidebar). - baseLayer: new GlanceboardHomeLayer(), + baseLayer: new GlanceboardHomeLayer((visible) => { + previewVisible = visible; + app.setHeightMode(heightMode()); + }), submitFrame: options.submitFrame, setSurfaceVisible: options.setSurfaceVisible, removeSurface: options.removeSurface, reconfigureSurface: options.reconfigureSurface, - onClosed: options.onClosed, + onClosed: () => { + unsubscribeSettings?.(); + options.onClosed(); + }, }); + unsubscribeSettings = onAnySettingChanged(() => app.setHeightMode(heightMode())); + return app; } diff --git a/app/apps/glanceboard/glanceboard-settings.ts b/app/apps/glanceboard/glanceboard-settings.ts index d42dd129..bcb13948 100644 --- a/app/apps/glanceboard/glanceboard-settings.ts +++ b/app/apps/glanceboard/glanceboard-settings.ts @@ -1,5 +1,5 @@ import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings"; -import { QUADRANT_LAYOUT, slotsVerticallyAdjacent, type GlanceLayout } from "./layout"; +import { QUADRANT_LAYOUT, SIX_SLOT_LAYOUT, slotsVerticallyAdjacent, type GlanceLayout } from "./layout"; import { type GlanceWidgetId } from "./widget"; import { glanceWidgetSpans } from "./widgets"; @@ -27,6 +27,19 @@ export function glanceSlotChoiceLabel(choice: GlanceSlotChoice): string { */ const DEFAULT_QUADRANT_CHOICES: readonly GlanceSlotChoice[] = ["system-card", "calendar", "music", "calendar"]; +export const glanceLayoutSetting = new ConfigSettingEnum<"2x2" | "2x3">({ + id: "glanceboard-layout", + label: "Layout", + storageKey: "glanceboard.layout", + defaultValue: "2x2", + values: ["2x2", "2x3"], + description: "Choose two columns with two or three rows of widgets.", +}); + +export function glanceLayout(): GlanceLayout { + return glanceLayoutSetting.get() === "2x3" ? SIX_SLOT_LAYOUT : QUADRANT_LAYOUT; +} + /** * Whether sleep-time gestures show the board at all. Off restores the plain * behaviour: a tap, hold or head-tilt while asleep does nothing (a head-tilt @@ -100,17 +113,19 @@ export const glanceShowLinesSetting = new ConfigSettingBoolean({ }); /** - * One picker per layout slot, keyed by layout id and slot index so a future - * layout gets its own stored choices rather than inheriting the quadrants'. + * Both grid sizes share the original storage keys so changing row count + * preserves the first four choices and remembers the hidden bottom row. */ -export function glanceSlotSettings(layout: GlanceLayout = QUADRANT_LAYOUT): ConfigSettingEnum[] { +export function glanceSlotSettings(layout: GlanceLayout = glanceLayout()): ConfigSettingEnum[] { + const grid = layout === QUADRANT_LAYOUT || layout === SIX_SLOT_LAYOUT; + const storageId = grid ? QUADRANT_LAYOUT.id : layout.id; return layout.slots.map( (slot, index) => new ConfigSettingEnum({ id: `glanceboard-${layout.id}-slot-${index}`, label: slot.label, - storageKey: `glanceboard.${layout.id}.slot.${index}`, - defaultValue: (layout === QUADRANT_LAYOUT ? DEFAULT_QUADRANT_CHOICES[index] : undefined) ?? "none", + storageKey: `glanceboard.${storageId}.slot.${index}`, + defaultValue: (grid ? DEFAULT_QUADRANT_CHOICES[index] : undefined) ?? "none", values: SLOT_CHOICES, formatValue: glanceSlotChoiceLabel, description: `Which widget fills the ${slot.label.toLowerCase()} slot of the Glanceboard.`, @@ -129,9 +144,13 @@ export function clearConflictingSlots(layout: GlanceLayout, changedIndex: number const settings = glanceSlotSettings(layout); const spans = glanceWidgetSpans(choice); const cleared: number[] = []; + let keptPartner = false; settings.forEach((setting, index) => { if (index === changedIndex || setting.get() !== choice) return; - if (spans && slotsVerticallyAdjacent(layout, changedIndex, index)) return; + if (spans && !keptPartner && slotsVerticallyAdjacent(layout, changedIndex, index)) { + keptPartner = true; + return; + } setting.set("none"); cleared.push(index); }); diff --git a/app/apps/glanceboard/index.ts b/app/apps/glanceboard/index.ts index b2295508..354bef25 100644 --- a/app/apps/glanceboard/index.ts +++ b/app/apps/glanceboard/index.ts @@ -3,12 +3,12 @@ import { GlanceBoard } from "./board"; import { createGlanceboardAppWindow, GLANCEBOARD_SURFACE_ID, GLANCEBOARD_WINDOW_ID } from "./glanceboard-app"; import { glanceboardEnabledSetting, + glanceLayout, glanceShowOnHeadTiltSetting, glanceShowOnLongPressSetting, glanceShowOnTap, glanceTapTimeoutMs, } from "./glanceboard-settings"; -import { QUADRANT_LAYOUT } from "./layout"; const glanceboardApp: AppDefinition = { appId: "glanceboard", @@ -16,13 +16,16 @@ const glanceboardApp: AppDefinition = { icon: "eye", launch: (ctx) => ctx.launchInProcessApp(GLANCEBOARD_WINDOW_ID, GLANCEBOARD_SURFACE_ID, createGlanceboardAppWindow), glanceboard: { - size: { width: QUADRANT_LAYOUT.width, height: QUADRANT_LAYOUT.height }, + get size() { + const layout = glanceLayout(); + return { width: layout.width, height: layout.height }; + }, isEnabled: () => glanceboardEnabledSetting.get(), showOnTap: glanceShowOnTap, tapTimeoutMs: glanceTapTimeoutMs, showOnLongPress: () => glanceShowOnLongPressSetting.get(), showOnHeadTilt: () => glanceShowOnHeadTiltSetting.get(), - createBoard: (requestRender) => new GlanceBoard(requestRender, QUADRANT_LAYOUT), + createBoard: (requestRender) => new GlanceBoard(requestRender), }, }; diff --git a/app/apps/glanceboard/layout.ts b/app/apps/glanceboard/layout.ts index 993d73c1..69d2fb2b 100644 --- a/app/apps/glanceboard/layout.ts +++ b/app/apps/glanceboard/layout.ts @@ -29,6 +29,20 @@ export const QUADRANT_LAYOUT: GlanceLayout = { ], }; +/** Six 288x144 slots: the same first two rows, plus a third at the bottom. */ +export const SIX_SLOT_LAYOUT: GlanceLayout = { + id: "six-slots", + width: GLANCEBOARD_WIDTH, + height: 432, + slots: [ + ...QUADRANT_LAYOUT.slots.slice(0, 2), + { label: "Middle left", rect: { x: 0, y: 144, width: 288, height: 144 } }, + { label: "Middle right", rect: { x: 288, y: 144, width: 288, height: 144 } }, + { label: "Bottom left", rect: { x: 0, y: 288, width: 288, height: 144 } }, + { label: "Bottom right", rect: { x: 288, y: 288, width: 288, height: 144 } }, + ], +}; + /** Whether two slots stack vertically with matching width (one region if merged). */ export function slotsVerticallyAdjacent(layout: GlanceLayout, a: number, b: number): boolean { const first = layout.slots[a]?.rect; diff --git a/tests/glanceboard-layout.test.cjs b/tests/glanceboard-layout.test.cjs index 8b602aba..680e890d 100644 --- a/tests/glanceboard-layout.test.cjs +++ b/tests/glanceboard-layout.test.cjs @@ -57,3 +57,25 @@ test("dividers: left edges of the right column, top edges of the bottom row, min { x0: 288, y0: 144, x1: 575, y1: 144 }, ]); }); + +test("2x3 adds a bottom row without moving or resizing the first four slots", () => { + const { SIX_SLOT_LAYOUT } = require("../.test-build/app/apps/glanceboard/layout.js"); + assert.equal(SIX_SLOT_LAYOUT.width, 576); + assert.equal(SIX_SLOT_LAYOUT.height, 432); + assert.deepEqual(SIX_SLOT_LAYOUT.slots.slice(0, 4).map((slot) => slot.rect), QUADRANT_LAYOUT.slots.map((slot) => slot.rect)); + assert.deepEqual(SIX_SLOT_LAYOUT.slots.slice(4).map((slot) => slot.rect), [ + { x: 0, y: 288, width: 288, height: 144 }, + { x: 288, y: 288, width: 288, height: 144 }, + ]); + const regions = resolveGlanceRegions(SIX_SLOT_LAYOUT, ["none", "none", "terminal", "none", "terminal", "music"], spans); + assert.deepEqual(regions, [ + { choice: "terminal", rect: { x: 0, y: 144, width: 288, height: 288 }, slots: [2, 4] }, + { choice: "music", rect: { x: 288, y: 288, width: 288, height: 144 }, slots: [5] }, + ]); + const { slotDividers } = require("../.test-build/app/apps/glanceboard/layout.js"); + const bottomDividers = slotDividers(SIX_SLOT_LAYOUT, regions).filter((line) => line.y0 === 288); + assert.deepEqual(bottomDividers, [ + { x0: 288, y0: 288, x1: 288, y1: 431 }, + { x0: 288, y0: 288, x1: 575, y1: 288 }, + ]); +}); diff --git a/tests/glanceboard-settings.test.cjs b/tests/glanceboard-settings.test.cjs new file mode 100644 index 00000000..ebb171f9 --- /dev/null +++ b/tests/glanceboard-settings.test.cjs @@ -0,0 +1,99 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); +const layout = require('../.test-build/app/apps/glanceboard/layout.js'); + +function load(file, dependencies) { + const context = { exports: {}, console, require: (name) => { + assert.ok(name in dependencies, name); + return dependencies[name]; + } }; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, context); + return context.exports; +} + +function harness() { + const values = new Map(); + const listeners = new Set(); + class Setting { + constructor(options) { Object.assign(this, options); } + get() { return values.get(this.storageKey) ?? this.defaultValue; } + set(value) { values.set(this.storageKey, value); } + } + const config = { + ConfigSettingBoolean: Setting, ConfigSettingEnum: Setting, + onAnySettingChanged: (listener) => { listeners.add(listener); return () => listeners.delete(listener); }, + }; + const widgets = { + glanceWidgetSpans: (choice) => choice === 'calendar' || choice === 'terminal', + findGlanceWidget: () => ({ create: () => ({ start() {}, stop() {}, paint() {} }) }), + }; + const settings = load('app/apps/glanceboard/glanceboard-settings.ts', { + '../../ui/dashboard-settings': config, './layout': layout, './widgets': widgets, + }); + class GrayImage { + constructor(width, height) { this.width = width; this.height = height; } + drawLine() {} + composeInto() {} + } + const { GlanceBoard } = load('app/apps/glanceboard/board.ts', { + '../../graphics/image': { GrayImage }, '../../ui/dashboard-settings': config, + './glanceboard-settings': settings, './layout': layout, './widgets': widgets, + }); + return { settings, GlanceBoard, values, listeners, notify: () => [...listeners].forEach((listener) => listener()) }; +} + +test('layout defaults to 2x2; expanding preserves stored slots and remembers the extra row', () => { + const { settings: s, values } = harness(); + values.set('glanceboard.quadrants.slot.0', 'compass'); + assert.equal(s.glanceLayoutSetting.get(), '2x2'); + assert.equal(s.glanceLayout(), layout.QUADRANT_LAYOUT); + s.glanceLayoutSetting.set('2x3'); + assert.equal(s.glanceLayout(), layout.SIX_SLOT_LAYOUT); + assert.deepEqual(Array.from(s.glanceSlotSettings(), (slot) => slot.get()), + ['compass', 'calendar', 'music', 'calendar', 'none', 'none']); + s.glanceSlotSettings()[5].set('nightscout'); + s.glanceLayoutSetting.set('2x2'); + assert.equal(s.glanceSlotSettings().length, 4); + s.glanceLayoutSetting.set('2x3'); + assert.equal(s.glanceSlotSettings()[5].get(), 'nightscout'); +}); + +test('a spanning widget in the middle row keeps only one adjacent partner', () => { + const { settings: s } = harness(); + s.glanceLayoutSetting.set('2x3'); + const slots = s.glanceSlotSettings(); + for (const index of [0, 2, 4]) slots[index].set('terminal'); + s.clearConflictingSlots(s.glanceLayout(), 2, 'terminal'); + assert.deepEqual(Array.from(slots, (slot) => slot.get()), + ['terminal', 'calendar', 'terminal', 'calendar', 'none', 'none']); + slots[4].set('terminal'); + s.clearConflictingSlots(s.glanceLayout(), 4, 'terminal'); + assert.equal(slots[0].get(), 'none'); + assert.equal(slots[2].get(), 'terminal'); +}); + +test('an active empty board resizes and requests a frame when the layout changes', () => { + const h = harness(); + for (const slot of h.settings.glanceSlotSettings(layout.SIX_SLOT_LAYOUT)) slot.set('none'); + let renders = 0; + const board = new h.GlanceBoard(() => renders++); + board.start(); + assert.equal(board.paint().height, 288); + h.settings.glanceLayoutSetting.set('2x3'); + h.notify(); + assert.equal(board.paint().width, 576); + assert.equal(board.paint().height, 432); + assert.equal(renders, 1); + h.settings.glanceLayoutSetting.set('2x2'); + h.notify(); + assert.equal(board.paint().height, 288); + assert.equal(renders, 2); + board.stop(); + assert.equal(h.listeners.size, 0); +});